From b82ba39aabce2a2cd83825a0aa0f77164ab8a276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l-Kristian=20Hamre?= Date: Mon, 21 Jul 2025 20:13:18 +0200 Subject: [PATCH] Added web dashboard. --- requirements.txt | 3 +- rstat_tool/dashboard.py | 54 +++++++++++++++++++++ rstat_tool/database.py | 88 ++++++++++++++++++++++++---------- rstat_tool/ticker_extractor.py | 74 +++++++++++++++------------- setup.py | 1 + subreddits.json | 4 +- templates/base.html | 87 +++++++++++++++++++++++++++++++++ templates/index.html | 35 ++++++++++++++ templates/subreddit.html | 35 ++++++++++++++ 9 files changed, 321 insertions(+), 60 deletions(-) create mode 100644 rstat_tool/dashboard.py create mode 100644 templates/base.html create mode 100644 templates/index.html create mode 100644 templates/subreddit.html diff --git a/requirements.txt b/requirements.txt index 630ba9c..a7a9c37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ yfinance praw python-dotenv -nltk \ No newline at end of file +nltk +Flask \ No newline at end of file diff --git a/rstat_tool/dashboard.py b/rstat_tool/dashboard.py new file mode 100644 index 0000000..0b848c3 --- /dev/null +++ b/rstat_tool/dashboard.py @@ -0,0 +1,54 @@ +# rstat_tool/dashboard.py + +from flask import Flask, render_template +from .database import ( + get_overall_summary, + get_subreddit_summary, + get_all_scanned_subreddits +) + +app = Flask(__name__, template_folder='../templates') + +@app.template_filter('format_mc') +def format_market_cap(mc): + """Formats a large number into a readable market cap string.""" + if mc is None or mc == 0: + return "N/A" + if mc >= 1e12: + return f"${mc/1e12:.2f}T" + elif mc >= 1e9: + return f"${mc/1e9:.2f}B" + elif mc >= 1e6: + return f"${mc/1e6:.2f}M" + else: + return f"${mc:,}" + +@app.context_processor +def inject_subreddits(): + """Makes the list of all scanned subreddits available to every template.""" + subreddits = get_all_scanned_subreddits() + return dict(subreddits=subreddits) + +@app.route("/") +def index(): + """The handler for the main dashboard page.""" + # --- CHANGE HERE: Limit the data to the top 10 --- + tickers = get_overall_summary(limit=10) + return render_template("index.html", tickers=tickers) + +@app.route("/subreddit/") +def subreddit_dashboard(name): + """A dynamic route for per-subreddit dashboards.""" + # --- CHANGE HERE: Limit the data to the top 10 --- + tickers = get_subreddit_summary(name, limit=10) + return render_template("subreddit.html", tickers=tickers, subreddit_name=name) + +def start_dashboard(): + """The main function called by the 'rstat-dashboard' command.""" + print("Starting Flask server...") + print("Open http://127.0.0.1:5000 in your browser.") + print("Press CTRL+C to stop the server.") + app.run(debug=True) + +if __name__ == "__main__": + start_dashboard() \ No newline at end of file diff --git a/rstat_tool/database.py b/rstat_tool/database.py index 335dfda..caf9b44 100644 --- a/rstat_tool/database.py +++ b/rstat_tool/database.py @@ -2,7 +2,6 @@ import sqlite3 import time -# --- IMPORT ADDED BACK IN --- from .ticker_extractor import COMMON_WORDS_BLACKLIST DB_FILE = "reddit_stocks.db" @@ -14,9 +13,13 @@ def get_db_connection(): return conn def initialize_db(): - # ... (This function is unchanged) + """ + Initializes the database and creates the necessary tables if they don't exist. + """ conn = get_db_connection() cursor = conn.cursor() + + # --- Create tickers table --- cursor.execute(""" CREATE TABLE IF NOT EXISTS tickers ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -25,12 +28,16 @@ def initialize_db(): last_updated INTEGER ) """) + + # --- Create subreddits table --- cursor.execute(""" CREATE TABLE IF NOT EXISTS subreddits ( id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL UNIQUE ) """) + + # --- Create mentions table --- cursor.execute(""" CREATE TABLE IF NOT EXISTS mentions ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -44,11 +51,11 @@ def initialize_db(): UNIQUE(ticker_id, post_id, sentiment_score) ) """) + conn.commit() conn.close() print("Database initialized successfully.") -# --- CLEANUP FUNCTION ADDED BACK IN --- def clean_stale_tickers(): """ Removes tickers and their associated mentions from the database @@ -58,7 +65,6 @@ def clean_stale_tickers(): conn = get_db_connection() cursor = conn.cursor() - # Find ticker IDs that match the blacklist placeholders = ','.join('?' for _ in COMMON_WORDS_BLACKLIST) query = f"SELECT id, symbol FROM tickers WHERE symbol IN ({placeholders})" @@ -74,11 +80,7 @@ def clean_stale_tickers(): ticker_id = ticker['id'] ticker_symbol = ticker['symbol'] print(f"Removing stale ticker '{ticker_symbol}' (ID: {ticker_id})...") - - # 1. Delete all mentions associated with this ticker ID cursor.execute("DELETE FROM mentions WHERE ticker_id = ?", (ticker_id,)) - - # 2. Delete the ticker itself cursor.execute("DELETE FROM tickers WHERE id = ?", (ticker_id,)) deleted_count = conn.total_changes @@ -88,7 +90,7 @@ def clean_stale_tickers(): def add_mention(conn, ticker_id, subreddit_id, post_id, timestamp, sentiment): - # ... (This function is unchanged) + """Adds a new mention with its sentiment score to the database.""" cursor = conn.cursor() try: cursor.execute( @@ -99,49 +101,44 @@ def add_mention(conn, ticker_id, subreddit_id, post_id, timestamp, sentiment): except sqlite3.IntegrityError: pass -# ... (get_or_create_entity, update_ticker_market_cap, get_ticker_info are unchanged) def get_or_create_entity(conn, table_name, column_name, value): - # ... + """Generic function to get or create an entity and return its ID.""" cursor = conn.cursor() cursor.execute(f"SELECT id FROM {table_name} WHERE {column_name} = ?", (value,)) result = cursor.fetchone() - if result: return result['id'] + if result: + return result['id'] else: cursor.execute(f"INSERT INTO {table_name} ({column_name}) VALUES (?)", (value,)) conn.commit() return cursor.lastrowid def update_ticker_market_cap(conn, ticker_id, market_cap): - # ... + """Updates the market cap and timestamp for a specific ticker.""" cursor = conn.cursor() current_timestamp = int(time.time()) cursor.execute("UPDATE tickers SET market_cap = ?, last_updated = ? WHERE id = ?", (market_cap, current_timestamp, ticker_id)) conn.commit() def get_ticker_info(conn, ticker_id): - # ... + """Retrieves all info for a specific ticker by its ID.""" cursor = conn.cursor() cursor.execute("SELECT * FROM tickers WHERE id = ?", (ticker_id,)) return cursor.fetchone() def generate_summary_report(limit=20): - # ... (This function is unchanged) + """Queries the DB to generate a summary for the command-line tool.""" print(f"\n--- Top {limit} Tickers by Mention Count ---") conn = get_db_connection() cursor = conn.cursor() query = """ SELECT - t.symbol, - t.market_cap, - COUNT(m.id) as mention_count, + t.symbol, t.market_cap, COUNT(m.id) as mention_count, SUM(CASE WHEN m.sentiment_score > 0.1 THEN 1 ELSE 0 END) as bullish_mentions, SUM(CASE WHEN m.sentiment_score < -0.1 THEN 1 ELSE 0 END) as bearish_mentions, SUM(CASE WHEN m.sentiment_score BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions - FROM mentions m - JOIN tickers t ON m.ticker_id = t.id - GROUP BY t.symbol, t.market_cap - ORDER BY mention_count DESC - LIMIT ?; + FROM mentions m JOIN tickers t ON m.ticker_id = t.id + GROUP BY t.symbol, t.market_cap ORDER BY mention_count DESC LIMIT ?; """ results = cursor.execute(query, (limit,)).fetchall() header = f"{'Ticker':<8} | {'Mentions':<8} | {'Bullish':<8} | {'Bearish':<8} | {'Neutral':<8} | {'Market Cap':<15}" @@ -155,4 +152,47 @@ def generate_summary_report(limit=20): elif mc >= 1e9: market_cap_str = f"${mc/1e9:.2f}B" else: market_cap_str = f"${mc/1e6:.2f}M" print(f"{row['symbol']:<8} | {row['mention_count']:<8} | {row['bullish_mentions']:<8} | {row['bearish_mentions']:<8} | {row['neutral_mentions']:<8} | {market_cap_str:<15}") - conn.close() \ No newline at end of file + conn.close() + +def get_overall_summary(limit=50): + """Gets the top tickers across all subreddits for the dashboard.""" + conn = get_db_connection() + query = """ + SELECT + t.symbol, t.market_cap, COUNT(m.id) as mention_count, + SUM(CASE WHEN m.sentiment_score > 0.1 THEN 1 ELSE 0 END) as bullish_mentions, + SUM(CASE WHEN m.sentiment_score < -0.1 THEN 1 ELSE 0 END) as bearish_mentions, + SUM(CASE WHEN m.sentiment_score BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions + FROM mentions m JOIN tickers t ON m.ticker_id = t.id + GROUP BY t.symbol, t.market_cap ORDER BY mention_count DESC LIMIT ?; + """ + results = conn.execute(query, (limit,)).fetchall() + conn.close() + return results + +def get_subreddit_summary(subreddit_name, limit=50): + """Gets the top tickers for a specific subreddit for the dashboard.""" + conn = get_db_connection() + query = """ + SELECT + t.symbol, t.market_cap, COUNT(m.id) as mention_count, + SUM(CASE WHEN m.sentiment_score > 0.1 THEN 1 ELSE 0 END) as bullish_mentions, + SUM(CASE WHEN m.sentiment_score < -0.1 THEN 1 ELSE 0 END) as bearish_mentions, + SUM(CASE WHEN m.sentiment_score BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions + FROM mentions m + JOIN tickers t ON m.ticker_id = t.id + JOIN subreddits s ON m.subreddit_id = s.id + WHERE s.name = ? + GROUP BY t.symbol, t.market_cap ORDER BY mention_count DESC LIMIT ?; + """ + results = conn.execute(query, (subreddit_name, limit)).fetchall() + conn.close() + return results + +def get_all_scanned_subreddits(): + """Gets a unique list of all subreddits we have data for.""" + # --- THIS IS THE CORRECTED LINE --- + conn = get_db_connection() + results = conn.execute("SELECT DISTINCT name FROM subreddits ORDER BY name ASC;").fetchall() + conn.close() + return [row['name'] for row in results] \ No newline at end of file diff --git a/rstat_tool/ticker_extractor.py b/rstat_tool/ticker_extractor.py index 7cd19cf..9a5523e 100644 --- a/rstat_tool/ticker_extractor.py +++ b/rstat_tool/ticker_extractor.py @@ -5,44 +5,50 @@ import re # A set of common English words and acronyms that look like stock tickers. # This helps reduce false positives. COMMON_WORDS_BLACKLIST = { - "401K", "403B", "457B", "ABOVE", "AI", "ALL", "ALPHA", "AMA", "AMEX", - "AND", "ANY", "AR", "ARE", "AROUND", "ASSET", "AT", "ATH", "ATL", "AUD", - "BE", "BEAR", "BELOW", "BETA", "BIG", "BIS", "BLEND", "BOE", "BOJ", - "BOND", "BRB", "BRL", "BTC", "BTW", "BULL", "BUT", "BUY", "BUZZ", "CAD", - "CAN", "CEO", "CFO", "CHF", "CIA", "CNY", "COME", "COST", "COULD", "CPI", + "401K", "403B", "457B", "ABOUT", "ABOVE", "ADAM", "AEDT", "AEST", "AH", + "AI", "ALL", "ALPHA", "ALSO", "AM", "AMA", "AMEX", "AND", "ANY", "AR", + "ARE", "AROUND", "ASAP", "ASS", "ASSET", "AT", "ATH", "ATL", "ATM", + "AUD", "BE", "BEAR", "BELOW", "BETA", "BIG", "BIS", "BLEND", "BOE", + "BOJ", "BOMB", "BOND", "BOTS", "BRB", "BRL", "BS", "BST", "BTC", "BTW", + "BULL", "BUT", "BUY", "BUZZ", "CAD", "CAN", "CEO", "CEST", "CET", "CFO", + "CHF", "CIA", "CLOSE", "CNY", "COME", "COST", "COULD", "CPI", "CST", "CTB", "CTO", "CYCLE", "CZK", "DAO", "DATE", "DAX", "DAY", "DCA", "DD", - "DEBT", "DIA", "DIV", "DJIA", "DKK", "DM", "DO", "DOGE", "DR", "EACH", - "EARLY", "EARN", "ECB", "EDGAR", "EDIT", "EPS", "ESG", "ETF", "ETH", - "EU", "EUR", "EV", "EVERY", "FAQ", "FAR", "FAST", "FBI", "FDA", "FIHTX", - "FINRA", "FINT", "FINTX", "FINTY", "FOMC", "FOMO", "FOR", "FRAUD", - "FRG", "FSPSX", "FTSE", "FUD", "FULL", "FUND", "FXAIX", "FXIAX", "FY", - "FYI", "FZROX", "GAIN", "GDP", "GET", "GBP", "GO", "GOAL", "GPU", "GRAB", - "GTG", "HAS", "HAVE", "HATE", "HEAR", "HEDGE", "HINT", "HKD", "HODL", - "HOLD", "HOUR", "HSA", "HUF", "IMHO", "IMO", "IN", "INR", "IPO", "IRA", - "IRS", "IS", "ISM", "IT", "IV", "IVV", "IWM", "JPY", "JUST", "KNOW", - "KRW", "LARGE", "LAST", "LATE", "LATER", "LBO", "LIKE", "LMAO", "LOL", - "LONG", "LOOK", "LOSS", "LOVE", "M&A", "MAKE", "MAX", "MC", "MID", "MIGHT", - "MIN", "ML", "MOASS", "MONTH", "MUST", "MXN", "MY", "NATO", "NEAR", - "NEED", "NEW", "NEXT", "NFA", "NFT", "NGMI", "NIGHT", "NO", "NOK", "NONE", - "NOT", "NOW", "NSA", "NULL", "NZD", "NYSE", "OF", "OK", "OLD", "ON", - "OP", "OR", "OTC", "OUGHT", "OUT", "OVER", "PE", "PEAK", "PEG", - "PLAN", "PLN", "PMI", "PPI", "PRICE", "PROFIT", "PSA", "Q1", "Q2", "Q3", - "Q4", "QQQ", "RBA", "RBNZ", "REIT", "REKT", "RH", "RISK", "ROE", "ROFL", - "ROI", "ROTH", "RSD", "RUB", "SAVE", "SCALP", "SCAM", "SCHB", "SEC", + "DEBT", "DIA", "DIV", "DJIA", "DKK", "DM", "DO", "DOGE", "DONT", "DR", + "EACH", "EARLY", "EARN", "ECB", "EDGAR", "EDIT", "EDT", "END", "EPS", + "ER", "ESG", "EST", "ETF", "ETH", "EU", "EUR", "EV", "EVERY", "FAQ", + "FAR", "FAST", "FBI", "FDA", "FIHTX", "FINRA", "FINT", "FINTX", "FINTY", + "FOMC", "FOMO", "FOR", "FRAUD", "FRG", "FSPSX", "FTSE", "FUCK", "FUD", + "FULL", "FUND", "FXAIX", "FXIAX", "FY", "FYI", "FZROX", "GAIN", "GBP", + "GDP", "GET", "GMT", "GO", "GOAL", "GPU", "GRAB", "GTG", "HAS", "HAVE", + "HATE", "HEAR", "HEDGE", "HIGH", "HINT", "HKD", "HODL", "HOLD", "HOUR", + "HSA", "HUF", "IF", "IMHO", "IMO", "IN", "INR", "IP", "IPO", "IRA", + "IRS", "IS", "ISM", "IST", "IT", "ITM", "IV", "IVV", "IWM", "JPOW", + "JPY", "JST", "JUST", "KARMA", "KNOW", "KO", "KRW", "LARGE", "LAST", + "LATE", "LATER", "LBO", "LEAP", "LEAPS", "LETS", "LFG", "LIKE", "LIMIT", + "LMAO", "LOL", "LONG", "LOOK", "LOSS", "LOVE", "LOW", "M&A", "MAKE", + "MAX", "MC", "ME", "MID", "MIGHT", "MIN", "ML", "MOASS", "MONTH", "MSK", + "MUST", "MXN", "MY", "NATO", "NEAR", "NEED", "NEVER", "NEW", "NEXT", + "NFA", "NFT", "NGMI", "NIGHT", "NO", "NOK", "NONE", "NOT", "NOW", "NSA", + "NULL", "NZD", "NYSE", "OF", "OK", "OLD", "ON", "OP", "OPEN", "OR", + "OTC", "OTM", "OUGHT", "OUT", "OVER", "OWN", "PC", "PDT", "PE", "PEAK", + "PEG", "PEW", "PLAN", "PLN", "PM", "PMI", "POS", "PPI", "PR", "PRICE", + "PROFIT", "PSA", "PST", "Q1", "Q2", "Q3", "Q4", "QQQ", "RBA", "RBNZ", + "RE", "REAL", "REIT", "REKT", "RH", "RIP", "RISK", "ROE", "ROFL", "ROI", + "ROTH", "RSD", "RUB", "RULE", "SAVE", "SCALP", "SCAM", "SCHB", "SEC", "SEE", "SEK", "SELL", "SEP", "SGD", "SHALL", "SHARE", "SHORT", "SO", - "SOME", "SOON", "SPAC", "SPEND", "SPLG", "SPX", "SPY", "STILL", "STOCK", - "SWING", "TAKE", "TERM", "THE", "THINK", "THIS", "TIME", "TL", "TL;DR", - "TLDR", "TODAY", "TO", "TOTAL", "TRADE", "TREND", "TRUE", "TRY", "TTYL", - "TWO", "UK", "UNDER", "UP", "US", "USA", "USD", "VTI", "VALUE", "VOO", - "VR", "WAGMI", "WANT", "WATCH", "WAY", "WE", "WEB3", "WEEK", "WHO", - "WHY", "WILL", "WORTH", "WOULD", "WSB", "YET", "YIELD", "YOLO", "YOU", + "SOME", "SOON", "SP", "SPAC", "SPEND", "SPLG", "SPX", "SPY", "START", + "STILL", "STOCK", "STOP", "SWING", "TAKE", "TERM", "THAT", "THE", "THINK", + "THIS", "TIME", "TITS", "TL", "TL;DR", "TLDR", "TO", "TODAY", "TOTAL", + "TRADE", "TREND", "TRUE", "TRY", "TTYL", "TWO", "UI", "UK", "UNDER", + "UP", "US", "USA", "USD", "UTC", "VTI", "VALUE", "VOO", "VR", "WAGMI", + "WANT", "WATCH", "WAY", "WE", "WEB3", "WEEK", "WHO", "WHY", "WILL", + "WORTH", "WOULD", "WSB", "WTF", "YET", "YIELD", "YES", "YOLO", "YOU", "ZAR", - "KARMA", "OTM", "ITM", "ATM", "JPOW", "OPEN", "CLOSE", "HIGH", "LOW", - "RE", "BS", "ASAP", "RULE", "REAL", "LIMIT", "STOP", "END", "START", "BOTS", - "UTC", "AH", "PM", "PR", "GMT", "EST", "CST", "PST", "BST", "AEDT", "AEST", - "CET", "CEST", "EDT", "IST", "JST", "MSK", "PDT", "PST", "YES", "NO", "OWN", - "BOMB", + "YOUR", "BABY", "BAG", "BAGS", "GL", "GLHF", "EOD", "EOW", "EOY", "GOING", "KEEP", + "MORE", "PUT", "CALL", "YTD", "BOTH", "BUST", "EVEN", "FROM", "GOAT", "HALF", + "SL", "OS", "SOLIS", "OEM", "MA", "DOE", "II", "CHIPS" } + def extract_tickers(text): """ Extracts potential stock tickers from a given piece of text. diff --git a/setup.py b/setup.py index 67a4712..af9394b 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ setup( 'console_scripts': [ # The path is now 'package_name.module_name:function_name' 'rstat=rstat_tool.main:main', + 'rstat-dashboard=rstat_tool.dashboard:start_dashboard', ], }, ) \ No newline at end of file diff --git a/subreddits.json b/subreddits.json index 742c938..48cfbef 100644 --- a/subreddits.json +++ b/subreddits.json @@ -2,6 +2,8 @@ "subreddits": [ "pennystocks", "Shortsqueeze", - "smallstreetbets" + "smallstreetbets", + "wallstreetbets", + "Wallstreetbetsnew" ] } diff --git a/templates/base.html b/templates/base.html new file mode 100644 index 0000000..3e8ffc8 --- /dev/null +++ b/templates/base.html @@ -0,0 +1,87 @@ + + + + + + {% block title %}Reddit Stock Dashboard{% endblock %} + + + +
+ Overall + {% for sub in subreddits %} + r/{{ sub }} + {% endfor %} +
+
+ {% block content %}{% endblock %} +
+ + \ No newline at end of file diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..5211a90 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,35 @@ +{% extends "base.html" %} + +{% block title %}Overall Dashboard{% endblock %} + +{% block content %} +

Top 10 Tickers (All Subreddits)

+ + + + + + + + + + + {% for ticker in tickers %} + + + + + + + {% endfor %} + +
TickerMentionsMarket CapSentiment
{{ ticker.symbol }}{{ ticker.mention_count }}{{ ticker.market_cap | format_mc }} + {% if ticker.bullish_mentions > ticker.bearish_mentions %} + Bullish + {% elif ticker.bearish_mentions > ticker.bullish_mentions %} + Bearish + {% else %} + Neutral + {% endif %} +
+{% endblock %} \ No newline at end of file diff --git a/templates/subreddit.html b/templates/subreddit.html new file mode 100644 index 0000000..037142c --- /dev/null +++ b/templates/subreddit.html @@ -0,0 +1,35 @@ +{% extends "base.html" %} + +{% block title %}r/{{ subreddit_name }} Dashboard{% endblock %} + +{% block content %} +

Top 10 Tickers in r/{{ subreddit_name }}

+ + + + + + + + + + + {% for ticker in tickers %} + + + + + + + {% endfor %} + +
TickerMentionsMarket CapSentiment
{{ ticker.symbol }}{{ ticker.mention_count }}{{ ticker.market_cap | format_mc }} + {% if ticker.bullish_mentions > ticker.bearish_mentions %} + Bullish + {% elif ticker.bearish_mentions > ticker.bullish_mentions %} + Bearish + {% else %} + Neutral + {% endif %} +
+{% endblock %} \ No newline at end of file