Added blacklist words.

rstat_tool/database.py

@@ -46,12 +46,12 @@ def initialize_db():
             ticker_id INTEGER,
             subreddit_id INTEGER,
             post_id TEXT NOT NULL,
-            mention_type TEXT NOT NULL, -- Can be 'post' or 'comment'
+            mention_type TEXT NOT NULL,
+            mention_sentiment REAL,         -- Renamed from sentiment_score for clarity
+            post_avg_sentiment REAL,      -- NEW: Stores the avg sentiment of the whole post
             mention_timestamp INTEGER NOT NULL,
-            sentiment_score REAL,
             FOREIGN KEY (ticker_id) REFERENCES tickers (id),
-            FOREIGN KEY (subreddit_id) REFERENCES subreddits (id),
-            UNIQUE(ticker_id, post_id, mention_type, sentiment_score)
+            FOREIGN KEY (subreddit_id) REFERENCES subreddits (id)
         )
     """)
 
@@ -105,13 +105,69 @@ def clean_stale_tickers():
     conn.close()
     print(f"Cleanup complete. Removed {deleted_count} records.")
 
+def get_db_connection():
+    conn = sqlite3.connect(DB_FILE)
+    conn.row_factory = sqlite3.Row
+    return conn
 
-def add_mention(conn, ticker_id, subreddit_id, post_id, mention_type, timestamp, sentiment):
+def initialize_db():
+    conn = get_db_connection()
+    cursor = conn.cursor()
+    cursor.execute("""
+        CREATE TABLE IF NOT EXISTS tickers (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            symbol TEXT NOT NULL UNIQUE,
+            market_cap INTEGER,
+            closing_price REAL,
+            last_updated INTEGER
+        )
+    """)
+    cursor.execute("""
+        CREATE TABLE IF NOT EXISTS subreddits (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            name TEXT NOT NULL UNIQUE
+        )
+    """)
+    cursor.execute("""
+        CREATE TABLE IF NOT EXISTS mentions (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            ticker_id INTEGER,
+            subreddit_id INTEGER,
+            post_id TEXT NOT NULL,
+            mention_type TEXT NOT NULL,
+            mention_sentiment REAL,
+            post_avg_sentiment REAL,
+            mention_timestamp INTEGER NOT NULL,
+            FOREIGN KEY (ticker_id) REFERENCES tickers (id),
+            FOREIGN KEY (subreddit_id) REFERENCES subreddits (id)
+        )
+    """)
+    cursor.execute("""
+        CREATE TABLE IF NOT EXISTS posts (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            post_id TEXT NOT NULL UNIQUE,
+            title TEXT NOT NULL,
+            post_url TEXT,
+            subreddit_id INTEGER,
+            post_timestamp INTEGER,
+            comment_count INTEGER,
+            avg_comment_sentiment REAL,
+            FOREIGN KEY (subreddit_id) REFERENCES subreddits (id)
+        )
+    """)
+    conn.commit()
+    conn.close()
+    print("Database initialized successfully.")
+
+def add_mention(conn, ticker_id, subreddit_id, post_id, mention_type, timestamp, mention_sentiment, post_avg_sentiment=None):
     cursor = conn.cursor()
     try:
         cursor.execute(
-            "INSERT INTO mentions (ticker_id, subreddit_id, post_id, mention_type, mention_timestamp, sentiment_score) VALUES (?, ?, ?, ?, ?, ?)",
-            (ticker_id, subreddit_id, post_id, mention_type, timestamp, sentiment)
+            """
+            INSERT INTO mentions (ticker_id, subreddit_id, post_id, mention_type, mention_timestamp, mention_sentiment, post_avg_sentiment)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+            """,
+            (ticker_id, subreddit_id, post_id, mention_type, timestamp, mention_sentiment, post_avg_sentiment)
         )
         conn.commit()
     except sqlite3.IntegrityError:
@@ -150,19 +206,26 @@ def generate_summary_report(limit=20):
     print(f"\n--- Top {limit} Tickers by Mention Count ---")
     conn = get_db_connection()
     cursor = conn.cursor()
+    
+    # --- UPDATED QUERY: Changed m.sentiment_score to m.mention_sentiment ---
     query = """
         SELECT
-            t.symbol, t.market_cap, COUNT(m.id) as mention_count,
-            SUM(CASE WHEN m.sentiment_score > 0.1 THEN 1 ELSE 0 END) as bullish_mentions,
-            SUM(CASE WHEN m.sentiment_score < -0.1 THEN 1 ELSE 0 END) as bearish_mentions,
-            SUM(CASE WHEN m.sentiment_score BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions
+            t.symbol, t.market_cap, t.closing_price,
+            COUNT(m.id) as mention_count,
+            SUM(CASE WHEN m.mention_sentiment > 0.1 THEN 1 ELSE 0 END) as bullish_mentions,
+            SUM(CASE WHEN m.mention_sentiment < -0.1 THEN 1 ELSE 0 END) as bearish_mentions,
+            SUM(CASE WHEN m.mention_sentiment BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions
         FROM mentions m JOIN tickers t ON m.ticker_id = t.id
-        GROUP BY t.symbol, t.market_cap ORDER BY mention_count DESC LIMIT ?;
+        GROUP BY t.symbol, t.market_cap, t.closing_price
+        ORDER BY mention_count DESC
+        LIMIT ?;
     """
     results = cursor.execute(query, (limit,)).fetchall()
-    header = f"{'Ticker':<8} | {'Mentions':<8} | {'Bullish':<8} | {'Bearish':<8} | {'Neutral':<8} | {'Market Cap':<15}"
+    
+    header = f"{'Ticker':<8} | {'Mentions':<8} | {'Bullish':<8} | {'Bearish':<8} | {'Neutral':<8} | {'Market Cap':<15} | {'Close Price':<12}"
     print(header)
-    print("-" * len(header))
+    print("-" * (len(header) + 2)) # Adjusted separator length
+
     for row in results:
         market_cap_str = "N/A"
         if row['market_cap'] and row['market_cap'] > 0:
@@ -170,45 +233,19 @@ def generate_summary_report(limit=20):
             if mc >= 1e12: market_cap_str = f"${mc/1e12:.2f}T"
             elif mc >= 1e9: market_cap_str = f"${mc/1e9:.2f}B"
             else: market_cap_str = f"${mc/1e6:.2f}M"
-        print(f"{row['symbol']:<8} | {row['mention_count']:<8} | {row['bullish_mentions']:<8} | {row['bearish_mentions']:<8} | {row['neutral_mentions']:<8} | {market_cap_str:<15}")
-    conn.close()
 
-def get_overall_summary(limit=50):
-    conn = get_db_connection()
-    query = """
-        SELECT
-            t.symbol, t.market_cap, t.closing_price, -- Added closing_price
-            COUNT(m.id) as mention_count,
-            SUM(CASE WHEN m.sentiment_score > 0.1 THEN 1 ELSE 0 END) as bullish_mentions,
-            SUM(CASE WHEN m.sentiment_score < -0.1 THEN 1 ELSE 0 END) as bearish_mentions,
-            SUM(CASE WHEN m.sentiment_score BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions
-        FROM mentions m JOIN tickers t ON m.ticker_id = t.id
-        GROUP BY t.symbol, t.market_cap, t.closing_price -- Added closing_price
-        ORDER BY mention_count DESC LIMIT ?;
-    """
-    results = conn.execute(query, (limit,)).fetchall()
-    conn.close()
-    return results
+        closing_price_str = f"${row['closing_price']:.2f}" if row['closing_price'] else "N/A"
 
-def get_subreddit_summary(subreddit_name, limit=50):
-    conn = get_db_connection()
-    query = """
-        SELECT
-            t.symbol, t.market_cap, t.closing_price, -- Added closing_price
-            COUNT(m.id) as mention_count,
-            SUM(CASE WHEN m.sentiment_score > 0.1 THEN 1 ELSE 0 END) as bullish_mentions,
-            SUM(CASE WHEN m.sentiment_score < -0.1 THEN 1 ELSE 0 END) as bearish_mentions,
-            SUM(CASE WHEN m.sentiment_score BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions
-        FROM mentions m
-        JOIN tickers t ON m.ticker_id = t.id
-        JOIN subreddits s ON m.subreddit_id = s.id
-        WHERE s.name = ?
-        GROUP BY t.symbol, t.market_cap, t.closing_price -- Added closing_price
-        ORDER BY mention_count DESC LIMIT ?;
-    """
-    results = conn.execute(query, (subreddit_name, limit)).fetchall()
+        print(
+            f"{row['symbol']:<8} | "
+            f"{row['mention_count']:<8} | "
+            f"{row['bullish_mentions']:<8} | "
+            f"{row['bearish_mentions']:<8} | "
+            f"{row['neutral_mentions']:<8} | "
+            f"{market_cap_str:<15} | "
+            f"{closing_price_str:<12}"
+        )
     conn.close()
-    return results
 
 def get_all_scanned_subreddits():
     """Gets a unique list of all subreddits we have data for."""
@@ -253,20 +290,52 @@ def get_deep_dive_details(ticker_symbol):
     conn.close()
     return results
 
-def get_image_view_summary(subreddit_name):
-    """
-    Gets a summary of tickers for the image view, including post, comment,
-    and sentiment counts.
-    """
+def get_overall_summary(limit=50):
+    conn = get_db_connection()
+    query = """
+        SELECT
+            t.symbol, t.market_cap, t.closing_price,
+            COUNT(m.id) as mention_count,
+            SUM(CASE WHEN m.mention_sentiment > 0.1 THEN 1 ELSE 0 END) as bullish_mentions,
+            SUM(CASE WHEN m.mention_sentiment < -0.1 THEN 1 ELSE 0 END) as bearish_mentions,
+            SUM(CASE WHEN m.mention_sentiment BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions
+        FROM mentions m JOIN tickers t ON m.ticker_id = t.id
+        GROUP BY t.symbol, t.market_cap, t.closing_price
+        ORDER BY mention_count DESC LIMIT ?;
+    """
+    results = conn.execute(query, (limit,)).fetchall()
+    conn.close()
+    return results
+
+def get_subreddit_summary(subreddit_name, limit=50):
+    conn = get_db_connection()
+    query = """
+        SELECT
+            t.symbol, t.market_cap, t.closing_price,
+            COUNT(m.id) as mention_count,
+            SUM(CASE WHEN m.mention_sentiment > 0.1 THEN 1 ELSE 0 END) as bullish_mentions,
+            SUM(CASE WHEN m.mention_sentiment < -0.1 THEN 1 ELSE 0 END) as bearish_mentions,
+            SUM(CASE WHEN m.mention_sentiment BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions
+        FROM mentions m
+        JOIN tickers t ON m.ticker_id = t.id
+        JOIN subreddits s ON m.subreddit_id = s.id
+        WHERE s.name = ?
+        GROUP BY t.symbol, t.market_cap, t.closing_price
+        ORDER BY mention_count DESC LIMIT ?;
+    """
+    results = conn.execute(query, (subreddit_name, limit)).fetchall()
+    conn.close()
+    return results
+
+def get_image_view_summary(subreddit_name):
     conn = get_db_connection()
-    # This query now also counts sentiment types
     query = """
         SELECT
             t.symbol,
             COUNT(CASE WHEN m.mention_type = 'post' THEN 1 END) as post_mentions,
             COUNT(CASE WHEN m.mention_type = 'comment' THEN 1 END) as comment_mentions,
-            COUNT(CASE WHEN m.sentiment_score > 0.1 THEN 1 END) as bullish_mentions,
-            COUNT(CASE WHEN m.sentiment_score < -0.1 THEN 1 END) as bearish_mentions
+            COUNT(CASE WHEN m.mention_sentiment > 0.1 THEN 1 END) as bullish_mentions,
+            COUNT(CASE WHEN m.mention_sentiment < -0.1 THEN 1 END) as bearish_mentions
         FROM mentions m
         JOIN tickers t ON m.ticker_id = t.id
         JOIN subreddits s ON m.subreddit_id = s.id
@@ -280,23 +349,16 @@ def get_image_view_summary(subreddit_name):
     return results
 
 def get_weekly_summary_for_subreddit(subreddit_name):
-    """
-    Gets a weekly summary for a specific subreddit for the image view.
-    """
     conn = get_db_connection()
-    
-    # Calculate the timestamp for 7 days ago
     seven_days_ago = datetime.utcnow() - timedelta(days=7)
     seven_days_ago_timestamp = int(seven_days_ago.timestamp())
-
-    # The query is the same as before, but with an added WHERE clause for the timestamp
     query = """
         SELECT
             t.symbol,
             COUNT(CASE WHEN m.mention_type = 'post' THEN 1 END) as post_mentions,
             COUNT(CASE WHEN m.mention_type = 'comment' THEN 1 END) as comment_mentions,
-            COUNT(CASE WHEN m.sentiment_score > 0.1 THEN 1 END) as bullish_mentions,
-            COUNT(CASE WHEN m.sentiment_score < -0.1 THEN 1 END) as bearish_mentions
+            COUNT(CASE WHEN m.mention_sentiment > 0.1 THEN 1 END) as bullish_mentions,
+            COUNT(CASE WHEN m.mention_sentiment < -0.1 THEN 1 END) as bearish_mentions
         FROM mentions m
         JOIN tickers t ON m.ticker_id = t.id
         JOIN subreddits s ON m.subreddit_id = s.id

rstat_tool/format_blacklist.py

@@ -3,50 +3,52 @@
 COMMON_WORDS_BLACKLIST = {
     "401K", "403B", "457B", "ABOUT", "ABOVE", "ADAM", "ADX", "AEDT", "AEST", "AH",
     "AI", "ALL", "ALPHA", "ALSO", "AM", "AMA", "AMEX", "AND", "ANY", "AR",
-    "ARE", "ARK", "AROUND", "ASAP", "ASS", "ASSET", "AT", "ATH", "ATL", "ATM",
-    "AUD", "AWS", "BABY", "BAG", "BAGS", "BE", "BEAR", "BELOW", "BETA", "BIG",
-    "BIS", "BLEND", "BOE", "BOJ", "BOLL", "BOMB", "BOND", "BOTH", "BOTS", "BRB",
-    "BRL", "BS", "BST", "BSU", "BTC", "BTW", "BULL", "BUST", "BUT", "BUY",
-    "BUZZ", "CAD", "CALL", "CAN", "CAP", "CBS", "CCI", "CEO", "CEST", "CET",
-    "CEX", "CFD", "CFO", "CHF", "CHIPS", "CIA", "CLOSE", "CNBC", "CNY", "COKE",
-    "COME", "COST", "COULD", "CPAP", "CPI", "CSE", "CST", "CTB", "CTO", "CYCLE",
-    "CZK", "DAO", "DATE", "DAX", "DAY", "DCA", "DD", "DEBT", "DEX", "DIA",
-    "DIV", "DJIA", "DKK", "DM", "DO", "DOE", "DOGE", "DOJ", "DONT", "DR",
-    "EACH", "EARLY", "EARN", "ECB", "EDGAR", "EDIT", "EDT", "EMA", "END", "EOD",
-    "EOW", "EOY", "EPA", "EPS", "ER", "ESG", "EST", "ETF", "ETFS", "ETH",
-    "EU", "EUR", "EV", "EVEN", "EVERY", "FAQ", "FAR", "FAST", "FBI", "FD",
-    "FDA", "FIHTX", "FINRA", "FINT", "FINTX", "FINTY", "FIRST", "FOMC", "FOMO", "FOR",
-    "FOREX", "FRAUD", "FRG", "FROM", "FSPSX", "FTSE", "FUCK", "FUD", "FULL", "FUND",
-    "FXAIX", "FXIAX", "FY", "FYI", "FZROX", "GAAP", "GAIN", "GBP", "GDP", "GET",
-    "GL", "GLHF", "GMT", "GO", "GOAL", "GOAT", "GOING", "GPT", "GPU", "GRAB",
-    "GTG", "HALF", "HAS", "HATE", "HAVE", "HEAR", "HEDGE", "HELP", "HIGH", "HINT",
-    "HKD", "HODL", "HOLD", "HOUR", "HSA", "HUF", "IF", "II", "IKZ", "IMHO",
-    "IMO", "IN", "INR", "IP", "IPO", "IRA", "IRS", "IS", "ISA", "ISM",
-    "IST", "IT", "ITM", "IV", "IVV", "IWM", "JD", "JPOW", "JPY", "JST",
-    "JUST", "KARMA", "KEEP", "KNOW", "KO", "KRW", "LANGT", "LARGE", "LAST", "LATE",
-    "LATER", "LBO", "LEAP", "LEAPS", "LETS", "LFG", "LIKE", "LIMIT", "LLC", "LLM",
-    "LMAO", "LOKO", "LOL", "LONG", "LOOK", "LOSS", "LOVE", "LOW", "M&A", "MA",
-    "MACD", "MAKE", "MAX", "MC", "ME", "MEME", "MERK", "MEXC", "MID", "MIGHT",
-    "MIN", "MIND", "ML", "MOASS", "MONTH", "MORE", "MSK", "MUSIC", "MUST", "MXN",
-    "MY", "NATO", "NEAR", "NEED", "NEVER", "NEW", "NEXT", "NFA", "NFC", "NFT",
-    "NGMI", "NIGHT", "NO", "NOK", "NONE", "NOT", "NOW", "NSA", "NULL", "NUT",
-    "NYSE", "NZD", "OBV", "OEM", "OF", "OG", "OK", "OLD", "ON", "ONE",
-    "ONLY", "OP", "OPEX", "OR", "OS", "OSCE", "OTC", "OTM", "OUGHT", "OUT",
-    "OVER", "OWN", "PANIC", "PC", "PDT", "PE", "PEAK", "PEG", "PEW", "PLAN",
-    "PLN", "PM", "PMI", "POC", "POS", "PPI", "PR", "PRICE", "PROFIT", "PSA",
-    "PST", "PT", "PUT", "Q1", "Q2", "Q3", "Q4", "QQQ", "QR", "RBA",
-    "RBNZ", "RE", "REAL", "REIT", "REKT", "RH", "RIGHT", "RIP", "RISK", "ROCK",
-    "ROE", "ROFL", "ROI", "ROTH", "RSD", "RSI", "RUB", "RULE", "SAME", "SAVE",
-    "SCALP", "SCAM", "SCHB", "SEC", "SEE", "SEK", "SELL", "SEP", "SGD", "SHALL",
-    "SHARE", "SHORT", "SL", "SMA", "SMALL", "SO", "SOLIS", "SOME", "SOON", "SP",
-    "SPAC", "SPEND", "SPLG", "SPX", "SPY", "START", "STILL", "STOCK", "STOP", "STOR",
-    "SWING", "TA", "TAG", "TAKE", "TERM", "THANK", "THAT", "THE", "THINK", "THIS",
-    "TIME", "TITS", "TL", "TL;DR", "TLDR", "TO", "TODAY", "TOTAL", "TRADE", "TREND",
-    "TRUE", "TRY", "TTYL", "TWO", "UI", "UK", "UNDER", "UP", "US", "USA",
-    "USD", "UTC", "VALUE", "VOO", "VP", "VR", "VTI", "WAGMI", "WANT", "WATCH",
-    "WAY", "WE", "WEB3", "WEEK", "WHALE", "WHO", "WHY", "WIDE", "WILL", "WORDS",
-    "WORTH", "WOULD", "WSB", "WTF", "XRP", "YES", "YET", "YIELD", "YOLO", "YOU",
-    "YOUR", "YOY", "YT", "YTD", "ZAR", "ZEN", "ZERO"
+    "ARE", "AREA", "ARK", "AROUND", "ASAP", "ASS", "ASSET", "AT", "ATH", "ATL",
+    "ATM", "AUD", "AWS", "BABY", "BAG", "BAGS", "BE", "BEAR", "BELOW", "BETA",
+    "BIG", "BIS", "BLEND", "BOE", "BOJ", "BOLL", "BOMB", "BOND", "BOTH", "BOTS",
+    "BRB", "BRL", "BROKE", "BS", "BST", "BSU", "BTC", "BTW", "BULL", "BUST",
+    "BUT", "BUY", "BUZZ", "CAD", "CALL", "CAN", "CAP", "CBS", "CCI", "CEO",
+    "CEST", "CET", "CEX", "CFD", "CFO", "CHF", "CHIPS", "CIA", "CLOSE", "CNBC",
+    "CNY", "COKE", "COME", "COST", "COULD", "CPAP", "CPI", "CSE", "CST", "CTB",
+    "CTO", "CYCLE", "CZK", "DAO", "DATE", "DAX", "DAY", "DAYS", "DCA", "DD",
+    "DEBT", "DEX", "DIA", "DIV", "DJIA", "DKK", "DM", "DO", "DOE", "DOGE",
+    "DOJ", "DONT", "DR", "EACH", "EARLY", "EARN", "EAST", "ECB", "EDGAR", "EDIT",
+    "EDT", "EMA", "END", "ENV", "EOD", "EOW", "EOY", "EPA", "EPS", "ER",
+    "ESG", "EST", "ETF", "ETFS", "ETH", "EU", "EUR", "EV", "EVEN", "EVERY",
+    "FAQ", "FAR", "FAST", "FBI", "FD", "FDA", "FIHTX", "FINRA", "FINT", "FINTX",
+    "FINTY", "FIRST", "FOMC", "FOMO", "FOR", "FOREX", "FRAUD", "FRG", "FROM", "FSPSX",
+    "FTD", "FTSE", "FUCK", "FUD", "FULL", "FUND", "FXAIX", "FXIAX", "FY", "FYI",
+    "FZROX", "GAAP", "GAIN", "GBP", "GDP", "GET", "GL", "GLHF", "GMT", "GO",
+    "GOAL", "GOAT", "GOING", "GPT", "GPU", "GRAB", "GTG", "GUH", "HALF", "HAS",
+    "HATE", "HAVE", "HEAR", "HEDGE", "HELP", "HIGH", "HINT", "HKD", "HODL", "HOLD",
+    "HOUR", "HSA", "HUF", "HUGE", "IBS", "IF", "II", "IKKE", "IKZ", "IMHO",
+    "IMO", "IN", "INR", "IP", "IPO", "IRA", "IRS", "IS", "ISA", "ISIN",
+    "ISM", "IST", "IT", "ITM", "IV", "IVV", "IWM", "JD", "JPOW", "JPY",
+    "JST", "JUST", "KARMA", "KEEP", "KNOW", "KO", "KRW", "LANGT", "LARGE", "LAST",
+    "LATE", "LATER", "LBO", "LEAP", "LEAPS", "LETS", "LFG", "LIKE", "LIMIT", "LLC",
+    "LLM", "LMAO", "LOKO", "LOL", "LONG", "LOOK", "LOSS", "LOVE", "LOW", "M&A",
+    "MA", "MACD", "MAKE", "MAX", "MC", "ME", "MEME", "MERK", "MEXC", "MID",
+    "MIGHT", "MIN", "MIND", "ML", "MOASS", "MONTH", "MORE", "MSK", "MUSIC", "MUST",
+    "MXN", "MY", "NASA", "NATO", "NEAR", "NEAT", "NEED", "NEVER", "NEW", "NEXT",
+    "NFA", "NFC", "NFT", "NGMI", "NIGHT", "NO", "NOK", "NONE", "NORTH", "NOT",
+    "NOW", "NSA", "NULL", "NUT", "NYSE", "NZ", "NZD", "OBV", "OEM", "OF",
+    "OG", "OK", "OLD", "ON", "ONE", "ONLY", "OP", "OPEX", "OR", "OS",
+    "OSCE", "OTC", "OTM", "OUGHT", "OUT", "OVER", "OWN", "PANIC", "PC", "PDT",
+    "PE", "PEAK", "PEG", "PEW", "PLAN", "PLN", "PM", "PMI", "POC", "POS",
+    "POSCO", "PPI", "PR", "PRICE", "PROFIT", "PSA", "PST", "PT", "PUT", "Q1",
+    "Q2", "Q3", "Q4", "QQQ", "QR", "RBA", "RBNZ", "RE", "REAL", "REIT",
+    "REKT", "RH", "RIGHT", "RIP", "RISK", "ROCK", "ROE", "ROFL", "ROI", "ROTH",
+    "RSD", "RSI", "RUB", "RULE", "SAME", "SAVE", "SCALP", "SCAM", "SCHB", "SEC",
+    "SEE", "SEK", "SELL", "SEP", "SGD", "SHALL", "SHARE", "SHORT", "SL", "SMA",
+    "SMALL", "SO", "SOLIS", "SOME", "SOON", "SOUTH", "SP", "SPAC", "SPEND", "SPLG",
+    "SPX", "SPY", "START", "STILL", "STOCK", "STOP", "STOR", "SWING", "TA", "TAG",
+    "TAKE", "TERM", "THANK", "THAT", "THE", "THINK", "THIS", "TIME", "TITS", "TL",
+    "TL;DR", "TLDR", "TO", "TODAY", "TOLD", "TOTAL", "TRADE", "TREND", "TRUE", "TRY",
+    "TTYL", "TWO", "UI", "UK", "UNDER", "UP", "US", "USA", "USD", "UTC",
+    "VALUE", "VOO", "VP", "VR", "VTI", "WAGMI", "WANT", "WATCH", "WAY", "WE",
+    "WEB3", "WEEK", "WEST", "WHALE", "WHO", "WHY", "WIDE", "WILL", "WIRE", "WORDS",
+    "WORTH", "WOULD", "WSB", "WTF", "XO", "XRP", "YES", "YET", "YIELD", "YOLO",
+    "YOU", "YOUR", "YOY", "YT", "YTD", "ZAR", "ZEN", "ZERO"
 }
 
 def format_and_print_list(word_set, words_per_line=10):

rstat_tool/main.py

@@ -43,6 +43,11 @@ def get_reddit_instance():
     return praw.Reddit(client_id=client_id, client_secret=client_secret, user_agent=user_agent)
 
 def scan_subreddits(reddit, subreddits_list, post_limit=100, comment_limit=100, days_to_scan=1):
+    """
+    Scans subreddits with a hybrid mention counting logic.
+    - If a ticker is in the title, it gets credit for all comments.
+    - If not, tickers only get credit for direct mentions in comments.
+    """
     conn = database.get_db_connection()
     post_age_limit = days_to_scan * 86400
     current_time = time.time()
@@ -56,53 +61,77 @@ def scan_subreddits(reddit, subreddits_list, post_limit=100, comment_limit=100,
             
             for submission in subreddit.new(limit=post_limit):
                 if (current_time - submission.created_utc) > post_age_limit:
-                    print(f"  -> Reached posts older than the {days_to_scan}-day limit. Moving to next subreddit.")
+                    print(f"  -> Reached posts older than the {days_to_scan}-day limit.")
                     break
 
-                post_text = submission.title + " " + submission.selftext
-                tickers_in_post = extract_tickers(post_text)
-                if tickers_in_post:
-                    post_sentiment = get_sentiment_score(submission.title)
-                    for ticker_symbol in set(tickers_in_post):
-                        ticker_id = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol)
-                        database.add_mention(conn, ticker_id, subreddit_id, submission.id, 'post', int(submission.created_utc), post_sentiment)
-                        
-                        ticker_info = database.get_ticker_info(conn, ticker_id)
-                        if not ticker_info['last_updated'] or (current_time - ticker_info['last_updated'] > MARKET_CAP_REFRESH_INTERVAL):
-                            print(f"  -> Fetching financial data for {ticker_symbol}...")
-                            financials = get_financial_data(ticker_symbol)
-                            database.update_ticker_financials(
-                                conn, ticker_id,
-                                financials['market_cap'] or ticker_info['market_cap'],
-                                financials['closing_price'] or ticker_info['closing_price']
-                            )
+                # --- NEW HYBRID LOGIC ---
+                
+                tickers_in_title = set(extract_tickers(submission.title))
+                all_tickers_found_in_post = set(tickers_in_title) # Start a set to track all tickers for financials
 
                 submission.comments.replace_more(limit=0)
-                all_comment_sentiments = []
-                for comment in submission.comments.list()[:comment_limit]:
-                    all_comment_sentiments.append(get_sentiment_score(comment.body))
-                    tickers_in_comment = extract_tickers(comment.body)
-                    if tickers_in_comment:
+                all_comments = submission.comments.list()[:comment_limit]
+                
+                # --- CASE A: Tickers were found in the title ---
+                if tickers_in_title:
+                    print(f"  -> Title Mention(s): {', '.join(tickers_in_title)}. Attributing all comments.")
+                    post_sentiment = get_sentiment_score(submission.title)
+                    
+                    # Add one 'post' mention for each title ticker
+                    for ticker_symbol in tickers_in_title:
+                        ticker_id = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol)
+                        database.add_mention(conn, ticker_id, subreddit_id, submission.id, 'post', int(submission.created_utc), post_sentiment)
+
+                    # Add one 'comment' mention for EACH comment FOR EACH title ticker
+                    for comment in all_comments:
                         comment_sentiment = get_sentiment_score(comment.body)
-                        for ticker_symbol in set(tickers_in_comment):
+                        for ticker_symbol in tickers_in_title:
                             ticker_id = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol)
                             database.add_mention(conn, ticker_id, subreddit_id, submission.id, 'comment', int(comment.created_utc), comment_sentiment)
+
+                # --- CASE B: No tickers in the title, scan comments individually ---
+                else:
+                    for comment in all_comments:
+                        tickers_in_comment = set(extract_tickers(comment.body))
+                        if tickers_in_comment:
+                            all_tickers_found_in_post.update(tickers_in_comment) # Add to our set for financials
+                            comment_sentiment = get_sentiment_score(comment.body)
+                            for ticker_symbol in tickers_in_comment:
+                                ticker_id = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol)
+                                database.add_mention(conn, ticker_id, subreddit_id, submission.id, 'comment', int(comment.created_utc), comment_sentiment)
                 
+                # --- EFFICIENT FINANCIALS UPDATE ---
+                # Now, update market cap once for every unique ticker found in the whole post
+                for ticker_symbol in all_tickers_found_in_post:
+                    ticker_id = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol)
+                    ticker_info = database.get_ticker_info(conn, ticker_id)
+                    if not ticker_info['last_updated'] or (current_time - ticker_info['last_updated'] > MARKET_CAP_REFRESH_INTERVAL):
+                        print(f"    -> Fetching financial data for {ticker_symbol}...")
+                        financials = get_financial_data(ticker_symbol)
+                        database.update_ticker_financials(
+                            conn, ticker_id,
+                            financials['market_cap'] or ticker_info['market_cap'],
+                            financials['closing_price'] or ticker_info['closing_price']
+                        )
+                
+                # --- DEEP DIVE SAVE (Still valuable) ---
+                all_comment_sentiments = [get_sentiment_score(c.body) for c in all_comments]
                 avg_sentiment = sum(all_comment_sentiments) / len(all_comment_sentiments) if all_comment_sentiments else 0
                 post_analysis_data = {
                     "post_id": submission.id, "title": submission.title,
-                    "post_url": f"https://reddit.com{submission.permalink}",
-                    "subreddit_id": subreddit_id, "post_timestamp": int(submission.created_utc),
-                    "comment_count": len(all_comment_sentiments), "avg_comment_sentiment": avg_sentiment
+                    "post_url": f"https://reddit.com{submission.permalink}", "subreddit_id": subreddit_id,
+                    "post_timestamp": int(submission.created_utc), "comment_count": len(all_comments),
+                    "avg_comment_sentiment": avg_sentiment
                 }
                 database.add_or_update_post_analysis(conn, post_analysis_data)
-
+                
         except Exception as e:
             print(f"Could not scan r/{subreddit_name}. Error: {e}")
             
     conn.close()
     print("\n--- Scan Complete ---")
 
+
 def main():
     """Main function to run the Reddit stock analysis tool."""
     parser = argparse.ArgumentParser(description="Analyze stock ticker mentions on Reddit.", formatter_class=argparse.RawTextHelpFormatter)

rstat_tool/ticker_extractor.py

@@ -7,50 +7,52 @@ import re
 COMMON_WORDS_BLACKLIST = {
     "401K", "403B", "457B", "ABOUT", "ABOVE", "ADAM", "ADX", "AEDT", "AEST", "AH",
     "AI", "ALL", "ALPHA", "ALSO", "AM", "AMA", "AMEX", "AND", "ANY", "AR",
-    "ARE", "ARK", "AROUND", "ASAP", "ASS", "ASSET", "AT", "ATH", "ATL", "ATM",
-    "AUD", "AWS", "BABY", "BAG", "BAGS", "BE", "BEAR", "BELOW", "BETA", "BIG",
-    "BIS", "BLEND", "BOE", "BOJ", "BOLL", "BOMB", "BOND", "BOTH", "BOTS", "BRB",
-    "BRL", "BS", "BST", "BSU", "BTC", "BTW", "BULL", "BUST", "BUT", "BUY",
-    "BUZZ", "CAD", "CALL", "CAN", "CAP", "CBS", "CCI", "CEO", "CEST", "CET",
-    "CEX", "CFD", "CFO", "CHF", "CHIPS", "CIA", "CLOSE", "CNBC", "CNY", "COKE",
-    "COME", "COST", "COULD", "CPAP", "CPI", "CSE", "CST", "CTB", "CTO", "CYCLE",
-    "CZK", "DAO", "DATE", "DAX", "DAY", "DCA", "DD", "DEBT", "DEX", "DIA",
-    "DIV", "DJIA", "DKK", "DM", "DO", "DOE", "DOGE", "DOJ", "DONT", "DR",
-    "EACH", "EARLY", "EARN", "ECB", "EDGAR", "EDIT", "EDT", "EMA", "END", "EOD",
-    "EOW", "EOY", "EPA", "EPS", "ER", "ESG", "EST", "ETF", "ETFS", "ETH",
-    "EU", "EUR", "EV", "EVEN", "EVERY", "FAQ", "FAR", "FAST", "FBI", "FD",
-    "FDA", "FIHTX", "FINRA", "FINT", "FINTX", "FINTY", "FIRST", "FOMC", "FOMO", "FOR",
-    "FOREX", "FRAUD", "FRG", "FROM", "FSPSX", "FTSE", "FUCK", "FUD", "FULL", "FUND",
-    "FXAIX", "FXIAX", "FY", "FYI", "FZROX", "GAAP", "GAIN", "GBP", "GDP", "GET",
-    "GL", "GLHF", "GMT", "GO", "GOAL", "GOAT", "GOING", "GPT", "GPU", "GRAB",
-    "GTG", "HALF", "HAS", "HATE", "HAVE", "HEAR", "HEDGE", "HELP", "HIGH", "HINT",
-    "HKD", "HODL", "HOLD", "HOUR", "HSA", "HUF", "IF", "II", "IKZ", "IMHO",
-    "IMO", "IN", "INR", "IP", "IPO", "IRA", "IRS", "IS", "ISA", "ISM",
-    "IST", "IT", "ITM", "IV", "IVV", "IWM", "JD", "JPOW", "JPY", "JST",
-    "JUST", "KARMA", "KEEP", "KNOW", "KO", "KRW", "LANGT", "LARGE", "LAST", "LATE",
-    "LATER", "LBO", "LEAP", "LEAPS", "LETS", "LFG", "LIKE", "LIMIT", "LLC", "LLM",
-    "LMAO", "LOKO", "LOL", "LONG", "LOOK", "LOSS", "LOVE", "LOW", "M&A", "MA",
-    "MACD", "MAKE", "MAX", "MC", "ME", "MEME", "MERK", "MEXC", "MID", "MIGHT",
-    "MIN", "MIND", "ML", "MOASS", "MONTH", "MORE", "MSK", "MUSIC", "MUST", "MXN",
-    "MY", "NATO", "NEAR", "NEED", "NEVER", "NEW", "NEXT", "NFA", "NFC", "NFT",
-    "NGMI", "NIGHT", "NO", "NOK", "NONE", "NOT", "NOW", "NSA", "NULL", "NUT",
-    "NYSE", "NZD", "OBV", "OEM", "OF", "OG", "OK", "OLD", "ON", "ONE",
-    "ONLY", "OP", "OPEX", "OR", "OS", "OSCE", "OTC", "OTM", "OUGHT", "OUT",
-    "OVER", "OWN", "PANIC", "PC", "PDT", "PE", "PEAK", "PEG", "PEW", "PLAN",
-    "PLN", "PM", "PMI", "POC", "POS", "PPI", "PR", "PRICE", "PROFIT", "PSA",
-    "PST", "PT", "PUT", "Q1", "Q2", "Q3", "Q4", "QQQ", "QR", "RBA",
-    "RBNZ", "RE", "REAL", "REIT", "REKT", "RH", "RIGHT", "RIP", "RISK", "ROCK",
-    "ROE", "ROFL", "ROI", "ROTH", "RSD", "RSI", "RUB", "RULE", "SAME", "SAVE",
-    "SCALP", "SCAM", "SCHB", "SEC", "SEE", "SEK", "SELL", "SEP", "SGD", "SHALL",
-    "SHARE", "SHORT", "SL", "SMA", "SMALL", "SO", "SOLIS", "SOME", "SOON", "SP",
-    "SPAC", "SPEND", "SPLG", "SPX", "SPY", "START", "STILL", "STOCK", "STOP", "STOR",
-    "SWING", "TA", "TAG", "TAKE", "TERM", "THANK", "THAT", "THE", "THINK", "THIS",
-    "TIME", "TITS", "TL", "TL;DR", "TLDR", "TO", "TODAY", "TOTAL", "TRADE", "TREND",
-    "TRUE", "TRY", "TTYL", "TWO", "UI", "UK", "UNDER", "UP", "US", "USA",
-    "USD", "UTC", "VALUE", "VOO", "VP", "VR", "VTI", "WAGMI", "WANT", "WATCH",
-    "WAY", "WE", "WEB3", "WEEK", "WHALE", "WHO", "WHY", "WIDE", "WILL", "WORDS",
-    "WORTH", "WOULD", "WSB", "WTF", "XRP", "YES", "YET", "YIELD", "YOLO", "YOU",
-    "YOUR", "YOY", "YT", "YTD", "ZAR", "ZEN", "ZERO"
+    "ARE", "AREA", "ARK", "AROUND", "ASAP", "ASS", "ASSET", "AT", "ATH", "ATL",
+    "ATM", "AUD", "AWS", "BABY", "BAG", "BAGS", "BE", "BEAR", "BELOW", "BETA",
+    "BIG", "BIS", "BLEND", "BOE", "BOJ", "BOLL", "BOMB", "BOND", "BOTH", "BOTS",
+    "BRB", "BRL", "BROKE", "BS", "BST", "BSU", "BTC", "BTW", "BULL", "BUST",
+    "BUT", "BUY", "BUZZ", "CAD", "CALL", "CAN", "CAP", "CBS", "CCI", "CEO",
+    "CEST", "CET", "CEX", "CFD", "CFO", "CHF", "CHIPS", "CIA", "CLOSE", "CNBC",
+    "CNY", "COKE", "COME", "COST", "COULD", "CPAP", "CPI", "CSE", "CST", "CTB",
+    "CTO", "CYCLE", "CZK", "DAO", "DATE", "DAX", "DAY", "DAYS", "DCA", "DD",
+    "DEBT", "DEX", "DIA", "DIV", "DJIA", "DKK", "DM", "DO", "DOE", "DOGE",
+    "DOJ", "DONT", "DR", "EACH", "EARLY", "EARN", "EAST", "ECB", "EDGAR", "EDIT",
+    "EDT", "EMA", "END", "ENV", "EOD", "EOW", "EOY", "EPA", "EPS", "ER",
+    "ESG", "EST", "ETF", "ETFS", "ETH", "EU", "EUR", "EV", "EVEN", "EVERY",
+    "FAQ", "FAR", "FAST", "FBI", "FD", "FDA", "FIHTX", "FINRA", "FINT", "FINTX",
+    "FINTY", "FIRST", "FOMC", "FOMO", "FOR", "FOREX", "FRAUD", "FRG", "FROM", "FSPSX",
+    "FTD", "FTSE", "FUCK", "FUD", "FULL", "FUND", "FXAIX", "FXIAX", "FY", "FYI",
+    "FZROX", "GAAP", "GAIN", "GBP", "GDP", "GET", "GL", "GLHF", "GMT", "GO",
+    "GOAL", "GOAT", "GOING", "GPT", "GPU", "GRAB", "GTG", "GUH", "HALF", "HAS",
+    "HATE", "HAVE", "HEAR", "HEDGE", "HELP", "HIGH", "HINT", "HKD", "HODL", "HOLD",
+    "HOUR", "HSA", "HUF", "HUGE", "IBS", "IF", "II", "IKKE", "IKZ", "IMHO",
+    "IMO", "IN", "INR", "IP", "IPO", "IRA", "IRS", "IS", "ISA", "ISIN",
+    "ISM", "IST", "IT", "ITM", "IV", "IVV", "IWM", "JD", "JPOW", "JPY",
+    "JST", "JUST", "KARMA", "KEEP", "KNOW", "KO", "KRW", "LANGT", "LARGE", "LAST",
+    "LATE", "LATER", "LBO", "LEAP", "LEAPS", "LETS", "LFG", "LIKE", "LIMIT", "LLC",
+    "LLM", "LMAO", "LOKO", "LOL", "LONG", "LOOK", "LOSS", "LOVE", "LOW", "M&A",
+    "MA", "MACD", "MAKE", "MAX", "MC", "ME", "MEME", "MERK", "MEXC", "MID",
+    "MIGHT", "MIN", "MIND", "ML", "MOASS", "MONTH", "MORE", "MSK", "MUSIC", "MUST",
+    "MXN", "MY", "NASA", "NATO", "NEAR", "NEAT", "NEED", "NEVER", "NEW", "NEXT",
+    "NFA", "NFC", "NFT", "NGMI", "NIGHT", "NO", "NOK", "NONE", "NORTH", "NOT",
+    "NOW", "NSA", "NULL", "NUT", "NYSE", "NZ", "NZD", "OBV", "OEM", "OF",
+    "OG", "OK", "OLD", "ON", "ONE", "ONLY", "OP", "OPEX", "OR", "OS",
+    "OSCE", "OTC", "OTM", "OUGHT", "OUT", "OVER", "OWN", "PANIC", "PC", "PDT",
+    "PE", "PEAK", "PEG", "PEW", "PLAN", "PLN", "PM", "PMI", "POC", "POS",
+    "POSCO", "PPI", "PR", "PRICE", "PROFIT", "PSA", "PST", "PT", "PUT", "Q1",
+    "Q2", "Q3", "Q4", "QQQ", "QR", "RBA", "RBNZ", "RE", "REAL", "REIT",
+    "REKT", "RH", "RIGHT", "RIP", "RISK", "ROCK", "ROE", "ROFL", "ROI", "ROTH",
+    "RSD", "RSI", "RUB", "RULE", "SAME", "SAVE", "SCALP", "SCAM", "SCHB", "SEC",
+    "SEE", "SEK", "SELL", "SEP", "SGD", "SHALL", "SHARE", "SHORT", "SL", "SMA",
+    "SMALL", "SO", "SOLIS", "SOME", "SOON", "SOUTH", "SP", "SPAC", "SPEND", "SPLG",
+    "SPX", "SPY", "START", "STILL", "STOCK", "STOP", "STOR", "SWING", "TA", "TAG",
+    "TAKE", "TERM", "THANK", "THAT", "THE", "THINK", "THIS", "TIME", "TITS", "TL",
+    "TL;DR", "TLDR", "TO", "TODAY", "TOLD", "TOTAL", "TRADE", "TREND", "TRUE", "TRY",
+    "TTYL", "TWO", "UI", "UK", "UNDER", "UP", "US", "USA", "USD", "UTC",
+    "VALUE", "VOO", "VP", "VR", "VTI", "WAGMI", "WANT", "WATCH", "WAY", "WE",
+    "WEB3", "WEEK", "WEST", "WHALE", "WHO", "WHY", "WIDE", "WILL", "WIRE", "WORDS",
+    "WORTH", "WOULD", "WSB", "WTF", "XO", "XRP", "YES", "YET", "YIELD", "YOLO",
+    "YOU", "YOUR", "YOY", "YT", "YTD", "ZAR", "ZEN", "ZERO"
 }
 
 def extract_tickers(text):