Modularized the tool.

rstat_tool/database.py

@@ -0,0 +1,158 @@
+# rstat_tool/database.py
+
+import sqlite3
+import time
+# --- IMPORT ADDED BACK IN ---
+from .ticker_extractor import COMMON_WORDS_BLACKLIST
+
+DB_FILE = "reddit_stocks.db"
+
+def get_db_connection():
+    """Establishes a connection to the SQLite database."""
+    conn = sqlite3.connect(DB_FILE)
+    conn.row_factory = sqlite3.Row
+    return conn
+
+def initialize_db():
+    # ... (This function is unchanged)
+    conn = get_db_connection()
+    cursor = conn.cursor()
+    cursor.execute("""
+        CREATE TABLE IF NOT EXISTS tickers (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            symbol TEXT NOT NULL UNIQUE,
+            market_cap INTEGER,
+            last_updated INTEGER
+        )
+    """)
+    cursor.execute("""
+        CREATE TABLE IF NOT EXISTS subreddits (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            name TEXT NOT NULL UNIQUE
+        )
+    """)
+    cursor.execute("""
+        CREATE TABLE IF NOT EXISTS mentions (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            ticker_id INTEGER,
+            subreddit_id INTEGER,
+            post_id TEXT NOT NULL,
+            mention_timestamp INTEGER NOT NULL,
+            sentiment_score REAL,
+            FOREIGN KEY (ticker_id) REFERENCES tickers (id),
+            FOREIGN KEY (subreddit_id) REFERENCES subreddits (id),
+            UNIQUE(ticker_id, post_id, sentiment_score)
+        )
+    """)
+    conn.commit()
+    conn.close()
+    print("Database initialized successfully.")
+
+# --- CLEANUP FUNCTION ADDED BACK IN ---
+def clean_stale_tickers():
+    """
+    Removes tickers and their associated mentions from the database
+    if the ticker symbol exists in the COMMON_WORDS_BLACKLIST.
+    """
+    print("\n--- Cleaning Stale Tickers from Database ---")
+    conn = get_db_connection()
+    cursor = conn.cursor()
+
+    # Find ticker IDs that match the blacklist
+    placeholders = ','.join('?' for _ in COMMON_WORDS_BLACKLIST)
+    query = f"SELECT id, symbol FROM tickers WHERE symbol IN ({placeholders})"
+    
+    cursor.execute(query, tuple(COMMON_WORDS_BLACKLIST))
+    stale_tickers = cursor.fetchall()
+
+    if not stale_tickers:
+        print("No stale tickers to clean.")
+        conn.close()
+        return
+
+    for ticker in stale_tickers:
+        ticker_id = ticker['id']
+        ticker_symbol = ticker['symbol']
+        print(f"Removing stale ticker '{ticker_symbol}' (ID: {ticker_id})...")
+        
+        # 1. Delete all mentions associated with this ticker ID
+        cursor.execute("DELETE FROM mentions WHERE ticker_id = ?", (ticker_id,))
+        
+        # 2. Delete the ticker itself
+        cursor.execute("DELETE FROM tickers WHERE id = ?", (ticker_id,))
+    
+    deleted_count = conn.total_changes
+    conn.commit()
+    conn.close()
+    print(f"Cleanup complete. Removed {deleted_count} records.")
+
+
+def add_mention(conn, ticker_id, subreddit_id, post_id, timestamp, sentiment):
+    # ... (This function is unchanged)
+    cursor = conn.cursor()
+    try:
+        cursor.execute(
+            "INSERT INTO mentions (ticker_id, subreddit_id, post_id, mention_timestamp, sentiment_score) VALUES (?, ?, ?, ?, ?)",
+            (ticker_id, subreddit_id, post_id, timestamp, sentiment)
+        )
+        conn.commit()
+    except sqlite3.IntegrityError:
+        pass
+
+# ... (get_or_create_entity, update_ticker_market_cap, get_ticker_info are unchanged)
+def get_or_create_entity(conn, table_name, column_name, value):
+    # ...
+    cursor = conn.cursor()
+    cursor.execute(f"SELECT id FROM {table_name} WHERE {column_name} = ?", (value,))
+    result = cursor.fetchone()
+    if result: return result['id']
+    else:
+        cursor.execute(f"INSERT INTO {table_name} ({column_name}) VALUES (?)", (value,))
+        conn.commit()
+        return cursor.lastrowid
+
+def update_ticker_market_cap(conn, ticker_id, market_cap):
+    # ...
+    cursor = conn.cursor()
+    current_timestamp = int(time.time())
+    cursor.execute("UPDATE tickers SET market_cap = ?, last_updated = ? WHERE id = ?", (market_cap, current_timestamp, ticker_id))
+    conn.commit()
+
+def get_ticker_info(conn, ticker_id):
+    # ...
+    cursor = conn.cursor()
+    cursor.execute("SELECT * FROM tickers WHERE id = ?", (ticker_id,))
+    return cursor.fetchone()
+
+def generate_summary_report(limit=20):
+    # ... (This function is unchanged)
+    print(f"\n--- Top {limit} Tickers by Mention Count ---")
+    conn = get_db_connection()
+    cursor = conn.cursor()
+    query = """
+        SELECT
+            t.symbol,
+            t.market_cap,
+            COUNT(m.id) as mention_count,
+            SUM(CASE WHEN m.sentiment_score > 0.1 THEN 1 ELSE 0 END) as bullish_mentions,
+            SUM(CASE WHEN m.sentiment_score < -0.1 THEN 1 ELSE 0 END) as bearish_mentions,
+            SUM(CASE WHEN m.sentiment_score BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions
+        FROM mentions m
+        JOIN tickers t ON m.ticker_id = t.id
+        GROUP BY t.symbol, t.market_cap
+        ORDER BY mention_count DESC
+        LIMIT ?;
+    """
+    results = cursor.execute(query, (limit,)).fetchall()
+    header = f"{'Ticker':<8} | {'Mentions':<8} | {'Bullish':<8} | {'Bearish':<8} | {'Neutral':<8} | {'Market Cap':<15}"
+    print(header)
+    print("-" * len(header))
+    for row in results:
+        market_cap_str = "N/A"
+        if row['market_cap'] and row['market_cap'] > 0:
+            mc = row['market_cap']
+            if mc >= 1e12: market_cap_str = f"${mc/1e12:.2f}T"
+            elif mc >= 1e9: market_cap_str = f"${mc/1e9:.2f}B"
+            else: market_cap_str = f"${mc/1e6:.2f}M"
+        print(f"{row['symbol']:<8} | {row['mention_count']:<8} | {row['bullish_mentions']:<8} | {row['bearish_mentions']:<8} | {row['neutral_mentions']:<8} | {market_cap_str:<15}")
+    conn.close()

rstat_tool/main.py

@@ -0,0 +1,149 @@
+# main.py
+
+import argparse
+import json
+import os
+import time
+
+import praw
+import yfinance as yf
+from dotenv import load_dotenv
+
+# Import local modules
+from . import database
+from .ticker_extractor import extract_tickers
+from .sentiment_analyzer import get_sentiment_score
+
+load_dotenv()
+MARKET_CAP_REFRESH_INTERVAL = 86400
+
+# (load_subreddits, get_market_cap, get_reddit_instance functions are unchanged)
+def load_subreddits(filepath):
+    try:
+        with open(filepath, 'r') as f:
+            return json.load(f).get("subreddits", [])
+    except (FileNotFoundError, json.JSONDecodeError) as e:
+        print(f"Error loading config: {e}")
+        return None
+
+def get_market_cap(ticker_symbol):
+    try:
+        ticker = yf.Ticker(ticker_symbol)
+        return ticker.fast_info.get('marketCap')
+    except Exception:
+        return None
+
+def get_reddit_instance():
+    client_id = os.getenv("REDDIT_CLIENT_ID")
+    client_secret = os.getenv("REDDIT_CLIENT_SECRET")
+    user_agent = os.getenv("REDDIT_USER_AGENT")
+    if not all([client_id, client_secret, user_agent]):
+        print("Error: Reddit API credentials not found.")
+        return None
+    return praw.Reddit(client_id=client_id, client_secret=client_secret, user_agent=user_agent)
+
+# --- UPDATED: Function now accepts post_limit and comment_limit ---
+def scan_subreddits(reddit, subreddits_list, post_limit=25, comment_limit=100):
+    """Scans subreddits, analyzes posts and comments, and stores results in the database."""
+    conn = database.get_db_connection()
+    
+    print(f"\nScanning {len(subreddits_list)} subreddits (Top {post_limit} posts, {comment_limit} comments/post)...")
+    for subreddit_name in subreddits_list:
+        try:
+            subreddit_id = database.get_or_create_entity(conn, 'subreddits', 'name', subreddit_name)
+            subreddit = reddit.subreddit(subreddit_name)
+            print(f"Scanning r/{subreddit_name}...")
+            
+            for submission in subreddit.hot(limit=post_limit):
+                # --- 1. Process the Post Title and Body ---
+                post_text = submission.title + " " + submission.selftext
+                tickers_in_post = extract_tickers(post_text)
+                post_sentiment = get_sentiment_score(submission.title)
+                
+                for ticker_symbol in set(tickers_in_post):
+                    ticker_id = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol)
+                    database.add_mention(conn, ticker_id, subreddit_id, submission.id, int(submission.created_utc), post_sentiment)
+                    # (Market cap logic remains the same)
+                    ticker_info = database.get_ticker_info(conn, ticker_id)
+                    current_time = int(time.time())
+                    if not ticker_info['last_updated'] or (current_time - ticker_info['last_updated'] > MARKET_CAP_REFRESH_INTERVAL):
+                        print(f"  -> Fetching market cap for {ticker_symbol}...")
+                        market_cap = get_market_cap(ticker_symbol)
+                        database.update_ticker_market_cap(conn, ticker_id, market_cap or ticker_info['market_cap'])
+                
+                # --- 2. Process the Comments ---
+                # Expand "MoreComments" objects. limit=None means we try to get all, but PRAW is protective.
+                # A limit of 32 is the max PRAW will do in a single call. We'll iterate to be safe.
+                submission.comments.replace_more(limit=10) 
+                comment_count = 0
+                for comment in submission.comments.list():
+                    if comment_count >= comment_limit:
+                        break # Stop processing comments for this post if we hit our limit
+
+                    tickers_in_comment = extract_tickers(comment.body)
+                    if not tickers_in_comment:
+                        continue # Skip comments that don't mention any tickers
+
+                    comment_sentiment = get_sentiment_score(comment.body)
+
+                    for ticker_symbol in set(tickers_in_comment):
+                        ticker_id = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol)
+                        # We use the submission.id as the post_id to group mentions correctly
+                        database.add_mention(conn, ticker_id, subreddit_id, submission.id, int(comment.created_utc), comment_sentiment)
+                    
+                    comment_count += 1
+
+        except Exception as e:
+            print(f"Could not scan r/{subreddit_name}. Error: {e}")
+            
+    conn.close()
+    print("\n--- Scan Complete ---")
+
+def main():
+    """Main function to run the Reddit stock analysis tool."""
+    parser = argparse.ArgumentParser(
+        description="Analyze stock ticker mentions on Reddit.",
+        formatter_class=argparse.RawTextHelpFormatter # For better help text formatting
+    )
+    
+    # --- Existing Argument ---
+    parser.add_argument("config_file", help="Path to the JSON file containing subreddits.")
+    
+    # --- NEW Arguments ---
+    parser.add_argument(
+        "-p", "--posts",
+        type=int,
+        default=25,
+        help="Number of posts to scan per subreddit.\n(Default: 25)"
+    )
+    parser.add_argument(
+        "-c", "--comments",
+        type=int,
+        default=100,
+        help="Number of comments to scan per post.\n(Default: 100)"
+    )
+    parser.add_argument(
+        "-l", "--limit",
+        type=int,
+        default=20,
+        help="Number of tickers to show in the final report.\n(Default: 20)"
+    )
+    args = parser.parse_args()
+
+    # --- Initialize and Run ---
+    database.initialize_db()
+    database.clean_stale_tickers()
+    
+    subreddits = load_subreddits(args.config_file)
+    if not subreddits: return
+
+    reddit = get_reddit_instance()
+    if not reddit: return
+
+    # Pass the command-line arguments to the functions
+    scan_subreddits(reddit, subreddits, post_limit=args.posts, comment_limit=args.comments)
+    database.generate_summary_report(limit=args.limit)
+
+
+if __name__ == "__main__":
+    main()

rstat_tool/sentiment_analyzer.py

@@ -0,0 +1,19 @@
+# sentiment_analyzer.py
+
+from nltk.sentiment.vader import SentimentIntensityAnalyzer
+
+# Initialize the VADER sentiment intensity analyzer
+# We only need to create one instance of this.
+_analyzer = SentimentIntensityAnalyzer()
+
+def get_sentiment_score(text):
+    """
+    Analyzes a piece of text and returns its sentiment score.
+    
+    The 'compound' score is a single metric that summarizes the sentiment.
+    It ranges from -1 (most negative) to +1 (most positive).
+    """
+    # The polarity_scores() method returns a dictionary with 'neg', 'neu', 'pos', and 'compound' scores.
+    # We are most interested in the 'compound' score.
+    scores = _analyzer.polarity_scores(text)
+    return scores['compound']

rstat_tool/setup_nltk.py

@@ -0,0 +1,11 @@
+import nltk
+
+# This will download the 'vader_lexicon' dataset
+# It only needs to be run once
+try:
+    nltk.data.find('sentiment/vader_lexicon.zip')
+    print("VADER lexicon is already downloaded.")
+except LookupError:
+    print("Downloading VADER lexicon...")
+    nltk.download('vader_lexicon')
+    print("Download complete.")

rstat_tool/ticker_extractor.py

@@ -0,0 +1,65 @@
+# ticker_extractor.py
+
+import re
+
+# A set of common English words and acronyms that look like stock tickers.
+# This helps reduce false positives.
+COMMON_WORDS_BLACKLIST = {
+    "401K", "403B", "457B", "ABOVE", "AI", "ALL", "ALPHA", "AMA", "AMEX",
+    "AND", "ANY", "AR", "ARE", "AROUND", "ASSET", "AT", "ATH", "ATL", "AUD",
+    "BE", "BEAR", "BELOW", "BETA", "BIG", "BIS", "BLEND", "BOE", "BOJ",
+    "BOND", "BRB", "BRL", "BTC", "BTW", "BULL", "BUT", "BUY", "BUZZ", "CAD",
+    "CAN", "CEO", "CFO", "CHF", "CIA", "CNY", "COME", "COST", "COULD", "CPI",
+    "CTB", "CTO", "CYCLE", "CZK", "DAO", "DATE", "DAX", "DAY", "DCA", "DD",
+    "DEBT", "DIA", "DIV", "DJIA", "DKK", "DM", "DO", "DOGE", "DR", "EACH",
+    "EARLY", "EARN", "ECB", "EDGAR", "EDIT", "EPS", "ESG", "ETF", "ETH",
+    "EU", "EUR", "EV", "EVERY", "FAQ", "FAR", "FAST", "FBI", "FDA", "FIHTX",
+    "FINRA", "FINT", "FINTX", "FINTY", "FOMC", "FOMO", "FOR", "FRAUD",
+    "FRG", "FSPSX", "FTSE", "FUD", "FULL", "FUND", "FXAIX", "FXIAX", "FY",
+    "FYI", "FZROX", "GAIN", "GDP", "GET", "GBP", "GO", "GOAL", "GPU", "GRAB",
+    "GTG", "HAS", "HAVE", "HATE", "HEAR", "HEDGE", "HINT", "HKD", "HODL",
+    "HOLD", "HOUR", "HSA", "HUF", "IMHO", "IMO", "IN", "INR", "IPO", "IRA",
+    "IRS", "IS", "ISM", "IT", "IV", "IVV", "IWM", "JPY", "JUST", "KNOW",
+    "KRW", "LARGE", "LAST", "LATE", "LATER", "LBO", "LIKE", "LMAO", "LOL",
+    "LONG", "LOOK", "LOSS", "LOVE", "M&A", "MAKE", "MAX", "MC", "MID", "MIGHT",
+    "MIN", "ML", "MOASS", "MONTH", "MUST", "MXN", "MY", "NATO", "NEAR",
+    "NEED", "NEW", "NEXT", "NFA", "NFT", "NGMI", "NIGHT", "NO", "NOK", "NONE",
+    "NOT", "NOW", "NSA", "NULL", "NZD", "NYSE", "OF", "OK", "OLD", "ON",
+    "OP", "OR", "OTC", "OUGHT", "OUT", "OVER", "PE", "PEAK", "PEG",
+    "PLAN", "PLN", "PMI", "PPI", "PRICE", "PROFIT", "PSA", "Q1", "Q2", "Q3",
+    "Q4", "QQQ", "RBA", "RBNZ", "REIT", "REKT", "RH", "RISK", "ROE", "ROFL",
+    "ROI", "ROTH", "RSD", "RUB", "SAVE", "SCALP", "SCAM", "SCHB", "SEC",
+    "SEE", "SEK", "SELL", "SEP", "SGD", "SHALL", "SHARE", "SHORT", "SO",
+    "SOME", "SOON", "SPAC", "SPEND", "SPLG", "SPX", "SPY", "STILL", "STOCK",
+    "SWING", "TAKE", "TERM", "THE", "THINK", "THIS", "TIME", "TL", "TL;DR",
+    "TLDR", "TODAY", "TO", "TOTAL", "TRADE", "TREND", "TRUE", "TRY", "TTYL",
+    "TWO", "UK", "UNDER", "UP", "US", "USA", "USD", "VTI", "VALUE", "VOO",
+    "VR", "WAGMI", "WANT", "WATCH", "WAY", "WE", "WEB3", "WEEK", "WHO",
+    "WHY", "WILL", "WORTH", "WOULD", "WSB", "YET", "YIELD", "YOLO", "YOU",
+    "ZAR",
+    "KARMA", "OTM", "ITM", "ATM", "JPOW", "OPEN", "CLOSE", "HIGH", "LOW",
+    "RE", "BS", "ASAP", "RULE", "REAL", "LIMIT", "STOP", "END", "START", "BOTS",
+    "UTC", "AH", "PM", "PR", "GMT", "EST", "CST", "PST", "BST", "AEDT", "AEST",
+    "CET", "CEST", "EDT", "IST", "JST", "MSK", "PDT", "PST", "YES", "NO", "OWN",
+    "BOMB", 
+}
+def extract_tickers(text):
+    """
+    Extracts potential stock tickers from a given piece of text.
+    A ticker is identified as a 1-5 character uppercase word, or a word prefixed with $.
+    """
+    # Regex to find potential tickers:
+    # 1. Words prefixed with $: $AAPL, $TSLA
+    # 2. All-caps words between 1 and 5 characters: GME, AMC
+    ticker_regex = r"\$[A-Z]{1,5}\b|\b[A-Z]{2,5}\b"
+
+    potential_tickers = re.findall(ticker_regex, text)
+
+    # Filter out common words and remove the '$' prefix
+    tickers = []
+    for ticker in potential_tickers:
+        cleaned_ticker = ticker.replace("$", "").upper()
+        if cleaned_ticker not in COMMON_WORDS_BLACKLIST:
+            tickers.append(cleaned_ticker)
+
+    return tickers