From 71890d1a57f4102a7cf13b3b7c6a1ec461c8f5d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l-Kristian=20Hamre?= Date: Mon, 21 Jul 2025 15:49:15 +0200 Subject: [PATCH] Modularized the tool. --- .gitignore | 1 + rstat_tool/__init__.py | 0 rstat_tool/database.py | 158 +++++++++++++++++++++++++++++++ rstat_tool/main.py | 149 +++++++++++++++++++++++++++++ rstat_tool/sentiment_analyzer.py | 19 ++++ rstat_tool/setup_nltk.py | 11 +++ rstat_tool/ticker_extractor.py | 65 +++++++++++++ setup.py | 23 +++++ 8 files changed, 426 insertions(+) create mode 100644 rstat_tool/__init__.py create mode 100644 rstat_tool/database.py create mode 100644 rstat_tool/main.py create mode 100644 rstat_tool/sentiment_analyzer.py create mode 100644 rstat_tool/setup_nltk.py create mode 100644 rstat_tool/ticker_extractor.py create mode 100644 setup.py diff --git a/.gitignore b/.gitignore index 9755e26..2f21318 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ __pycache__/ *.sqlite3 *.db *.log +reddit_stock_analyzer.egg-info/ \ No newline at end of file diff --git a/rstat_tool/__init__.py b/rstat_tool/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rstat_tool/database.py b/rstat_tool/database.py new file mode 100644 index 0000000..335dfda --- /dev/null +++ b/rstat_tool/database.py @@ -0,0 +1,158 @@ +# rstat_tool/database.py + +import sqlite3 +import time +# --- IMPORT ADDED BACK IN --- +from .ticker_extractor import COMMON_WORDS_BLACKLIST + +DB_FILE = "reddit_stocks.db" + +def get_db_connection(): + """Establishes a connection to the SQLite database.""" + conn = sqlite3.connect(DB_FILE) + conn.row_factory = sqlite3.Row + return conn + +def initialize_db(): + # ... (This function is unchanged) + conn = get_db_connection() + cursor = conn.cursor() + cursor.execute(""" + CREATE TABLE IF NOT EXISTS tickers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + symbol TEXT NOT NULL UNIQUE, + market_cap INTEGER, + last_updated INTEGER + ) + """) + cursor.execute(""" + CREATE TABLE IF NOT EXISTS subreddits ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE + ) + """) + cursor.execute(""" + CREATE TABLE IF NOT EXISTS mentions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ticker_id INTEGER, + subreddit_id INTEGER, + post_id TEXT NOT NULL, + mention_timestamp INTEGER NOT NULL, + sentiment_score REAL, + FOREIGN KEY (ticker_id) REFERENCES tickers (id), + FOREIGN KEY (subreddit_id) REFERENCES subreddits (id), + UNIQUE(ticker_id, post_id, sentiment_score) + ) + """) + conn.commit() + conn.close() + print("Database initialized successfully.") + +# --- CLEANUP FUNCTION ADDED BACK IN --- +def clean_stale_tickers(): + """ + Removes tickers and their associated mentions from the database + if the ticker symbol exists in the COMMON_WORDS_BLACKLIST. + """ + print("\n--- Cleaning Stale Tickers from Database ---") + conn = get_db_connection() + cursor = conn.cursor() + + # Find ticker IDs that match the blacklist + placeholders = ','.join('?' for _ in COMMON_WORDS_BLACKLIST) + query = f"SELECT id, symbol FROM tickers WHERE symbol IN ({placeholders})" + + cursor.execute(query, tuple(COMMON_WORDS_BLACKLIST)) + stale_tickers = cursor.fetchall() + + if not stale_tickers: + print("No stale tickers to clean.") + conn.close() + return + + for ticker in stale_tickers: + ticker_id = ticker['id'] + ticker_symbol = ticker['symbol'] + print(f"Removing stale ticker '{ticker_symbol}' (ID: {ticker_id})...") + + # 1. Delete all mentions associated with this ticker ID + cursor.execute("DELETE FROM mentions WHERE ticker_id = ?", (ticker_id,)) + + # 2. Delete the ticker itself + cursor.execute("DELETE FROM tickers WHERE id = ?", (ticker_id,)) + + deleted_count = conn.total_changes + conn.commit() + conn.close() + print(f"Cleanup complete. Removed {deleted_count} records.") + + +def add_mention(conn, ticker_id, subreddit_id, post_id, timestamp, sentiment): + # ... (This function is unchanged) + cursor = conn.cursor() + try: + cursor.execute( + "INSERT INTO mentions (ticker_id, subreddit_id, post_id, mention_timestamp, sentiment_score) VALUES (?, ?, ?, ?, ?)", + (ticker_id, subreddit_id, post_id, timestamp, sentiment) + ) + conn.commit() + except sqlite3.IntegrityError: + pass + +# ... (get_or_create_entity, update_ticker_market_cap, get_ticker_info are unchanged) +def get_or_create_entity(conn, table_name, column_name, value): + # ... + cursor = conn.cursor() + cursor.execute(f"SELECT id FROM {table_name} WHERE {column_name} = ?", (value,)) + result = cursor.fetchone() + if result: return result['id'] + else: + cursor.execute(f"INSERT INTO {table_name} ({column_name}) VALUES (?)", (value,)) + conn.commit() + return cursor.lastrowid + +def update_ticker_market_cap(conn, ticker_id, market_cap): + # ... + cursor = conn.cursor() + current_timestamp = int(time.time()) + cursor.execute("UPDATE tickers SET market_cap = ?, last_updated = ? WHERE id = ?", (market_cap, current_timestamp, ticker_id)) + conn.commit() + +def get_ticker_info(conn, ticker_id): + # ... + cursor = conn.cursor() + cursor.execute("SELECT * FROM tickers WHERE id = ?", (ticker_id,)) + return cursor.fetchone() + +def generate_summary_report(limit=20): + # ... (This function is unchanged) + print(f"\n--- Top {limit} Tickers by Mention Count ---") + conn = get_db_connection() + cursor = conn.cursor() + query = """ + SELECT + t.symbol, + t.market_cap, + COUNT(m.id) as mention_count, + SUM(CASE WHEN m.sentiment_score > 0.1 THEN 1 ELSE 0 END) as bullish_mentions, + SUM(CASE WHEN m.sentiment_score < -0.1 THEN 1 ELSE 0 END) as bearish_mentions, + SUM(CASE WHEN m.sentiment_score BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions + FROM mentions m + JOIN tickers t ON m.ticker_id = t.id + GROUP BY t.symbol, t.market_cap + ORDER BY mention_count DESC + LIMIT ?; + """ + results = cursor.execute(query, (limit,)).fetchall() + header = f"{'Ticker':<8} | {'Mentions':<8} | {'Bullish':<8} | {'Bearish':<8} | {'Neutral':<8} | {'Market Cap':<15}" + print(header) + print("-" * len(header)) + for row in results: + market_cap_str = "N/A" + if row['market_cap'] and row['market_cap'] > 0: + mc = row['market_cap'] + if mc >= 1e12: market_cap_str = f"${mc/1e12:.2f}T" + elif mc >= 1e9: market_cap_str = f"${mc/1e9:.2f}B" + else: market_cap_str = f"${mc/1e6:.2f}M" + print(f"{row['symbol']:<8} | {row['mention_count']:<8} | {row['bullish_mentions']:<8} | {row['bearish_mentions']:<8} | {row['neutral_mentions']:<8} | {market_cap_str:<15}") + conn.close() \ No newline at end of file diff --git a/rstat_tool/main.py b/rstat_tool/main.py new file mode 100644 index 0000000..725a1f9 --- /dev/null +++ b/rstat_tool/main.py @@ -0,0 +1,149 @@ +# main.py + +import argparse +import json +import os +import time + +import praw +import yfinance as yf +from dotenv import load_dotenv + +# Import local modules +from . import database +from .ticker_extractor import extract_tickers +from .sentiment_analyzer import get_sentiment_score + +load_dotenv() +MARKET_CAP_REFRESH_INTERVAL = 86400 + +# (load_subreddits, get_market_cap, get_reddit_instance functions are unchanged) +def load_subreddits(filepath): + try: + with open(filepath, 'r') as f: + return json.load(f).get("subreddits", []) + except (FileNotFoundError, json.JSONDecodeError) as e: + print(f"Error loading config: {e}") + return None + +def get_market_cap(ticker_symbol): + try: + ticker = yf.Ticker(ticker_symbol) + return ticker.fast_info.get('marketCap') + except Exception: + return None + +def get_reddit_instance(): + client_id = os.getenv("REDDIT_CLIENT_ID") + client_secret = os.getenv("REDDIT_CLIENT_SECRET") + user_agent = os.getenv("REDDIT_USER_AGENT") + if not all([client_id, client_secret, user_agent]): + print("Error: Reddit API credentials not found.") + return None + return praw.Reddit(client_id=client_id, client_secret=client_secret, user_agent=user_agent) + +# --- UPDATED: Function now accepts post_limit and comment_limit --- +def scan_subreddits(reddit, subreddits_list, post_limit=25, comment_limit=100): + """Scans subreddits, analyzes posts and comments, and stores results in the database.""" + conn = database.get_db_connection() + + print(f"\nScanning {len(subreddits_list)} subreddits (Top {post_limit} posts, {comment_limit} comments/post)...") + for subreddit_name in subreddits_list: + try: + subreddit_id = database.get_or_create_entity(conn, 'subreddits', 'name', subreddit_name) + subreddit = reddit.subreddit(subreddit_name) + print(f"Scanning r/{subreddit_name}...") + + for submission in subreddit.hot(limit=post_limit): + # --- 1. Process the Post Title and Body --- + post_text = submission.title + " " + submission.selftext + tickers_in_post = extract_tickers(post_text) + post_sentiment = get_sentiment_score(submission.title) + + for ticker_symbol in set(tickers_in_post): + ticker_id = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol) + database.add_mention(conn, ticker_id, subreddit_id, submission.id, int(submission.created_utc), post_sentiment) + # (Market cap logic remains the same) + ticker_info = database.get_ticker_info(conn, ticker_id) + current_time = int(time.time()) + if not ticker_info['last_updated'] or (current_time - ticker_info['last_updated'] > MARKET_CAP_REFRESH_INTERVAL): + print(f" -> Fetching market cap for {ticker_symbol}...") + market_cap = get_market_cap(ticker_symbol) + database.update_ticker_market_cap(conn, ticker_id, market_cap or ticker_info['market_cap']) + + # --- 2. Process the Comments --- + # Expand "MoreComments" objects. limit=None means we try to get all, but PRAW is protective. + # A limit of 32 is the max PRAW will do in a single call. We'll iterate to be safe. + submission.comments.replace_more(limit=10) + comment_count = 0 + for comment in submission.comments.list(): + if comment_count >= comment_limit: + break # Stop processing comments for this post if we hit our limit + + tickers_in_comment = extract_tickers(comment.body) + if not tickers_in_comment: + continue # Skip comments that don't mention any tickers + + comment_sentiment = get_sentiment_score(comment.body) + + for ticker_symbol in set(tickers_in_comment): + ticker_id = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol) + # We use the submission.id as the post_id to group mentions correctly + database.add_mention(conn, ticker_id, subreddit_id, submission.id, int(comment.created_utc), comment_sentiment) + + comment_count += 1 + + except Exception as e: + print(f"Could not scan r/{subreddit_name}. Error: {e}") + + conn.close() + print("\n--- Scan Complete ---") + +def main(): + """Main function to run the Reddit stock analysis tool.""" + parser = argparse.ArgumentParser( + description="Analyze stock ticker mentions on Reddit.", + formatter_class=argparse.RawTextHelpFormatter # For better help text formatting + ) + + # --- Existing Argument --- + parser.add_argument("config_file", help="Path to the JSON file containing subreddits.") + + # --- NEW Arguments --- + parser.add_argument( + "-p", "--posts", + type=int, + default=25, + help="Number of posts to scan per subreddit.\n(Default: 25)" + ) + parser.add_argument( + "-c", "--comments", + type=int, + default=100, + help="Number of comments to scan per post.\n(Default: 100)" + ) + parser.add_argument( + "-l", "--limit", + type=int, + default=20, + help="Number of tickers to show in the final report.\n(Default: 20)" + ) + args = parser.parse_args() + + # --- Initialize and Run --- + database.initialize_db() + database.clean_stale_tickers() + + subreddits = load_subreddits(args.config_file) + if not subreddits: return + + reddit = get_reddit_instance() + if not reddit: return + + # Pass the command-line arguments to the functions + scan_subreddits(reddit, subreddits, post_limit=args.posts, comment_limit=args.comments) + database.generate_summary_report(limit=args.limit) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/rstat_tool/sentiment_analyzer.py b/rstat_tool/sentiment_analyzer.py new file mode 100644 index 0000000..32b08e8 --- /dev/null +++ b/rstat_tool/sentiment_analyzer.py @@ -0,0 +1,19 @@ +# sentiment_analyzer.py + +from nltk.sentiment.vader import SentimentIntensityAnalyzer + +# Initialize the VADER sentiment intensity analyzer +# We only need to create one instance of this. +_analyzer = SentimentIntensityAnalyzer() + +def get_sentiment_score(text): + """ + Analyzes a piece of text and returns its sentiment score. + + The 'compound' score is a single metric that summarizes the sentiment. + It ranges from -1 (most negative) to +1 (most positive). + """ + # The polarity_scores() method returns a dictionary with 'neg', 'neu', 'pos', and 'compound' scores. + # We are most interested in the 'compound' score. + scores = _analyzer.polarity_scores(text) + return scores['compound'] \ No newline at end of file diff --git a/rstat_tool/setup_nltk.py b/rstat_tool/setup_nltk.py new file mode 100644 index 0000000..bfd5209 --- /dev/null +++ b/rstat_tool/setup_nltk.py @@ -0,0 +1,11 @@ +import nltk + +# This will download the 'vader_lexicon' dataset +# It only needs to be run once +try: + nltk.data.find('sentiment/vader_lexicon.zip') + print("VADER lexicon is already downloaded.") +except LookupError: + print("Downloading VADER lexicon...") + nltk.download('vader_lexicon') + print("Download complete.") \ No newline at end of file diff --git a/rstat_tool/ticker_extractor.py b/rstat_tool/ticker_extractor.py new file mode 100644 index 0000000..7cd19cf --- /dev/null +++ b/rstat_tool/ticker_extractor.py @@ -0,0 +1,65 @@ +# ticker_extractor.py + +import re + +# A set of common English words and acronyms that look like stock tickers. +# This helps reduce false positives. +COMMON_WORDS_BLACKLIST = { + "401K", "403B", "457B", "ABOVE", "AI", "ALL", "ALPHA", "AMA", "AMEX", + "AND", "ANY", "AR", "ARE", "AROUND", "ASSET", "AT", "ATH", "ATL", "AUD", + "BE", "BEAR", "BELOW", "BETA", "BIG", "BIS", "BLEND", "BOE", "BOJ", + "BOND", "BRB", "BRL", "BTC", "BTW", "BULL", "BUT", "BUY", "BUZZ", "CAD", + "CAN", "CEO", "CFO", "CHF", "CIA", "CNY", "COME", "COST", "COULD", "CPI", + "CTB", "CTO", "CYCLE", "CZK", "DAO", "DATE", "DAX", "DAY", "DCA", "DD", + "DEBT", "DIA", "DIV", "DJIA", "DKK", "DM", "DO", "DOGE", "DR", "EACH", + "EARLY", "EARN", "ECB", "EDGAR", "EDIT", "EPS", "ESG", "ETF", "ETH", + "EU", "EUR", "EV", "EVERY", "FAQ", "FAR", "FAST", "FBI", "FDA", "FIHTX", + "FINRA", "FINT", "FINTX", "FINTY", "FOMC", "FOMO", "FOR", "FRAUD", + "FRG", "FSPSX", "FTSE", "FUD", "FULL", "FUND", "FXAIX", "FXIAX", "FY", + "FYI", "FZROX", "GAIN", "GDP", "GET", "GBP", "GO", "GOAL", "GPU", "GRAB", + "GTG", "HAS", "HAVE", "HATE", "HEAR", "HEDGE", "HINT", "HKD", "HODL", + "HOLD", "HOUR", "HSA", "HUF", "IMHO", "IMO", "IN", "INR", "IPO", "IRA", + "IRS", "IS", "ISM", "IT", "IV", "IVV", "IWM", "JPY", "JUST", "KNOW", + "KRW", "LARGE", "LAST", "LATE", "LATER", "LBO", "LIKE", "LMAO", "LOL", + "LONG", "LOOK", "LOSS", "LOVE", "M&A", "MAKE", "MAX", "MC", "MID", "MIGHT", + "MIN", "ML", "MOASS", "MONTH", "MUST", "MXN", "MY", "NATO", "NEAR", + "NEED", "NEW", "NEXT", "NFA", "NFT", "NGMI", "NIGHT", "NO", "NOK", "NONE", + "NOT", "NOW", "NSA", "NULL", "NZD", "NYSE", "OF", "OK", "OLD", "ON", + "OP", "OR", "OTC", "OUGHT", "OUT", "OVER", "PE", "PEAK", "PEG", + "PLAN", "PLN", "PMI", "PPI", "PRICE", "PROFIT", "PSA", "Q1", "Q2", "Q3", + "Q4", "QQQ", "RBA", "RBNZ", "REIT", "REKT", "RH", "RISK", "ROE", "ROFL", + "ROI", "ROTH", "RSD", "RUB", "SAVE", "SCALP", "SCAM", "SCHB", "SEC", + "SEE", "SEK", "SELL", "SEP", "SGD", "SHALL", "SHARE", "SHORT", "SO", + "SOME", "SOON", "SPAC", "SPEND", "SPLG", "SPX", "SPY", "STILL", "STOCK", + "SWING", "TAKE", "TERM", "THE", "THINK", "THIS", "TIME", "TL", "TL;DR", + "TLDR", "TODAY", "TO", "TOTAL", "TRADE", "TREND", "TRUE", "TRY", "TTYL", + "TWO", "UK", "UNDER", "UP", "US", "USA", "USD", "VTI", "VALUE", "VOO", + "VR", "WAGMI", "WANT", "WATCH", "WAY", "WE", "WEB3", "WEEK", "WHO", + "WHY", "WILL", "WORTH", "WOULD", "WSB", "YET", "YIELD", "YOLO", "YOU", + "ZAR", + "KARMA", "OTM", "ITM", "ATM", "JPOW", "OPEN", "CLOSE", "HIGH", "LOW", + "RE", "BS", "ASAP", "RULE", "REAL", "LIMIT", "STOP", "END", "START", "BOTS", + "UTC", "AH", "PM", "PR", "GMT", "EST", "CST", "PST", "BST", "AEDT", "AEST", + "CET", "CEST", "EDT", "IST", "JST", "MSK", "PDT", "PST", "YES", "NO", "OWN", + "BOMB", +} +def extract_tickers(text): + """ + Extracts potential stock tickers from a given piece of text. + A ticker is identified as a 1-5 character uppercase word, or a word prefixed with $. + """ + # Regex to find potential tickers: + # 1. Words prefixed with $: $AAPL, $TSLA + # 2. All-caps words between 1 and 5 characters: GME, AMC + ticker_regex = r"\$[A-Z]{1,5}\b|\b[A-Z]{2,5}\b" + + potential_tickers = re.findall(ticker_regex, text) + + # Filter out common words and remove the '$' prefix + tickers = [] + for ticker in potential_tickers: + cleaned_ticker = ticker.replace("$", "").upper() + if cleaned_ticker not in COMMON_WORDS_BLACKLIST: + tickers.append(cleaned_ticker) + + return tickers \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..67a4712 --- /dev/null +++ b/setup.py @@ -0,0 +1,23 @@ +# setup.py + +from setuptools import setup, find_packages + +with open('requirements.txt') as f: + requirements = f.read().splitlines() + +setup( + name='reddit-stock-analyzer', + version='0.0.1', + author='Pål-Kristian Hamre', + author_email='its@pkhamre.com', + description='A command-line tool to analyze stock ticker mentions on Reddit.', + # This now correctly finds your 'rstat_tool' package + packages=find_packages(), + install_requires=requirements, + entry_points={ + 'console_scripts': [ + # The path is now 'package_name.module_name:function_name' + 'rstat=rstat_tool.main:main', + ], + }, +) \ No newline at end of file