From d330f31950bba81c69d254840d9d04bffd72cb04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l-Kristian=20Hamre?= Date: Mon, 21 Jul 2025 23:44:27 +0200 Subject: [PATCH] Bugfix. --- rstat_tool/database.py | 196 ++++++++++++++++++++++----------- rstat_tool/format_blacklist.py | 5 +- rstat_tool/main.py | 85 +++++++++----- 3 files changed, 190 insertions(+), 96 deletions(-) diff --git a/rstat_tool/database.py b/rstat_tool/database.py index 88bb078..4f576a7 100644 --- a/rstat_tool/database.py +++ b/rstat_tool/database.py @@ -46,12 +46,12 @@ def initialize_db(): ticker_id INTEGER, subreddit_id INTEGER, post_id TEXT NOT NULL, - mention_type TEXT NOT NULL, -- Can be 'post' or 'comment' + mention_type TEXT NOT NULL, + mention_sentiment REAL, -- Renamed from sentiment_score for clarity + post_avg_sentiment REAL, -- NEW: Stores the avg sentiment of the whole post mention_timestamp INTEGER NOT NULL, - sentiment_score REAL, FOREIGN KEY (ticker_id) REFERENCES tickers (id), - FOREIGN KEY (subreddit_id) REFERENCES subreddits (id), - UNIQUE(ticker_id, post_id, mention_type, sentiment_score) + FOREIGN KEY (subreddit_id) REFERENCES subreddits (id) ) """) @@ -105,13 +105,69 @@ def clean_stale_tickers(): conn.close() print(f"Cleanup complete. Removed {deleted_count} records.") +def get_db_connection(): + conn = sqlite3.connect(DB_FILE) + conn.row_factory = sqlite3.Row + return conn -def add_mention(conn, ticker_id, subreddit_id, post_id, mention_type, timestamp, sentiment): +def initialize_db(): + conn = get_db_connection() + cursor = conn.cursor() + cursor.execute(""" + CREATE TABLE IF NOT EXISTS tickers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + symbol TEXT NOT NULL UNIQUE, + market_cap INTEGER, + closing_price REAL, + last_updated INTEGER + ) + """) + cursor.execute(""" + CREATE TABLE IF NOT EXISTS subreddits ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE + ) + """) + cursor.execute(""" + CREATE TABLE IF NOT EXISTS mentions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ticker_id INTEGER, + subreddit_id INTEGER, + post_id TEXT NOT NULL, + mention_type TEXT NOT NULL, + mention_sentiment REAL, + post_avg_sentiment REAL, + mention_timestamp INTEGER NOT NULL, + FOREIGN KEY (ticker_id) REFERENCES tickers (id), + FOREIGN KEY (subreddit_id) REFERENCES subreddits (id) + ) + """) + cursor.execute(""" + CREATE TABLE IF NOT EXISTS posts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + post_id TEXT NOT NULL UNIQUE, + title TEXT NOT NULL, + post_url TEXT, + subreddit_id INTEGER, + post_timestamp INTEGER, + comment_count INTEGER, + avg_comment_sentiment REAL, + FOREIGN KEY (subreddit_id) REFERENCES subreddits (id) + ) + """) + conn.commit() + conn.close() + print("Database initialized successfully.") + +def add_mention(conn, ticker_id, subreddit_id, post_id, mention_type, timestamp, mention_sentiment, post_avg_sentiment=None): cursor = conn.cursor() try: cursor.execute( - "INSERT INTO mentions (ticker_id, subreddit_id, post_id, mention_type, mention_timestamp, sentiment_score) VALUES (?, ?, ?, ?, ?, ?)", - (ticker_id, subreddit_id, post_id, mention_type, timestamp, sentiment) + """ + INSERT INTO mentions (ticker_id, subreddit_id, post_id, mention_type, mention_timestamp, mention_sentiment, post_avg_sentiment) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + (ticker_id, subreddit_id, post_id, mention_type, timestamp, mention_sentiment, post_avg_sentiment) ) conn.commit() except sqlite3.IntegrityError: @@ -150,19 +206,26 @@ def generate_summary_report(limit=20): print(f"\n--- Top {limit} Tickers by Mention Count ---") conn = get_db_connection() cursor = conn.cursor() + + # --- UPDATED QUERY: Changed m.sentiment_score to m.mention_sentiment --- query = """ SELECT - t.symbol, t.market_cap, COUNT(m.id) as mention_count, - SUM(CASE WHEN m.sentiment_score > 0.1 THEN 1 ELSE 0 END) as bullish_mentions, - SUM(CASE WHEN m.sentiment_score < -0.1 THEN 1 ELSE 0 END) as bearish_mentions, - SUM(CASE WHEN m.sentiment_score BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions + t.symbol, t.market_cap, t.closing_price, + COUNT(m.id) as mention_count, + SUM(CASE WHEN m.mention_sentiment > 0.1 THEN 1 ELSE 0 END) as bullish_mentions, + SUM(CASE WHEN m.mention_sentiment < -0.1 THEN 1 ELSE 0 END) as bearish_mentions, + SUM(CASE WHEN m.mention_sentiment BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions FROM mentions m JOIN tickers t ON m.ticker_id = t.id - GROUP BY t.symbol, t.market_cap ORDER BY mention_count DESC LIMIT ?; + GROUP BY t.symbol, t.market_cap, t.closing_price + ORDER BY mention_count DESC + LIMIT ?; """ results = cursor.execute(query, (limit,)).fetchall() - header = f"{'Ticker':<8} | {'Mentions':<8} | {'Bullish':<8} | {'Bearish':<8} | {'Neutral':<8} | {'Market Cap':<15}" + + header = f"{'Ticker':<8} | {'Mentions':<8} | {'Bullish':<8} | {'Bearish':<8} | {'Neutral':<8} | {'Market Cap':<15} | {'Close Price':<12}" print(header) - print("-" * len(header)) + print("-" * (len(header) + 2)) # Adjusted separator length + for row in results: market_cap_str = "N/A" if row['market_cap'] and row['market_cap'] > 0: @@ -170,45 +233,19 @@ def generate_summary_report(limit=20): if mc >= 1e12: market_cap_str = f"${mc/1e12:.2f}T" elif mc >= 1e9: market_cap_str = f"${mc/1e9:.2f}B" else: market_cap_str = f"${mc/1e6:.2f}M" - print(f"{row['symbol']:<8} | {row['mention_count']:<8} | {row['bullish_mentions']:<8} | {row['bearish_mentions']:<8} | {row['neutral_mentions']:<8} | {market_cap_str:<15}") - conn.close() -def get_overall_summary(limit=50): - conn = get_db_connection() - query = """ - SELECT - t.symbol, t.market_cap, t.closing_price, -- Added closing_price - COUNT(m.id) as mention_count, - SUM(CASE WHEN m.sentiment_score > 0.1 THEN 1 ELSE 0 END) as bullish_mentions, - SUM(CASE WHEN m.sentiment_score < -0.1 THEN 1 ELSE 0 END) as bearish_mentions, - SUM(CASE WHEN m.sentiment_score BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions - FROM mentions m JOIN tickers t ON m.ticker_id = t.id - GROUP BY t.symbol, t.market_cap, t.closing_price -- Added closing_price - ORDER BY mention_count DESC LIMIT ?; - """ - results = conn.execute(query, (limit,)).fetchall() - conn.close() - return results + closing_price_str = f"${row['closing_price']:.2f}" if row['closing_price'] else "N/A" -def get_subreddit_summary(subreddit_name, limit=50): - conn = get_db_connection() - query = """ - SELECT - t.symbol, t.market_cap, t.closing_price, -- Added closing_price - COUNT(m.id) as mention_count, - SUM(CASE WHEN m.sentiment_score > 0.1 THEN 1 ELSE 0 END) as bullish_mentions, - SUM(CASE WHEN m.sentiment_score < -0.1 THEN 1 ELSE 0 END) as bearish_mentions, - SUM(CASE WHEN m.sentiment_score BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions - FROM mentions m - JOIN tickers t ON m.ticker_id = t.id - JOIN subreddits s ON m.subreddit_id = s.id - WHERE s.name = ? - GROUP BY t.symbol, t.market_cap, t.closing_price -- Added closing_price - ORDER BY mention_count DESC LIMIT ?; - """ - results = conn.execute(query, (subreddit_name, limit)).fetchall() + print( + f"{row['symbol']:<8} | " + f"{row['mention_count']:<8} | " + f"{row['bullish_mentions']:<8} | " + f"{row['bearish_mentions']:<8} | " + f"{row['neutral_mentions']:<8} | " + f"{market_cap_str:<15} | " + f"{closing_price_str:<12}" + ) conn.close() - return results def get_all_scanned_subreddits(): """Gets a unique list of all subreddits we have data for.""" @@ -253,20 +290,52 @@ def get_deep_dive_details(ticker_symbol): conn.close() return results -def get_image_view_summary(subreddit_name): - """ - Gets a summary of tickers for the image view, including post, comment, - and sentiment counts. - """ +def get_overall_summary(limit=50): + conn = get_db_connection() + query = """ + SELECT + t.symbol, t.market_cap, t.closing_price, + COUNT(m.id) as mention_count, + SUM(CASE WHEN m.mention_sentiment > 0.1 THEN 1 ELSE 0 END) as bullish_mentions, + SUM(CASE WHEN m.mention_sentiment < -0.1 THEN 1 ELSE 0 END) as bearish_mentions, + SUM(CASE WHEN m.mention_sentiment BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions + FROM mentions m JOIN tickers t ON m.ticker_id = t.id + GROUP BY t.symbol, t.market_cap, t.closing_price + ORDER BY mention_count DESC LIMIT ?; + """ + results = conn.execute(query, (limit,)).fetchall() + conn.close() + return results + +def get_subreddit_summary(subreddit_name, limit=50): + conn = get_db_connection() + query = """ + SELECT + t.symbol, t.market_cap, t.closing_price, + COUNT(m.id) as mention_count, + SUM(CASE WHEN m.mention_sentiment > 0.1 THEN 1 ELSE 0 END) as bullish_mentions, + SUM(CASE WHEN m.mention_sentiment < -0.1 THEN 1 ELSE 0 END) as bearish_mentions, + SUM(CASE WHEN m.mention_sentiment BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions + FROM mentions m + JOIN tickers t ON m.ticker_id = t.id + JOIN subreddits s ON m.subreddit_id = s.id + WHERE s.name = ? + GROUP BY t.symbol, t.market_cap, t.closing_price + ORDER BY mention_count DESC LIMIT ?; + """ + results = conn.execute(query, (subreddit_name, limit)).fetchall() + conn.close() + return results + +def get_image_view_summary(subreddit_name): conn = get_db_connection() - # This query now also counts sentiment types query = """ SELECT t.symbol, COUNT(CASE WHEN m.mention_type = 'post' THEN 1 END) as post_mentions, COUNT(CASE WHEN m.mention_type = 'comment' THEN 1 END) as comment_mentions, - COUNT(CASE WHEN m.sentiment_score > 0.1 THEN 1 END) as bullish_mentions, - COUNT(CASE WHEN m.sentiment_score < -0.1 THEN 1 END) as bearish_mentions + COUNT(CASE WHEN m.mention_sentiment > 0.1 THEN 1 END) as bullish_mentions, + COUNT(CASE WHEN m.mention_sentiment < -0.1 THEN 1 END) as bearish_mentions FROM mentions m JOIN tickers t ON m.ticker_id = t.id JOIN subreddits s ON m.subreddit_id = s.id @@ -280,23 +349,16 @@ def get_image_view_summary(subreddit_name): return results def get_weekly_summary_for_subreddit(subreddit_name): - """ - Gets a weekly summary for a specific subreddit for the image view. - """ conn = get_db_connection() - - # Calculate the timestamp for 7 days ago seven_days_ago = datetime.utcnow() - timedelta(days=7) seven_days_ago_timestamp = int(seven_days_ago.timestamp()) - - # The query is the same as before, but with an added WHERE clause for the timestamp query = """ SELECT t.symbol, COUNT(CASE WHEN m.mention_type = 'post' THEN 1 END) as post_mentions, COUNT(CASE WHEN m.mention_type = 'comment' THEN 1 END) as comment_mentions, - COUNT(CASE WHEN m.sentiment_score > 0.1 THEN 1 END) as bullish_mentions, - COUNT(CASE WHEN m.sentiment_score < -0.1 THEN 1 END) as bearish_mentions + COUNT(CASE WHEN m.mention_sentiment > 0.1 THEN 1 END) as bullish_mentions, + COUNT(CASE WHEN m.mention_sentiment < -0.1 THEN 1 END) as bearish_mentions FROM mentions m JOIN tickers t ON m.ticker_id = t.id JOIN subreddits s ON m.subreddit_id = s.id diff --git a/rstat_tool/format_blacklist.py b/rstat_tool/format_blacklist.py index 5f304cb..958a54b 100644 --- a/rstat_tool/format_blacklist.py +++ b/rstat_tool/format_blacklist.py @@ -46,7 +46,10 @@ COMMON_WORDS_BLACKLIST = { "USD", "UTC", "VALUE", "VOO", "VP", "VR", "VTI", "WAGMI", "WANT", "WATCH", "WAY", "WE", "WEB3", "WEEK", "WHALE", "WHO", "WHY", "WIDE", "WILL", "WORDS", "WORTH", "WOULD", "WSB", "WTF", "XRP", "YES", "YET", "YIELD", "YOLO", "YOU", - "YOUR", "YOY", "YT", "YTD", "ZAR", "ZEN", "ZERO" + "YOUR", "YOY", "YT", "YTD", "ZAR", "ZEN", "ZERO", + + "SOUTH", "WIRE", "NORTH", "EAST", "WEST", "AREA", "FTD", "NEAT", "ISIN", "BROKE", "TOLD", + "HUGE", "XO", "NASA", "DAYS", "ENV", "NZ", "IBS", "POSCO", "GUH", "IKKE" } def format_and_print_list(word_set, words_per_line=10): diff --git a/rstat_tool/main.py b/rstat_tool/main.py index 2750009..50c7461 100644 --- a/rstat_tool/main.py +++ b/rstat_tool/main.py @@ -43,6 +43,11 @@ def get_reddit_instance(): return praw.Reddit(client_id=client_id, client_secret=client_secret, user_agent=user_agent) def scan_subreddits(reddit, subreddits_list, post_limit=100, comment_limit=100, days_to_scan=1): + """ + Scans subreddits with a hybrid mention counting logic. + - If a ticker is in the title, it gets credit for all comments. + - If not, tickers only get credit for direct mentions in comments. + """ conn = database.get_db_connection() post_age_limit = days_to_scan * 86400 current_time = time.time() @@ -56,53 +61,77 @@ def scan_subreddits(reddit, subreddits_list, post_limit=100, comment_limit=100, for submission in subreddit.new(limit=post_limit): if (current_time - submission.created_utc) > post_age_limit: - print(f" -> Reached posts older than the {days_to_scan}-day limit. Moving to next subreddit.") + print(f" -> Reached posts older than the {days_to_scan}-day limit.") break - post_text = submission.title + " " + submission.selftext - tickers_in_post = extract_tickers(post_text) - if tickers_in_post: - post_sentiment = get_sentiment_score(submission.title) - for ticker_symbol in set(tickers_in_post): - ticker_id = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol) - database.add_mention(conn, ticker_id, subreddit_id, submission.id, 'post', int(submission.created_utc), post_sentiment) - - ticker_info = database.get_ticker_info(conn, ticker_id) - if not ticker_info['last_updated'] or (current_time - ticker_info['last_updated'] > MARKET_CAP_REFRESH_INTERVAL): - print(f" -> Fetching financial data for {ticker_symbol}...") - financials = get_financial_data(ticker_symbol) - database.update_ticker_financials( - conn, ticker_id, - financials['market_cap'] or ticker_info['market_cap'], - financials['closing_price'] or ticker_info['closing_price'] - ) + # --- NEW HYBRID LOGIC --- + + tickers_in_title = set(extract_tickers(submission.title)) + all_tickers_found_in_post = set(tickers_in_title) # Start a set to track all tickers for financials submission.comments.replace_more(limit=0) - all_comment_sentiments = [] - for comment in submission.comments.list()[:comment_limit]: - all_comment_sentiments.append(get_sentiment_score(comment.body)) - tickers_in_comment = extract_tickers(comment.body) - if tickers_in_comment: + all_comments = submission.comments.list()[:comment_limit] + + # --- CASE A: Tickers were found in the title --- + if tickers_in_title: + print(f" -> Title Mention(s): {', '.join(tickers_in_title)}. Attributing all comments.") + post_sentiment = get_sentiment_score(submission.title) + + # Add one 'post' mention for each title ticker + for ticker_symbol in tickers_in_title: + ticker_id = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol) + database.add_mention(conn, ticker_id, subreddit_id, submission.id, 'post', int(submission.created_utc), post_sentiment) + + # Add one 'comment' mention for EACH comment FOR EACH title ticker + for comment in all_comments: comment_sentiment = get_sentiment_score(comment.body) - for ticker_symbol in set(tickers_in_comment): + for ticker_symbol in tickers_in_title: ticker_id = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol) database.add_mention(conn, ticker_id, subreddit_id, submission.id, 'comment', int(comment.created_utc), comment_sentiment) + + # --- CASE B: No tickers in the title, scan comments individually --- + else: + for comment in all_comments: + tickers_in_comment = set(extract_tickers(comment.body)) + if tickers_in_comment: + all_tickers_found_in_post.update(tickers_in_comment) # Add to our set for financials + comment_sentiment = get_sentiment_score(comment.body) + for ticker_symbol in tickers_in_comment: + ticker_id = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol) + database.add_mention(conn, ticker_id, subreddit_id, submission.id, 'comment', int(comment.created_utc), comment_sentiment) + # --- EFFICIENT FINANCIALS UPDATE --- + # Now, update market cap once for every unique ticker found in the whole post + for ticker_symbol in all_tickers_found_in_post: + ticker_id = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol) + ticker_info = database.get_ticker_info(conn, ticker_id) + if not ticker_info['last_updated'] or (current_time - ticker_info['last_updated'] > MARKET_CAP_REFRESH_INTERVAL): + print(f" -> Fetching financial data for {ticker_symbol}...") + financials = get_financial_data(ticker_symbol) + database.update_ticker_financials( + conn, ticker_id, + financials['market_cap'] or ticker_info['market_cap'], + financials['closing_price'] or ticker_info['closing_price'] + ) + + # --- DEEP DIVE SAVE (Still valuable) --- + all_comment_sentiments = [get_sentiment_score(c.body) for c in all_comments] avg_sentiment = sum(all_comment_sentiments) / len(all_comment_sentiments) if all_comment_sentiments else 0 post_analysis_data = { "post_id": submission.id, "title": submission.title, - "post_url": f"https://reddit.com{submission.permalink}", - "subreddit_id": subreddit_id, "post_timestamp": int(submission.created_utc), - "comment_count": len(all_comment_sentiments), "avg_comment_sentiment": avg_sentiment + "post_url": f"https://reddit.com{submission.permalink}", "subreddit_id": subreddit_id, + "post_timestamp": int(submission.created_utc), "comment_count": len(all_comments), + "avg_comment_sentiment": avg_sentiment } database.add_or_update_post_analysis(conn, post_analysis_data) - + except Exception as e: print(f"Could not scan r/{subreddit_name}. Error: {e}") conn.close() print("\n--- Scan Complete ---") + def main(): """Main function to run the Reddit stock analysis tool.""" parser = argparse.ArgumentParser(description="Analyze stock ticker mentions on Reddit.", formatter_class=argparse.RawTextHelpFormatter)