pkhamre/reddit_stock_analyzer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bugfix.

Browse Source
This commit is contained in:
Pål-Kristian Hamre
2025-07-21 23:44:27 +02:00
parent 728fe43571
commit d330f31950
3 changed files with 190 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

196
rstat_tool/database.py
View File

@@ -46,12 +46,12 @@ def initialize_db():
ticker_id INTEGER,
subreddit_id INTEGER,
post_id TEXT NOT NULL,
mention_type TEXT NOT NULL, -- Can be 'post' or 'comment'
mention_type TEXT NOT NULL,
mention_sentiment REAL, -- Renamed from sentiment_score for clarity
post_avg_sentiment REAL, -- NEW: Stores the avg sentiment of the whole post
mention_timestamp INTEGER NOT NULL,
sentiment_score REAL,
FOREIGN KEY (ticker_id) REFERENCES tickers (id),
FOREIGN KEY (subreddit_id) REFERENCES subreddits (id),
UNIQUE(ticker_id, post_id, mention_type, sentiment_score)
FOREIGN KEY (subreddit_id) REFERENCES subreddits (id)
)
""")
@@ -105,13 +105,69 @@ def clean_stale_tickers():
conn.close()
print(f"Cleanup complete. Removed {deleted_count} records.")
def get_db_connection():
conn = sqlite3.connect(DB_FILE)
conn.row_factory = sqlite3.Row
return conn
def add_mention(conn, ticker_id, subreddit_id, post_id, mention_type, timestamp, sentiment):
def initialize_db():
conn = get_db_connection()
cursor = conn.cursor()
cursor.execute("""
CREATE TABLE IF NOT EXISTS tickers (
id INTEGER PRIMARY KEY AUTOINCREMENT,
symbol TEXT NOT NULL UNIQUE,
market_cap INTEGER,
closing_price REAL,
last_updated INTEGER
)
""")
cursor.execute("""
CREATE TABLE IF NOT EXISTS subreddits (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE
)
""")
cursor.execute("""
CREATE TABLE IF NOT EXISTS mentions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ticker_id INTEGER,
subreddit_id INTEGER,
post_id TEXT NOT NULL,
mention_type TEXT NOT NULL,
mention_sentiment REAL,
post_avg_sentiment REAL,
mention_timestamp INTEGER NOT NULL,
FOREIGN KEY (ticker_id) REFERENCES tickers (id),
FOREIGN KEY (subreddit_id) REFERENCES subreddits (id)
)
""")
cursor.execute("""
CREATE TABLE IF NOT EXISTS posts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
post_id TEXT NOT NULL UNIQUE,
title TEXT NOT NULL,
post_url TEXT,
subreddit_id INTEGER,
post_timestamp INTEGER,
comment_count INTEGER,
avg_comment_sentiment REAL,
FOREIGN KEY (subreddit_id) REFERENCES subreddits (id)
)
""")
conn.commit()
conn.close()
print("Database initialized successfully.")
def add_mention(conn, ticker_id, subreddit_id, post_id, mention_type, timestamp, mention_sentiment, post_avg_sentiment=None):
cursor = conn.cursor()
try:
cursor.execute(
"INSERT INTO mentions (ticker_id, subreddit_id, post_id, mention_type, mention_timestamp, sentiment_score) VALUES (?, ?, ?, ?, ?, ?)",
(ticker_id, subreddit_id, post_id, mention_type, timestamp, sentiment)
"""
INSERT INTO mentions (ticker_id, subreddit_id, post_id, mention_type, mention_timestamp, mention_sentiment, post_avg_sentiment)
VALUES (?, ?, ?, ?, ?, ?, ?)
""",
(ticker_id, subreddit_id, post_id, mention_type, timestamp, mention_sentiment, post_avg_sentiment)
)
conn.commit()
except sqlite3.IntegrityError:
@@ -150,19 +206,26 @@ def generate_summary_report(limit=20):
print(f"\n--- Top {limit} Tickers by Mention Count ---")
conn = get_db_connection()
cursor = conn.cursor()
# --- UPDATED QUERY: Changed m.sentiment_score to m.mention_sentiment ---
query = """
SELECT
t.symbol, t.market_cap, COUNT(m.id) as mention_count,
SUM(CASE WHEN m.sentiment_score > 0.1 THEN 1 ELSE 0 END) as bullish_mentions,
SUM(CASE WHEN m.sentiment_score < -0.1 THEN 1 ELSE 0 END) as bearish_mentions,
SUM(CASE WHEN m.sentiment_score BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions
t.symbol, t.market_cap, t.closing_price,
COUNT(m.id) as mention_count,
SUM(CASE WHEN m.mention_sentiment > 0.1 THEN 1 ELSE 0 END) as bullish_mentions,
SUM(CASE WHEN m.mention_sentiment < -0.1 THEN 1 ELSE 0 END) as bearish_mentions,
SUM(CASE WHEN m.mention_sentiment BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions
FROM mentions m JOIN tickers t ON m.ticker_id = t.id
GROUP BY t.symbol, t.market_cap ORDER BY mention_count DESC LIMIT ?;
GROUP BY t.symbol, t.market_cap, t.closing_price
ORDER BY mention_count DESC
LIMIT ?;
"""
results = cursor.execute(query, (limit,)).fetchall()
header = f"{'Ticker':<8} | {'Mentions':<8} | {'Bullish':<8} | {'Bearish':<8} | {'Neutral':<8} | {'Market Cap':<15}"
header = f"{'Ticker':<8} | {'Mentions':<8} | {'Bullish':<8} | {'Bearish':<8} | {'Neutral':<8} | {'Market Cap':<15} | {'Close Price':<12}"
print(header)
print("-" * len(header))
print("-" * (len(header) + 2)) # Adjusted separator length
for row in results:
market_cap_str = "N/A"
if row['market_cap'] and row['market_cap'] > 0:
@@ -170,45 +233,19 @@ def generate_summary_report(limit=20):
if mc >= 1e12: market_cap_str = f"${mc/1e12:.2f}T"
elif mc >= 1e9: market_cap_str = f"${mc/1e9:.2f}B"
else: market_cap_str = f"${mc/1e6:.2f}M"
print(f"{row['symbol']:<8} | {row['mention_count']:<8} | {row['bullish_mentions']:<8} | {row['bearish_mentions']:<8} | {row['neutral_mentions']:<8} | {market_cap_str:<15}")
conn.close()
def get_overall_summary(limit=50):
conn = get_db_connection()
query = """
SELECT
t.symbol, t.market_cap, t.closing_price, -- Added closing_price
COUNT(m.id) as mention_count,
SUM(CASE WHEN m.sentiment_score > 0.1 THEN 1 ELSE 0 END) as bullish_mentions,
SUM(CASE WHEN m.sentiment_score < -0.1 THEN 1 ELSE 0 END) as bearish_mentions,
SUM(CASE WHEN m.sentiment_score BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions
FROM mentions m JOIN tickers t ON m.ticker_id = t.id
GROUP BY t.symbol, t.market_cap, t.closing_price -- Added closing_price
ORDER BY mention_count DESC LIMIT ?;
"""
results = conn.execute(query, (limit,)).fetchall()
conn.close()
return results
closing_price_str = f"${row['closing_price']:.2f}" if row['closing_price'] else "N/A"
def get_subreddit_summary(subreddit_name, limit=50):
conn = get_db_connection()
query = """
SELECT
t.symbol, t.market_cap, t.closing_price, -- Added closing_price
COUNT(m.id) as mention_count,
SUM(CASE WHEN m.sentiment_score > 0.1 THEN 1 ELSE 0 END) as bullish_mentions,
SUM(CASE WHEN m.sentiment_score < -0.1 THEN 1 ELSE 0 END) as bearish_mentions,
SUM(CASE WHEN m.sentiment_score BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions
FROM mentions m
JOIN tickers t ON m.ticker_id = t.id
JOIN subreddits s ON m.subreddit_id = s.id
WHERE s.name = ?
GROUP BY t.symbol, t.market_cap, t.closing_price -- Added closing_price
ORDER BY mention_count DESC LIMIT ?;
"""
results = conn.execute(query, (subreddit_name, limit)).fetchall()
print(
f"{row['symbol']:<8} | "
f"{row['mention_count']:<8} | "
f"{row['bullish_mentions']:<8} | "
f"{row['bearish_mentions']:<8} | "
f"{row['neutral_mentions']:<8} | "
f"{market_cap_str:<15} | "
f"{closing_price_str:<12}"
)
conn.close()
return results
def get_all_scanned_subreddits():
"""Gets a unique list of all subreddits we have data for."""
@@ -253,20 +290,52 @@ def get_deep_dive_details(ticker_symbol):
conn.close()
return results
def get_image_view_summary(subreddit_name):
"""
Gets a summary of tickers for the image view, including post, comment,
and sentiment counts.
"""
def get_overall_summary(limit=50):
conn = get_db_connection()
query = """
SELECT
t.symbol, t.market_cap, t.closing_price,
COUNT(m.id) as mention_count,
SUM(CASE WHEN m.mention_sentiment > 0.1 THEN 1 ELSE 0 END) as bullish_mentions,
SUM(CASE WHEN m.mention_sentiment < -0.1 THEN 1 ELSE 0 END) as bearish_mentions,
SUM(CASE WHEN m.mention_sentiment BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions
FROM mentions m JOIN tickers t ON m.ticker_id = t.id
GROUP BY t.symbol, t.market_cap, t.closing_price
ORDER BY mention_count DESC LIMIT ?;
"""
results = conn.execute(query, (limit,)).fetchall()
conn.close()
return results
def get_subreddit_summary(subreddit_name, limit=50):
conn = get_db_connection()
query = """
SELECT
t.symbol, t.market_cap, t.closing_price,
COUNT(m.id) as mention_count,
SUM(CASE WHEN m.mention_sentiment > 0.1 THEN 1 ELSE 0 END) as bullish_mentions,
SUM(CASE WHEN m.mention_sentiment < -0.1 THEN 1 ELSE 0 END) as bearish_mentions,
SUM(CASE WHEN m.mention_sentiment BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions
FROM mentions m
JOIN tickers t ON m.ticker_id = t.id
JOIN subreddits s ON m.subreddit_id = s.id
WHERE s.name = ?
GROUP BY t.symbol, t.market_cap, t.closing_price
ORDER BY mention_count DESC LIMIT ?;
"""
results = conn.execute(query, (subreddit_name, limit)).fetchall()
conn.close()
return results
def get_image_view_summary(subreddit_name):
conn = get_db_connection()
# This query now also counts sentiment types
query = """
SELECT
t.symbol,
COUNT(CASE WHEN m.mention_type = 'post' THEN 1 END) as post_mentions,
COUNT(CASE WHEN m.mention_type = 'comment' THEN 1 END) as comment_mentions,
COUNT(CASE WHEN m.sentiment_score > 0.1 THEN 1 END) as bullish_mentions,
COUNT(CASE WHEN m.sentiment_score < -0.1 THEN 1 END) as bearish_mentions
COUNT(CASE WHEN m.mention_sentiment > 0.1 THEN 1 END) as bullish_mentions,
COUNT(CASE WHEN m.mention_sentiment < -0.1 THEN 1 END) as bearish_mentions
FROM mentions m
JOIN tickers t ON m.ticker_id = t.id
JOIN subreddits s ON m.subreddit_id = s.id
@@ -280,23 +349,16 @@ def get_image_view_summary(subreddit_name):
return results
def get_weekly_summary_for_subreddit(subreddit_name):
"""
Gets a weekly summary for a specific subreddit for the image view.
"""
conn = get_db_connection()
# Calculate the timestamp for 7 days ago
seven_days_ago = datetime.utcnow() - timedelta(days=7)
seven_days_ago_timestamp = int(seven_days_ago.timestamp())
# The query is the same as before, but with an added WHERE clause for the timestamp
query = """
SELECT
t.symbol,
COUNT(CASE WHEN m.mention_type = 'post' THEN 1 END) as post_mentions,
COUNT(CASE WHEN m.mention_type = 'comment' THEN 1 END) as comment_mentions,
COUNT(CASE WHEN m.sentiment_score > 0.1 THEN 1 END) as bullish_mentions,
COUNT(CASE WHEN m.sentiment_score < -0.1 THEN 1 END) as bearish_mentions
COUNT(CASE WHEN m.mention_sentiment > 0.1 THEN 1 END) as bullish_mentions,
COUNT(CASE WHEN m.mention_sentiment < -0.1 THEN 1 END) as bearish_mentions
FROM mentions m
JOIN tickers t ON m.ticker_id = t.id
JOIN subreddits s ON m.subreddit_id = s.id

5
rstat_tool/format_blacklist.py
View File

@@ -46,7 +46,10 @@ COMMON_WORDS_BLACKLIST = {
"USD", "UTC", "VALUE", "VOO", "VP", "VR", "VTI", "WAGMI", "WANT", "WATCH",
"WAY", "WE", "WEB3", "WEEK", "WHALE", "WHO", "WHY", "WIDE", "WILL", "WORDS",
"WORTH", "WOULD", "WSB", "WTF", "XRP", "YES", "YET", "YIELD", "YOLO", "YOU",
"YOUR", "YOY", "YT", "YTD", "ZAR", "ZEN", "ZERO"
"YOUR", "YOY", "YT", "YTD", "ZAR", "ZEN", "ZERO",
"SOUTH", "WIRE", "NORTH", "EAST", "WEST", "AREA", "FTD", "NEAT", "ISIN", "BROKE", "TOLD",
"HUGE", "XO", "NASA", "DAYS", "ENV", "NZ", "IBS", "POSCO", "GUH", "IKKE"
}
def format_and_print_list(word_set, words_per_line=10):

85
rstat_tool/main.py
View File

@@ -43,6 +43,11 @@ def get_reddit_instance():
return praw.Reddit(client_id=client_id, client_secret=client_secret, user_agent=user_agent)
def scan_subreddits(reddit, subreddits_list, post_limit=100, comment_limit=100, days_to_scan=1):
"""
Scans subreddits with a hybrid mention counting logic.
- If a ticker is in the title, it gets credit for all comments.
- If not, tickers only get credit for direct mentions in comments.
"""
conn = database.get_db_connection()
post_age_limit = days_to_scan * 86400
current_time = time.time()
@@ -56,53 +61,77 @@ def scan_subreddits(reddit, subreddits_list, post_limit=100, comment_limit=100,
for submission in subreddit.new(limit=post_limit):
if (current_time - submission.created_utc) > post_age_limit:
print(f" -> Reached posts older than the {days_to_scan}-day limit. Moving to next subreddit.")
print(f" -> Reached posts older than the {days_to_scan}-day limit.")
break
post_text = submission.title + " " + submission.selftext
tickers_in_post = extract_tickers(post_text)
if tickers_in_post:
post_sentiment = get_sentiment_score(submission.title)
for ticker_symbol in set(tickers_in_post):
ticker_id = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol)
database.add_mention(conn, ticker_id, subreddit_id, submission.id, 'post', int(submission.created_utc), post_sentiment)
ticker_info = database.get_ticker_info(conn, ticker_id)
if not ticker_info['last_updated'] or (current_time - ticker_info['last_updated'] > MARKET_CAP_REFRESH_INTERVAL):
print(f" -> Fetching financial data for {ticker_symbol}...")
financials = get_financial_data(ticker_symbol)
database.update_ticker_financials(
conn, ticker_id,
financials['market_cap'] or ticker_info['market_cap'],
financials['closing_price'] or ticker_info['closing_price']
)
# --- NEW HYBRID LOGIC ---
tickers_in_title = set(extract_tickers(submission.title))
all_tickers_found_in_post = set(tickers_in_title) # Start a set to track all tickers for financials
submission.comments.replace_more(limit=0)
all_comment_sentiments = []
for comment in submission.comments.list()[:comment_limit]:
all_comment_sentiments.append(get_sentiment_score(comment.body))
tickers_in_comment = extract_tickers(comment.body)
if tickers_in_comment:
all_comments = submission.comments.list()[:comment_limit]
# --- CASE A: Tickers were found in the title ---
if tickers_in_title:
print(f" -> Title Mention(s): {', '.join(tickers_in_title)}. Attributing all comments.")
post_sentiment = get_sentiment_score(submission.title)
# Add one 'post' mention for each title ticker
for ticker_symbol in tickers_in_title:
ticker_id = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol)
database.add_mention(conn, ticker_id, subreddit_id, submission.id, 'post', int(submission.created_utc), post_sentiment)
# Add one 'comment' mention for EACH comment FOR EACH title ticker
for comment in all_comments:
comment_sentiment = get_sentiment_score(comment.body)
for ticker_symbol in set(tickers_in_comment):
for ticker_symbol in tickers_in_title:
ticker_id = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol)
database.add_mention(conn, ticker_id, subreddit_id, submission.id, 'comment', int(comment.created_utc), comment_sentiment)
# --- CASE B: No tickers in the title, scan comments individually ---
else:
for comment in all_comments:
tickers_in_comment = set(extract_tickers(comment.body))
if tickers_in_comment:
all_tickers_found_in_post.update(tickers_in_comment) # Add to our set for financials
comment_sentiment = get_sentiment_score(comment.body)
for ticker_symbol in tickers_in_comment:
ticker_id = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol)
database.add_mention(conn, ticker_id, subreddit_id, submission.id, 'comment', int(comment.created_utc), comment_sentiment)
# --- EFFICIENT FINANCIALS UPDATE ---
# Now, update market cap once for every unique ticker found in the whole post
for ticker_symbol in all_tickers_found_in_post:
ticker_id = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol)
ticker_info = database.get_ticker_info(conn, ticker_id)
if not ticker_info['last_updated'] or (current_time - ticker_info['last_updated'] > MARKET_CAP_REFRESH_INTERVAL):
print(f" -> Fetching financial data for {ticker_symbol}...")
financials = get_financial_data(ticker_symbol)
database.update_ticker_financials(
conn, ticker_id,
financials['market_cap'] or ticker_info['market_cap'],
financials['closing_price'] or ticker_info['closing_price']
)
# --- DEEP DIVE SAVE (Still valuable) ---
all_comment_sentiments = [get_sentiment_score(c.body) for c in all_comments]
avg_sentiment = sum(all_comment_sentiments) / len(all_comment_sentiments) if all_comment_sentiments else 0
post_analysis_data = {
"post_id": submission.id, "title": submission.title,
"post_url": f"https://reddit.com{submission.permalink}",
"subreddit_id": subreddit_id, "post_timestamp": int(submission.created_utc),
"comment_count": len(all_comment_sentiments), "avg_comment_sentiment": avg_sentiment
"post_url": f"https://reddit.com{submission.permalink}", "subreddit_id": subreddit_id,
"post_timestamp": int(submission.created_utc), "comment_count": len(all_comments),
"avg_comment_sentiment": avg_sentiment
}
database.add_or_update_post_analysis(conn, post_analysis_data)
except Exception as e:
print(f"Could not scan r/{subreddit_name}. Error: {e}")
conn.close()
print("\n--- Scan Complete ---")
def main():
"""Main function to run the Reddit stock analysis tool."""
parser = argparse.ArgumentParser(description="Analyze stock ticker mentions on Reddit.", formatter_class=argparse.RawTextHelpFormatter)