pkhamre/reddit_stock_analyzer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Format all code.

Browse Source
This commit is contained in:
Pål-Kristian Hamre
2025-07-25 23:22:37 +02:00
parent f940470de3
commit 56e0965a5f
19 changed files with 850 additions and 353 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
rstat_tool/database.py
View File

@@ -9,6 +9,7 @@ from datetime import datetime, timedelta, timezone
DB_FILE = "reddit_stocks.db"
MARKET_CAP_REFRESH_INTERVAL = 86400
def clean_stale_tickers():
"""
Removes tickers and their associated mentions from the database
@@ -18,9 +19,9 @@ def clean_stale_tickers():
conn = get_db_connection()
cursor = conn.cursor()
placeholders = ','.join('?' for _ in COMMON_WORDS_BLACKLIST)
placeholders = ",".join("?" for _ in COMMON_WORDS_BLACKLIST)
query = f"SELECT id, symbol FROM tickers WHERE symbol IN ({placeholders})"
cursor.execute(query, tuple(COMMON_WORDS_BLACKLIST))
stale_tickers = cursor.fetchall()
@@ -30,17 +31,18 @@ def clean_stale_tickers():
return
for ticker in stale_tickers:
ticker_id = ticker['id']
ticker_symbol = ticker['symbol']
ticker_id = ticker["id"]
ticker_symbol = ticker["symbol"]
log.info(f"Removing stale ticker '{ticker_symbol}' (ID: {ticker_id})...")
cursor.execute("DELETE FROM mentions WHERE ticker_id = ?", (ticker_id,))
cursor.execute("DELETE FROM tickers WHERE id = ?", (ticker_id,))
deleted_count = conn.total_changes
conn.commit()
conn.close()
log.info(f"Cleanup complete. Removed {deleted_count} records.")
def clean_stale_subreddits(active_subreddits):
"""
Removes all data associated with subreddits that are NOT in the active list.
@@ -57,9 +59,9 @@ def clean_stale_subreddits(active_subreddits):
db_subreddits = cursor.fetchall()
stale_sub_ids = []
for sub in db_subreddits:
if sub['name'] not in active_subreddits_lower:
if sub["name"] not in active_subreddits_lower:
log.info(f"Found stale subreddit to remove: r/{sub['name']}")
stale_sub_ids.append(sub['id'])
stale_sub_ids.append(sub["id"])
if not stale_sub_ids:
log.info("No stale subreddits to clean.")
conn.close()
@@ -73,15 +75,18 @@ def clean_stale_subreddits(active_subreddits):
conn.close()
log.info("Stale subreddit cleanup complete.")
def get_db_connection():
conn = sqlite3.connect(DB_FILE)
conn.row_factory = sqlite3.Row
return conn
def initialize_db():
conn = get_db_connection()
cursor = conn.cursor()
cursor.execute("""
cursor.execute(
"""
CREATE TABLE IF NOT EXISTS tickers (
id INTEGER PRIMARY KEY AUTOINCREMENT,
symbol TEXT NOT NULL UNIQUE,
@@ -89,14 +94,18 @@ def initialize_db():
closing_price REAL,
last_updated INTEGER
)
""")
cursor.execute("""
"""
)
cursor.execute(
"""
CREATE TABLE IF NOT EXISTS subreddits (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE
)
""")
cursor.execute("""
"""
)
cursor.execute(
"""
CREATE TABLE IF NOT EXISTS mentions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ticker_id INTEGER,
@@ -109,8 +118,10 @@ def initialize_db():
FOREIGN KEY (ticker_id) REFERENCES tickers (id),
FOREIGN KEY (subreddit_id) REFERENCES subreddits (id)
)
""")
cursor.execute("""
"""
)
cursor.execute(
"""
CREATE TABLE IF NOT EXISTS posts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
post_id TEXT NOT NULL UNIQUE,
@@ -122,12 +133,23 @@ def initialize_db():
avg_comment_sentiment REAL,
FOREIGN KEY (subreddit_id) REFERENCES subreddits (id)
)
""")
"""
)
conn.commit()
conn.close()
log.info("Database initialized successfully.")
def add_mention(conn, ticker_id, subreddit_id, post_id, mention_type, timestamp, mention_sentiment, post_avg_sentiment=None):
def add_mention(
conn,
ticker_id,
subreddit_id,
post_id,
mention_type,
timestamp,
mention_sentiment,
post_avg_sentiment=None,
):
cursor = conn.cursor()
try:
cursor.execute(
@@ -135,40 +157,52 @@ def add_mention(conn, ticker_id, subreddit_id, post_id, mention_type, timestamp,
INSERT INTO mentions (ticker_id, subreddit_id, post_id, mention_type, mention_timestamp, mention_sentiment, post_avg_sentiment)
VALUES (?, ?, ?, ?, ?, ?, ?)
""",
(ticker_id, subreddit_id, post_id, mention_type, timestamp, mention_sentiment, post_avg_sentiment)
(
ticker_id,
subreddit_id,
post_id,
mention_type,
timestamp,
mention_sentiment,
post_avg_sentiment,
),
)
conn.commit()
except sqlite3.IntegrityError:
pass
def get_or_create_entity(conn, table_name, column_name, value):
"""Generic function to get or create an entity and return its ID."""
cursor = conn.cursor()
cursor.execute(f"SELECT id FROM {table_name} WHERE {column_name} = ?", (value,))
result = cursor.fetchone()
if result:
return result['id']
return result["id"]
else:
cursor.execute(f"INSERT INTO {table_name} ({column_name}) VALUES (?)", (value,))
conn.commit()
return cursor.lastrowid
def update_ticker_financials(conn, ticker_id, market_cap, closing_price):
"""Updates the financials and timestamp for a specific ticker."""
cursor = conn.cursor()
current_timestamp = int(time.time())
cursor.execute(
"UPDATE tickers SET market_cap = ?, closing_price = ?, last_updated = ? WHERE id = ?",
(market_cap, closing_price, current_timestamp, ticker_id)
(market_cap, closing_price, current_timestamp, ticker_id),
)
conn.commit()
def get_ticker_info(conn, ticker_id):
"""Retrieves all info for a specific ticker by its ID."""
cursor = conn.cursor()
cursor.execute("SELECT * FROM tickers WHERE id = ?", (ticker_id,))
return cursor.fetchone()
def get_week_start_end(for_date):
"""
Calculates the start (Monday, 00:00:00) and end (Sunday, 23:59:59)
@@ -178,13 +212,14 @@ def get_week_start_end(for_date):
# Monday is 0, Sunday is 6
start_of_week = for_date - timedelta(days=for_date.weekday())
end_of_week = start_of_week + timedelta(days=6)
# Set time to the very beginning and very end of the day for an inclusive range
start_of_week = start_of_week.replace(hour=0, minute=0, second=0, microsecond=0)
end_of_week = end_of_week.replace(hour=23, minute=59, second=59, microsecond=999999)
return start_of_week, end_of_week
def add_or_update_post_analysis(conn, post_data):
"""
Inserts a new post analysis record or updates an existing one.
@@ -200,10 +235,11 @@ def add_or_update_post_analysis(conn, post_data):
comment_count = excluded.comment_count,
avg_comment_sentiment = excluded.avg_comment_sentiment;
""",
post_data
post_data,
)
conn.commit()
def get_overall_summary(limit=10):
"""
Gets the top tickers across all subreddits from the LAST 24 HOURS.
@@ -211,7 +247,7 @@ def get_overall_summary(limit=10):
conn = get_db_connection()
one_day_ago = datetime.now(timezone.utc) - timedelta(days=1)
one_day_ago_timestamp = int(one_day_ago.timestamp())
query = """
SELECT t.symbol, t.market_cap, t.closing_price, COUNT(m.id) as mention_count,
SUM(CASE WHEN m.mention_sentiment > 0.1 THEN 1 ELSE 0 END) as bullish_mentions,
@@ -226,6 +262,7 @@ def get_overall_summary(limit=10):
conn.close()
return results
def get_subreddit_summary(subreddit_name, limit=10):
"""
Gets the top tickers for a specific subreddit from the LAST 24 HOURS.
@@ -233,7 +270,7 @@ def get_subreddit_summary(subreddit_name, limit=10):
conn = get_db_connection()
one_day_ago = datetime.now(timezone.utc) - timedelta(days=1)
one_day_ago_timestamp = int(one_day_ago.timestamp())
query = """
SELECT t.symbol, t.market_cap, t.closing_price, COUNT(m.id) as mention_count,
SUM(CASE WHEN m.mention_sentiment > 0.1 THEN 1 ELSE 0 END) as bullish_mentions,
@@ -244,12 +281,15 @@ def get_subreddit_summary(subreddit_name, limit=10):
GROUP BY t.symbol, t.market_cap, t.closing_price
ORDER BY mention_count DESC LIMIT ?;
"""
results = conn.execute(query, (subreddit_name, one_day_ago_timestamp, limit)).fetchall()
results = conn.execute(
query, (subreddit_name, one_day_ago_timestamp, limit)
).fetchall()
conn.close()
return results
def get_daily_summary_for_subreddit(subreddit_name):
""" Gets a summary for the DAILY image view (last 24 hours). """
"""Gets a summary for the DAILY image view (last 24 hours)."""
conn = get_db_connection()
one_day_ago = datetime.now(timezone.utc) - timedelta(days=1)
one_day_ago_timestamp = int(one_day_ago.timestamp())
@@ -268,8 +308,9 @@ def get_daily_summary_for_subreddit(subreddit_name):
conn.close()
return results
def get_weekly_summary_for_subreddit(subreddit_name, for_date):
""" Gets a summary for the WEEKLY image view (full week). """
"""Gets a summary for the WEEKLY image view (full week)."""
conn = get_db_connection()
start_of_week, end_of_week = get_week_start_end(for_date)
start_timestamp = int(start_of_week.timestamp())
@@ -285,10 +326,13 @@ def get_weekly_summary_for_subreddit(subreddit_name, for_date):
GROUP BY t.symbol, t.market_cap, t.closing_price
ORDER BY total_mentions DESC LIMIT 10;
"""
results = conn.execute(query, (subreddit_name, start_timestamp, end_timestamp)).fetchall()
results = conn.execute(
query, (subreddit_name, start_timestamp, end_timestamp)
).fetchall()
conn.close()
return results, start_of_week, end_of_week
def get_overall_image_view_summary():
"""
Gets a summary of top tickers across ALL subreddits for the DAILY image view (last 24 hours).
@@ -311,6 +355,7 @@ def get_overall_image_view_summary():
conn.close()
return results
def get_overall_daily_summary():
"""
Gets the top tickers across all subreddits from the LAST 24 HOURS.
@@ -332,13 +377,16 @@ def get_overall_daily_summary():
conn.close()
return results
def get_overall_weekly_summary():
"""
Gets the top tickers across all subreddits for the LAST 7 DAYS.
"""
conn = get_db_connection()
today = datetime.now(timezone.utc)
start_of_week, end_of_week = get_week_start_end(today - timedelta(days=7)) # Get last week's boundaries
start_of_week, end_of_week = get_week_start_end(
today - timedelta(days=7)
) # Get last week's boundaries
start_timestamp = int(start_of_week.timestamp())
end_timestamp = int(end_of_week.timestamp())
query = """
@@ -354,8 +402,9 @@ def get_overall_weekly_summary():
conn.close()
return results, start_of_week, end_of_week
def get_deep_dive_details(ticker_symbol):
""" Gets all analyzed posts that mention a specific ticker. """
"""Gets all analyzed posts that mention a specific ticker."""
conn = get_db_connection()
query = """
SELECT DISTINCT p.*, s.name as subreddit_name FROM posts p
@@ -367,12 +416,16 @@ def get_deep_dive_details(ticker_symbol):
conn.close()
return results
def get_all_scanned_subreddits():
""" Gets a unique list of all subreddits we have data for. """
"""Gets a unique list of all subreddits we have data for."""
conn = get_db_connection()
results = conn.execute("SELECT DISTINCT name FROM subreddits ORDER BY name ASC;").fetchall()
results = conn.execute(
"SELECT DISTINCT name FROM subreddits ORDER BY name ASC;"
).fetchall()
conn.close()
return [row['name'] for row in results]
return [row["name"] for row in results]
def get_all_tickers():
"""Retrieves the ID and symbol of every ticker in the database."""
@@ -381,6 +434,7 @@ def get_all_tickers():
conn.close()
return results
def get_ticker_by_symbol(symbol):
"""
Retrieves a single ticker's ID and symbol from the database.
@@ -388,11 +442,14 @@ def get_ticker_by_symbol(symbol):
"""
conn = get_db_connection()
cursor = conn.cursor()
cursor.execute("SELECT id, symbol FROM tickers WHERE LOWER(symbol) = LOWER(?)", (symbol,))
cursor.execute(
"SELECT id, symbol FROM tickers WHERE LOWER(symbol) = LOWER(?)", (symbol,)
)
result = cursor.fetchone()
conn.close()
return result
def get_top_daily_ticker_symbols():
"""Gets a simple list of the Top 10 ticker symbols from the last 24 hours."""
conn = get_db_connection()
@@ -405,7 +462,8 @@ def get_top_daily_ticker_symbols():
"""
results = conn.execute(query, (one_day_ago_timestamp,)).fetchall()
conn.close()
return [row['symbol'] for row in results] # Return a simple list of strings
return [row["symbol"] for row in results] # Return a simple list of strings
def get_top_weekly_ticker_symbols():
"""Gets a simple list of the Top 10 ticker symbols from the last 7 days."""
@@ -419,7 +477,8 @@ def get_top_weekly_ticker_symbols():
"""
results = conn.execute(query, (seven_days_ago_timestamp,)).fetchall()
conn.close()
return [row['symbol'] for row in results] # Return a simple list of strings
return [row["symbol"] for row in results] # Return a simple list of strings
def get_top_daily_ticker_symbols_for_subreddit(subreddit_name):
"""Gets a list of the Top 10 daily ticker symbols for a specific subreddit."""
@@ -432,9 +491,16 @@ def get_top_daily_ticker_symbols_for_subreddit(subreddit_name):
WHERE LOWER(s.name) = LOWER(?) AND m.mention_timestamp >= ?
GROUP BY t.symbol ORDER BY COUNT(m.id) DESC LIMIT 10;
"""
results = conn.execute(query, (subreddit_name, one_day_ago_timestamp,)).fetchall()
results = conn.execute(
query,
(
subreddit_name,
one_day_ago_timestamp,
),
).fetchall()
conn.close()
return [row['symbol'] for row in results]
return [row["symbol"] for row in results]
def get_top_weekly_ticker_symbols_for_subreddit(subreddit_name):
"""Gets a list of the Top 10 weekly ticker symbols for a specific subreddit."""
@@ -447,6 +513,12 @@ def get_top_weekly_ticker_symbols_for_subreddit(subreddit_name):
WHERE LOWER(s.name) = LOWER(?) AND m.mention_timestamp >= ?
GROUP BY t.symbol ORDER BY COUNT(m.id) DESC LIMIT 10;
"""
results = conn.execute(query, (subreddit_name, seven_days_ago_timestamp,)).fetchall()
results = conn.execute(
query,
(
subreddit_name,
seven_days_ago_timestamp,
),
).fetchall()
conn.close()
return [row['symbol'] for row in results]
return [row["symbol"] for row in results]