pkhamre/reddit_stock_analyzer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactored scraping logic.

Browse Source
This commit is contained in:
Pål-Kristian Hamre
2025-07-31 22:23:52 +02:00
parent 2f43380970
commit 8ebaaf8b36
3 changed files with 138 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files

193
rstat_tool/database.py
View File

@@ -2,7 +2,7 @@
import sqlite3
import time
from .ticker_extractor import COMMON_WORDS_BLACKLIST
from .ticker_extractor import COMMON_WORDS_BLACKLIST, extract_golden_tickers, extract_potential_tickers
from .logger_setup import logger as log
from datetime import datetime, timedelta, timezone
@@ -203,23 +203,6 @@ def get_ticker_info(conn, ticker_id):
return cursor.fetchone()
def get_week_start_end(for_date):
"""
Calculates the start (Monday, 00:00:00) and end (Sunday, 23:59:59)
of the week that a given date falls into.
Returns two datetime objects.
"""
# Monday is 0, Sunday is 6
start_of_week = for_date - timedelta(days=for_date.weekday())
end_of_week = start_of_week + timedelta(days=6)
# Set time to the very beginning and very end of the day for an inclusive range
start_of_week = start_of_week.replace(hour=0, minute=0, second=0, microsecond=0)
end_of_week = end_of_week.replace(hour=23, minute=59, second=59, microsecond=999999)
return start_of_week, end_of_week
def add_or_update_post_analysis(conn, post_data):
"""
Inserts a new post analysis record or updates an existing one.
@@ -240,127 +223,16 @@ def add_or_update_post_analysis(conn, post_data):
conn.commit()
def get_overall_summary(limit=10):
"""
Gets the top tickers across all subreddits from the LAST 24 HOURS.
"""
conn = get_db_connection()
one_day_ago = datetime.now(timezone.utc) - timedelta(days=1)
one_day_ago_timestamp = int(one_day_ago.timestamp())
query = """
SELECT t.symbol, t.market_cap, t.closing_price, COUNT(m.id) as mention_count,
SUM(CASE WHEN m.mention_sentiment > 0.1 THEN 1 ELSE 0 END) as bullish_mentions,
SUM(CASE WHEN m.mention_sentiment < -0.1 THEN 1 ELSE 0 END) as bearish_mentions,
SUM(CASE WHEN m.mention_sentiment BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions
FROM mentions m JOIN tickers t ON m.ticker_id = t.id
WHERE m.mention_timestamp >= ? -- <-- ADDED TIME FILTER
GROUP BY t.symbol, t.market_cap, t.closing_price
ORDER BY mention_count DESC LIMIT ?;
"""
results = conn.execute(query, (one_day_ago_timestamp, limit)).fetchall()
conn.close()
return results
def get_subreddit_summary(subreddit_name, limit=10):
"""
Gets the top tickers for a specific subreddit from the LAST 24 HOURS.
"""
conn = get_db_connection()
one_day_ago = datetime.now(timezone.utc) - timedelta(days=1)
one_day_ago_timestamp = int(one_day_ago.timestamp())
query = """
SELECT t.symbol, t.market_cap, t.closing_price, COUNT(m.id) as mention_count,
SUM(CASE WHEN m.mention_sentiment > 0.1 THEN 1 ELSE 0 END) as bullish_mentions,
SUM(CASE WHEN m.mention_sentiment < -0.1 THEN 1 ELSE 0 END) as bearish_mentions,
SUM(CASE WHEN m.mention_sentiment BETWEEN -0.1 AND 0.1 THEN 1 ELSE 0 END) as neutral_mentions
FROM mentions m JOIN tickers t ON m.ticker_id = t.id JOIN subreddits s ON m.subreddit_id = s.id
WHERE LOWER(s.name) = LOWER(?) AND m.mention_timestamp >= ? -- <-- ADDED TIME FILTER
GROUP BY t.symbol, t.market_cap, t.closing_price
ORDER BY mention_count DESC LIMIT ?;
"""
results = conn.execute(
query, (subreddit_name, one_day_ago_timestamp, limit)
).fetchall()
conn.close()
return results
def get_daily_summary_for_subreddit(subreddit_name):
"""Gets a summary for the DAILY image view (last 24 hours)."""
conn = get_db_connection()
one_day_ago = datetime.now(timezone.utc) - timedelta(days=1)
one_day_ago_timestamp = int(one_day_ago.timestamp())
query = """
SELECT
t.symbol, t.market_cap, t.closing_price,
COUNT(m.id) as total_mentions,
COUNT(CASE WHEN m.mention_sentiment > 0.1 THEN 1 END) as bullish_mentions,
COUNT(CASE WHEN m.mention_sentiment < -0.1 THEN 1 END) as bearish_mentions
FROM mentions m JOIN tickers t ON m.ticker_id = t.id JOIN subreddits s ON m.subreddit_id = s.id
WHERE LOWER(s.name) = LOWER(?) AND m.mention_timestamp >= ?
GROUP BY t.symbol, t.market_cap, t.closing_price
ORDER BY total_mentions DESC LIMIT 10;
"""
results = conn.execute(query, (subreddit_name, one_day_ago_timestamp)).fetchall()
conn.close()
return results
def get_weekly_summary_for_subreddit(subreddit_name, for_date):
"""Gets a summary for the WEEKLY image view (full week)."""
conn = get_db_connection()
start_of_week, end_of_week = get_week_start_end(for_date)
start_timestamp = int(start_of_week.timestamp())
end_timestamp = int(end_of_week.timestamp())
query = """
SELECT
t.symbol, t.market_cap, t.closing_price,
COUNT(m.id) as total_mentions,
COUNT(CASE WHEN m.mention_sentiment > 0.1 THEN 1 END) as bullish_mentions,
COUNT(CASE WHEN m.mention_sentiment < -0.1 THEN 1 END) as bearish_mentions
FROM mentions m JOIN tickers t ON m.ticker_id = t.id JOIN subreddits s ON m.subreddit_id = s.id
WHERE LOWER(s.name) = LOWER(?) AND m.mention_timestamp BETWEEN ? AND ?
GROUP BY t.symbol, t.market_cap, t.closing_price
ORDER BY total_mentions DESC LIMIT 10;
"""
results = conn.execute(
query, (subreddit_name, start_timestamp, end_timestamp)
).fetchall()
conn.close()
return results, start_of_week, end_of_week
def get_overall_image_view_summary():
"""
Gets a summary of top tickers across ALL subreddits for the DAILY image view (last 24 hours).
"""
conn = get_db_connection()
one_day_ago = datetime.now(timezone.utc) - timedelta(days=1)
one_day_ago_timestamp = int(one_day_ago.timestamp())
query = """
SELECT
t.symbol, t.market_cap, t.closing_price,
COUNT(m.id) as total_mentions,
COUNT(CASE WHEN m.mention_sentiment > 0.1 THEN 1 END) as bullish_mentions,
COUNT(CASE WHEN m.mention_sentiment < -0.1 THEN 1 END) as bearish_mentions
FROM mentions m JOIN tickers t ON m.ticker_id = t.id
WHERE m.mention_timestamp >= ? -- <-- ADDED TIME FILTER
GROUP BY t.symbol, t.market_cap, t.closing_price
ORDER BY total_mentions DESC LIMIT 10;
"""
results = conn.execute(query, (one_day_ago_timestamp,)).fetchall()
conn.close()
return results
def get_week_start_end(for_date):
"""Calculates the start (Monday) and end (Sunday) of the week."""
start_of_week = for_date - timedelta(days=for_date.weekday())
end_of_week = start_of_week + timedelta(days=6)
start_of_week = start_of_week.replace(hour=0, minute=0, second=0, microsecond=0)
end_of_week = end_of_week.replace(hour=23, minute=59, second=59, microsecond=999999)
return start_of_week, end_of_week
def get_overall_daily_summary():
"""
Gets the top tickers across all subreddits from the LAST 24 HOURS.
(This is a copy of get_overall_summary, renamed for clarity).
"""
"""Gets the top tickers across all subreddits from the LAST 24 HOURS."""
conn = get_db_connection()
one_day_ago = datetime.now(timezone.utc) - timedelta(days=1)
one_day_ago_timestamp = int(one_day_ago.timestamp())
@@ -377,16 +249,12 @@ def get_overall_daily_summary():
conn.close()
return results
def get_overall_weekly_summary():
"""
Gets the top tickers across all subreddits for the LAST 7 DAYS.
"""
"""Gets the top tickers across all subreddits for LAST WEEK (Mon-Sun)."""
conn = get_db_connection()
today = datetime.now(timezone.utc)
start_of_week, end_of_week = get_week_start_end(
today - timedelta(days=7)
) # Get last week's boundaries
target_date_for_last_week = today - timedelta(days=7)
start_of_week, end_of_week = get_week_start_end(target_date_for_last_week)
start_timestamp = int(start_of_week.timestamp())
end_timestamp = int(end_of_week.timestamp())
query = """
@@ -402,6 +270,43 @@ def get_overall_weekly_summary():
conn.close()
return results, start_of_week, end_of_week
def get_daily_summary_for_subreddit(subreddit_name):
"""Gets a summary for a subreddit's DAILY view (last 24 hours)."""
conn = get_db_connection()
one_day_ago = datetime.now(timezone.utc) - timedelta(days=1)
one_day_ago_timestamp = int(one_day_ago.timestamp())
query = """
SELECT t.symbol, t.market_cap, t.closing_price, COUNT(m.id) as total_mentions,
SUM(CASE WHEN m.mention_sentiment > 0.1 THEN 1 ELSE 0 END) as bullish_mentions,
SUM(CASE WHEN m.mention_sentiment < -0.1 THEN 1 ELSE 0 END) as bearish_mentions
FROM mentions m JOIN tickers t ON m.ticker_id = t.id JOIN subreddits s ON m.subreddit_id = s.id
WHERE LOWER(s.name) = LOWER(?) AND m.mention_timestamp >= ?
GROUP BY t.symbol, t.market_cap, t.closing_price
ORDER BY total_mentions DESC LIMIT 10;
"""
results = conn.execute(query, (subreddit_name, one_day_ago_timestamp)).fetchall()
conn.close()
return results
def get_weekly_summary_for_subreddit(subreddit_name, for_date):
"""Gets a summary for a subreddit's WEEKLY view (for a specific week)."""
conn = get_db_connection()
start_of_week, end_of_week = get_week_start_end(for_date)
start_timestamp = int(start_of_week.timestamp())
end_timestamp = int(end_of_week.timestamp())
query = """
SELECT t.symbol, t.market_cap, t.closing_price, COUNT(m.id) as total_mentions,
SUM(CASE WHEN m.mention_sentiment > 0.1 THEN 1 ELSE 0 END) as bullish_mentions,
SUM(CASE WHEN m.mention_sentiment < -0.1 THEN 1 ELSE 0 END) as bearish_mentions
FROM mentions m JOIN tickers t ON m.ticker_id = t.id JOIN subreddits s ON m.subreddit_id = s.id
WHERE LOWER(s.name) = LOWER(?) AND m.mention_timestamp BETWEEN ? AND ?
GROUP BY t.symbol, t.market_cap, t.closing_price
ORDER BY total_mentions DESC LIMIT 10;
"""
results = conn.execute(query, (subreddit_name, start_timestamp, end_timestamp)).fetchall()
conn.close()
return results, start_of_week, end_of_week
def get_deep_dive_details(ticker_symbol):
"""Gets all analyzed posts that mention a specific ticker."""