pkhamre/reddit_stock_analyzer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Format all code.

Browse Source
This commit is contained in:
Pål-Kristian Hamre
2025-07-25 23:22:37 +02:00
parent f940470de3
commit 56e0965a5f
19 changed files with 850 additions and 353 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
rstat_tool/cleanup.py
View File

@@ -3,45 +3,56 @@
import argparse
from . import database
from .logger_setup import setup_logging, logger as log
# We can't reuse load_subreddits from main anymore if it's not in the same file
# So we will duplicate it here. It's small and keeps this script self-contained.
import json
def load_subreddits(filepath):
"""Loads a list of subreddits from a JSON file."""
try:
with open(filepath, 'r') as f:
with open(filepath, "r") as f:
data = json.load(f)
return data.get("subreddits", [])
except (FileNotFoundError, json.JSONDecodeError) as e:
log.error(f"Error loading config file '{filepath}': {e}")
return None
def run_cleanup():
"""Main function for the cleanup tool."""
parser = argparse.ArgumentParser(
description="A tool to clean stale data from the RSTAT database.",
formatter_class=argparse.RawTextHelpFormatter
formatter_class=argparse.RawTextHelpFormatter,
)
parser.add_argument("--tickers", action="store_true", help="Clean tickers that are in the blacklist.")
parser.add_argument(
"--tickers",
action="store_true",
help="Clean tickers that are in the blacklist.",
)
# --- UPDATED ARGUMENT DEFINITION ---
# nargs='?': Makes the argument optional.
# const='subreddits.json': The value used if the flag is present with no argument.
# default=None: The value if the flag is not present at all.
parser.add_argument(
"--subreddits",
nargs='?',
const='subreddits.json',
"--subreddits",
nargs="?",
const="subreddits.json",
default=None,
help="Clean data from subreddits NOT in the specified config file.\n(Defaults to 'subreddits.json' if flag is used without a value)."
help="Clean data from subreddits NOT in the specified config file.\n(Defaults to 'subreddits.json' if flag is used without a value).",
)
parser.add_argument("--all", action="store_true", help="Run all available cleanup tasks.")
parser.add_argument("--stdout", action="store_true", help="Print all log messages to the console.")
parser.add_argument(
"--all", action="store_true", help="Run all available cleanup tasks."
)
parser.add_argument(
"--stdout", action="store_true", help="Print all log messages to the console."
)
args = parser.parse_args()
setup_logging(console_verbose=args.stdout)
run_any_task = False
@@ -57,7 +68,7 @@ def run_cleanup():
if args.all or args.subreddits is not None:
run_any_task = True
# If --all is used, default to 'subreddits.json' if --subreddits wasn't also specified
config_file = args.subreddits or 'subreddits.json'
config_file = args.subreddits or "subreddits.json"
log.info(f"\nCleaning subreddits based on active list in: {config_file}")
active_subreddits = load_subreddits(config_file)
if active_subreddits is not None:
@@ -65,10 +76,13 @@ def run_cleanup():
if not run_any_task:
parser.print_help()
log.error("\nError: Please provide at least one cleanup option (e.g., --tickers, --subreddits, --all).")
log.error(
"\nError: Please provide at least one cleanup option (e.g., --tickers, --subreddits, --all)."
)
return
log.critical("\nCleanup finished.")
if __name__ == "__main__":
run_cleanup()
run_cleanup()

45
rstat_tool/dashboard.py
View File

@@ -8,13 +8,14 @@ from .database import (
get_deep_dive_details,
get_daily_summary_for_subreddit,
get_weekly_summary_for_subreddit,
get_overall_daily_summary, # Now correctly imported
get_overall_weekly_summary # Now correctly imported
get_overall_daily_summary, # Now correctly imported
get_overall_weekly_summary, # Now correctly imported
)
app = Flask(__name__, template_folder='../templates')
app = Flask(__name__, template_folder="../templates")
@app.template_filter('format_mc')
@app.template_filter("format_mc")
def format_market_cap(mc):
"""Formats a large number into a readable market cap string."""
if mc is None or mc == 0:
@@ -28,26 +29,28 @@ def format_market_cap(mc):
else:
return f"${mc:,}"
@app.context_processor
def inject_subreddits():
"""Makes the list of all subreddits available to every template for the navbar."""
return dict(all_subreddits=get_all_scanned_subreddits())
@app.route("/")
def overall_dashboard():
"""Handler for the main, overall dashboard."""
view_type = request.args.get('view', 'daily')
is_image_mode = request.args.get('image') == 'true'
if view_type == 'weekly':
view_type = request.args.get("view", "daily")
is_image_mode = request.args.get("image") == "true"
if view_type == "weekly":
tickers, start, end = get_overall_weekly_summary()
date_string = f"{start.strftime('%b %d')} - {end.strftime('%b %d, %Y')}"
subtitle = "All Subreddits - Top 10 Weekly"
else: # Default to daily
else: # Default to daily
tickers = get_overall_daily_summary()
date_string = datetime.now(timezone.utc).strftime("%Y-%m-%d")
subtitle = "All Subreddits - Top 10 Daily"
return render_template(
"dashboard_view.html",
title="Overall Dashboard",
@@ -57,26 +60,27 @@ def overall_dashboard():
view_type=view_type,
subreddit_name=None,
is_image_mode=is_image_mode,
base_url="/"
base_url="/",
)
@app.route("/subreddit/<name>")
def subreddit_dashboard(name):
"""Handler for per-subreddit dashboards."""
view_type = request.args.get('view', 'daily')
is_image_mode = request.args.get('image') == 'true'
if view_type == 'weekly':
view_type = request.args.get("view", "daily")
is_image_mode = request.args.get("image") == "true"
if view_type == "weekly":
today = datetime.now(timezone.utc)
target_date = today - timedelta(days=7)
tickers, start, end = get_weekly_summary_for_subreddit(name, target_date)
date_string = f"{start.strftime('%b %d')} - {end.strftime('%b %d, %Y')}"
subtitle = f"r/{name} - Top 10 Weekly"
else: # Default to daily
else: # Default to daily
tickers = get_daily_summary_for_subreddit(name)
date_string = datetime.now(timezone.utc).strftime("%Y-%m-%d")
subtitle = f"r/{name} - Top 10 Daily"
return render_template(
"dashboard_view.html",
title=f"r/{name} Dashboard",
@@ -86,9 +90,10 @@ def subreddit_dashboard(name):
view_type=view_type,
subreddit_name=name,
is_image_mode=is_image_mode,
base_url=f"/subreddit/{name}"
base_url=f"/subreddit/{name}",
)
@app.route("/deep-dive/<symbol>")
def deep_dive(symbol):
"""The handler for the deep-dive page for a specific ticker."""
@@ -96,6 +101,7 @@ def deep_dive(symbol):
posts = get_deep_dive_details(symbol)
return render_template("deep_dive.html", posts=posts, symbol=symbol)
def start_dashboard():
"""The main function called by the 'rstat-dashboard' command."""
log.info("Starting Flask server...")
@@ -103,5 +109,6 @@ def start_dashboard():
log.info("Press CTRL+C to stop the server.")
app.run(debug=True)
if __name__ == "__main__":
start_dashboard()
start_dashboard()

152
rstat_tool/database.py
View File

@@ -9,6 +9,7 @@ from datetime import datetime, timedelta, timezone
DB_FILE = "reddit_stocks.db"
MARKET_CAP_REFRESH_INTERVAL = 86400
def clean_stale_tickers():
"""
Removes tickers and their associated mentions from the database
@@ -18,9 +19,9 @@ def clean_stale_tickers():
conn = get_db_connection()
cursor = conn.cursor()
placeholders = ','.join('?' for _ in COMMON_WORDS_BLACKLIST)
placeholders = ",".join("?" for _ in COMMON_WORDS_BLACKLIST)
query = f"SELECT id, symbol FROM tickers WHERE symbol IN ({placeholders})"
cursor.execute(query, tuple(COMMON_WORDS_BLACKLIST))
stale_tickers = cursor.fetchall()
@@ -30,17 +31,18 @@ def clean_stale_tickers():
return
for ticker in stale_tickers:
ticker_id = ticker['id']
ticker_symbol = ticker['symbol']
ticker_id = ticker["id"]
ticker_symbol = ticker["symbol"]
log.info(f"Removing stale ticker '{ticker_symbol}' (ID: {ticker_id})...")
cursor.execute("DELETE FROM mentions WHERE ticker_id = ?", (ticker_id,))
cursor.execute("DELETE FROM tickers WHERE id = ?", (ticker_id,))
deleted_count = conn.total_changes
conn.commit()
conn.close()
log.info(f"Cleanup complete. Removed {deleted_count} records.")
def clean_stale_subreddits(active_subreddits):
"""
Removes all data associated with subreddits that are NOT in the active list.
@@ -57,9 +59,9 @@ def clean_stale_subreddits(active_subreddits):
db_subreddits = cursor.fetchall()
stale_sub_ids = []
for sub in db_subreddits:
if sub['name'] not in active_subreddits_lower:
if sub["name"] not in active_subreddits_lower:
log.info(f"Found stale subreddit to remove: r/{sub['name']}")
stale_sub_ids.append(sub['id'])
stale_sub_ids.append(sub["id"])
if not stale_sub_ids:
log.info("No stale subreddits to clean.")
conn.close()
@@ -73,15 +75,18 @@ def clean_stale_subreddits(active_subreddits):
conn.close()
log.info("Stale subreddit cleanup complete.")
def get_db_connection():
conn = sqlite3.connect(DB_FILE)
conn.row_factory = sqlite3.Row
return conn
def initialize_db():
conn = get_db_connection()
cursor = conn.cursor()
cursor.execute("""
cursor.execute(
"""
CREATE TABLE IF NOT EXISTS tickers (
id INTEGER PRIMARY KEY AUTOINCREMENT,
symbol TEXT NOT NULL UNIQUE,
@@ -89,14 +94,18 @@ def initialize_db():
closing_price REAL,
last_updated INTEGER
)
""")
cursor.execute("""
"""
)
cursor.execute(
"""
CREATE TABLE IF NOT EXISTS subreddits (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL UNIQUE
)
""")
cursor.execute("""
"""
)
cursor.execute(
"""
CREATE TABLE IF NOT EXISTS mentions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ticker_id INTEGER,
@@ -109,8 +118,10 @@ def initialize_db():
FOREIGN KEY (ticker_id) REFERENCES tickers (id),
FOREIGN KEY (subreddit_id) REFERENCES subreddits (id)
)
""")
cursor.execute("""
"""
)
cursor.execute(
"""
CREATE TABLE IF NOT EXISTS posts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
post_id TEXT NOT NULL UNIQUE,
@@ -122,12 +133,23 @@ def initialize_db():
avg_comment_sentiment REAL,
FOREIGN KEY (subreddit_id) REFERENCES subreddits (id)
)
""")
"""
)
conn.commit()
conn.close()
log.info("Database initialized successfully.")
def add_mention(conn, ticker_id, subreddit_id, post_id, mention_type, timestamp, mention_sentiment, post_avg_sentiment=None):
def add_mention(
conn,
ticker_id,
subreddit_id,
post_id,
mention_type,
timestamp,
mention_sentiment,
post_avg_sentiment=None,
):
cursor = conn.cursor()
try:
cursor.execute(
@@ -135,40 +157,52 @@ def add_mention(conn, ticker_id, subreddit_id, post_id, mention_type, timestamp,
INSERT INTO mentions (ticker_id, subreddit_id, post_id, mention_type, mention_timestamp, mention_sentiment, post_avg_sentiment)
VALUES (?, ?, ?, ?, ?, ?, ?)
""",
(ticker_id, subreddit_id, post_id, mention_type, timestamp, mention_sentiment, post_avg_sentiment)
(
ticker_id,
subreddit_id,
post_id,
mention_type,
timestamp,
mention_sentiment,
post_avg_sentiment,
),
)
conn.commit()
except sqlite3.IntegrityError:
pass
def get_or_create_entity(conn, table_name, column_name, value):
"""Generic function to get or create an entity and return its ID."""
cursor = conn.cursor()
cursor.execute(f"SELECT id FROM {table_name} WHERE {column_name} = ?", (value,))
result = cursor.fetchone()
if result:
return result['id']
return result["id"]
else:
cursor.execute(f"INSERT INTO {table_name} ({column_name}) VALUES (?)", (value,))
conn.commit()
return cursor.lastrowid
def update_ticker_financials(conn, ticker_id, market_cap, closing_price):
"""Updates the financials and timestamp for a specific ticker."""
cursor = conn.cursor()
current_timestamp = int(time.time())
cursor.execute(
"UPDATE tickers SET market_cap = ?, closing_price = ?, last_updated = ? WHERE id = ?",
(market_cap, closing_price, current_timestamp, ticker_id)
(market_cap, closing_price, current_timestamp, ticker_id),
)
conn.commit()
def get_ticker_info(conn, ticker_id):
"""Retrieves all info for a specific ticker by its ID."""
cursor = conn.cursor()
cursor.execute("SELECT * FROM tickers WHERE id = ?", (ticker_id,))
return cursor.fetchone()
def get_week_start_end(for_date):
"""
Calculates the start (Monday, 00:00:00) and end (Sunday, 23:59:59)
@@ -178,13 +212,14 @@ def get_week_start_end(for_date):
# Monday is 0, Sunday is 6
start_of_week = for_date - timedelta(days=for_date.weekday())
end_of_week = start_of_week + timedelta(days=6)
# Set time to the very beginning and very end of the day for an inclusive range
start_of_week = start_of_week.replace(hour=0, minute=0, second=0, microsecond=0)
end_of_week = end_of_week.replace(hour=23, minute=59, second=59, microsecond=999999)
return start_of_week, end_of_week
def add_or_update_post_analysis(conn, post_data):
"""
Inserts a new post analysis record or updates an existing one.
@@ -200,10 +235,11 @@ def add_or_update_post_analysis(conn, post_data):
comment_count = excluded.comment_count,
avg_comment_sentiment = excluded.avg_comment_sentiment;
""",
post_data
post_data,
)
conn.commit()
def get_overall_summary(limit=10):
"""
Gets the top tickers across all subreddits from the LAST 24 HOURS.
@@ -211,7 +247,7 @@ def get_overall_summary(limit=10):
conn = get_db_connection()
one_day_ago = datetime.now(timezone.utc) - timedelta(days=1)
one_day_ago_timestamp = int(one_day_ago.timestamp())
query = """
SELECT t.symbol, t.market_cap, t.closing_price, COUNT(m.id) as mention_count,
SUM(CASE WHEN m.mention_sentiment > 0.1 THEN 1 ELSE 0 END) as bullish_mentions,
@@ -226,6 +262,7 @@ def get_overall_summary(limit=10):
conn.close()
return results
def get_subreddit_summary(subreddit_name, limit=10):
"""
Gets the top tickers for a specific subreddit from the LAST 24 HOURS.
@@ -233,7 +270,7 @@ def get_subreddit_summary(subreddit_name, limit=10):
conn = get_db_connection()
one_day_ago = datetime.now(timezone.utc) - timedelta(days=1)
one_day_ago_timestamp = int(one_day_ago.timestamp())
query = """
SELECT t.symbol, t.market_cap, t.closing_price, COUNT(m.id) as mention_count,
SUM(CASE WHEN m.mention_sentiment > 0.1 THEN 1 ELSE 0 END) as bullish_mentions,
@@ -244,12 +281,15 @@ def get_subreddit_summary(subreddit_name, limit=10):
GROUP BY t.symbol, t.market_cap, t.closing_price
ORDER BY mention_count DESC LIMIT ?;
"""
results = conn.execute(query, (subreddit_name, one_day_ago_timestamp, limit)).fetchall()
results = conn.execute(
query, (subreddit_name, one_day_ago_timestamp, limit)
).fetchall()
conn.close()
return results
def get_daily_summary_for_subreddit(subreddit_name):
""" Gets a summary for the DAILY image view (last 24 hours). """
"""Gets a summary for the DAILY image view (last 24 hours)."""
conn = get_db_connection()
one_day_ago = datetime.now(timezone.utc) - timedelta(days=1)
one_day_ago_timestamp = int(one_day_ago.timestamp())
@@ -268,8 +308,9 @@ def get_daily_summary_for_subreddit(subreddit_name):
conn.close()
return results
def get_weekly_summary_for_subreddit(subreddit_name, for_date):
""" Gets a summary for the WEEKLY image view (full week). """
"""Gets a summary for the WEEKLY image view (full week)."""
conn = get_db_connection()
start_of_week, end_of_week = get_week_start_end(for_date)
start_timestamp = int(start_of_week.timestamp())
@@ -285,10 +326,13 @@ def get_weekly_summary_for_subreddit(subreddit_name, for_date):
GROUP BY t.symbol, t.market_cap, t.closing_price
ORDER BY total_mentions DESC LIMIT 10;
"""
results = conn.execute(query, (subreddit_name, start_timestamp, end_timestamp)).fetchall()
results = conn.execute(
query, (subreddit_name, start_timestamp, end_timestamp)
).fetchall()
conn.close()
return results, start_of_week, end_of_week
def get_overall_image_view_summary():
"""
Gets a summary of top tickers across ALL subreddits for the DAILY image view (last 24 hours).
@@ -311,6 +355,7 @@ def get_overall_image_view_summary():
conn.close()
return results
def get_overall_daily_summary():
"""
Gets the top tickers across all subreddits from the LAST 24 HOURS.
@@ -332,13 +377,16 @@ def get_overall_daily_summary():
conn.close()
return results
def get_overall_weekly_summary():
"""
Gets the top tickers across all subreddits for the LAST 7 DAYS.
"""
conn = get_db_connection()
today = datetime.now(timezone.utc)
start_of_week, end_of_week = get_week_start_end(today - timedelta(days=7)) # Get last week's boundaries
start_of_week, end_of_week = get_week_start_end(
today - timedelta(days=7)
) # Get last week's boundaries
start_timestamp = int(start_of_week.timestamp())
end_timestamp = int(end_of_week.timestamp())
query = """
@@ -354,8 +402,9 @@ def get_overall_weekly_summary():
conn.close()
return results, start_of_week, end_of_week
def get_deep_dive_details(ticker_symbol):
""" Gets all analyzed posts that mention a specific ticker. """
"""Gets all analyzed posts that mention a specific ticker."""
conn = get_db_connection()
query = """
SELECT DISTINCT p.*, s.name as subreddit_name FROM posts p
@@ -367,12 +416,16 @@ def get_deep_dive_details(ticker_symbol):
conn.close()
return results
def get_all_scanned_subreddits():
""" Gets a unique list of all subreddits we have data for. """
"""Gets a unique list of all subreddits we have data for."""
conn = get_db_connection()
results = conn.execute("SELECT DISTINCT name FROM subreddits ORDER BY name ASC;").fetchall()
results = conn.execute(
"SELECT DISTINCT name FROM subreddits ORDER BY name ASC;"
).fetchall()
conn.close()
return [row['name'] for row in results]
return [row["name"] for row in results]
def get_all_tickers():
"""Retrieves the ID and symbol of every ticker in the database."""
@@ -381,6 +434,7 @@ def get_all_tickers():
conn.close()
return results
def get_ticker_by_symbol(symbol):
"""
Retrieves a single ticker's ID and symbol from the database.
@@ -388,11 +442,14 @@ def get_ticker_by_symbol(symbol):
"""
conn = get_db_connection()
cursor = conn.cursor()
cursor.execute("SELECT id, symbol FROM tickers WHERE LOWER(symbol) = LOWER(?)", (symbol,))
cursor.execute(
"SELECT id, symbol FROM tickers WHERE LOWER(symbol) = LOWER(?)", (symbol,)
)
result = cursor.fetchone()
conn.close()
return result
def get_top_daily_ticker_symbols():
"""Gets a simple list of the Top 10 ticker symbols from the last 24 hours."""
conn = get_db_connection()
@@ -405,7 +462,8 @@ def get_top_daily_ticker_symbols():
"""
results = conn.execute(query, (one_day_ago_timestamp,)).fetchall()
conn.close()
return [row['symbol'] for row in results] # Return a simple list of strings
return [row["symbol"] for row in results] # Return a simple list of strings
def get_top_weekly_ticker_symbols():
"""Gets a simple list of the Top 10 ticker symbols from the last 7 days."""
@@ -419,7 +477,8 @@ def get_top_weekly_ticker_symbols():
"""
results = conn.execute(query, (seven_days_ago_timestamp,)).fetchall()
conn.close()
return [row['symbol'] for row in results] # Return a simple list of strings
return [row["symbol"] for row in results] # Return a simple list of strings
def get_top_daily_ticker_symbols_for_subreddit(subreddit_name):
"""Gets a list of the Top 10 daily ticker symbols for a specific subreddit."""
@@ -432,9 +491,16 @@ def get_top_daily_ticker_symbols_for_subreddit(subreddit_name):
WHERE LOWER(s.name) = LOWER(?) AND m.mention_timestamp >= ?
GROUP BY t.symbol ORDER BY COUNT(m.id) DESC LIMIT 10;
"""
results = conn.execute(query, (subreddit_name, one_day_ago_timestamp,)).fetchall()
results = conn.execute(
query,
(
subreddit_name,
one_day_ago_timestamp,
),
).fetchall()
conn.close()
return [row['symbol'] for row in results]
return [row["symbol"] for row in results]
def get_top_weekly_ticker_symbols_for_subreddit(subreddit_name):
"""Gets a list of the Top 10 weekly ticker symbols for a specific subreddit."""
@@ -447,6 +513,12 @@ def get_top_weekly_ticker_symbols_for_subreddit(subreddit_name):
WHERE LOWER(s.name) = LOWER(?) AND m.mention_timestamp >= ?
GROUP BY t.symbol ORDER BY COUNT(m.id) DESC LIMIT 10;
"""
results = conn.execute(query, (subreddit_name, seven_days_ago_timestamp,)).fetchall()
results = conn.execute(
query,
(
subreddit_name,
seven_days_ago_timestamp,
),
).fetchall()
conn.close()
return [row['symbol'] for row in results]
return [row["symbol"] for row in results]

19
rstat_tool/format_blacklist.py
View File

@@ -115,7 +115,7 @@ COMMON_WORDS_BLACKLIST = {
def format_and_print_list(word_set, words_per_line=10):
"""
Sorts a set of words and prints it in a specific format.
Args:
word_set (set): The set of words to process.
words_per_line (int): The number of words to print on each line.
@@ -123,32 +123,33 @@ def format_and_print_list(word_set, words_per_line=10):
# 1. Convert the set to a list to ensure order, and sort it alphabetically.
# The set is also used to remove any duplicates from the initial list.
sorted_words = sorted(list(word_set))
# 2. Start printing the output
print("COMMON_WORDS_BLACKLIST = {")
# 3. Iterate through the sorted list and print words, respecting the line limit
for i in range(0, len(sorted_words), words_per_line):
# Get a chunk of words for the current line
line_chunk = sorted_words[i:i + words_per_line]
line_chunk = sorted_words[i : i + words_per_line]
# Format each word with double quotes
formatted_words = [f'"{word}"' for word in line_chunk]
# Join the words with a comma and a space
line_content = ", ".join(formatted_words)
# Add a trailing comma if it's not the last line
is_last_line = (i + words_per_line) >= len(sorted_words)
if not is_last_line:
line_content += ","
# Print the indented line
print(f" {line_content}")
# 4. Print the closing brace
print("}")
# --- Main execution ---
if __name__ == "__main__":
format_and_print_list(COMMON_WORDS_BLACKLIST)
format_and_print_list(COMMON_WORDS_BLACKLIST)

19
rstat_tool/logger_setup.py
View File

@@ -5,6 +5,7 @@ import sys
logger = logging.getLogger("rstat_app")
def setup_logging(console_verbose=False, debug_mode=False):
"""
Configures the application's logger with a new DEBUG level.
@@ -12,30 +13,32 @@ def setup_logging(console_verbose=False, debug_mode=False):
# The logger itself must be set to the lowest possible level (DEBUG).
log_level = logging.DEBUG if debug_mode else logging.INFO
logger.setLevel(log_level)
logger.propagate = False
if logger.hasHandlers():
logger.handlers.clear()
# File Handler (Always verbose at INFO level or higher)
file_handler = logging.FileHandler("rstat.log", mode='a')
file_handler.setLevel(logging.INFO) # We don't need debug spam in the file usually
file_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
file_handler = logging.FileHandler("rstat.log", mode="a")
file_handler.setLevel(logging.INFO) # We don't need debug spam in the file usually
file_formatter = logging.Formatter(
"%(asctime)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
)
file_handler.setFormatter(file_formatter)
logger.addHandler(file_handler)
# Console Handler (Verbosity is controlled)
console_handler = logging.StreamHandler(sys.stdout)
console_formatter = logging.Formatter('%(message)s')
console_formatter = logging.Formatter("%(message)s")
console_handler.setFormatter(console_formatter)
if debug_mode:
console_handler.setLevel(logging.DEBUG)
elif console_verbose:
console_handler.setLevel(logging.INFO)
else:
console_handler.setLevel(logging.CRITICAL)
logger.addHandler(console_handler)
# YFINANCE LOGGER CAPTURE
@@ -45,4 +48,4 @@ def setup_logging(console_verbose=False, debug_mode=False):
yfinance_logger.handlers.clear()
yfinance_logger.setLevel(logging.WARNING)
yfinance_logger.addHandler(console_handler)
yfinance_logger.addHandler(file_handler)
yfinance_logger.addHandler(file_handler)

315
rstat_tool/main.py
View File

@@ -16,27 +16,32 @@ from .ticker_extractor import extract_tickers
from .sentiment_analyzer import get_sentiment_score
from .logger_setup import setup_logging, logger as log
def load_subreddits(filepath):
"""Loads a list of subreddits from a JSON file."""
try:
with open(filepath, 'r') as f:
with open(filepath, "r") as f:
return json.load(f).get("subreddits", [])
except (FileNotFoundError, json.JSONDecodeError) as e:
log.error(f"Error loading config file '{filepath}': {e}")
return None
def get_reddit_instance():
"""Initializes and returns a PRAW Reddit instance."""
env_path = Path(__file__).parent.parent / '.env'
env_path = Path(__file__).parent.parent / ".env"
load_dotenv(dotenv_path=env_path)
client_id = os.getenv("REDDIT_CLIENT_ID")
client_secret = os.getenv("REDDIT_CLIENT_SECRET")
user_agent = os.getenv("REDDIT_USER_AGENT")
if not all([client_id, client_secret, user_agent]):
log.error("Error: Reddit API credentials not found in .env file.")
return None
return praw.Reddit(client_id=client_id, client_secret=client_secret, user_agent=user_agent)
return praw.Reddit(
client_id=client_id, client_secret=client_secret, user_agent=user_agent
)
def get_financial_data_via_fetcher(ticker_symbol):
"""
@@ -48,38 +53,45 @@ def get_financial_data_via_fetcher(ticker_symbol):
# --- Call 1: Get Market Cap ---
try:
mc_script_path = project_root / 'fetch_market_cap.py'
mc_script_path = project_root / "fetch_market_cap.py"
command_mc = [sys.executable, str(mc_script_path), ticker_symbol]
result_mc = subprocess.run(command_mc, capture_output=True, text=True, check=True, timeout=30)
result_mc = subprocess.run(
command_mc, capture_output=True, text=True, check=True, timeout=30
)
financials.update(json.loads(result_mc.stdout))
except Exception as e:
log.warning(f"Market cap fetcher failed for {ticker_symbol}: {e}")
# --- Call 2: Get Closing Price ---
try:
cp_script_path = project_root / 'fetch_close_price.py'
cp_script_path = project_root / "fetch_close_price.py"
command_cp = [sys.executable, str(cp_script_path), ticker_symbol]
result_cp = subprocess.run(command_cp, capture_output=True, text=True, check=True, timeout=30)
result_cp = subprocess.run(
command_cp, capture_output=True, text=True, check=True, timeout=30
)
financials.update(json.loads(result_cp.stdout))
except Exception as e:
log.warning(f"Closing price fetcher failed for {ticker_symbol}: {e}")
return financials
# --- HELPER FUNCTION: Contains all the optimized logic for one post ---
def _process_submission(submission, subreddit_id, conn, comment_limit, fetch_financials):
def _process_submission(
submission, subreddit_id, conn, comment_limit, fetch_financials
):
"""
Processes a single Reddit submission with optimized logic.
- Uses a single loop over comments.
- Caches ticker IDs to reduce DB lookups.
"""
current_time = time.time()
# 1. Initialize data collectors for this post
tickers_in_title = set(extract_tickers(submission.title))
all_tickers_found_in_post = set(tickers_in_title)
all_comment_sentiments = []
ticker_id_cache = {} # In-memory cache for ticker IDs for this post
ticker_id_cache = {} # In-memory cache for ticker IDs for this post
submission.comments.replace_more(limit=0)
all_comments = submission.comments.list()[:comment_limit]
@@ -88,8 +100,8 @@ def _process_submission(submission, subreddit_id, conn, comment_limit, fetch_fin
# We gather all necessary information in one pass.
for comment in all_comments:
comment_sentiment = get_sentiment_score(comment.body)
all_comment_sentiments.append(comment_sentiment) # For the deep dive
all_comment_sentiments.append(comment_sentiment) # For the deep dive
tickers_in_comment = set(extract_tickers(comment.body))
if not tickers_in_comment:
continue
@@ -101,147 +113,266 @@ def _process_submission(submission, subreddit_id, conn, comment_limit, fetch_fin
# If the title has tickers, every comment is a mention for them
for ticker_symbol in tickers_in_title:
if ticker_symbol not in ticker_id_cache:
ticker_id_cache[ticker_symbol] = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol)
ticker_id_cache[ticker_symbol] = database.get_or_create_entity(
conn, "tickers", "symbol", ticker_symbol
)
ticker_id = ticker_id_cache[ticker_symbol]
database.add_mention(conn, ticker_id, subreddit_id, submission.id, 'comment', int(comment.created_utc), comment_sentiment)
database.add_mention(
conn,
ticker_id,
subreddit_id,
submission.id,
"comment",
int(comment.created_utc),
comment_sentiment,
)
else:
# If no title tickers, only direct mentions in comments count
for ticker_symbol in tickers_in_comment:
if ticker_symbol not in ticker_id_cache:
ticker_id_cache[ticker_symbol] = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol)
ticker_id_cache[ticker_symbol] = database.get_or_create_entity(
conn, "tickers", "symbol", ticker_symbol
)
ticker_id = ticker_id_cache[ticker_symbol]
database.add_mention(conn, ticker_id, subreddit_id, submission.id, 'comment', int(comment.created_utc), comment_sentiment)
database.add_mention(
conn,
ticker_id,
subreddit_id,
submission.id,
"comment",
int(comment.created_utc),
comment_sentiment,
)
# 3. Process title mentions (if any)
if tickers_in_title:
log.info(f" -> Title Mention(s): {', '.join(tickers_in_title)}. Attributing all comments.")
log.info(
f" -> Title Mention(s): {', '.join(tickers_in_title)}. Attributing all comments."
)
post_sentiment = get_sentiment_score(submission.title)
for ticker_symbol in tickers_in_title:
if ticker_symbol not in ticker_id_cache:
ticker_id_cache[ticker_symbol] = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol)
ticker_id_cache[ticker_symbol] = database.get_or_create_entity(
conn, "tickers", "symbol", ticker_symbol
)
ticker_id = ticker_id_cache[ticker_symbol]
database.add_mention(conn, ticker_id, subreddit_id, submission.id, 'post', int(submission.created_utc), post_sentiment)
database.add_mention(
conn,
ticker_id,
subreddit_id,
submission.id,
"post",
int(submission.created_utc),
post_sentiment,
)
# 4. Fetch financial data if enabled
if fetch_financials:
for ticker_symbol in all_tickers_found_in_post:
ticker_id = ticker_id_cache[ticker_symbol] # Guaranteed to be in cache
ticker_id = ticker_id_cache[ticker_symbol] # Guaranteed to be in cache
ticker_info = database.get_ticker_info(conn, ticker_id)
if not ticker_info['last_updated'] or (current_time - ticker_info['last_updated'] > database.MARKET_CAP_REFRESH_INTERVAL):
if not ticker_info["last_updated"] or (
current_time - ticker_info["last_updated"]
> database.MARKET_CAP_REFRESH_INTERVAL
):
log.info(f" -> Fetching financial data for {ticker_symbol}...")
financials = get_financial_data_via_fetcher(ticker_symbol)
database.update_ticker_financials(conn, ticker_id, financials.get('market_cap'), financials.get('closing_price'))
database.update_ticker_financials(
conn,
ticker_id,
financials.get("market_cap"),
financials.get("closing_price"),
)
# 5. Save deep dive analysis
avg_sentiment = sum(all_comment_sentiments) / len(all_comment_sentiments) if all_comment_sentiments else 0
avg_sentiment = (
sum(all_comment_sentiments) / len(all_comment_sentiments)
if all_comment_sentiments
else 0
)
post_analysis_data = {
"post_id": submission.id, "title": submission.title,
"post_url": f"https://reddit.com{submission.permalink}", "subreddit_id": subreddit_id,
"post_timestamp": int(submission.created_utc), "comment_count": len(all_comments),
"avg_comment_sentiment": avg_sentiment
"post_id": submission.id,
"title": submission.title,
"post_url": f"https://reddit.com{submission.permalink}",
"subreddit_id": subreddit_id,
"post_timestamp": int(submission.created_utc),
"comment_count": len(all_comments),
"avg_comment_sentiment": avg_sentiment,
}
database.add_or_update_post_analysis(conn, post_analysis_data)
def scan_subreddits(reddit, subreddits_list, post_limit=100, comment_limit=100, days_to_scan=1, fetch_financials=True):
def scan_subreddits(
reddit,
subreddits_list,
post_limit=100,
comment_limit=100,
days_to_scan=1,
fetch_financials=True,
):
conn = database.get_db_connection()
post_age_limit = days_to_scan * 86400
current_time = time.time()
log.info(f"Scanning {len(subreddits_list)} subreddit(s) for NEW posts in the last {days_to_scan} day(s)...")
log.info(
f"Scanning {len(subreddits_list)} subreddit(s) for NEW posts in the last {days_to_scan} day(s)..."
)
if not fetch_financials:
log.warning("NOTE: Financial data fetching is disabled for this run.")
for subreddit_name in subreddits_list:
try:
normalized_sub_name = subreddit_name.lower()
subreddit_id = database.get_or_create_entity(conn, 'subreddits', 'name', normalized_sub_name)
subreddit_id = database.get_or_create_entity(
conn, "subreddits", "name", normalized_sub_name
)
subreddit = reddit.subreddit(normalized_sub_name)
log.info(f"Scanning r/{normalized_sub_name}...")
for submission in subreddit.new(limit=post_limit):
if (current_time - submission.created_utc) > post_age_limit:
log.info(f" -> Reached posts older than the {days_to_scan}-day limit.")
log.info(
f" -> Reached posts older than the {days_to_scan}-day limit."
)
break
# Call the new helper function for each post
_process_submission(submission, subreddit_id, conn, comment_limit, fetch_financials)
_process_submission(
submission, subreddit_id, conn, comment_limit, fetch_financials
)
except Exception as e:
log.error(f"Could not scan r/{normalized_sub_name}. Error: {e}", exc_info=True)
log.error(
f"Could not scan r/{normalized_sub_name}. Error: {e}", exc_info=True
)
conn.close()
log.critical("\n--- Scan Complete ---")
def main():
"""Main function to run the Reddit stock analysis tool."""
parser = argparse.ArgumentParser(description="Analyze stock ticker mentions on Reddit.", formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument("-f", "--config", default="subreddits.json", help="Path to the JSON file for scanning. (Default: subreddits.json)")
parser.add_argument("-s", "--subreddit", help="Scan a single subreddit, ignoring the config file.")
parser.add_argument("-d", "--days", type=int, default=1, help="Number of past days to scan for new posts. (Default: 1)")
parser.add_argument("-p", "--posts", type=int, default=200, help="Max posts to check per subreddit. (Default: 200)")
parser.add_argument("-c", "--comments", type=int, default=100, help="Number of comments to scan per post. (Default: 100)")
parser.add_argument("-n", "--no-financials", action="store_true", help="Disable fetching of financial data during the Reddit scan.")
parser.add_argument("--update-top-tickers", action="store_true", help="Update financial data only for tickers currently in the Top 10 daily/weekly dashboards.")
parser.add_argument(
"-u", "--update-financials-only",
nargs='?',
const="ALL_TICKERS", # A special value to signify "update all"
default=None,
metavar='TICKER',
help="Update financials. Provide a ticker symbol to update just one,\nor use the flag alone to update all tickers in the database."
parser = argparse.ArgumentParser(
description="Analyze stock ticker mentions on Reddit.",
formatter_class=argparse.RawTextHelpFormatter,
)
parser.add_argument("--debug", action="store_true", help="Enable detailed debug logging to the console.")
parser.add_argument("--stdout", action="store_true", help="Print all log messages to the console.")
parser.add_argument(
"-f",
"--config",
default="subreddits.json",
help="Path to the JSON file for scanning. (Default: subreddits.json)",
)
parser.add_argument(
"-s", "--subreddit", help="Scan a single subreddit, ignoring the config file."
)
parser.add_argument(
"-d",
"--days",
type=int,
default=1,
help="Number of past days to scan for new posts. (Default: 1)",
)
parser.add_argument(
"-p",
"--posts",
type=int,
default=200,
help="Max posts to check per subreddit. (Default: 200)",
)
parser.add_argument(
"-c",
"--comments",
type=int,
default=100,
help="Number of comments to scan per post. (Default: 100)",
)
parser.add_argument(
"-n",
"--no-financials",
action="store_true",
help="Disable fetching of financial data during the Reddit scan.",
)
parser.add_argument(
"--update-top-tickers",
action="store_true",
help="Update financial data only for tickers currently in the Top 10 daily/weekly dashboards.",
)
parser.add_argument(
"-u",
"--update-financials-only",
nargs="?",
const="ALL_TICKERS", # A special value to signify "update all"
default=None,
metavar="TICKER",
help="Update financials. Provide a ticker symbol to update just one,\nor use the flag alone to update all tickers in the database.",
)
parser.add_argument(
"--debug",
action="store_true",
help="Enable detailed debug logging to the console.",
)
parser.add_argument(
"--stdout", action="store_true", help="Print all log messages to the console."
)
args = parser.parse_args()
setup_logging(console_verbose=args.stdout, debug_mode=args.debug)
database.initialize_db()
if args.update_top_tickers:
log.critical("--- Starting Financial Data Update for Top Tickers ---")
# 1. Start with an empty set to hold all unique tickers
tickers_to_update = set()
# 2. Get the overall top tickers
log.info("-> Checking overall top daily and weekly tickers...")
top_daily_overall = database.get_top_daily_ticker_symbols()
top_weekly_overall = database.get_top_weekly_ticker_symbols()
tickers_to_update.update(top_daily_overall)
tickers_to_update.update(top_weekly_overall)
# 3. Get all subreddits and loop through them
all_subreddits = database.get_all_scanned_subreddits()
log.info(f"-> Checking top tickers for {len(all_subreddits)} individual subreddit(s)...")
log.info(
f"-> Checking top tickers for {len(all_subreddits)} individual subreddit(s)..."
)
for sub_name in all_subreddits:
log.debug(f" -> Checking r/{sub_name}...")
top_daily_sub = database.get_top_daily_ticker_symbols_for_subreddit(sub_name)
top_weekly_sub = database.get_top_weekly_ticker_symbols_for_subreddit(sub_name)
top_daily_sub = database.get_top_daily_ticker_symbols_for_subreddit(
sub_name
)
top_weekly_sub = database.get_top_weekly_ticker_symbols_for_subreddit(
sub_name
)
tickers_to_update.update(top_daily_sub)
tickers_to_update.update(top_weekly_sub)
unique_top_tickers = sorted(list(tickers_to_update))
if not unique_top_tickers:
log.info("No top tickers found in the last week. Nothing to update.")
else:
log.info(f"Found {len(unique_top_tickers)} unique top tickers to update: {', '.join(unique_top_tickers)}")
log.info(
f"Found {len(unique_top_tickers)} unique top tickers to update: {', '.join(unique_top_tickers)}"
)
conn = database.get_db_connection()
for ticker_symbol in unique_top_tickers:
# 4. Find the ticker's ID to perform the update
ticker_info = database.get_ticker_by_symbol(ticker_symbol)
if ticker_info:
log.info(f" -> Updating financials for {ticker_info['symbol']}...")
financials = get_financial_data_via_fetcher(ticker_info['symbol'])
financials = get_financial_data_via_fetcher(ticker_info["symbol"])
database.update_ticker_financials(
conn, ticker_info['id'],
financials.get('market_cap'),
financials.get('closing_price')
conn,
ticker_info["id"],
financials.get("market_cap"),
financials.get("closing_price"),
)
conn.close()
log.critical("--- Top Ticker Financial Data Update Complete ---")
elif args.update_financials_only:
@@ -253,31 +384,37 @@ def main():
log.info(f"Found {len(all_tickers)} tickers in the database to update.")
conn = database.get_db_connection()
for ticker in all_tickers:
symbol = ticker['symbol']
symbol = ticker["symbol"]
log.info(f" -> Updating financials for {symbol}...")
financials = get_financial_data_via_fetcher(symbol)
database.update_ticker_financials(
conn, ticker['id'],
financials.get('market_cap'),
financials.get('closing_price')
conn,
ticker["id"],
financials.get("market_cap"),
financials.get("closing_price"),
)
conn.close()
else:
ticker_symbol_to_update = update_mode
log.critical(f"--- Starting Financial Data Update for single ticker: {ticker_symbol_to_update} ---")
log.critical(
f"--- Starting Financial Data Update for single ticker: {ticker_symbol_to_update} ---"
)
ticker_info = database.get_ticker_by_symbol(ticker_symbol_to_update)
if ticker_info:
conn = database.get_db_connection()
log.info(f" -> Updating financials for {ticker_info['symbol']}...")
financials = get_financial_data_via_fetcher(ticker_info['symbol'])
financials = get_financial_data_via_fetcher(ticker_info["symbol"])
database.update_ticker_financials(
conn, ticker_info['id'],
financials.get('market_cap'),
financials.get('closing_price')
conn,
ticker_info["id"],
financials.get("market_cap"),
financials.get("closing_price"),
)
conn.close()
else:
log.error(f"Ticker '{ticker_symbol_to_update}' not found in the database.")
log.error(
f"Ticker '{ticker_symbol_to_update}' not found in the database."
)
log.critical("--- Financial Data Update Complete ---")
else:
@@ -288,14 +425,15 @@ def main():
log.info(f"Targeted Scan Mode: Focusing on r/{args.subreddit}")
else:
log.info(f"Config Scan Mode: Loading subreddits from {args.config}")
subreddits_to_scan = load_subreddits(args.config)
subreddits_to_scan = load_subreddits(args.config)
if not subreddits_to_scan:
log.error("Error: No subreddits to scan.")
return
reddit = get_reddit_instance()
if not reddit: return
if not reddit:
return
scan_subreddits(
reddit,
@@ -303,8 +441,9 @@ def main():
post_limit=args.posts,
comment_limit=args.comments,
days_to_scan=args.days,
fetch_financials=(not args.no_financials)
fetch_financials=(not args.no_financials),
)
if __name__ == "__main__":
main()
main()

4
rstat_tool/sentiment_analyzer.py
View File

@@ -9,11 +9,11 @@ _analyzer = SentimentIntensityAnalyzer()
def get_sentiment_score(text):
"""
Analyzes a piece of text and returns its sentiment score.
The 'compound' score is a single metric that summarizes the sentiment.
It ranges from -1 (most negative) to +1 (most positive).
"""
# The polarity_scores() method returns a dictionary with 'neg', 'neu', 'pos', and 'compound' scores.
# We are most interested in the 'compound' score.
scores = _analyzer.polarity_scores(text)
return scores['compound']
return scores["compound"]

6
rstat_tool/setup_nltk.py
View File

@@ -3,9 +3,9 @@ import nltk
# This will download the 'vader_lexicon' dataset
# It only needs to be run once
try:
nltk.data.find('sentiment/vader_lexicon.zip')
nltk.data.find("sentiment/vader_lexicon.zip")
print("VADER lexicon is already downloaded.")
except LookupError:
print("Downloading VADER lexicon...")
nltk.download('vader_lexicon')
print("Download complete.")
nltk.download("vader_lexicon")
print("Download complete.")

2
rstat_tool/ticker_extractor.py
View File

@@ -135,4 +135,4 @@ def extract_tickers(text):
if cleaned_ticker not in COMMON_WORDS_BLACKLIST:
tickers.append(cleaned_ticker)
return tickers
return tickers