pkhamre/reddit_stock_analyzer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Format all code.

Browse Source
This commit is contained in:
Pål-Kristian Hamre
2025-07-25 23:22:37 +02:00
parent f940470de3
commit 56e0965a5f
19 changed files with 850 additions and 353 deletions
Download Patch File Download Diff File Expand all files Collapse all files

315
rstat_tool/main.py
View File

@@ -16,27 +16,32 @@ from .ticker_extractor import extract_tickers
from .sentiment_analyzer import get_sentiment_score
from .logger_setup import setup_logging, logger as log
def load_subreddits(filepath):
"""Loads a list of subreddits from a JSON file."""
try:
with open(filepath, 'r') as f:
with open(filepath, "r") as f:
return json.load(f).get("subreddits", [])
except (FileNotFoundError, json.JSONDecodeError) as e:
log.error(f"Error loading config file '{filepath}': {e}")
return None
def get_reddit_instance():
"""Initializes and returns a PRAW Reddit instance."""
env_path = Path(__file__).parent.parent / '.env'
env_path = Path(__file__).parent.parent / ".env"
load_dotenv(dotenv_path=env_path)
client_id = os.getenv("REDDIT_CLIENT_ID")
client_secret = os.getenv("REDDIT_CLIENT_SECRET")
user_agent = os.getenv("REDDIT_USER_AGENT")
if not all([client_id, client_secret, user_agent]):
log.error("Error: Reddit API credentials not found in .env file.")
return None
return praw.Reddit(client_id=client_id, client_secret=client_secret, user_agent=user_agent)
return praw.Reddit(
client_id=client_id, client_secret=client_secret, user_agent=user_agent
)
def get_financial_data_via_fetcher(ticker_symbol):
"""
@@ -48,38 +53,45 @@ def get_financial_data_via_fetcher(ticker_symbol):
# --- Call 1: Get Market Cap ---
try:
mc_script_path = project_root / 'fetch_market_cap.py'
mc_script_path = project_root / "fetch_market_cap.py"
command_mc = [sys.executable, str(mc_script_path), ticker_symbol]
result_mc = subprocess.run(command_mc, capture_output=True, text=True, check=True, timeout=30)
result_mc = subprocess.run(
command_mc, capture_output=True, text=True, check=True, timeout=30
)
financials.update(json.loads(result_mc.stdout))
except Exception as e:
log.warning(f"Market cap fetcher failed for {ticker_symbol}: {e}")
# --- Call 2: Get Closing Price ---
try:
cp_script_path = project_root / 'fetch_close_price.py'
cp_script_path = project_root / "fetch_close_price.py"
command_cp = [sys.executable, str(cp_script_path), ticker_symbol]
result_cp = subprocess.run(command_cp, capture_output=True, text=True, check=True, timeout=30)
result_cp = subprocess.run(
command_cp, capture_output=True, text=True, check=True, timeout=30
)
financials.update(json.loads(result_cp.stdout))
except Exception as e:
log.warning(f"Closing price fetcher failed for {ticker_symbol}: {e}")
return financials
# --- HELPER FUNCTION: Contains all the optimized logic for one post ---
def _process_submission(submission, subreddit_id, conn, comment_limit, fetch_financials):
def _process_submission(
submission, subreddit_id, conn, comment_limit, fetch_financials
):
"""
Processes a single Reddit submission with optimized logic.
- Uses a single loop over comments.
- Caches ticker IDs to reduce DB lookups.
"""
current_time = time.time()
# 1. Initialize data collectors for this post
tickers_in_title = set(extract_tickers(submission.title))
all_tickers_found_in_post = set(tickers_in_title)
all_comment_sentiments = []
ticker_id_cache = {} # In-memory cache for ticker IDs for this post
ticker_id_cache = {} # In-memory cache for ticker IDs for this post
submission.comments.replace_more(limit=0)
all_comments = submission.comments.list()[:comment_limit]
@@ -88,8 +100,8 @@ def _process_submission(submission, subreddit_id, conn, comment_limit, fetch_fin
# We gather all necessary information in one pass.
for comment in all_comments:
comment_sentiment = get_sentiment_score(comment.body)
all_comment_sentiments.append(comment_sentiment) # For the deep dive
all_comment_sentiments.append(comment_sentiment) # For the deep dive
tickers_in_comment = set(extract_tickers(comment.body))
if not tickers_in_comment:
continue
@@ -101,147 +113,266 @@ def _process_submission(submission, subreddit_id, conn, comment_limit, fetch_fin
# If the title has tickers, every comment is a mention for them
for ticker_symbol in tickers_in_title:
if ticker_symbol not in ticker_id_cache:
ticker_id_cache[ticker_symbol] = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol)
ticker_id_cache[ticker_symbol] = database.get_or_create_entity(
conn, "tickers", "symbol", ticker_symbol
)
ticker_id = ticker_id_cache[ticker_symbol]
database.add_mention(conn, ticker_id, subreddit_id, submission.id, 'comment', int(comment.created_utc), comment_sentiment)
database.add_mention(
conn,
ticker_id,
subreddit_id,
submission.id,
"comment",
int(comment.created_utc),
comment_sentiment,
)
else:
# If no title tickers, only direct mentions in comments count
for ticker_symbol in tickers_in_comment:
if ticker_symbol not in ticker_id_cache:
ticker_id_cache[ticker_symbol] = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol)
ticker_id_cache[ticker_symbol] = database.get_or_create_entity(
conn, "tickers", "symbol", ticker_symbol
)
ticker_id = ticker_id_cache[ticker_symbol]
database.add_mention(conn, ticker_id, subreddit_id, submission.id, 'comment', int(comment.created_utc), comment_sentiment)
database.add_mention(
conn,
ticker_id,
subreddit_id,
submission.id,
"comment",
int(comment.created_utc),
comment_sentiment,
)
# 3. Process title mentions (if any)
if tickers_in_title:
log.info(f" -> Title Mention(s): {', '.join(tickers_in_title)}. Attributing all comments.")
log.info(
f" -> Title Mention(s): {', '.join(tickers_in_title)}. Attributing all comments."
)
post_sentiment = get_sentiment_score(submission.title)
for ticker_symbol in tickers_in_title:
if ticker_symbol not in ticker_id_cache:
ticker_id_cache[ticker_symbol] = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol)
ticker_id_cache[ticker_symbol] = database.get_or_create_entity(
conn, "tickers", "symbol", ticker_symbol
)
ticker_id = ticker_id_cache[ticker_symbol]
database.add_mention(conn, ticker_id, subreddit_id, submission.id, 'post', int(submission.created_utc), post_sentiment)
database.add_mention(
conn,
ticker_id,
subreddit_id,
submission.id,
"post",
int(submission.created_utc),
post_sentiment,
)
# 4. Fetch financial data if enabled
if fetch_financials:
for ticker_symbol in all_tickers_found_in_post:
ticker_id = ticker_id_cache[ticker_symbol] # Guaranteed to be in cache
ticker_id = ticker_id_cache[ticker_symbol] # Guaranteed to be in cache
ticker_info = database.get_ticker_info(conn, ticker_id)
if not ticker_info['last_updated'] or (current_time - ticker_info['last_updated'] > database.MARKET_CAP_REFRESH_INTERVAL):
if not ticker_info["last_updated"] or (
current_time - ticker_info["last_updated"]
> database.MARKET_CAP_REFRESH_INTERVAL
):
log.info(f" -> Fetching financial data for {ticker_symbol}...")
financials = get_financial_data_via_fetcher(ticker_symbol)
database.update_ticker_financials(conn, ticker_id, financials.get('market_cap'), financials.get('closing_price'))
database.update_ticker_financials(
conn,
ticker_id,
financials.get("market_cap"),
financials.get("closing_price"),
)
# 5. Save deep dive analysis
avg_sentiment = sum(all_comment_sentiments) / len(all_comment_sentiments) if all_comment_sentiments else 0
avg_sentiment = (
sum(all_comment_sentiments) / len(all_comment_sentiments)
if all_comment_sentiments
else 0
)
post_analysis_data = {
"post_id": submission.id, "title": submission.title,
"post_url": f"https://reddit.com{submission.permalink}", "subreddit_id": subreddit_id,
"post_timestamp": int(submission.created_utc), "comment_count": len(all_comments),
"avg_comment_sentiment": avg_sentiment
"post_id": submission.id,
"title": submission.title,
"post_url": f"https://reddit.com{submission.permalink}",
"subreddit_id": subreddit_id,
"post_timestamp": int(submission.created_utc),
"comment_count": len(all_comments),
"avg_comment_sentiment": avg_sentiment,
}
database.add_or_update_post_analysis(conn, post_analysis_data)
def scan_subreddits(reddit, subreddits_list, post_limit=100, comment_limit=100, days_to_scan=1, fetch_financials=True):
def scan_subreddits(
reddit,
subreddits_list,
post_limit=100,
comment_limit=100,
days_to_scan=1,
fetch_financials=True,
):
conn = database.get_db_connection()
post_age_limit = days_to_scan * 86400
current_time = time.time()
log.info(f"Scanning {len(subreddits_list)} subreddit(s) for NEW posts in the last {days_to_scan} day(s)...")
log.info(
f"Scanning {len(subreddits_list)} subreddit(s) for NEW posts in the last {days_to_scan} day(s)..."
)
if not fetch_financials:
log.warning("NOTE: Financial data fetching is disabled for this run.")
for subreddit_name in subreddits_list:
try:
normalized_sub_name = subreddit_name.lower()
subreddit_id = database.get_or_create_entity(conn, 'subreddits', 'name', normalized_sub_name)
subreddit_id = database.get_or_create_entity(
conn, "subreddits", "name", normalized_sub_name
)
subreddit = reddit.subreddit(normalized_sub_name)
log.info(f"Scanning r/{normalized_sub_name}...")
for submission in subreddit.new(limit=post_limit):
if (current_time - submission.created_utc) > post_age_limit:
log.info(f" -> Reached posts older than the {days_to_scan}-day limit.")
log.info(
f" -> Reached posts older than the {days_to_scan}-day limit."
)
break
# Call the new helper function for each post
_process_submission(submission, subreddit_id, conn, comment_limit, fetch_financials)
_process_submission(
submission, subreddit_id, conn, comment_limit, fetch_financials
)
except Exception as e:
log.error(f"Could not scan r/{normalized_sub_name}. Error: {e}", exc_info=True)
log.error(
f"Could not scan r/{normalized_sub_name}. Error: {e}", exc_info=True
)
conn.close()
log.critical("\n--- Scan Complete ---")
def main():
"""Main function to run the Reddit stock analysis tool."""
parser = argparse.ArgumentParser(description="Analyze stock ticker mentions on Reddit.", formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument("-f", "--config", default="subreddits.json", help="Path to the JSON file for scanning. (Default: subreddits.json)")
parser.add_argument("-s", "--subreddit", help="Scan a single subreddit, ignoring the config file.")
parser.add_argument("-d", "--days", type=int, default=1, help="Number of past days to scan for new posts. (Default: 1)")
parser.add_argument("-p", "--posts", type=int, default=200, help="Max posts to check per subreddit. (Default: 200)")
parser.add_argument("-c", "--comments", type=int, default=100, help="Number of comments to scan per post. (Default: 100)")
parser.add_argument("-n", "--no-financials", action="store_true", help="Disable fetching of financial data during the Reddit scan.")
parser.add_argument("--update-top-tickers", action="store_true", help="Update financial data only for tickers currently in the Top 10 daily/weekly dashboards.")
parser.add_argument(
"-u", "--update-financials-only",
nargs='?',
const="ALL_TICKERS", # A special value to signify "update all"
default=None,
metavar='TICKER',
help="Update financials. Provide a ticker symbol to update just one,\nor use the flag alone to update all tickers in the database."
parser = argparse.ArgumentParser(
description="Analyze stock ticker mentions on Reddit.",
formatter_class=argparse.RawTextHelpFormatter,
)
parser.add_argument("--debug", action="store_true", help="Enable detailed debug logging to the console.")
parser.add_argument("--stdout", action="store_true", help="Print all log messages to the console.")
parser.add_argument(
"-f",
"--config",
default="subreddits.json",
help="Path to the JSON file for scanning. (Default: subreddits.json)",
)
parser.add_argument(
"-s", "--subreddit", help="Scan a single subreddit, ignoring the config file."
)
parser.add_argument(
"-d",
"--days",
type=int,
default=1,
help="Number of past days to scan for new posts. (Default: 1)",
)
parser.add_argument(
"-p",
"--posts",
type=int,
default=200,
help="Max posts to check per subreddit. (Default: 200)",
)
parser.add_argument(
"-c",
"--comments",
type=int,
default=100,
help="Number of comments to scan per post. (Default: 100)",
)
parser.add_argument(
"-n",
"--no-financials",
action="store_true",
help="Disable fetching of financial data during the Reddit scan.",
)
parser.add_argument(
"--update-top-tickers",
action="store_true",
help="Update financial data only for tickers currently in the Top 10 daily/weekly dashboards.",
)
parser.add_argument(
"-u",
"--update-financials-only",
nargs="?",
const="ALL_TICKERS", # A special value to signify "update all"
default=None,
metavar="TICKER",
help="Update financials. Provide a ticker symbol to update just one,\nor use the flag alone to update all tickers in the database.",
)
parser.add_argument(
"--debug",
action="store_true",
help="Enable detailed debug logging to the console.",
)
parser.add_argument(
"--stdout", action="store_true", help="Print all log messages to the console."
)
args = parser.parse_args()
setup_logging(console_verbose=args.stdout, debug_mode=args.debug)
database.initialize_db()
if args.update_top_tickers:
log.critical("--- Starting Financial Data Update for Top Tickers ---")
# 1. Start with an empty set to hold all unique tickers
tickers_to_update = set()
# 2. Get the overall top tickers
log.info("-> Checking overall top daily and weekly tickers...")
top_daily_overall = database.get_top_daily_ticker_symbols()
top_weekly_overall = database.get_top_weekly_ticker_symbols()
tickers_to_update.update(top_daily_overall)
tickers_to_update.update(top_weekly_overall)
# 3. Get all subreddits and loop through them
all_subreddits = database.get_all_scanned_subreddits()
log.info(f"-> Checking top tickers for {len(all_subreddits)} individual subreddit(s)...")
log.info(
f"-> Checking top tickers for {len(all_subreddits)} individual subreddit(s)..."
)
for sub_name in all_subreddits:
log.debug(f" -> Checking r/{sub_name}...")
top_daily_sub = database.get_top_daily_ticker_symbols_for_subreddit(sub_name)
top_weekly_sub = database.get_top_weekly_ticker_symbols_for_subreddit(sub_name)
top_daily_sub = database.get_top_daily_ticker_symbols_for_subreddit(
sub_name
)
top_weekly_sub = database.get_top_weekly_ticker_symbols_for_subreddit(
sub_name
)
tickers_to_update.update(top_daily_sub)
tickers_to_update.update(top_weekly_sub)
unique_top_tickers = sorted(list(tickers_to_update))
if not unique_top_tickers:
log.info("No top tickers found in the last week. Nothing to update.")
else:
log.info(f"Found {len(unique_top_tickers)} unique top tickers to update: {', '.join(unique_top_tickers)}")
log.info(
f"Found {len(unique_top_tickers)} unique top tickers to update: {', '.join(unique_top_tickers)}"
)
conn = database.get_db_connection()
for ticker_symbol in unique_top_tickers:
# 4. Find the ticker's ID to perform the update
ticker_info = database.get_ticker_by_symbol(ticker_symbol)
if ticker_info:
log.info(f" -> Updating financials for {ticker_info['symbol']}...")
financials = get_financial_data_via_fetcher(ticker_info['symbol'])
financials = get_financial_data_via_fetcher(ticker_info["symbol"])
database.update_ticker_financials(
conn, ticker_info['id'],
financials.get('market_cap'),
financials.get('closing_price')
conn,
ticker_info["id"],
financials.get("market_cap"),
financials.get("closing_price"),
)
conn.close()
log.critical("--- Top Ticker Financial Data Update Complete ---")
elif args.update_financials_only:
@@ -253,31 +384,37 @@ def main():
log.info(f"Found {len(all_tickers)} tickers in the database to update.")
conn = database.get_db_connection()
for ticker in all_tickers:
symbol = ticker['symbol']
symbol = ticker["symbol"]
log.info(f" -> Updating financials for {symbol}...")
financials = get_financial_data_via_fetcher(symbol)
database.update_ticker_financials(
conn, ticker['id'],
financials.get('market_cap'),
financials.get('closing_price')
conn,
ticker["id"],
financials.get("market_cap"),
financials.get("closing_price"),
)
conn.close()
else:
ticker_symbol_to_update = update_mode
log.critical(f"--- Starting Financial Data Update for single ticker: {ticker_symbol_to_update} ---")
log.critical(
f"--- Starting Financial Data Update for single ticker: {ticker_symbol_to_update} ---"
)
ticker_info = database.get_ticker_by_symbol(ticker_symbol_to_update)
if ticker_info:
conn = database.get_db_connection()
log.info(f" -> Updating financials for {ticker_info['symbol']}...")
financials = get_financial_data_via_fetcher(ticker_info['symbol'])
financials = get_financial_data_via_fetcher(ticker_info["symbol"])
database.update_ticker_financials(
conn, ticker_info['id'],
financials.get('market_cap'),
financials.get('closing_price')
conn,
ticker_info["id"],
financials.get("market_cap"),
financials.get("closing_price"),
)
conn.close()
else:
log.error(f"Ticker '{ticker_symbol_to_update}' not found in the database.")
log.error(
f"Ticker '{ticker_symbol_to_update}' not found in the database."
)
log.critical("--- Financial Data Update Complete ---")
else:
@@ -288,14 +425,15 @@ def main():
log.info(f"Targeted Scan Mode: Focusing on r/{args.subreddit}")
else:
log.info(f"Config Scan Mode: Loading subreddits from {args.config}")
subreddits_to_scan = load_subreddits(args.config)
subreddits_to_scan = load_subreddits(args.config)
if not subreddits_to_scan:
log.error("Error: No subreddits to scan.")
return
reddit = get_reddit_instance()
if not reddit: return
if not reddit:
return
scan_subreddits(
reddit,
@@ -303,8 +441,9 @@ def main():
post_limit=args.posts,
comment_limit=args.comments,
days_to_scan=args.days,
fetch_financials=(not args.no_financials)
fetch_financials=(not args.no_financials),
)
if __name__ == "__main__":
main()
main()