Process comments.
This commit is contained in:
75
main.py
75
main.py
@@ -11,29 +11,28 @@ from dotenv import load_dotenv
|
||||
|
||||
import database
|
||||
from ticker_extractor import extract_tickers
|
||||
from sentiment_analyzer import get_sentiment_score # <-- IMPORT OUR NEW MODULE
|
||||
from sentiment_analyzer import get_sentiment_score
|
||||
|
||||
load_dotenv()
|
||||
MARKET_CAP_REFRESH_INTERVAL = 86400
|
||||
|
||||
# ... (load_subreddits, get_market_cap, get_reddit_instance functions are unchanged)
|
||||
# (load_subreddits, get_market_cap, get_reddit_instance functions are unchanged)
|
||||
def load_subreddits(filepath):
|
||||
# ...
|
||||
try:
|
||||
with open(filepath, 'r') as f: return json.load(f).get("subreddits", [])
|
||||
with open(filepath, 'r') as f:
|
||||
return json.load(f).get("subreddits", [])
|
||||
except (FileNotFoundError, json.JSONDecodeError) as e:
|
||||
print(f"Error loading config: {e}")
|
||||
return None
|
||||
|
||||
def get_market_cap(ticker_symbol):
|
||||
# ...
|
||||
try:
|
||||
ticker = yf.Ticker(ticker_symbol)
|
||||
return ticker.fast_info.get('marketCap')
|
||||
except Exception: return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def get_reddit_instance():
|
||||
# ...
|
||||
client_id = os.getenv("REDDIT_CLIENT_ID")
|
||||
client_secret = os.getenv("REDDIT_CLIENT_SECRET")
|
||||
user_agent = os.getenv("REDDIT_USER_AGENT")
|
||||
@@ -42,12 +41,12 @@ def get_reddit_instance():
|
||||
return None
|
||||
return praw.Reddit(client_id=client_id, client_secret=client_secret, user_agent=user_agent)
|
||||
|
||||
|
||||
def scan_subreddits(reddit, subreddits_list, post_limit=25):
|
||||
"""Scans subreddits, performs sentiment analysis, and stores results in the database."""
|
||||
# --- UPDATED: Function now accepts post_limit and comment_limit ---
|
||||
def scan_subreddits(reddit, subreddits_list, post_limit=25, comment_limit=100):
|
||||
"""Scans subreddits, analyzes posts and comments, and stores results in the database."""
|
||||
conn = database.get_db_connection()
|
||||
|
||||
print(f"\nScanning {len(subreddits_list)} subreddits for top {post_limit} posts...")
|
||||
print(f"\nScanning {len(subreddits_list)} subreddits (Top {post_limit} posts, {comment_limit} comments/post)...")
|
||||
for subreddit_name in subreddits_list:
|
||||
try:
|
||||
subreddit_id = database.get_or_create_entity(conn, 'subreddits', 'name', subreddit_name)
|
||||
@@ -55,28 +54,15 @@ def scan_subreddits(reddit, subreddits_list, post_limit=25):
|
||||
print(f"Scanning r/{subreddit_name}...")
|
||||
|
||||
for submission in subreddit.hot(limit=post_limit):
|
||||
# We analyze the title for sentiment as it's often the most concise summary.
|
||||
# Analyzing all comments could be a future enhancement.
|
||||
text_to_analyze = submission.title
|
||||
tickers_in_post = extract_tickers(text_to_analyze + " " + submission.selftext)
|
||||
|
||||
# --- NEW: Get sentiment score for the post's title ---
|
||||
sentiment = get_sentiment_score(text_to_analyze)
|
||||
# --- 1. Process the Post Title and Body ---
|
||||
post_text = submission.title + " " + submission.selftext
|
||||
tickers_in_post = extract_tickers(post_text)
|
||||
post_sentiment = get_sentiment_score(submission.title)
|
||||
|
||||
for ticker_symbol in set(tickers_in_post):
|
||||
ticker_id = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol)
|
||||
|
||||
# --- NEW: Pass the sentiment score to the database ---
|
||||
database.add_mention(
|
||||
conn,
|
||||
ticker_id=ticker_id,
|
||||
subreddit_id=subreddit_id,
|
||||
post_id=submission.id,
|
||||
timestamp=int(submission.created_utc),
|
||||
sentiment=sentiment # Pass the score here
|
||||
)
|
||||
|
||||
# (The market cap update logic remains the same)
|
||||
database.add_mention(conn, ticker_id, subreddit_id, submission.id, int(submission.created_utc), post_sentiment)
|
||||
# (Market cap logic remains the same)
|
||||
ticker_info = database.get_ticker_info(conn, ticker_id)
|
||||
current_time = int(time.time())
|
||||
if not ticker_info['last_updated'] or (current_time - ticker_info['last_updated'] > MARKET_CAP_REFRESH_INTERVAL):
|
||||
@@ -84,6 +70,28 @@ def scan_subreddits(reddit, subreddits_list, post_limit=25):
|
||||
market_cap = get_market_cap(ticker_symbol)
|
||||
database.update_ticker_market_cap(conn, ticker_id, market_cap or ticker_info['market_cap'])
|
||||
|
||||
# --- 2. Process the Comments ---
|
||||
# Expand "MoreComments" objects. limit=None means we try to get all, but PRAW is protective.
|
||||
# A limit of 32 is the max PRAW will do in a single call. We'll iterate to be safe.
|
||||
submission.comments.replace_more(limit=10)
|
||||
comment_count = 0
|
||||
for comment in submission.comments.list():
|
||||
if comment_count >= comment_limit:
|
||||
break # Stop processing comments for this post if we hit our limit
|
||||
|
||||
tickers_in_comment = extract_tickers(comment.body)
|
||||
if not tickers_in_comment:
|
||||
continue # Skip comments that don't mention any tickers
|
||||
|
||||
comment_sentiment = get_sentiment_score(comment.body)
|
||||
|
||||
for ticker_symbol in set(tickers_in_comment):
|
||||
ticker_id = database.get_or_create_entity(conn, 'tickers', 'symbol', ticker_symbol)
|
||||
# We use the submission.id as the post_id to group mentions correctly
|
||||
database.add_mention(conn, ticker_id, subreddit_id, submission.id, int(comment.created_utc), comment_sentiment)
|
||||
|
||||
comment_count += 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"Could not scan r/{subreddit_name}. Error: {e}")
|
||||
|
||||
@@ -91,10 +99,6 @@ def scan_subreddits(reddit, subreddits_list, post_limit=25):
|
||||
print("\n--- Scan Complete ---")
|
||||
|
||||
def main():
|
||||
# --- IMPORTANT: Delete your old DB file before running! ---
|
||||
# Since we changed the schema and logic, old data won't have sentiment.
|
||||
# It's best to start fresh. Delete the `reddit_stocks.db` file now.
|
||||
|
||||
parser = argparse.ArgumentParser(description="Analyze stock ticker mentions on Reddit.")
|
||||
parser.add_argument("config_file", help="Path to the JSON file containing subreddits.")
|
||||
args = parser.parse_args()
|
||||
@@ -105,7 +109,8 @@ def main():
|
||||
reddit = get_reddit_instance()
|
||||
if not reddit: return
|
||||
|
||||
scan_subreddits(reddit, subreddits)
|
||||
# We now pass the limits to the scan function
|
||||
scan_subreddits(reddit, subreddits, post_limit=25, comment_limit=100)
|
||||
database.generate_summary_report()
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
{
|
||||
"subreddits": [
|
||||
"wallstreetbets",
|
||||
"stocks",
|
||||
"investing",
|
||||
"pennystocks",
|
||||
"Shortsqueeze"
|
||||
"Shortsqueeze",
|
||||
"smallstreetbets"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -5,47 +5,44 @@ import re
|
||||
# A set of common English words and acronyms that look like stock tickers.
|
||||
# This helps reduce false positives.
|
||||
COMMON_WORDS_BLACKLIST = {
|
||||
"A", "I", "DD", "CEO", "CFO", "CTO", "EPS", "IPO", "YOLO", "FOMO",
|
||||
"TLDR", "EDIT", "THE", "AND", "FOR", "ARE", "BUT", "NOT", "YOU",
|
||||
"ALL", "ANY", "CAN", "HAS", "NEW", "NOW", "OLD", "SEE", "TWO",
|
||||
"WAY", "WHO", "WHY", "BIG", "BUY", "SELL", "HOLD", "BE", "GO",
|
||||
"ON", "AT", "IN", "IS", "IT", "OF", "OR", "TO", "WE", "UP",
|
||||
"OUT", "SO", "RH", "SEC", "IRS", "USA", "UK", "EU",
|
||||
"AI", "ML", "AR", "VR", "NFT", "DAO", "WEB3", "ETH", "BTC", "DOGE",
|
||||
"USD", "EUR", "GBP", "JPY", "CNY", "INR", "AUD", "CAD", "CHF",
|
||||
"RUB", "ZAR", "BRL", "MXN", "HKD", "SGD", "NZD", "RSD",
|
||||
"JPY", "KRW", "SEK", "NOK", "DKK", "PLN", "CZK", "HUF", "TRY",
|
||||
"US", "IRA", "FDA", "SEC", "FBI", "CIA", "NSA", "NATO", "FINRA",
|
||||
"NASDAQ", "NYSE", "AMEX", "FTSE", "DAX", "WSB", "SPX", "DJIA",
|
||||
"EDGAR", "GDP", "CPI", "PPI", "PMI", "ISM", "FOMC", "ECB", "BOE",
|
||||
"BOJ", "RBA", "RBNZ", "BIS", "NFA", "P", "VOO", "CTB", "DR",
|
||||
"ETF", "EV", "ESG", "REIT", "SPAC", "IPO", "M&A", "LBO", "PE",
|
||||
"Q1", "Q2", "Q3", "Q4", "FY", "FAQ", "ROI", "ROE", "EPS", "P/E", "PEG",
|
||||
"FRG", "FXAIX", "FXIAX", "FZROX", "BULL", "BEAR", "BULLISH", "BEARISH",
|
||||
"QQQ", "SPY", "DIA", "IWM", "VTI", "VOO", "IVV", "SCHB", "SPLG",
|
||||
"ROTH", "IRA", "401K", "403B", "457B", "SEP", "SIMPLE", "HSA",
|
||||
"LONG", "SHORT", "LEVERAGE", "MARGIN", "HEDGE", "SWING", "DAY",
|
||||
"GRAB", "GPU", "MY", "PSA", "AMA", "DM", "OP", "SPAC", "FIHTX",
|
||||
"FINTX", "FINT", "FINTX", "FINTY", "FSPSX", "TOTAL", "LARGE", "MID", "SMALL",
|
||||
"GROWTH", "VALUE", "BLEND", "INCOME", "DIV", "YIELD", "BETA", "ALPHA", "VOLATILITY",
|
||||
"RISK", "RETURN", "SHARPE", "SORTINO", "MAX", "MIN", "STDDEV", "VARIANCE",
|
||||
"PDF", "FULL", "PEAK", "LATE", "EARLY", "MIDDAY", "NIGHT", "MORNING", "AFTERNOON",
|
||||
"CYCLE", "TREND", "PATTERN", "BREAKOUT", "PULLBACK", "REVERSAL", "CONSOLIDATION",
|
||||
"OTC", "TRUE", "FALSE", "NULL", "NONE", "ALL", "ANY", "SOME", "EACH", "EVERY",
|
||||
"STILL", "TERM", "TIME", "DATE", "YEAR", "MONTH", "WEEK", "HOUR", "MINUTE", "SECOND",
|
||||
"JUST", "ALREADY", "STILL", "YET", "NOW", "LATER", "SOON", "EARLIER", "TODAY", "TOMORROW",
|
||||
"YESTERDAY", "TONIGHT", "THIS", "LAST", "NEXT", "WOULD", "SHOULD", "COULD", "MIGHT",
|
||||
"WILL", "CAN", "MUST", "SHALL", "OUGHT", "TAKE", "MAKE", "HAVE", "GET", "DO", "BE",
|
||||
"GO", "COME", "SEE", "LOOK", "WATCH", "HEAR", "YES", "NO", "OK", "LIKE", "LOVE", "HATE",
|
||||
"WANT", "NEED", "THINK", "BELIEVE", "KNOW", "PRICE", "COST", "VALUE", "WORTH",
|
||||
"EXPENSE", "SPEND", "SAVE", "EARN", "PROFIT", "LOSS", "GAIN", "DEBT", "CREDIT",
|
||||
"BOND", "STOCK", "SHARE", "FUND", "ASSET", "LIABILITY", "BUZZ", "UNDER", "OVER", "BETWEEN",
|
||||
"FRAUD", "SCAM", "RISK", "REWARD", "RETURN", "INVEST", "TRADE", "BUY", "SELL", "HOLD",
|
||||
"SHORT", "LONG", "LEVERAGE", "MARGIN", "HEDGE", "SCALP", "POSITION",
|
||||
"PLAN", "GOAL", "WILL", "FAST", "HINT", "ABOVE", "BELOW", "AROUND", "NEAR", "FAR",
|
||||
"TL",
|
||||
"401K", "403B", "457B", "ABOVE", "AI", "ALL", "ALPHA", "AMA", "AMEX",
|
||||
"AND", "ANY", "AR", "ARE", "AROUND", "ASSET", "AT", "ATH", "ATL", "AUD",
|
||||
"BE", "BEAR", "BELOW", "BETA", "BIG", "BIS", "BLEND", "BOE", "BOJ",
|
||||
"BOND", "BRB", "BRL", "BTC", "BTW", "BULL", "BUT", "BUY", "BUZZ", "CAD",
|
||||
"CAN", "CEO", "CFO", "CHF", "CIA", "CNY", "COME", "COST", "COULD", "CPI",
|
||||
"CTB", "CTO", "CYCLE", "CZK", "DAO", "DATE", "DAX", "DAY", "DCA", "DD",
|
||||
"DEBT", "DIA", "DIV", "DJIA", "DKK", "DM", "DO", "DOGE", "DR", "EACH",
|
||||
"EARLY", "EARN", "ECB", "EDGAR", "EDIT", "EPS", "ESG", "ETF", "ETH",
|
||||
"EU", "EUR", "EV", "EVERY", "FAQ", "FAR", "FAST", "FBI", "FDA", "FIHTX",
|
||||
"FINRA", "FINT", "FINTX", "FINTY", "FOMC", "FOMO", "FOR", "FRAUD",
|
||||
"FRG", "FSPSX", "FTSE", "FUD", "FULL", "FUND", "FXAIX", "FXIAX", "FY",
|
||||
"FYI", "FZROX", "GAIN", "GDP", "GET", "GBP", "GO", "GOAL", "GPU", "GRAB",
|
||||
"GTG", "HAS", "HAVE", "HATE", "HEAR", "HEDGE", "HINT", "HKD", "HODL",
|
||||
"HOLD", "HOUR", "HSA", "HUF", "IMHO", "IMO", "IN", "INR", "IPO", "IRA",
|
||||
"IRS", "IS", "ISM", "IT", "IV", "IVV", "IWM", "JPY", "JUST", "KNOW",
|
||||
"KRW", "LARGE", "LAST", "LATE", "LATER", "LBO", "LIKE", "LMAO", "LOL",
|
||||
"LONG", "LOOK", "LOSS", "LOVE", "M&A", "MAKE", "MAX", "MC", "MID", "MIGHT",
|
||||
"MIN", "ML", "MOASS", "MONTH", "MUST", "MXN", "MY", "NATO", "NEAR",
|
||||
"NEED", "NEW", "NEXT", "NFA", "NFT", "NGMI", "NIGHT", "NO", "NOK", "NONE",
|
||||
"NOT", "NOW", "NSA", "NULL", "NZD", "NYSE", "OF", "OK", "OLD", "ON",
|
||||
"OP", "OR", "OTC", "OUGHT", "OUT", "OVER", "PE", "PEAK", "PEG",
|
||||
"PLAN", "PLN", "PMI", "PPI", "PRICE", "PROFIT", "PSA", "Q1", "Q2", "Q3",
|
||||
"Q4", "QQQ", "RBA", "RBNZ", "REIT", "REKT", "RH", "RISK", "ROE", "ROFL",
|
||||
"ROI", "ROTH", "RSD", "RUB", "SAVE", "SCALP", "SCAM", "SCHB", "SEC",
|
||||
"SEE", "SEK", "SELL", "SEP", "SGD", "SHALL", "SHARE", "SHORT", "SO",
|
||||
"SOME", "SOON", "SPAC", "SPEND", "SPLG", "SPX", "SPY", "STILL", "STOCK",
|
||||
"SWING", "TAKE", "TERM", "THE", "THINK", "THIS", "TIME", "TL", "TL;DR",
|
||||
"TLDR", "TODAY", "TO", "TOTAL", "TRADE", "TREND", "TRUE", "TRY", "TTYL",
|
||||
"TWO", "UK", "UNDER", "UP", "US", "USA", "USD", "VTI", "VALUE", "VOO",
|
||||
"VR", "WAGMI", "WANT", "WATCH", "WAY", "WE", "WEB3", "WEEK", "WHO",
|
||||
"WHY", "WILL", "WORTH", "WOULD", "WSB", "YET", "YIELD", "YOLO", "YOU",
|
||||
"ZAR",
|
||||
"KARMA", "OTM", "ITM", "ATM", "JPOW", "OPEN", "CLOSE", "HIGH", "LOW",
|
||||
"RE", "BS", "ASAP", "RULE", "REAL", "LIMIT", "STOP", "END", "START", "BOTS",
|
||||
"UTC", "AH", "PM", "PR", "GMT", "EST", "CST", "PST", "BST", "AEDT", "AEST",
|
||||
"CET", "CEST", "EDT", "IST", "JST", "MSK", "PDT", "PST", "YES", "NO", "OWN",
|
||||
"BOMB",
|
||||
}
|
||||
|
||||
def extract_tickers(text):
|
||||
"""
|
||||
Extracts potential stock tickers from a given piece of text.
|
||||
@@ -54,7 +51,7 @@ def extract_tickers(text):
|
||||
# Regex to find potential tickers:
|
||||
# 1. Words prefixed with $: $AAPL, $TSLA
|
||||
# 2. All-caps words between 1 and 5 characters: GME, AMC
|
||||
ticker_regex = r"\$[A-Z]{1,5}\b|\b[A-Z]{1,5}\b"
|
||||
ticker_regex = r"\$[A-Z]{1,5}\b|\b[A-Z]{2,5}\b"
|
||||
|
||||
potential_tickers = re.findall(ticker_regex, text)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user