diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0a72cce --- /dev/null +++ b/requirements.txt @@ -0,0 +1,29 @@ +beautifulsoup4==4.9.3 +bs4==0.0.1 +certifi==2020.12.5 +chardet==4.0.0 +elasticsearch==7.10.1 +html5lib==1.1 +idna==2.10 +lxml==4.6.2 +marshmallow==3.10.0 +marshmallow-sqlalchemy==0.24.1 +multitasking==0.0.9 +numpy==1.19.5 +pandas==1.2.1 +praw==7.1.0 +prawcore==1.5.0 +pygal==2.4.0 +python-dateutil==2.8.1 +pytz==2020.5 +requests==2.25.1 +simplejson==3.17.2 +six==1.15.0 +soupsieve==2.1 +SQLAlchemy==1.3.22 +update-checker==0.18.0 +urllib3==1.26.2 +webencodings==0.5.1 +websocket-client==0.57.0 +wsbfin @ file:///home/wes/wsbfin +yfinance==0.1.55 diff --git a/setup.py b/setup.py index ec74b4d..9b22c72 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,9 @@ setuptools.setup( requires_python=">=3.7", install_requires = [ "yfinance", - "praw" + "praw", + "sqlalchemy", + "pygal" ], classifiers= [ "Programming Language :: Python :: 3", diff --git a/README.md b/wsbfin/__init__.py similarity index 100% rename from README.md rename to wsbfin/__init__.py diff --git a/wsbfin/ingest/__init__.py b/wsbfin/ingest/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/wsbfin/ingest/model.py b/wsbfin/ingest/model.py new file mode 100644 index 0000000..0fe0151 --- /dev/null +++ b/wsbfin/ingest/model.py @@ -0,0 +1,11 @@ +from marshmallow import Schema, fields +from datetime import date + +class Symbol: + symbol = fields.Str() + name = fields.Str() + count = fields.Integer() + day = fields.Date() + +class DD: + pass diff --git a/wsbfin/wsbfin.py b/wsbfin/wsbfin.py index f5e1865..9528eb4 100644 --- a/wsbfin/wsbfin.py +++ b/wsbfin/wsbfin.py @@ -1,9 +1,19 @@ +import datetime import praw import yfinance as yf -from os import environ +import wsbfin.ingest.model + from collections import defaultdict +from os import environ from queue import Queue as queue from re import search +from sqlalchemy import Column, Text, Integer, Date +from sqlalchemy import create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker + +engine = create_engine("sqlite:///stocks.db", echo=True) +session = sessionmaker(bind=engine)() reddit = praw.Reddit(client_id=environ.get("CLIENT_ID"), client_secret=environ.get("SECRET"), @@ -11,23 +21,39 @@ reddit = praw.Reddit(client_id=environ.get("CLIENT_ID"), user_agent="testscript by u/weskerfoot", username="weskerfoot") +# finra bonds +# seekingalpha +# sec filings +# open insider +# moodys +# twitter +# whale wisdom +# stockcharts + +# tag with industries! + + +ignored = {"DD", "USA", "USA", "WSB", "FD", "TV", "GOAT", "YOLO"} + +Base = declarative_base() + +class Stock(Base): + __tablename__ = "stocks" -ignored = {"DD", "USA", "USA", "WBS", "FD"} -symbols = defaultdict(int) + id = Column(Integer, primary_key=True) + symbol = Column(Text) + name = Column(Text) + count = Column(Integer) + day = Column(Date) -def normalize_symbol(text): + def __repr__(self): + return "%s, %s, %s" % (self.name, self.symbol, self.day) + +def get_stock_info(text): """ - Try to extract a stock symbol from a word, and return it. + Try to extract a stock symbol from a word, and return info about it. """ result = search(r"\$?[A-Z]{2,5}", text) or search(r"\$[A-Z]{2,5}", text) - if result and result.group(0).upper() in symbols: - sym = result.group(0).upper() - - if sym.startswith("$"): - sym = sym[1:] - - symbols[sym] += 1 - return if result: sym = result.group(0).upper() @@ -38,10 +64,9 @@ def normalize_symbol(text): sym = sym[1:] try: - yf.Ticker(sym).info # it's a real symbol - symbols[sym] += 1 - except: - pass + return yf.Ticker(sym).info + except KeyError: # XXX what exceptions can yf throw? + return None return None # use bloom filter to skip seen submissions/comments/etc @@ -57,7 +82,23 @@ def submissions(sr): continue for reply in comment.replies: if hasattr(reply, "body"): - yield normalize_symbol(reply.body) + info = get_stock_info(reply.body) + if not info: + continue + print(info.get("symbol")) + stock = session.query(Stock).filter_by(symbol=info.get("symbol"), day=datetime.datetime.now()).one_or_none() + + if stock: + stock.count = stock.count + 1 + else: + new_stock = Stock(symbol=info.get("symbol"), + name=info.get("longName"), + count=1, + day=datetime.datetime.now()) + session.add(new_stock) + session.commit() -for comment in submissions("wallstreetbets"): - print(symbols) +if __name__ == "__main__": + #Stock.__table__.create(engine) + for comment in submissions("wallstreetbets"): + print(comment)