update with changes made

requirements.txt

@@ -0,0 +1,29 @@
+beautifulsoup4==4.9.3
+bs4==0.0.1
+certifi==2020.12.5
+chardet==4.0.0
+elasticsearch==7.10.1
+html5lib==1.1
+idna==2.10
+lxml==4.6.2
+marshmallow==3.10.0
+marshmallow-sqlalchemy==0.24.1
+multitasking==0.0.9
+numpy==1.19.5
+pandas==1.2.1
+praw==7.1.0
+prawcore==1.5.0
+pygal==2.4.0
+python-dateutil==2.8.1
+pytz==2020.5
+requests==2.25.1
+simplejson==3.17.2
+six==1.15.0
+soupsieve==2.1
+SQLAlchemy==1.3.22
+update-checker==0.18.0
+urllib3==1.26.2
+webencodings==0.5.1
+websocket-client==0.57.0
+wsbfin @ file:///home/wes/wsbfin
+yfinance==0.1.55

setup.py

@@ -17,7 +17,9 @@ setuptools.setup(
     requires_python=">=3.7",
     install_requires = [
         "yfinance",
-        "praw"
+        "praw",
+        "sqlalchemy",
+        "pygal"
     ],
     classifiers= [
         "Programming Language :: Python :: 3",

wsbfin/ingest/model.py

@@ -0,0 +1,11 @@
+from marshmallow import Schema, fields
+from datetime import date
+
+class Symbol:
+    symbol = fields.Str()
+    name = fields.Str()
+    count = fields.Integer()
+    day = fields.Date()
+
+class DD:
+    pass

wsbfin/wsbfin.py

@@ -1,9 +1,19 @@
+import datetime
 import praw
 import yfinance as yf
-from os import environ
+import wsbfin.ingest.model
+
 from collections import defaultdict
+from os import environ
 from queue import Queue as queue
 from re import search
+from sqlalchemy import Column, Text, Integer, Date
+from sqlalchemy import create_engine
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+
+engine = create_engine("sqlite:///stocks.db", echo=True)
+session = sessionmaker(bind=engine)()
 
 reddit = praw.Reddit(client_id=environ.get("CLIENT_ID"),
                      client_secret=environ.get("SECRET"),
@@ -11,23 +21,39 @@ reddit = praw.Reddit(client_id=environ.get("CLIENT_ID"),
                      user_agent="testscript by u/weskerfoot",
                      username="weskerfoot")
 
+# finra bonds
+# seekingalpha
+# sec filings
+# open insider
+# moodys
+# twitter
+# whale wisdom
+# stockcharts
+
+# tag with industries!
+
+
+ignored = {"DD", "USA", "USA", "WSB", "FD", "TV", "GOAT", "YOLO"}
+
+Base = declarative_base()
+
+class Stock(Base):
+    __tablename__ = "stocks"
 
-ignored = {"DD", "USA", "USA", "WBS", "FD"}
-symbols = defaultdict(int)
+    id = Column(Integer, primary_key=True)
+    symbol = Column(Text)
+    name = Column(Text)
+    count = Column(Integer)
+    day = Column(Date)
 
-def normalize_symbol(text):
+    def __repr__(self):
+        return "%s, %s, %s" % (self.name, self.symbol, self.day)
+
+def get_stock_info(text):
     """
-    Try to extract a stock symbol from a word, and return it.
+    Try to extract a stock symbol from a word, and return info about it.
     """
     result = search(r"\$?[A-Z]{2,5}", text) or search(r"\$[A-Z]{2,5}", text)
-    if result and result.group(0).upper() in symbols:
-        sym = result.group(0).upper()
-
-        if sym.startswith("$"):
-            sym = sym[1:]
-
-        symbols[sym] += 1
-        return
 
     if result:
         sym = result.group(0).upper()
@@ -38,10 +64,9 @@ def normalize_symbol(text):
             sym = sym[1:]
 
         try:
-            yf.Ticker(sym).info # it's a real symbol
-            symbols[sym] += 1
-        except:
-            pass
+            return yf.Ticker(sym).info
+        except KeyError: # XXX what exceptions can yf throw?
+            return None
     return None
 
 # use bloom filter to skip seen submissions/comments/etc
@@ -57,7 +82,23 @@ def submissions(sr):
                 continue
             for reply in comment.replies:
                 if hasattr(reply, "body"):
-                    yield normalize_symbol(reply.body)
+                    info = get_stock_info(reply.body)
+                    if not info:
+                        continue
+                    print(info.get("symbol"))
+                    stock = session.query(Stock).filter_by(symbol=info.get("symbol"), day=datetime.datetime.now()).one_or_none()
+
+                    if stock:
+                        stock.count = stock.count + 1
+                    else:
+                        new_stock = Stock(symbol=info.get("symbol"),
+                                          name=info.get("longName"),
+                                          count=1,
+                                          day=datetime.datetime.now())
+                        session.add(new_stock)
+                    session.commit()
 
-for comment in submissions("wallstreetbets"):
-    print(symbols)
+if __name__ == "__main__":
+    #Stock.__table__.create(engine)
+    for comment in submissions("wallstreetbets"):
+        print(comment)