Merge pull request #1 from weskerfoot/webserver

Refactor it into a web app

.gitignore

@@ -1,2 +1,3 @@
 .envrc
 twit2blog
+twit2blog.db

README.md

@@ -1,6 +1,6 @@
 ### Twit2Blog
 
-A simple tool to help make it easier to turn your tweet rants into real blog posts
+A simple tool to help make it easier to generate readable tweets you can browse.
 
 ### Building
 `nimble build` is all you'll need at this point.
@@ -9,18 +9,9 @@ A simple tool to help make it easier to turn your tweet rants into real blog pos
 Requires `TWITTER_CONSUMER_KEY` and `TWITTER_CONSUMER_SECRET`, both of which you can only get if you have a registered developer account and an application created for twitter.
 
 ```
-Usage: twit2blog [opts]
+./twit2blog
-
-Options:
-  -u, --user   The screen name of the twitter user. E.g. If your twitter is https://twitter.com/foobar, then `foobar`.
-  -t, --thread The ID of the last tweet in your thread. E.g. 12345.
-
-For more information read the Github readme:
-  https://github.com/weskerfoot/Twit2Blog#readme
 ```
 
-Example: `twit2blog -t:1234 -u:alice | pandoc --from=markdown --to=html > thread.html`
-
 You must provide the ID of the *last* tweet from the thread you want rendered. E.g. [https://twitter.com/weskerfoot/status/1199466868953759750](https://twitter.com/weskerfoot/status/1199466868953759750) is the last tweet in one of my threads. The reason for this is that the twitter API does not provide an easy way to search for replies to a given tweet.
 
-You can see the output generated from it [here](https://wesk.tech/tweet_example.html)
+Example: `http://localhost:8080/thread/weskerfoot/status/1221552400852451329`

config.nims

@@ -1 +1,2 @@
 switch("define", "ssl")
+switch("threads", "on")

src/twit2blog.nim

@@ -1,34 +1,6 @@
-import twit2blogpkg/twitter, twit2blogpkg/help
+import twit2blogpkg/twitter, twit2blogpkg/server
-import os, system, parseopt, strutils, tables
+import threadpool
 
 when isMainModule:
-  var args = initOptParser(commandLineParams().join(" "))
+  spawn handleRenders()
-  var twitterParams = initTable[string, string]()
+  startServer()
-  let validArgs = @["u", "t", "user", "thread"]
-  var currentKey : string
-
-  while true:
-    args.next()
-    case args.kind
-      of cmdEnd: break
-      of cmdShortOption, cmdLongOption:
-        if (args.key == "help") or (args.key == "h"):
-          writeHelp()
-        if args.val == "":
-          continue
-        else:
-          if validArgs.contains(args.key):
-            twitterParams[args.key] = args.val
-      of cmdArgument:
-        if validArgs.contains(currentKey):
-          twitterParams[currentKey] = args.val
-
-  if twitterParams.hasKey("u"):
-    twitterParams["user"] = twitterParams["u"]
-  if twitterParams.hasKey("t"):
-    twitterParams["thread"] = twitterParams["t"]
-
-  if not (twitterParams.hasKey("user") and twitterParams.hasKey("thread")):
-    writeHelp()
-
-  echo twitterParams["thread"].renderThread(twitterParams["user"])

src/twit2blogpkg/help.nim

@@ -1,18 +0,0 @@
-import system
-
-const
-  help = """
-Usage: twit2blog [opts]
-
-Options:
-  -u, --user   The screen name of the twitter user. E.g. If your twitter is https://twitter.com/foobar, then `foobar`.
-  -t, --thread The ID of the last tweet in your thread. E.g. 12345.
-
-For more information read the Github readme:
-  https://github.com/weskerfoot/Twit2Blog#readme
-"""
-
-proc writeHelp*(quit=true) =
-  echo(help)
-  if quit:
-    quit(1)

src/twit2blogpkg/server.nim

@@ -0,0 +1,89 @@
+import strutils, sets, options, sugar, sequtils, asyncdispatch, threadpool, db_sqlite
+import twitter
+import xander
+
+# one thread just receives messages with thread ID / username
+# thread then passes messages to worker threads in round-robin fashion
+# worker threads gather thread contents, then update Redis DB (or sqlite) with thread ID mapped to content
+# user can go back to page with thread ID / user combo (or unique ID we give them?) and see compiled thread
+
+type
+  ThreadRequest = object
+    tweetID: string
+    author: string
+
+type TwitterThread = ref object of RootObj
+  tweetID: string
+  author: string
+  tweets: string
+
+var chan : Channel[ThreadRequest]
+
+# Max 20 items processing
+chan.open(20)
+
+var server = newAsyncHttpServer()
+
+let db = open("twit2blog.db", "", "", "")
+
+proc createTweetTable() =
+  db.exec(sql"""CREATE TABLE IF NOT EXISTS threads (
+                   id INTEGER PRIMARY KEY,
+                   tid TEXT,
+                   author TEXT,
+                   tweets TEXT
+                )""")
+
+proc threadExists(threadID : string, author : string) : Option[TwitterThread] =
+  let row = db.getRow(sql"SELECT * FROM threads WHERE tid=? AND author=?", threadID, author)
+
+  if row.all(col => col == ""):
+    return none(TwitterThread)
+  some(
+    TwitterThread(tweetID: row[1],
+                  author: row[2],
+                  tweets: row[3])
+  )
+
+proc insertThread(thread : TwitterThread) =
+  db.exec(sql"INSERT INTO threads (tid, author, tweets) VALUES (?, ?, ?)",
+          thread.tweetID,
+          thread.author,
+          thread.tweets)
+
+get "/thread/:author/status/:threadID":
+  let threadID = data{"threadID"}.getStr()
+  let author = data{"author"}.getStr()
+
+  let thread = threadExists(threadID, author)
+
+  if thread.isSome:
+    respond thread.get.tweets
+  else:
+    chan.send(ThreadRequest(tweetID: data{"threadID"}.getStr(), author: data{"author"}.getStr()))
+    respond "Hang on, we're grabbing your thread :) Come back to this page later."
+
+proc startServer* =
+  createTweetTable()
+  defer: db.close()
+  runForever(8080)
+
+proc handleRenders* =
+  var processing = initHashSet[string]()
+
+  while true:
+    let t : ThreadRequest = chan.recv()
+
+    if processing.contains(t.author & t.tweetID):
+      continue
+
+    processing.incl(t.author & t.tweetID)
+
+    let tweets = t.tweetID.renderThread(t.author)
+
+    if tweets.isSome:
+      insertThread(
+        TwitterThread(tweetID: t.tweetID,
+                      author: t.author,
+                      tweets: tweets.get.join("\n"))
+      )

src/twit2blogpkg/twitter.nim

@@ -1,4 +1,4 @@
-import httpClient, base64, uri, json, os, strformat, sequtils, strutils
+import httpClient, base64, uri, json, os, strformat, sequtils, strutils, options
 
 proc buildAuthHeader() : string =
   let consumerKey = "TWITTER_CONSUMER_KEY".getEnv
@@ -82,6 +82,8 @@ proc convertWords(tweet : string) : string =
         continue
   stripped.join(" ")
 
-proc renderThread*(tweetID : string, user : string) : string =
+proc renderThread*(tweetID : string, user : string) : Option[seq[string]] =
-  let thread = tweetID.getThread(user).map(convertWords).map(capitalizeAscii).join("\n\n")
+  let thread = tweetID.getThread(user).map(convertWords).map(capitalizeAscii)
-  fmt"### By {user}" & "\n" & fmt"{thread}"
+  if thread.len == 0:
+    return none(seq[string])
+  some(thread)

twit2blog.nimble

@@ -12,4 +12,5 @@ bin           = @["twit2blog"]
 
 # Dependencies
 
-requires "nim >= 1.0.9"
+requires "nim >= 1.0.9", "regex"
+requires "https://github.com/sunjohanday/xander"