Browse Source

Refactor it all into a clone of threadreaderapp ;)

pull/1/head
Wesley Kerfoot 4 years ago
parent
commit
1a1ee3f207
  1. 15
      README.md
  2. 1
      config.nims
  3. 36
      src/twit2blog.nim
  4. 18
      src/twit2blogpkg/help.nim
  5. 89
      src/twit2blogpkg/server.nim
  6. 10
      src/twit2blogpkg/twitter.nim
  7. 3
      twit2blog.nimble

15
README.md

@ -1,6 +1,6 @@
### Twit2Blog
A simple tool to help make it easier to turn your tweet rants into real blog posts
A simple tool to help make it easier to generate readable tweets you can browse.
### Building
`nimble build` is all you'll need at this point.
@ -9,18 +9,9 @@ A simple tool to help make it easier to turn your tweet rants into real blog pos
Requires `TWITTER_CONSUMER_KEY` and `TWITTER_CONSUMER_SECRET`, both of which you can only get if you have a registered developer account and an application created for twitter.
```
Usage: twit2blog [opts]
Options:
-u, --user The screen name of the twitter user. E.g. If your twitter is https://twitter.com/foobar, then `foobar`.
-t, --thread The ID of the last tweet in your thread. E.g. 12345.
For more information read the Github readme:
https://github.com/weskerfoot/Twit2Blog#readme
./twit2blog
```
Example: `twit2blog -t:1234 -u:alice | pandoc --from=markdown --to=html > thread.html`
You must provide the ID of the *last* tweet from the thread you want rendered. E.g. [https://twitter.com/weskerfoot/status/1199466868953759750](https://twitter.com/weskerfoot/status/1199466868953759750) is the last tweet in one of my threads. The reason for this is that the twitter API does not provide an easy way to search for replies to a given tweet.
You can see the output generated from it [here](https://wesk.tech/tweet_example.html)
Example: `http://localhost:8080/thread/weskerfoot/status/1221552400852451329`

1
config.nims

@ -1 +1,2 @@
switch("define", "ssl")
switch("threads", "on")

36
src/twit2blog.nim

@ -1,34 +1,6 @@
import twit2blogpkg/twitter, twit2blogpkg/help
import os, system, parseopt, strutils, tables
import twit2blogpkg/twitter, twit2blogpkg/server
import threadpool
when isMainModule:
var args = initOptParser(commandLineParams().join(" "))
var twitterParams = initTable[string, string]()
let validArgs = @["u", "t", "user", "thread"]
var currentKey : string
while true:
args.next()
case args.kind
of cmdEnd: break
of cmdShortOption, cmdLongOption:
if (args.key == "help") or (args.key == "h"):
writeHelp()
if args.val == "":
continue
else:
if validArgs.contains(args.key):
twitterParams[args.key] = args.val
of cmdArgument:
if validArgs.contains(currentKey):
twitterParams[currentKey] = args.val
if twitterParams.hasKey("u"):
twitterParams["user"] = twitterParams["u"]
if twitterParams.hasKey("t"):
twitterParams["thread"] = twitterParams["t"]
if not (twitterParams.hasKey("user") and twitterParams.hasKey("thread")):
writeHelp()
echo twitterParams["thread"].renderThread(twitterParams["user"])
spawn handleRenders()
startServer()

18
src/twit2blogpkg/help.nim

@ -1,18 +0,0 @@
import system
const
help = """
Usage: twit2blog [opts]
Options:
-u, --user The screen name of the twitter user. E.g. If your twitter is https://twitter.com/foobar, then `foobar`.
-t, --thread The ID of the last tweet in your thread. E.g. 12345.
For more information read the Github readme:
https://github.com/weskerfoot/Twit2Blog#readme
"""
proc writeHelp*(quit=true) =
echo(help)
if quit:
quit(1)

89
src/twit2blogpkg/server.nim

@ -0,0 +1,89 @@
import strutils, sets, options, sugar, sequtils, asyncdispatch, threadpool, db_sqlite
import twitter
import xander
# one thread just receives messages with thread ID / username
# thread then passes messages to worker threads in round-robin fashion
# worker threads gather thread contents, then update Redis DB (or sqlite) with thread ID mapped to content
# user can go back to page with thread ID / user combo (or unique ID we give them?) and see compiled thread
type
ThreadRequest = object
tweetID: string
author: string
type TwitterThread = ref object of RootObj
tweetID: string
author: string
tweets: string
var chan : Channel[ThreadRequest]
# Max 20 items processing
chan.open(20)
var server = newAsyncHttpServer()
let db = open("twit2blog.db", "", "", "")
proc createTweetTable() =
db.exec(sql"""CREATE TABLE IF NOT EXISTS threads (
id INTEGER PRIMARY KEY,
tid TEXT,
author TEXT,
tweets TEXT
)""")
proc threadExists(threadID : string, author : string) : Option[TwitterThread] =
let row = db.getRow(sql"SELECT * FROM threads WHERE tid=? AND author=?", threadID, author)
if row.all(col => col == ""):
return none(TwitterThread)
some(
TwitterThread(tweetID: row[1],
author: row[2],
tweets: row[3])
)
proc insertThread(thread : TwitterThread) =
db.exec(sql"INSERT INTO threads (tid, author, tweets) VALUES (?, ?, ?)",
thread.tweetID,
thread.author,
thread.tweets)
get "/thread/:author/status/:threadID":
let threadID = data{"threadID"}.getStr()
let author = data{"author"}.getStr()
let thread = threadExists(threadID, author)
if thread.isSome:
respond thread.get.tweets
else:
chan.send(ThreadRequest(tweetID: data{"threadID"}.getStr(), author: data{"author"}.getStr()))
respond "Hang on, we're grabbing your thread :) Come back to this page later."
proc startServer* =
createTweetTable()
defer: db.close()
runForever(8080)
proc handleRenders* =
var processing = initHashSet[string]()
while true:
let t : ThreadRequest = chan.recv()
if processing.contains(t.author & t.tweetID):
continue
processing.incl(t.author & t.tweetID)
let tweets = t.tweetID.renderThread(t.author)
if tweets.isSome:
insertThread(
TwitterThread(tweetID: t.tweetID,
author: t.author,
tweets: tweets.get.join("\n"))
)

10
src/twit2blogpkg/twitter.nim

@ -1,4 +1,4 @@
import httpClient, base64, uri, json, os, strformat, sequtils, strutils
import httpClient, base64, uri, json, os, strformat, sequtils, strutils, options
proc buildAuthHeader() : string =
let consumerKey = "TWITTER_CONSUMER_KEY".getEnv
@ -82,6 +82,8 @@ proc convertWords(tweet : string) : string =
continue
stripped.join(" ")
proc renderThread*(tweetID : string, user : string) : string =
let thread = tweetID.getThread(user).map(convertWords).map(capitalizeAscii).join("\n\n")
fmt"### By {user}" & "\n" & fmt"{thread}"
proc renderThread*(tweetID : string, user : string) : Option[seq[string]] =
let thread = tweetID.getThread(user).map(convertWords).map(capitalizeAscii)
if thread.len == 0:
return none(seq[string])
some(thread)

3
twit2blog.nimble

@ -12,4 +12,5 @@ bin = @["twit2blog"]
# Dependencies
requires "nim >= 1.0.9"
requires "nim >= 1.0.9", "regex"
requires "https://github.com/sunjohanday/xander"

Loading…
Cancel
Save