diff --git a/src/tweetlog.nim b/src/tweetlog.nim index 4099d1b..bb5b3b6 100644 --- a/src/tweetlog.nim +++ b/src/tweetlog.nim @@ -1,20 +1,21 @@ -import tweetlogpkg/twitter, tweetlogpkg/server +import tweetlogpkg/twitter_api, tweetlogpkg/server import threadpool - +import tweetlogpkg/auth import httpClient, base64, uri, json, os, strformat, sequtils, strutils, options import timezones, times from xmltree import escape when isMainModule: - echo "Running" - for tweet in "1355971359168466945".getThread: - echo "" - echo tweet.text - echo "" + #echo "weskerfoot".listTweets + #echo 10.getHome + #for tweet in "1355971359168466945".getThread: + #echo "" + #echo tweet.text + #echo "" #for tweet in "strivev4".listTweets2(){"data"}: #echo tweet #echo tweet{"id"}.getStr.getTweetConvo - #spawn handleRenders() - #startServer() + spawn handleRenders() + startServer() diff --git a/src/tweetlogpkg/auth.nim b/src/tweetlogpkg/auth.nim new file mode 100644 index 0000000..fbc9862 --- /dev/null +++ b/src/tweetlogpkg/auth.nim @@ -0,0 +1,177 @@ +import httpClient, base64, uri, json, os, strformat, sequtils, strutils, options, sugar, times, types +import tables, algorithm, base64, math, options +import nimcrypto +import threadpool + +proc realEncodeUrl*(s: string): string = + ## Exclude A..Z a..z 0..9 - . _ ~ + ## See https://dev.twitter.com/oauth/overview/percent-encoding-parameters + result = newStringOfCap(s.len + s.len shr 2) # assume 12% non-alnum-chars + for i in 0..s.len-1: + case s[i] + of 'a'..'z', 'A'..'Z', '0'..'9', '_', '-', '.', '~': + add(result, s[i]) + else: + add(result, '%') + add(result, toHex(ord(s[i]), 2)) + +proc tweetClient*(token : string) : HttpClient = + var client = newHttpClient() + + client.headers = newHttpHeaders( + { + "Authorization" : token + } + ) + client + +# 3-legged OAuth stuff + +proc parseQueryString(qstr : string) : Table[string, string] = + toTable( + map(qstr.split("&"), + proc(pair : string) : tuple[a: string, b: string] = + let split = pair.split("=") + (split[0], split[1]))) + +proc generateNonce() : string {.gcsafe.} = + let alphabet = map(toSeq(65..90).concat( + toSeq(97..122)).concat( + toSeq(49..57)), (c) {.gcsafe.} => char(c)) + + var randBytes : array[50, uint8] + discard randomBytes(randBytes) + return map(randBytes, (c) => alphabet[(c.int %% alphabet.len)]).join + +proc constructEncodedString(params : Params, sep : string, include_quotes : bool) : string = + var encodedPairs : seq[string] = @[] + var keyPairs : seq[tuple[key: string, value: string]] = toSeq(params.pairs) + + keyPairs.sort((a, b) => cmp(a.key.realEncodeUrl, b.key.realEncodeUrl)) + + for pair in keyPairs: + if include_quotes: + encodedPairs &= pair[0] & "=" & "\"" & pair[1].realEncodeUrl & "\"" + else: + encodedPairs &= pair[0] & "=" & pair[1].realEncodeUrl + + encodedPairs.join(sep) + +proc constructParameterString(params : Params) : string = + result = params.constructEncodedString("&", include_quotes=false) + echo fmt"parameter string = {result}" + +proc constructHeaderString(params : Params) : string = + "OAuth " & params.constructEncodedString(", ", include_quotes=true) + +proc sign(reqMethod : string, + paramString : string, + baseUrl : string) : string = + + let sigBaseString : string = reqMethod.toUpperAscii & "&" & baseUrl.realEncodeUrl & "&" & paramString.realEncodeUrl + var signingKey : string + + signingKey = getEnv("TWITTER_CONSUMER_SECRET").realEncodeUrl & "&" + + result = sha1.hmac(signingKey, sigBaseString).data.encode + +proc signRequest*(requestUrl : string, + requestMethod : string) : Params = + # Return params along with signature that signs request for API request + let oauth_consumer_key = getEnv("TWITTER_CONSUMER_KEY") + let oauth_nonce = generateNonce() + + result["oauth_callback"] = getEnv("TWITTER_OAUTH_CALLBACK") + + # The twitter documentation uses SHA1, but this works and is future-proof + let oauth_signature_method = "HMAC-SHA1".realEncodeUrl + let oauth_timestamp : string = $trunc(epochTime()).uint64 + + result["oauth_nonce"] = oauth_nonce + result["oauth_signature_method"] = oauth_signature_method + result["oauth_timestamp"] = oauth_timestamp + result["oauth_consumer_key"] = oauth_consumer_key + result["oauth_version"] = "1.0" + + let paramString = result.constructParameterString + + result["oauth_signature"] = sign(requestMethod, paramString, requestUrl) + +proc getAuthRequestSigned(requestUrl : string) : Params = + signRequest(requestUrl, "POST") + +proc requestToken*(requestUrl : string) : Option[OAuthToken] = + var headers = newHttpHeaders([]) + + let params = getAuthRequestSigned(requestUrl) + let client = tweetClient(params.constructHeaderString) + let resp = client.request(requestUrl, httpMethod = HttpPost, headers = headers, body = "") + + if resp.status != "200 OK": + echo resp.body + return none(OAuthToken) + + let keyPairs = resp.body.parseQueryString + + if not (keyPairs.hasKey("oauth_token") and keyPairs.hasKey("oauth_token_secret")): + return none(OAuthToken) + + some((oauth_token: keyPairs["oauth_token"], oauth_token_secret: keyPairs["oauth_token_secret"])) + +proc getTokenRedirect*() : string = + let req = "https://api.twitter.com/oauth/request_token".requestToken + fmt"https://api.twitter.com/oauth/authenticate?oauth_token={req.get.oauth_token}" + +proc getAccessToken*(oauth_token : string, oauth_verifier : string) : Option[AccessToken] = + var params : Params + let client = tweetClient(params.constructHeaderString) + let requestUrl = fmt"https://api.twitter.com/oauth/access_token?oauth_token={oauth_token}&oauth_verifier={oauth_verifier}" + let resp = client.request(requestUrl, httpMethod = HttpPost) + + if resp.status != "200 OK": + return none(AccessToken) + + let keyPairs = resp.body.parseQueryString + + some((access_token: keyPairs["oauth_token"], + access_token_secret: keyPairs["oauth_token_secret"], + screen_name: keyPairs["screen_name"], + user_id: keyPairs["user_id"])) + +import jwt + +proc generateJWT*(token : AccessToken) : string = + # take an access token and return an encrypted JWT + let secret = getEnv("JWT_SECRET") + var encoded = toJWT(%*{ + "header" : { + "alg" : "HS256", + "typ" : "JWT" + }, + "claims" : { + "access_token" : token.access_token, + "access_token_secret" : token.access_token_secret, + "screen_name" : token.screen_name, + "user_id" : token.user_id + } + }) + + encoded.sign(secret) + $encoded + +proc verify*(token: string): bool = + let secret = getEnv("JWT_SECRET") + try: + let jwtToken = token.toJWT() + result = jwtToken.verify(secret, HS256) + except InvalidToken: + result = false + +proc decode*(token: string): Option[AccessToken] = + if not token.verify: + return none(AccessToken) + let claims = token.toJWT().claims + some((access_token: claims["access_token"].node.str, + access_token_secret: claims["access_token_secret"].node.str, + screen_name: claims["screen_name"].node.str, + user_id: claims["user_id"].node.str)) diff --git a/src/tweetlogpkg/server.nim b/src/tweetlogpkg/server.nim index 29fb1d6..8281c05 100644 --- a/src/tweetlogpkg/server.nim +++ b/src/tweetlogpkg/server.nim @@ -1,7 +1,8 @@ import strutils, options, sugar, sequtils, asyncdispatch, threadpool, db_sqlite, json, strformat, uri, strscans, times -import twitter +import twitter_api import templates import jester +import auth, types type Author = object name: string @@ -10,6 +11,7 @@ type Author = object type ThreadRequest = object tweetID: string author: Author + token: AccessToken type TwitterThread = ref object of RootObj tweetID: string @@ -20,7 +22,7 @@ type TwitterThread = ref object of RootObj # DateTime format string in ISO8601 format const dateFmt = "YYYY-MM-dd'T'hh:mm:ss'Z'" -proc parseTweetUrl(url : string) : Option[ThreadRequest] = +proc parseTweetUrl(url : string, token : AccessToken) : Option[ThreadRequest] = let path = url.parseUri.path var author : string var tweetID : int @@ -28,7 +30,8 @@ proc parseTweetUrl(url : string) : Option[ThreadRequest] = some( ThreadRequest( tweetID : $tweetID, - author: Author(name: author) + author: Author(name: author), + token: token ) ) else: @@ -123,19 +126,78 @@ proc insertThread(thread : TwitterThread) = # Routes +# If using the web app: +# go to login link, log in, get redirected +# jwt generated (of access token and other info) and stored in httponly cookie, sent along with req to api endpoints +# api decodes it using secret +# +# If using api: +# generate your own oauth token and oauth secret +# pass to api +# api generates jwt (of access token and other info) and it is stored client side wherever client wants (filesystem, etc) +# it is sent along with req to api endpoints +# api decodes it using secret +# +# In both cases, expire tokens after n hours +# When re-auth is needed, redirect the user for web app or return response code as appropriate (401 error) and let client refresh +# need a way to get refresh tokens +# should I only refresh when the underlying oauth access token expires? + +proc decodeToken(cookies: Table[string, string]) : Option[AccessToken] = + # take cookies, get jwt if it exists, and try to decode it + # this can definitely fail + + if cookies.hasKey("twitterjwt"): + let token = cookies["twitterjwt"] + return token.decode + else: + none(AccessToken) + router twitblog: + # TODO make me configurable + get "/tweetlog/auth": + let params = request.params + if not ("oauth_token" in params and "oauth_verifier" in params): + redirect getTokenRedirect() + else: + let oauth_token = params["oauth_token"] + let oauth_verifier = params["oauth_verifier"] + let access_tok = getAccessToken(oauth_token, oauth_verifier) + + if access_tok.isSome: + echo "Setting cookie" + + # XXX insecure for now + setCookie("twitterjwt", access_tok.get.generateJWT, domain="localhost", sameSite=Lax, path="/") + + redirect("http://localhost:3030/") + + #resp(200.HttpCode, $(%*{"jwt" : access_tok.get.generateJWT}), contentType="application/json") + else: + resp(500.HttpCode, $(%*{"error" : "Failed to create token"}), contentType="application/json") + get "/": # Lists all authors + let token = request.cookies.decodeToken + + if token.isNone: + redirect "/tweetlog/auth" + let authors = allAuthors.toSeq let title = "Authors" resp authors.mainPage post "/thread": + let token = request.cookies.decodeToken + + if token.isNone: + redirect "/tweetlog/auth" + let params = request.params if not ("tweetURL" in params): resp "Invalid" - let threadURL = params["tweetURL"].parseTweetUrl + let threadURL = params["tweetURL"].parseTweetUrl(token.get) if threadURL.isSome: redirect (fmt"/thread/{threadURL.get.author}/status/{threadURL.get.tweetID}") @@ -156,10 +218,14 @@ router twitblog: else: # Send it off to the rendering thread for processing # Let them know to check back later + let token = request.cookies.decodeToken + if token.isNone: + redirect "/tweetlog/auth" chan.send( ThreadRequest( tweetID: tweetID, - author: Author(name: author) + author: Author(name: author), + token: token.get ) ) resp checkBack() @@ -170,12 +236,15 @@ router twitblog: let threads = toSeq(threadIDs(author)) resp author.listThreads(threads) + get "/tweetlog/auth": + resp "" + # Entry points proc startServer* = createTweetTables() defer: db.close() - let port = 8080.Port + let port = 3030.Port let settings = newSettings(port=port) var jester = initJester(twitblog, settings=settings) jester.serve() @@ -185,10 +254,14 @@ proc handleRenders* = while true: let t : ThreadRequest = chan.recv() + echo t + if threadExists(t.tweetID, t.author.name).isSome: continue - let tweets = t.tweetID.renderThread + let tweets = t.tweetID.renderThread(t.token) + + echo $tweets if tweets.isSome: insertThread( diff --git a/src/tweetlogpkg/twitter.nim b/src/tweetlogpkg/twitter.nim deleted file mode 100644 index be65184..0000000 --- a/src/tweetlogpkg/twitter.nim +++ /dev/null @@ -1,135 +0,0 @@ -import httpClient, base64, uri, json, os, strformat, sequtils, strutils, options -import timezones, times -import types - -from xmltree import escape - -proc parseTwitterTS(ts : string) : DateTime = - ts.parse("ddd MMM dd hh:mm:ss YYYY") - -# echo "Sun Feb 16 18:19:17 +0000 2020".parseTwitterTS.repr - -proc buildAuthHeader() : string = - let consumerKey = "TWITTER_CONSUMER_KEY".getEnv - let secret = "TWITTER_CONSUMER_SECRET".getEnv - "Basic " & (consumerKey.encodeUrl & ":" & secret.encodeUrl).encode - -proc getToken*() : string = - var client = newHttpClient() - client.headers = newHttpHeaders( - { - "Content-Type" : "application/x-www-form-urlencoded;charset=UTF-8", - "Authorization" : buildAuthHeader() - } - ) - - let body = "grant_type=client_credentials" - - let response = client.request("https://api.twitter.com/oauth2/token", - httpMethod = HttpPost, - body = body).body.parseJson - - let responseType = response["token_type"].getStr - - assert(responseType == "bearer") - - "Bearer " & response["access_token"].getStr - -proc tweetClient() : HttpClient = - var client = newHttpClient() - client.headers = newHttpHeaders( - { - "Authorization" : getToken() - } - ) - client - -proc listTweets2*(user : string) : JsonNode = - let client = tweetClient() - let userIdReq = fmt"/2/users/by?usernames={user}" - var url = fmt"https://api.twitter.com{userIdReq}" - - let userId = client.request(url, httpMethod = HttpGet).body.parseJson{"data"}[0]{"id"}.getStr - - let tweetsReq = fmt"/2/users/{userId}/tweets" - url = fmt"https://api.twitter.com{tweetsReq}" - return client.request(url, httpMethod = HttpGet).body.parseJson - -proc getTweetConvo*(tweetID : string) : JsonNode = - let client = tweetClient() - let userIdReq = fmt"/2/tweets?ids={tweetID}&tweet.fields=conversation_id,author_id" - var url = fmt"https://api.twitter.com{userIdReq}" - - let tweetInfo = client.request(url, httpMethod = HttpGet).body.parseJson - - tweetInfo - -proc listTweets*(user : string) : JsonNode = - let client = tweetClient() - let reqTarget = fmt"/1.1/statuses/user_timeline.json?count=100&screen_name={user}" - let url = fmt"https://api.twitter.com{reqTarget}" - - client.request(url, httpMethod = HttpGet).body.parseJson - -proc getTweet*(tweetID : string) : string = - let client = tweetClient() - let reqTarget = fmt"/1.1/statuses/show.json?id={tweetID}&tweet_mode=extended" - let url = fmt"https://api.twitter.com{reqTarget}" - - client.request(url, httpMethod = HttpGet).body - -iterator getThread*(tweetStart : string) : Tweet = - let client = tweetClient() - var reqTarget = fmt"/2/tweets/search/recent?query=conversation_id:{tweetStart}&tweet.fields=in_reply_to_user_id,author_id,created_at,conversation_id" - var url = fmt"https://api.twitter.com{reqTarget}" - - var currentPage : JsonNode - - currentPage = client.request(url, httpMethod = HttpGet).body.parseJson - - while true: - if currentPage{"meta", "result_count"}.getInt == 0: - break - for tweet in currentPage{"data"}: - yield Tweet( - id: tweet{"id"}.getStr, - in_reply: tweet{"in_reply_to_user_id"}.getStr, - author_id: tweet{"author_id"}.getStr, - text: tweet{"text"}.getStr, - created_at: tweet{"created_at"}.getStr, - conversation_id: tweet{"conversation_id"}.getStr - ) - - let paginationToken = currentPage{"meta"}{"next_token"} - - if paginationToken == nil: - break - - reqTarget = fmt"/2/tweets/search/recent?query=conversation_id:{tweetStart}&tweet.fields=in_reply_to_user_id,author_id,created_at,conversation_id&next_token={paginationToken.getStr}" - url = fmt"https://api.twitter.com{reqTarget}" - currentPage = client.request(url, httpMethod = HttpGet).body.parseJson - -proc convertWords(tweet : Tweet) : string = - let words = tweet.text.split(" ") - var stripped : seq[string] - for chunk in words: - for word in chunk.splitLines: - if word.len > 3 and word[0..3] == "http": - let parsedUri = word.parseUri - let scheme = parsedUri.scheme - let hostname = parsedUri.hostname - let path = parsedUri.path - if (scheme.len > 0 and hostname.len > 0): - let url = xmltree.escape(fmt"{scheme}://{hostname}{path}") - stripped &= url - elif word.len > 0 and word[0] != '@': - stripped &= word - else: - continue - stripped.join(" ") - -proc renderThread*(tweetID : string) : Option[seq[string]] = - let thread = toSeq(getThread(tweetID)).map(convertWords).map(capitalizeAscii) - if thread.len == 0: - return none(seq[string]) - some(thread) diff --git a/src/tweetlogpkg/twitter_api.nim b/src/tweetlogpkg/twitter_api.nim new file mode 100644 index 0000000..ee629b8 --- /dev/null +++ b/src/tweetlogpkg/twitter_api.nim @@ -0,0 +1,129 @@ +import httpClient, uri, json, os, strformat, sequtils, strutils, options, sugar, types +import tables, options, times, twitter +import auth + +from nimcrypto.sysrand import randomBytes +from xmltree import escape + +proc parseTwitterTS(ts : string) : DateTime = + ts.parse("ddd MMM dd hh:mm:ss YYYY") + +proc listTweets*(user : string, token : AccessToken) : JsonNode = + # Lists tweets from a given user + # XXX use Tweet type + let client = tweetClient("Bearer " & token.access_token) + let userIdReq = fmt"/2/users/by?usernames={user}" + var url = fmt"https://api.twitter.com{userIdReq}" + + let userId = client.request(url, httpMethod = HttpGet).body.parseJson{"data"}[0]{"id"}.getStr + + let tweetsReq = fmt"/2/users/{userId}/tweets" + url = fmt"https://api.twitter.com{tweetsReq}" + return client.request(url, httpMethod = HttpGet).body.parseJson + +proc getTweetConvo*(tweetID : string, token : AccessToken) : JsonNode = + # Gets the conversation info for a given tweet + let client = tweetClient("Bearer " & token.access_token) + let userIdReq = fmt"/2/tweets?ids={tweetID}&tweet.fields=conversation_id,author_id" + var url = fmt"https://api.twitter.com{userIdReq}" + + let tweetInfo = client.request(url, httpMethod = HttpGet).body.parseJson + + tweetInfo + +proc getTweet*(tweetID : string, token : AccessToken) : string = + # Grabs a single tweet + # XXX use Tweet type + let client = tweetClient("Bearer " & token.access_token) + let reqTarget = fmt"/1.1/statuses/show.json?id={tweetID}&tweet_mode=extended" + let url = fmt"https://api.twitter.com{reqTarget}" + + client.request(url, httpMethod = HttpGet).body + +proc getHome*(count: int, token : AccessToken) : string = + # Gets your home timeline + let client = tweetClient("Bearer " & token.access_token) + let reqTarget = fmt"/1.1/statuses/user_timeline.json?count={count}&trim_user=1&exclude_replies=1" + let url = fmt"https://api.twitter.com{reqTarget}" + + client.request(url, httpMethod = HttpGet).body + +iterator getThread*(tweetStart : string, token : AccessToken) : Tweet = + + var reqParams : Params # params for the actual API request + + let consumerKey = "TWITTER_CONSUMER_KEY".getEnv + let secret = "TWITTER_CONSUMER_SECRET".getEnv + + var consumerToken = newConsumerToken(consumerKey, secret) + + var twitterAPI = newTwitterAPI(consumerToken, token.access_token, token.access_token_secret) + + reqParams["status"] = "testing123" + reqParams["include_entities"] = "true" + echo fmt"tweetStart = {tweetStart}" + + #var reqTarget = fmt"/2/tweets/search/recent?query=conversation_id:{tweetStart}&tweet.fields=in_reply_to_user_id,author_id,created_at,conversation_id" + var url = "https://api.twitter.com/1.1/statuses/update.json" + + var currentPage : string + + echo fmt"url = {url}" + + # Simply get. + var resp = twitterAPI.get("account/verify_credentials.json") + echo resp.status + + # Using proc corresponding twitter REST APIs. + resp = twitterAPI.statusesUpdate("testing 1 2 3") + echo parseJson(resp.body) + + #while true: + #if currentPage{"meta", "result_count"}.getInt == 0: + #break + #for tweet in currentPage{"data"}: + #yield Tweet( + #id: tweet{"id"}.getStr, + #in_reply: tweet{"in_reply_to_user_id"}.getStr, + #author_id: tweet{"author_id"}.getStr, + #text: tweet{"text"}.getStr, + #created_at: tweet{"created_at"}.getStr, + #conversation_id: tweet{"conversation_id"}.getStr + #) + + #let paginationToken = currentPage{"meta"}{"next_token"} + + #if paginationToken == nil: + #break + + #echo "Getting next page" + + #reqTarget = fmt"/2/tweets/search/recent?query=conversation_id:{tweetStart}&tweet.fields=in_reply_to_user_id,author_id,created_at,conversation_id&next_token={paginationToken.getStr}" + #url = fmt"https://api.twitter.com{reqTarget}" + #currentPage = client.request(url, httpMethod = HttpGet).body.parseJson + +proc convertWords(tweet : Tweet) : string = + let words = tweet.text.split(" ") + var stripped : seq[string] + for chunk in words: + for word in chunk.splitLines: + if word.len > 3 and word[0..3] == "http": + let parsedUri = word.parseUri + let scheme = parsedUri.scheme + let hostname = parsedUri.hostname + let path = parsedUri.path + if (scheme.len > 0 and hostname.len > 0): + let url = xmltree.escape(fmt"{scheme}://{hostname}{path}") + stripped &= url + elif word.len > 0 and word[0] != '@': + stripped &= word + else: + continue + stripped.join(" ") + +proc renderThread*(tweetID : string, token : AccessToken) : Option[seq[string]] = + let thread = toSeq(getThread(tweetID, token)).map(convertWords).map(capitalizeAscii) + echo $thread + if thread.len == 0: + return none(seq[string]) + some(thread) diff --git a/src/tweetlogpkg/types.nim b/src/tweetlogpkg/types.nim index 592772b..00f550e 100644 --- a/src/tweetlogpkg/types.nim +++ b/src/tweetlogpkg/types.nim @@ -1,3 +1,13 @@ +import tables + +type Params* = Table[string, string] +type OAuthToken* = tuple[oauth_token: string, oauth_token_secret: string] + +type AccessToken* = tuple[access_token : string, + access_token_secret: string, + screen_name: string, + user_id: string] + type Tweet* = ref object of RootObj id*: string diff --git a/tweetlog.nimble b/tweetlog.nimble index fb71867..cc36d36 100644 --- a/tweetlog.nimble +++ b/tweetlog.nimble @@ -12,6 +12,10 @@ bin = @["tweetlog"] requires "nim >= 1.0" requires "https://github.com/dom96/jester" requires "https://github.com/pragmagic/karax" +requires "https://github.com/GULPF/timezones" +requires "https://github.com/cheatfate/nimcrypto" +requires "https://github.com/yglukhov/nim-jwt" +requires "https://github.com/snus-kin/twitter.nim" task bookmark, "Builds the minified bookmarklet": "echo -n 'javascript:' > ./bookmarklet.min.js".exec