7 changed files with 410 additions and 151 deletions
@ -1,20 +1,21 @@ |
|||||
import tweetlogpkg/twitter, tweetlogpkg/server |
import tweetlogpkg/twitter_api, tweetlogpkg/server |
||||
import threadpool |
import threadpool |
||||
|
import tweetlogpkg/auth |
||||
import httpClient, base64, uri, json, os, strformat, sequtils, strutils, options |
import httpClient, base64, uri, json, os, strformat, sequtils, strutils, options |
||||
import timezones, times |
import timezones, times |
||||
|
|
||||
from xmltree import escape |
from xmltree import escape |
||||
|
|
||||
when isMainModule: |
when isMainModule: |
||||
echo "Running" |
#echo "weskerfoot".listTweets |
||||
for tweet in "1355971359168466945".getThread: |
#echo 10.getHome |
||||
echo "" |
#for tweet in "1355971359168466945".getThread: |
||||
echo tweet.text |
#echo "" |
||||
echo "" |
#echo tweet.text |
||||
|
#echo "" |
||||
|
|
||||
#for tweet in "strivev4".listTweets2(){"data"}: |
#for tweet in "strivev4".listTweets2(){"data"}: |
||||
#echo tweet |
#echo tweet |
||||
#echo tweet{"id"}.getStr.getTweetConvo |
#echo tweet{"id"}.getStr.getTweetConvo |
||||
#spawn handleRenders() |
spawn handleRenders() |
||||
#startServer() |
startServer() |
||||
|
|||||
@ -0,0 +1,177 @@ |
|||||
|
import httpClient, base64, uri, json, os, strformat, sequtils, strutils, options, sugar, times, types |
||||
|
import tables, algorithm, base64, math, options |
||||
|
import nimcrypto |
||||
|
import threadpool |
||||
|
|
||||
|
proc realEncodeUrl*(s: string): string = |
||||
|
## Exclude A..Z a..z 0..9 - . _ ~ |
||||
|
## See https://dev.twitter.com/oauth/overview/percent-encoding-parameters |
||||
|
result = newStringOfCap(s.len + s.len shr 2) # assume 12% non-alnum-chars |
||||
|
for i in 0..s.len-1: |
||||
|
case s[i] |
||||
|
of 'a'..'z', 'A'..'Z', '0'..'9', '_', '-', '.', '~': |
||||
|
add(result, s[i]) |
||||
|
else: |
||||
|
add(result, '%') |
||||
|
add(result, toHex(ord(s[i]), 2)) |
||||
|
|
||||
|
proc tweetClient*(token : string) : HttpClient = |
||||
|
var client = newHttpClient() |
||||
|
|
||||
|
client.headers = newHttpHeaders( |
||||
|
{ |
||||
|
"Authorization" : token |
||||
|
} |
||||
|
) |
||||
|
client |
||||
|
|
||||
|
# 3-legged OAuth stuff |
||||
|
|
||||
|
proc parseQueryString(qstr : string) : Table[string, string] = |
||||
|
toTable( |
||||
|
map(qstr.split("&"), |
||||
|
proc(pair : string) : tuple[a: string, b: string] = |
||||
|
let split = pair.split("=") |
||||
|
(split[0], split[1]))) |
||||
|
|
||||
|
proc generateNonce() : string {.gcsafe.} = |
||||
|
let alphabet = map(toSeq(65..90).concat( |
||||
|
toSeq(97..122)).concat( |
||||
|
toSeq(49..57)), (c) {.gcsafe.} => char(c)) |
||||
|
|
||||
|
var randBytes : array[50, uint8] |
||||
|
discard randomBytes(randBytes) |
||||
|
return map(randBytes, (c) => alphabet[(c.int %% alphabet.len)]).join |
||||
|
|
||||
|
proc constructEncodedString(params : Params, sep : string, include_quotes : bool) : string = |
||||
|
var encodedPairs : seq[string] = @[] |
||||
|
var keyPairs : seq[tuple[key: string, value: string]] = toSeq(params.pairs) |
||||
|
|
||||
|
keyPairs.sort((a, b) => cmp(a.key.realEncodeUrl, b.key.realEncodeUrl)) |
||||
|
|
||||
|
for pair in keyPairs: |
||||
|
if include_quotes: |
||||
|
encodedPairs &= pair[0] & "=" & "\"" & pair[1].realEncodeUrl & "\"" |
||||
|
else: |
||||
|
encodedPairs &= pair[0] & "=" & pair[1].realEncodeUrl |
||||
|
|
||||
|
encodedPairs.join(sep) |
||||
|
|
||||
|
proc constructParameterString(params : Params) : string = |
||||
|
result = params.constructEncodedString("&", include_quotes=false) |
||||
|
echo fmt"parameter string = {result}" |
||||
|
|
||||
|
proc constructHeaderString(params : Params) : string = |
||||
|
"OAuth " & params.constructEncodedString(", ", include_quotes=true) |
||||
|
|
||||
|
proc sign(reqMethod : string, |
||||
|
paramString : string, |
||||
|
baseUrl : string) : string = |
||||
|
|
||||
|
let sigBaseString : string = reqMethod.toUpperAscii & "&" & baseUrl.realEncodeUrl & "&" & paramString.realEncodeUrl |
||||
|
var signingKey : string |
||||
|
|
||||
|
signingKey = getEnv("TWITTER_CONSUMER_SECRET").realEncodeUrl & "&" |
||||
|
|
||||
|
result = sha1.hmac(signingKey, sigBaseString).data.encode |
||||
|
|
||||
|
proc signRequest*(requestUrl : string, |
||||
|
requestMethod : string) : Params = |
||||
|
# Return params along with signature that signs request for API request |
||||
|
let oauth_consumer_key = getEnv("TWITTER_CONSUMER_KEY") |
||||
|
let oauth_nonce = generateNonce() |
||||
|
|
||||
|
result["oauth_callback"] = getEnv("TWITTER_OAUTH_CALLBACK") |
||||
|
|
||||
|
# The twitter documentation uses SHA1, but this works and is future-proof |
||||
|
let oauth_signature_method = "HMAC-SHA1".realEncodeUrl |
||||
|
let oauth_timestamp : string = $trunc(epochTime()).uint64 |
||||
|
|
||||
|
result["oauth_nonce"] = oauth_nonce |
||||
|
result["oauth_signature_method"] = oauth_signature_method |
||||
|
result["oauth_timestamp"] = oauth_timestamp |
||||
|
result["oauth_consumer_key"] = oauth_consumer_key |
||||
|
result["oauth_version"] = "1.0" |
||||
|
|
||||
|
let paramString = result.constructParameterString |
||||
|
|
||||
|
result["oauth_signature"] = sign(requestMethod, paramString, requestUrl) |
||||
|
|
||||
|
proc getAuthRequestSigned(requestUrl : string) : Params = |
||||
|
signRequest(requestUrl, "POST") |
||||
|
|
||||
|
proc requestToken*(requestUrl : string) : Option[OAuthToken] = |
||||
|
var headers = newHttpHeaders([]) |
||||
|
|
||||
|
let params = getAuthRequestSigned(requestUrl) |
||||
|
let client = tweetClient(params.constructHeaderString) |
||||
|
let resp = client.request(requestUrl, httpMethod = HttpPost, headers = headers, body = "") |
||||
|
|
||||
|
if resp.status != "200 OK": |
||||
|
echo resp.body |
||||
|
return none(OAuthToken) |
||||
|
|
||||
|
let keyPairs = resp.body.parseQueryString |
||||
|
|
||||
|
if not (keyPairs.hasKey("oauth_token") and keyPairs.hasKey("oauth_token_secret")): |
||||
|
return none(OAuthToken) |
||||
|
|
||||
|
some((oauth_token: keyPairs["oauth_token"], oauth_token_secret: keyPairs["oauth_token_secret"])) |
||||
|
|
||||
|
proc getTokenRedirect*() : string = |
||||
|
let req = "https://api.twitter.com/oauth/request_token".requestToken |
||||
|
fmt"https://api.twitter.com/oauth/authenticate?oauth_token={req.get.oauth_token}" |
||||
|
|
||||
|
proc getAccessToken*(oauth_token : string, oauth_verifier : string) : Option[AccessToken] = |
||||
|
var params : Params |
||||
|
let client = tweetClient(params.constructHeaderString) |
||||
|
let requestUrl = fmt"https://api.twitter.com/oauth/access_token?oauth_token={oauth_token}&oauth_verifier={oauth_verifier}" |
||||
|
let resp = client.request(requestUrl, httpMethod = HttpPost) |
||||
|
|
||||
|
if resp.status != "200 OK": |
||||
|
return none(AccessToken) |
||||
|
|
||||
|
let keyPairs = resp.body.parseQueryString |
||||
|
|
||||
|
some((access_token: keyPairs["oauth_token"], |
||||
|
access_token_secret: keyPairs["oauth_token_secret"], |
||||
|
screen_name: keyPairs["screen_name"], |
||||
|
user_id: keyPairs["user_id"])) |
||||
|
|
||||
|
import jwt |
||||
|
|
||||
|
proc generateJWT*(token : AccessToken) : string = |
||||
|
# take an access token and return an encrypted JWT |
||||
|
let secret = getEnv("JWT_SECRET") |
||||
|
var encoded = toJWT(%*{ |
||||
|
"header" : { |
||||
|
"alg" : "HS256", |
||||
|
"typ" : "JWT" |
||||
|
}, |
||||
|
"claims" : { |
||||
|
"access_token" : token.access_token, |
||||
|
"access_token_secret" : token.access_token_secret, |
||||
|
"screen_name" : token.screen_name, |
||||
|
"user_id" : token.user_id |
||||
|
} |
||||
|
}) |
||||
|
|
||||
|
encoded.sign(secret) |
||||
|
$encoded |
||||
|
|
||||
|
proc verify*(token: string): bool = |
||||
|
let secret = getEnv("JWT_SECRET") |
||||
|
try: |
||||
|
let jwtToken = token.toJWT() |
||||
|
result = jwtToken.verify(secret, HS256) |
||||
|
except InvalidToken: |
||||
|
result = false |
||||
|
|
||||
|
proc decode*(token: string): Option[AccessToken] = |
||||
|
if not token.verify: |
||||
|
return none(AccessToken) |
||||
|
let claims = token.toJWT().claims |
||||
|
some((access_token: claims["access_token"].node.str, |
||||
|
access_token_secret: claims["access_token_secret"].node.str, |
||||
|
screen_name: claims["screen_name"].node.str, |
||||
|
user_id: claims["user_id"].node.str)) |
||||
@ -1,135 +0,0 @@ |
|||||
import httpClient, base64, uri, json, os, strformat, sequtils, strutils, options |
|
||||
import timezones, times |
|
||||
import types |
|
||||
|
|
||||
from xmltree import escape |
|
||||
|
|
||||
proc parseTwitterTS(ts : string) : DateTime = |
|
||||
ts.parse("ddd MMM dd hh:mm:ss YYYY") |
|
||||
|
|
||||
# echo "Sun Feb 16 18:19:17 +0000 2020".parseTwitterTS.repr |
|
||||
|
|
||||
proc buildAuthHeader() : string = |
|
||||
let consumerKey = "TWITTER_CONSUMER_KEY".getEnv |
|
||||
let secret = "TWITTER_CONSUMER_SECRET".getEnv |
|
||||
"Basic " & (consumerKey.encodeUrl & ":" & secret.encodeUrl).encode |
|
||||
|
|
||||
proc getToken*() : string = |
|
||||
var client = newHttpClient() |
|
||||
client.headers = newHttpHeaders( |
|
||||
{ |
|
||||
"Content-Type" : "application/x-www-form-urlencoded;charset=UTF-8", |
|
||||
"Authorization" : buildAuthHeader() |
|
||||
} |
|
||||
) |
|
||||
|
|
||||
let body = "grant_type=client_credentials" |
|
||||
|
|
||||
let response = client.request("https://api.twitter.com/oauth2/token", |
|
||||
httpMethod = HttpPost, |
|
||||
body = body).body.parseJson |
|
||||
|
|
||||
let responseType = response["token_type"].getStr |
|
||||
|
|
||||
assert(responseType == "bearer") |
|
||||
|
|
||||
"Bearer " & response["access_token"].getStr |
|
||||
|
|
||||
proc tweetClient() : HttpClient = |
|
||||
var client = newHttpClient() |
|
||||
client.headers = newHttpHeaders( |
|
||||
{ |
|
||||
"Authorization" : getToken() |
|
||||
} |
|
||||
) |
|
||||
client |
|
||||
|
|
||||
proc listTweets2*(user : string) : JsonNode = |
|
||||
let client = tweetClient() |
|
||||
let userIdReq = fmt"/2/users/by?usernames={user}" |
|
||||
var url = fmt"https://api.twitter.com{userIdReq}" |
|
||||
|
|
||||
let userId = client.request(url, httpMethod = HttpGet).body.parseJson{"data"}[0]{"id"}.getStr |
|
||||
|
|
||||
let tweetsReq = fmt"/2/users/{userId}/tweets" |
|
||||
url = fmt"https://api.twitter.com{tweetsReq}" |
|
||||
return client.request(url, httpMethod = HttpGet).body.parseJson |
|
||||
|
|
||||
proc getTweetConvo*(tweetID : string) : JsonNode = |
|
||||
let client = tweetClient() |
|
||||
let userIdReq = fmt"/2/tweets?ids={tweetID}&tweet.fields=conversation_id,author_id" |
|
||||
var url = fmt"https://api.twitter.com{userIdReq}" |
|
||||
|
|
||||
let tweetInfo = client.request(url, httpMethod = HttpGet).body.parseJson |
|
||||
|
|
||||
tweetInfo |
|
||||
|
|
||||
proc listTweets*(user : string) : JsonNode = |
|
||||
let client = tweetClient() |
|
||||
let reqTarget = fmt"/1.1/statuses/user_timeline.json?count=100&screen_name={user}" |
|
||||
let url = fmt"https://api.twitter.com{reqTarget}" |
|
||||
|
|
||||
client.request(url, httpMethod = HttpGet).body.parseJson |
|
||||
|
|
||||
proc getTweet*(tweetID : string) : string = |
|
||||
let client = tweetClient() |
|
||||
let reqTarget = fmt"/1.1/statuses/show.json?id={tweetID}&tweet_mode=extended" |
|
||||
let url = fmt"https://api.twitter.com{reqTarget}" |
|
||||
|
|
||||
client.request(url, httpMethod = HttpGet).body |
|
||||
|
|
||||
iterator getThread*(tweetStart : string) : Tweet = |
|
||||
let client = tweetClient() |
|
||||
var reqTarget = fmt"/2/tweets/search/recent?query=conversation_id:{tweetStart}&tweet.fields=in_reply_to_user_id,author_id,created_at,conversation_id" |
|
||||
var url = fmt"https://api.twitter.com{reqTarget}" |
|
||||
|
|
||||
var currentPage : JsonNode |
|
||||
|
|
||||
currentPage = client.request(url, httpMethod = HttpGet).body.parseJson |
|
||||
|
|
||||
while true: |
|
||||
if currentPage{"meta", "result_count"}.getInt == 0: |
|
||||
break |
|
||||
for tweet in currentPage{"data"}: |
|
||||
yield Tweet( |
|
||||
id: tweet{"id"}.getStr, |
|
||||
in_reply: tweet{"in_reply_to_user_id"}.getStr, |
|
||||
author_id: tweet{"author_id"}.getStr, |
|
||||
text: tweet{"text"}.getStr, |
|
||||
created_at: tweet{"created_at"}.getStr, |
|
||||
conversation_id: tweet{"conversation_id"}.getStr |
|
||||
) |
|
||||
|
|
||||
let paginationToken = currentPage{"meta"}{"next_token"} |
|
||||
|
|
||||
if paginationToken == nil: |
|
||||
break |
|
||||
|
|
||||
reqTarget = fmt"/2/tweets/search/recent?query=conversation_id:{tweetStart}&tweet.fields=in_reply_to_user_id,author_id,created_at,conversation_id&next_token={paginationToken.getStr}" |
|
||||
url = fmt"https://api.twitter.com{reqTarget}" |
|
||||
currentPage = client.request(url, httpMethod = HttpGet).body.parseJson |
|
||||
|
|
||||
proc convertWords(tweet : Tweet) : string = |
|
||||
let words = tweet.text.split(" ") |
|
||||
var stripped : seq[string] |
|
||||
for chunk in words: |
|
||||
for word in chunk.splitLines: |
|
||||
if word.len > 3 and word[0..3] == "http": |
|
||||
let parsedUri = word.parseUri |
|
||||
let scheme = parsedUri.scheme |
|
||||
let hostname = parsedUri.hostname |
|
||||
let path = parsedUri.path |
|
||||
if (scheme.len > 0 and hostname.len > 0): |
|
||||
let url = xmltree.escape(fmt"{scheme}://{hostname}{path}") |
|
||||
stripped &= url |
|
||||
elif word.len > 0 and word[0] != '@': |
|
||||
stripped &= word |
|
||||
else: |
|
||||
continue |
|
||||
stripped.join(" ") |
|
||||
|
|
||||
proc renderThread*(tweetID : string) : Option[seq[string]] = |
|
||||
let thread = toSeq(getThread(tweetID)).map(convertWords).map(capitalizeAscii) |
|
||||
if thread.len == 0: |
|
||||
return none(seq[string]) |
|
||||
some(thread) |
|
||||
@ -0,0 +1,129 @@ |
|||||
|
import httpClient, uri, json, os, strformat, sequtils, strutils, options, sugar, types |
||||
|
import tables, options, times, twitter |
||||
|
import auth |
||||
|
|
||||
|
from nimcrypto.sysrand import randomBytes |
||||
|
from xmltree import escape |
||||
|
|
||||
|
proc parseTwitterTS(ts : string) : DateTime = |
||||
|
ts.parse("ddd MMM dd hh:mm:ss YYYY") |
||||
|
|
||||
|
proc listTweets*(user : string, token : AccessToken) : JsonNode = |
||||
|
# Lists tweets from a given user |
||||
|
# XXX use Tweet type |
||||
|
let client = tweetClient("Bearer " & token.access_token) |
||||
|
let userIdReq = fmt"/2/users/by?usernames={user}" |
||||
|
var url = fmt"https://api.twitter.com{userIdReq}" |
||||
|
|
||||
|
let userId = client.request(url, httpMethod = HttpGet).body.parseJson{"data"}[0]{"id"}.getStr |
||||
|
|
||||
|
let tweetsReq = fmt"/2/users/{userId}/tweets" |
||||
|
url = fmt"https://api.twitter.com{tweetsReq}" |
||||
|
return client.request(url, httpMethod = HttpGet).body.parseJson |
||||
|
|
||||
|
proc getTweetConvo*(tweetID : string, token : AccessToken) : JsonNode = |
||||
|
# Gets the conversation info for a given tweet |
||||
|
let client = tweetClient("Bearer " & token.access_token) |
||||
|
let userIdReq = fmt"/2/tweets?ids={tweetID}&tweet.fields=conversation_id,author_id" |
||||
|
var url = fmt"https://api.twitter.com{userIdReq}" |
||||
|
|
||||
|
let tweetInfo = client.request(url, httpMethod = HttpGet).body.parseJson |
||||
|
|
||||
|
tweetInfo |
||||
|
|
||||
|
proc getTweet*(tweetID : string, token : AccessToken) : string = |
||||
|
# Grabs a single tweet |
||||
|
# XXX use Tweet type |
||||
|
let client = tweetClient("Bearer " & token.access_token) |
||||
|
let reqTarget = fmt"/1.1/statuses/show.json?id={tweetID}&tweet_mode=extended" |
||||
|
let url = fmt"https://api.twitter.com{reqTarget}" |
||||
|
|
||||
|
client.request(url, httpMethod = HttpGet).body |
||||
|
|
||||
|
proc getHome*(count: int, token : AccessToken) : string = |
||||
|
# Gets your home timeline |
||||
|
let client = tweetClient("Bearer " & token.access_token) |
||||
|
let reqTarget = fmt"/1.1/statuses/user_timeline.json?count={count}&trim_user=1&exclude_replies=1" |
||||
|
let url = fmt"https://api.twitter.com{reqTarget}" |
||||
|
|
||||
|
client.request(url, httpMethod = HttpGet).body |
||||
|
|
||||
|
iterator getThread*(tweetStart : string, token : AccessToken) : Tweet = |
||||
|
|
||||
|
var reqParams : Params # params for the actual API request |
||||
|
|
||||
|
let consumerKey = "TWITTER_CONSUMER_KEY".getEnv |
||||
|
let secret = "TWITTER_CONSUMER_SECRET".getEnv |
||||
|
|
||||
|
var consumerToken = newConsumerToken(consumerKey, secret) |
||||
|
|
||||
|
var twitterAPI = newTwitterAPI(consumerToken, token.access_token, token.access_token_secret) |
||||
|
|
||||
|
reqParams["status"] = "testing123" |
||||
|
reqParams["include_entities"] = "true" |
||||
|
echo fmt"tweetStart = {tweetStart}" |
||||
|
|
||||
|
#var reqTarget = fmt"/2/tweets/search/recent?query=conversation_id:{tweetStart}&tweet.fields=in_reply_to_user_id,author_id,created_at,conversation_id" |
||||
|
var url = "https://api.twitter.com/1.1/statuses/update.json" |
||||
|
|
||||
|
var currentPage : string |
||||
|
|
||||
|
echo fmt"url = {url}" |
||||
|
|
||||
|
# Simply get. |
||||
|
var resp = twitterAPI.get("account/verify_credentials.json") |
||||
|
echo resp.status |
||||
|
|
||||
|
# Using proc corresponding twitter REST APIs. |
||||
|
resp = twitterAPI.statusesUpdate("testing 1 2 3") |
||||
|
echo parseJson(resp.body) |
||||
|
|
||||
|
#while true: |
||||
|
#if currentPage{"meta", "result_count"}.getInt == 0: |
||||
|
#break |
||||
|
#for tweet in currentPage{"data"}: |
||||
|
#yield Tweet( |
||||
|
#id: tweet{"id"}.getStr, |
||||
|
#in_reply: tweet{"in_reply_to_user_id"}.getStr, |
||||
|
#author_id: tweet{"author_id"}.getStr, |
||||
|
#text: tweet{"text"}.getStr, |
||||
|
#created_at: tweet{"created_at"}.getStr, |
||||
|
#conversation_id: tweet{"conversation_id"}.getStr |
||||
|
#) |
||||
|
|
||||
|
#let paginationToken = currentPage{"meta"}{"next_token"} |
||||
|
|
||||
|
#if paginationToken == nil: |
||||
|
#break |
||||
|
|
||||
|
#echo "Getting next page" |
||||
|
|
||||
|
#reqTarget = fmt"/2/tweets/search/recent?query=conversation_id:{tweetStart}&tweet.fields=in_reply_to_user_id,author_id,created_at,conversation_id&next_token={paginationToken.getStr}" |
||||
|
#url = fmt"https://api.twitter.com{reqTarget}" |
||||
|
#currentPage = client.request(url, httpMethod = HttpGet).body.parseJson |
||||
|
|
||||
|
proc convertWords(tweet : Tweet) : string = |
||||
|
let words = tweet.text.split(" ") |
||||
|
var stripped : seq[string] |
||||
|
for chunk in words: |
||||
|
for word in chunk.splitLines: |
||||
|
if word.len > 3 and word[0..3] == "http": |
||||
|
let parsedUri = word.parseUri |
||||
|
let scheme = parsedUri.scheme |
||||
|
let hostname = parsedUri.hostname |
||||
|
let path = parsedUri.path |
||||
|
if (scheme.len > 0 and hostname.len > 0): |
||||
|
let url = xmltree.escape(fmt"{scheme}://{hostname}{path}") |
||||
|
stripped &= url |
||||
|
elif word.len > 0 and word[0] != '@': |
||||
|
stripped &= word |
||||
|
else: |
||||
|
continue |
||||
|
stripped.join(" ") |
||||
|
|
||||
|
proc renderThread*(tweetID : string, token : AccessToken) : Option[seq[string]] = |
||||
|
let thread = toSeq(getThread(tweetID, token)).map(convertWords).map(capitalizeAscii) |
||||
|
echo $thread |
||||
|
if thread.len == 0: |
||||
|
return none(seq[string]) |
||||
|
some(thread) |
||||
Loading…
Reference in new issue