Wesley Kerfoot
3 years ago
committed by
GitHub
7 changed files with 410 additions and 151 deletions
@ -1,20 +1,21 @@ |
|||
import tweetlogpkg/twitter, tweetlogpkg/server |
|||
import tweetlogpkg/twitter_api, tweetlogpkg/server |
|||
import threadpool |
|||
|
|||
import tweetlogpkg/auth |
|||
import httpClient, base64, uri, json, os, strformat, sequtils, strutils, options |
|||
import timezones, times |
|||
|
|||
from xmltree import escape |
|||
|
|||
when isMainModule: |
|||
echo "Running" |
|||
for tweet in "1355971359168466945".getThread: |
|||
echo "" |
|||
echo tweet.text |
|||
echo "" |
|||
#echo "weskerfoot".listTweets |
|||
#echo 10.getHome |
|||
#for tweet in "1355971359168466945".getThread: |
|||
#echo "" |
|||
#echo tweet.text |
|||
#echo "" |
|||
|
|||
#for tweet in "strivev4".listTweets2(){"data"}: |
|||
#echo tweet |
|||
#echo tweet{"id"}.getStr.getTweetConvo |
|||
#spawn handleRenders() |
|||
#startServer() |
|||
spawn handleRenders() |
|||
startServer() |
|||
|
@ -0,0 +1,177 @@ |
|||
import httpClient, base64, uri, json, os, strformat, sequtils, strutils, options, sugar, times, types |
|||
import tables, algorithm, base64, math, options |
|||
import nimcrypto |
|||
import threadpool |
|||
|
|||
proc realEncodeUrl*(s: string): string = |
|||
## Exclude A..Z a..z 0..9 - . _ ~ |
|||
## See https://dev.twitter.com/oauth/overview/percent-encoding-parameters |
|||
result = newStringOfCap(s.len + s.len shr 2) # assume 12% non-alnum-chars |
|||
for i in 0..s.len-1: |
|||
case s[i] |
|||
of 'a'..'z', 'A'..'Z', '0'..'9', '_', '-', '.', '~': |
|||
add(result, s[i]) |
|||
else: |
|||
add(result, '%') |
|||
add(result, toHex(ord(s[i]), 2)) |
|||
|
|||
proc tweetClient*(token : string) : HttpClient = |
|||
var client = newHttpClient() |
|||
|
|||
client.headers = newHttpHeaders( |
|||
{ |
|||
"Authorization" : token |
|||
} |
|||
) |
|||
client |
|||
|
|||
# 3-legged OAuth stuff |
|||
|
|||
proc parseQueryString(qstr : string) : Table[string, string] = |
|||
toTable( |
|||
map(qstr.split("&"), |
|||
proc(pair : string) : tuple[a: string, b: string] = |
|||
let split = pair.split("=") |
|||
(split[0], split[1]))) |
|||
|
|||
proc generateNonce() : string {.gcsafe.} = |
|||
let alphabet = map(toSeq(65..90).concat( |
|||
toSeq(97..122)).concat( |
|||
toSeq(49..57)), (c) {.gcsafe.} => char(c)) |
|||
|
|||
var randBytes : array[50, uint8] |
|||
discard randomBytes(randBytes) |
|||
return map(randBytes, (c) => alphabet[(c.int %% alphabet.len)]).join |
|||
|
|||
proc constructEncodedString(params : Params, sep : string, include_quotes : bool) : string = |
|||
var encodedPairs : seq[string] = @[] |
|||
var keyPairs : seq[tuple[key: string, value: string]] = toSeq(params.pairs) |
|||
|
|||
keyPairs.sort((a, b) => cmp(a.key.realEncodeUrl, b.key.realEncodeUrl)) |
|||
|
|||
for pair in keyPairs: |
|||
if include_quotes: |
|||
encodedPairs &= pair[0] & "=" & "\"" & pair[1].realEncodeUrl & "\"" |
|||
else: |
|||
encodedPairs &= pair[0] & "=" & pair[1].realEncodeUrl |
|||
|
|||
encodedPairs.join(sep) |
|||
|
|||
proc constructParameterString(params : Params) : string = |
|||
result = params.constructEncodedString("&", include_quotes=false) |
|||
echo fmt"parameter string = {result}" |
|||
|
|||
proc constructHeaderString(params : Params) : string = |
|||
"OAuth " & params.constructEncodedString(", ", include_quotes=true) |
|||
|
|||
proc sign(reqMethod : string, |
|||
paramString : string, |
|||
baseUrl : string) : string = |
|||
|
|||
let sigBaseString : string = reqMethod.toUpperAscii & "&" & baseUrl.realEncodeUrl & "&" & paramString.realEncodeUrl |
|||
var signingKey : string |
|||
|
|||
signingKey = getEnv("TWITTER_CONSUMER_SECRET").realEncodeUrl & "&" |
|||
|
|||
result = sha1.hmac(signingKey, sigBaseString).data.encode |
|||
|
|||
proc signRequest*(requestUrl : string, |
|||
requestMethod : string) : Params = |
|||
# Return params along with signature that signs request for API request |
|||
let oauth_consumer_key = getEnv("TWITTER_CONSUMER_KEY") |
|||
let oauth_nonce = generateNonce() |
|||
|
|||
result["oauth_callback"] = getEnv("TWITTER_OAUTH_CALLBACK") |
|||
|
|||
# The twitter documentation uses SHA1, but this works and is future-proof |
|||
let oauth_signature_method = "HMAC-SHA1".realEncodeUrl |
|||
let oauth_timestamp : string = $trunc(epochTime()).uint64 |
|||
|
|||
result["oauth_nonce"] = oauth_nonce |
|||
result["oauth_signature_method"] = oauth_signature_method |
|||
result["oauth_timestamp"] = oauth_timestamp |
|||
result["oauth_consumer_key"] = oauth_consumer_key |
|||
result["oauth_version"] = "1.0" |
|||
|
|||
let paramString = result.constructParameterString |
|||
|
|||
result["oauth_signature"] = sign(requestMethod, paramString, requestUrl) |
|||
|
|||
proc getAuthRequestSigned(requestUrl : string) : Params = |
|||
signRequest(requestUrl, "POST") |
|||
|
|||
proc requestToken*(requestUrl : string) : Option[OAuthToken] = |
|||
var headers = newHttpHeaders([]) |
|||
|
|||
let params = getAuthRequestSigned(requestUrl) |
|||
let client = tweetClient(params.constructHeaderString) |
|||
let resp = client.request(requestUrl, httpMethod = HttpPost, headers = headers, body = "") |
|||
|
|||
if resp.status != "200 OK": |
|||
echo resp.body |
|||
return none(OAuthToken) |
|||
|
|||
let keyPairs = resp.body.parseQueryString |
|||
|
|||
if not (keyPairs.hasKey("oauth_token") and keyPairs.hasKey("oauth_token_secret")): |
|||
return none(OAuthToken) |
|||
|
|||
some((oauth_token: keyPairs["oauth_token"], oauth_token_secret: keyPairs["oauth_token_secret"])) |
|||
|
|||
proc getTokenRedirect*() : string = |
|||
let req = "https://api.twitter.com/oauth/request_token".requestToken |
|||
fmt"https://api.twitter.com/oauth/authenticate?oauth_token={req.get.oauth_token}" |
|||
|
|||
proc getAccessToken*(oauth_token : string, oauth_verifier : string) : Option[AccessToken] = |
|||
var params : Params |
|||
let client = tweetClient(params.constructHeaderString) |
|||
let requestUrl = fmt"https://api.twitter.com/oauth/access_token?oauth_token={oauth_token}&oauth_verifier={oauth_verifier}" |
|||
let resp = client.request(requestUrl, httpMethod = HttpPost) |
|||
|
|||
if resp.status != "200 OK": |
|||
return none(AccessToken) |
|||
|
|||
let keyPairs = resp.body.parseQueryString |
|||
|
|||
some((access_token: keyPairs["oauth_token"], |
|||
access_token_secret: keyPairs["oauth_token_secret"], |
|||
screen_name: keyPairs["screen_name"], |
|||
user_id: keyPairs["user_id"])) |
|||
|
|||
import jwt |
|||
|
|||
proc generateJWT*(token : AccessToken) : string = |
|||
# take an access token and return an encrypted JWT |
|||
let secret = getEnv("JWT_SECRET") |
|||
var encoded = toJWT(%*{ |
|||
"header" : { |
|||
"alg" : "HS256", |
|||
"typ" : "JWT" |
|||
}, |
|||
"claims" : { |
|||
"access_token" : token.access_token, |
|||
"access_token_secret" : token.access_token_secret, |
|||
"screen_name" : token.screen_name, |
|||
"user_id" : token.user_id |
|||
} |
|||
}) |
|||
|
|||
encoded.sign(secret) |
|||
$encoded |
|||
|
|||
proc verify*(token: string): bool = |
|||
let secret = getEnv("JWT_SECRET") |
|||
try: |
|||
let jwtToken = token.toJWT() |
|||
result = jwtToken.verify(secret, HS256) |
|||
except InvalidToken: |
|||
result = false |
|||
|
|||
proc decode*(token: string): Option[AccessToken] = |
|||
if not token.verify: |
|||
return none(AccessToken) |
|||
let claims = token.toJWT().claims |
|||
some((access_token: claims["access_token"].node.str, |
|||
access_token_secret: claims["access_token_secret"].node.str, |
|||
screen_name: claims["screen_name"].node.str, |
|||
user_id: claims["user_id"].node.str)) |
@ -1,135 +0,0 @@ |
|||
import httpClient, base64, uri, json, os, strformat, sequtils, strutils, options |
|||
import timezones, times |
|||
import types |
|||
|
|||
from xmltree import escape |
|||
|
|||
proc parseTwitterTS(ts : string) : DateTime = |
|||
ts.parse("ddd MMM dd hh:mm:ss YYYY") |
|||
|
|||
# echo "Sun Feb 16 18:19:17 +0000 2020".parseTwitterTS.repr |
|||
|
|||
proc buildAuthHeader() : string = |
|||
let consumerKey = "TWITTER_CONSUMER_KEY".getEnv |
|||
let secret = "TWITTER_CONSUMER_SECRET".getEnv |
|||
"Basic " & (consumerKey.encodeUrl & ":" & secret.encodeUrl).encode |
|||
|
|||
proc getToken*() : string = |
|||
var client = newHttpClient() |
|||
client.headers = newHttpHeaders( |
|||
{ |
|||
"Content-Type" : "application/x-www-form-urlencoded;charset=UTF-8", |
|||
"Authorization" : buildAuthHeader() |
|||
} |
|||
) |
|||
|
|||
let body = "grant_type=client_credentials" |
|||
|
|||
let response = client.request("https://api.twitter.com/oauth2/token", |
|||
httpMethod = HttpPost, |
|||
body = body).body.parseJson |
|||
|
|||
let responseType = response["token_type"].getStr |
|||
|
|||
assert(responseType == "bearer") |
|||
|
|||
"Bearer " & response["access_token"].getStr |
|||
|
|||
proc tweetClient() : HttpClient = |
|||
var client = newHttpClient() |
|||
client.headers = newHttpHeaders( |
|||
{ |
|||
"Authorization" : getToken() |
|||
} |
|||
) |
|||
client |
|||
|
|||
proc listTweets2*(user : string) : JsonNode = |
|||
let client = tweetClient() |
|||
let userIdReq = fmt"/2/users/by?usernames={user}" |
|||
var url = fmt"https://api.twitter.com{userIdReq}" |
|||
|
|||
let userId = client.request(url, httpMethod = HttpGet).body.parseJson{"data"}[0]{"id"}.getStr |
|||
|
|||
let tweetsReq = fmt"/2/users/{userId}/tweets" |
|||
url = fmt"https://api.twitter.com{tweetsReq}" |
|||
return client.request(url, httpMethod = HttpGet).body.parseJson |
|||
|
|||
proc getTweetConvo*(tweetID : string) : JsonNode = |
|||
let client = tweetClient() |
|||
let userIdReq = fmt"/2/tweets?ids={tweetID}&tweet.fields=conversation_id,author_id" |
|||
var url = fmt"https://api.twitter.com{userIdReq}" |
|||
|
|||
let tweetInfo = client.request(url, httpMethod = HttpGet).body.parseJson |
|||
|
|||
tweetInfo |
|||
|
|||
proc listTweets*(user : string) : JsonNode = |
|||
let client = tweetClient() |
|||
let reqTarget = fmt"/1.1/statuses/user_timeline.json?count=100&screen_name={user}" |
|||
let url = fmt"https://api.twitter.com{reqTarget}" |
|||
|
|||
client.request(url, httpMethod = HttpGet).body.parseJson |
|||
|
|||
proc getTweet*(tweetID : string) : string = |
|||
let client = tweetClient() |
|||
let reqTarget = fmt"/1.1/statuses/show.json?id={tweetID}&tweet_mode=extended" |
|||
let url = fmt"https://api.twitter.com{reqTarget}" |
|||
|
|||
client.request(url, httpMethod = HttpGet).body |
|||
|
|||
iterator getThread*(tweetStart : string) : Tweet = |
|||
let client = tweetClient() |
|||
var reqTarget = fmt"/2/tweets/search/recent?query=conversation_id:{tweetStart}&tweet.fields=in_reply_to_user_id,author_id,created_at,conversation_id" |
|||
var url = fmt"https://api.twitter.com{reqTarget}" |
|||
|
|||
var currentPage : JsonNode |
|||
|
|||
currentPage = client.request(url, httpMethod = HttpGet).body.parseJson |
|||
|
|||
while true: |
|||
if currentPage{"meta", "result_count"}.getInt == 0: |
|||
break |
|||
for tweet in currentPage{"data"}: |
|||
yield Tweet( |
|||
id: tweet{"id"}.getStr, |
|||
in_reply: tweet{"in_reply_to_user_id"}.getStr, |
|||
author_id: tweet{"author_id"}.getStr, |
|||
text: tweet{"text"}.getStr, |
|||
created_at: tweet{"created_at"}.getStr, |
|||
conversation_id: tweet{"conversation_id"}.getStr |
|||
) |
|||
|
|||
let paginationToken = currentPage{"meta"}{"next_token"} |
|||
|
|||
if paginationToken == nil: |
|||
break |
|||
|
|||
reqTarget = fmt"/2/tweets/search/recent?query=conversation_id:{tweetStart}&tweet.fields=in_reply_to_user_id,author_id,created_at,conversation_id&next_token={paginationToken.getStr}" |
|||
url = fmt"https://api.twitter.com{reqTarget}" |
|||
currentPage = client.request(url, httpMethod = HttpGet).body.parseJson |
|||
|
|||
proc convertWords(tweet : Tweet) : string = |
|||
let words = tweet.text.split(" ") |
|||
var stripped : seq[string] |
|||
for chunk in words: |
|||
for word in chunk.splitLines: |
|||
if word.len > 3 and word[0..3] == "http": |
|||
let parsedUri = word.parseUri |
|||
let scheme = parsedUri.scheme |
|||
let hostname = parsedUri.hostname |
|||
let path = parsedUri.path |
|||
if (scheme.len > 0 and hostname.len > 0): |
|||
let url = xmltree.escape(fmt"{scheme}://{hostname}{path}") |
|||
stripped &= url |
|||
elif word.len > 0 and word[0] != '@': |
|||
stripped &= word |
|||
else: |
|||
continue |
|||
stripped.join(" ") |
|||
|
|||
proc renderThread*(tweetID : string) : Option[seq[string]] = |
|||
let thread = toSeq(getThread(tweetID)).map(convertWords).map(capitalizeAscii) |
|||
if thread.len == 0: |
|||
return none(seq[string]) |
|||
some(thread) |
@ -0,0 +1,129 @@ |
|||
import httpClient, uri, json, os, strformat, sequtils, strutils, options, sugar, types |
|||
import tables, options, times, twitter |
|||
import auth |
|||
|
|||
from nimcrypto.sysrand import randomBytes |
|||
from xmltree import escape |
|||
|
|||
proc parseTwitterTS(ts : string) : DateTime = |
|||
ts.parse("ddd MMM dd hh:mm:ss YYYY") |
|||
|
|||
proc listTweets*(user : string, token : AccessToken) : JsonNode = |
|||
# Lists tweets from a given user |
|||
# XXX use Tweet type |
|||
let client = tweetClient("Bearer " & token.access_token) |
|||
let userIdReq = fmt"/2/users/by?usernames={user}" |
|||
var url = fmt"https://api.twitter.com{userIdReq}" |
|||
|
|||
let userId = client.request(url, httpMethod = HttpGet).body.parseJson{"data"}[0]{"id"}.getStr |
|||
|
|||
let tweetsReq = fmt"/2/users/{userId}/tweets" |
|||
url = fmt"https://api.twitter.com{tweetsReq}" |
|||
return client.request(url, httpMethod = HttpGet).body.parseJson |
|||
|
|||
proc getTweetConvo*(tweetID : string, token : AccessToken) : JsonNode = |
|||
# Gets the conversation info for a given tweet |
|||
let client = tweetClient("Bearer " & token.access_token) |
|||
let userIdReq = fmt"/2/tweets?ids={tweetID}&tweet.fields=conversation_id,author_id" |
|||
var url = fmt"https://api.twitter.com{userIdReq}" |
|||
|
|||
let tweetInfo = client.request(url, httpMethod = HttpGet).body.parseJson |
|||
|
|||
tweetInfo |
|||
|
|||
proc getTweet*(tweetID : string, token : AccessToken) : string = |
|||
# Grabs a single tweet |
|||
# XXX use Tweet type |
|||
let client = tweetClient("Bearer " & token.access_token) |
|||
let reqTarget = fmt"/1.1/statuses/show.json?id={tweetID}&tweet_mode=extended" |
|||
let url = fmt"https://api.twitter.com{reqTarget}" |
|||
|
|||
client.request(url, httpMethod = HttpGet).body |
|||
|
|||
proc getHome*(count: int, token : AccessToken) : string = |
|||
# Gets your home timeline |
|||
let client = tweetClient("Bearer " & token.access_token) |
|||
let reqTarget = fmt"/1.1/statuses/user_timeline.json?count={count}&trim_user=1&exclude_replies=1" |
|||
let url = fmt"https://api.twitter.com{reqTarget}" |
|||
|
|||
client.request(url, httpMethod = HttpGet).body |
|||
|
|||
iterator getThread*(tweetStart : string, token : AccessToken) : Tweet = |
|||
|
|||
var reqParams : Params # params for the actual API request |
|||
|
|||
let consumerKey = "TWITTER_CONSUMER_KEY".getEnv |
|||
let secret = "TWITTER_CONSUMER_SECRET".getEnv |
|||
|
|||
var consumerToken = newConsumerToken(consumerKey, secret) |
|||
|
|||
var twitterAPI = newTwitterAPI(consumerToken, token.access_token, token.access_token_secret) |
|||
|
|||
reqParams["status"] = "testing123" |
|||
reqParams["include_entities"] = "true" |
|||
echo fmt"tweetStart = {tweetStart}" |
|||
|
|||
#var reqTarget = fmt"/2/tweets/search/recent?query=conversation_id:{tweetStart}&tweet.fields=in_reply_to_user_id,author_id,created_at,conversation_id" |
|||
var url = "https://api.twitter.com/1.1/statuses/update.json" |
|||
|
|||
var currentPage : string |
|||
|
|||
echo fmt"url = {url}" |
|||
|
|||
# Simply get. |
|||
var resp = twitterAPI.get("account/verify_credentials.json") |
|||
echo resp.status |
|||
|
|||
# Using proc corresponding twitter REST APIs. |
|||
resp = twitterAPI.statusesUpdate("testing 1 2 3") |
|||
echo parseJson(resp.body) |
|||
|
|||
#while true: |
|||
#if currentPage{"meta", "result_count"}.getInt == 0: |
|||
#break |
|||
#for tweet in currentPage{"data"}: |
|||
#yield Tweet( |
|||
#id: tweet{"id"}.getStr, |
|||
#in_reply: tweet{"in_reply_to_user_id"}.getStr, |
|||
#author_id: tweet{"author_id"}.getStr, |
|||
#text: tweet{"text"}.getStr, |
|||
#created_at: tweet{"created_at"}.getStr, |
|||
#conversation_id: tweet{"conversation_id"}.getStr |
|||
#) |
|||
|
|||
#let paginationToken = currentPage{"meta"}{"next_token"} |
|||
|
|||
#if paginationToken == nil: |
|||
#break |
|||
|
|||
#echo "Getting next page" |
|||
|
|||
#reqTarget = fmt"/2/tweets/search/recent?query=conversation_id:{tweetStart}&tweet.fields=in_reply_to_user_id,author_id,created_at,conversation_id&next_token={paginationToken.getStr}" |
|||
#url = fmt"https://api.twitter.com{reqTarget}" |
|||
#currentPage = client.request(url, httpMethod = HttpGet).body.parseJson |
|||
|
|||
proc convertWords(tweet : Tweet) : string = |
|||
let words = tweet.text.split(" ") |
|||
var stripped : seq[string] |
|||
for chunk in words: |
|||
for word in chunk.splitLines: |
|||
if word.len > 3 and word[0..3] == "http": |
|||
let parsedUri = word.parseUri |
|||
let scheme = parsedUri.scheme |
|||
let hostname = parsedUri.hostname |
|||
let path = parsedUri.path |
|||
if (scheme.len > 0 and hostname.len > 0): |
|||
let url = xmltree.escape(fmt"{scheme}://{hostname}{path}") |
|||
stripped &= url |
|||
elif word.len > 0 and word[0] != '@': |
|||
stripped &= word |
|||
else: |
|||
continue |
|||
stripped.join(" ") |
|||
|
|||
proc renderThread*(tweetID : string, token : AccessToken) : Option[seq[string]] = |
|||
let thread = toSeq(getThread(tweetID, token)).map(convertWords).map(capitalizeAscii) |
|||
echo $thread |
|||
if thread.len == 0: |
|||
return none(seq[string]) |
|||
some(thread) |
Loading…
Reference in new issue