nitter/src/apiutils.nim

130 lines
3.8 KiB
Nim
Raw Normal View History

2021-12-27 01:37:38 +00:00
# SPDX-License-Identifier: AGPL-3.0-only
2022-01-16 17:28:40 +00:00
import httpclient, asyncdispatch, options, strutils, uri
2022-01-16 05:00:11 +00:00
import jsony, packedjson, zippy
import types, tokens, consts, parserutils, http_pool
2022-01-16 17:28:40 +00:00
import experimental/types/common
2022-01-05 21:48:45 +00:00
const
rlRemaining = "x-rate-limit-remaining"
rlReset = "x-rate-limit-reset"
2020-06-01 00:16:24 +00:00
2022-01-02 10:21:03 +00:00
var pool: HttpPool
2022-01-06 02:57:14 +00:00
proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
2020-06-16 22:20:34 +00:00
count="20"; ext=true): seq[(string, string)] =
2020-06-01 00:16:24 +00:00
result = timelineParams
for p in pars:
result &= p
2020-06-16 22:20:34 +00:00
if ext:
result &= ("ext", "mediaStats")
result &= ("include_ext_alt_text", "true")
result &= ("include_ext_media_availability", "true")
2020-06-16 22:20:34 +00:00
if count.len > 0:
result &= ("count", count)
2021-12-28 07:07:15 +00:00
if cursor.len > 0:
# The raw cursor often has plus signs, which sometimes get turned into spaces,
2022-12-19 09:07:24 +00:00
# so we need to turn them back into a plus
2021-12-28 07:07:15 +00:00
if " " in cursor:
result &= ("cursor", cursor.replace(" ", "+"))
else:
result &= ("cursor", cursor)
2020-06-01 00:16:24 +00:00
proc genHeaders*(token: Token = nil): HttpHeaders =
2020-06-01 00:16:24 +00:00
result = newHttpHeaders({
"connection": "keep-alive",
2020-06-01 00:16:24 +00:00
"authorization": auth,
"content-type": "application/json",
"x-guest-token": if token == nil: "" else: token.tok,
"x-twitter-active-user": "yes",
"authority": "api.twitter.com",
"accept-encoding": "gzip",
2020-06-01 00:16:24 +00:00
"accept-language": "en-US,en;q=0.9",
"accept": "*/*",
"DNT": "1"
2020-06-01 00:16:24 +00:00
})
2022-01-16 05:00:11 +00:00
template updateToken() =
if api != Api.search and resp.headers.hasKey(rlRemaining):
let
remaining = parseInt(resp.headers[rlRemaining])
reset = parseInt(resp.headers[rlReset])
token.setRateLimit(api, remaining, reset)
template fetchImpl(result, fetchBody) {.dirty.} =
once:
pool = HttpPool()
2022-01-05 21:48:45 +00:00
var token = await getToken(api)
2020-07-09 07:18:14 +00:00
if token.tok.len == 0:
2021-01-07 21:31:29 +00:00
raise rateLimitError()
2020-06-01 00:16:24 +00:00
try:
2022-01-23 01:29:03 +00:00
var resp: AsyncResponse
pool.use(genHeaders(token)):
template getContent =
resp = await c.get($url)
result = await resp.body
getContent()
# Twitter randomly returns 401 errors with an empty body quite often.
# Retrying the request usually works.
2022-12-19 09:07:24 +00:00
if resp.status == "401 Unauthorized" and result.len == 0:
getContent()
if resp.status == $Http503:
badClient = true
raise newException(InternalError, result)
2021-12-30 00:39:00 +00:00
2022-01-16 05:00:11 +00:00
if result.len > 0:
2021-12-30 00:39:00 +00:00
if resp.headers.getOrDefault("content-encoding") == "gzip":
2022-01-16 05:00:11 +00:00
result = uncompress(result, dfGzip)
2021-12-30 00:39:00 +00:00
else:
2022-01-16 05:00:11 +00:00
echo "non-gzip body, url: ", url, ", body: ", result
2022-01-16 05:00:11 +00:00
fetchBody
2020-06-01 00:16:24 +00:00
2022-01-16 05:00:11 +00:00
release(token, used=true)
2021-12-28 04:41:41 +00:00
if resp.status == $Http400:
raise newException(InternalError, $url)
except InternalError as e:
raise e
except Exception as e:
2021-12-28 04:41:41 +00:00
echo "error: ", e.name, ", msg: ", e.msg, ", token: ", token[], ", url: ", url
if "length" notin e.msg and "descriptor" notin e.msg:
release(token, invalid=true)
2021-01-07 21:31:29 +00:00
raise rateLimitError()
2022-01-16 05:00:11 +00:00
proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
var body: string
fetchImpl body:
if body.startsWith('{') or body.startsWith('['):
result = parseJson(body)
else:
2022-01-23 07:38:38 +00:00
echo resp.status, ": ", body, " --- url: ", url
2022-01-16 05:00:11 +00:00
result = newJNull()
updateToken()
let error = result.getError
if error in {invalidToken, badToken}:
2022-01-16 05:00:11 +00:00
echo "fetch error: ", result.getError
release(token, invalid=true)
raise rateLimitError()
proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
fetchImpl result:
if not (result.startsWith('{') or result.startsWith('[')):
2022-01-23 07:38:38 +00:00
echo resp.status, ": ", result, " --- url: ", url
2022-01-16 05:00:11 +00:00
result.setLen(0)
updateToken()
if result.startsWith("{\"errors"):
2022-01-16 17:28:40 +00:00
let errors = result.fromJson(Errors)
if errors in {invalidToken, badToken}:
2022-01-16 05:00:11 +00:00
echo "fetch error: ", errors
release(token, invalid=true)
raise rateLimitError()