From 3da6c50cb8cc8cfa2c361465fa5099fa86d80d68 Mon Sep 17 00:00:00 2001 From: Zed Date: Sun, 26 Dec 2021 06:49:27 +0100 Subject: [PATCH] Use gzip for API calls to lower bandwidth and RAM --- nitter.nimble | 1 + src/apiutils.nim | 5 +++-- src/http_pool.nim | 3 ++- src/tokens.nim | 4 +++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/nitter.nimble b/nitter.nimble index d8ffa98..48f9b74 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -22,6 +22,7 @@ requires "supersnappy#2.1.1" requires "redpool#f880f49" requires "https://github.com/zedeus/redis#d0a0e6f" requires "https://github.com/disruptek/frosty#0.3.1" +requires "zippy#0.7.3" # Tasks diff --git a/src/apiutils.nim b/src/apiutils.nim index 6b2b83f..cee5ab5 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -1,5 +1,5 @@ import httpclient, asyncdispatch, options, times, strutils, uri -import packedjson +import packedjson, zippy import types, tokens, consts, parserutils, http_pool const rl = "x-rate-limit-" @@ -26,6 +26,7 @@ proc genHeaders*(token: Token = nil): HttpHeaders = "x-guest-token": if token == nil: "" else: token.tok, "x-twitter-active-user": "yes", "authority": "api.twitter.com", + "accept-encoding": "gzip", "accept-language": "en-US,en;q=0.9", "accept": "*/*", "DNT": "1" @@ -44,7 +45,7 @@ proc fetch*(url: Uri; oldApi=false): Future[JsonNode] {.async.} = var resp: AsyncResponse let body = pool.use(headers): resp = await c.get($url) - await resp.body + uncompress(await resp.body) if body.startsWith('{') or body.startsWith('['): result = parseJson(body) diff --git a/src/http_pool.nim b/src/http_pool.nim index 2daf030..9fb8457 100644 --- a/src/http_pool.nim +++ b/src/http_pool.nim @@ -7,7 +7,8 @@ type var maxConns {.threadvar.}: int let keepAlive* = newHttpHeaders({ - "Connection": "Keep-Alive" + "connection": "Keep-Alive", + "accept-encoding": "gzip" }) proc setMaxHttpConns*(n: int) = diff --git a/src/tokens.nim b/src/tokens.nim index 1cf35cf..9ce7bcf 100644 --- a/src/tokens.nim +++ b/src/tokens.nim @@ -1,5 +1,6 @@ import asyncdispatch, httpclient, times, sequtils, json, math, random import strutils, strformat +import zippy import types, agents, consts, http_pool const @@ -28,6 +29,7 @@ proc fetchToken(): Future[Token] {.async.} = let headers = newHttpHeaders({ "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "accept-encoding": "gzip", "accept-language": "en-US,en;q=0.5", "connection": "keep-alive", "user-agent": getAgent(), @@ -41,7 +43,7 @@ proc fetchToken(): Future[Token] {.async.} = try: resp = clientPool.use(headers): await c.postContent(activate) - tokNode = parseJson(resp)["guest_token"] + tokNode = parseJson(uncompress(resp))["guest_token"] tok = tokNode.getStr($(tokNode.getInt)) let time = getTime()