From fb591e43b8c485f61946495447178ef9048e9c6d Mon Sep 17 00:00:00 2001 From: Zed Date: Tue, 2 Jun 2020 16:22:44 +0200 Subject: [PATCH] Switch to packedjson to try lowering memory usage --- nitter.nimble | 3 +-- src/api.nim | 3 ++- src/apiutils.nim | 14 ++++++++------ src/parser.nim | 25 +++++++++++++------------ src/parserutils.nim | 25 +++++++++++++------------ 5 files changed, 37 insertions(+), 33 deletions(-) diff --git a/nitter.nimble b/nitter.nimble index bd8680b..6ee8d5e 100644 --- a/nitter.nimble +++ b/nitter.nimble @@ -11,10 +11,8 @@ bin = @["nitter"] # Dependencies requires "nim >= 1.2.0" -requires "norm#head" requires "jester#head" requires "regex#2d96bab" -requires "q >= 0.0.7" requires "nimcrypto >= 0.4.11" requires "karax >= 1.1.2" requires "sass" @@ -22,6 +20,7 @@ requires "markdown#head" requires "https://github.com/zedeus/redis#head" requires "redpool#head" requires "msgpack4nim >= 0.3.1" +requires "packedjson" # Tasks diff --git a/src/api.nim b/src/api.nim index 9d8d048..44a69a9 100644 --- a/src/api.nim +++ b/src/api.nim @@ -1,4 +1,5 @@ -import asyncdispatch, httpclient, uri, strutils, json +import asyncdispatch, httpclient, uri, strutils +import packedjson import types, query, formatters, consts, apiutils, parser proc getGraphProfile*(username: string): Future[Profile] {.async.} = diff --git a/src/apiutils.nim b/src/apiutils.nim index db8015f..ea00bc1 100644 --- a/src/apiutils.nim +++ b/src/apiutils.nim @@ -1,5 +1,6 @@ -import httpclient, asyncdispatch, options, times, strutils, json, uri -import types, agents, tokens, consts +import httpclient, asyncdispatch, options, times, strutils, uri +import packedjson +import types, agents, tokens, consts, parserutils proc genParams*(pars: openarray[(string, string)] = @[]; cursor=""): seq[(string, string)] = @@ -41,19 +42,20 @@ proc fetch*(url: Uri; retried=false; oldApi=false): Future[JsonNode] {.async.} = if resp.status != $Http200: if "Bad guest token" in body: - return + keepToken = false + return newJNull() elif not body.startsWith('{'): echo resp.status, " ", body - return + return newJNull() result = parseJson(body) - if result{"errors"} != nil and result.getError == forbidden: + if result{"errors"}.notNull and result.getError == forbidden: keepToken = false echo "bad token" except: echo "error: ", url - result = nil + result = newJNull() finally: if keepToken: token.release() diff --git a/src/parser.nim b/src/parser.nim index f6726a2..563b515 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -1,8 +1,9 @@ -import json, strutils, options, tables, times, math +import strutils, options, tables, times, math +import packedjson import types, parserutils proc parseProfile(js: JsonNode; id=""): Profile = - if js == nil: return + if js.isNull: return result = Profile( id: if id.len > 0: id else: js{"id_str"}.getStr, username: js{"screen_name"}.getStr, @@ -24,7 +25,7 @@ proc parseProfile(js: JsonNode; id=""): Profile = result.expandProfileEntities(js) proc parseUserShow*(js: JsonNode; username: string): Profile = - if js == nil: return + if js.isNull: return with error, js{"errors"}: result = Profile(username: username) if error.getError == suspended: @@ -34,7 +35,7 @@ proc parseUserShow*(js: JsonNode; username: string): Profile = result = parseProfile(js) proc parseGraphProfile*(js: JsonNode; username: string): Profile = - if js == nil: return + if js.isNull: return with error, js{"errors"}: result = Profile(username: username) if error.getError == suspended: @@ -46,12 +47,12 @@ proc parseGraphProfile*(js: JsonNode; username: string): Profile = parseProfile(user, id) proc parseGraphList*(js: JsonNode): List = - if js == nil: return + if js.isNull: return var list = js{"data", "user_by_screen_name", "list"} - if list == nil: + if list.isNull: list = js{"data", "list"} - if list == nil: + if list.isNull: return result = List( @@ -70,7 +71,7 @@ proc parseListMembers*(js: JsonNode; cursor: string): Result[Profile] = query: Query(kind: userList) ) - if js == nil: return + if js.isNull: return result.top = js{"previous_cursor_str"}.getStr result.bottom = js{"next_cursor_str"}.getStr @@ -116,7 +117,7 @@ proc parseVideo(js: JsonNode): Video = for v in js{"video_info", "variants"}: result.variants.add VideoVariant( - videoType: v{"content_type"}.to(VideoType), + videoType: parseEnum[VideoType](v{"content_type"}.getStr("summary")), bitrate: v{"bitrate"}.getInt, url: v{"url"}.getStr ) @@ -192,14 +193,14 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card = break proc parseTweet(js: JsonNode): Tweet = - if js == nil: return + if js.isNull: return result = Tweet( id: js{"id_str"}.getId, threadId: js{"conversation_id_str"}.getId, replyId: js{"in_reply_to_status_id_str"}.getId, text: js{"full_text"}.getStr, time: js{"created_at"}.getTime, - hasThread: js{"self_thread"} != nil, + hasThread: js{"self_thread"}.notNull, available: true, profile: Profile(id: js{"user_id_str"}.getStr), stats: TweetStats( @@ -354,7 +355,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline = if instructions.len == 0: return for i in instructions: - if result.beginning and i{"pinEntry"} != nil: + if result.beginning and i{"pinEntry"}.notNull: with pin, parsePin(i, global): result.content.add pin else: diff --git a/src/parserutils.nim b/src/parserutils.nim index 0ee78fa..d29db85 100644 --- a/src/parserutils.nim +++ b/src/parserutils.nim @@ -1,5 +1,5 @@ -import json, strutils, times, macros, htmlgen, uri, unicode, options -import regex +import strutils, times, macros, htmlgen, uri, unicode, options +import regex, packedjson import types, utils, formatters const @@ -11,9 +11,12 @@ const let localTimezone = local() +template isNull*(js: JsonNode): bool = js.kind == JNull +template notNull*(js: JsonNode): bool = js.kind != JNull + template `?`*(js: JsonNode): untyped = let j = js - if j == nil: return + if j.isNull: return else: j template `with`*(ident, value, body): untyped = @@ -24,8 +27,7 @@ template `with`*(ident, value, body): untyped = template `with`*(ident; value: JsonNode; body): untyped = block: let ident {.inject.} = value - if ident != nil and ident.kind != JNull: - body + if value.notNull: body template getCursor*(js: JsonNode): string = js{"content", "operation", "cursor", "value"}.getStr @@ -50,7 +52,6 @@ proc getId*(id: string): string {.inline.} = id[start + 1 ..< id.len] proc getId*(js: JsonNode): int64 {.inline.} = - if js == nil: return case js.kind of JString: return parseBiggestInt(js.getStr("0")) of JInt: return js.getBiggestInt() @@ -117,10 +118,9 @@ template getSlice(text: string; slice: seq[int]): string = text.runeSubStr(slice[0], slice[1] - slice[0]) proc getSlice(text: string; js: JsonNode): string = - if js == nil or js.kind != JArray or js.len < 2 or - js[0].kind != JInt: return text + if js.kind != JArray or js.len < 2 or js[0].kind != JInt: return text - let slice = js.to(seq[int]) + let slice = @[js{0}.getInt, js{1}.getInt] text.getSlice(slice) proc expandUrl(text: var string; js: JsonNode; tLen: int; hideTwitter=false) = @@ -130,9 +130,9 @@ proc expandUrl(text: var string; js: JsonNode; tLen: int; hideTwitter=false) = let url = js{"expanded_url"}.getStr - slice = js{"indices"}.to(seq[int]) + slice = js{"indices"}[1].getInt - if hideTwitter and slice[1] >= tLen and url.isTwitterUrl: + if hideTwitter and slice >= tLen and url.isTwitterUrl: text = text.replace(u, "") text.removeSuffix(' ') text.removeSuffix('\n') @@ -178,7 +178,8 @@ proc expandProfileEntities*(profile: var Profile; js: JsonNode) = proc expandTweetEntities*(tweet: Tweet; js: JsonNode) = let orig = tweet.text - slice = js{"display_text_range"}.to(seq[int]) + textRange = js{"display_text_range"} + slice = @[textRange{0}.getInt, textRange{1}.getInt] hasQuote = js{"is_quote_status"}.getBool hasCard = tweet.card.isSome