From c6215876faaae5fcd0f41c67e0f2ab00702150dd Mon Sep 17 00:00:00 2001 From: Zed Date: Wed, 29 Apr 2020 18:09:13 +0200 Subject: [PATCH] Implement proper infinite scroll for replies Fixes #125 --- public/js/infiniteScroll.js | 17 ++++++++++++--- src/api/consts.nim | 1 + src/api/media.nim | 13 +++++++++-- src/api/search.nim | 4 ++-- src/api/tweet.nim | 29 +++++++++++++++++++++++++ src/parser.nim | 43 +++++++++++++++++++------------------ src/routes/status.nim | 6 +++++- src/types.nim | 16 +++++++------- src/views/status.nim | 17 +++++++++------ 9 files changed, 102 insertions(+), 44 deletions(-) diff --git a/public/js/infiniteScroll.js b/public/js/infiniteScroll.js index 4208cb0..9090222 100644 --- a/public/js/infiniteScroll.js +++ b/public/js/infiniteScroll.js @@ -6,8 +6,16 @@ function getLoadMore(doc) { return doc.querySelector('.show-more:not(.timeline-item)'); } +function isDuplicate(item, itemClass) { + const tweet = item.querySelector(".tweet-link"); + if (tweet == null) return false; + const href = tweet.getAttribute("href"); + return document.querySelector(itemClass + " .tweet-link[href='" + href + "']") != null; +} + window.onload = function() { - const isTweet = window.location.pathname.indexOf("/status/") !== -1; + const url = window.location.pathname; + const isTweet = url.indexOf("/status/") !== -1; const containerClass = isTweet ? ".replies" : ".timeline"; const itemClass = isTweet ? ".thread-line" : ".timeline-item"; @@ -36,13 +44,16 @@ window.onload = function() { for (var item of doc.querySelectorAll(itemClass)) { if (item.className == "timeline-item show-more") continue; + if (isDuplicate(item, itemClass)) continue; if (isTweet) container.appendChild(item); else insertBeforeLast(container, item); } - if (isTweet) container.appendChild(getLoadMore(doc)); - else insertBeforeLast(container, getLoadMore(doc)); loading = false; + const newLoadMore = getLoadMore(doc); + if (newLoadMore == null) return; + if (isTweet) container.appendChild(newLoadMore); + else insertBeforeLast(container, newLoadMore); }).catch(function (err) { console.warn('Something went wrong.', err); loading = true; diff --git a/src/api/consts.nim b/src/api/consts.nim index 61b7508..9c76162 100644 --- a/src/api/consts.nim +++ b/src/api/consts.nim @@ -17,6 +17,7 @@ const profileIntentUrl* = "intent/user" searchUrl* = "i/search/timeline" tweetUrl* = "status" + repliesUrl* = "i/$1/conversation/$2" videoUrl* = "videos/tweet/config/$1.json" tokenUrl* = "guest/activate.json" cardUrl* = "i/cards/tfw/v1/$1" diff --git a/src/api/media.nim b/src/api/media.nim index 41257d1..0e17d72 100644 --- a/src/api/media.nim +++ b/src/api/media.nim @@ -16,6 +16,7 @@ macro genMediaGet(media: untyped; token=false) = mediaName = capitalizeAscii($media) multi = ident("get" & mediaName & "s") convo = ident("getConversation" & mediaName & "s") + replies = ident("getReplies" & mediaName & "s") single = ident("get" & mediaName) quote do: @@ -29,6 +30,14 @@ macro genMediaGet(media: untyped; token=false) = else: await all(`media`.mapIt(`single`(it, agent))) + proc `replies`*(replies: Result[Chain]; agent: string; token="") {.async.} = + when `token`: + var gToken = token + if gToken.len == 0: gToken = await getGuestToken(agent) + await all(replies.content.mapIt(`multi`(it, agent, token=gToken))) + else: + await all(replies.content.mapIt(`multi`(it, agent))) + proc `convo`*(convo: Conversation; agent: string) {.async.} = var futs: seq[Future[void]] when `token`: @@ -37,13 +46,13 @@ macro genMediaGet(media: untyped; token=false) = futs.add `multi`(convo.before, agent, token=token) futs.add `multi`(convo.after, agent, token=token) if convo.replies != nil: - futs.add convo.replies.content.mapIt(`multi`(it, agent, token=token)) + futs.add `replies`(convo.replies, agent, token=token) else: futs.add `single`(convo.tweet, agent) futs.add `multi`(convo.before, agent) futs.add `multi`(convo.after, agent) if convo.replies != nil: - futs.add convo.replies.content.mapIt(`multi`(it, agent)) + futs.add `replies`(convo.replies, agent) await all(futs) proc getGuestToken(agent: string; force=false): Future[string] {.async.} = diff --git a/src/api/search.nim b/src/api/search.nim index aadb728..af80688 100644 --- a/src/api/search.nim +++ b/src/api/search.nim @@ -8,8 +8,8 @@ proc getResult*[T](json: JsonNode; query: Query; after: string): Result[T] = if json == nil: return Result[T](beginning: true, query: query) Result[T]( hasMore: json{"has_more_items"}.getBool(false), - maxId: json{"max_position"}.getStr(""), - minId: json{"min_position"}.getStr(""), + maxId: json{"max_position"}.getStr, + minId: json{"min_position"}.getStr, query: query, beginning: after.len == 0 ) diff --git a/src/api/tweet.nim b/src/api/tweet.nim index f724134..2ba8df7 100644 --- a/src/api/tweet.nim +++ b/src/api/tweet.nim @@ -30,3 +30,32 @@ proc getTweet*(username, id, after, agent: string): Future[Conversation] {.async await all(getConversationVideos(result, agent), getConversationCards(result, agent), getConversationPolls(result, agent)) + +proc getReplies*(username, id, after, agent: string): Future[Result[Chain]] {.async.} = + let + headers = genHeaders({ + "pragma": "no-cache", + "x-previous-page-name": "permalink", + "accept": htmlAccept + }, agent, base, xml=true) + + params = { + "include_available_features": "1", + "include_entities": "1", + "max_position": after, + } + + url = base / (repliesUrl % [username, id]) ? params + + let json = await fetchJson(url, headers) + if json == nil or not json.hasKey("items_html"): return + let html = parseHtml(json{"items_html"}.getStr) + + result = parseReplies(html) + result.minId = json{"min_position"}.getStr(result.minId) + if result.minId.len > 0: + result.hasMore = true + + await all(getRepliesVideos(result, agent), + getRepliesCards(result, agent), + getRepliesPolls(result, agent)) diff --git a/src/parser.nim b/src/parser.nim index 929f0aa..a48bba2 100644 --- a/src/parser.nim +++ b/src/parser.nim @@ -160,6 +160,19 @@ proc parseChain*(nodes: XmlNode): Chain = else: result.content.add parseTweet(n) +proc parseReplies*(replies: XmlNode; skipFirst=false): Result[Chain] = + new(result) + for i, reply in replies.filterIt(it.kind != xnText): + if skipFirst and i == 0: continue + let class = reply.attr("class").toLower() + if "lone" in class: + result.content.add parseChain(reply) + elif "showmore" in class: + result.minId = reply.selectAttr("button", "data-cursor") + result.hasMore = true + else: + result.content.add parseChain(reply.select(".stream-items")) + proc parseConversation*(node: XmlNode; after: string): Conversation = let tweet = node.select(".permalink-tweet-container") @@ -169,11 +182,6 @@ proc parseConversation*(node: XmlNode; after: string): Conversation = result = Conversation( tweet: parseTweet(tweet), before: parseChain(node.select(".in-reply-to .stream-items")), - replies: Result[Chain]( - minId: node.selectAttr(".replies-to .stream-container", "data-min-position"), - hasMore: node.select(".stream-footer .has-more-items") != nil, - beginning: after.len == 0 - ) ) if result.before != nil: @@ -181,26 +189,19 @@ proc parseConversation*(node: XmlNode; after: string): Conversation = if maxId.len > 0: result.before.more = -1 - let showMore = node.selectAttr(".ThreadedConversation-showMoreThreads button", - "data-cursor") - - if showMore.len > 0: - result.replies.minId = showMore - result.replies.hasMore = true - let replies = node.select(".replies-to .stream-items") if replies == nil: return - for i, reply in replies.filterIt(it.kind != xnText): - let class = reply.attr("class").toLower() - let thread = reply.select(".stream-items") + let nodes = replies.filterIt(it.kind != xnText and "self" in it.attr("class")) + if nodes.len > 0 and "self" in nodes[0].attr("class"): + result.after = parseChain(nodes[0].select(".stream-items")) - if i == 0 and "self" in class: - result.after = parseChain(thread) - elif "lone" in class: - result.replies.content.add parseChain(reply) - else: - result.replies.content.add parseChain(thread) + result.replies = parseReplies(replies, result.after != nil) + + result.replies.beginning = after.len == 0 + if result.replies.minId.len == 0: + result.replies.minId = node.selectAttr(".replies-to .stream-container", "data-min-position") + result.replies.hasMore = node.select(".stream-footer .has-more-items") != nil proc parseTimeline*(node: XmlNode; after: string): Timeline = if node == nil: return Timeline() diff --git a/src/routes/status.nim b/src/routes/status.nim index 25948f5..109266a 100644 --- a/src/routes/status.nim +++ b/src/routes/status.nim @@ -1,6 +1,6 @@ import asyncdispatch, strutils, sequtils, uri, options -import jester +import jester, karax/vdom import router_utils import ".."/[api, types, formatters, agents] @@ -17,6 +17,10 @@ proc createStatusRouter*(cfg: Config) = cond '.' notin @"name" let prefs = cookiePrefs() + if @"scroll".len > 0: + let replies = await getReplies(@"name", @"id", @"max_position", getAgent()) + resp $renderReplies(replies, prefs, getPath()) + let conversation = await getTweet(@"name", @"id", @"max_position", getAgent()) if conversation == nil or conversation.tweet.id == 0: var error = "Tweet not found" diff --git a/src/types.nim b/src/types.nim index 80a8a50..b0a96a0 100644 --- a/src/types.nim +++ b/src/types.nim @@ -77,14 +77,6 @@ type near*: string sep*: string - Result*[T] = ref object - content*: seq[T] - minId*: string - maxId*: string - hasMore*: bool - beginning*: bool - query*: Query - Gif* = object url*: string thumb*: string @@ -166,6 +158,14 @@ type photos*: seq[string] poll*: Option[Poll] + Result*[T] = ref object + content*: seq[T] + minId*: string + maxId*: string + hasMore*: bool + beginning*: bool + query*: Query + Chain* = ref object content*: seq[Tweet] more*: int64 diff --git a/src/views/status.nim b/src/views/status.nim index 81311e2..ad5c83f 100644 --- a/src/views/status.nim +++ b/src/views/status.nim @@ -29,6 +29,15 @@ proc renderReplyThread(thread: Chain; prefs: Prefs; path: string): VNode = if thread.more != 0: renderMoreReplies(thread) +proc renderReplies*(replies: Result[Chain]; prefs: Prefs; path: string): VNode = + buildHtml(tdiv(class="replies", id="r")): + for thread in replies.content: + if thread == nil: continue + renderReplyThread(thread, prefs, path) + + if replies.hasMore: + renderMore(Query(), replies.minId, focus="#r") + proc renderConversation*(conversation: Conversation; prefs: Prefs; path: string): VNode = let hasAfter = conversation.after != nil let showReplies = not prefs.hideReplies @@ -60,10 +69,4 @@ proc renderConversation*(conversation: Conversation; prefs: Prefs; path: string) renderNewer(Query(), getLink(conversation.tweet)) if conversation.replies.content.len > 0 and showReplies: - tdiv(class="replies", id="r"): - for thread in conversation.replies.content: - if thread == nil: continue - renderReplyThread(thread, prefs, path) - - if conversation.replies.hasMore and showReplies: - renderMore(Query(), conversation.replies.minId, focus="#r") + renderReplies(conversation.replies, prefs, path)