Merge 7f3426ce21 into 95893eedaa

src/api.nim

@@ -51,6 +51,12 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
   let url = graphListMembers ? {"variables": $variables}
   result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
 
+proc getGraphArticle*(id: string): Future[Article] {.async.} =
+  let
+    variables = %*{"twitterArticleId": id}
+    url = graphArticle ? {"variables": $variables}
+  result = parseGraphArticle(await fetch(url, Api.userRestId))
+
 proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} =
   if id.len == 0: return
   let

src/consts.nim

@@ -25,6 +25,7 @@ const
   graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
   graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug"
   graphListMembers* = graphql / "Ke6urWMeCV2UlKXGRy4sow/ListMembers"
+  graphArticle* = graphql / "rJMGbcr9LTsjVycjUmcnEg/TwitterArticleByRestId"
 
   timelineParams* = {
     "include_profile_interstitial_type": "0",

src/formatters.nim

@@ -26,6 +26,8 @@ let
   userPicRegex = re"_(normal|bigger|mini|200x200|400x400)(\.[A-z]+)$"
   extRegex = re"(\.[A-z]+)$"
   illegalXmlRegex = re"(*UTF8)[^\x09\x0A\x0D\x20-\x{D7FF}\x{E000}-\x{FFFD}\x{10000}-\x{10FFFF}]"
+  hashtagRegex = re"\B#(\w*[A-Za-z]\w*)\b"
+  mentionRegex = re"\B@(\w{1,15})\b"
 
 proc getUrlPrefix*(cfg: Config): string =
   if cfg.useHttps: https & cfg.hostname
@@ -72,6 +74,13 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
   if absolute.len > 0 and "href" in result:
     result = result.replace("href=\"/", &"href=\"{absolute}/")
 
+proc replaceHashtagsAndMentions*(body: string): string =
+  result = body
+  result = result.replacef(hashtagRegex, a(
+    "#$1", href = "/search?q=%23$1"))
+  result = result.replacef(mentionRegex, a(
+    "@$1", href = "/$1"))
+
 proc getM3u8Url*(content: string): string =
   var matches: array[1, string]
   if re.find(content, m3u8Regex, matches) != -1:
@@ -121,14 +130,14 @@ proc getTime*(tweet: Tweet): string =
 proc getRfc822Time*(tweet: Tweet): string =
   tweet.time.format("ddd', 'dd MMM yyyy HH:mm:ss 'GMT'")
 
-proc getShortTime*(tweet: Tweet): string =
+proc getShortTime*(time: DateTime): string =
   let now = now()
-  let since = now - tweet.time
+  let since = now - time
 
-  if now.year != tweet.time.year:
-    result = tweet.time.format("d MMM yyyy")
+  if now.year != time.year:
+    result = time.format("d MMM yyyy")
   elif since.inDays >= 1:
-    result = tweet.time.format("MMM d")
+    result = time.format("MMM d")
   elif since.inHours >= 1:
     result = $since.inHours & "h"
   elif since.inMinutes >= 1:

src/nitter.nim

@@ -10,7 +10,7 @@ import types, config, prefs, formatters, redis_cache, http_pool, tokens
 import views/[general, about]
 import routes/[
   preferences, timeline, status, media, search, rss, list, debug,
-  unsupported, embed, resolver, router_utils]
+  unsupported, embed, notes, resolver, router_utils]
 
 const instancesUrl = "https://github.com/zedeus/nitter/wiki/Instances"
 const issuesUrl = "https://github.com/zedeus/nitter/issues"
@@ -48,6 +48,7 @@ createListRouter(cfg)
 createStatusRouter(cfg)
 createSearchRouter(cfg)
 createMediaRouter(cfg)
+createNotesRouter(cfg)
 createEmbedRouter(cfg)
 createRssRouter(cfg)
 createDebugRouter(cfg)
@@ -100,4 +101,5 @@ routes:
   extend status, ""
   extend media, ""
   extend embed, ""
+  extend notes, ""
   extend debug, ""

src/parser.nim

@@ -435,3 +435,71 @@ proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
         result.replies.content.add thread
     elif entryId.startsWith("cursor-bottom"):
       result.replies.bottom = e{"content", "itemContent", "value"}.getStr
+
+proc parseGraphArticle*(js: JsonNode): Article =
+  if not js{"errors"}.isNull:
+    return
+
+  let article = js{"data", "twitterArticle"}
+  let meta = article{"metadata"}
+
+  result = Article(
+    title: article{"title"}.getStr,
+    coverImage: article{"cover_image", "media_info", "original_img_url"}.getStr,
+    user: meta{"authorResults", "result", "legacy"}.parseUser,
+    time: meta{"publishedAtMs"}.getStr.parseInt.div(1000).fromUnix.utc,
+  )
+
+  let
+    content = article{"data", "contentStateJson"}.getStr.parseJson
+  
+  for p in content{"blocks"}:
+    var paragraph = ArticleParagraph(
+      text: p{"text"}.getStr,
+      baseType: parseEnum[ArticleType](p{"type"}.getStr)
+    )
+    for sr in p{"inlineStyleRanges"}:
+      paragraph.inlineStyleRanges.add ArticleStyleRange(
+        offset: sr{"offset"}.getInt,
+        length: sr{"length"}.getInt,
+        style: parseEnum[ArticleStyle](sr{"style"}.getStr)
+      )
+    for er in p{"entityRanges"}:
+      paragraph.entityRanges.add ArticleEntityRange(
+        offset: er{"offset"}.getInt,
+        length: er{"length"}.getInt,
+        key: er{"key"}.getInt
+      )
+    result.paragraphs.add paragraph
+  
+  # Note: This is a map but the indices are integers so it's fine.
+  for _, jEntity in content{"entityMap"}:
+    var entity = ArticleEntity(
+      entityType: parseEnum[ArticleEntityType] jEntity{"type"}.getStr,
+    )
+    case entity.entityType
+    of ArticleEntityType.link:
+      entity.url = jEntity{"data", "url"}.getStr
+    of ArticleEntityType.media:
+      for jMedia in jEntity{"data", "mediaItems"}:
+        entity.mediaIds.add jMedia{"mediaId"}.getStr
+    of ArticleEntityType.tweet:
+      entity.tweetId = jEntity{"data", "tweetId"}.getStr
+    of ArticleEntityType.twemoji:
+      entity.twemoji = jEntity{"data", "url"}.getStr
+    else: discard
+
+    result.entities.add entity
+
+  for m in article{"media"}:
+    let mediaInfo = m{"media_info"}
+    var media = ArticleMedia(
+      mediaType: parseEnum[ArticleMediaType](mediaInfo{"__typename"}.getStr)
+    )
+    case media.mediaType
+    of ArticleMediaType.image:
+      media.url = mediaInfo{"original_img_url"}.getStr
+    of ArticleMediaType.gif:
+      media.url = mediaInfo{"variants"}[0]{"url"}.getStr
+    else: discard
+    result.media[m{"media_id"}.getStr] = media

src/routes/notes.nim

@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import asyncdispatch, tables, asyncfutures, options
+import jester, karax/vdom
+import ".."/[types, api]
+import ../views/[notes, tweet, general]
+import router_utils
+
+export api, notes, vdom, tweet, general, router_utils
+
+proc createNotesRouter*(cfg: Config) =
+  router notes:
+    get "/i/notes/@id":
+      let article = await getGraphArticle(@"id")
+
+      if article == nil:
+        resp Http404
+
+      var tweetFutures: seq[Future[Conversation]]
+      for e in article.entities:
+        if e.entityType == ArticleEntityType.tweet:
+          tweetFutures.add getTweet(e.tweetId)
+
+      let convs = await tweetFutures.all
+
+      var tweets = initTable[int64, Tweet]()
+      for c in convs:
+        if c != nil and c.tweet != nil:
+          tweets[c.tweet.id] = c.tweet
+
+      let
+        path = getPath()
+        prefs = cookiePrefs()
+        note = renderNote(article, tweets, path, prefs)
+      resp renderMain(note, request, cfg, prefs, titleText=article.title)

src/routes/unsupported.nim

@@ -21,5 +21,5 @@ proc createUnsupportedRouter*(cfg: Config) =
       feature()
 
     get "/i/@i?/?@j?":
-      cond @"i" notin ["status", "lists" , "user"]
+      cond @"i" notin ["status", "lists" , "user", "notes"]
       feature()

src/sass/index.scss

@@ -7,6 +7,7 @@
 @import 'inputs';
 @import 'timeline';
 @import 'search';
+@import 'note';
 
 body {
     // colors

src/sass/note.scss

@@ -0,0 +1,60 @@
+.note {
+    width: 600px;
+    margin: 0 auto;
+    background-color: var(--bg_panel);
+    font-family: sans-serif;
+
+    img.cover {
+        margin: 0;
+        width: 100%;
+    }
+
+    article {
+        padding: 20px;
+
+        &>h1 {
+            display: inherit;
+            font-size: 2.5rem;
+            margin: 30px 0;
+        }
+
+        &>p,
+        li {
+            font-size: 18px;
+        }
+
+        &>p {
+            line-height: 1.5em;
+            margin: 30px 0;
+            word-wrap: break-word;
+            white-space: break-spaces;
+        }
+
+        &>span.image {
+            text-align: center;
+            width: 100%;
+
+            img,
+            video {
+                max-width: 100%;
+                border-radius: 20px;
+                margin: 0 auto;
+            }
+        }
+
+        img.twemoji {
+            width: 18px;
+            height: 18px;
+        }
+
+        &>ul>li,
+        &>ol>li {
+            line-height: 2em;
+        }
+
+        &>iframe {
+            width: 100%;
+            height: 400px;
+        }
+    }
+}

src/types.nim

@@ -38,6 +38,7 @@ type
     protectedUser = 22
     couldntAuth = 32
     doesntExist = 34
+    invalidPermission = 37
     userNotFound = 50
     suspended = 63
     rateLimited = 88
@@ -120,6 +121,70 @@ type
 
   PhotoRail* = seq[GalleryPhoto]
 
+  Article* = ref object
+    title*: string
+    coverImage*: string
+    user*: User
+    time*: DateTime
+    paragraphs*: seq[ArticleParagraph]
+    entities*: seq[ArticleEntity]
+    media*: Table[string, ArticleMedia]
+
+  ArticleParagraph* = object
+    text*: string
+    baseType*: ArticleType
+    inlineStyleRanges*: seq[ArticleStyleRange]
+    entityRanges*: seq[ArticleEntityRange]
+  
+  ArticleType* {.pure.} = enum
+    headerOne = "header-one"
+    headerTwo = "header-two"
+    headerThree = "header-three"
+    orderedListItem = "ordered-list-item"
+    unorderedListItem = "unordered-list-item"
+    unstyled = "unstyled"
+    atomic = "atomic"
+    unknown
+
+  ArticleStyleRange* = object
+    offset*: int
+    length*: int
+    style*: ArticleStyle
+  
+  ArticleStyle* {.pure.} = enum
+    bold = "BOLD"
+    italic = "ITALIC"
+    strikethrough = "STRIKETHROUGH"
+    unknown
+
+  ArticleEntityRange* = object
+    offset*: int
+    length*: int
+    key*: int
+
+  ArticleEntity* = object
+    entityType*: ArticleEntityType
+    url*: string
+    mediaIds*: seq[string]
+    tweetId*: string
+    twemoji*: string
+
+  ArticleEntityType* {.pure.} = enum
+    link = "LINK"
+    media = "MEDIA"
+    tweet = "TWEET"
+    twemoji = "TWEMOJI"
+    unknown
+
+  ArticleMedia* = object
+    mediaType*: ArticleMediaType
+    url*: string
+
+  ArticleMediaType* {.pure.} = enum
+    image = "ApiImage"
+    gif = "ApiGif"
+    unknown
+
   Poll* = object
     options*: seq[string]
     values*: seq[int]

src/utils.nim

@@ -15,6 +15,7 @@ const
     "pic.twitter.com",
     "twimg.com",
     "abs.twimg.com",
+    "abs-0.twimg.com",
     "pbs.twimg.com",
     "video.twimg.com"
   ]

src/views/general.nim

@@ -52,7 +52,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
   let opensearchUrl = getUrlPrefix(cfg) & "/opensearch"
 
   buildHtml(head):
-    link(rel="stylesheet", type="text/css", href="/css/style.css?v=18")
+    link(rel="stylesheet", type="text/css", href="/css/style.css?v=19")
     link(rel="stylesheet", type="text/css", href="/css/fontello.css?v=2")
 
     if theme.len > 0:

src/views/notes.nim

@@ -0,0 +1,195 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+import strutils, tables, unicode, bitops
+import karax/[karaxdsl, vdom]
+from jester import Request
+
+import renderutils, tweet
+import ".."/[types, utils, formatters]
+
+const doctype = "<!DOCTYPE html>\n"
+
+proc getSmallPic(url: string): string =
+  result = url
+  if "?" notin url and not url.endsWith("placeholder.png"):
+    result &= "?name=small"
+  result = getPicUrl(result)
+
+proc renderMiniAvatar(user: User; prefs: Prefs): VNode =
+  let url = getPicUrl(user.getUserPic("_mini"))
+  buildHtml():
+    img(class=(prefs.getAvatarClass & " mini"), src=url)
+
+proc renderNoteParagraph(articleParagraph: ArticleParagraph; article: Article; tweets: Table[int64, Tweet]; path: string; prefs: Prefs): VNode =
+  if articleParagraph.baseType == ArticleType.atomic:
+    let er = articleParagraph.entityRanges[0]
+    let entity = article.entities[er.key]
+
+    case entity.entityType
+    of ArticleEntityType.media:
+      for id in entity.mediaIds:
+        let media = article.media.getOrDefault(id)
+        if media.url == "":
+          discard
+        case media.mediaType:
+        of ArticleMediaType.image:
+          let image = buildHtml(span(class="image")):
+            img(src=media.url.getSmallPic, alt="")
+          result = image
+        of ArticleMediaType.gif:
+          let video = buildHtml(span(class="image")):
+            video(src=media.url.getVidUrl, controls="", autoplay="", loop="")
+          result = video
+        else: discard
+    of ArticleEntityType.tweet:
+      let tweet = tweets.getOrDefault(entity.tweetId.parseInt, nil)
+      if tweet == nil: discard
+      result = renderTweet(tweet, prefs, path, mainTweet=true)
+    else: discard
+  else:
+    let text = articleParagraph.text
+
+    case articleParagraph.baseType
+    of ArticleType.headerOne:
+      result = h1.newVNode()
+    of ArticleType.headerTwo:
+      result = h2.newVNode()
+    of ArticleType.headerThree:
+      result = h3.newVNode()
+    of ArticleType.orderedListItem:
+      result = li.newVNode()
+    of ArticleType.unorderedListItem:
+      result = li.newVNode()
+    of ArticleType.atomic:
+      result = nil
+    else:
+      result = p.newVNode()
+
+    proc flushPlainText(target: VNode; start: int; len: int): void =
+      if articleParagraph.inlineStyleRanges.len == 0:
+        target.add verbatim text.runeSubStr(start, len).replaceHashtagsAndMentions
+      else:
+
+        proc flushInternal(start: int, len: int, style: int): void =
+          let content = text.runeSubStr(start, len).replaceHashtagsAndMentions
+          if style == 0:
+            target.add text content
+          else:
+            let container = span.newVNode()
+            container.add text content
+            var styleStr = ""
+            if style.testBit(0):
+              styleStr.add "font-weight:bold;"
+            if style.testBit(1):
+              styleStr.add "font-style:italic;"
+            if style.testBit(2):
+              styleStr.add "text-decoration:line-through;"
+            container.setAttr("style", styleStr)
+            target.add container
+
+        var
+          lastStyle = 0
+          lastStart = start
+
+        for i in start..(start + len):
+          var style = 0
+          for styleRange in articleParagraph.inlineStyleRanges:
+            let
+              styleStart = styleRange.offset
+              styleEnd = styleStart + styleRange.length
+            if styleStart <= i and styleEnd > i:
+              case styleRange.style:
+              of ArticleStyle.bold:
+                style.setBit(0)
+              of ArticleStyle.italic:
+                style.setBit(1)
+              of ArticleStyle.strikethrough:
+                style.setBit(2)
+              else: discard
+          
+          if style != lastStyle:
+            if i > lastStart:
+              flushInternal(lastStart, i - lastStart, lastStyle)
+
+            lastStyle = style
+            lastStart = i
+            
+        if lastStart < len:
+          flushInternal(lastStart, len - lastStart, lastStyle)
+
+    var last = 0
+    for er in articleParagraph.entityRanges:
+      # prevent karax from inserting whitespaces to fix wrapping
+      result.add text ""
+
+      if er.offset > last:
+        flushPlainText(result, last, er.offset - last)
+
+      let entity = article.entities[er.key]
+      case entity.entityType
+      of ArticleEntityType.link:
+        let link = buildHtml(a(href=entity.url)):
+          text text.runeSubStr(er.offset, er.length)
+        result.add link
+      of ArticleEntityType.twemoji:
+        let url = entity.twemoji.getSmallPic
+        let emoji = buildHtml(img(class="twemoji", src=url, alt=""))
+        result.add emoji
+      else: discard
+
+      last = er.offset + er.length
+
+    # flush remaining text
+    if last < text.len:
+      flushPlainText(result, last, text.len - last)
+
+proc renderNote*(article: Article; tweets: Table[int64, Tweet]; path: string; prefs: Prefs): VNode =
+  let cover = getSmallPic(article.coverImage)
+  let author = article.user
+
+  # build header
+  let main = buildHtml(article):
+    h1: text article.title
+    
+    tdiv(class="author"):
+      renderMiniAvatar(author, prefs)
+      linkUser(author, class="fullname")
+      linkUser(author, class="username")
+      text " · "
+      text article.time.getShortTime
+
+  # add paragraphs
+  var listType = ArticleType.unknown
+  var list: VNode = nil
+
+  proc flushList() =
+    if list != nil:
+      main.add list
+      list = nil
+      listType = ArticleType.unknown
+
+  for paragraph in article.paragraphs:
+    let node = renderNoteParagraph(paragraph, article, tweets, path, prefs)
+    
+    let currentType = paragraph.baseType
+    if currentType in [ArticleType.orderedListItem, ArticleType.unorderedListItem]:
+      if currentType != listType:
+        flushList()
+
+        case currentType:
+        of ArticleType.orderedListItem:
+          list = ol.newVNode()
+        of ArticleType.unorderedListItem:
+          list = ul.newVNode()
+        else: discard
+        listType = currentType
+      list.add node
+    else:
+      flushList()
+      main.add node
+  
+  flushList()
+
+  buildHtml(tdiv(class="note")):
+    img(class="cover", src=(cover), alt="")
+    
+    main

src/views/tweet.nim

@@ -38,7 +38,7 @@ proc renderHeader(tweet: Tweet; retweet: string; prefs: Prefs): VNode =
 
         span(class="tweet-date"):
           a(href=getLink(tweet), title=tweet.getTime):
-            text tweet.getShortTime
+            text tweet.time.getShortTime
 
 proc renderAlbum(tweet: Tweet): VNode =
   let
@@ -253,7 +253,7 @@ proc renderQuote(quote: Tweet; prefs: Prefs; path: string): VNode =
 
       span(class="tweet-date"):
         a(href=getLink(quote), title=quote.getTime):
-          text quote.getShortTime
+          text quote.time.getShortTime
 
     if quote.reply.len > 0:
       renderReply(quote)