mirror of
https://github.com/zedeus/nitter.git
synced 2024-06-02 13:29:25 +00:00
295 lines
9.2 KiB
Nim
295 lines
9.2 KiB
Nim
import xmltree, times, uri, options, json
|
|
import strtabs, strformat, strutils, sequtils
|
|
import regex
|
|
|
|
import types, formatters
|
|
|
|
from q import nil
|
|
from htmlgen import a
|
|
|
|
const
|
|
thumbRegex = re".+:url\('([^']+)'\)"
|
|
gifRegex = re".+thumb/([^\.']+)\.[jpng].*"
|
|
reColor = re"a:active \{\n +color: (#[A-Z0-9]+)"
|
|
|
|
proc selectAll*(node: XmlNode; selector: string): seq[XmlNode] =
|
|
if node == nil: return
|
|
q.select(node, selector)
|
|
|
|
proc select*(node: XmlNode; selector: string): XmlNode =
|
|
if node == nil: return
|
|
let nodes = node.selectAll(selector)
|
|
if nodes.len > 0: nodes[0] else: nil
|
|
|
|
proc selectAttr*(node: XmlNode; selector: string; attr: string): string =
|
|
let res = node.select(selector)
|
|
if res == nil: "" else: res.attr(attr)
|
|
|
|
proc selectText*(node: XmlNode; selector: string): string =
|
|
let res = node.select(selector)
|
|
result = if res == nil: "" else: res.innerText()
|
|
|
|
proc getHeader(profile: XmlNode): XmlNode =
|
|
result = profile.select(".permalink-header")
|
|
if result == nil:
|
|
result = profile.select(".stream-item-header")
|
|
if result == nil:
|
|
result = profile.select(".ProfileCard-userFields")
|
|
if result == nil:
|
|
result = profile
|
|
|
|
proc isVerified*(profile: XmlNode): bool =
|
|
getHeader(profile).select(".Icon.Icon--verified") != nil
|
|
|
|
proc isProtected*(profile: XmlNode): bool =
|
|
getHeader(profile).select(".Icon.Icon--protected") != nil
|
|
|
|
proc parseText*(text: XmlNode; skipLink=""): string =
|
|
if text == nil: return
|
|
for el in text:
|
|
case el.kind
|
|
of xnText:
|
|
result.add el
|
|
of xnElement:
|
|
if el.attrs == nil:
|
|
if el.tag == "strong":
|
|
result.add $el
|
|
continue
|
|
|
|
let class = el.attr("class")
|
|
if "data-expanded-url" in el.attrs:
|
|
let url = el.attr("data-expanded-url")
|
|
if url == skipLink: continue
|
|
if "u-hidden" in class and result.len > 0:
|
|
result.add "\n"
|
|
result.add a(shortLink(url), href=url)
|
|
elif "ashtag" in class or "hashflag" in class:
|
|
let hash = el.innerText()
|
|
result.add a(hash, href=("/search?q=" & encodeUrl(hash)))
|
|
elif "atreply" in class:
|
|
result.add a(el.innerText(), href=el.attr("href"))
|
|
elif "Emoji" in class:
|
|
result.add el.attr("alt")
|
|
else: discard
|
|
|
|
proc getQuoteText*(tweet: XmlNode): string =
|
|
parseText(tweet.select(".QuoteTweet-text"))
|
|
|
|
proc getTweetText*(tweet: XmlNode): string =
|
|
let
|
|
quote = tweet.select(".QuoteTweet")
|
|
text = tweet.select(".tweet-text")
|
|
link = text.selectAttr("a.twitter-timeline-link.u-hidden", "data-expanded-url")
|
|
parseText(text, if quote != nil: link else: "")
|
|
|
|
proc getTimestamp*(tweet: XmlNode): Time =
|
|
let time = tweet.selectAttr(".js-short-timestamp", "data-time")
|
|
fromUnix(if time.len > 0: parseBiggestInt(time) else: 0)
|
|
|
|
proc getShortTime*(tweet: XmlNode): string =
|
|
tweet.selectText(".js-short-timestamp")
|
|
|
|
proc getDate*(node: XmlNode; selector: string): Time =
|
|
let date = node.select(selector)
|
|
if date == nil: return
|
|
parseTime(date.attr("title"), "h:mm tt - d MMM YYYY", utc())
|
|
|
|
proc getName*(profile: XmlNode; selector: string): string =
|
|
profile.selectText(selector).stripText()
|
|
|
|
proc getUsername*(profile: XmlNode; selector: string): string =
|
|
profile.selectText(selector).strip(chars={'@', ' ', '\n'})
|
|
|
|
proc getBio*(profile: XmlNode; selector: string; fallback=""): string =
|
|
var bio = profile.select(selector)
|
|
if bio == nil and fallback.len > 0:
|
|
bio = profile.select(fallback)
|
|
parseText(bio)
|
|
|
|
proc getLocation*(profile: XmlNode): string =
|
|
let sel = ".ProfileHeaderCard-locationText"
|
|
result = profile.selectText(sel).stripText()
|
|
|
|
let link = profile.selectAttr(sel & " a", "data-place-id")
|
|
if link.len > 0:
|
|
result &= ":" & link
|
|
|
|
proc getAvatar*(profile: XmlNode; selector: string): string =
|
|
profile.selectAttr(selector, "src").getUserpic()
|
|
|
|
proc getBanner*(node: XmlNode): string =
|
|
let url = node.selectAttr("svg > image", "xlink:href")
|
|
if url.len > 0:
|
|
result = url.replace("600x200", "1500x500")
|
|
else:
|
|
result = node.selectAttr(".ProfileCard-bg", "style")
|
|
result = result.replace("background-color: ", "")
|
|
|
|
if result.len == 0:
|
|
result = "#161616"
|
|
|
|
proc getTimelineBanner*(node: XmlNode): string =
|
|
let banner = node.select(".ProfileCanopy-headerBg img")
|
|
let img = banner.attr("src")
|
|
if img.len > 0:
|
|
return img
|
|
|
|
let style = node.select("style").innerText()
|
|
var m: RegexMatch
|
|
if style.find(reColor, m):
|
|
return style[m.group(0)[0]]
|
|
|
|
proc getMediaCount*(node: XmlNode): string =
|
|
let text = node.selectText(".PhotoRail-headingWithCount")
|
|
return text.stripText().split(" ")[0]
|
|
|
|
proc getProfileStats*(profile: var Profile; node: XmlNode) =
|
|
for s in node.selectAll( ".ProfileNav-stat"):
|
|
let text = s.attr("title").split(" ")[0]
|
|
case s.attr("data-nav")
|
|
of "followers": profile.followers = text
|
|
of "following": profile.following = text
|
|
of "favorites": profile.likes = text
|
|
of "tweets": profile.tweets = text
|
|
|
|
proc getPopupStats*(profile: var Profile; node: XmlNode) =
|
|
for s in node.selectAll( ".ProfileCardStats-statLink"):
|
|
let text = s.attr("title").split(" ")[0]
|
|
case s.attr("href").split("/")[^1]
|
|
of "followers": profile.followers = text
|
|
of "following": profile.following = text
|
|
else: profile.tweets = text
|
|
|
|
proc getIntentStats*(profile: var Profile; node: XmlNode) =
|
|
profile.tweets = "?"
|
|
for s in node.selectAll( "dd.count > a"):
|
|
let text = s.innerText()
|
|
case s.attr("href").split("/")[^1]
|
|
of "followers": profile.followers = text
|
|
of "following": profile.following = text
|
|
|
|
proc parseTweetStats*(node: XmlNode): TweetStats =
|
|
result = TweetStats()
|
|
for action in node.selectAll(".ProfileTweet-actionCountForAria"):
|
|
let text = action.innerText.split()
|
|
case text[1][0 .. 2]
|
|
of "ret": result.retweets = text[0].parseInt
|
|
of "rep": result.replies = text[0].parseInt
|
|
of "lik": result.likes = text[0].parseInt
|
|
|
|
proc parseTweetReply*(node: XmlNode): seq[string] =
|
|
let reply = node.select(".ReplyingToContextBelowAuthor")
|
|
if reply == nil: return
|
|
|
|
let selector = if "Quote" in node.attr("class"): "b"
|
|
else: "a b"
|
|
|
|
result = reply.selectAll(selector).map(innerText)
|
|
|
|
proc getGif(player: XmlNode): Gif =
|
|
let
|
|
thumb = player.attr("style").replace(thumbRegex, "$1")
|
|
id = thumb.replace(gifRegex, "$1")
|
|
url = &"https://video.twimg.com/tweet_video/{id}.mp4"
|
|
Gif(url: url, thumb: thumb)
|
|
|
|
proc getTweetMedia*(tweet: Tweet; node: XmlNode) =
|
|
for photo in node.selectAll(".AdaptiveMedia-photoContainer"):
|
|
tweet.photos.add photo.attrs["data-image-url"]
|
|
|
|
let player = node.select(".PlayableMedia")
|
|
if player == nil: return
|
|
|
|
let attrib = player.select(".PlayableMedia-attribution")
|
|
if attrib != nil:
|
|
tweet.attribution = some Profile(
|
|
username: attrib.attr("href").strip(chars={'/'}),
|
|
fullname: attrib.selectText(".fullname"),
|
|
userpic: attrib.selectAttr(".avatar", "src")
|
|
)
|
|
|
|
if "gif" in player.attr("class"):
|
|
tweet.gif = some getGif(player.select(".PlayableMedia-player"))
|
|
elif "video" in player.attr("class"):
|
|
let
|
|
thumb = player.selectAttr(".PlayableMedia-player", "style").split("'")
|
|
desc = player.selectText(".PlayableMedia-description")
|
|
title = player.selectText(".PlayableMedia-title")
|
|
var video = Video(title: title, description: desc)
|
|
if thumb.len > 1:
|
|
video.thumb = thumb[^2]
|
|
tweet.video = some video
|
|
|
|
proc getQuoteMedia*(quote: var Quote; node: XmlNode) =
|
|
if node.select(".QuoteTweet--sensitive") != nil:
|
|
quote.sensitive = true
|
|
return
|
|
|
|
let media = node.select(".QuoteMedia")
|
|
if media != nil:
|
|
quote.thumb = media.selectAttr("img", "src")
|
|
|
|
let badge = node.select(".AdaptiveMedia-badgeText")
|
|
let gifBadge = node.select(".Icon--gifBadge")
|
|
|
|
if badge != nil:
|
|
quote.badge = badge.innerText()
|
|
elif gifBadge != nil:
|
|
quote.badge = "GIF"
|
|
|
|
proc getTweetCard*(tweet: Tweet; node: XmlNode) =
|
|
if node.attr("data-has-cards") == "false": return
|
|
var cardType = node.attr("data-card2-type")
|
|
|
|
if ":" in cardType:
|
|
cardType = cardType.split(":")[^1]
|
|
|
|
if "poll" in cardType:
|
|
tweet.poll = some Poll()
|
|
return
|
|
|
|
if "message_me" in cardType:
|
|
return
|
|
|
|
let cardDiv = node.select(".card2 > .js-macaw-cards-iframe-container")
|
|
if cardDiv == nil: return
|
|
|
|
var card = Card(
|
|
id: $tweet.id,
|
|
query: cardDiv.attr("data-src")
|
|
)
|
|
|
|
try:
|
|
card.kind = parseEnum[CardKind](cardType)
|
|
except ValueError:
|
|
card.kind = summary
|
|
|
|
let cardUrl = cardDiv.attr("data-card-url")
|
|
for n in node.selectAll(".tweet-text a"):
|
|
if n.attr("href") == cardUrl:
|
|
card.url = n.attr("data-expanded-url")
|
|
|
|
tweet.card = some card
|
|
|
|
proc getMoreReplies*(node: XmlNode): int64 =
|
|
let text = node.innerText().strip()
|
|
try:
|
|
result = parseBiggestInt(text.split(" ")[0])
|
|
except:
|
|
result = -1
|
|
|
|
proc getMediaTags*(node: XmlNode): seq[Profile] =
|
|
let usernames = node.attr("data-tagged")
|
|
if usernames.len == 0: return
|
|
let users = parseJson(node.attr("data-reply-to-users-json"))
|
|
for user in users:
|
|
let un = user["screen_name"].getStr
|
|
if un notin usernames: continue
|
|
result.add Profile(username: un, fullname: user["name"].getStr)
|
|
|
|
proc getTweetLocation*(node: XmlNode): string =
|
|
let geo = node.select(".js-geo-pivot-link")
|
|
if geo == nil: return
|
|
result = geo.innerText().stripText()
|
|
result &= ":" & geo.attr("data-place-id")
|