Better video/gif support

This commit is contained in:
Zed 2019-06-24 05:14:14 +02:00
parent 8f7c61eab6
commit 861ac7a593
7 changed files with 151 additions and 39 deletions

View file

@ -593,3 +593,8 @@ nav {
top: calc(50% - 20px);
font-size: 20px;
}
video {
height: 100%;
width: 100%;
}

View file

@ -5,12 +5,18 @@ import nimquery, regex
import ./types, ./parser
const
base = parseUri("https://twitter.com/")
agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
auth = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
base = parseUri("https://twitter.com/")
apiBase = parseUri("https://api.twitter.com/1.1/")
timelineUrl = "i/profiles/show/$1/timeline/tweets?include_available_features=1&include_entities=1&include_new_items_bar=true"
profilePopupUrl = "i/profiles/popup"
profileIntentUrl = "intent/user"
tweetUrl = "i/status/"
videoUrl = "videos/tweet/config/$1.json"
tokenUrl = "guest/activate.json"
proc fetchHtml(url: Uri; headers: HttpHeaders; jsonKey = ""): Future[XmlNode] {.async.} =
var client = newAsyncHttpClient()
@ -30,6 +36,20 @@ proc fetchHtml(url: Uri; headers: HttpHeaders; jsonKey = ""): Future[XmlNode] {.
else:
return parseHtml(resp)
proc fetchJson(url: Uri; headers: HttpHeaders): Future[JsonNode] {.async.} =
var client = newAsyncHttpClient()
defer: client.close()
client.headers = headers
var resp = ""
try:
resp = await client.getContent($url)
except:
return nil
return parseJson(resp)
proc getProfileFallback(username: string; headers: HttpHeaders): Future[Profile] {.async.} =
let
url = base / profileIntentUrl ? {"screen_name": username}
@ -61,6 +81,63 @@ proc getProfile*(username: string): Future[Profile] {.async.} =
result = parsePopupProfile(html)
proc getGuestToken(): Future[string] {.async.} =
let headers = newHttpHeaders({
"Accept": "application/json, text/javascript, */*; q=0.01",
"Referer": $base,
"User-Agent": agent,
"Authorization": auth
})
let client = newAsyncHttpClient()
client.headers = headers
let
url = apibase / tokenUrl
json = parseJson(await client.postContent($url))
result = json["guest_token"].to(string)
proc getVideo*(tweet: Tweet; token: string) {.async.} =
let headers = newHttpHeaders({
"Accept": "application/json, text/javascript, */*; q=0.01",
"Referer": tweet.link,
"User-Agent": agent,
"Authorization": auth,
"x-guest-token": token
})
let
url = apiBase / (videoUrl % tweet.id)
json = await fetchJson(url, headers)
tweet.video = some(parseVideo(json))
proc getVideos*(tweets: Tweets; token="") {.async.} =
if not tweets.anyIt(it.video.isSome): return
var
token = if token.len > 0: token else: await getGuestToken()
videoFuts: seq[Future[void]]
for tweet in tweets:
if tweet.video.isSome:
videoFuts.add getVideo(tweet, token)
await all(videoFuts)
proc getConversationVideos*(convo: Conversation) {.async.} =
var token = await getGuestToken()
var futs: seq[Future[void]]
futs.add getVideo(convo.tweet, token)
futs.add getVideos(convo.before, token=token)
futs.add getVideos(convo.after, token=token)
futs.add convo.replies.mapIt(getVideos(it, token=token))
await all(futs)
proc getTimeline*(username: string; after=""): Future[Tweets] {.async.} =
let headers = newHttpHeaders({
"Accept": "application/json, text/javascript, */*; q=0.01",
@ -78,6 +155,7 @@ proc getTimeline*(username: string; after=""): Future[Tweets] {.async.} =
let html = await fetchHtml(base / url, headers, jsonKey="items_html")
result = parseTweets(html)
await getVideos(result)
proc getTweet*(id: string): Future[Conversation] {.async.} =
let headers = newHttpHeaders({
@ -96,3 +174,4 @@ proc getTweet*(id: string): Future[Conversation] {.async.} =
html = await fetchHtml(url, headers)
result = parseConversation(html)
await getConversationVideos(result)

View file

@ -65,12 +65,6 @@ proc getUserpic*(userpic: string; style=""): string =
proc getUserpic*(profile: Profile; style=""): string =
getUserPic(profile.userpic, style)
proc getGifSrc*(tweet: Tweet): string =
fmt"https://video.twimg.com/tweet_video/{tweet.gif.get()}.mp4"
proc getGifThumb*(tweet: Tweet): string =
fmt"https://pbs.twimg.com/tweet_video_thumb/{tweet.gif.get()}.jpg"
proc formatName*(profile: Profile): string =
result = xmltree.escape(profile.fullname)
if profile.verified:

View file

@ -1,4 +1,4 @@
import xmltree, sequtils, strtabs, strutils, strformat
import xmltree, sequtils, strtabs, strutils, strformat, json
import nimquery
import ./types, ./parserutils
@ -40,7 +40,6 @@ proc parseTweetProfile*(profile: XmlNode): Profile =
proc parseQuote*(tweet: XmlNode): Tweet =
let tweet = tweet.querySelector(".QuoteTweet-innerContainer")
result = Tweet(
id: tweet.getAttr("data-item-id"),
link: tweet.getAttr("href"),
@ -77,8 +76,10 @@ proc parseTweets*(node: XmlNode): Tweets =
node.querySelectorAll(".tweet").map(parseTweet)
proc parseConversation*(node: XmlNode): Conversation =
result.tweet = parseTweet(node.querySelector(".permalink-tweet-container > .tweet"))
result.before = parseTweets(node.querySelector(".in-reply-to"))
result = Conversation(
tweet: parseTweet(node.querySelector(".permalink-tweet-container > .tweet")),
before: parseTweets(node.querySelector(".in-reply-to"))
)
let replies = node.querySelector(".replies-to")
if replies.isNil: return
@ -89,3 +90,14 @@ proc parseConversation*(node: XmlNode): Conversation =
let thread = parseTweets(reply)
if not thread.anyIt(it in result.after):
result.replies.add thread
proc parseVideo*(node: JsonNode): Video =
let track = node{"track"}
result = Video(
thumb: node["posterImage"].to(string),
id: track["contentId"].to(string),
length: track["durationMs"].to(int),
views: track["viewCount"].to(string),
url: track["playbackUrl"].to(string),
available: track{"mediaAvailability"}["status"].to(string) == "available"
)

View file

@ -1,7 +1,7 @@
import xmltree, strtabs, times
import xmltree, strtabs, strformat, times
import nimquery, regex
import ./types, ./formatters
import ./types, ./formatters, ./api
const
thumbRegex = re".+:url\('([^']+)'\)"
@ -84,11 +84,10 @@ proc getIntentStats*(profile: var Profile; node: XmlNode) =
of "followers": profile.followers = text
of "following": profile.following = text
proc getTweetStats*(tweet: var Tweet; node: XmlNode) =
proc getTweetStats*(tweet: Tweet; node: XmlNode) =
tweet.replies = "0"
tweet.retweets = "0"
tweet.likes = "0"
for action in node.querySelectorAll(".ProfileTweet-actionCountForAria"):
let text = action.innerText.split()
case text[1]
@ -96,16 +95,22 @@ proc getTweetStats*(tweet: var Tweet; node: XmlNode) =
of "likes": tweet.likes = text[0]
of "retweets": tweet.retweets = text[0]
proc getTweetMedia*(tweet: var Tweet; node: XmlNode) =
proc getGif(player: XmlNode): Gif =
let
thumb = player.getAttr("style").replace(thumbRegex, "$1")
id = thumb.replace(gifRegex, "$1")
url = fmt"https://video.twimg.com/tweet_video/{id}.mp4"
Gif(url: url, thumb: thumb)
proc getTweetMedia*(tweet: Tweet; node: XmlNode) =
for photo in node.querySelectorAll(".AdaptiveMedia-photoContainer"):
tweet.photos.add photo.attrs["data-image-url"]
let player = node.selectAttr(".PlayableMedia-player", "style")
if player.len == 0:
let player = node.querySelector(".PlayableMedia")
if player.isNil:
return
let thumb = player.replace(thumbRegex, "$1")
if "tweet_video" in thumb:
tweet.gif = some(thumb.replace(gifRegex, "$1"))
if "gif" in player.getAttr("class"):
tweet.gif = some(getGif(player.querySelector(".PlayableMedia-player")))
else:
tweet.videoThumb = some(thumb)
tweet.video = some(Video())

View file

@ -31,7 +31,26 @@ db("cache.db", "", "", ""):
.}: Time
type
Tweet* = object
Video* = object
id*: string
url*: string
thumb*: string
length*: int
views*: string
available*: bool
Gif* = object
url*: string
thumb*: string
Quote* = ref object
id*: string
profile*: Profile
link*: string
text*: string
video*: Option[Video]
Tweet* = ref object
id*: string
profile*: Profile
link*: string
@ -42,16 +61,16 @@ type
retweets*: string
likes*: string
pinned*: bool
photos*: seq[string]
quote*: Option[Quote]
retweetBy*: Option[string]
gif*: Option[string]
video*: Option[string]
videoThumb*: Option[string]
retweetId*: Option[string]
gif*: Option[Gif]
video*: Option[Video]
photos*: seq[string]
Tweets* = seq[Tweet]
Conversation* = object
Conversation* = ref object
tweet*: Tweet
before*: Tweets
after*: Tweets

View file

@ -54,11 +54,11 @@
</div>
#end proc
#
#proc renderVideo(tweet: Tweet): string =
#proc renderVideo(video: Video): string =
<div class="attachments media-body">
<div class="gallery-row" style="max-height: unset;">
<div class="attachment image">
<video poster=${tweet.videoThumb.get()} style="width: 100%; height: 100%;" autoplay muted loop></video>
<video poster=${video.thumb.getSigUrl("pic")} autoplay muted loop></video>
<div class="video-overlay">
<p>Video playback not supported</p>
</div>
@ -67,14 +67,12 @@
</div>
#end proc
#
#proc renderGif(tweet: Tweet): string =
#let thumbUrl = getGifThumb(tweet).getSigUrl("pic")
#let videoUrl = getGifSrc(tweet).getSigUrl("video")
<div class="attachments media-body">
#proc renderGif(gif: Gif): string =
<div class="attachments media-body" style="display: table-cell;">
<div class="gallery-row" style="max-height: unset;">
<div class="attachment image">
<video poster=${thumbUrl} style="width: 100%; height: 100%;" autoplay muted loop>
<source src=${videoUrl} type="video/mp4">
<video class="gif" poster=${gif.thumb.getSigUrl("pic")} autoplay muted loop>
<source src=${gif.url.getSigUrl("video")} type="video/mp4">
</video>
</div>
</div>
@ -103,10 +101,10 @@
</div>
#if tweet.photos.len > 0:
${renderMediaGroup(tweet)}
#elif tweet.videoThumb.isSome:
${renderVideo(tweet)}
#elif tweet.video.isSome:
${renderVideo(tweet.video.get())}
#elif tweet.gif.isSome:
${renderGif(tweet)}
${renderGif(tweet.gif.get())}
#end if
${renderStats(tweet)}
</div>