Add experimental x-client-transaction-id support (#1324)

* Add experimental x-client-transaction-id support

* Remove broken test

Author: zedeus

nitter.example.conf

@@ -26,6 +26,7 @@ enableRSS = true  # set this to false to disable RSS feeds
 enableDebug = false  # enable request logs and debug endpoints (/.sessions)
 proxy = ""  # http/https url, SOCKS proxies are not supported
 proxyAuth = ""
+disableTid = false # enable this if cookie-based auth is failing
 
 # Change default preferences here, see src/prefs_impl.nim for a complete list
 [Preferences]

src/api.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, httpclient, uri, strutils, sequtils, sugar, tables
+import asyncdispatch, httpclient, strutils, sequtils, sugar
 import packedjson
 import types, query, formatters, consts, apiutils, parser
 import experimental/parser as newParser
@@ -11,95 +11,91 @@ proc genParams(variables: string; fieldToggles = ""): seq[(string, string)] =
   if fieldToggles.len > 0:
     result.add ("fieldToggles", fieldToggles)
 
-proc mediaUrl(id: string; cursor: string): SessionAwareUrl =
-  let
-    cookieVars = userMediaVars % [id, cursor]
-    oauthVars = restIdVars % [id, cursor]
-  result = SessionAwareUrl(
-    cookieUrl: graphUserMedia ? genParams(cookieVars),
-    oauthUrl: graphUserMediaV2 ? genParams(oauthVars)
+proc apiUrl(endpoint, variables: string; fieldToggles = ""): ApiUrl =
+  return ApiUrl(endpoint: endpoint, params: genParams(variables, fieldToggles))
+
+proc apiReq(endpoint, variables: string; fieldToggles = ""): ApiReq =
+  let url = apiUrl(endpoint, variables, fieldToggles)
+  return ApiReq(cookie: url, oauth: url)
+
+proc mediaUrl(id: string; cursor: string): ApiReq =
+  result = ApiReq(
+    cookie: apiUrl(graphUserMedia, userMediaVars % [id, cursor]),
+    oauth: apiUrl(graphUserMediaV2, restIdVars % [id, cursor])
   )
 
-proc userTweetsUrl(id: string; cursor: string): SessionAwareUrl =
-  let
-    cookieVars = userTweetsVars % [id, cursor]
-    oauthVars = restIdVars % [id, cursor]
-  result = SessionAwareUrl(
-    # cookieUrl: graphUserTweets ? genParams(cookieVars, userTweetsFieldToggles),
-    oauthUrl: graphUserTweetsV2 ? genParams(oauthVars)
+proc userTweetsUrl(id: string; cursor: string): ApiReq =
+  result = ApiReq(
+    # cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles),
+    oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor])
   )
   # might change this in the future pending testing
-  result.cookieUrl = result.oauthUrl
+  result.cookie = result.oauth
 
-proc userTweetsAndRepliesUrl(id: string; cursor: string): SessionAwareUrl =
-  let
-    cookieVars = userTweetsAndRepliesVars % [id, cursor]
-    oauthVars = restIdVars % [id, cursor]
-  result = SessionAwareUrl(
-    cookieUrl: graphUserTweetsAndReplies ? genParams(cookieVars, userTweetsFieldToggles),
-    oauthUrl: graphUserTweetsAndRepliesV2 ? genParams(oauthVars)
+proc userTweetsAndRepliesUrl(id: string; cursor: string): ApiReq =
+  let cookieVars = userTweetsAndRepliesVars % [id, cursor]
+  result = ApiReq(
+    cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles),
+    oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor])
   )
 
-proc tweetDetailUrl(id: string; cursor: string): SessionAwareUrl =
-  let
-    cookieVars = tweetDetailVars % [id, cursor]
-    oauthVars = tweetVars % [id, cursor]
-  result = SessionAwareUrl(
-    cookieUrl: graphTweetDetail ? genParams(cookieVars, tweetDetailFieldToggles),
-    oauthUrl: graphTweet ? genParams(oauthVars)
+proc tweetDetailUrl(id: string; cursor: string): ApiReq =
+  let cookieVars = tweetDetailVars % [id, cursor]
+  result = ApiReq(
+    cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles),
+    oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
   )
 
-proc userUrl(username: string): SessionAwareUrl =
-  let
-    cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username
-    oauthVars = """{"screen_name": "$1"}""" % username
-  result = SessionAwareUrl(
-    cookieUrl: graphUser ? genParams(cookieVars, tweetDetailFieldToggles),
-    oauthUrl: graphUserV2 ? genParams(oauthVars)
+proc userUrl(username: string): ApiReq =
+  let cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username
+  result = ApiReq(
+    cookie: apiUrl(graphUser, cookieVars, tweetDetailFieldToggles),
+    oauth: apiUrl(graphUserV2, """{"screen_name": "$1"}""" % username)
   )
 
 proc getGraphUser*(username: string): Future[User] {.async.} =
   if username.len == 0: return
-  let js = await fetchRaw(userUrl(username), Api.userScreenName)
+  let js = await fetchRaw(userUrl(username))
   result = parseGraphUser(js)
 
 proc getGraphUserById*(id: string): Future[User] {.async.} =
   if id.len == 0 or id.any(c => not c.isDigit): return
   let
-    url = graphUserById ? genParams("""{"rest_id": "$1"}""" % id)
-    js = await fetchRaw(url, Api.userRestId)
+    url = apiReq(graphUserById, """{"rest_id": "$1"}""" % id)
+    js = await fetchRaw(url)
   result = parseGraphUser(js)
 
 proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Profile] {.async.} =
   if id.len == 0: return
   let
     cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    js = case kind
-      of TimelineKind.tweets:
-        await fetch(userTweetsUrl(id, cursor), Api.userTweets)
-      of TimelineKind.replies:
-        await fetch(userTweetsAndRepliesUrl(id, cursor), Api.userTweetsAndReplies)
-      of TimelineKind.media:
-        await fetch(mediaUrl(id, cursor), Api.userMedia)
+    url = case kind
+      of TimelineKind.tweets: userTweetsUrl(id, cursor)
+      of TimelineKind.replies: userTweetsAndRepliesUrl(id, cursor)
+      of TimelineKind.media: mediaUrl(id, cursor)
+    js = await fetch(url)
   result = parseGraphTimeline(js, after)
 
 proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
   if id.len == 0: return
   let
     cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    url = graphListTweets ? genParams(restIdVars % [id, cursor])
-  result = parseGraphTimeline(await fetch(url, Api.listTweets), after).tweets
+    url = apiReq(graphListTweets, restIdVars % [id, cursor])
+    js = await fetch(url)
+  result = parseGraphTimeline(js, after).tweets
 
 proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
   let
     variables = %*{"screenName": name, "listSlug": list}
-    url = graphListBySlug ? genParams($variables)
-  result = parseGraphList(await fetch(url, Api.listBySlug))
+    url = apiReq(graphListBySlug, $variables)
+    js = await fetch(url)
+  result = parseGraphList(js)
 
 proc getGraphList*(id: string): Future[List] {.async.} =
-  let
-    url = graphListById ? genParams("""{"listId": "$1"}""" % id)
-  result = parseGraphList(await fetch(url, Api.list))
+  let 
+    url = apiReq(graphListById, """{"listId": "$1"}""" % id)
+    js = await fetch(url)
+  result = parseGraphList(js)
 
 proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} =
   if list.id.len == 0: return
@@ -113,22 +109,23 @@ proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.}
     }
   if after.len > 0:
     variables["cursor"] = % after
-  let url = graphListMembers ? genParams($variables)
-  result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
+  let 
+    url = apiReq(graphListMembers, $variables)
+    js = await fetchRaw(url)
+  result = parseGraphListMembers(js, after)
 
 proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
   if id.len == 0: return
   let
-    variables = """{"rest_id": "$1"}""" % id
-    params = {"variables": variables, "features": gqlFeatures}
-    js = await fetch(graphTweetResult ? params, Api.tweetResult)
+    url = apiReq(graphTweetResult, """{"rest_id": "$1"}""" % id)
+    js = await fetch(url)
   result = parseGraphTweetResult(js)
 
 proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
   if id.len == 0: return
   let
     cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
-    js = await fetch(tweetDetailUrl(id, cursor), Api.tweetDetail)
+    js = await fetch(tweetDetailUrl(id, cursor))
   result = parseGraphConversation(js, id)
 
 proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
@@ -157,8 +154,10 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
     }
   if after.len > 0:
     variables["cursor"] = % after
-  let url = graphSearchTimeline ? genParams($variables)
-  result = parseGraphSearch[Tweets](await fetch(url, Api.search), after)
+  let 
+    url = apiReq(graphSearchTimeline, $variables)
+    js = await fetch(url)
+  result = parseGraphSearch[Tweets](js, after)
   result.query = query
 
 proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.} =
@@ -179,13 +178,15 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.}
     variables["cursor"] = % after
     result.beginning = false
 
-  let url = graphSearchTimeline ? genParams($variables)
-  result = parseGraphSearch[User](await fetch(url, Api.search), after)
+  let 
+    url = apiReq(graphSearchTimeline, $variables)
+    js = await fetch(url)
+  result = parseGraphSearch[User](js, after)
   result.query = query
 
 proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} =
   if id.len == 0: return
-  let js = await fetch(mediaUrl(id, ""), Api.userMedia)
+  let js = await fetch(mediaUrl(id, ""))
   result = parseGraphPhotoRail(js)
 
 proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =

src/apiutils.nim

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
 import jsony, packedjson, zippy, oauth1
-import types, auth, consts, parserutils, http_pool
+import types, auth, consts, parserutils, http_pool, tid
 import experimental/types/common
 
 const
@@ -10,7 +10,21 @@ const
   rlLimit = "x-rate-limit-limit"
   errorsToSkip = {null, doesntExist, tweetNotFound, timeout, unauthorized, badRequest}
 
-var pool: HttpPool
+var 
+  pool: HttpPool
+  disableTid: bool
+
+proc setDisableTid*(disable: bool) =
+  disableTid = disable
+
+proc toUrl(req: ApiReq; sessionKind: SessionKind): Uri =
+  case sessionKind
+  of oauth:  
+    let o = req.oauth
+    parseUri("https://api.x.com/graphql")   / o.endpoint ? o.params
+  of cookie: 
+    let c = req.cookie
+    parseUri("https://x.com/i/api/graphql") / c.endpoint ? c.params
 
 proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
   let
@@ -32,31 +46,36 @@ proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
 proc getCookieHeader(authToken, ct0: string): string =
   "auth_token=" & authToken & "; ct0=" & ct0
 
-proc genHeaders*(session: Session, url: string): HttpHeaders =
+proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} =
   result = newHttpHeaders({
     "connection": "keep-alive",
     "content-type": "application/json",
     "x-twitter-active-user": "yes",
     "x-twitter-client-language": "en",
-    "authority": "api.x.com",
+    "origin": "https://x.com",
     "accept-encoding": "gzip",
-    "accept-language": "en-US,en;q=0.9",
+    "accept-language": "en-US,en;q=0.5",
     "accept": "*/*",
     "DNT": "1",
     "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
   })
 
   case session.kind
   of SessionKind.oauth:
-    result["authorization"] = getOauthHeader(url, session.oauthToken, session.oauthSecret)
+    result["authority"] = "api.x.com"
+    result["authorization"] = getOauthHeader($url, session.oauthToken, session.oauthSecret)
   of SessionKind.cookie:
-    result["authorization"] = "Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F"
     result["x-twitter-auth-type"] = "OAuth2Session"
     result["x-csrf-token"] = session.ct0
     result["cookie"] = getCookieHeader(session.authToken, session.ct0)
-
-proc getAndValidateSession*(api: Api): Future[Session] {.async.} =
-  result = await getSession(api)
+    if disableTid:
+      result["authorization"] = bearerToken2
+    else:
+      result["authorization"] = bearerToken
+      result["x-client-transaction-id"] = await genTid(url.path)
+
+proc getAndValidateSession*(req: ApiReq): Future[Session] {.async.} =
+  result = await getSession(req)
   case result.kind
   of SessionKind.oauth:
     if result.oauthToken.len == 0:
@@ -73,7 +92,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
 
   try:
     var resp: AsyncResponse
-    pool.use(genHeaders(session, $url)):
+    pool.use(await genHeaders(session, url)):
       template getContent =
         resp = await c.get($url)
         result = await resp.body
@@ -89,7 +108,7 @@ template fetchImpl(result, fetchBody) {.dirty.} =
         remaining = parseInt(resp.headers[rlRemaining])
         reset = parseInt(resp.headers[rlReset])
         limit = parseInt(resp.headers[rlLimit])
-      session.setRateLimit(api, remaining, reset, limit)
+      session.setRateLimit(req, remaining, reset, limit)
 
     if result.len > 0:
       if resp.headers.getOrDefault("content-encoding") == "gzip":
@@ -98,24 +117,22 @@ template fetchImpl(result, fetchBody) {.dirty.} =
       if result.startsWith("{\"errors"):
         let errors = result.fromJson(Errors)
         if errors notin errorsToSkip:
-          echo "Fetch error, API: ", api, ", errors: ", errors
+          echo "Fetch error, API: ", url.path, ", errors: ", errors
           if errors in {expiredToken, badToken, locked}:
             invalidate(session)
             raise rateLimitError()
           elif errors in {rateLimited}:
             # rate limit hit, resets after 24 hours
-            setLimited(session, api)
+            setLimited(session, req)
             raise rateLimitError()
       elif result.startsWith("429 Too Many Requests"):
-        echo "[sessions] 429 error, API: ", api, ", session: ", session.pretty
-        session.apis[api].remaining = 0
-        # rate limit hit, resets after the 15 minute window
+        echo "[sessions] 429 error, API: ", url.path, ", session: ", session.pretty
         raise rateLimitError()
 
     fetchBody
 
     if resp.status == $Http400:
-      echo "ERROR 400, ", api, ": ", result
+      echo "ERROR 400, ", url.path, ": ", result
       raise newException(InternalError, $url)
   except InternalError as e:
     raise e
@@ -134,19 +151,16 @@ template retry(bod) =
   try:
     bod
   except RateLimitError:
-    echo "[sessions] Rate limited, retrying ", api, " request..."
+    echo "[sessions] Rate limited, retrying ", req.cookie.endpoint, " request..."
     bod
 
-proc fetch*(url: Uri | SessionAwareUrl; api: Api): Future[JsonNode] {.async.} =
+proc fetch*(req: ApiReq): Future[JsonNode] {.async.} =
   retry:
     var 
       body: string
-      session = await getAndValidateSession(api)
+      session = await getAndValidateSession(req)
 
-    when url is SessionAwareUrl:
-      let url = case session.kind
-        of SessionKind.oauth: url.oauthUrl
-        of SessionKind.cookie: url.cookieUrl
+    let url = req.toUrl(session.kind)
 
     fetchImpl body:
       if body.startsWith('{') or body.startsWith('['):
@@ -157,19 +171,15 @@ proc fetch*(url: Uri | SessionAwareUrl; api: Api): Future[JsonNode] {.async.} =
 
       let error = result.getError
       if error != null and error notin errorsToSkip:
-        echo "Fetch error, API: ", api, ", error: ", error
+        echo "Fetch error, API: ", url.path, ", error: ", error
         if error in {expiredToken, badToken, locked}:
           invalidate(session)
           raise rateLimitError()
 
-proc fetchRaw*(url: Uri | SessionAwareUrl; api: Api): Future[string] {.async.} =
+proc fetchRaw*(req: ApiReq): Future[string] {.async.} =
   retry:
-    var session = await getAndValidateSession(api)
-
-    when url is SessionAwareUrl:
-      let url = case session.kind
-        of SessionKind.oauth: url.oauthUrl
-        of SessionKind.cookie: url.cookieUrl
+    var session = await getAndValidateSession(req)
+    let url = req.toUrl(session.kind)
 
     fetchImpl result:
       if not (result.startsWith('{') or result.startsWith('[')):