Merge pull request 'November update' (#2) from github/origin into alterware

Reviewed-on: #2

.github/workflows/python-tests.yml

@@ -6,23 +6,30 @@ jobs:
   build:
     runs-on: ubuntu-latest
     environment: test
-    strategy:
-      matrix:
-        python-version: ["3.10", "3.11", "3.12"]
-
     steps:
-      - uses: actions/checkout@v3
+      - name: Check out files
-      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/checkout@v4
-        uses: actions/setup-python@v4
+
-        with:
+      - name: Install Python
-          python-version: ${{ matrix.python-version }}
+        run: |
+          apt-get update
+          apt-get install -y python3 python3-pip python3-venv
+
+      - name: Set up virtual environment
+        run: |
+          python3 -m venv venv
+          . venv/bin/activate
+          python -m pip install --upgrade pip
+
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          . venv/bin/activate
           pip install pytest pytest-cov pytest-env
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
       - name: Test with pytest
         env:
           VXTWITTER_WORKAROUND_TOKENS: ${{ secrets.VXTWITTER_WORKAROUND_TOKENS }}
         run: |
+          . venv/bin/activate
           pytest --cov-config=.coveragerc --cov=.

activity.py

@@ -0,0 +1,79 @@
+import datetime
+import msgs
+from utils import determineEmbedTweet, determineMediaToEmbed
+from copy import deepcopy
+
+def tweetDataToActivity(tweetData,embedIndex = -1):
+    content=""
+
+    if tweetData['replyingTo'] is not None:
+        content += f"<blockquote>↪️ <i>Replying to @{tweetData['replyingTo']}</i></blockquote>"
+    content+=f"<p>{tweetData['text']}</p>"
+
+    attachments=[]
+    if tweetData['qrt'] is not None:
+        content += f"<blockquote><b>QRT: <a href=\"{tweetData['qrtURL']}\">{tweetData['qrt']['user_screen_name']}</a></b><br>{tweetData['qrt']['text']}</blockquote>"
+    if tweetData['pollData'] is not None:
+        content += f"<p>{msgs.genPollDisplay(tweetData['pollData'])}</p>"
+        content += "</p>"
+    content = content.replace("\n","<br>")
+    #if media is not None:
+    #    attachments.append({"type":mediatype,"url":media})
+    likes = tweetData['likes']
+    retweets = tweetData['retweets']
+
+    # convert date epoch to iso format
+    date = tweetData['date_epoch']
+    date = datetime.datetime.fromtimestamp(date).isoformat() + "Z"
+
+    embedTweetData = determineEmbedTweet(tweetData)
+    embeddingMedia = embedTweetData['hasMedia']
+    media = None
+    if embeddingMedia:
+        media = determineMediaToEmbed(embedTweetData,embedIndex)
+
+    if media is not None:
+        media = deepcopy(media)
+        if media['type'] == "gif":
+            media['type'] = "gifv"
+        if 'thumbnail_url' not in media:
+            media['thumbnail_url'] = media['url']
+        if media['type'] == "image" and "?" not in media['url']:
+            media['url'] += "?name=orig"
+        attachments.append({
+            "id": "114163769487684704",
+            "type": media['type'],
+            "url": media['url'],
+            "preview_url": media['thumbnail_url'],
+        })
+
+    # https://docs.joinmastodon.org/methods/statuses/
+    return {
+	"id": tweetData['tweetID'],
+	"url": f"https://x.com/{tweetData['user_screen_name']}/status/{tweetData['tweetID']}",
+	"uri": f"https://x.com/{tweetData['user_screen_name']}/status/{tweetData['tweetID']}",
+	"created_at": date,
+	"edited_at": None,
+	"reblog": None,
+	"in_reply_to_account_id": None,
+	"language": "en",
+	"content": content,
+	"spoiler_text": "",
+	"visibility": "public",
+	"application": {
+		"website": None
+	},
+	"media_attachments": attachments,
+	"account": {
+		"display_name": tweetData['user_name'],
+		"username": tweetData['user_screen_name'],
+		"acct": tweetData['user_screen_name'],
+		"url": f"https://x.com/{tweetData['user_screen_name']}/status/{tweetData['tweetID']}",
+		"uri": f"https://x.com/{tweetData['user_screen_name']}/status/{tweetData['tweetID']}",
+		"locked": False,
+		"avatar": tweetData['user_profile_image_url'],
+		"avatar_static": tweetData['user_profile_image_url'],
+		"hide_collections": False,
+		"noindex": False,
+	},
+}

cache.py

@@ -49,7 +49,9 @@ def addVnfToTweetIdCache(tweet_id, vnf):
     global link_cache
     try:
         if link_cache_system == "db":
-                out = db.linkCache.update_one(vnf)
+                filter_query = {'tweet': tweet_id}
+                update_operation = {'$set': vnf}
+                out = db.linkCache.update_one(filter_query, update_operation, upsert=True)
                 log.debug("Link added to DB cache ")
                 return True
         elif link_cache_system == "json":
@@ -86,7 +88,7 @@ def getVnfFromTweetIdCache(tweet_id):
         collection = db.linkCache
         vnf        = collection.find_one({'tweet': tweet_id})
         if vnf != None: 
-            hits   = ( vnf['hits'] + 1 ) 
+            hits   = ( vnf.get('hits', 0) + 1 ) 
             log.debug("Link located in DB cache.")
             query  = { 'tweet': tweet_id }
             change = { "$set" : { "hits" : hits } }

config.json

@@ -2,14 +2,14 @@
     "config": {
         "appname": "vxTwitter",
         "color": "#1DA1F2",
-        "database": "[url to mongo database goes here]",
+        "database": "mongodb://localhost:27017/bettervxtwitter",
-        "link_cache": "ram",
+        "link_cache": "db",
         "method": "hybrid",
-        "repo": "https://github.com/dylanpdx/BetterTwitFix",
+        "repo": "https://git.alterware.dev/alterware/BetterTwitFix",
-        "table": "[database table here]",
+        "table": "links",
-        "url": "https://vxtwitter.com",
+        "url": "https://girlcock.alterware.dev",
         "combination_method": "local",
         "gifConvertAPI": "local",
-        "workaroundTokens":null
+        "workaroundTokens": null
     }
 }

requirements.txt

@@ -1,5 +1,5 @@
 pymongo==4.8.0
-boto3==1.35.18
+boto3==1.36.6
 requests==2.32.3
 Pillow==10.4.0
 Flask==2.2.3
@@ -8,3 +8,4 @@ Werkzeug==2.3.7
 numerize==0.12
 oauthlib==3.2.2
 PyRTF3==0.47.5
+XClientTransaction==0.0.2

serverless.yml

@@ -47,7 +47,7 @@ functions:
     handler: wsgi_handler.handler
     url: true
     timeout: 15
-    memorySize: 500
+    memorySize: 128
     layers:
       - Ref: PythonRequirementsLambdaLayer
 

templates/tweetCommon.html

@@ -2,7 +2,21 @@
 <meta content="{{ color }}" name="theme-color" />
 <meta property="og:site_name" content="{{ appname }}">
 
-<meta name="twitter:title" content="{{ tweet['user_name'] }} (@{{ tweet['user_screen_name'] }})" />
+<meta name="og:title" content="{{ tweet['user_name'] }} (@{{ tweet['user_screen_name'] }})" />
+
+{% if activityLink %}
+<link type="application/activity+json" href="{{ activityLink|safe }}" />
+{% endif %}
+
+{% if sicon %}
+<!-- Video icon created by Rizki Ahmad Fauzi - Flaticon -->
+<!-- Picture and Text icon created by Freepik - Flaticon -->
+<!-- Gif file icons created by Grand Iconic - Flaticon -->
+<link rel="apple-touch-icon" sizes="128x128" href="/{{ sicon }}.png">
+{% else %}
+<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png">
+{% endif %}
+
 <script src="/openInApp.js"></script>
 <script>document.addEventListener('DOMContentLoaded',function(){openTweet('{{ tweet["tweetID"] }}')})</script>
 <style>

templates/user.html

@@ -8,7 +8,7 @@
 <meta content="{{ color }}" name="theme-color" />
 <meta property="og:site_name" content="{{ appname }}">
 
-<meta name="twitter:title" content="{{ user['name'] }} (@{{ user['screen_name'] }})" />
+<meta name="og:title" content="{{ user['name'] }} (@{{ user['screen_name'] }})" />
 
 <meta name="twitter:image" content="{{ user['profile_image_url'] }}" />
 <meta name="twitter:creator" content="@{{ user['name'] }}" />

test_api.py

@@ -3,6 +3,17 @@ import twitfix,twExtract
 from flask.testing import FlaskClient
 client = FlaskClient(twitfix.app)
 
+def test_api_get_tweet():
+    resp = client.get(testTextTweet.replace("https://twitter.com","https://api.vxtwitter.com")+"?include_txt=true",headers={"User-Agent":"test"})
+    jData = resp.get_json()
+    assert resp.status_code==200
+    assert jData['text'] == 'just setting up my twttr'
+
+def test_api_get_invalid_tweet():
+    resp = client.get("https://vxtwitter.com/test/status/None",headers={"User-Agent":"test"})
+    jData = resp.get_json()
+    assert resp.status_code!=200
+
 def test_api_include_txt():
     resp = client.get(testTextTweet.replace("https://twitter.com","https://api.vxtwitter.com")+"?include_txt=true",headers={"User-Agent":"test"})
     jData = resp.get_json()
@@ -37,8 +48,42 @@ def test_api_include_rtf_nomedia():
     assert resp.status_code==200
     assert not any(".rtf" in i for i in jData["mediaURLs"])
 
+def test_api_mixedmedia():
+    resp = client.get(testMixedMediaTweet.replace("https://twitter.com","https://api.vxtwitter.com")+"?include_txt=true",headers={"User-Agent":"test"})
+    assert resp.status_code==200
+
 def test_api_user():
-    resp = client.get(testUser.replace("https://twitter.com","https://api.vxtwitter.com")+"?include_rtf=true",headers={"User-Agent":"test"})
+    resp = client.get(testUser.replace("https://twitter.com","https://api.vxtwitter.com"),headers={"User-Agent":"test"})
     jData = resp.get_json()
     assert resp.status_code==200
     assert jData["screen_name"]=="jack"
+
+def test_api_user_suspended():
+    resp = client.get(testUserSuspended.replace("https://twitter.com","https://api.vxtwitter.com"),headers={"User-Agent":"test"})
+    jData = resp.get_json()
+    assert 'suspended' in jData["error"]
+
+def test_api_user_private():
+    resp = client.get(testUserPrivate.replace("https://twitter.com","https://api.vxtwitter.com")+"?with_tweets=true",headers={"User-Agent":"test"})
+    jData = resp.get_json()
+    assert jData['protected'] == True
+    assert len(jData["latest_tweets"])==0
+
+def test_api_user_invalid():
+    resp = client.get(testUserInvalid.replace("https://twitter.com","https://api.vxtwitter.com")+"?with_tweets=true",headers={"User-Agent":"test"})
+    jData = resp.get_json()
+    assert resp.status_code==404
+
+def test_api_user_feed():
+    resp = client.get(testUser.replace("https://twitter.com","https://api.vxtwitter.com")+"?with_tweets=true",headers={"User-Agent":"test"})
+    jData = resp.get_json()
+    assert resp.status_code==200
+    assert jData["screen_name"]=="jack"
+    assert len(jData["latest_tweets"])>0
+
+def test_api_retweet():
+    resp = client.get(testRetweetTweet.replace("https://twitter.com","https://api.vxtwitter.com"),headers={"User-Agent":"test"})
+    jData = resp.get_json()
+    assert jData['retweetURL'] == 'https://twitter.com/i/status/1828569456231993456'
+    assert jData['retweet'] != None
+    assert jData['retweet']['text'].startswith("If you want to try")

test_vx_embeds.py

@@ -194,3 +194,14 @@ def test_embed_rtf():
     resp = client.get(testTextTweet.replace("https://twitter.com","")+".rtf",headers={"User-Agent":"test"})
     assert resp.status_code==200
     assert testTextTweet_compare["text"] in str(resp.data)
+
+def test_embed_action():
+    cache.clearCache()
+    resp = client.get(testTextTweet.replace("https://twitter.com",""),headers={"User-Agent":"test"})
+    assert resp.status_code==200
+    assert "application/activity+json" in str(resp.data)
+    assert "%F0%9F%92%96" in str(resp.data) # 💖
+    resp = client.get(testTextTweet.replace("https://twitter.com",""),headers={"User-Agent":"Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)"})
+    assert resp.status_code==200
+    assert "application/activity+json" in str(resp.data)
+    assert "%F0%9F%92%96" not in str(resp.data) # 💖

test_vx_extract.py

@@ -3,19 +3,20 @@ import os
 import twExtract
 import utils
 from vx_testdata import *
+import twitfix
 
 def test_twextract_syndicationAPI():
     tweet = twExtract.extractStatus_syndication(testMediaTweet,workaroundTokens=tokens)
     assert utils.stripEndTCO(utils.stripEndTCO(tweet["full_text"]))==testMediaTweet_compare['text']
 
-def test_twextract_extractStatusV2Anon():
+def test_twextract_extractStatusV2Rest():
-    tweet = twExtract.extractStatusV2Anon(testTextTweet,None)['legacy']
+    tweet = twExtract.extractStatusV2Rest(testTextTweet,None)['legacy']
     assert utils.stripEndTCO(tweet["full_text"])==testTextTweet_compare['text']
-    tweet = twExtract.extractStatusV2Anon(testVideoTweet,None)['legacy']
+    tweet = twExtract.extractStatusV2Rest(testVideoTweet,None)['legacy']
     assert utils.stripEndTCO(tweet["full_text"])==testVideoTweet_compare['text']
-    tweet = twExtract.extractStatusV2Anon(testMediaTweet,None)['legacy']
+    tweet = twExtract.extractStatusV2Rest(testMediaTweet,None)['legacy']
     assert utils.stripEndTCO(tweet["full_text"])==testMediaTweet_compare['text']
-    tweet = twExtract.extractStatusV2Anon(testMultiMediaTweet,None)['legacy']
+    tweet = twExtract.extractStatusV2Rest(testMultiMediaTweet,None)['legacy']
     assert utils.stripEndTCO(tweet["full_text"])[:94]==testMultiMediaTweet_compare['text'][:94]
     
 
@@ -32,30 +33,28 @@ def test_twextract_extractStatusV2TweetDetails():
     assert utils.stripEndTCO(tweet["full_text"])==testMediaTweet_compare['text']
 
 ## Tweet retrieve tests ##
-def test_twextract_textTweetExtract():
-    tweet = twExtract.extractStatus(testTextTweet,workaroundTokens=tokens)
-    assert utils.stripEndTCO(tweet["legacy"]["full_text"])==testTextTweet_compare['text']
-    assert tweet["user"]["screen_name"]=="jack"
-    assert 'extended_entities' not in tweet
     
-def test_twextract_extractV2(): # remove this when v2 is default
+def test_twextract_extractV2():
     tweet = twExtract.extractStatusV2(testTextTweet,workaroundTokens=tokens)
 
 def test_twextract_UserExtract():
-    user = twExtract.extractUser(testUser,workaroundTokens=tokens)
+    rawUserData = twExtract.extractUser(testUser,workaroundTokens=tokens)
+    user = twitfix.getApiUserResponse(rawUserData)
     assert user["screen_name"]=="jack"
     assert user["id"]==12
     assert user["created_at"] == "Tue Mar 21 20:50:14 +0000 2006"
 
 def test_twextract_UserExtractID():
-    user = twExtract.extractUser(testUserIDUrl,workaroundTokens=tokens)
+    rawUserData = twExtract.extractUser(testUserIDUrl,workaroundTokens=tokens)
+    user = twitfix.getApiUserResponse(rawUserData)
     assert user["screen_name"]=="jack"
     assert user["id"]==12
     assert user["created_at"] == "Tue Mar 21 20:50:14 +0000 2006"
 
 def test_twextract_UserExtractWeirdURLs():
     for url in testUserWeirdURLs:
-        user = twExtract.extractUser(url,workaroundTokens=tokens)
+        rawUserData = twExtract.extractUser(url,workaroundTokens=tokens)
+        user = twitfix.getApiUserResponse(rawUserData)
         assert user["screen_name"]=="jack"
         assert user["id"]==12
         assert user["created_at"] == "Tue Mar 21 20:50:14 +0000 2006"
@@ -101,7 +100,6 @@ def test_twextract_pollTweetExtract(): # basic check if poll data exists
 def test_twextract_NSFW_TweetExtract():
     tweet = twExtract.extractStatus(testNSFWTweet,workaroundTokens=tokens) # For now just test that there's no error
 
-'''
 def test_twextract_feed():
-    tweet = twExtract.extractUserFeedFromId(testUserID,workaroundTokens=tokens)
+    tweets = twExtract.extractUserFeedFromId(testUserID,workaroundTokens=tokens) # For now just test that there's no error
-'''
+    assert len(tweets)>0

test_vx_misc.py

@@ -1,10 +1,17 @@
-import twitfix, cache, twExtract
+import twitfix, cache, twExtract, utils
 from vx_testdata import *
 from twExtract import twUtils
 
 def test_calcSyndicationToken():
     assert twUtils.calcSyndicationToken("1691389765483200513") == "43lnobuxzql"
 
+def test_stripEndTCO():
+    assert utils.stripEndTCO("Hello World https://t.co/abc123") == "Hello World"
+    assert utils.stripEndTCO("Hello\nWorld https://t.co/abc123") == "Hello\nWorld"
+    assert utils.stripEndTCO("Hello\nWorld\nhttps://t.co/abc123") == "Hello\nWorld"
+    assert utils.stripEndTCO("Hello\nWorld\n https://t.co/abc123") == "Hello\nWorld"
+    assert utils.stripEndTCO("Hello\nWorld \nhttps://t.co/abc123") == "Hello\nWorld"
+
 def test_addToCache():
     cache.clearCache()
     twitfix.getTweetData(testTextTweet)

testgen.py

@@ -12,6 +12,7 @@ tests = {
     "testPollTweet": "https://twitter.com/norm/status/651169346518056960",
     "testMixedMediaTweet":"https://twitter.com/bigbeerfest/status/1760638922084741177",
     "testVinePlayerTweet":"https://twitter.com/Roblox/status/583302104342638592",
+    "testRetweetTweet":"https://twitter.com/pdxdylan/status/1828570470222045294",
 }
 
 def getVNFFromLink(link):
@@ -31,5 +32,6 @@ with open('generated.txt', 'w',encoding='utf-8') as f:
         del VNF['user_name']
         del VNF['user_profile_image_url']
         del VNF['communityNote']
+        del VNF['fetched_on']
         # write in a format that can be copy-pasted into a python file, i.e testTextTweet={...
         f.write(f"{test}_compare={VNF}\n")

twExtract/__init__.py

@@ -9,12 +9,14 @@ from oauthlib import oauth1
 import sys
 sys.path.append(os.path.dirname(os.path.realpath(__file__)))
 import twUtils
+import concurrent.futures
 bearer="Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
 v2bearer="Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
 androidBearer="Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F"
-tweetdeckBearer="Bearer AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
 
-bearerTokens=[tweetdeckBearer,bearer,v2bearer,androidBearer]
+requestUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0"
+
+bearerTokens=[bearer,v2bearer,androidBearer]
 
 guestToken=None
 guestTokenUses=0
@@ -25,21 +27,28 @@ userIDregex = r"\/i\/user\/(\d+)"
 v2Features='{"longform_notetweets_inline_media_enabled":true,"super_follow_badge_privacy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"super_follow_user_api_enabled":true,"super_follow_tweet_api_enabled":true,"android_graphql_skip_api_media_color_palette":true,"creator_subscriptions_tweet_preview_api_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"creator_subscriptions_subscription_count_enabled":true,"tweetypie_unmention_optimization_enabled":true,"longform_notetweets_consumption_enabled":true,"subscriptions_verification_info_enabled":true,"blue_business_profile_image_shape_enabled":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"super_follow_exclusive_tweet_notifications_enabled":true}'
 v2graphql_api="2OOZWmw8nAtUHVnXXQhgaA"
 
-v2AnonFeatures='{"creator_subscriptions_tweet_preview_api_enabled":true,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"articles_preview_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"tweet_with_visibility_results_prefer_gql_media_interstitial_enabled":true,"rweb_video_timestamps_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false}'
+v2AnonFeatures='{"creator_subscriptions_tweet_preview_api_enabled":true,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":false,"responsive_web_jetfuel_frame":true,"responsive_web_grok_share_attachment_enabled":true,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":false,"responsive_web_grok_analysis_button_from_backend":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"payments_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"verified_phone_label_enabled":false,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_grok_imagine_annotation_enabled":true,"responsive_web_grok_community_note_auto_translation_is_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false}'
-v2AnonGraphql_api="7xflPyRiUxGVbJd4uWmbfg"
+v2AnonGraphql_api="wqi5M7wZ7tW-X9S2t-Mqcg"
 gt_pattern = r'document\.cookie="gt=([^;]+);'
 
-androidGraphqlFeatures='{"longform_notetweets_inline_media_enabled":true,"super_follow_badge_privacy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"super_follow_user_api_enabled":true,"unified_cards_ad_metadata_container_dynamic_card_content_query_enabled":true,"super_follow_tweet_api_enabled":true,"articles_api_enabled":true,"android_graphql_skip_api_media_color_palette":true,"creator_subscriptions_tweet_preview_api_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"tweetypie_unmention_optimization_enabled":true,"longform_notetweets_consumption_enabled":true,"subscriptions_verification_info_enabled":true,"blue_business_profile_image_shape_enabled":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"immersive_video_status_linkable_timestamps":true,"super_follow_exclusive_tweet_notifications_enabled":true}'
+androidGraphqlFeatures='{"grok_translations_community_note_translation_is_enabled":false,"super_follow_badge_privacy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"super_follow_user_api_enabled":true,"profile_label_improvements_pcf_label_in_profile_enabled":true,"premium_content_api_read_enabled":false,"grok_translations_community_note_auto_translation_is_enabled":false,"android_graphql_skip_api_media_color_palette":true,"tweetypie_unmention_optimization_enabled":true,"longform_notetweets_consumption_enabled":true,"subscriptions_verification_info_enabled":true,"blue_business_profile_image_shape_enabled":true,"super_follow_exclusive_tweet_notifications_enabled":true,"longform_notetweets_inline_media_enabled":true,"grok_android_analyze_trend_fetch_enabled":false,"unified_cards_ad_metadata_container_dynamic_card_content_query_enabled":true,"super_follow_tweet_api_enabled":true,"articles_api_enabled":true,"creator_subscriptions_tweet_preview_api_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"grok_translations_timeline_user_bio_auto_translation_is_enabled":false,"grok_translations_post_auto_translation_is_enabled":false,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"immersive_video_status_linkable_timestamps":true,"profile_label_improvements_pcf_label_in_post_enabled":true}'
-androidGraphql_api="llQH5PFIRlenVrlKJU8jNA"
+androidGraphql_api="k3rtLsS9kG5hI-Jr0dTMCg"
 
 tweetDetailGraphqlFeatures='{"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"articles_preview_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"rweb_video_timestamps_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_enhance_cards_enabled":false}'
-tweetDetailGraphql_api="e7RKseIxLu7HgkWNKZ6qnw"
+tweetDetailGraphql_api="YVyS4SfwYW7Uw5qwy0mQCA"
 
 # this is for UserTweets endpoint
-tweetFeedGraphqlFeatures='{"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":false,"responsive_web_grok_share_attachment_enabled":true,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_analysis_button_from_backend":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"rweb_video_timestamps_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_enhance_cards_enabled":false}'
+tweetFeedGraphqlFeatures='{"rweb_video_screen_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":false,"responsive_web_grok_share_attachment_enabled":true,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":false,"responsive_web_grok_analysis_button_from_backend":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_enhance_cards_enabled":false}'
-tweetFeedGraphql_api="Y9WM4Id6UcGFE8Z-hbnixw"
+tweetFeedGraphql_api="OAx9yEcW3JA9bPo63pcYlA"
+
+userByScreenNameGraphqlFeatures='{"rweb_xchat_enabled":false,"hidden_profile_subscriptions_enabled":true,"payments_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"verified_phone_label_enabled":false,"subscriptions_verification_info_is_identity_verified_enabled":true,"subscriptions_verification_info_verified_since_enabled":true,"highlights_tweets_tab_ui_enabled":true,"responsive_web_twitter_article_notes_tab_enabled":true,"subscriptions_feature_can_gift_premium":true,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true}'
+userByScreenNameGraphql_api="96tVxbPqMZDoYB5pmzezKA"
+userByRestIdGraphql_api="8r5oa_2vD0WkhIAOkY4TTA"
 
 twitterUrl = "x.com" # doubt this will change but just in case
+
+simultaneousRequests = int(os.getenv("VXTWITTER_SIMULTANEOUS_REQUESTS",1))
+
 class TwExtractError(Exception):
     def __init__(self, code, message):
         self.code = code
@@ -48,6 +57,37 @@ class TwExtractError(Exception):
     def __str__(self):
         return self.msg
 
+def parallel_token_request(twid, tokens, request_function):
+    results = []
+    errors = []
+    def try_token(token):
+        try:
+            result = request_function(twid, token)
+            return {'success': True, 'result': result}
+        except Exception as e:
+            return {'success': False, 'error': str(e)}
+    
+    with concurrent.futures.ThreadPoolExecutor(max_workers=min(simultaneousRequests, len(tokens))) as executor:
+        futures = {executor.submit(try_token, token): token for token in tokens}
+        for future in concurrent.futures.as_completed(futures):
+            result = future.result()
+            if result['success']:
+                results.append(result)
+            else:
+                errors.append(result)
+            
+            # Early return if success
+            if result['success']:
+                for f in futures: # Cancel remaining futures
+                    if not f.done():
+                        f.cancel()
+                return result['result']
+    
+    # all tokens failed
+    if errors:
+        raise TwExtractError(400, f"All tokens failed. Last error: {errors[-1]['error']}")
+    return None
+
 def cycleBearerTokenGet(url,headers):
     global bearerTokens
     rateLimitRemaining = None
@@ -75,7 +115,7 @@ def cycleBearerTokenGet(url,headers):
 
 def twitterApiGet(url,btoken=None,authToken=None,guestToken=None):
 
-    if authToken.startswith("oa|"):
+    if authToken != None and authToken.startswith("oa|"):
         url = url.replace("https://x.com/i/api/graphql/","https://api.twitter.com/graphql/")
         authToken = authToken[3:]
         key = authToken.split("|")[0]
@@ -91,7 +131,8 @@ def twitterApiGet(url,btoken=None,authToken=None,guestToken=None):
         response = requests.get(url,headers=headers)
     else:
         if btoken is None:
-            return cycleBearerTokenGet(url,getAuthHeaders(bearer,authToken=authToken,guestToken=guestToken))
+            btoken = v2bearer
+            #return cycleBearerTokenGet(url,getAuthHeaders(bearer,authToken=authToken,guestToken=guestToken))
         headers = getAuthHeaders(btoken,authToken=authToken,guestToken=guestToken)
         response = requests.get(url, headers=headers)
 
@@ -99,7 +140,7 @@ def twitterApiGet(url,btoken=None,authToken=None,guestToken=None):
 
 def getAuthHeaders(btoken,authToken=None,guestToken=None):
     csrfToken=str(uuid.uuid4()).replace('-', '')
-    headers = {"x-twitter-active-user":"yes","x-twitter-client-language":"en","x-csrf-token":csrfToken,"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0"}
+    headers = {"x-twitter-active-user":"yes","x-twitter-client-language":"en","x-csrf-token":csrfToken,"User-Agent":requestUserAgent}
     headers['Authorization'] = btoken
 
     if authToken is not None:
@@ -114,7 +155,7 @@ def getGuestToken():
     global guestToken
     global guestTokenUses
     if guestToken is None:
-        r = requests.get(f"https://{twitterUrl}",headers={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0","Cookie":"night_mode=2"},allow_redirects=False)
+        r = requests.get(f"https://{twitterUrl}",headers={"User-Agent":requestUserAgent,"Cookie":"night_mode=2"},allow_redirects=False)
         m = re.search(gt_pattern, r.text)
         if m is None:
             r = requests.post(f"https://api.{twitterUrl}/1.1/guest/activate.json", headers={"Authorization":bearer})
@@ -235,51 +276,47 @@ def extractStatusV2(url,workaroundTokens):
     # get tweet
     tokens = workaroundTokens
     random.shuffle(tokens)
-    for authToken in tokens:
+    def request_with_token(twid, authToken):
+        vars = json.loads('{"includeTweetImpression":true,"includeHasBirdwatchNotes":false,"includeEditPerspective":false,"rest_ids":["x"],"includeEditControl":true,"includeCommunityTweetRelationship":true,"includeTweetVisibilityNudge":true}')
+        vars['rest_ids'][0] = str(twid)
+        tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2graphql_api}/TweetResultsByIdsQuery?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2Features)}",authToken=authToken)
         try:
-            vars = json.loads('{"includeTweetImpression":true,"includeHasBirdwatchNotes":false,"includeEditPerspective":false,"rest_ids":["x"],"includeEditControl":true,"includeCommunityTweetRelationship":true,"includeTweetVisibilityNudge":true}')
+            rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining")
-            vars['rest_ids'][0] = str(twid)
+            print(f"Twitter Token Rate limit remaining: {rateLimitRemaining}")
-            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2graphql_api}/TweetResultsByIdsQuery?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2Features)}",authToken=authToken)
+        except: # for some reason the header is not always present
-            try:
+            pass
-                rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining")
+        if tweet.status_code == 429:
-                print(f"Twitter Token Rate limit remaining: {rateLimitRemaining}")
+            print("Rate limit reached for token (429)")
-            except: # for some reason the header is not always present
+            # try another token
-                pass
+            raise TwExtractError(400, "Extract error: rate limit reached")
-            if tweet.status_code == 429:
+        output = tweet.json()
-                print("Rate limit reached for token (429)")
-                # try another token
-                continue
-            output = tweet.json()
         
-            if "errors" in output:
+        if "errors" in output:
-                print(f"Error in output: {json.dumps(output['errors'])}")
+            print(f"Error in output: {json.dumps(output['errors'])}")
-                # try another token
+            # try another token
+            raise TwExtractError(400, "Extract error: errors in output - "+json.dumps(output['errors']))
+        entries=output['data']['tweet_results']
+        tweetEntry=None
+        for entry in entries:
+            if 'result' not in entry:
+                print("Tweet result not found in entry")
                 continue
-            entries=output['data']['tweet_results']
+            result = entry['result']
-            tweetEntry=None
+            if '__typename' in result and result['__typename'] == 'TweetWithVisibilityResults':
-            for entry in entries:
+                result=result['tweet']
-                if 'result' not in entry:
+            elif '__typename' in result and result['__typename'] == 'TweetUnavailable':
-                    print("Tweet result not found in entry")
+                if 'reason' in result:
-                    continue
+                    return {'error':'Tweet unavailable: '+result['reason']}
-                result = entry['result']
+                return {'error':'Tweet unavailable'}
-                if '__typename' in result and result['__typename'] == 'TweetWithVisibilityResults':
+            if 'rest_id' in result and result['rest_id'] == twid:
-                    result=result['tweet']
+                tweetEntry=result
-                elif '__typename' in result and result['__typename'] == 'TweetUnavailable':
+                break
-                    if 'reason' in result:
+        tweet=tweetEntry
-                        return {'error':'Tweet unavailable: '+result['reason']}
+        if tweet is None:
-                    return {'error':'Tweet unavailable'}
+            print("Tweet 404")
-                if 'rest_id' in result and result['rest_id'] == twid:
+            return {'error':'Tweet not found (404); May be due to invalid tweet, changes in Twitter\'s API, or a protected account.'}
-                    tweetEntry=result
-                    break
-            tweet=tweetEntry
-            if tweet is None:
-                print("Tweet 404")
-                return {'error':'Tweet not found (404); May be due to invalid tweet, changes in Twitter\'s API, or a protected account.'}
-        except Exception as e:
-            print(f"Exception in extractStatusV2: {str(e)}")
-            continue
         return tweet
-    raise TwExtractError(400, "Extract error")
+    return parallel_token_request(twid, tokens, request_with_token)
 
 def extractStatusV2Android(url,workaroundTokens):
     # get tweet ID
@@ -289,15 +326,13 @@ def extractStatusV2Android(url,workaroundTokens):
     twid = m.group(2)
     if workaroundTokens == None:
         raise TwExtractError(400, "Extract error (no tokens defined)")
-    # get tweet
     tokens = workaroundTokens
     random.shuffle(tokens)
-    for authToken in tokens:
+    def request_with_token(twid, authToken):
         try:
-            
             vars = json.loads('{"referrer":"home","includeTweetImpression":true,"includeHasBirdwatchNotes":false,"isReaderMode":false,"includeEditPerspective":false,"includeEditControl":true,"focalTweetId":0,"includeCommunityTweetRelationship":true,"includeTweetVisibilityNudge":true}')
             vars['focalTweetId'] = int(twid)
-            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{androidGraphql_api}/ConversationTimelineV2?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(androidGraphqlFeatures)}", authToken=authToken)
+            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{androidGraphql_api}/ConversationTimelineV2?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(androidGraphqlFeatures)}", authToken=authToken,btoken=androidBearer)
             try:
                 rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining")
                 print(f"Twitter Android Token Rate limit remaining: {rateLimitRemaining}")
@@ -306,14 +341,18 @@ def extractStatusV2Android(url,workaroundTokens):
             if tweet.status_code == 429:
                 print("Rate limit reached for android token")
                 # try another token
-                continue
+                raise TwExtractError(400, "Extract error: rate limit reached")
             output = tweet.json()
             
             if "errors" in output:
                 print(f"Error in output: {json.dumps(output['errors'])}")
                 # try another token
-                continue
+                raise TwExtractError(400, "Extract error: errors in output - "+json.dumps(output['errors']))
-            entries=output['data']['timeline_response']['instructions'][0]['entries']
+            entries = None
+            for instruction in output['data']['timeline_response']['instructions']:
+                if instruction["__typename"] == "TimelineAddEntries":
+                    entries = instruction['entries']
+                    break
             tweetEntry=None
             for entry in entries:
                 if 'content' not in entry:
@@ -332,11 +371,11 @@ def extractStatusV2Android(url,workaroundTokens):
                 print("Tweet 404")
                 return {'error':'Tweet not found (404); May be due to invalid tweet, changes in Twitter\'s API, or a protected account.'}
         except Exception as e:
-            print(f"Exception in extractStatusV2: {str(e)}")
+            print(f"Exception in extractStatusV2Android: {str(e)}")
-            continue
+            raise TwExtractError(400, "Extract error")
 
         return tweet
-    raise TwExtractError(400, "Extract error")
+    return parallel_token_request(twid, tokens, request_with_token)
 
 def extractStatusV2TweetDetail(url,workaroundTokens):
     # get tweet ID
@@ -349,12 +388,11 @@ def extractStatusV2TweetDetail(url,workaroundTokens):
     # get tweet
     tokens = workaroundTokens
     random.shuffle(tokens)
-    for authToken in tokens:
+    def request_with_token(twid, authToken):
         try:
-            
             vars = json.loads('{"focalTweetId":"0","with_rux_injections":false,"includePromotedContent":true,"withCommunity":true,"withQuickPromoteEligibilityTweetFields":true,"withBirdwatchNotes":true,"withVoice":true,"withV2Timeline":true}')
             vars['focalTweetId'] = str(twid)
-            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{tweetDetailGraphql_api}/TweetDetail?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(tweetDetailGraphqlFeatures)}", authToken=authToken)
+            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{tweetDetailGraphql_api}/TweetDetail?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(tweetDetailGraphqlFeatures)}", authToken=authToken,btoken=v2bearer)
             try:
                 rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining")
                 print(f"Twitter Token Rate limit remaining: {rateLimitRemaining}")
@@ -363,14 +401,18 @@ def extractStatusV2TweetDetail(url,workaroundTokens):
             if tweet.status_code == 429:
                 print("Rate limit reached for token")
                 # try another token
-                continue
+                raise TwExtractError(400, "Extract error: rate limit reached")
             output = tweet.json()
             
             if "errors" in output:
                 print(f"Error in output: {json.dumps(output['errors'])}")
                 # try another token
-                continue
+                raise TwExtractError(400, "Extract error: errors in output - "+json.dumps(output['errors']))
-            entries=output['data']['threaded_conversation_with_injections_v2']['instructions'][0]['entries']
+            entries = None
+            for instruction in output['data']['threaded_conversation_with_injections_v2']['instructions']:
+                if instruction["type"] == "TimelineAddEntries":
+                    entries = instruction['entries']
+                    break
             tweetEntry=None
             for entry in entries:
                 if 'content' not in entry:
@@ -390,12 +432,15 @@ def extractStatusV2TweetDetail(url,workaroundTokens):
                 return {'error':'Tweet not found (404); May be due to invalid tweet, changes in Twitter\'s API, or a protected account.'}
         except Exception as e:
             print(f"Exception in extractStatusV2: {str(e)}")
-            continue
+            raise TwExtractError(400, "Extract error")
 
         return tweet
-    raise TwExtractError(400, "Extract error")
+    return parallel_token_request(twid, tokens, request_with_token)
 
-def extractStatusV2Anon(url,x):
+def extractStatusV2Rest_Anon(url,workaroundTokens):
+    return extractStatusV2Rest(url,None)
+
+def extractStatusV2Rest(url,workaroundTokens):
     # get tweet ID
     m = re.search(pathregex, url)
     if m is None:
@@ -408,7 +453,17 @@ def extractStatusV2Anon(url,x):
     try:
         vars = json.loads('{"tweetId":"0","withCommunity":false,"includePromotedContent":false,"withVoice":false}')
         vars['tweetId'] = str(twid)
-        tweet = requests.get(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", headers=getAuthHeaders(v2bearer,guestToken=guestToken))
+        if workaroundTokens is not None and len(workaroundTokens) > 0:
+            tokens = workaroundTokens
+            random.shuffle(tokens)
+            for authToken in tokens:
+                try:
+                    tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", btoken=v2bearer,authToken=authToken,guestToken=guestToken)
+                except Exception as e:
+                    continue
+        else:
+            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", btoken=v2bearer,guestToken=guestToken)
+        
         try:
             rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining")
             print(f"Twitter Anon Token Rate limit remaining: {rateLimitRemaining}")
@@ -455,7 +510,8 @@ def fixTweetData(tweet):
     return tweet
 
 def extractStatus(url,workaroundTokens=None):
-    methods=[extractStatusV2Anon,extractStatusV2TweetDetail,extractStatusV2Android,extractStatusV2]
+    # TODO: commented out methods are too slow/unreliable at the moment
+    methods=[extractStatusV2Rest_Anon,extractStatusV2,extractStatusV2Rest,extractStatusV2Android]#,extractStatusV2TweetDetail]
     for method in methods:
         try:
             result = method(url,workaroundTokens)
@@ -486,19 +542,28 @@ def extractUser(url,workaroundTokens):
         if authToken.startswith("oa|"): # oauth token not supported atm
             continue
         try:
-            
+            reqHeaders = getAuthHeaders(v2bearer,authToken=authToken)
-            reqHeaders = getAuthHeaders(bearer,authToken=authToken)
             if not useId:
-                user = requests.get(f"https://api.{twitterUrl}/1.1/users/show.json?screen_name={screen_name}",headers=reqHeaders)
+                vars=json.loads('{"screen_name":"","withGrokTranslatedBio":false}')
+                vars['screen_name'] = screen_name
+                user = requests.get(f"https://x.com/i/api/graphql/{userByScreenNameGraphql_api}/UserByScreenName",{'variables':json.dumps(vars),'features':userByScreenNameGraphqlFeatures,'fieldToggles':'{"withAuxiliaryUserLabels":true}'},headers=reqHeaders)
             else:
-                user = requests.get(f"https://api.{twitterUrl}/1.1/users/show.json?user_id={screen_name}",headers=reqHeaders)
+                vars=json.loads('{"userId":"","withGrokTranslatedBio":false}')
+                vars['userId'] = screen_name
+                user = requests.get(f"https://x.com/i/api/graphql/{userByRestIdGraphql_api}/UserByRestId",{'variables':json.dumps(vars),'features':userByScreenNameGraphqlFeatures,'fieldToggles':'{"withAuxiliaryUserLabels":true}'},headers=reqHeaders)
             output = user.json()
             if "errors" in output:
                 # pick the first error and create a twExtractError
                 error = output["errors"][0]
                 raise TwExtractError(error["code"], error["message"])
+            elif 'user' not in output['data']:
+                raise TwExtractError(404, "User not found.")
+            elif output['data']['user']['result']['__typename'] == 'UserUnavailable':
+                raise TwExtractError(404, output['data']['user']['result']['message'])
             return output
         except Exception as e:
+            if hasattr(e,"msg") and ('suspended' in e.msg or e.msg == 'User not found.'):
+                raise e
             continue
     raise TwExtractError(400, "Extract error")
 
@@ -510,25 +575,41 @@ def extractUserFeedFromId(userId,workaroundTokens):
             # TODO: https://api.twitter.com/graphql/x31u1gdnjcqtiVZFc1zWnQ/UserWithProfileTweetsQueryV2?variables={"cursor":"?","includeTweetImpression":true,"includeHasBirdwatchNotes":false,"includeEditPerspective":false,"includeEditControl":true,"count":40,"rest_id":"12","includeTweetVisibilityNudge":true,"autoplay_enabled":true}&features={"longform_notetweets_inline_media_enabled":true,"super_follow_badge_privacy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"super_follow_user_api_enabled":true,"unified_cards_ad_metadata_container_dynamic_card_content_query_enabled":true,"super_follow_tweet_api_enabled":true,"articles_api_enabled":true,"android_graphql_skip_api_media_color_palette":true,"creator_subscriptions_tweet_preview_api_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"tweetypie_unmention_optimization_enabled":true,"longform_notetweets_consumption_enabled":true,"subscriptions_verification_info_enabled":true,"blue_business_profile_image_shape_enabled":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"immersive_video_status_linkable_timestamps":false,"super_follow_exclusive_tweet_notifications_enabled":true}
             continue
         try:
-            vars = json.loads('{"userId":"x","count":20,"includePromotedContent":true,"withQuickPromoteEligibilityTweetFields":true,"withVoice":true,"withV2Timeline":true}')
+            vars = json.loads('{"userId":"0","count":20,"includePromotedContent":true,"withCommunity":true,"withVoice":true}')
             vars['userId'] = str(userId)
             vars['includePromotedContent'] = False # idk if this works
-            reqHeaders = getAuthHeaders(bearer,authToken=authToken)
+            reqHeaders = getAuthHeaders(v2bearer,authToken=authToken)
-            reqHeaders["x-client-transaction-id"] = twUtils.generate_transaction_id("GET","/i/api/graphql/x31u1gdnjcqtiVZFc1zWnQ/UserWithProfileTweetsQueryV2")
+            endpoint=f"/i/api/graphql/{tweetFeedGraphql_api}/UserTweetsAndReplies"
-            feed = requests.get(f"https://{twitterUrl}/i/api/graphql/{tweetFeedGraphql_api}/UserTweets?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(tweetFeedGraphqlFeatures)}", reqHeaders)
+            reqHeaders["x-client-transaction-id"] = twUtils.generate_transaction_id("GET",endpoint)
+            feed = requests.get(f"https://{twitterUrl}{endpoint}", {'variables':json.dumps(vars),'features':tweetFeedGraphqlFeatures,'fieldToggles':'{"withArticlePlainText":false}'},headers=reqHeaders)
+            if feed.status_code == 403 or feed.status_code == 404:
+                raise TwExtractError(403, "Extract error")
             output = feed.json()
             if "errors" in output:
                 # pick the first error and create a twExtractError
                 error = output["errors"][0]
                 raise TwExtractError(error["code"], error["message"])
-            return output
+            timelineInstructions = output['data']['user']['result']['timeline']['timeline']['instructions']
+            #tweetIds=None
+            tweets=None
+            for instruction in timelineInstructions:
+                if 'type' in instruction and instruction['type'] == 'TimelineAddEntries':
+                    entries = instruction['entries']
+                    #tweetIds = []
+                    tweets = []
+                    for entry in entries:
+                        if entry['entryId'].startswith("tweet-"):
+                            # get the tweet ID from the entryId
+                            #tweetId = entry['entryId'].split("-")[1]
+                            #tweetIds.append(tweetId)
+                            tweet = entry['content']['itemContent']['tweet_results']['result']
+                            tweets.append(tweet)
+            return tweets
         except Exception as e:
+            print(f"Exception in extractUserFeedFromId: {str(e)}")
             continue
     raise TwExtractError(400, "Extract error")
 
-def extractUserFeed(username,workaroundTokens):
-    pass
-
 def lambda_handler(event, context):
     if ("queryStringParameters" not in event):
         return {

twExtract/twUtils.py

@@ -2,6 +2,9 @@ import math
 import hashlib
 import base64
 import uuid
+from x_client_transaction import ClientTransaction
+from x_client_transaction.utils import handle_x_migration
+import requests
 digits = "0123456789abcdefghijklmnopqrstuvwxyz"
 
 def baseConversion(x, base):
@@ -31,5 +34,21 @@ def calcSyndicationToken(idStr):
         c = '0'
     return c
 
-def generate_transaction_id(method: str, path: str) -> str:
+def get_twitter_homepage(headers=None):
-    return "?" # not implemented
+    if headers is None:
+        headers = {"Authority": "x.com",
+            "Accept-Language": "en-US,en;q=0.9",
+            "Cache-Control": "no-cache",
+            "Referer": "https://x.com",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
+            "X-Twitter-Active-User": "yes",
+            "X-Twitter-Client-Language": "en"}
+    if 'Authorization' in headers:
+        del headers['Authorization']
+    response = requests.get("https://x.com/home", headers=headers)
+    return response
+
+def generate_transaction_id(method: str, path: str,headers=None) -> str:
+    ct = ClientTransaction(get_twitter_homepage(headers=headers))
+    transaction_id = ct.generate_transaction_id(method=method, path=path)
+    return transaction_id

twitfix.py

@@ -1,5 +1,4 @@
-from weakref import finalize
+from flask import Flask, render_template, request, redirect, abort, Response, send_from_directory, send_file
-from flask import Flask, render_template, request, redirect, abort, Response, send_from_directory, url_for, send_file, make_response, jsonify
 
 from configHandler import config
 remoteCombine='combination_method' in config['config'] and config['config']['combination_method'] != "local"
@@ -15,14 +14,16 @@ import msgs
 import twExtract as twExtract
 from cache import addVnfToLinkCache,getVnfFromLinkCache
 import vxlogging as log
-from utils import getTweetIdFromUrl, pathregex
+from utils import getTweetIdFromUrl, pathregex, determineMediaToEmbed, determineEmbedTweet, BytesIOWrapper, fixMedia
 from vxApi import getApiResponse, getApiUserResponse
 from urllib.parse import urlparse 
 from PyRTF.Elements import Document
 from PyRTF.document.section import Section
 from PyRTF.document.paragraph import Paragraph
-from utils import BytesIOWrapper
 from copy import deepcopy
+import json
+import datetime
+import activity as activitymg
 app = Flask(__name__)
 CORS(app)
 user_agent=""
@@ -34,6 +35,10 @@ staticFiles = { # TODO: Use flask static files instead of this
     "preferences": {"mime": "text/html","path": "preferences.html"},
     "style.css": {"mime": "text/css","path": "style.css"},
     "Roboto-Regular.ttf": {"mime": "font/ttf","path": "Roboto-Regular.ttf"},
+    "gif.png": {"mime": "image/png","path": "richEmbed/gif.png"},
+    "video.png": {"mime": "image/png","path": "richEmbed/video.png"},
+    "image.png": {"mime": "image/png","path": "richEmbed/image.png"},
+    "text.png": {"mime": "image/png","path": "richEmbed/text.png"},
 }
 
 generate_embed_user_agents = [
@@ -61,23 +66,34 @@ def isValidUserAgent(user_agent):
         return True
     return False
 
-def fixMedia(mediaInfo):
-    # This is for the iOS Discord app, which has issues when serving URLs ending in .mp4 (https://github.com/dylanpdx/BetterTwitFix/issues/210)
-    if 'video.twimg.com' not in mediaInfo['url'] or 'convert?url=' in mediaInfo['url']:
-        return mediaInfo
-    mediaInfo['url'] = mediaInfo['url'].replace("https://video.twimg.com",f"{config['config']['url']}/tvid").replace(".mp4","")
-    return mediaInfo
-
 def message(text):
-    return render_template(
+    rendered = render_template(
         'default.html', 
         message = text, 
         color   = config['config']['color'], 
         appname = config['config']['appname'], 
         repo    = config['config']['repo'], 
         url     = config['config']['url'] )
+    return Response(rendered, mimetype='text/html',headers={"Cache-Tag": "message", "Cache-Control": "max-age=1760, public"})
 
-def renderImageTweetEmbed(tweetData,image,appnameSuffix=""):
+def generateActivityLink(tweetData,media=None,mediatype=None,embedIndex=-1):
+    global user_agent
+    if 'LegacyEmbed' in user_agent: # TODO: Clean up; This is a hacky fix to make the new activity embed not trigger
+        return None
+    try:
+        embedIndex = embedIndex+1
+        return f"{config['config']['url']}/users/{tweetData['user_screen_name']}/statuses/{str(embedIndex)}{tweetData['tweetID']}"
+    except Exception as e:
+        log.error("Error generating activity link: "+str(e))
+        return None
+
+def getAppName(tweetData,appnameSuffix=""):
+    appName = config['config']['appname']+appnameSuffix
+    if 'Discord' not in user_agent:
+        appName = msgs.formatProvider(config['config']['appname']+appnameSuffix,tweetData)
+    return appName
+
+def renderImageTweetEmbed(tweetData,image,appnameSuffix="",embedIndex=-1):
     qrt = tweetData['qrt']
     embedDesc = msgs.formatEmbedDesc("Image",tweetData['text'],qrt,tweetData['pollData'])
 
@@ -91,15 +107,22 @@ def renderImageTweetEmbed(tweetData,image,appnameSuffix=""):
                     desc=embedDesc,
                     urlEncodedDesc=urllib.parse.quote(embedDesc),
                     tweetLink=f'https://twitter.com/{tweetData["user_screen_name"]}/status/{tweetData["tweetID"]}',
-                    appname=msgs.formatProvider(config['config']['appname']+appnameSuffix,tweetData),
+                    appname=getAppName(tweetData,appnameSuffix),
-                    color=config['config']['color']
+                    color=config['config']['color'],
+                    sicon="image",
+                    activityLink=generateActivityLink(tweetData,image,"image/png",embedIndex)
                     )
 
-def renderVideoTweetEmbed(tweetData,mediaInfo,appnameSuffix=""):
+def renderVideoTweetEmbed(tweetData,mediaInfo,appnameSuffix="",embedIndex=-1):
     qrt = tweetData['qrt']
     embedDesc = msgs.formatEmbedDesc("Video",tweetData['text'],qrt,tweetData['pollData'])
 
     mediaInfo=fixMedia(mediaInfo)
+
+    appName = config['config']['appname']+appnameSuffix
+    if 'Discord' not in user_agent:
+        appName = msgs.formatProvider(config['config']['appname']+appnameSuffix,tweetData)
+
     return render_template("video.html",
                     tweet=tweetData,
                     media=mediaInfo,
@@ -107,26 +130,32 @@ def renderVideoTweetEmbed(tweetData,mediaInfo,appnameSuffix=""):
                     desc=embedDesc,
                     urlEncodedDesc=urllib.parse.quote(embedDesc),
                     tweetLink=f'https://twitter.com/{tweetData["user_screen_name"]}/status/{tweetData["tweetID"]}',
-                    appname=msgs.formatProvider(config['config']['appname']+appnameSuffix,tweetData),
+                    appname=appName,
-                    color=config['config']['color']
+                    color=config['config']['color'],
+                    sicon="video",
+                    activityLink=generateActivityLink(tweetData,mediaInfo['url'],"video/mp4",embedIndex)
                     )
 
 def renderTextTweetEmbed(tweetData,appnameSuffix=""):
     qrt = tweetData['qrt']
     embedDesc = msgs.formatEmbedDesc("Text",tweetData['text'],qrt,tweetData['pollData'])
+
     return render_template("text.html",
                     tweet=tweetData,
                     host=config['config']['url'],
                     desc=embedDesc,
                     urlEncodedDesc=urllib.parse.quote(embedDesc),
                     tweetLink=f'https://twitter.com/{tweetData["user_screen_name"]}/status/{tweetData["tweetID"]}',
-                    appname=msgs.formatProvider(config['config']['appname']+appnameSuffix,tweetData),
+                    appname=getAppName(tweetData,appnameSuffix),
-                    color=config['config']['color']
+                    color=config['config']['color'],
+                    activityLink=generateActivityLink(tweetData),
+                    sicon="text"
                     )
 
 def renderArticleTweetEmbed(tweetData,appnameSuffix=""):
     articlePreview=tweetData['article']["title"]+"\n\n"+tweetData['article']["preview_text"]+"…"
     embedDesc = msgs.formatEmbedDesc("Image",articlePreview,None,None)
+
     return render_template("image.html",
                     tweet=tweetData,
                     pic=[tweetData['article']["image"]],
@@ -134,8 +163,9 @@ def renderArticleTweetEmbed(tweetData,appnameSuffix=""):
                     desc=embedDesc,
                     urlEncodedDesc=urllib.parse.quote(embedDesc),
                     tweetLink=f'https://twitter.com/{tweetData["user_screen_name"]}/status/{tweetData["tweetID"]}',
-                    appname=msgs.formatProvider(config['config']['appname']+appnameSuffix,tweetData),
+                    appname=getAppName(tweetData,appnameSuffix),
-                    color=config['config']['color']
+                    color=config['config']['color'],
+                    sicon="image"
                     )
 
 def renderUserEmbed(userData,appnameSuffix=""):
@@ -166,25 +196,83 @@ def oembedend():
     provName = request.args.get("provider",None)
     return  oEmbedGen(desc, user, link, ttype,providerName=provName)
 
+@app.route('/activity.json')
+def activity():
+    tweetId = request.args.get("id", None)
+    publishedDate = request.args.get("published", None)
+    likes = request.args.get("likes", None)
+    retweets = request.args.get("retweets", None)
+    userAttrTo = request.args.get("user", None)
+    content = request.args.get("content", None)
+    attachments = json.loads(request.args.get("attachments", "[]"))
+
+    ##
+
+    attachmentsRaw = []
+    for attachment in attachments:
+        attachmentsRaw.append({
+            "type": "Document",
+            "mediaType": attachment["type"],
+            "url": attachment["url"],
+            "preview_url": "https://pbs.twimg.com/ext_tw_video_thumb/1906073839441735680/pu/img/2xqg6tlK9mK0mSOR.jpg",
+    })
+
+    return {
+        "id": "https://x.com/i/status/"+tweetId,
+        "type": "Note",
+        "summary": None,
+        "inReplyTo": None,
+        "published": publishedDate,
+        "url": "https://x.com/i/status/"+tweetId,
+        "attributedTo": userAttrTo,
+        "content": content,
+        "attachment": attachmentsRaw,
+        "likes": {
+            "type": "Collection",
+            "totalItems": int(likes)
+        },
+        "shares": {
+            "type": "Collection",
+            "totalItems": int(retweets)
+        },
+    }
+
+@app.route('/user.json')
+def userJson():
+
+    screen_name = request.args.get("screen_name", None)
+    name = request.args.get("name", None)
+    pfp = request.args.get("pfp", None)
+
+    return {
+        "id": screen_name,
+        "type": "Person",
+        "preferredUsername": screen_name,
+        "name": name,
+        "summary": "",
+        "url": "https://x.com/"+screen_name,
+        "tag": [],
+        "attachment": [],
+        "icon": {
+            "type": "Image",
+            "mediaType": "image/jpeg",
+            "url": pfp
+        },
+    }
+
 def getTweetData(twitter_url,include_txt="false",include_rtf="false"):
     cachedVNF = getVnfFromLinkCache(twitter_url)
     if cachedVNF is not None and include_txt == "false" and include_rtf == "false":
         return cachedVNF
 
     try:
-        rawTweetData = twExtract.extractStatusV2Anon(twitter_url, None)
+        if config['config']['workaroundTokens'] is not None:
+            workaroundTokens = config['config']['workaroundTokens'].split(",")
+        else:
+            workaroundTokens = None
+        rawTweetData = twExtract.extractStatus(twitter_url,workaroundTokens=workaroundTokens)
     except:
         rawTweetData = None
-    if rawTweetData is None:
-        try:
-            if config['config']['workaroundTokens'] is not None:
-                workaroundTokens = config['config']['workaroundTokens'].split(",")
-            else:
-                workaroundTokens = None
-            
-            rawTweetData = twExtract.extractStatus(twitter_url,workaroundTokens=workaroundTokens)
-        except:
-            rawTweetData = None
     if rawTweetData == None or 'error' in rawTweetData:
         return None
 
@@ -197,27 +285,36 @@ def getTweetData(twitter_url,include_txt="false",include_rtf="false"):
         addVnfToLinkCache(twitter_url,tweetData)
     return tweetData
 
-def getUserData(twitter_url):
+def getUserData(twitter_url,includeFeed=False):
-    rawUserData = twExtract.extractUser(twitter_url,workaroundTokens=config['config']['workaroundTokens'].split(','))
+    if config['config']['workaroundTokens'] is not None:
+        workaroundTokens = config['config']['workaroundTokens'].split(",")
+    else:
+        workaroundTokens = None
+    rawUserData = twExtract.extractUser(twitter_url,workaroundTokens=workaroundTokens)
     userData = getApiUserResponse(rawUserData)
-    return userData
 
-def determineEmbedTweet(tweetData):
+    if includeFeed:
-    # Determine which tweet, i.e main or QRT, to embed the media from.
+        if userData['protected']:
-    # if there is no QRT, return the main tweet => default behavior
+            userData['latest_tweets']=[]
-    # if both don't have media, return the main tweet => embedding qrt text will be handled in the embed description
+        else:
-    # if both have media, return the main tweet => priority is given to the main tweet's media
+            feed = twExtract.extractUserFeedFromId(userData['id'],workaroundTokens=workaroundTokens)
-    # if only the QRT has media, return the QRT => show the QRT's media, not the main tweet's
+            apiFeed = []
-    # if only the main tweet has media, return the main tweet => show the main tweet's media, embedding QRT text will be handled in the embed description
+            for tweet in feed:
-    if tweetData['qrt'] is None:
+                apiFeed.append(getApiResponse(tweet))
-        return tweetData
+            userData['latest_tweets'] = apiFeed
-    if tweetData['qrt']['hasMedia'] and not tweetData['hasMedia']:
+
-        return tweetData['qrt']
+    return userData
-    return tweetData
 
 @app.route('/<path:sub_path>') # Default endpoint used by everything
 def twitfix(sub_path):
+    global user_agent
+    user_agent = request.headers.get('User-Agent', None)
+    if user_agent is None:
+        user_agent = "unknown"
+
     isApiRequest=request.url.startswith("https://api.vx") or request.url.startswith("http://api.vx")
+    if not isApiRequest and (request.url.startswith("https://l.vx") or request.url.startswith("https://old.vx")) and "Discord" in user_agent:
+        user_agent = user_agent.replace("Discord","LegacyEmbed") # TODO: Clean up; This is a hacky fix to make the new activity embed not trigger
     if sub_path in staticFiles:
         if 'path' not in staticFiles[sub_path] or staticFiles[sub_path]["path"] == None:
             staticFiles[sub_path]["path"] = sub_path
@@ -235,7 +332,15 @@ def twitfix(sub_path):
             username=sub_path.split("/")[0]
             extra = sub_path.split("/")[1]
         if extra in [None,"with_replies","media","likes","highlights","superfollows","media",''] and username != "" and username != None:
-            userData = getUserData(f"https://twitter.com/{username}")
+            try:
+                userData = getUserData(f"https://twitter.com/{username}","with_tweets" in request.args)
+            except twExtract.TwExtractError as e:
+                if isApiRequest:
+                    status=500
+                    if 'not found' in e.msg:
+                        status=404
+                    return Response(json.dumps({"error": e.msg}), status=status,mimetype='application/json')
+                return message("Error getting user data: "+str(e.msg))
             if isApiRequest:
                 if userData is None:
                     abort(404)
@@ -264,6 +369,12 @@ def twitfix(sub_path):
     if 'qrtURL' in tweetData and tweetData['qrtURL'] is not None:
         qrt = getTweetData(tweetData['qrtURL'])
     tweetData['qrt'] = qrt
+
+    retweet = None
+    if 'retweetURL' in tweetData and tweetData['retweetURL'] is not None:
+        retweet = getTweetData(tweetData['retweetURL'])
+    tweetData['retweet'] = retweet
+
     tweetData = deepcopy(tweetData)
     log.success("Tweet Data Get success")
     if '?' in request.url:
@@ -300,48 +411,36 @@ def twitfix(sub_path):
     if isApiRequest: # Directly return the API response if the request is from the API
         return tweetData
     elif directEmbed: # direct embed
+        embeddingMedia = tweetData['hasMedia']
+        renderMedia = None
+        if embeddingMedia:
+            renderMedia = determineMediaToEmbed(tweetData,embedIndex,convertGif=False)
         # direct embeds should always prioritize the main tweet, so don't check for qrt
         # determine what type of media we're dealing with
-        if not tweetData['hasMedia'] and qrt is None:
+        if not embeddingMedia and qrt is None:
             return renderTextTweetEmbed(tweetData)
-        elif tweetData['allSameType'] and tweetData['media_extended'][0]['type'] == "image" and embedIndex == -1 and tweetData['combinedMediaUrl'] != None:
-            return render_template("rawimage.html",media={"url":tweetData['combinedMediaUrl']})
         else:
-            # this means we have mixed media or video, and we're only going to embed one
+            if renderMedia['type'] == "image":
-            if embedIndex == -1: # if the user didn't specify an index, we'll just use the first one
+                return Response(render_template("rawimage.html",media=renderMedia),headers={"Cache-Tag": "embed"})
-                embedIndex = 0
+            elif renderMedia['type'] == "video" or renderMedia['type'] == "gif":
-            media = tweetData['media_extended'][embedIndex]
+                return Response(render_template("rawvideo.html",media=renderMedia),headers={"Cache-Tag": "embed"})
-            media=fixMedia(media)
-            if media['type'] == "image":
-                return render_template("rawimage.html",media=media)
-            elif media['type'] == "video" or media['type'] == "gif":
-                return render_template("rawvideo.html",media=media)
     else: # full embed
         embedTweetData = determineEmbedTweet(tweetData)
+        embeddingMedia = embedTweetData['hasMedia']
+        
         if "article" in embedTweetData and embedTweetData["article"] is not None:
-            return renderArticleTweetEmbed(tweetData," - See original tweet for full article")
+            return Response(renderArticleTweetEmbed(tweetData," • See original tweet for full article"),headers={"Cache-Tag": "embed"})
-        elif not embedTweetData['hasMedia']:
+        elif not embeddingMedia:
-            return renderTextTweetEmbed(tweetData)
+            return Response(renderTextTweetEmbed(tweetData),headers={"Cache-Tag": "embed"})
-        elif embedTweetData['allSameType'] and embedTweetData['media_extended'][0]['type'] == "image" and embedIndex == -1 and embedTweetData['combinedMediaUrl'] != None:
-            return renderImageTweetEmbed(tweetData,embedTweetData['combinedMediaUrl'],appnameSuffix=" - See original tweet for full quality")
         else:
-            # this means we have mixed media or video, and we're only going to embed one
+            media = determineMediaToEmbed(embedTweetData,embedIndex)
-            if embedIndex == -1: # if the user didn't specify an index, we'll just use the first one
+            suffix=""
-                embedIndex = 0
+            if "suffix" in media:
-            media = embedTweetData['media_extended'][embedIndex]
+                suffix = media["suffix"]
-            if len(embedTweetData["media_extended"]) > 1:
-                suffix = f' - Media {embedIndex+1}/{len(embedTweetData["media_extended"])}'
-            else:
-                suffix = ''
             if media['type'] == "image":
-                return renderImageTweetEmbed(tweetData,media['url'] , appnameSuffix=suffix)
+                return Response(renderImageTweetEmbed(tweetData,media['url'] , appnameSuffix=suffix,embedIndex=embedIndex),headers={"Cache-Tag": "embed"})
             elif media['type'] == "video" or media['type'] == "gif":
-                if media['type'] == "gif":
+                return Response(renderVideoTweetEmbed(tweetData,media,appnameSuffix=suffix,embedIndex=embedIndex),headers={"Cache-Tag": "embed"})
-                    if config['config']['gifConvertAPI'] != "" and config['config']['gifConvertAPI'] != "none":
-                        vurl=media['originalUrl'] if 'originalUrl' in media else media['url']
-                        media['url'] = config['config']['gifConvertAPI'] + "/convert?url=" + vurl
-                        suffix += " - GIF"
-                return renderVideoTweetEmbed(tweetData,media,appnameSuffix=suffix)
 
     return message(msgs.failedToScan)
 
@@ -377,6 +476,29 @@ def rendercombined():
     imgIo.seek(0)
     return send_file(imgIo, mimetype='image/jpeg',max_age=86400)
 
+@app.route("/api/v1/statuses/<string:tweet_id>")
+def api_v1_status(tweet_id):
+    embedIndex = int(tweet_id[0])-1
+    tweet_id = int(tweet_id[1:])
+    twitter_url=f"https://twitter.com/i/status/{tweet_id}"
+    tweetData = getTweetData(twitter_url)
+    if tweetData is None:
+        log.error("Tweet Data Get failed for "+twitter_url)
+        return message(msgs.failedToScan)
+    qrt = None
+    if 'qrtURL' in tweetData and tweetData['qrtURL'] is not None:
+        qrt = getTweetData(tweetData['qrtURL'])
+    tweetData['qrt'] = qrt
+
+    retweet = None
+    if 'retweetURL' in tweetData and tweetData['retweetURL'] is not None:
+        retweet = getTweetData(tweetData['retweetURL'])
+    tweetData['retweet'] = retweet
+
+    if tweetData is None:
+        abort(500) # this should cause Discord to fall back to the default embed
+    return activitymg.tweetDataToActivity(tweetData,embedIndex)
+
 def oEmbedGen(description, user, video_link, ttype,providerName=None):
     if providerName == None:
         providerName = config['config']['appname']
@@ -394,4 +516,4 @@ def oEmbedGen(description, user, video_link, ttype,providerName=None):
 
 if __name__ == "__main__":
     app.config['SERVER_NAME']='localhost:80'
-    app.run(host='0.0.0.0')
+    app.run(host='0.0.0.0', port=8080)

twitfix.service

@@ -3,11 +3,11 @@ Description=Init file for twitfix uwsgi instance
 After=network.target
 
 [Service]
-User=dylan
+User=carbonara
-Group=dylan
+Group=carbonara
-WorkingDirectory=/home/dylan/BetterTwitFix
+WorkingDirectory=/home/carbonara/twitter/BetterTwitFix
-Environment="PATH=/home/dylan/BetterTwitFix/venv/bin"
+Environment="PATH=/home/carbonara/twitter/BetterTwitFix/venv/bin"
-ExecStart=/home/dylan/BetterTwitFix/venv/bin/uwsgi --ini twitfix.ini
+ExecStart=/home/carbonara/twitter/BetterTwitFix/venv/bin/uwsgi --ini twitfix.ini
 Restart=always
 RestartSec=3
 

utils.py

@@ -1,8 +1,9 @@
 import re
 import io
+from configHandler import config
 
 pathregex = re.compile("\\w{1,15}\\/(status|statuses)\\/(\\d{2,20})")
-endTCOregex = re.compile("(^.*?) +https:\/\/t.co\/.*?$")
+endTCOregex = re.compile("(^.*?)[ \n]+https:\/\/t.co\/.*?$",flags=re.DOTALL)
 
 def getTweetIdFromUrl(url):
     match = pathregex.search(url)
@@ -41,3 +42,50 @@ class BytesIOWrapper(io.BufferedReader):
 
     def peek(self, size=-1):
         return self._encoding_call('peek', size)
+    
+def fixMedia(mediaInfo):
+    # This is for the iOS Discord app, which has issues when serving URLs ending in .mp4 (https://github.com/dylanpdx/BetterTwitFix/issues/210)
+    if 'video.twimg.com' not in mediaInfo['url'] or 'convert?url=' in mediaInfo['url'] or 'originalUrl' in mediaInfo:
+        return mediaInfo
+    mediaInfo["originalUrl"] = mediaInfo['url']
+    mediaInfo['url'] = mediaInfo['url'].replace("https://video.twimg.com",f"{config['config']['url']}/tvid").replace(".mp4","")
+    return mediaInfo
+
+def determineEmbedTweet(tweetData):
+    # Determine which tweet, i.e main or QRT, to embed the media from.
+    # if there is no QRT, return the main tweet => default behavior
+    # if both don't have media, return the main tweet => embedding qrt text will be handled in the embed description
+    # if both have media, return the main tweet => priority is given to the main tweet's media
+    # if only the QRT has media, return the QRT => show the QRT's media, not the main tweet's
+    # if only the main tweet has media, return the main tweet => show the main tweet's media, embedding QRT text will be handled in the embed description
+    if tweetData['qrt'] is None:
+        return tweetData
+    if tweetData['qrt']['hasMedia'] and not tweetData['hasMedia']:
+        return tweetData['qrt']
+    return tweetData
+
+def determineMediaToEmbed(tweetData,embedIndex = -1,convertGif = True):
+    if tweetData['allSameType'] and tweetData['media_extended'][0]['type'] == "image" and embedIndex == -1 and tweetData['combinedMediaUrl'] != None:
+        return {"url":tweetData['combinedMediaUrl'],"type":"image"}
+    else:
+        # this means we have mixed media or video, and we're only going to embed one
+        if embedIndex == -1: # if the user didn't specify an index, we'll just use the first one
+            embedIndex = 0
+        media = tweetData['media_extended'][embedIndex]
+        media=fixMedia(media)
+        suffix=""
+        if len(tweetData["media_extended"]) > 1:
+            suffix = f' • Media {embedIndex+1}/{len(tweetData["media_extended"])}'
+        else:
+            suffix = ''
+        media["suffix"] = suffix
+        if media['type'] == "image":
+            return media
+        elif media['type'] == "video" or media['type'] == "gif":
+            if media['type'] == "gif" and convertGif:
+                if config['config']['gifConvertAPI'] != "" and config['config']['gifConvertAPI'] != "none":
+                    vurl=media['originalUrl'] if 'originalUrl' in media else media['url']
+                    media['url'] = config['config']['gifConvertAPI'] + "/convert?url=" + vurl
+                    suffix += " • GIF"
+                    media["suffix"] = suffix
+        return media

vxApi.py

@@ -1,29 +1,75 @@
 import html
 from datetime import datetime
+from flask import json
 from configHandler import config
 from utils import stripEndTCO
 
 def getApiUserResponse(user):
+    userResult = user["data"]["user"]["result"]
     return {
-        "id": user["id"],
+        "id": int(userResult["rest_id"]),
-        "screen_name": user["screen_name"],
+        "screen_name": userResult["core"]["screen_name"],
-        "name": user["name"],
+        "name": userResult["core"]["name"],
-        "profile_image_url": user["profile_image_url_https"],
+        "profile_image_url": userResult['avatar']["image_url"],
-        "description": user["description"],
+        "description": userResult["legacy"]["description"],
-        "location": user["location"],
+        "location": userResult["location"]["location"],
-        "followers_count": user["followers_count"],
+        "followers_count": userResult["legacy"]["followers_count"],
-        "following_count": user["friends_count"],
+        "following_count": userResult["legacy"]["friends_count"],
-        "tweet_count": user["statuses_count"],
+        "tweet_count": userResult["legacy"]["statuses_count"],
-        "created_at": user["created_at"],
+        "created_at": userResult["core"]["created_at"],
-        "protected": user["protected"],
+        "protected": userResult["privacy"]["protected"],
+        "fetched_on": int(datetime.now().timestamp()),
     }
 
+def getBestMediaUrl(mediaList):
+    # find the highest bitrate
+    best_bitrate = -1
+    besturl=""
+    for j in mediaList:
+        if j['content_type'] == "video/mp4" and '/hevc/' not in j["url"] and j['bitrate'] > best_bitrate:
+            besturl = j["url"]
+            best_bitrate = j['bitrate']
+    if "?tag=" in besturl:
+        besturl = besturl[:besturl.index("?tag=")]
+    return besturl
+
+def getExtendedVideoOrGifInfo(mediaEntry):
+    videoInfo = mediaEntry["video_info"]
+    info = {
+        "url": getBestMediaUrl(videoInfo["variants"]),
+        "type": "gif" if mediaEntry.get("type", "") == "animated_gif" else "video",
+        "size": {
+            "width": mediaEntry['original_info']["width"],
+            "height": mediaEntry['original_info']["height"]
+        },
+        "duration_millis": videoInfo.get("duration_millis", 0),
+        "thumbnail_url": mediaEntry.get("media_url_https", None),
+        "altText": mediaEntry.get("ext_alt_text", None),
+        "id_str": mediaEntry.get("id_str", None)
+    }
+    return info
+
+def getExtendedImageInfo(mediaEntry):
+    info = {
+        "url": mediaEntry.get("media_url_https", None),
+        "type": "image",
+        "size": {
+            "width": mediaEntry["original_info"]["width"],
+            "height": mediaEntry["original_info"]["height"]
+        },
+        "thumbnail_url": mediaEntry.get("media_url_https", None),
+        "altText": mediaEntry.get("ext_alt_text", None),
+        "id_str": mediaEntry.get("id_str", None)
+    }
+    return info
+    
 def getApiResponse(tweet,include_txt=False,include_rtf=False):
     tweetL = tweet["legacy"]
     if "user_result" in tweet["core"]:
-        userL = tweet["core"]["user_result"]["result"]["legacy"]
+        user = tweet["core"]["user_result"]["result"]
     elif "user_results" in tweet["core"]:
-        userL = tweet["core"]["user_results"]["result"]["legacy"]
+        user = tweet["core"]["user_results"]["result"]
+    userL = user["legacy"]
     media=[]
     media_extended=[]
     hashtags=[]
@@ -31,6 +77,14 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False):
     oldTweetVersion = False
     tweetArticle=None
     lang=None
+
+    if "screen_name" not in userL:
+        userL["screen_name"] = user["core"]["screen_name"]
+    if "name" not in userL:
+        userL["name"] = user["core"]["name"]
+    if "profile_image_url_https" not in userL:
+        userL["profile_image_url_https"] = user["avatar"]["image_url"]
+
     #editedTweet=False
     try:
         if "birdwatch_pivot" in tweet:
@@ -58,61 +112,55 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False):
             for i in tmedia:
                 extendedInfo={}
                 if "video_info" in i:
-                    # find the highest bitrate
+                    extendedInfo = getExtendedVideoOrGifInfo(i)
-                    best_bitrate = -1
+                    media.append(extendedInfo["url"])
-                    besturl=""
-                    for j in i["video_info"]["variants"]:
-                        if j['content_type'] == "video/mp4" and '/hevc/' not in j["url"] and j['bitrate'] > best_bitrate:
-                            besturl = j['url']
-                            best_bitrate = j['bitrate']
-                    if "?tag=" in besturl:
-                        besturl = besturl[:besturl.index("?tag=")]
-                    media.append(besturl)
-                    extendedInfo["url"] = besturl
-                    extendedInfo["type"] = "video"
-                    if (i["type"] == "animated_gif"):
-                        extendedInfo["type"] = "gif"
-                    altText = None
-                    extendedInfo["size"] = {"width":i["original_info"]["width"],"height":i["original_info"]["height"]}
-                    if "ext_alt_text" in i:
-                        altText=i["ext_alt_text"]
-                    if "duration_millis" in i["video_info"]:
-                        extendedInfo["duration_millis"] = i["video_info"]["duration_millis"]
-                    else:
-                        extendedInfo["duration_millis"] = 0
-                    extendedInfo["thumbnail_url"] = i["media_url_https"]
-                    extendedInfo["altText"] = altText
                     media_extended.append(extendedInfo)
                 else:
-                    media.append(i["media_url_https"])
+                    extendedInfo = getExtendedImageInfo(i)
-                    extendedInfo["url"] = i["media_url_https"]
-                    altText=None
-                    if "ext_alt_text" in i:
-                        altText=i["ext_alt_text"]
-                    extendedInfo["altText"] = altText
-                    extendedInfo["type"] = "image"
-                    extendedInfo["size"] = {"width":i["original_info"]["width"],"height":i["original_info"]["height"]}
-                    extendedInfo["thumbnail_url"] = i["media_url_https"]
                     media_extended.append(extendedInfo)
+                    media.append(extendedInfo["url"])
 
         if "hashtags" in tweetL["entities"]:
             for i in tweetL["entities"]["hashtags"]:
                 hashtags.append(i["text"])
-    elif "card" in tweet and tweet['card']['name'] == "player":
+    elif "card" in tweet or "tweet_card" in tweet:
-        width = None
+        cardData = tweet["card" if "card" in tweet else "tweet_card"]
-        height = None
+        bindingValues = None
-        vidUrl = None
+        if 'binding_values' in cardData:
-        for i in tweet['card']['binding_values']:
+            bindingValues = cardData['binding_values']
-            if i['key'] == 'player_stream_url':
+        elif 'legacy' in cardData and 'binding_values' in cardData['legacy']:
-                vidUrl = i['value']['string_value']
+            bindingValues = cardData['legacy']['binding_values']
-            elif i['key'] == 'player_width':
+        if bindingValues != None:
-                width = int(i['value']['string_value'])
+            if 'name' in cardData and cardData['name'] == "player":
-            elif i['key'] == 'player_height':
+                width = None
-                height = int(i['value']['string_value'])
+                height = None
-        if vidUrl != None and width != None and height != None:
+                vidUrl = None
-            media.append(vidUrl)
+                for i in bindingValues:
-            media_extended.append({"url":vidUrl,"type":"video","size":{"width":width,"height":height}})
+                    if i['key'] == 'player_stream_url':
-
+                        vidUrl = i['value']['string_value']
+                    elif i['key'] == 'player_width':
+                        width = int(i['value']['string_value'])
+                    elif i['key'] == 'player_height':
+                        height = int(i['value']['string_value'])
+                if vidUrl != None and width != None and height != None:
+                    media.append(vidUrl)
+                    media_extended.append({"url":vidUrl,"type":"video","size":{"width":width,"height":height}})
+            else:
+                for i in bindingValues:
+                    if i['key'] == 'unified_card' and 'value' in i and 'string_value' in i['value']:
+                        cardData = json.loads(i['value']['string_value'])
+                        media_key = cardData['component_objects']['media_1']['data']['id']
+                        media_entry = cardData['media_entities'][media_key]
+                        extendedInfo = getExtendedVideoOrGifInfo(media_entry)
+                        media.append(extendedInfo['url'])
+                        media_extended.append(extendedInfo)
+                        break
+                    elif i['key'] == 'photo_image_full_size_large' and 'value' in i and 'image_value' in i['value']:
+                        imgData = i['value']['image_value']
+                        imgurl = imgData['url']
+                        media.append(imgurl)
+                        media_extended.append({"url":imgurl,"type":"image","size":{"width":imgData['width'],"height":imgData['height']}})
+                        break
     if "article" in tweet:
         try:
             result = tweet["article"]["article_results"]["result"]
@@ -143,6 +191,10 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False):
     if 'quoted_status_id_str' in tweetL:
         qrtURL = "https://twitter.com/i/status/" + tweetL['quoted_status_id_str']
 
+    retweetURL = None
+    if 'retweeted_status_result' in tweetL:
+        retweetURL = "https://twitter.com/i/status/" + tweetL['retweeted_status_result']['result']['rest_id']
+
     if 'possibly_sensitive' not in tweetL:
         tweetL['possibly_sensitive'] = False
 
@@ -155,6 +207,8 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False):
 
     if 'entities' in tweetL and 'urls' in tweetL['entities']:
         for eurl in tweetL['entities']['urls']:
+            if 'expanded_url' not in eurl:
+                continue
             if "/status/" in eurl["expanded_url"] and eurl["expanded_url"].startswith("https://twitter.com/"):
                 twText = twText.replace(eurl["url"], "")
             else:
@@ -210,7 +264,7 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False):
                 totalVotes += option["votes"]
                 pollData["options"].append(option)
         for i in pollData["options"]:
-            i["percent"] = round((i["votes"]/totalVotes)*100,2)
+            i["percent"] = round((i["votes"]/totalVotes)*100,2) if totalVotes > 0 else 0
         
     if 'lang' in tweetL:
         lang = tweetL['lang']
@@ -219,6 +273,10 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False):
     if 'in_reply_to_screen_name' in tweetL and tweetL['in_reply_to_screen_name'] != None:
         replyingTo = tweetL['in_reply_to_screen_name']
 
+    replyingToID = None
+    if 'in_reply_to_status_id_str' in tweetL and tweetL['in_reply_to_status_id_str'] != None:
+        replyingToID = tweetL['in_reply_to_status_id_str']
+
     apiObject = {
         "text": twText,
         "likes": tweetL["favorite_count"],
@@ -244,6 +302,9 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False):
         "article": tweetArticle,
         "lang": lang,
         "replyingTo": replyingTo,
+        "replyingToID": replyingToID,
+        "fetched_on": int(datetime.now().timestamp()),
+        "retweetURL":retweetURL,
     }
     try:
         apiObject["date_epoch"] = int(datetime.strptime(tweetL["created_at"], "%a %b %d %H:%M:%S %z %Y").timestamp())

vx_testdata.py

@@ -6,26 +6,30 @@ testVideoTweet="https://twitter.com/pdxdylan/status/1540398733669666818"
 testMediaTweet="https://twitter.com/pdxdylan/status/1534672932106035200"
 testMultiMediaTweet="https://twitter.com/pdxdylan/status/1532006436703715331"
 testQRTTweet="https://twitter.com/pdxdylan/status/1611477137319514129"
-testQrtCeptionTweet="https://twitter.com/CatherineShu/status/585253766271672320" # TODO: tweet is deleted
+testQrtCeptionTweet="https://twitter.com/CatherineShu/status/585253766271672320"
 testQrtVideoTweet="https://twitter.com/pdxdylan/status/1674561759422578690"
 testNSFWTweet="https://twitter.com/kuyacoy/status/1581185279376838657"
 testPollTweet="https://twitter.com/norm/status/651169346518056960"
 testMixedMediaTweet="https://twitter.com/bigbeerfest/status/1760638922084741177"
 testVinePlayerTweet="https://twitter.com/Roblox/status/583302104342638592"
+testRetweetTweet="https://twitter.com/pdxdylan/status/1828570470222045294"
 
-testTextTweet_compare={'text': 'just setting up my twttr', 'date': 'Tue Mar 21 20:50:14 +0000 2006', 'tweetURL': 'https://twitter.com/jack/status/20', 'tweetID': '20', 'conversationID': '20', 'mediaURLs': [], 'media_extended': [], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': False, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'date_epoch': 1142974214}
+testTextTweet_compare={'text': 'just setting up my twttr', 'date': 'Tue Mar 21 20:50:14 +0000 2006', 'tweetURL': 'https://twitter.com/jack/status/20', 'tweetID': '20', 'conversationID': '20', 'mediaURLs': [], 'media_extended': [], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': False, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1142974214}
-testVideoTweet_compare={'text': 'TikTok embeds on Discord/Telegram bait you with a fake play button, but to see the actual video you have to go to their website.\nAs a request from a friend, I made it so that if you add "vx" before "tiktok" on any link, it fixes that. https://t.co/QYpiVXUIrW', 'date': 'Fri Jun 24 18:17:31 +0000 2022', 'tweetURL': 'https://twitter.com/pdxdylan/status/1540398733669666818', 'tweetID': '1540398733669666818', 'conversationID': '1540398733669666818', 'mediaURLs': ['https://video.twimg.com/ext_tw_video/1540396699037929472/pu/vid/762x528/YxbXbT3X7vq4LWfC.mp4'], 'media_extended': [{'url': 'https://video.twimg.com/ext_tw_video/1540396699037929472/pu/vid/762x528/YxbXbT3X7vq4LWfC.mp4', 'type': 'video', 'size': {'width': 762, 'height': 528}, 'duration_millis': 13650, 'thumbnail_url': 'https://pbs.twimg.com/ext_tw_video_thumb/1540396699037929472/pu/img/l187Z6B9AHHxUKPV.jpg', 'altText': None}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'date_epoch': 1656094651}
+testVideoTweet_compare={'text': 'TikTok embeds on Discord/Telegram bait you with a fake play button, but to see the actual video you have to go to their website.\nAs a request from a friend, I made it so that if you add "vx" before "tiktok" on any link, it fixes that.', 'date': 'Fri Jun 24 18:17:31 +0000 2022', 'tweetURL': 'https://twitter.com/pdxdylan/status/1540398733669666818', 'tweetID': '1540398733669666818', 'conversationID': '1540398733669666818', 'mediaURLs': ['https://video.twimg.com/ext_tw_video/1540396699037929472/pu/vid/762x528/YxbXbT3X7vq4LWfC.mp4'], 'media_extended': [{'url': 'https://video.twimg.com/ext_tw_video/1540396699037929472/pu/vid/762x528/YxbXbT3X7vq4LWfC.mp4', 'type': 'video', 'size': {'width': 762, 'height': 528}, 'duration_millis': 13650, 'thumbnail_url': 'https://pbs.twimg.com/ext_tw_video_thumb/1540396699037929472/pu/img/l187Z6B9AHHxUKPV.jpg', 'altText': None, 'id_str': '1540396699037929472'}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1656094651}
-testMediaTweet_compare={'text': 'oh.', 'date': 'Wed Jun 08 23:05:14 +0000 2022', 'tweetURL': 'https://twitter.com/pdxdylan/status/1534672932106035200', 'tweetID': '1534672932106035200', 'conversationID': '1534672673422381057', 'mediaURLs': ['https://pbs.twimg.com/media/FUxAt5LWUAMol0N.png'], 'media_extended': [{'url': 'https://pbs.twimg.com/media/FUxAt5LWUAMol0N.png', 'altText': None, 'type': 'image', 'size': {'width': 927, 'height': 534}, 'thumbnail_url': 'https://pbs.twimg.com/media/FUxAt5LWUAMol0N.png'}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'date_epoch': 1654729514}
+testMediaTweet_compare={'text': 'oh.', 'date': 'Wed Jun 08 23:05:14 +0000 2022', 'tweetURL': 'https://twitter.com/pdxdylan/status/1534672932106035200', 'tweetID': '1534672932106035200', 'conversationID': '1534672673422381057', 'mediaURLs': ['https://pbs.twimg.com/media/FUxAt5LWUAMol0N.png'], 'media_extended': [{'url': 'https://pbs.twimg.com/media/FUxAt5LWUAMol0N.png', 'altText': None, 'type': 'image', 'size': {'width': 927, 'height': 534}, 'thumbnail_url': 'https://pbs.twimg.com/media/FUxAt5LWUAMol0N.png', 'id_str': '1534672730213208067'}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'und', 'replyingTo': 'pdxdylan', 'replyingToID': '1534672673422381057', 'retweetURL': None, 'date_epoch': 1654729514}
-testMultiMediaTweet_compare={'text': 'Released #Retro64 1.0.9. Besides a lot of internal bug-fixes, this adds quicksand blocks, fixes the rendering for the castle stairs block, and adds a new model, Sonic! \nhttps://github.com/Retro64Mod/Retro64Mod/releases/tag/1.18.2-1.0.9 https://t.co/CWZaw4hzyg', 'date': 'Wed Jun 01 14:29:32 +0000 2022', 'tweetURL': 'https://twitter.com/pdxdylan/status/1532006436703715331', 'tweetID': '1532006436703715331', 'conversationID': '1532006436703715331', 'mediaURLs': ['https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png', 'https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png', 'https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png'], 'media_extended': [{'url': 'https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png', 'altText': None, 'type': 'image', 'size': {'width': 507, 'height': 507}, 'thumbnail_url': 'https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png'}, {'url': 'https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png', 'altText': None, 'type': 'image', 'size': {'width': 396, 'height': 431}, 'thumbnail_url': 'https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png'}, {'url': 'https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png', 'altText': None, 'type': 'image', 'size': {'width': 399, 'height': 341}, 'thumbnail_url': 'https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png'}], 'possibly_sensitive': False, 'hashtags': ['Retro64'], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': 'https://vxtwitter.com/rendercombined.jpg?imgs=https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png,https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png,https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png', 'pollData': None, 'article': None, 'date_epoch': 1654093772}
+testMultiMediaTweet_compare={'text': 'Released #Retro64 1.0.9. Besides a lot of internal bug-fixes, this adds quicksand blocks, fixes the rendering for the castle stairs block, and adds a new model, Sonic! \nhttps://github.com/Retro64Mod/Retro64Mod/releases/tag/1.18.2-1.0.9', 'date': 'Wed Jun 01 14:29:32 +0000 2022', 'tweetURL': 'https://twitter.com/pdxdylan/status/1532006436703715331', 'tweetID': '1532006436703715331', 'conversationID': '1532006436703715331', 'mediaURLs': ['https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png', 'https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png', 'https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png'], 'media_extended': [{'url': 'https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png', 'altText': None, 'type': 'image', 'size': {'width': 507, 'height': 507}, 'thumbnail_url': 'https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png', 'id_str': '1532004485966577667'}, {'url': 'https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png', 'altText': None, 'type': 'image', 'size': {'width': 396, 'height': 431}, 'thumbnail_url': 'https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png', 'id_str': '1532004975269797890'}, {'url': 'https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png', 'altText': None, 'type': 'image', 'size': {'width': 399, 'height': 341}, 'thumbnail_url': 'https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png', 'id_str': '1532005117553164291'}], 'possibly_sensitive': False, 'hashtags': ['Retro64'], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': 'https://vxtwitter.com/rendercombined.jpg?imgs=https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png,https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png,https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png', 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1654093772}
-testQRTTweet_compare={'text': "vxTwitter has gotten a *ton* of usage recently, so I'd appreciate a donation to keep things running!\n", 'date': 'Fri Jan 06 21:37:43 +0000 2023', 'tweetURL': 'https://twitter.com/pdxdylan/status/1611477137319514129', 'tweetID': '1611477137319514129', 'conversationID': '1611476665821003776', 'mediaURLs': [], 'media_extended': [], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': 'https://twitter.com/i/status/1518309187515781125', 'allSameType': True, 'hasMedia': False, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'date_epoch': 1673041063}
+testQRTTweet_compare={'text': "vxTwitter has gotten a *ton* of usage recently, so I'd appreciate a donation to keep things running!\nhttps://x.com/pdxdylan/status/1518309187515781125", 'date': 'Fri Jan 06 21:37:43 +0000 2023', 'tweetURL': 'https://twitter.com/pdxdylan/status/1611477137319514129', 'tweetID': '1611477137319514129', 'conversationID': '1611476665821003776', 'mediaURLs': [], 'media_extended': [], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': 'https://twitter.com/i/status/1518309187515781125', 'allSameType': True, 'hasMedia': False, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': 'pdxdylan', 'replyingToID': '1611476665821003776', 'retweetURL': None, 'date_epoch': 1673041063}
-testQrtCeptionTweet_compare={'text': 'Testing retweetception ', 'date': 'Tue Apr 07 01:32:26 +0000 2015', 'tweetURL': 'https://twitter.com/CatherineShu/status/585253766271672320', 'tweetID': '585253766271672320', 'conversationID': '585253766271672320', 'mediaURLs': [], 'media_extended': [], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': 'https://twitter.com/i/status/585253161260216320', 'allSameType': True, 'hasMedia': False, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'date_epoch': 1428370346}
+testQrtVideoTweet_compare={'text': 'good', 'date': 'Thu Jun 29 23:33:29 +0000 2023', 'tweetURL': 'https://twitter.com/pdxdylan/status/1674561759422578690', 'tweetID': '1674561759422578690', 'conversationID': '1674561759422578690', 'mediaURLs': [], 'media_extended': [], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': 'https://twitter.com/i/status/1674197531301904388', 'allSameType': True, 'hasMedia': False, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1688081609}
-testQrtVideoTweet_compare={'text': 'good', 'date': 'Thu Jun 29 23:33:29 +0000 2023', 'tweetURL': 'https://twitter.com/pdxdylan/status/1674561759422578690', 'tweetID': '1674561759422578690', 'conversationID': '1674561759422578690', 'mediaURLs': [], 'media_extended': [], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': 'https://twitter.com/i/status/1674197531301904388', 'allSameType': True, 'hasMedia': False, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'date_epoch': 1688081609}
+testNSFWTweet_compare={'text': "ngl, I'm scared on finding out the cute Sprigatito's final evolution..\n\nso i had a bot generate it for me.... and I'm forever scarred", 'date': 'Sat Oct 15 07:28:42 +0000 2022', 'tweetURL': 'https://twitter.com/kuyacoy/status/1581185279376838657', 'tweetID': '1581185279376838657', 'conversationID': '1581185279376838657', 'mediaURLs': ['https://pbs.twimg.com/media/FfF_gKwXgAIpnpD.jpg'], 'media_extended': [{'url': 'https://pbs.twimg.com/media/FfF_gKwXgAIpnpD.jpg', 'altText': None, 'type': 'image', 'size': {'width': 760, 'height': 926}, 'thumbnail_url': 'https://pbs.twimg.com/media/FfF_gKwXgAIpnpD.jpg', 'id_str': '1581185134803517442'}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1665818922}
-testNSFWTweet_compare={'text': "ngl, I'm scared on finding out the cute Sprigatito's final evolution..\n\nso i had a bot generate it for me.... and I'm forever scarred https://t.co/itMay87vcS", 'date': 'Sat Oct 15 07:28:42 +0000 2022', 'tweetURL': 'https://twitter.com/kuyacoy/status/1581185279376838657', 'tweetID': '1581185279376838657', 'conversationID': '1581185279376838657', 'mediaURLs': ['https://pbs.twimg.com/media/FfF_gKwXgAIpnpD.jpg'], 'media_extended': [{'url': 'https://pbs.twimg.com/media/FfF_gKwXgAIpnpD.jpg', 'altText': None, 'type': 'image', 'size': {'width': 760, 'height': 926}, 'thumbnail_url': 'https://pbs.twimg.com/media/FfF_gKwXgAIpnpD.jpg'}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'date_epoch': 1665818922}
+testPollTweet_compare={'text': 'I know when that hotline bling, that can only:', 'date': 'Mon Oct 05 22:57:25 +0000 2015', 'tweetURL': 'https://twitter.com/norm/status/651169346518056960', 'tweetID': '651169346518056960', 'conversationID': '651169346518056960', 'mediaURLs': [], 'media_extended': [], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': False, 'combinedMediaUrl': None, 'pollData': {'options': [{'name': 'Mean one thing', 'votes': 124875, 'percent': 78.82}, {'name': 'Mean multiple things', 'votes': 33554, 'percent': 21.18}]}, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1444085845}
-testPollTweet_compare={'text': 'I know when that hotline bling, that can only:', 'date': 'Mon Oct 05 22:57:25 +0000 2015', 'tweetURL': 'https://twitter.com/norm/status/651169346518056960', 'tweetID': '651169346518056960', 'conversationID': '651169346518056960', 'mediaURLs': [], 'media_extended': [], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': False, 'combinedMediaUrl': None, 'pollData': {'options': [{'name': 'Mean one thing', 'votes': 124875, 'percent': 78.82}, {'name': 'Mean multiple things', 'votes': 33554, 'percent': 21.18}]}, 'article': None, 'date_epoch': 1444085845}
+testMixedMediaTweet_compare={'text': 'Some of us here are definitely big nerds about beer, and could talk your ear off about it for days on end, but some of us are just "beer is nice"', 'date': 'Thu Feb 22 12:13:24 +0000 2024', 'tweetURL': 'https://twitter.com/salebeerfest/status/1760638922084741177', 'tweetID': '1760638922084741177', 'conversationID': '1760638922084741177', 'mediaURLs': ['https://pbs.twimg.com/media/GG8LwfuWoAANKhs.jpg', 'https://video.twimg.com/tweet_video/GG8LwqWX0AAZch0.mp4'], 'media_extended': [{'url': 'https://pbs.twimg.com/media/GG8LwfuWoAANKhs.jpg', 'altText': None, 'type': 'image', 'size': {'width': 858, 'height': 960}, 'thumbnail_url': 'https://pbs.twimg.com/media/GG8LwfuWoAANKhs.jpg', 'id_str': '1760638907102699520'}, {'url': 'https://video.twimg.com/tweet_video/GG8LwqWX0AAZch0.mp4', 'type': 'gif', 'size': {'width': 500, 'height': 500}, 'duration_millis': 0, 'thumbnail_url': 'https://pbs.twimg.com/tweet_video_thumb/GG8LwqWX0AAZch0.jpg', 'altText': None, 'id_str': '1760638909954904064'}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': False, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1708604004}
-testMixedMediaTweet_compare={'text': 'Some of us here are definitely big nerds about beer, and could talk your ear off about it for days on end, but some of us are just "beer is nice"', 'date': 'Thu Feb 22 12:13:24 +0000 2024', 'tweetURL': 'https://twitter.com/salebeerfest/status/1760638922084741177', 'tweetID': '1760638922084741177', 'conversationID': '1760638922084741177', 'mediaURLs': ['https://pbs.twimg.com/media/GG8LwfuWoAANKhs.jpg', 'https://video.twimg.com/tweet_video/GG8LwqWX0AAZch0.mp4'], 'media_extended': [{'url': 'https://pbs.twimg.com/media/GG8LwfuWoAANKhs.jpg', 'altText': None, 'type': 'image', 'size': {'width': 858, 'height': 960}, 'thumbnail_url': 'https://pbs.twimg.com/media/GG8LwfuWoAANKhs.jpg'}, {'url': 'https://video.twimg.com/tweet_video/GG8LwqWX0AAZch0.mp4', 'type': 'gif', 'size': {'width': 500, 'height': 500}, 'duration_millis': 0, 'thumbnail_url': 'https://pbs.twimg.com/tweet_video_thumb/GG8LwqWX0AAZch0.jpg', 'altText': None}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': False, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'date_epoch': 1708604004}
+testVinePlayerTweet_compare={'text': 'You wanted old ROBLOX back, you got it. Check out our sweet "new" look! #BringBackOldROBLOX https://vine.co/v/OL9VqvM6wJh', 'date': 'Wed Apr 01 16:17:13 +0000 2015', 'tweetURL': 'https://twitter.com/Roblox/status/583302104342638592', 'tweetID': '583302104342638592', 'conversationID': '583302104342638592', 'mediaURLs': ['https://v.cdn.vine.co/r/videos/20A1BE53011195086166081318912_3fe3b526b1a.1.5.3156516531034157495.mp4?versionId=DI1mMu7EI6zcLbvgucyp3GHebdz8.9cQ'], 'media_extended': [{'url': 'https://v.cdn.vine.co/r/videos/20A1BE53011195086166081318912_3fe3b526b1a.1.5.3156516531034157495.mp4?versionId=DI1mMu7EI6zcLbvgucyp3GHebdz8.9cQ', 'type': 'video', 'size': {'width': 435, 'height': 435}}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': {'options': []}, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1427905033}
-testVinePlayerTweet_compare={'text': 'You wanted old ROBLOX back, you got it. Check out our sweet "new" look! #BringBackOldROBLOX https://vine.co/v/OL9VqvM6wJh', 'date': 'Wed Apr 01 16:17:13 +0000 2015', 'tweetURL': 'https://twitter.com/Roblox/status/583302104342638592', 'tweetID': '583302104342638592', 'conversationID': '583302104342638592', 'mediaURLs': ['https://v.cdn.vine.co/r/videos/20A1BE53011195086166081318912_3fe3b526b1a.1.5.3156516531034157495.mp4?versionId=DI1mMu7EI6zcLbvgucyp3GHebdz8.9cQ'], 'media_extended': [{'url': 'https://v.cdn.vine.co/r/videos/20A1BE53011195086166081318912_3fe3b526b1a.1.5.3156516531034157495.mp4?versionId=DI1mMu7EI6zcLbvgucyp3GHebdz8.9cQ', 'type': 'video', 'size': {'width': 435, 'height': 435}}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': {'options': []}, 'article': None, 'date_epoch': 1427905033}
+testRetweetTweet_compare={'text': 'RT @pdxdylan: If you want to try this out, on your mobile device, head over to https://vxtwitter.com/preferences and enable "Open links in app". Hope…', 'date': 'Tue Aug 27 23:09:07 +0000 2024', 'tweetURL': 'https://twitter.com/pdxdylan/status/1828570470222045294', 'tweetID': '1828570470222045294', 'conversationID': '1828570470222045294', 'mediaURLs': [], 'media_extended': [], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': False, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': 'https://twitter.com/i/status/1828569456231993456', 'date_epoch': 1724800147}
 
 testUser="https://twitter.com/jack"
+testUserSuspended="https://twitter.com/twitter"
+testUserPrivate="https://twitter.com/PrestigeIsKey"
+testUserInvalid="https://twitter.com/.a"
 testUserID=12 # could also be 170824883
 testUserIDUrl = "https://twitter.com/i/user/"+str(testUserID)
 testUserWeirdURLs=["https://twitter.com/jack?lang=en","https://twitter.com/jack/with_replies","https://twitter.com/jack/media","https://twitter.com/jack/likes","https://twitter.com/jack/with_replies?lang=en","https://twitter.com/jack/media?lang=en","https://twitter.com/jack/likes?lang=en","https://twitter.com/jack/"]