Merge branch 'main' of https://github.com/dylanpdx/BetterTwitFix into github/origin

Support alternate card key

.github/workflows/python-tests.yml

@@ -6,23 +6,30 @@ jobs:
   build:
     runs-on: ubuntu-latest
     environment: test
-    strategy:
-      matrix:
-        python-version: ["3.10", "3.11", "3.12"]
-
     steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
+      - name: Check out files
+        uses: actions/checkout@v4
+
+      - name: Install Python
+        run: |
+          apt-get update
+          apt-get install -y python3 python3-pip python3-venv
+
+      - name: Set up virtual environment
+        run: |
+          python3 -m venv venv
+          . venv/bin/activate
+          python -m pip install --upgrade pip
+
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          . venv/bin/activate
           pip install pytest pytest-cov pytest-env
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
       - name: Test with pytest
         env:
           VXTWITTER_WORKAROUND_TOKENS: ${{ secrets.VXTWITTER_WORKAROUND_TOKENS }}
         run: |
+          . venv/bin/activate
           pytest --cov-config=.coveragerc --cov=.

config.json

@@ -2,12 +2,12 @@
     "config": {
         "appname": "vxTwitter",
         "color": "#1DA1F2",
-        "database": "[url to mongo database goes here]",
-        "link_cache": "ram",
+        "database": "mongodb://localhost:27017/bettervxtwitter",
+        "link_cache": "db",
         "method": "hybrid",
-        "repo": "https://github.com/dylanpdx/BetterTwitFix",
-        "table": "[database table here]",
-        "url": "https://vxtwitter.com",
+        "repo": "https://git.alterware.dev/alterware/BetterTwitFix",
+        "table": "links",
+        "url": "https://girlcock.alterware.dev",
         "combination_method": "local",
         "gifConvertAPI": "local",
         "workaroundTokens": null

test_vx_extract.py

@@ -9,14 +9,14 @@ def test_twextract_syndicationAPI():
     tweet = twExtract.extractStatus_syndication(testMediaTweet,workaroundTokens=tokens)
     assert utils.stripEndTCO(utils.stripEndTCO(tweet["full_text"]))==testMediaTweet_compare['text']
 
-def test_twextract_extractStatusV2Anon():
-    tweet = twExtract.extractStatusV2Anon(testTextTweet,None)['legacy']
+def test_twextract_extractStatusV2Rest():
+    tweet = twExtract.extractStatusV2Rest(testTextTweet,None)['legacy']
     assert utils.stripEndTCO(tweet["full_text"])==testTextTweet_compare['text']
-    tweet = twExtract.extractStatusV2Anon(testVideoTweet,None)['legacy']
+    tweet = twExtract.extractStatusV2Rest(testVideoTweet,None)['legacy']
     assert utils.stripEndTCO(tweet["full_text"])==testVideoTweet_compare['text']
-    tweet = twExtract.extractStatusV2Anon(testMediaTweet,None)['legacy']
+    tweet = twExtract.extractStatusV2Rest(testMediaTweet,None)['legacy']
     assert utils.stripEndTCO(tweet["full_text"])==testMediaTweet_compare['text']
-    tweet = twExtract.extractStatusV2Anon(testMultiMediaTweet,None)['legacy']
+    tweet = twExtract.extractStatusV2Rest(testMultiMediaTweet,None)['legacy']
     assert utils.stripEndTCO(tweet["full_text"])[:94]==testMultiMediaTweet_compare['text'][:94]
     
 
@@ -33,11 +33,6 @@ def test_twextract_extractStatusV2TweetDetails():
     assert utils.stripEndTCO(tweet["full_text"])==testMediaTweet_compare['text']
 
 ## Tweet retrieve tests ##
-def test_twextract_textTweetExtract():
-    tweet = twExtract.extractStatus(testTextTweet,workaroundTokens=tokens)
-    assert utils.stripEndTCO(tweet["legacy"]["full_text"])==testTextTweet_compare['text']
-    assert tweet["user"]["screen_name"]=="jack"
-    assert 'extended_entities' not in tweet
     
 def test_twextract_extractV2():
     tweet = twExtract.extractStatusV2(testTextTweet,workaroundTokens=tokens)

test_vx_misc.py

@@ -1,10 +1,17 @@
-import twitfix, cache, twExtract
+import twitfix, cache, twExtract, utils
 from vx_testdata import *
 from twExtract import twUtils
 
 def test_calcSyndicationToken():
     assert twUtils.calcSyndicationToken("1691389765483200513") == "43lnobuxzql"
 
+def test_stripEndTCO():
+    assert utils.stripEndTCO("Hello World https://t.co/abc123") == "Hello World"
+    assert utils.stripEndTCO("Hello\nWorld https://t.co/abc123") == "Hello\nWorld"
+    assert utils.stripEndTCO("Hello\nWorld\nhttps://t.co/abc123") == "Hello\nWorld"
+    assert utils.stripEndTCO("Hello\nWorld\n https://t.co/abc123") == "Hello\nWorld"
+    assert utils.stripEndTCO("Hello\nWorld \nhttps://t.co/abc123") == "Hello\nWorld"
+
 def test_addToCache():
     cache.clearCache()
     twitfix.getTweetData(testTextTweet)

twExtract/__init__.py

@@ -13,11 +13,10 @@ import concurrent.futures
 bearer="Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
 v2bearer="Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
 androidBearer="Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F"
-tweetdeckBearer="Bearer AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
 
 requestUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0"
 
-bearerTokens=[tweetdeckBearer,bearer,v2bearer,androidBearer]
+bearerTokens=[bearer,v2bearer,androidBearer]
 
 guestToken=None
 guestTokenUses=0
@@ -32,11 +31,11 @@ v2AnonFeatures='{"creator_subscriptions_tweet_preview_api_enabled":true,"premium
 v2AnonGraphql_api="wqi5M7wZ7tW-X9S2t-Mqcg"
 gt_pattern = r'document\.cookie="gt=([^;]+);'
 
-androidGraphqlFeatures='{"longform_notetweets_inline_media_enabled":true,"super_follow_badge_privacy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"super_follow_user_api_enabled":true,"unified_cards_ad_metadata_container_dynamic_card_content_query_enabled":true,"super_follow_tweet_api_enabled":true,"articles_api_enabled":true,"android_graphql_skip_api_media_color_palette":true,"creator_subscriptions_tweet_preview_api_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"tweetypie_unmention_optimization_enabled":true,"longform_notetweets_consumption_enabled":true,"subscriptions_verification_info_enabled":true,"blue_business_profile_image_shape_enabled":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"immersive_video_status_linkable_timestamps":true,"super_follow_exclusive_tweet_notifications_enabled":true}'
-androidGraphql_api="llQH5PFIRlenVrlKJU8jNA"
+androidGraphqlFeatures='{"grok_translations_community_note_translation_is_enabled":false,"super_follow_badge_privacy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"super_follow_user_api_enabled":true,"profile_label_improvements_pcf_label_in_profile_enabled":true,"premium_content_api_read_enabled":false,"grok_translations_community_note_auto_translation_is_enabled":false,"android_graphql_skip_api_media_color_palette":true,"tweetypie_unmention_optimization_enabled":true,"longform_notetweets_consumption_enabled":true,"subscriptions_verification_info_enabled":true,"blue_business_profile_image_shape_enabled":true,"super_follow_exclusive_tweet_notifications_enabled":true,"longform_notetweets_inline_media_enabled":true,"grok_android_analyze_trend_fetch_enabled":false,"unified_cards_ad_metadata_container_dynamic_card_content_query_enabled":true,"super_follow_tweet_api_enabled":true,"articles_api_enabled":true,"creator_subscriptions_tweet_preview_api_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"grok_translations_timeline_user_bio_auto_translation_is_enabled":false,"grok_translations_post_auto_translation_is_enabled":false,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"immersive_video_status_linkable_timestamps":true,"profile_label_improvements_pcf_label_in_post_enabled":true}'
+androidGraphql_api="k3rtLsS9kG5hI-Jr0dTMCg"
 
 tweetDetailGraphqlFeatures='{"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"articles_preview_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"rweb_video_timestamps_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_enhance_cards_enabled":false}'
-tweetDetailGraphql_api="e7RKseIxLu7HgkWNKZ6qnw"
+tweetDetailGraphql_api="YVyS4SfwYW7Uw5qwy0mQCA"
 
 # this is for UserTweets endpoint
 tweetFeedGraphqlFeatures='{"rweb_video_screen_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":false,"responsive_web_grok_share_attachment_enabled":true,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":false,"responsive_web_grok_analysis_button_from_backend":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_enhance_cards_enabled":false}'
@@ -116,7 +115,7 @@ def cycleBearerTokenGet(url,headers):
 
 def twitterApiGet(url,btoken=None,authToken=None,guestToken=None):
 
-    if authToken.startswith("oa|"):
+    if authToken != None and authToken.startswith("oa|"):
         url = url.replace("https://x.com/i/api/graphql/","https://api.twitter.com/graphql/")
         authToken = authToken[3:]
         key = authToken.split("|")[0]
@@ -132,7 +131,8 @@ def twitterApiGet(url,btoken=None,authToken=None,guestToken=None):
         response = requests.get(url,headers=headers)
     else:
         if btoken is None:
-            return cycleBearerTokenGet(url,getAuthHeaders(bearer,authToken=authToken,guestToken=guestToken))
+            btoken = v2bearer
+            #return cycleBearerTokenGet(url,getAuthHeaders(bearer,authToken=authToken,guestToken=guestToken))
         headers = getAuthHeaders(btoken,authToken=authToken,guestToken=guestToken)
         response = requests.get(url, headers=headers)
 
@@ -279,7 +279,7 @@ def extractStatusV2(url,workaroundTokens):
     def request_with_token(twid, authToken):
         vars = json.loads('{"includeTweetImpression":true,"includeHasBirdwatchNotes":false,"includeEditPerspective":false,"rest_ids":["x"],"includeEditControl":true,"includeCommunityTweetRelationship":true,"includeTweetVisibilityNudge":true}')
         vars['rest_ids'][0] = str(twid)
-        tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2graphql_api}/TweetResultsByIdsQuery?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2Features)}",authToken=authToken,btoken=v2bearer)
+        tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2graphql_api}/TweetResultsByIdsQuery?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2Features)}",authToken=authToken)
         try:
             rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining")
             print(f"Twitter Token Rate limit remaining: {rateLimitRemaining}")
@@ -437,7 +437,10 @@ def extractStatusV2TweetDetail(url,workaroundTokens):
         return tweet
     return parallel_token_request(twid, tokens, request_with_token)
 
-def extractStatusV2Anon(url,x):
+def extractStatusV2Rest_Anon(url,workaroundTokens):
+    return extractStatusV2Rest(url,None)
+
+def extractStatusV2Rest(url,workaroundTokens):
     # get tweet ID
     m = re.search(pathregex, url)
     if m is None:
@@ -450,7 +453,17 @@ def extractStatusV2Anon(url,x):
     try:
         vars = json.loads('{"tweetId":"0","withCommunity":false,"includePromotedContent":false,"withVoice":false}')
         vars['tweetId'] = str(twid)
-        tweet = requests.get(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", headers=getAuthHeaders(v2bearer,guestToken=guestToken))
+        if workaroundTokens is not None and len(workaroundTokens) > 0:
+            tokens = workaroundTokens
+            random.shuffle(tokens)
+            for authToken in tokens:
+                try:
+                    tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", btoken=v2bearer,authToken=authToken,guestToken=guestToken)
+                except Exception as e:
+                    continue
+        else:
+            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", btoken=v2bearer,guestToken=guestToken)
+        
         try:
             rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining")
             print(f"Twitter Anon Token Rate limit remaining: {rateLimitRemaining}")
@@ -498,7 +511,7 @@ def fixTweetData(tweet):
 
 def extractStatus(url,workaroundTokens=None):
     # TODO: commented out methods are too slow/unreliable at the moment
-    methods=[extractStatusV2Anon,extractStatusV2]#,extractStatusV2Android,extractStatusV2TweetDetail] # 
+    methods=[extractStatusV2Rest_Anon,extractStatusV2,extractStatusV2Rest,extractStatusV2Android]#,extractStatusV2TweetDetail]
     for method in methods:
         try:
             result = method(url,workaroundTokens)

twitfix.py

@@ -516,4 +516,4 @@ def oEmbedGen(description, user, video_link, ttype,providerName=None):
 
 if __name__ == "__main__":
     app.config['SERVER_NAME']='localhost:80'
-    app.run(host='0.0.0.0')
+    app.run(host='0.0.0.0', port=8080)

twitfix.service

@@ -3,11 +3,11 @@ Description=Init file for twitfix uwsgi instance
 After=network.target
 
 [Service]
-User=dylan
-Group=dylan
-WorkingDirectory=/home/dylan/BetterTwitFix
-Environment="PATH=/home/dylan/BetterTwitFix/venv/bin"
-ExecStart=/home/dylan/BetterTwitFix/venv/bin/uwsgi --ini twitfix.ini
+User=carbonara
+Group=carbonara
+WorkingDirectory=/home/carbonara/twitter/BetterTwitFix
+Environment="PATH=/home/carbonara/twitter/BetterTwitFix/venv/bin"
+ExecStart=/home/carbonara/twitter/BetterTwitFix/venv/bin/uwsgi --ini twitfix.ini
 Restart=always
 RestartSec=3
 

utils.py

@@ -3,7 +3,7 @@ import io
 from configHandler import config
 
 pathregex = re.compile("\\w{1,15}\\/(status|statuses)\\/(\\d{2,20})")
-endTCOregex = re.compile("(^.*?) +https:\/\/t.co\/.*?$")
+endTCOregex = re.compile("(^.*?)[ \n]+https:\/\/t.co\/.*?$",flags=re.DOTALL)
 
 def getTweetIdFromUrl(url):
     match = pathregex.search(url)

vxApi.py

@@ -1,5 +1,6 @@
 import html
 from datetime import datetime
+from flask import json
 from configHandler import config
 from utils import stripEndTCO
 
@@ -20,6 +21,48 @@ def getApiUserResponse(user):
         "fetched_on": int(datetime.now().timestamp()),
     }
 
+def getBestMediaUrl(mediaList):
+    # find the highest bitrate
+    best_bitrate = -1
+    besturl=""
+    for j in mediaList:
+        if j['content_type'] == "video/mp4" and '/hevc/' not in j["url"] and j['bitrate'] > best_bitrate:
+            besturl = j["url"]
+            best_bitrate = j['bitrate']
+    if "?tag=" in besturl:
+        besturl = besturl[:besturl.index("?tag=")]
+    return besturl
+
+def getExtendedVideoOrGifInfo(mediaEntry):
+    videoInfo = mediaEntry["video_info"]
+    info = {
+        "url": getBestMediaUrl(videoInfo["variants"]),
+        "type": "gif" if mediaEntry.get("type", "") == "animated_gif" else "video",
+        "size": {
+            "width": mediaEntry['original_info']["width"],
+            "height": mediaEntry['original_info']["height"]
+        },
+        "duration_millis": videoInfo.get("duration_millis", 0),
+        "thumbnail_url": mediaEntry.get("media_url_https", None),
+        "altText": mediaEntry.get("ext_alt_text", None),
+        "id_str": mediaEntry.get("id_str", None)
+    }
+    return info
+
+def getExtendedImageInfo(mediaEntry):
+    info = {
+        "url": mediaEntry.get("media_url_https", None),
+        "type": "image",
+        "size": {
+            "width": mediaEntry["original_info"]["width"],
+            "height": mediaEntry["original_info"]["height"]
+        },
+        "thumbnail_url": mediaEntry.get("media_url_https", None),
+        "altText": mediaEntry.get("ext_alt_text", None),
+        "id_str": mediaEntry.get("id_str", None)
+    }
+    return info
+    
 def getApiResponse(tweet,include_txt=False,include_rtf=False):
     tweetL = tweet["legacy"]
     if "user_result" in tweet["core"]:
@@ -69,53 +112,30 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False):
             for i in tmedia:
                 extendedInfo={}
                 if "video_info" in i:
-                    # find the highest bitrate
-                    best_bitrate = -1
-                    besturl=""
-                    for j in i["video_info"]["variants"]:
-                        if j['content_type'] == "video/mp4" and '/hevc/' not in j["url"] and j['bitrate'] > best_bitrate:
-                            besturl = j['url']
-                            best_bitrate = j['bitrate']
-                    if "?tag=" in besturl:
-                        besturl = besturl[:besturl.index("?tag=")]
-                    media.append(besturl)
-                    extendedInfo["url"] = besturl
-                    extendedInfo["type"] = "video"
-                    if (i["type"] == "animated_gif"):
-                        extendedInfo["type"] = "gif"
-                    altText = None
-                    extendedInfo["size"] = {"width":i["original_info"]["width"],"height":i["original_info"]["height"]}
-                    if "ext_alt_text" in i:
-                        altText=i["ext_alt_text"]
-                    if "duration_millis" in i["video_info"]:
-                        extendedInfo["duration_millis"] = i["video_info"]["duration_millis"]
-                    else:
-                        extendedInfo["duration_millis"] = 0
-                    extendedInfo["thumbnail_url"] = i["media_url_https"]
-                    extendedInfo["altText"] = altText
-                    extendedInfo["id_str"] = i["id_str"]
+                    extendedInfo = getExtendedVideoOrGifInfo(i)
+                    media.append(extendedInfo["url"])
                     media_extended.append(extendedInfo)
                 else:
-                    media.append(i["media_url_https"])
-                    extendedInfo["url"] = i["media_url_https"]
-                    altText=None
-                    if "ext_alt_text" in i:
-                        altText=i["ext_alt_text"]
-                    extendedInfo["altText"] = altText
-                    extendedInfo["type"] = "image"
-                    extendedInfo["size"] = {"width":i["original_info"]["width"],"height":i["original_info"]["height"]}
-                    extendedInfo["thumbnail_url"] = i["media_url_https"]
-                    extendedInfo["id_str"] = i["id_str"]
+                    extendedInfo = getExtendedImageInfo(i)
                     media_extended.append(extendedInfo)
+                    media.append(extendedInfo["url"])
 
         if "hashtags" in tweetL["entities"]:
             for i in tweetL["entities"]["hashtags"]:
                 hashtags.append(i["text"])
-    elif "card" in tweet and 'name' in tweet['card'] and tweet['card']['name'] == "player":
+    elif "card" in tweet or "tweet_card" in tweet:
+        cardData = tweet["card" if "card" in tweet else "tweet_card"]
+        bindingValues = None
+        if 'binding_values' in cardData:
+            bindingValues = cardData['binding_values']
+        elif 'legacy' in cardData and 'binding_values' in cardData['legacy']:
+            bindingValues = cardData['legacy']['binding_values']
+        if bindingValues != None:
+            if 'name' in cardData and cardData['name'] == "player":
                 width = None
                 height = None
                 vidUrl = None
-        for i in tweet['card']['binding_values']:
+                for i in bindingValues:
                     if i['key'] == 'player_stream_url':
                         vidUrl = i['value']['string_value']
                     elif i['key'] == 'player_width':
@@ -125,7 +145,22 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False):
                 if vidUrl != None and width != None and height != None:
                     media.append(vidUrl)
                     media_extended.append({"url":vidUrl,"type":"video","size":{"width":width,"height":height}})
-
+            else:
+                for i in bindingValues:
+                    if i['key'] == 'unified_card' and 'value' in i and 'string_value' in i['value']:
+                        cardData = json.loads(i['value']['string_value'])
+                        media_key = cardData['component_objects']['media_1']['data']['id']
+                        media_entry = cardData['media_entities'][media_key]
+                        extendedInfo = getExtendedVideoOrGifInfo(media_entry)
+                        media.append(extendedInfo['url'])
+                        media_extended.append(extendedInfo)
+                        break
+                    elif i['key'] == 'photo_image_full_size_large' and 'value' in i and 'image_value' in i['value']:
+                        imgData = i['value']['image_value']
+                        imgurl = imgData['url']
+                        media.append(imgurl)
+                        media_extended.append({"url":imgurl,"type":"image","size":{"width":imgData['width'],"height":imgData['height']}})
+                        break
     if "article" in tweet:
         try:
             result = tweet["article"]["article_results"]["result"]

vx_testdata.py

@@ -15,12 +15,12 @@ testVinePlayerTweet="https://twitter.com/Roblox/status/583302104342638592"
 testRetweetTweet="https://twitter.com/pdxdylan/status/1828570470222045294"
 
 testTextTweet_compare={'text': 'just setting up my twttr', 'date': 'Tue Mar 21 20:50:14 +0000 2006', 'tweetURL': 'https://twitter.com/jack/status/20', 'tweetID': '20', 'conversationID': '20', 'mediaURLs': [], 'media_extended': [], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': False, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1142974214}
-testVideoTweet_compare={'text': 'TikTok embeds on Discord/Telegram bait you with a fake play button, but to see the actual video you have to go to their website.\nAs a request from a friend, I made it so that if you add "vx" before "tiktok" on any link, it fixes that. https://t.co/QYpiVXUIrW', 'date': 'Fri Jun 24 18:17:31 +0000 2022', 'tweetURL': 'https://twitter.com/pdxdylan/status/1540398733669666818', 'tweetID': '1540398733669666818', 'conversationID': '1540398733669666818', 'mediaURLs': ['https://video.twimg.com/ext_tw_video/1540396699037929472/pu/vid/762x528/YxbXbT3X7vq4LWfC.mp4'], 'media_extended': [{'url': 'https://video.twimg.com/ext_tw_video/1540396699037929472/pu/vid/762x528/YxbXbT3X7vq4LWfC.mp4', 'type': 'video', 'size': {'width': 762, 'height': 528}, 'duration_millis': 13650, 'thumbnail_url': 'https://pbs.twimg.com/ext_tw_video_thumb/1540396699037929472/pu/img/l187Z6B9AHHxUKPV.jpg', 'altText': None, 'id_str': '1540396699037929472'}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1656094651}
+testVideoTweet_compare={'text': 'TikTok embeds on Discord/Telegram bait you with a fake play button, but to see the actual video you have to go to their website.\nAs a request from a friend, I made it so that if you add "vx" before "tiktok" on any link, it fixes that.', 'date': 'Fri Jun 24 18:17:31 +0000 2022', 'tweetURL': 'https://twitter.com/pdxdylan/status/1540398733669666818', 'tweetID': '1540398733669666818', 'conversationID': '1540398733669666818', 'mediaURLs': ['https://video.twimg.com/ext_tw_video/1540396699037929472/pu/vid/762x528/YxbXbT3X7vq4LWfC.mp4'], 'media_extended': [{'url': 'https://video.twimg.com/ext_tw_video/1540396699037929472/pu/vid/762x528/YxbXbT3X7vq4LWfC.mp4', 'type': 'video', 'size': {'width': 762, 'height': 528}, 'duration_millis': 13650, 'thumbnail_url': 'https://pbs.twimg.com/ext_tw_video_thumb/1540396699037929472/pu/img/l187Z6B9AHHxUKPV.jpg', 'altText': None, 'id_str': '1540396699037929472'}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1656094651}
 testMediaTweet_compare={'text': 'oh.', 'date': 'Wed Jun 08 23:05:14 +0000 2022', 'tweetURL': 'https://twitter.com/pdxdylan/status/1534672932106035200', 'tweetID': '1534672932106035200', 'conversationID': '1534672673422381057', 'mediaURLs': ['https://pbs.twimg.com/media/FUxAt5LWUAMol0N.png'], 'media_extended': [{'url': 'https://pbs.twimg.com/media/FUxAt5LWUAMol0N.png', 'altText': None, 'type': 'image', 'size': {'width': 927, 'height': 534}, 'thumbnail_url': 'https://pbs.twimg.com/media/FUxAt5LWUAMol0N.png', 'id_str': '1534672730213208067'}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'und', 'replyingTo': 'pdxdylan', 'replyingToID': '1534672673422381057', 'retweetURL': None, 'date_epoch': 1654729514}
-testMultiMediaTweet_compare={'text': 'Released #Retro64 1.0.9. Besides a lot of internal bug-fixes, this adds quicksand blocks, fixes the rendering for the castle stairs block, and adds a new model, Sonic! \nhttps://github.com/Retro64Mod/Retro64Mod/releases/tag/1.18.2-1.0.9 https://t.co/CWZaw4hzyg', 'date': 'Wed Jun 01 14:29:32 +0000 2022', 'tweetURL': 'https://twitter.com/pdxdylan/status/1532006436703715331', 'tweetID': '1532006436703715331', 'conversationID': '1532006436703715331', 'mediaURLs': ['https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png', 'https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png', 'https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png'], 'media_extended': [{'url': 'https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png', 'altText': None, 'type': 'image', 'size': {'width': 507, 'height': 507}, 'thumbnail_url': 'https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png', 'id_str': '1532004485966577667'}, {'url': 'https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png', 'altText': None, 'type': 'image', 'size': {'width': 396, 'height': 431}, 'thumbnail_url': 'https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png', 'id_str': '1532004975269797890'}, {'url': 'https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png', 'altText': None, 'type': 'image', 'size': {'width': 399, 'height': 341}, 'thumbnail_url': 'https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png', 'id_str': '1532005117553164291'}], 'possibly_sensitive': False, 'hashtags': ['Retro64'], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': 'https://vxtwitter.com/rendercombined.jpg?imgs=https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png,https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png,https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png', 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1654093772}
+testMultiMediaTweet_compare={'text': 'Released #Retro64 1.0.9. Besides a lot of internal bug-fixes, this adds quicksand blocks, fixes the rendering for the castle stairs block, and adds a new model, Sonic! \nhttps://github.com/Retro64Mod/Retro64Mod/releases/tag/1.18.2-1.0.9', 'date': 'Wed Jun 01 14:29:32 +0000 2022', 'tweetURL': 'https://twitter.com/pdxdylan/status/1532006436703715331', 'tweetID': '1532006436703715331', 'conversationID': '1532006436703715331', 'mediaURLs': ['https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png', 'https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png', 'https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png'], 'media_extended': [{'url': 'https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png', 'altText': None, 'type': 'image', 'size': {'width': 507, 'height': 507}, 'thumbnail_url': 'https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png', 'id_str': '1532004485966577667'}, {'url': 'https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png', 'altText': None, 'type': 'image', 'size': {'width': 396, 'height': 431}, 'thumbnail_url': 'https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png', 'id_str': '1532004975269797890'}, {'url': 'https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png', 'altText': None, 'type': 'image', 'size': {'width': 399, 'height': 341}, 'thumbnail_url': 'https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png', 'id_str': '1532005117553164291'}], 'possibly_sensitive': False, 'hashtags': ['Retro64'], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': 'https://vxtwitter.com/rendercombined.jpg?imgs=https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png,https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png,https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png', 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1654093772}
 testQRTTweet_compare={'text': "vxTwitter has gotten a *ton* of usage recently, so I'd appreciate a donation to keep things running!\nhttps://x.com/pdxdylan/status/1518309187515781125", 'date': 'Fri Jan 06 21:37:43 +0000 2023', 'tweetURL': 'https://twitter.com/pdxdylan/status/1611477137319514129', 'tweetID': '1611477137319514129', 'conversationID': '1611476665821003776', 'mediaURLs': [], 'media_extended': [], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': 'https://twitter.com/i/status/1518309187515781125', 'allSameType': True, 'hasMedia': False, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': 'pdxdylan', 'replyingToID': '1611476665821003776', 'retweetURL': None, 'date_epoch': 1673041063}
 testQrtVideoTweet_compare={'text': 'good', 'date': 'Thu Jun 29 23:33:29 +0000 2023', 'tweetURL': 'https://twitter.com/pdxdylan/status/1674561759422578690', 'tweetID': '1674561759422578690', 'conversationID': '1674561759422578690', 'mediaURLs': [], 'media_extended': [], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': 'https://twitter.com/i/status/1674197531301904388', 'allSameType': True, 'hasMedia': False, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1688081609}
-testNSFWTweet_compare={'text': "ngl, I'm scared on finding out the cute Sprigatito's final evolution..\n\nso i had a bot generate it for me.... and I'm forever scarred https://t.co/itMay87vcS", 'date': 'Sat Oct 15 07:28:42 +0000 2022', 'tweetURL': 'https://twitter.com/kuyacoy/status/1581185279376838657', 'tweetID': '1581185279376838657', 'conversationID': '1581185279376838657', 'mediaURLs': ['https://pbs.twimg.com/media/FfF_gKwXgAIpnpD.jpg'], 'media_extended': [{'url': 'https://pbs.twimg.com/media/FfF_gKwXgAIpnpD.jpg', 'altText': None, 'type': 'image', 'size': {'width': 760, 'height': 926}, 'thumbnail_url': 'https://pbs.twimg.com/media/FfF_gKwXgAIpnpD.jpg', 'id_str': '1581185134803517442'}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1665818922}
+testNSFWTweet_compare={'text': "ngl, I'm scared on finding out the cute Sprigatito's final evolution..\n\nso i had a bot generate it for me.... and I'm forever scarred", 'date': 'Sat Oct 15 07:28:42 +0000 2022', 'tweetURL': 'https://twitter.com/kuyacoy/status/1581185279376838657', 'tweetID': '1581185279376838657', 'conversationID': '1581185279376838657', 'mediaURLs': ['https://pbs.twimg.com/media/FfF_gKwXgAIpnpD.jpg'], 'media_extended': [{'url': 'https://pbs.twimg.com/media/FfF_gKwXgAIpnpD.jpg', 'altText': None, 'type': 'image', 'size': {'width': 760, 'height': 926}, 'thumbnail_url': 'https://pbs.twimg.com/media/FfF_gKwXgAIpnpD.jpg', 'id_str': '1581185134803517442'}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1665818922}
 testPollTweet_compare={'text': 'I know when that hotline bling, that can only:', 'date': 'Mon Oct 05 22:57:25 +0000 2015', 'tweetURL': 'https://twitter.com/norm/status/651169346518056960', 'tweetID': '651169346518056960', 'conversationID': '651169346518056960', 'mediaURLs': [], 'media_extended': [], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': False, 'combinedMediaUrl': None, 'pollData': {'options': [{'name': 'Mean one thing', 'votes': 124875, 'percent': 78.82}, {'name': 'Mean multiple things', 'votes': 33554, 'percent': 21.18}]}, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1444085845}
 testMixedMediaTweet_compare={'text': 'Some of us here are definitely big nerds about beer, and could talk your ear off about it for days on end, but some of us are just "beer is nice"', 'date': 'Thu Feb 22 12:13:24 +0000 2024', 'tweetURL': 'https://twitter.com/salebeerfest/status/1760638922084741177', 'tweetID': '1760638922084741177', 'conversationID': '1760638922084741177', 'mediaURLs': ['https://pbs.twimg.com/media/GG8LwfuWoAANKhs.jpg', 'https://video.twimg.com/tweet_video/GG8LwqWX0AAZch0.mp4'], 'media_extended': [{'url': 'https://pbs.twimg.com/media/GG8LwfuWoAANKhs.jpg', 'altText': None, 'type': 'image', 'size': {'width': 858, 'height': 960}, 'thumbnail_url': 'https://pbs.twimg.com/media/GG8LwfuWoAANKhs.jpg', 'id_str': '1760638907102699520'}, {'url': 'https://video.twimg.com/tweet_video/GG8LwqWX0AAZch0.mp4', 'type': 'gif', 'size': {'width': 500, 'height': 500}, 'duration_millis': 0, 'thumbnail_url': 'https://pbs.twimg.com/tweet_video_thumb/GG8LwqWX0AAZch0.jpg', 'altText': None, 'id_str': '1760638909954904064'}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': False, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1708604004}
 testVinePlayerTweet_compare={'text': 'You wanted old ROBLOX back, you got it. Check out our sweet "new" look! #BringBackOldROBLOX https://vine.co/v/OL9VqvM6wJh', 'date': 'Wed Apr 01 16:17:13 +0000 2015', 'tweetURL': 'https://twitter.com/Roblox/status/583302104342638592', 'tweetID': '583302104342638592', 'conversationID': '583302104342638592', 'mediaURLs': ['https://v.cdn.vine.co/r/videos/20A1BE53011195086166081318912_3fe3b526b1a.1.5.3156516531034157495.mp4?versionId=DI1mMu7EI6zcLbvgucyp3GHebdz8.9cQ'], 'media_extended': [{'url': 'https://v.cdn.vine.co/r/videos/20A1BE53011195086166081318912_3fe3b526b1a.1.5.3156516531034157495.mp4?versionId=DI1mMu7EI6zcLbvgucyp3GHebdz8.9cQ', 'type': 'video', 'size': {'width': 435, 'height': 435}}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': {'options': []}, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1427905033}