Merge branch 'main' of https://github.com/dylanpdx/BetterTwitFix into update

Reviewed-on: #2

.github/workflows/deploy.yml

@@ -71,3 +71,4 @@ jobs:
           VXTWITTER_WORKAROUND_TOKENS: ${{ secrets.VXTWITTER_WORKAROUND_TOKENS }}
           VXTWITTER_PROXIES: ${{ secrets.VXTWITTER_PROXIES }}
           SERVERLESS_ACCESS_KEY: ${{ secrets.SERVERLESS_ACCESS_KEY }}
+          DEEPL_API_KEY: ${{ secrets.DEEPL_API_KEY }}

.github/workflows/python-tests.yml

@@ -6,23 +6,30 @@ jobs:
   build:
     runs-on: ubuntu-latest
     environment: test
-    strategy:
-      matrix:
-        python-version: ["3.10", "3.11", "3.12"]
-
     steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
+      - name: Check out files
+        uses: actions/checkout@v4
+
+      - name: Install Python
+        run: |
+          apt-get update
+          apt-get install -y python3 python3-pip python3-venv
+
+      - name: Set up virtual environment
+        run: |
+          python3 -m venv venv
+          . venv/bin/activate
+          python -m pip install --upgrade pip
+
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          . venv/bin/activate
           pip install pytest pytest-cov pytest-env
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
       - name: Test with pytest
         env:
           VXTWITTER_WORKAROUND_TOKENS: ${{ secrets.VXTWITTER_WORKAROUND_TOKENS }}
         run: |
+          . venv/bin/activate
           pytest --cov-config=.coveragerc --cov=.

activity.py

@@ -2,17 +2,27 @@ import datetime
 import msgs
 from utils import determineEmbedTweet, determineMediaToEmbed
 from copy import deepcopy
+import html
 
 def tweetDataToActivity(tweetData,embedIndex = -1):
     content=""
 
     if tweetData['replyingTo'] is not None:
         content += f"<blockquote>↪️ <i>Replying to @{tweetData['replyingTo']}</i></blockquote>"
-    content+=f"<p>{tweetData['text']}</p>"
+
+    mainText = html.escape(tweetData['text'])
+    if tweetData["translation"] is not None:
+        content += f"<blockquote>🌐 <i>{tweetData['translation']['source_language'].upper()}→{tweetData['translation']['destination_language'].upper()}</i></blockquote>"
+        mainText=html.escape(tweetData['translation']["text"])
+        
+    content+=f"<p>{mainText}</p>"
 
     attachments=[]
     if tweetData['qrt'] is not None:
-        content += f"<blockquote><b>QRT: <a href=\"{tweetData['qrtURL']}\">{tweetData['qrt']['user_screen_name']}</a></b><br>{tweetData['qrt']['text']}</blockquote>"
+        qrtText = tweetData['qrt']['text']
+        if "translation" in tweetData['qrt'] and tweetData['qrt']["translation"] is not None:
+            qrtText = tweetData['qrt']["translation"]["text"]
+        content += f"<blockquote><b>QRT: <a href=\"{tweetData['qrtURL']}\">{tweetData['qrt']['user_screen_name']}</a></b><br>{qrtText}</blockquote>"
     if tweetData['pollData'] is not None:
         content += f"<p>{msgs.genPollDisplay(tweetData['pollData'])}</p>"
         content += "</p>"

config.json

@@ -2,14 +2,14 @@
     "config": {
         "appname": "vxTwitter",
         "color": "#1DA1F2",
-        "database": "[url to mongo database goes here]",
-        "link_cache": "ram",
+        "database": "mongodb://localhost:27017/bettervxtwitter",
+        "link_cache": "db",
         "method": "hybrid",
-        "repo": "https://github.com/dylanpdx/BetterTwitFix",
-        "table": "[database table here]",
-        "url": "https://vxtwitter.com",
+        "repo": "https://git.alterware.dev/alterware/BetterTwitFix",
+        "table": "links",
+        "url": "https://girlcock.alterware.dev",
         "combination_method": "local",
         "gifConvertAPI": "local",
-        "workaroundTokens":null
+        "workaroundTokens": null
     }
 }

configHandler.py

@@ -13,7 +13,8 @@ if ('RUNNING_SERVERLESS' in os.environ and os.environ['RUNNING_SERVERLESS'] == '
                 "url": os.getenv("VXTWITTER_URL","https://vxtwitter.com"),
                 "combination_method": os.getenv("VXTWITTER_COMBINATION_METHOD","local"), # can either be 'local' or a URL to a server handling requests in the same format
                 "gifConvertAPI":os.getenv("VXTWITTER_GIF_CONVERT_API",""),
-                "workaroundTokens":os.getenv("VXTWITTER_WORKAROUND_TOKENS",None)
+                "workaroundTokens":os.getenv("VXTWITTER_WORKAROUND_TOKENS",None),
+                "deeplKey":os.getenv("DEEPL_API_KEY",None),
             }
         }
 else:
@@ -31,7 +32,8 @@ else:
                     "url": "https://vxtwitter.com",
                     "combination_method": "local", # can either be 'local' or a URL to a server handling requests in the same format
                     "gifConvertAPI":"",
-                    "workaroundTokens":None
+                    "workaroundTokens":None,
+                    "deeplKey":None
                 }
             }
 

msgs.py

@@ -17,8 +17,11 @@ def genLikesDisplay(vnf):
         return ("💖 " + numerize.numerize(vnf['likes']))
 
 def genQrtDisplay(qrt):
+    text = qrt['text']
+    if "translation" in qrt and qrt["translation"] is not None:
+        text = qrt["translation"]["text"]
     verifiedCheck = "☑️" if ('verified' in qrt and qrt['verified']) else ""
-    return ("\n\n【QRT of " + qrt['user_name'] + " (@" + qrt['user_screen_name'] + ")"+ verifiedCheck+":】\n\n'" + qrt['text'] + "'")
+    return ("\n\n【QRT of " + qrt['user_name'] + " (@" + qrt['user_screen_name'] + ")"+ verifiedCheck+":】\n\n'" + text + "'")
 
 def genPollDisplay(poll):
     pctSplit=10
@@ -27,6 +30,9 @@ def genPollDisplay(poll):
         output+=choice["name"]+"\n"+("█"*int(choice["percent"]/pctSplit)) +" "+str(choice["percent"])+"%\n"
     return output
 
+def genTranslationDisplay(translation):
+    return f"【TL "+translation["source_language"]+"→"+translation["destination_language"]+"】\n\n"+translation["text"]
+
 # formats the top text of the embed
 def formatProvider(base,vnf):
     finalText = base
@@ -38,13 +44,16 @@ def formatProvider(base,vnf):
         finalText = base
     return finalText
 
-def formatEmbedDesc(type,body,qrt,pollData):
+def formatEmbedDesc(type,body,qrt,pollData,translation):
     # Trim the embed description to 248 characters, prioritizing poll and likes
 
     qrtType=None
     if qrt!=None:
         qrtType="Text"
 
+    if translation is not None:
+        body = genTranslationDisplay(translation)
+
     limit = videoDescLimit if type=="Video" or (qrt!=None and (qrtType=="Video")) else tweetDescLimit
 
     output = ""
@@ -73,6 +82,6 @@ def formatEmbedDesc(type,body,qrt,pollData):
         diff = len(output)-limit
         # remove the characters from body, add ellipsis
         body = body[:-(diff+1)]+"…"
-        return formatEmbedDesc(type,body,qrt,pollData)
+        return formatEmbedDesc(type,body,qrt,pollData,None)
     else:
         return output

requirements.txt

@@ -8,4 +8,5 @@ Werkzeug==2.3.7
 numerize==0.12
 oauthlib==3.2.2
 PyRTF3==0.47.5
-XClientTransaction==0.0.2
+XClientTransaction==1.0.2
+deepl==1.3.0

serverless.yml

@@ -29,6 +29,7 @@ provider:
     VXTWITTER_COMBINATION_METHOD: ${env:VXTWITTER_COMBINATION_METHOD, 'local'}
     VXTWITTER_GIF_CONVERT_API: ${env:VXTWITTER_GIF_CONVERT_API, ''}
     VXTWITTER_WORKAROUND_TOKENS: ${env:VXTWITTER_WORKAROUND_TOKENS, ''}
+    DEEPL_API_KEY: ${env:DEEPL_API_KEY, ''}
     #VXTWITTER_PROXIES: ${env:VXTWITTER_PROXIES, ''}
 
 package:

test_vx_embeds.py

@@ -205,3 +205,19 @@ def test_embed_action():
     assert resp.status_code==200
     assert "application/activity+json" in str(resp.data)
     assert "%F0%9F%92%96" not in str(resp.data) # 💖
+
+def test_embed_translated():
+    resp = client.get(testTextTweet.replace("https://twitter.com","")+"/jp",headers={"User-Agent":"test"})
+    assert resp.status_code==200
+    assert "→" in resp.text
+    resp = client.get(testTextTweet.replace("https://twitter.com","")+"/ja",headers={"User-Agent":"test"})
+    assert resp.status_code==200
+    assert "→" in resp.text
+
+def test_embed_translated_subdomain():
+    resp = client.get(testTextTweet.replace("https://twitter.com","https://jp.vxtwitter.com"),headers={"User-Agent":"test"})
+    assert resp.status_code==200
+    assert "→" in resp.text
+    resp = client.get(testTextTweet.replace("https://twitter.com","https://ja.vxtwitter.com"),headers={"User-Agent":"test"})
+    assert resp.status_code==200
+    assert "→" in resp.text

translation.py

@@ -0,0 +1,27 @@
+import deepl
+from configHandler import config
+
+validLanguages = ['AF', 'AN', 'AR', 'AS', 'AY', 'AZ', 'BA', 'BE', 'BG', 'BN', 'BR', 'BS', 'CA', 'CS', 'CY', 'DA', 'DE', 'DE-DE', 'EL', 'EN', 'EO', 'ES', 'ES-419', 'ET', 'EU', 'FA', 'FI', 'FR', 'FR-FR', 'GA', 'GL', 'GN', 'GU', 'HA', 'HE', 'HI', 'HR', 'HT', 'HU', 'HY', 'ID', 'IG', 'IS', 'IT', 'JA', 'JV', 'KA', 'KK', 'KO', 'KY', 'LA', 'LB', 'LN', 'LT', 'LV', 'MG', 'MI', 'MK', 'ML', 'MN', 'MR', 'MS', 'MT', 'MY', 'NB', 'NE', 'NL', 'OC', 'OM', 'PA', 'PL', 'PS', 'PT-BR', 'PT-PT', 'QU', 'RO', 'RU', 'SA', 'SK', 'SL', 'SQ', 'SR', 'ST', 'SU', 'SV', 'SW', 'TA', 'TE', 'TG', 'TH', 'TK', 'TL', 'TN', 'TR', 'TS', 'TT', 'UK', 'UR', 'UZ', 'VI', 'WO', 'XH', 'YI', 'ZH', 'ZH-HANS', 'ZH-HANT', 'ZU']
+languageAliases = {
+    #"EN":"EN-US",
+    "JP":"JA",
+}
+
+mergedLangs=(validLanguages+list(languageAliases.keys()))
+
+def getDeeplTranslation(text,target):
+    target = target.upper()
+    if target == "EN":
+        target = "EN-US"
+    try:
+        if 'deeplKey' not in config["config"] or config["config"]["deeplKey"] == None:
+            return None
+        deepl_client = deepl.Translator(config["config"]["deeplKey"])
+        result = deepl_client.translate_text(text, target_lang=target)
+        return {
+            "text":result.text,
+            "source_language":result.detected_source_lang,
+            "destination_language":target
+        }
+    except Exception as e:
+        return None

twExtract/__init__.py

@@ -14,7 +14,7 @@ bearer="Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DT
 v2bearer="Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
 androidBearer="Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F"
 
-requestUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0"
+requestUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0"
 
 bearerTokens=[bearer,v2bearer,androidBearer]
 
@@ -113,7 +113,7 @@ def cycleBearerTokenGet(url,headers):
         return tweet
     raise TwExtractError(400, "Extract error")
 
-def twitterApiGet(url,btoken=None,authToken=None,guestToken=None,userAgent=None):
+def twitterApiGet(url,btoken=None,authToken=None,guestToken=None,userAgent=None,language=None):
 
     if authToken != None and authToken.startswith("oa|"):
         url = url.replace("https://x.com/i/api/graphql/","https://api.twitter.com/graphql/")
@@ -122,7 +122,7 @@ def twitterApiGet(url,btoken=None,authToken=None,guestToken=None,userAgent=None)
         secret = authToken.split("|")[1]
 
         twt = oauth1.Client(client_key='3nVuSoBZnx6U4vzUxf5w',client_secret='Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys',resource_owner_key=key,resource_owner_secret=secret)
-        hdr = getAuthHeaders(androidBearer)
+        hdr = getAuthHeaders(androidBearer,language=language)
         del hdr["Authorization"]
         hdr["X-Twitter-Client"] = "TwitterAndroid"
         if userAgent is not None:
@@ -135,12 +135,12 @@ def twitterApiGet(url,btoken=None,authToken=None,guestToken=None,userAgent=None)
         if btoken is None:
             btoken = v2bearer
             #return cycleBearerTokenGet(url,getAuthHeaders(bearer,authToken=authToken,guestToken=guestToken))
-        headers = getAuthHeaders(btoken,authToken=authToken,guestToken=guestToken)
+        headers = getAuthHeaders(btoken,authToken=authToken,guestToken=guestToken,language=language)
         response = requests.get(url, headers=headers)
 
     return response
 
-def getAuthHeaders(btoken,authToken=None,guestToken=None):
+def getAuthHeaders(btoken,authToken=None,guestToken=None,language=None):
 
     csrfToken=str(uuid.uuid4()).replace('-', '')
     headers = {"x-twitter-active-user":"yes","x-twitter-client-language":"en","x-csrf-token":csrfToken,"User-Agent":requestUserAgent}
@@ -152,19 +152,17 @@ def getAuthHeaders(btoken,authToken=None,guestToken=None):
     if guestToken is not None:
         headers["x-guest-token"] = guestToken
 
+    if language is not None:
+        headers["x-twitter-client-language"] = language
+
     return headers
 
 def getGuestToken():
     global guestToken
     global guestTokenUses
     if guestToken is None:
-        r = requests.get(f"https://{twitterUrl}",headers={"User-Agent":requestUserAgent,"Cookie":"night_mode=2"},allow_redirects=False)
-        m = re.search(gt_pattern, r.text)
-        if m is None:
-            r = requests.post(f"https://api.{twitterUrl}/1.1/guest/activate.json", headers={"Authorization":bearer})
-            guestToken = json.loads(r.text)["guest_token"]
-        else:
-            guestToken = m.group(1)
+        r = requests.post(f"https://api.{twitterUrl}/1.1/guest/activate.json", headers={"Authorization":v2bearer,"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0"})
+        guestToken = json.loads(r.text)["guest_token"]
         guestTokenUses = 0
     else:
         guestTokenUses+=1
@@ -216,7 +214,7 @@ def extractStatus_guestToken(url):
         raise TwExtractError(error["code"], error["message"])
     return output
 
-def extractStatus_syndication(url,workaroundTokens=None):
+def extractStatus_syndication(url,workaroundTokens=None,tlLanguage=None):
     # https://github.com/mikf/gallery-dl/blob/46cae04aa3a113c7b6bbee1bb468669564b14ae8/gallery_dl/extractor/twitter.py#L1784
     m = re.search(pathregex, url)
     if m is None:
@@ -268,7 +266,7 @@ def extractStatus_twExtractProxy(url,workaroundTokens=None):
             continue
         return output
 
-def extractStatusV2(url,workaroundTokens):
+def extractStatusV2(url,workaroundTokens,tlLanguage=None):
     # get tweet ID
     m = re.search(pathregex, url)
     if m is None:
@@ -282,7 +280,7 @@ def extractStatusV2(url,workaroundTokens):
     def request_with_token(twid, authToken):
         vars = json.loads('{"includeTweetImpression":true,"includeHasBirdwatchNotes":false,"includeEditPerspective":false,"rest_ids":["x"],"includeEditControl":true,"includeCommunityTweetRelationship":true,"includeTweetVisibilityNudge":true}')
         vars['rest_ids'][0] = str(twid)
-        tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2graphql_api}/TweetResultsByIdsQuery?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2Features)}",authToken=authToken)
+        tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2graphql_api}/TweetResultsByIdsQuery?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2Features)}",authToken=authToken,language=tlLanguage)
         try:
             rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining")
             print(f"Twitter Token Rate limit remaining: {rateLimitRemaining}")
@@ -321,7 +319,7 @@ def extractStatusV2(url,workaroundTokens):
         return tweet
     return parallel_token_request(twid, tokens, request_with_token)
 
-def extractStatusV2Android(url,workaroundTokens):
+def extractStatusV2Android(url,workaroundTokens,tlLanguage=None):
     # get tweet ID
     m = re.search(pathregex, url)
     if m is None:
@@ -335,7 +333,7 @@ def extractStatusV2Android(url,workaroundTokens):
         try:
             vars = json.loads('{"referrer":"home","includeTweetImpression":true,"includeHasBirdwatchNotes":false,"isReaderMode":false,"includeEditPerspective":false,"includeEditControl":true,"focalTweetId":0,"includeCommunityTweetRelationship":true,"includeTweetVisibilityNudge":true}')
             vars['focalTweetId'] = int(twid)
-            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{androidGraphql_api}/ConversationTimelineV2?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(androidGraphqlFeatures)}", authToken=authToken,btoken=androidBearer,userAgent="TwitterAndroid/11.61.0-release.0 (311610000-r-0) G011A/9 (google;G011A;google;G011A;0;;1;2016)")
+            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{androidGraphql_api}/ConversationTimelineV2?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(androidGraphqlFeatures)}", authToken=authToken,btoken=androidBearer,userAgent="TwitterAndroid/11.61.0-release.0 (311610000-r-0) G011A/9 (google;G011A;google;G011A;0;;1;2016)",language=tlLanguage)
             try:
                 rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining")
                 print(f"Twitter Android Token Rate limit remaining: {rateLimitRemaining}")
@@ -380,7 +378,7 @@ def extractStatusV2Android(url,workaroundTokens):
         return tweet
     return parallel_token_request(twid, tokens, request_with_token)
 
-def extractStatusV2TweetDetail(url,workaroundTokens):
+def extractStatusV2TweetDetail(url,workaroundTokens,tlLanguage=None):
     # get tweet ID
     m = re.search(pathregex, url)
     if m is None:
@@ -395,7 +393,7 @@ def extractStatusV2TweetDetail(url,workaroundTokens):
         try:
             vars = json.loads('{"focalTweetId":"0","with_rux_injections":false,"includePromotedContent":true,"withCommunity":true,"withQuickPromoteEligibilityTweetFields":true,"withBirdwatchNotes":true,"withVoice":true,"withV2Timeline":true}')
             vars['focalTweetId'] = str(twid)
-            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{tweetDetailGraphql_api}/TweetDetail?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(tweetDetailGraphqlFeatures)}", authToken=authToken,btoken=v2bearer)
+            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{tweetDetailGraphql_api}/TweetDetail?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(tweetDetailGraphqlFeatures)}", authToken=authToken,btoken=v2bearer,language=tlLanguage)
             try:
                 rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining")
                 print(f"Twitter Token Rate limit remaining: {rateLimitRemaining}")
@@ -440,10 +438,10 @@ def extractStatusV2TweetDetail(url,workaroundTokens):
         return tweet
     return parallel_token_request(twid, tokens, request_with_token)
 
-def extractStatusV2Rest_Anon(url,workaroundTokens):
-    return extractStatusV2Rest(url,None)
+def extractStatusV2Rest_Anon(url,workaroundTokens,tlLanguage=None):
+    return extractStatusV2Rest(url,None,tlLanguage=tlLanguage)
 
-def extractStatusV2Rest(url,workaroundTokens):
+def extractStatusV2Rest(url,workaroundTokens,tlLanguage=None):
     # get tweet ID
     m = re.search(pathregex, url)
     if m is None:
@@ -461,11 +459,11 @@ def extractStatusV2Rest(url,workaroundTokens):
             random.shuffle(tokens)
             for authToken in tokens:
                 try:
-                    tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", btoken=v2bearer,authToken=authToken,guestToken=guestToken)
+                    tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", btoken=v2bearer,authToken=authToken,guestToken=guestToken,language=tlLanguage)
                 except Exception as e:
                     continue
         else:
-            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", btoken=v2bearer,guestToken=guestToken)
+            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", btoken=v2bearer,guestToken=guestToken,language=tlLanguage)
         
         try:
             rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining")
@@ -512,12 +510,13 @@ def fixTweetData(tweet):
         pass
     return tweet
 
-def extractStatus(url,workaroundTokens=None):
-    # TODO: commented out methods are too slow/unreliable at the moment
+def extractStatus(url,workaroundTokens=None,tlLanguage=None):
     methods=[extractStatusV2Rest_Anon,extractStatusV2,extractStatusV2Rest,extractStatusV2Android]#,extractStatusV2TweetDetail]
+    if tlLanguage is not None: # prioritize endpoints that return translations
+        methods = [extractStatusV2Rest,extractStatusV2TweetDetail,extractStatusV2Android,extractStatusV2,extractStatusV2Rest_Anon]
     for method in methods:
         try:
-            result = method(url,workaroundTokens)
+            result = method(url,workaroundTokens,tlLanguage=tlLanguage)
             if 'legacy' not in result:
                 print(f"{method.__name__} method failed: Legacy not found for {url}")
                 # try another method

twExtract/twUtils.py

@@ -52,7 +52,7 @@ def get_twitter_homepage(headers=None):
 def generate_transaction_id(method: str, path: str,headers=None) -> str:
     session = requests.Session()
     session.headers = generate_headers()
-    home_page = session.get(url="https://x.com")
+    home_page = session.get(url="https://x.com/home")
     home_page_response = bs4.BeautifulSoup(home_page.content, 'html.parser')
     ondemand_file_url = get_ondemand_file_url(response=home_page_response)
     ondemand_file = session.get(url=ondemand_file_url)

twitfix.py

@@ -14,7 +14,7 @@ import msgs
 import twExtract as twExtract
 from cache import addVnfToLinkCache,getVnfFromLinkCache
 import vxlogging as log
-from utils import getTweetIdFromUrl, pathregex, determineMediaToEmbed, determineEmbedTweet, BytesIOWrapper, fixMedia
+from utils import getTweetIdFromUrl, pathregex, determineMediaToEmbed, determineEmbedTweet, BytesIOWrapper, fixMedia, indexOfAny
 from vxApi import getApiResponse, getApiUserResponse
 from urllib.parse import urlparse 
 from PyRTF.Elements import Document
@@ -22,12 +22,14 @@ from PyRTF.document.section import Section
 from PyRTF.document.paragraph import Paragraph
 from copy import deepcopy
 import json
-import datetime
+import translation
 import activity as activitymg
 app = Flask(__name__)
 CORS(app)
 user_agent=""
 
+activityseparator="zqj"
+
 staticFiles = { # TODO: Use flask static files instead of this
     "favicon.ico": {"mime": "image/vnd.microsoft.icon","path": "favicon.ico"},
     "apple-touch-icon.png": {"mime": "image/png","path": "apple-touch-icon.png"},
@@ -82,7 +84,11 @@ def generateActivityLink(tweetData,media=None,mediatype=None,embedIndex=-1):
         return None
     try:
         embedIndex = embedIndex+1
-        return f"{config['config']['url']}/users/{tweetData['user_screen_name']}/statuses/{str(embedIndex)}{tweetData['tweetID']}"
+
+        extras=""
+        if tweetData["translation"] is not None:
+            extras+=activityseparator+tweetData["translation"]["destination_language"]
+        return f"{config['config']['url']}/users/{tweetData['user_screen_name']}/statuses/{str(embedIndex)}{activityseparator}{tweetData['tweetID']}{extras}"
     except Exception as e:
         log.error("Error generating activity link: "+str(e))
         return None
@@ -95,7 +101,7 @@ def getAppName(tweetData,appnameSuffix=""):
 
 def renderImageTweetEmbed(tweetData,image,appnameSuffix="",embedIndex=-1):
     qrt = tweetData['qrt']
-    embedDesc = msgs.formatEmbedDesc("Image",tweetData['text'],qrt,tweetData['pollData'])
+    embedDesc = msgs.formatEmbedDesc("Image",tweetData['text'],qrt,tweetData['pollData'],tweetData["translation"])
 
     if image.startswith("https://pbs.twimg.com") and "?" not in image:
         image = f"{image}?name=orig"
@@ -115,7 +121,7 @@ def renderImageTweetEmbed(tweetData,image,appnameSuffix="",embedIndex=-1):
 
 def renderVideoTweetEmbed(tweetData,mediaInfo,appnameSuffix="",embedIndex=-1):
     qrt = tweetData['qrt']
-    embedDesc = msgs.formatEmbedDesc("Video",tweetData['text'],qrt,tweetData['pollData'])
+    embedDesc = msgs.formatEmbedDesc("Video",tweetData['text'],qrt,tweetData['pollData'],tweetData["translation"])
 
     mediaInfo=fixMedia(mediaInfo)
 
@@ -138,7 +144,7 @@ def renderVideoTweetEmbed(tweetData,mediaInfo,appnameSuffix="",embedIndex=-1):
 
 def renderTextTweetEmbed(tweetData,appnameSuffix=""):
     qrt = tweetData['qrt']
-    embedDesc = msgs.formatEmbedDesc("Text",tweetData['text'],qrt,tweetData['pollData'])
+    embedDesc = msgs.formatEmbedDesc("Text",tweetData['text'],qrt,tweetData['pollData'],tweetData["translation"])
 
     return render_template("text.html",
                     tweet=tweetData,
@@ -154,7 +160,7 @@ def renderTextTweetEmbed(tweetData,appnameSuffix=""):
 
 def renderArticleTweetEmbed(tweetData,appnameSuffix=""):
     articlePreview=tweetData['article']["title"]+"\n\n"+tweetData['article']["preview_text"]+"…"
-    embedDesc = msgs.formatEmbedDesc("Image",articlePreview,None,None)
+    embedDesc = msgs.formatEmbedDesc("Image",articlePreview,None,None,None)
 
     return render_template("image.html",
                     tweet=tweetData,
@@ -260,9 +266,9 @@ def userJson():
         },
     }
 
-def getTweetData(twitter_url,include_txt="false",include_rtf="false"):
+def getTweetData(twitter_url,include_txt="false",include_rtf="false",tlLanguage=None):
     cachedVNF = getVnfFromLinkCache(twitter_url)
-    if cachedVNF is not None and include_txt == "false" and include_rtf == "false":
+    if cachedVNF is not None and include_txt == "false" and include_rtf == "false" and tlLanguage is None:
         return cachedVNF
 
     try:
@@ -270,7 +276,7 @@ def getTweetData(twitter_url,include_txt="false",include_rtf="false"):
             workaroundTokens = config['config']['workaroundTokens'].split(",")
         else:
             workaroundTokens = None
-        rawTweetData = twExtract.extractStatus(twitter_url,workaroundTokens=workaroundTokens)
+        rawTweetData = twExtract.extractStatus(twitter_url,workaroundTokens=workaroundTokens,tlLanguage=tlLanguage)
     except:
         rawTweetData = None
     if rawTweetData == None or 'error' in rawTweetData:
@@ -281,7 +287,15 @@ def getTweetData(twitter_url,include_txt="false",include_rtf="false"):
     tweetData = getApiResponse(rawTweetData,include_txt,include_rtf)
     if tweetData is None:
         return None
-    if include_txt == "false" and include_rtf == "false":
+    
+    if tweetData["translation"] is None and tlLanguage is not None and tlLanguage != tweetData["lang"]:
+        # translation probably failed on twitter's end
+        tweetData["translation"] = translation.getDeeplTranslation(tweetData["text"],tlLanguage)
+
+    if tweetData["translation"] is not None and tlLanguage is None:
+        tweetData["translation"] = None # didn't ask for TL
+
+    if include_txt == "false" and include_rtf == "false" and tlLanguage is None:
         addVnfToLinkCache(twitter_url,tweetData)
     return tweetData
 
@@ -307,13 +321,19 @@ def getUserData(twitter_url,includeFeed=False):
 
 @app.route('/<path:sub_path>') # Default endpoint used by everything
 def twitfix(sub_path):
+    splitPath=sub_path.split("/")
+    if "|" in splitPath[-1]: # discord doesn't strip this when a link is spoilered
+        splitPath[-1] = splitPath[-1].replace("|","")
+
+    subdomain = request.host.rsplit(".",2)[0]
+
     global user_agent
     user_agent = request.headers.get('User-Agent', None)
     if user_agent is None:
         user_agent = "unknown"
 
-    isApiRequest=request.url.startswith("https://api.vx") or request.url.startswith("http://api.vx")
-    if not isApiRequest and (request.url.startswith("https://l.vx") or request.url.startswith("https://old.vx")) and "Discord" in user_agent:
+    isApiRequest=subdomain=="api"
+    if not isApiRequest and (subdomain=="l" or subdomain=="old") and "Discord" in user_agent:
         user_agent = user_agent.replace("Discord","LegacyEmbed") # TODO: Clean up; This is a hacky fix to make the new activity embed not trigger
     if sub_path in staticFiles:
         if 'path' not in staticFiles[sub_path] or staticFiles[sub_path]["path"] == None:
@@ -355,24 +375,38 @@ def twitfix(sub_path):
     include_txt="false"
     include_rtf="false"
 
+    
+    translationIndex=indexOfAny(splitPath,translation.mergedLangs,caseInsensitive=True)
+    translationLang = None
+    if translationIndex != -1:
+        if splitPath[translationIndex].upper() in translation.languageAliases:
+            translationLang = translation.languageAliases[splitPath[translationIndex].upper()]
+        else:
+            translationLang = splitPath[translationIndex].lower()
+    elif subdomain.upper() in translation.mergedLangs:
+        if subdomain.upper() in translation.languageAliases:
+            translationLang = translation.languageAliases[subdomain.upper()]
+        else:
+            translationLang = subdomain.lower()
+
     if isApiRequest:
         if "include_txt" in request.args:
             include_txt = request.args.get("include_txt")
         if "include_rtf" in request.args:
             include_rtf = request.args.get("include_rtf")
 
-    tweetData = getTweetData(twitter_url,include_txt,include_rtf)
+    tweetData = getTweetData(twitter_url,include_txt,include_rtf,tlLanguage=translationLang)
     if tweetData is None:
         log.error("Tweet Data Get failed for "+twitter_url)
         return message(msgs.failedToScan)
     qrt = None
     if 'qrtURL' in tweetData and tweetData['qrtURL'] is not None:
-        qrt = getTweetData(tweetData['qrtURL'])
+        qrt = getTweetData(tweetData['qrtURL'],tlLanguage=translationLang)
     tweetData['qrt'] = qrt
 
     retweet = None
     if 'retweetURL' in tweetData and tweetData['retweetURL'] is not None:
-        retweet = getTweetData(tweetData['retweetURL'])
+        retweet = getTweetData(tweetData['retweetURL'],tlLanguage=translationLang)
     tweetData['retweet'] = retweet
 
     tweetData = deepcopy(tweetData)
@@ -383,7 +417,7 @@ def twitfix(sub_path):
         requestUrlWithoutQuery = request.url
 
     directEmbed=False
-    if requestUrlWithoutQuery.startswith("https://d.vx") or requestUrlWithoutQuery.endswith(".mp4") or requestUrlWithoutQuery.endswith(".png"):
+    if subdomain=="d" or requestUrlWithoutQuery.endswith(".mp4") or requestUrlWithoutQuery.endswith(".png"):
         directEmbed = True
         # remove the .mp4 from the end of the URL
         if requestUrlWithoutQuery.endswith(".mp4") or requestUrlWithoutQuery.endswith(".png"):
@@ -403,9 +437,10 @@ def twitfix(sub_path):
         return send_file(BytesIOWrapper(rtf), mimetype='application/rtf', as_attachment=True, download_name=f'{tweetData["user_screen_name"]}_{tweetData["tweetID"]}.rtf')
 
     embedIndex = -1
-    # if url ends with /1, /2, /3, or /4, we'll use that as the index
-    if sub_path[-2:] in ["/1","/2","/3","/4"]:
-        embedIndex = int(sub_path[-1])-1
+    # if url has /1, /2, /3, or /4, we'll use that as the index
+    numIdx=indexOfAny(splitPath,["1","2","3","4"]);
+    if numIdx != -1:
+        embedIndex = int(splitPath[numIdx])-1
         sub_path = sub_path[:-2]
         
     if isApiRequest: # Directly return the API response if the request is from the API
@@ -484,21 +519,29 @@ def rendercombined():
 
 @app.route("/api/v1/statuses/<string:tweet_id>")
 def api_v1_status(tweet_id):
-    embedIndex = int(tweet_id[0])-1
-    tweet_id = int(tweet_id[1:])
+    lang = None
+    if activityseparator not in tweet_id: #old embeds
+        embedIndex = int(tweet_id[0])-1
+        tweet_id = int(tweet_id[1:])
+    else:
+        args = tweet_id.split(activityseparator)
+        embedIndex = int(args[0])-1
+        tweet_id = int(args[1])
+        if len(args)>2:
+            lang=args[2]
     twitter_url=f"https://twitter.com/i/status/{tweet_id}"
-    tweetData = getTweetData(twitter_url)
+    tweetData = getTweetData(twitter_url,tlLanguage=lang)
     if tweetData is None:
         log.error("Tweet Data Get failed for "+twitter_url)
         return message(msgs.failedToScan)
     qrt = None
     if 'qrtURL' in tweetData and tweetData['qrtURL'] is not None:
-        qrt = getTweetData(tweetData['qrtURL'])
+        qrt = getTweetData(tweetData['qrtURL'],tlLanguage=lang)
     tweetData['qrt'] = qrt
 
     retweet = None
     if 'retweetURL' in tweetData and tweetData['retweetURL'] is not None:
-        retweet = getTweetData(tweetData['retweetURL'])
+        retweet = getTweetData(tweetData['retweetURL'],tlLanguage=lang)
     tweetData['retweet'] = retweet
 
     if tweetData is None:
@@ -522,4 +565,4 @@ def oEmbedGen(description, user, video_link, ttype,providerName=None):
 
 if __name__ == "__main__":
     app.config['SERVER_NAME']='localhost:80'
-    app.run(host='0.0.0.0')
+    app.run(host='0.0.0.0', port=8080)

twitfix.service

@@ -3,11 +3,11 @@ Description=Init file for twitfix uwsgi instance
 After=network.target
 
 [Service]
-User=dylan
-Group=dylan
-WorkingDirectory=/home/dylan/BetterTwitFix
-Environment="PATH=/home/dylan/BetterTwitFix/venv/bin"
-ExecStart=/home/dylan/BetterTwitFix/venv/bin/uwsgi --ini twitfix.ini
+User=carbonara
+Group=carbonara
+WorkingDirectory=/home/carbonara/twitter/BetterTwitFix
+Environment="PATH=/home/carbonara/twitter/BetterTwitFix/venv/bin"
+ExecStart=/home/carbonara/twitter/BetterTwitFix/venv/bin/uwsgi --ini twitfix.ini
 Restart=always
 RestartSec=3
 

utils.py

@@ -92,3 +92,14 @@ def determineMediaToEmbed(tweetData,embedIndex = -1,convertGif = True):
                     suffix += " • GIF"
                     media["suffix"] = suffix
         return media
+    
+def indexOfAny(list,items,caseInsensitive=False):
+    if caseInsensitive:
+        list = [item.upper() for item in list]
+        items = [item.upper() for item in items]
+    for item in items:
+        try:
+            return list.index(item)
+        except:
+            continue
+    return -1

vxApi.py

@@ -77,6 +77,7 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False):
     oldTweetVersion = False
     tweetArticle=None
     lang=None
+    translation=None
 
     if "screen_name" not in userL:
         userL["screen_name"] = user["core"]["screen_name"]
@@ -85,6 +86,14 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False):
     if "profile_image_url_https" not in userL:
         userL["profile_image_url_https"] = user["avatar"]["image_url"]
 
+    if "grok_translated_post_with_availability" in tweet and tweet["grok_translated_post_with_availability"]["is_available"] and "data" in tweet["grok_translated_post_with_availability"]:
+        tlData = tweet["grok_translated_post_with_availability"]["data"]
+        translation = {
+            "source_language":tlData["source_language"],
+            "destination_language":tlData["destination_language"],
+            "text":tlData["translation"]
+        }
+
     #editedTweet=False
     try:
         if "birdwatch_pivot" in tweet:
@@ -305,6 +314,7 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False):
         "replyingToID": replyingToID,
         "fetched_on": int(datetime.now().timestamp()),
         "retweetURL":retweetURL,
+        "translation":translation
     }
     try:
         apiObject["date_epoch"] = int(datetime.strptime(tweetL["created_at"], "%a %b %d %H:%M:%S %z %Y").timestamp())