Merge pull request 'YOLO UPDATE' (#3) from update into alterware

Reviewed-on: #3

.github/workflows/deploy.yml

@@ -70,4 +70,5 @@ jobs:
           VXTWITTER_GIF_CONVERT_API: ${{ secrets.VXTWITTER_GIF_CONVERT_API }}
           VXTWITTER_WORKAROUND_TOKENS: ${{ secrets.VXTWITTER_WORKAROUND_TOKENS }}
           VXTWITTER_PROXIES: ${{ secrets.VXTWITTER_PROXIES }}
-          SERVERLESS_ACCESS_KEY: ${{ secrets.SERVERLESS_ACCESS_KEY }}
+          SERVERLESS_ACCESS_KEY: ${{ secrets.SERVERLESS_ACCESS_KEY }}
+          DEEPL_API_KEY: ${{ secrets.DEEPL_API_KEY }}

activity.py

@@ -2,17 +2,27 @@ import datetime
 import msgs
 from utils import determineEmbedTweet, determineMediaToEmbed
 from copy import deepcopy
+import html
 
 def tweetDataToActivity(tweetData,embedIndex = -1):
     content=""
 
     if tweetData['replyingTo'] is not None:
         content += f"<blockquote>↪️ <i>Replying to @{tweetData['replyingTo']}</i></blockquote>"
-    content+=f"<p>{tweetData['text']}</p>"
+
+    mainText = html.escape(tweetData['text'])
+    if tweetData["translation"] is not None:
+        content += f"<blockquote>🌐 <i>{tweetData['translation']['source_language'].upper()}→{tweetData['translation']['destination_language'].upper()}</i></blockquote>"
+        mainText=html.escape(tweetData['translation']["text"])
+        
+    content+=f"<p>{mainText}</p>"
 
     attachments=[]
     if tweetData['qrt'] is not None:
-        content += f"<blockquote><b>QRT: <a href=\"{tweetData['qrtURL']}\">{tweetData['qrt']['user_screen_name']}</a></b><br>{tweetData['qrt']['text']}</blockquote>"
+        qrtText = tweetData['qrt']['text']
+        if "translation" in tweetData['qrt'] and tweetData['qrt']["translation"] is not None:
+            qrtText = tweetData['qrt']["translation"]["text"]
+        content += f"<blockquote><b>QRT: <a href=\"{tweetData['qrtURL']}\">{tweetData['qrt']['user_screen_name']}</a></b><br>{qrtText}</blockquote>"
     if tweetData['pollData'] is not None:
         content += f"<p>{msgs.genPollDisplay(tweetData['pollData'])}</p>"
         content += "</p>"
@@ -35,7 +45,10 @@ def tweetDataToActivity(tweetData,embedIndex = -1):
     if media is not None:
         media = deepcopy(media)
         if media['type'] == "gif":
-            media['type'] = "gifv"
+            if "/convert.avif" in media['url']:
+                media['type'] = "image"
+            else:
+                media['type'] = "gifv"
         if 'thumbnail_url' not in media:
             media['thumbnail_url'] = media['url']
         if media['type'] == "image" and "?" not in media['url']:

configHandler.py

@@ -13,7 +13,8 @@ if ('RUNNING_SERVERLESS' in os.environ and os.environ['RUNNING_SERVERLESS'] == '
                 "url": os.getenv("VXTWITTER_URL","https://vxtwitter.com"),
                 "combination_method": os.getenv("VXTWITTER_COMBINATION_METHOD","local"), # can either be 'local' or a URL to a server handling requests in the same format
                 "gifConvertAPI":os.getenv("VXTWITTER_GIF_CONVERT_API",""),
-                "workaroundTokens":os.getenv("VXTWITTER_WORKAROUND_TOKENS",None)
+                "workaroundTokens":os.getenv("VXTWITTER_WORKAROUND_TOKENS",None),
+                "deeplKey":os.getenv("DEEPL_API_KEY",None),
             }
         }
 else:
@@ -31,7 +32,8 @@ else:
                     "url": "https://vxtwitter.com",
                     "combination_method": "local", # can either be 'local' or a URL to a server handling requests in the same format
                     "gifConvertAPI":"",
-                    "workaroundTokens":None
+                    "workaroundTokens":None,
+                    "deeplKey":None
                 }
             }
 

gifConvert/Dockerfile

@@ -1,30 +1,76 @@
-FROM public.ecr.aws/lambda/python:3.8 AS builder
-RUN yum -y install git curl
-RUN yum -y groupinstall 'Development Tools'
-RUN git clone https://github.com/kohler/gifsicle
-WORKDIR gifsicle
-RUN autoreconf -i
-RUN ./configure --disable-gifview --disable-gifdiff
-RUN make
-RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
-WORKDIR /var/task
-RUN git clone https://github.com/ImageOptim/gifski
-WORKDIR gifski
-RUN /root/.cargo/bin/cargo build --release
+ARG TARGETARCH
+FROM public.ecr.aws/lambda/python:3.12 AS builder
+RUN dnf -y install git cargo && dnf clean all
+RUN git clone https://github.com/ImageOptim/gifski /gifski
+WORKDIR /gifski
+RUN cargo build --release
 
+#FROM public.ecr.aws/lambda/python:3.12 AS ffmpeg-linux-amd64
+#RUN dnf -y update
+#RUN dnf -y install git wget tar.x86_64 xz && dnf clean all
+#WORKDIR /ffmpeg
+#RUN wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz
+#RUN tar -xvf ffmpeg-release-amd64-static.tar.xz
 
-FROM public.ecr.aws/lambda/python:3.8
-RUN yum -y update
-RUN yum -y install git && yum -y install wget && yum -y install tar.x86_64 && yum -y install xz && yum clean all
-RUN wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz
-RUN tar -xvf ffmpeg-release-amd64-static.tar.xz
-RUN mv ff*/ffmpeg . && mv ff*/ffprobe . && rm *.tar.xz && rm -rf ff*/
-COPY --from=builder /var/task/gifsicle/src/gifsicle ./
-COPY --from=builder /var/task/gifski/target/release/gifski ./
+#FROM public.ecr.aws/lambda/python:3.12 AS ffmpeg-linux-arm64
+#RUN dnf -y update
+#RUN dnf -y install git wget tar xz && dnf clean all
+#WORKDIR /ffmpeg
+#RUN wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-arm64-static.tar.xz
+#RUN tar -xvf ffmpeg-release-arm64-static.tar.xz
+
+#FROM ffmpeg-linux-${TARGETARCH} AS ffmpeg
+
+FROM public.ecr.aws/lambda/python:3.12 AS ffmpeg
+# tl;dr: build ffmpeg from source with svt-av1 support and statically linked binaries
+
+# Install dependencies
+RUN dnf remove -y microdnf-dnf && microdnf install -y dnf
+RUN dnf group install "Development Tools" -y
+
+RUN mkdir /ffmpeg_sources
+WORKDIR /ffmpeg_sources
+RUN git clone https://git.ffmpeg.org/ffmpeg.git .
+RUN git clone --branch stable --depth 1 https://code.videolan.org/videolan/x264.git
+RUN git clone https://gitlab.com/AOMediaCodec/SVT-AV1.git
+
+RUN dnf install curl nasm glibc-static libstdc++-static cmake --allowerasing -y
+
+# Install x264
+WORKDIR /ffmpeg_sources/x264
+RUN PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" ./configure --prefix="/ffmpeg_build" --bindir="/bin" --enable-static
+RUN make -j
+RUN make install
+
+# Install SVT-AV1
+WORKDIR /ffmpeg_sources
+WORKDIR /ffmpeg_sources/SVT-AV1/Build
+
+RUN PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" cmake .. -G"Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DENABLE_STATIC=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_C_FLAGS_INIT="-static"
+
+RUN make -j
+RUN make install
+
+# Install FFmpeg
+WORKDIR /ffmpeg_sources
+RUN PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" ./configure --prefix="/ffmpeg_build" \
+    --pkg-config-flags="--static" --extra-ldexeflags="-static" --extra-libs="-lpthread -lm" --bindir="/bin" --disable-shared --enable-static --enable-gpl --enable-pthreads \
+    --enable-libx264 --enable-libsvtav1 --disable-ffplay
+
+RUN make -j 8
+
+FROM public.ecr.aws/lambda/python:3.12
+
+#COPY --from=builder /gifski/target/release/gifski /usr/local/bin/gifski
+#COPY --from=ffmpeg /ffmpeg/ffmpeg-*-static/ffmpeg /usr/local/bin/ffmpeg
+#COPY --from=ffmpeg /ffmpeg/ffmpeg-*-static/ffprobe /usr/local/bin/ffprobe
+COPY --from=ffmpeg /ffmpeg_sources/ffmpeg /usr/local/bin/ffmpeg
+COPY --from=ffmpeg /ffmpeg_sources/ffprobe /usr/local/bin/ffprobe
+
+RUN pip install requests==2.32.3
 
 # Copy function code
 COPY __init__.py ${LAMBDA_TASK_ROOT}/app.py
-COPY conv.sh ${LAMBDA_TASK_ROOT}/conv.sh
 
 # Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
 CMD [ "app.lambda_handler" ] 

gifConvert/__init__.py

@@ -1,100 +1,144 @@
 import base64
 import os
 import subprocess
-import json
-import sys
 import tempfile
+import urllib.request
+import re
+import botocore
+import boto3
+import requests
+
+useBucket=False
+bucketname=os.getenv('CF_BUCKET')
+s3=None
+if bucketname is None:
+    useBucket=False
+else:
+    useBucket=True
+    s3 = boto3.client('s3',endpoint_url=os.getenv('CF_ENDPOINT'),aws_access_key_id=os.getenv('CF_KEY'),aws_secret_access_key=os.getenv('CF_KEY_SECRET'))
+
 
 def extractStatus(url):
     return ""
 
-def get_video_frame_rate(filename):
-    result = subprocess.run(
-        [
-            "./ffprobe",
-            "-v",
-            "error",
-            "-select_streams",
-            "v",
-            "-of",
-            "default=noprint_wrappers=1:nokey=1",
-            "-show_entries",
-            "stream=r_frame_rate",
-            filename,
-        ],
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-    )
-    result_string = result.stdout.decode('utf-8').split()[0].split('/')
-    fps = float(result_string[0])/float(result_string[1])
-    return fps
+def convert_video_to_gif(filename):
+    try:
+        new_filename = tempfile.mkstemp(suffix=".gif")[1]
+        print("converting gif w gifski")
+        p_ffmpeg = subprocess.Popen(["ffmpeg", "-i", filename, "-f", "yuv4mpegpipe", "-"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+        p_gifski = subprocess.Popen(["gifski","--quality","70","--lossy-quality","30","-o", new_filename, "-"], stdin=p_ffmpeg.stdout, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
 
-def get_video_length_seconds(filename):
-    result = subprocess.run(
-        [
-            "./ffprobe",
-            "-v",
-            "error",
-            "-show_entries",
-            "format=duration",
-            "-of",
-            "default=noprint_wrappers=1:nokey=1",
-            filename,
-        ],
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-    )
-    result_string = result.stdout.decode('utf-8').split()[0]
-    return float(result_string)
-
-def calcEdits(vlen,loopTimes):
-    st="r"
-    for i in range(loopTimes):
-        st+=f'e{str((vlen*i))}-9999,0'
-    return st
-
-def loop_video_until_length(filename, length):
-    # use stream_loop to loop video until it's at least length seconds long
-    video_length = get_video_length_seconds(filename)
-    if video_length < length:
-        loops = int(length/video_length)
-        new_filename = tempfile.mkstemp(suffix=".mp4")[1]
-        #edits = calcEdits(video_length,loops)
-        out = subprocess.call(["ffmpeg","-stream_loop",str(loops),"-i",filename,"-c","copy","-y",new_filename],stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT)
-        #subprocess.run(["./MP4Box", "-add",filename,"-edits",f'1={edits}',new_filename])
-        return new_filename
-    else:
+        p_ffmpeg.stdout.close()
+        ret_gifski = p_gifski.wait()
+        ret_ffmpeg = p_ffmpeg.wait()
+        if ret_gifski != 0:
+            print("err: gifski exited with code:", ret_gifski)
+        if ret_ffmpeg != 0:
+            print("err: ffmpeg exited with code:", ret_ffmpeg)
+        if os.path.isfile(new_filename) and os.path.getsize(new_filename) > 0:
+            return new_filename
+        else:
+            print("gifski failed to convert gif")
+            return filename
+    except Exception as e:
+        print("error converting gif (convert_video_to_gif):")
+        print(e)
+        return filename
+    
+def convert_video_to_avif(filename):
+    try:
+        fd,new_filename = tempfile.mkstemp(suffix=".avif")
+        os.close(fd)
+        print("converting gif w ffmpeg to avif")
+        subprocess.call(["ffmpeg","-nostdin","-y", "-i", filename,"-pix_fmt","yuv420p","-an","-c:v","libsvtav1","-crf","30","-b:v","0","-threads","4", new_filename], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
+        if os.path.isfile(new_filename) and os.path.getsize(new_filename) > 0:
+            return new_filename
+        else:
+            print("ffmpeg failed to convert avif")
+            return filename
+    except Exception as e:
+        print("error converting avif (convert_video_to_avif):")
+        print(e)
         return filename
 
-
+def redir(url):
+    return {
+        "statusCode": 307,
+        "headers": {
+            "Location": url
+        }
+    }
 
 def lambda_handler(event, context):
     if ("queryStringParameters" not in event):
         return {
             "statusCode": 400,
-            "body": "Invalid request."
+            "body": "Invalid request!"
         }
     
     url = event["queryStringParameters"].get("url","")
-
-    # download video
-    videoLocation = tempfile.mkstemp(suffix=".mp4")[1]
-    subprocess.call(["wget","-O",videoLocation,url],stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT)
-
-    videoLocationLooped = loop_video_until_length(videoLocation, 30)
-    if videoLocationLooped != videoLocation:
-        os.remove(videoLocation)
-        videoLocation = videoLocationLooped
-
-    with open(videoLocation, "rb") as image_file:
-        encoded_string = base64.b64encode(image_file.read()).decode('ascii')
-    os.remove(videoLocation)
-    return {
-        'statusCode': 200,
-        "headers": 
-        {
-            "Content-Type": "video/mp4"
-        },
-        'body': encoded_string,
-        'isBase64Encoded': True
-    }
+    try:
+        if url == "":
+            return {
+                "statusCode": 400,
+                "body": "Invalid request!!"
+            }
+        if not url.startswith("https://video.twimg.com/tweet_video/"):
+            return redir(url)
+    
+        if useBucket:
+            id=re.search(r"https:\/\/video\.twimg\.com\/tweet_video\/(.*?)\..*",url).group(1)
+            bfilename = str(id)+".avif"
+            furl=f"https://gifs.vxtwitter.com/{bfilename}" #f"https://{bucketname}.s3.amazonaws.com/{bfilename}"
+            print("get req for: "+url)
+            try:
+                s3.head_object(Bucket=bucketname, Key=bfilename)
+                print("found existing already: "+bfilename)
+                return redir(furl)
+            except botocore.exceptions.ClientError:
+                # Not found
+                pass
+        # download video
+        print("downloading: "+url)
+        videoLocation = tempfile.mkstemp(suffix=".mp4")[1]
+        response = requests.get(url, stream=True)
+        if response.status_code == 200:
+            with open(videoLocation, 'wb') as f:
+                for chunk in response.iter_content(chunk_size=8192):
+                    f.write(chunk)
+        else:
+            print("error downloading video")
+            return redir(url)
+    
+        videoLocationLooped = convert_video_to_avif(videoLocation)
+        if videoLocationLooped != videoLocation:
+            os.remove(videoLocation)
+            videoLocation = videoLocationLooped
+        else:
+            os.remove(videoLocation)
+            return redir(url)
+        
+        if not useBucket:
+            with open(videoLocation, "rb") as image_file:
+                encoded_string = base64.b64encode(image_file.read()).decode('ascii')
+                os.remove(videoLocation)
+                return {
+                    'statusCode': 200,
+                    "headers": 
+                    {
+                        "Content-Type": "image/avif",
+                        "Cache-Control": "public, max-age=604800, immutable"
+                    },
+                    'body': encoded_string,
+                    'isBase64Encoded': True
+                }
+        else:
+            with open(videoLocation, "rb") as image_file:
+                s3.upload_fileobj(image_file, bucketname, bfilename)
+            os.remove(videoLocation)
+            print("converted: "+url+" -> "+furl)
+            return redir(furl)
+    except Exception as e:
+        print("error converting gif: ")
+        print(e)
+        return redir(url)

gifConvert/conv.sh

@@ -1,64 +0,0 @@
-#!/bin/bash -e
-
-usage(){
-    echo "Usage: $0 [options] output"
-    echo "Options:"
-    echo "  --help              Show this help"
-    echo "  -u, --url           URL of the video"
-    echo "  -w, --max-width     Maximum width of the output"
-    echo "  -h, --max-height    Maximum height of the output"
-    echo "  -t, --threads       Number of threads to use"
-    exit 1
-}
-
-URL=""
-MAXW=400
-MAXH=267
-THREADS=1
-OUTPUT="out.gif"
-FPS=10
-
-while [ $# -gt 0 ]; do
-    case "$1" in
-        --help)
-            usage
-            ;;
-        -u|--url)
-            URL="$2"
-            shift
-            ;;
-        -w|--max-width)
-            MAXW="$2"
-            shift
-            ;;
-        -h|--max-height)
-            MAXH="$2"
-            shift
-            ;;
-        -t|--threads)
-            THREADS="$2"
-            shift
-            ;;
-        -f|--fps)
-            FPS="$2"
-            shift
-            ;;
-        -*)
-            echo "Unknown option: $1"
-            usage
-            ;;
-        *)
-            OUTPUT="$1"
-            ;;
-    esac
-    shift
-done
-
-# make unique temp directory
-TEMPDIR=$( mktemp -d )
-
-./ffmpeg -i "$URL" -vf "scale=if(gte(iw\,ih)\,min($MAXW\,iw)\,-2):if(lt(iw\,ih)\,min($MAXH\,ih)\,-2)" -threads $THREADS "$TEMPDIR/frame%04d.png"
-./gifski -o "$TEMPDIR/out.gif" --fast --fps $FPS --quality=90 $TEMPDIR/frame*.png
-#./gifsicle -O3 "$TEMPDIR/out.gif" -o "$OUTPUT"
-mv "$TEMPDIR/out.gif" "$OUTPUT"
-rm -rf "$TEMPDIR"

msgs.py

@@ -17,8 +17,11 @@ def genLikesDisplay(vnf):
         return ("💖 " + numerize.numerize(vnf['likes']))
 
 def genQrtDisplay(qrt):
+    text = qrt['text']
+    if "translation" in qrt and qrt["translation"] is not None:
+        text = qrt["translation"]["text"]
     verifiedCheck = "☑️" if ('verified' in qrt and qrt['verified']) else ""
-    return ("\n\n【QRT of " + qrt['user_name'] + " (@" + qrt['user_screen_name'] + ")"+ verifiedCheck+":】\n\n'" + qrt['text'] + "'")
+    return ("\n\n【QRT of " + qrt['user_name'] + " (@" + qrt['user_screen_name'] + ")"+ verifiedCheck+":】\n\n'" + text + "'")
 
 def genPollDisplay(poll):
     pctSplit=10
@@ -27,6 +30,9 @@ def genPollDisplay(poll):
         output+=choice["name"]+"\n"+("█"*int(choice["percent"]/pctSplit)) +" "+str(choice["percent"])+"%\n"
     return output
 
+def genTranslationDisplay(translation):
+    return f"【TL "+translation["source_language"]+"→"+translation["destination_language"]+"】\n\n"+translation["text"]
+
 # formats the top text of the embed
 def formatProvider(base,vnf):
     finalText = base
@@ -38,13 +44,16 @@ def formatProvider(base,vnf):
         finalText = base
     return finalText
 
-def formatEmbedDesc(type,body,qrt,pollData):
+def formatEmbedDesc(type,body,qrt,pollData,translation):
     # Trim the embed description to 248 characters, prioritizing poll and likes
 
     qrtType=None
     if qrt!=None:
         qrtType="Text"
 
+    if translation is not None:
+        body = genTranslationDisplay(translation)
+
     limit = videoDescLimit if type=="Video" or (qrt!=None and (qrtType=="Video")) else tweetDescLimit
 
     output = ""
@@ -73,6 +82,6 @@ def formatEmbedDesc(type,body,qrt,pollData):
         diff = len(output)-limit
         # remove the characters from body, add ellipsis
         body = body[:-(diff+1)]+"…"
-        return formatEmbedDesc(type,body,qrt,pollData)
+        return formatEmbedDesc(type,body,qrt,pollData,None)
     else:
         return output

requirements.txt

@@ -8,4 +8,5 @@ Werkzeug==2.3.7
 numerize==0.12
 oauthlib==3.2.2
 PyRTF3==0.47.5
-XClientTransaction==0.0.2
+XClientTransaction==1.0.2
+deepl==1.3.0

serverless.yml

@@ -29,6 +29,7 @@ provider:
     VXTWITTER_COMBINATION_METHOD: ${env:VXTWITTER_COMBINATION_METHOD, 'local'}
     VXTWITTER_GIF_CONVERT_API: ${env:VXTWITTER_GIF_CONVERT_API, ''}
     VXTWITTER_WORKAROUND_TOKENS: ${env:VXTWITTER_WORKAROUND_TOKENS, ''}
+    DEEPL_API_KEY: ${env:DEEPL_API_KEY, ''}
     #VXTWITTER_PROXIES: ${env:VXTWITTER_PROXIES, ''}
 
 package:

test_vx_embeds.py

@@ -204,4 +204,20 @@ def test_embed_action():
     resp = client.get(testTextTweet.replace("https://twitter.com",""),headers={"User-Agent":"Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)"})
     assert resp.status_code==200
     assert "application/activity+json" in str(resp.data)
-    assert "%F0%9F%92%96" not in str(resp.data) # 💖
+    assert "%F0%9F%92%96" not in str(resp.data) # 💖
+
+def test_embed_translated():
+    resp = client.get(testTextTweet.replace("https://twitter.com","")+"/jp",headers={"User-Agent":"test"})
+    assert resp.status_code==200
+    assert "→" in resp.text
+    resp = client.get(testTextTweet.replace("https://twitter.com","")+"/ja",headers={"User-Agent":"test"})
+    assert resp.status_code==200
+    assert "→" in resp.text
+
+def test_embed_translated_subdomain():
+    resp = client.get(testTextTweet.replace("https://twitter.com","https://jp.vxtwitter.com"),headers={"User-Agent":"test"})
+    assert resp.status_code==200
+    assert "→" in resp.text
+    resp = client.get(testTextTweet.replace("https://twitter.com","https://ja.vxtwitter.com"),headers={"User-Agent":"test"})
+    assert resp.status_code==200
+    assert "→" in resp.text

test_vx_extract.py

@@ -20,6 +20,10 @@ def test_twextract_extractStatusV2Rest():
     assert utils.stripEndTCO(tweet["full_text"])[:94]==testMultiMediaTweet_compare['text'][:94]
     
 
+def test_twextract_v2API_Anon():
+    tweet = twExtract.extractStatusV2Rest_Anon(testMediaTweet,workaroundTokens=tokens)['legacy']
+    assert utils.stripEndTCO(tweet["full_text"])==testMediaTweet_compare['text']
+
 def test_twextract_v2API():
     tweet = twExtract.extractStatusV2(testMediaTweet,workaroundTokens=tokens)['legacy']
     assert utils.stripEndTCO(tweet["full_text"])==testMediaTweet_compare['text']

translation.py

@@ -0,0 +1,27 @@
+import deepl
+from configHandler import config
+
+validLanguages = ['AF', 'AN', 'AR', 'AS', 'AY', 'AZ', 'BA', 'BE', 'BG', 'BN', 'BR', 'BS', 'CA', 'CS', 'CY', 'DA', 'DE', 'DE-DE', 'EL', 'EN', 'EO', 'ES', 'ES-419', 'ET', 'EU', 'FA', 'FI', 'FR', 'FR-FR', 'GA', 'GL', 'GN', 'GU', 'HA', 'HE', 'HI', 'HR', 'HT', 'HU', 'HY', 'ID', 'IG', 'IS', 'IT', 'JA', 'JV', 'KA', 'KK', 'KO', 'KY', 'LA', 'LB', 'LN', 'LT', 'LV', 'MG', 'MI', 'MK', 'ML', 'MN', 'MR', 'MS', 'MT', 'MY', 'NB', 'NE', 'NL', 'OC', 'OM', 'PA', 'PL', 'PS', 'PT-BR', 'PT-PT', 'QU', 'RO', 'RU', 'SA', 'SK', 'SL', 'SQ', 'SR', 'ST', 'SU', 'SV', 'SW', 'TA', 'TE', 'TG', 'TH', 'TK', 'TL', 'TN', 'TR', 'TS', 'TT', 'UK', 'UR', 'UZ', 'VI', 'WO', 'XH', 'YI', 'ZH', 'ZH-HANS', 'ZH-HANT', 'ZU']
+languageAliases = {
+    #"EN":"EN-US",
+    "JP":"JA",
+}
+
+mergedLangs=(validLanguages+list(languageAliases.keys()))
+
+def getDeeplTranslation(text,target):
+    target = target.upper()
+    if target == "EN":
+        target = "EN-US"
+    try:
+        if 'deeplKey' not in config["config"] or config["config"]["deeplKey"] == None:
+            return None
+        deepl_client = deepl.Translator(config["config"]["deeplKey"])
+        result = deepl_client.translate_text(text, target_lang=target)
+        return {
+            "text":result.text,
+            "source_language":result.detected_source_lang,
+            "destination_language":target
+        }
+    except Exception as e:
+        return None

twExtract/__init__.py

@@ -14,7 +14,7 @@ bearer="Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DT
 v2bearer="Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
 androidBearer="Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F"
 
-requestUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0"
+requestUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0"
 
 bearerTokens=[bearer,v2bearer,androidBearer]
 
@@ -27,15 +27,15 @@ userIDregex = r"\/i\/user\/(\d+)"
 v2Features='{"longform_notetweets_inline_media_enabled":true,"super_follow_badge_privacy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"super_follow_user_api_enabled":true,"super_follow_tweet_api_enabled":true,"android_graphql_skip_api_media_color_palette":true,"creator_subscriptions_tweet_preview_api_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"creator_subscriptions_subscription_count_enabled":true,"tweetypie_unmention_optimization_enabled":true,"longform_notetweets_consumption_enabled":true,"subscriptions_verification_info_enabled":true,"blue_business_profile_image_shape_enabled":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"super_follow_exclusive_tweet_notifications_enabled":true}'
 v2graphql_api="2OOZWmw8nAtUHVnXXQhgaA"
 
-v2AnonFeatures='{"creator_subscriptions_tweet_preview_api_enabled":true,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":false,"responsive_web_jetfuel_frame":true,"responsive_web_grok_share_attachment_enabled":true,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":false,"responsive_web_grok_analysis_button_from_backend":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"payments_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"verified_phone_label_enabled":false,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_grok_imagine_annotation_enabled":true,"responsive_web_grok_community_note_auto_translation_is_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false}'
-v2AnonGraphql_api="wqi5M7wZ7tW-X9S2t-Mqcg"
+v2AnonFeatures='{"creator_subscriptions_tweet_preview_api_enabled":true,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":true,"responsive_web_grok_share_attachment_enabled":true,"responsive_web_grok_annotations_enabled":false,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":true,"responsive_web_grok_analysis_button_from_backend":true,"post_ctas_fetch_enabled":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"profile_label_improvements_pcf_label_in_post_enabled":true,"responsive_web_profile_redirect_enabled":false,"rweb_tipjar_consumption_enabled":false,"verified_phone_label_enabled":false,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_grok_imagine_annotation_enabled":true,"responsive_web_grok_community_note_auto_translation_is_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false}'
+v2AnonGraphql_api="0aTrQMKgj95K791yXeNDRA"
 gt_pattern = r'document\.cookie="gt=([^;]+);'
 
-androidGraphqlFeatures='{"grok_translations_community_note_translation_is_enabled":false,"super_follow_badge_privacy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"super_follow_user_api_enabled":true,"profile_label_improvements_pcf_label_in_profile_enabled":true,"premium_content_api_read_enabled":false,"grok_translations_community_note_auto_translation_is_enabled":false,"android_graphql_skip_api_media_color_palette":true,"tweetypie_unmention_optimization_enabled":true,"longform_notetweets_consumption_enabled":true,"subscriptions_verification_info_enabled":true,"blue_business_profile_image_shape_enabled":true,"super_follow_exclusive_tweet_notifications_enabled":true,"longform_notetweets_inline_media_enabled":true,"grok_android_analyze_trend_fetch_enabled":false,"unified_cards_ad_metadata_container_dynamic_card_content_query_enabled":true,"super_follow_tweet_api_enabled":true,"articles_api_enabled":true,"creator_subscriptions_tweet_preview_api_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"grok_translations_timeline_user_bio_auto_translation_is_enabled":false,"grok_translations_post_auto_translation_is_enabled":false,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"immersive_video_status_linkable_timestamps":true,"profile_label_improvements_pcf_label_in_post_enabled":true}'
-androidGraphql_api="k3rtLsS9kG5hI-Jr0dTMCg"
+androidGraphqlFeatures='{"grok_translations_community_note_translation_is_enabled":false,"super_follow_badge_privacy_enabled":true,"unified_cards_destination_url_params_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"super_follow_user_api_enabled":true,"profile_label_improvements_pcf_label_in_profile_enabled":true,"premium_content_api_read_enabled":false,"grok_translations_community_note_auto_translation_is_enabled":false,"android_graphql_skip_api_media_color_palette":true,"tweetypie_unmention_optimization_enabled":true,"longform_notetweets_consumption_enabled":true,"subscriptions_verification_info_enabled":true,"blue_business_profile_image_shape_enabled":true,"super_follow_exclusive_tweet_notifications_enabled":true,"longform_notetweets_inline_media_enabled":true,"grok_android_analyze_trend_fetch_enabled":false,"unified_cards_ad_metadata_container_dynamic_card_content_query_enabled":true,"super_follow_tweet_api_enabled":true,"articles_api_enabled":true,"android_ad_formats_media_component_render_overlay_enabled":true,"creator_subscriptions_tweet_preview_api_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"grok_translations_timeline_user_bio_auto_translation_is_enabled":false,"grok_translations_post_auto_translation_is_enabled":false,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"immersive_video_status_linkable_timestamps":true,"profile_label_improvements_pcf_label_in_post_enabled":true}'
+androidGraphql_api="Wrspae-uyGj-nWPyUqdUag"
 
-tweetDetailGraphqlFeatures='{"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"articles_preview_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"rweb_video_timestamps_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_enhance_cards_enabled":false}'
-tweetDetailGraphql_api="YVyS4SfwYW7Uw5qwy0mQCA"
+tweetDetailGraphqlFeatures='{"rweb_video_screen_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"responsive_web_profile_redirect_enabled":false,"rweb_tipjar_consumption_enabled":false,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":true,"responsive_web_grok_share_attachment_enabled":true,"responsive_web_grok_annotations_enabled":false,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":true,"responsive_web_grok_analysis_button_from_backend":true,"post_ctas_fetch_enabled":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_grok_imagine_annotation_enabled":true,"responsive_web_grok_community_note_auto_translation_is_enabled":false,"responsive_web_enhance_cards_enabled":false}'
+tweetDetailGraphql_api="Kzfv17rukSzjT96BerOWZA"
 
 # this is for UserTweets endpoint
 tweetFeedGraphqlFeatures='{"rweb_video_screen_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":false,"responsive_web_grok_share_attachment_enabled":true,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":false,"responsive_web_grok_analysis_button_from_backend":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_enhance_cards_enabled":false}'
@@ -113,7 +113,7 @@ def cycleBearerTokenGet(url,headers):
         return tweet
     raise TwExtractError(400, "Extract error")
 
-def twitterApiGet(url,btoken=None,authToken=None,guestToken=None):
+def twitterApiGet(url,btoken=None,authToken=None,guestToken=None,userAgent=None,language=None):
 
     if authToken != None and authToken.startswith("oa|"):
         url = url.replace("https://x.com/i/api/graphql/","https://api.twitter.com/graphql/")
@@ -122,9 +122,11 @@ def twitterApiGet(url,btoken=None,authToken=None,guestToken=None):
         secret = authToken.split("|")[1]
 
         twt = oauth1.Client(client_key='3nVuSoBZnx6U4vzUxf5w',client_secret='Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys',resource_owner_key=key,resource_owner_secret=secret)
-        hdr = getAuthHeaders(androidBearer)
+        hdr = getAuthHeaders(androidBearer,language=language)
         del hdr["Authorization"]
         hdr["X-Twitter-Client"] = "TwitterAndroid"
+        if userAgent is not None:
+            hdr["User-Agent"] = userAgent
 
         uri, headers, body = twt.sign(url, headers=hdr,realm="http://api.twitter.com/")
 
@@ -133,12 +135,13 @@ def twitterApiGet(url,btoken=None,authToken=None,guestToken=None):
         if btoken is None:
             btoken = v2bearer
             #return cycleBearerTokenGet(url,getAuthHeaders(bearer,authToken=authToken,guestToken=guestToken))
-        headers = getAuthHeaders(btoken,authToken=authToken,guestToken=guestToken)
+        headers = getAuthHeaders(btoken,authToken=authToken,guestToken=guestToken,language=language)
         response = requests.get(url, headers=headers)
 
     return response
 
-def getAuthHeaders(btoken,authToken=None,guestToken=None):
+def getAuthHeaders(btoken,authToken=None,guestToken=None,language=None):
+
     csrfToken=str(uuid.uuid4()).replace('-', '')
     headers = {"x-twitter-active-user":"yes","x-twitter-client-language":"en","x-csrf-token":csrfToken,"User-Agent":requestUserAgent}
     headers['Authorization'] = btoken
@@ -149,19 +152,17 @@ def getAuthHeaders(btoken,authToken=None,guestToken=None):
     if guestToken is not None:
         headers["x-guest-token"] = guestToken
 
+    if language is not None:
+        headers["x-twitter-client-language"] = language
+
     return headers
 
 def getGuestToken():
     global guestToken
     global guestTokenUses
     if guestToken is None:
-        r = requests.get(f"https://{twitterUrl}",headers={"User-Agent":requestUserAgent,"Cookie":"night_mode=2"},allow_redirects=False)
-        m = re.search(gt_pattern, r.text)
-        if m is None:
-            r = requests.post(f"https://api.{twitterUrl}/1.1/guest/activate.json", headers={"Authorization":bearer})
-            guestToken = json.loads(r.text)["guest_token"]
-        else:
-            guestToken = m.group(1)
+        r = requests.post(f"https://api.{twitterUrl}/1.1/guest/activate.json", headers={"Authorization":v2bearer,"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0"})
+        guestToken = json.loads(r.text)["guest_token"]
         guestTokenUses = 0
     else:
         guestTokenUses+=1
@@ -213,7 +214,7 @@ def extractStatus_guestToken(url):
         raise TwExtractError(error["code"], error["message"])
     return output
 
-def extractStatus_syndication(url,workaroundTokens=None):
+def extractStatus_syndication(url,workaroundTokens=None,tlLanguage=None):
     # https://github.com/mikf/gallery-dl/blob/46cae04aa3a113c7b6bbee1bb468669564b14ae8/gallery_dl/extractor/twitter.py#L1784
     m = re.search(pathregex, url)
     if m is None:
@@ -265,7 +266,7 @@ def extractStatus_twExtractProxy(url,workaroundTokens=None):
             continue
         return output
 
-def extractStatusV2(url,workaroundTokens):
+def extractStatusV2(url,workaroundTokens,tlLanguage=None):
     # get tweet ID
     m = re.search(pathregex, url)
     if m is None:
@@ -279,7 +280,7 @@ def extractStatusV2(url,workaroundTokens):
     def request_with_token(twid, authToken):
         vars = json.loads('{"includeTweetImpression":true,"includeHasBirdwatchNotes":false,"includeEditPerspective":false,"rest_ids":["x"],"includeEditControl":true,"includeCommunityTweetRelationship":true,"includeTweetVisibilityNudge":true}')
         vars['rest_ids'][0] = str(twid)
-        tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2graphql_api}/TweetResultsByIdsQuery?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2Features)}",authToken=authToken)
+        tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2graphql_api}/TweetResultsByIdsQuery?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2Features)}",authToken=authToken,language=tlLanguage)
         try:
             rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining")
             print(f"Twitter Token Rate limit remaining: {rateLimitRemaining}")
@@ -318,7 +319,7 @@ def extractStatusV2(url,workaroundTokens):
         return tweet
     return parallel_token_request(twid, tokens, request_with_token)
 
-def extractStatusV2Android(url,workaroundTokens):
+def extractStatusV2Android(url,workaroundTokens,tlLanguage=None):
     # get tweet ID
     m = re.search(pathregex, url)
     if m is None:
@@ -332,7 +333,7 @@ def extractStatusV2Android(url,workaroundTokens):
         try:
             vars = json.loads('{"referrer":"home","includeTweetImpression":true,"includeHasBirdwatchNotes":false,"isReaderMode":false,"includeEditPerspective":false,"includeEditControl":true,"focalTweetId":0,"includeCommunityTweetRelationship":true,"includeTweetVisibilityNudge":true}')
             vars['focalTweetId'] = int(twid)
-            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{androidGraphql_api}/ConversationTimelineV2?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(androidGraphqlFeatures)}", authToken=authToken,btoken=androidBearer)
+            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{androidGraphql_api}/ConversationTimelineV2?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(androidGraphqlFeatures)}", authToken=authToken,btoken=androidBearer,userAgent="TwitterAndroid/11.61.0-release.0 (311610000-r-0) G011A/9 (google;G011A;google;G011A;0;;1;2016)",language=tlLanguage)
             try:
                 rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining")
                 print(f"Twitter Android Token Rate limit remaining: {rateLimitRemaining}")
@@ -377,7 +378,7 @@ def extractStatusV2Android(url,workaroundTokens):
         return tweet
     return parallel_token_request(twid, tokens, request_with_token)
 
-def extractStatusV2TweetDetail(url,workaroundTokens):
+def extractStatusV2TweetDetail(url,workaroundTokens,tlLanguage=None):
     # get tweet ID
     m = re.search(pathregex, url)
     if m is None:
@@ -392,7 +393,7 @@ def extractStatusV2TweetDetail(url,workaroundTokens):
         try:
             vars = json.loads('{"focalTweetId":"0","with_rux_injections":false,"includePromotedContent":true,"withCommunity":true,"withQuickPromoteEligibilityTweetFields":true,"withBirdwatchNotes":true,"withVoice":true,"withV2Timeline":true}')
             vars['focalTweetId'] = str(twid)
-            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{tweetDetailGraphql_api}/TweetDetail?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(tweetDetailGraphqlFeatures)}", authToken=authToken,btoken=v2bearer)
+            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{tweetDetailGraphql_api}/TweetDetail?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(tweetDetailGraphqlFeatures)}", authToken=authToken,btoken=v2bearer,language=tlLanguage)
             try:
                 rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining")
                 print(f"Twitter Token Rate limit remaining: {rateLimitRemaining}")
@@ -437,10 +438,10 @@ def extractStatusV2TweetDetail(url,workaroundTokens):
         return tweet
     return parallel_token_request(twid, tokens, request_with_token)
 
-def extractStatusV2Rest_Anon(url,workaroundTokens):
-    return extractStatusV2Rest(url,None)
+def extractStatusV2Rest_Anon(url,workaroundTokens,tlLanguage=None):
+    return extractStatusV2Rest(url,None,tlLanguage=tlLanguage)
 
-def extractStatusV2Rest(url,workaroundTokens):
+def extractStatusV2Rest(url,workaroundTokens,tlLanguage=None):
     # get tweet ID
     m = re.search(pathregex, url)
     if m is None:
@@ -451,18 +452,18 @@ def extractStatusV2Rest(url,workaroundTokens):
 
     # get tweet
     try:
-        vars = json.loads('{"tweetId":"0","withCommunity":false,"includePromotedContent":false,"withVoice":false}')
+        vars = json.loads('{"tweetId":"0","includePromotedContent":true,"withBirdwatchNotes":true,"withVoice":true,"withCommunity":true}')
         vars['tweetId'] = str(twid)
         if workaroundTokens is not None and len(workaroundTokens) > 0:
             tokens = workaroundTokens
             random.shuffle(tokens)
             for authToken in tokens:
                 try:
-                    tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", btoken=v2bearer,authToken=authToken,guestToken=guestToken)
+                    tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", btoken=v2bearer,authToken=authToken,guestToken=guestToken,language=tlLanguage)
                 except Exception as e:
                     continue
         else:
-            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", btoken=v2bearer,guestToken=guestToken)
+            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", btoken=v2bearer,guestToken=guestToken,language=tlLanguage)
         
         try:
             rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining")
@@ -509,12 +510,13 @@ def fixTweetData(tweet):
         pass
     return tweet
 
-def extractStatus(url,workaroundTokens=None):
-    # TODO: commented out methods are too slow/unreliable at the moment
+def extractStatus(url,workaroundTokens=None,tlLanguage=None):
     methods=[extractStatusV2Rest_Anon,extractStatusV2,extractStatusV2Rest,extractStatusV2Android]#,extractStatusV2TweetDetail]
+    if tlLanguage is not None: # prioritize endpoints that return translations
+        methods = [extractStatusV2Rest,extractStatusV2TweetDetail,extractStatusV2Android,extractStatusV2,extractStatusV2Rest_Anon]
     for method in methods:
         try:
-            result = method(url,workaroundTokens)
+            result = method(url,workaroundTokens,tlLanguage=tlLanguage)
             if 'legacy' not in result:
                 print(f"{method.__name__} method failed: Legacy not found for {url}")
                 # try another method

twExtract/twUtils.py

@@ -3,8 +3,9 @@ import hashlib
 import base64
 import uuid
 from x_client_transaction import ClientTransaction
-from x_client_transaction.utils import handle_x_migration
+from x_client_transaction.utils import get_ondemand_file_url, generate_headers
 import requests
+import bs4
 digits = "0123456789abcdefghijklmnopqrstuvwxyz"
 
 def baseConversion(x, base):
@@ -49,6 +50,14 @@ def get_twitter_homepage(headers=None):
     return response
 
 def generate_transaction_id(method: str, path: str,headers=None) -> str:
-    ct = ClientTransaction(get_twitter_homepage(headers=headers))
+    session = requests.Session()
+    session.headers = generate_headers()
+    home_page = session.get(url="https://x.com/home")
+    home_page_response = bs4.BeautifulSoup(home_page.content, 'html.parser')
+    ondemand_file_url = get_ondemand_file_url(response=home_page_response)
+    ondemand_file = session.get(url=ondemand_file_url)
+    ondemand_file_response = bs4.BeautifulSoup(ondemand_file.content, 'html.parser')
+
+    ct = ClientTransaction(home_page_response=home_page_response, ondemand_file_response=ondemand_file_response)
     transaction_id = ct.generate_transaction_id(method=method, path=path)
     return transaction_id

twitfix.py

@@ -14,7 +14,7 @@ import msgs
 import twExtract as twExtract
 from cache import addVnfToLinkCache,getVnfFromLinkCache
 import vxlogging as log
-from utils import getTweetIdFromUrl, pathregex, determineMediaToEmbed, determineEmbedTweet, BytesIOWrapper, fixMedia
+from utils import getTweetIdFromUrl, pathregex, determineMediaToEmbed, determineEmbedTweet, BytesIOWrapper, fixMedia, indexOfAny
 from vxApi import getApiResponse, getApiUserResponse
 from urllib.parse import urlparse 
 from PyRTF.Elements import Document
@@ -22,12 +22,14 @@ from PyRTF.document.section import Section
 from PyRTF.document.paragraph import Paragraph
 from copy import deepcopy
 import json
-import datetime
+import translation
 import activity as activitymg
 app = Flask(__name__)
 CORS(app)
 user_agent=""
 
+activityseparator="zqj"
+
 staticFiles = { # TODO: Use flask static files instead of this
     "favicon.ico": {"mime": "image/vnd.microsoft.icon","path": "favicon.ico"},
     "apple-touch-icon.png": {"mime": "image/png","path": "apple-touch-icon.png"},
@@ -82,7 +84,11 @@ def generateActivityLink(tweetData,media=None,mediatype=None,embedIndex=-1):
         return None
     try:
         embedIndex = embedIndex+1
-        return f"{config['config']['url']}/users/{tweetData['user_screen_name']}/statuses/{str(embedIndex)}{tweetData['tweetID']}"
+
+        extras=""
+        if tweetData["translation"] is not None:
+            extras+=activityseparator+tweetData["translation"]["destination_language"]
+        return f"{config['config']['url']}/users/{tweetData['user_screen_name']}/statuses/{str(embedIndex)}{activityseparator}{tweetData['tweetID']}{extras}"
     except Exception as e:
         log.error("Error generating activity link: "+str(e))
         return None
@@ -95,7 +101,7 @@ def getAppName(tweetData,appnameSuffix=""):
 
 def renderImageTweetEmbed(tweetData,image,appnameSuffix="",embedIndex=-1):
     qrt = tweetData['qrt']
-    embedDesc = msgs.formatEmbedDesc("Image",tweetData['text'],qrt,tweetData['pollData'])
+    embedDesc = msgs.formatEmbedDesc("Image",tweetData['text'],qrt,tweetData['pollData'],tweetData["translation"])
 
     if image.startswith("https://pbs.twimg.com") and "?" not in image:
         image = f"{image}?name=orig"
@@ -115,7 +121,7 @@ def renderImageTweetEmbed(tweetData,image,appnameSuffix="",embedIndex=-1):
 
 def renderVideoTweetEmbed(tweetData,mediaInfo,appnameSuffix="",embedIndex=-1):
     qrt = tweetData['qrt']
-    embedDesc = msgs.formatEmbedDesc("Video",tweetData['text'],qrt,tweetData['pollData'])
+    embedDesc = msgs.formatEmbedDesc("Video",tweetData['text'],qrt,tweetData['pollData'],tweetData["translation"])
 
     mediaInfo=fixMedia(mediaInfo)
 
@@ -138,7 +144,7 @@ def renderVideoTweetEmbed(tweetData,mediaInfo,appnameSuffix="",embedIndex=-1):
 
 def renderTextTweetEmbed(tweetData,appnameSuffix=""):
     qrt = tweetData['qrt']
-    embedDesc = msgs.formatEmbedDesc("Text",tweetData['text'],qrt,tweetData['pollData'])
+    embedDesc = msgs.formatEmbedDesc("Text",tweetData['text'],qrt,tweetData['pollData'],tweetData["translation"])
 
     return render_template("text.html",
                     tweet=tweetData,
@@ -154,7 +160,7 @@ def renderTextTweetEmbed(tweetData,appnameSuffix=""):
 
 def renderArticleTweetEmbed(tweetData,appnameSuffix=""):
     articlePreview=tweetData['article']["title"]+"\n\n"+tweetData['article']["preview_text"]+"…"
-    embedDesc = msgs.formatEmbedDesc("Image",articlePreview,None,None)
+    embedDesc = msgs.formatEmbedDesc("Image",articlePreview,None,None,None)
 
     return render_template("image.html",
                     tweet=tweetData,
@@ -260,9 +266,9 @@ def userJson():
         },
     }
 
-def getTweetData(twitter_url,include_txt="false",include_rtf="false"):
+def getTweetData(twitter_url,include_txt="false",include_rtf="false",tlLanguage=None):
     cachedVNF = getVnfFromLinkCache(twitter_url)
-    if cachedVNF is not None and include_txt == "false" and include_rtf == "false":
+    if cachedVNF is not None and include_txt == "false" and include_rtf == "false" and tlLanguage is None:
         return cachedVNF
 
     try:
@@ -270,7 +276,7 @@ def getTweetData(twitter_url,include_txt="false",include_rtf="false"):
             workaroundTokens = config['config']['workaroundTokens'].split(",")
         else:
             workaroundTokens = None
-        rawTweetData = twExtract.extractStatus(twitter_url,workaroundTokens=workaroundTokens)
+        rawTweetData = twExtract.extractStatus(twitter_url,workaroundTokens=workaroundTokens,tlLanguage=tlLanguage)
     except:
         rawTweetData = None
     if rawTweetData == None or 'error' in rawTweetData:
@@ -281,7 +287,15 @@ def getTweetData(twitter_url,include_txt="false",include_rtf="false"):
     tweetData = getApiResponse(rawTweetData,include_txt,include_rtf)
     if tweetData is None:
         return None
-    if include_txt == "false" and include_rtf == "false":
+    
+    if tweetData["translation"] is None and tlLanguage is not None and tlLanguage != tweetData["lang"]:
+        # translation probably failed on twitter's end
+        tweetData["translation"] = translation.getDeeplTranslation(tweetData["text"],tlLanguage)
+
+    if tweetData["translation"] is not None and tlLanguage is None:
+        tweetData["translation"] = None # didn't ask for TL
+
+    if include_txt == "false" and include_rtf == "false" and tlLanguage is None:
         addVnfToLinkCache(twitter_url,tweetData)
     return tweetData
 
@@ -307,13 +321,19 @@ def getUserData(twitter_url,includeFeed=False):
 
 @app.route('/<path:sub_path>') # Default endpoint used by everything
 def twitfix(sub_path):
+    splitPath=sub_path.split("/")
+    if "|" in splitPath[-1]: # discord doesn't strip this when a link is spoilered
+        splitPath[-1] = splitPath[-1].replace("|","")
+
+    subdomain = request.host.rsplit(".",2)[0]
+
     global user_agent
     user_agent = request.headers.get('User-Agent', None)
     if user_agent is None:
         user_agent = "unknown"
 
-    isApiRequest=request.url.startswith("https://api.vx") or request.url.startswith("http://api.vx")
-    if not isApiRequest and (request.url.startswith("https://l.vx") or request.url.startswith("https://old.vx")) and "Discord" in user_agent:
+    isApiRequest=subdomain=="api"
+    if not isApiRequest and (subdomain=="l" or subdomain=="old") and "Discord" in user_agent:
         user_agent = user_agent.replace("Discord","LegacyEmbed") # TODO: Clean up; This is a hacky fix to make the new activity embed not trigger
     if sub_path in staticFiles:
         if 'path' not in staticFiles[sub_path] or staticFiles[sub_path]["path"] == None:
@@ -355,24 +375,38 @@ def twitfix(sub_path):
     include_txt="false"
     include_rtf="false"
 
+    
+    translationIndex=indexOfAny(splitPath,translation.mergedLangs,caseInsensitive=True)
+    translationLang = None
+    if translationIndex != -1:
+        if splitPath[translationIndex].upper() in translation.languageAliases:
+            translationLang = translation.languageAliases[splitPath[translationIndex].upper()]
+        else:
+            translationLang = splitPath[translationIndex].lower()
+    elif subdomain.upper() in translation.mergedLangs:
+        if subdomain.upper() in translation.languageAliases:
+            translationLang = translation.languageAliases[subdomain.upper()]
+        else:
+            translationLang = subdomain.lower()
+
     if isApiRequest:
         if "include_txt" in request.args:
             include_txt = request.args.get("include_txt")
         if "include_rtf" in request.args:
             include_rtf = request.args.get("include_rtf")
 
-    tweetData = getTweetData(twitter_url,include_txt,include_rtf)
+    tweetData = getTweetData(twitter_url,include_txt,include_rtf,tlLanguage=translationLang)
     if tweetData is None:
         log.error("Tweet Data Get failed for "+twitter_url)
         return message(msgs.failedToScan)
     qrt = None
     if 'qrtURL' in tweetData and tweetData['qrtURL'] is not None:
-        qrt = getTweetData(tweetData['qrtURL'])
+        qrt = getTweetData(tweetData['qrtURL'],tlLanguage=translationLang)
     tweetData['qrt'] = qrt
 
     retweet = None
     if 'retweetURL' in tweetData and tweetData['retweetURL'] is not None:
-        retweet = getTweetData(tweetData['retweetURL'])
+        retweet = getTweetData(tweetData['retweetURL'],tlLanguage=translationLang)
     tweetData['retweet'] = retweet
 
     tweetData = deepcopy(tweetData)
@@ -383,7 +417,7 @@ def twitfix(sub_path):
         requestUrlWithoutQuery = request.url
 
     directEmbed=False
-    if requestUrlWithoutQuery.startswith("https://d.vx") or requestUrlWithoutQuery.endswith(".mp4") or requestUrlWithoutQuery.endswith(".png"):
+    if subdomain=="d" or requestUrlWithoutQuery.endswith(".mp4") or requestUrlWithoutQuery.endswith(".png"):
         directEmbed = True
         # remove the .mp4 from the end of the URL
         if requestUrlWithoutQuery.endswith(".mp4") or requestUrlWithoutQuery.endswith(".png"):
@@ -403,9 +437,10 @@ def twitfix(sub_path):
         return send_file(BytesIOWrapper(rtf), mimetype='application/rtf', as_attachment=True, download_name=f'{tweetData["user_screen_name"]}_{tweetData["tweetID"]}.rtf')
 
     embedIndex = -1
-    # if url ends with /1, /2, /3, or /4, we'll use that as the index
-    if sub_path[-2:] in ["/1","/2","/3","/4"]:
-        embedIndex = int(sub_path[-1])-1
+    # if url has /1, /2, /3, or /4, we'll use that as the index
+    numIdx=indexOfAny(splitPath,["1","2","3","4"]);
+    if numIdx != -1:
+        embedIndex = int(splitPath[numIdx])-1
         sub_path = sub_path[:-2]
         
     if isApiRequest: # Directly return the API response if the request is from the API
@@ -433,14 +468,20 @@ def twitfix(sub_path):
         elif not embeddingMedia:
             return Response(renderTextTweetEmbed(tweetData),headers={"Cache-Tag": "embed"})
         else:
-            media = determineMediaToEmbed(embedTweetData,embedIndex)
+            tryGifConversion = "Discord" in user_agent
+            media = determineMediaToEmbed(embedTweetData,embedIndex,convertGif=tryGifConversion)
             suffix=""
             if "suffix" in media:
                 suffix = media["suffix"]
             if media['type'] == "image":
                 return Response(renderImageTweetEmbed(tweetData,media['url'] , appnameSuffix=suffix,embedIndex=embedIndex),headers={"Cache-Tag": "embed"})
-            elif media['type'] == "video" or media['type'] == "gif":
+            elif media['type'] == "video":
                 return Response(renderVideoTweetEmbed(tweetData,media,appnameSuffix=suffix,embedIndex=embedIndex),headers={"Cache-Tag": "embed"})
+            elif media['type'] == "gif":
+                if "originalUrl" in media and media["url"] != media["originalUrl"] and tryGifConversion:
+                    return Response(renderImageTweetEmbed(tweetData,media['url'] , appnameSuffix=suffix,embedIndex=embedIndex),headers={"Cache-Tag": "embed"})
+                else:
+                    return Response(renderVideoTweetEmbed(tweetData,media,appnameSuffix=suffix,embedIndex=embedIndex),headers={"Cache-Tag": "embed"})
 
     return message(msgs.failedToScan)
 
@@ -478,21 +519,29 @@ def rendercombined():
 
 @app.route("/api/v1/statuses/<string:tweet_id>")
 def api_v1_status(tweet_id):
-    embedIndex = int(tweet_id[0])-1
-    tweet_id = int(tweet_id[1:])
+    lang = None
+    if activityseparator not in tweet_id: #old embeds
+        embedIndex = int(tweet_id[0])-1
+        tweet_id = int(tweet_id[1:])
+    else:
+        args = tweet_id.split(activityseparator)
+        embedIndex = int(args[0])-1
+        tweet_id = int(args[1])
+        if len(args)>2:
+            lang=args[2]
     twitter_url=f"https://twitter.com/i/status/{tweet_id}"
-    tweetData = getTweetData(twitter_url)
+    tweetData = getTweetData(twitter_url,tlLanguage=lang)
     if tweetData is None:
         log.error("Tweet Data Get failed for "+twitter_url)
         return message(msgs.failedToScan)
     qrt = None
     if 'qrtURL' in tweetData and tweetData['qrtURL'] is not None:
-        qrt = getTweetData(tweetData['qrtURL'])
+        qrt = getTweetData(tweetData['qrtURL'],tlLanguage=lang)
     tweetData['qrt'] = qrt
 
     retweet = None
     if 'retweetURL' in tweetData and tweetData['retweetURL'] is not None:
-        retweet = getTweetData(tweetData['retweetURL'])
+        retweet = getTweetData(tweetData['retweetURL'],tlLanguage=lang)
     tweetData['retweet'] = retweet
 
     if tweetData is None:

utils.py

@@ -83,9 +83,23 @@ def determineMediaToEmbed(tweetData,embedIndex = -1,convertGif = True):
             return media
         elif media['type'] == "video" or media['type'] == "gif":
             if media['type'] == "gif" and convertGif:
-                if config['config']['gifConvertAPI'] != "" and config['config']['gifConvertAPI'] != "none":
+                if config['config']['gifConvertAPI'] != "" and config['config']['gifConvertAPI'] != "none" and config['config']['gifConvertAPI'] != "local":
+                    gcApi = config['config']['gifConvertAPI']
+                    #if gcApi == "local": # TODO
+                        #gcApi = f"{config['config']['url']}/gifconvert"
                     vurl=media['originalUrl'] if 'originalUrl' in media else media['url']
-                    media['url'] = config['config']['gifConvertAPI'] + "/convert?url=" + vurl
+                    media['url'] = gcApi + "/convert.avif?url=" + vurl
                     suffix += " • GIF"
                     media["suffix"] = suffix
-        return media
+        return media
+    
+def indexOfAny(list,items,caseInsensitive=False):
+    if caseInsensitive:
+        list = [item.upper() for item in list]
+        items = [item.upper() for item in items]
+    for item in items:
+        try:
+            return list.index(item)
+        except:
+            continue
+    return -1

vxApi.py

@@ -77,6 +77,7 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False):
     oldTweetVersion = False
     tweetArticle=None
     lang=None
+    translation=None
 
     if "screen_name" not in userL:
         userL["screen_name"] = user["core"]["screen_name"]
@@ -85,6 +86,14 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False):
     if "profile_image_url_https" not in userL:
         userL["profile_image_url_https"] = user["avatar"]["image_url"]
 
+    if "grok_translated_post_with_availability" in tweet and tweet["grok_translated_post_with_availability"]["is_available"] and "data" in tweet["grok_translated_post_with_availability"]:
+        tlData = tweet["grok_translated_post_with_availability"]["data"]
+        translation = {
+            "source_language":tlData["source_language"],
+            "destination_language":tlData["destination_language"],
+            "text":tlData["translation"]
+        }
+
     #editedTweet=False
     try:
         if "birdwatch_pivot" in tweet:
@@ -305,6 +314,7 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False):
         "replyingToID": replyingToID,
         "fetched_on": int(datetime.now().timestamp()),
         "retweetURL":retweetURL,
+        "translation":translation
     }
     try:
         apiObject["date_epoch"] = int(datetime.strptime(tweetL["created_at"], "%a %b %d %H:%M:%S %z %Y").timestamp())