From d262fe10143f44e6c18b9a4c72d42922f2577011 Mon Sep 17 00:00:00 2001 From: Dylan Date: Fri, 30 Jan 2026 22:52:38 +0000 Subject: [PATCH 01/16] Update "gif" conversion logic --- gifConvert/Dockerfile | 37 ++++++------ gifConvert/__init__.py | 128 +++++++++++++++++++++++++++++------------ gifConvert/conv.sh | 64 --------------------- utils.py | 7 ++- 4 files changed, 112 insertions(+), 124 deletions(-) delete mode 100644 gifConvert/conv.sh diff --git a/gifConvert/Dockerfile b/gifConvert/Dockerfile index 8be33eb..20ffa30 100644 --- a/gifConvert/Dockerfile +++ b/gifConvert/Dockerfile @@ -1,30 +1,25 @@ -FROM public.ecr.aws/lambda/python:3.8 AS builder -RUN yum -y install git curl -RUN yum -y groupinstall 'Development Tools' -RUN git clone https://github.com/kohler/gifsicle -WORKDIR gifsicle -RUN autoreconf -i -RUN ./configure --disable-gifview --disable-gifdiff -RUN make -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y -WORKDIR /var/task -RUN git clone https://github.com/ImageOptim/gifski -WORKDIR gifski -RUN /root/.cargo/bin/cargo build --release +FROM public.ecr.aws/lambda/python:3.12 AS builder +RUN dnf -y install git cargo && dnf clean all +RUN git clone https://github.com/ImageOptim/gifski /gifski +WORKDIR /gifski +RUN cargo build --release - -FROM public.ecr.aws/lambda/python:3.8 -RUN yum -y update -RUN yum -y install git && yum -y install wget && yum -y install tar.x86_64 && yum -y install xz && yum clean all +FROM public.ecr.aws/lambda/python:3.12 AS ffmpeg +RUN dnf -y update +RUN dnf -y install git wget tar.x86_64 xz && dnf clean all +WORKDIR /ffmpeg RUN wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz RUN tar -xvf ffmpeg-release-amd64-static.tar.xz -RUN mv ff*/ffmpeg . && mv ff*/ffprobe . && rm *.tar.xz && rm -rf ff*/ -COPY --from=builder /var/task/gifsicle/src/gifsicle ./ -COPY --from=builder /var/task/gifski/target/release/gifski ./ + + +FROM public.ecr.aws/lambda/python:3.12 +COPY --from=builder /gifski/target/release/gifski ./ +COPY --from=ffmpeg /ffmpeg/ffmpeg-*-amd64-static/ffmpeg /usr/local/bin/ffmpeg +COPY --from=ffmpeg /ffmpeg/ffmpeg-*-amd64-static/ffprobe /usr/local/bin/ffprobe +RUN pip install requests==2.32.3 # Copy function code COPY __init__.py ${LAMBDA_TASK_ROOT}/app.py -COPY conv.sh ${LAMBDA_TASK_ROOT}/conv.sh # Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile) CMD [ "app.lambda_handler" ] \ No newline at end of file diff --git a/gifConvert/__init__.py b/gifConvert/__init__.py index 6d3f19b..6dff671 100644 --- a/gifConvert/__init__.py +++ b/gifConvert/__init__.py @@ -1,9 +1,22 @@ import base64 import os import subprocess -import json -import sys import tempfile +import urllib.request +import re +import botocore +import boto3 +import requests + +useBucket=False +bucketname=os.getenv('CF_BUCKET') +s3=None +if bucketname is None: + useBucket=False +else: + useBucket=True + s3 = boto3.client('s3',endpoint_url=os.getenv('CF_ENDPOINT'),aws_access_key_id=os.getenv('CF_KEY'),aws_secret_access_key=os.getenv('CF_KEY_SECRET')) + def extractStatus(url): return "" @@ -11,7 +24,7 @@ def extractStatus(url): def get_video_frame_rate(filename): result = subprocess.run( [ - "./ffprobe", + "ffprobe", "-v", "error", "-select_streams", @@ -32,7 +45,7 @@ def get_video_frame_rate(filename): def get_video_length_seconds(filename): result = subprocess.run( [ - "./ffprobe", + "ffprobe", "-v", "error", "-show_entries", @@ -47,54 +60,95 @@ def get_video_length_seconds(filename): result_string = result.stdout.decode('utf-8').split()[0] return float(result_string) -def calcEdits(vlen,loopTimes): - st="r" - for i in range(loopTimes): - st+=f'e{str((vlen*i))}-9999,0' - return st - def loop_video_until_length(filename, length): # use stream_loop to loop video until it's at least length seconds long video_length = get_video_length_seconds(filename) if video_length < length: loops = int(length/video_length) new_filename = tempfile.mkstemp(suffix=".mp4")[1] - #edits = calcEdits(video_length,loops) - out = subprocess.call(["ffmpeg","-stream_loop",str(loops),"-i",filename,"-c","copy","-y",new_filename],stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT) - #subprocess.run(["./MP4Box", "-add",filename,"-edits",f'1={edits}',new_filename]) + subprocess.call(["ffmpeg","-stream_loop",str(loops),"-i",filename,"-c","copy","-y",new_filename],stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT) + return new_filename else: return filename - +def redir(url): + return { + "statusCode": 307, + "headers": { + "Location": url + } + } def lambda_handler(event, context): if ("queryStringParameters" not in event): return { "statusCode": 400, - "body": "Invalid request." + "body": "Invalid request!" } url = event["queryStringParameters"].get("url","") - - # download video - videoLocation = tempfile.mkstemp(suffix=".mp4")[1] - subprocess.call(["wget","-O",videoLocation,url],stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT) - - videoLocationLooped = loop_video_until_length(videoLocation, 30) - if videoLocationLooped != videoLocation: - os.remove(videoLocation) - videoLocation = videoLocationLooped - - with open(videoLocation, "rb") as image_file: - encoded_string = base64.b64encode(image_file.read()).decode('ascii') - os.remove(videoLocation) - return { - 'statusCode': 200, - "headers": - { - "Content-Type": "video/mp4" - }, - 'body': encoded_string, - 'isBase64Encoded': True - } \ No newline at end of file + try: + if url == "": + return { + "statusCode": 400, + "body": "Invalid request!!" + } + if not url.startswith("https://video.twimg.com/tweet_video/"): + return redir(url) + + if useBucket: + id=re.search(r"https:\/\/video\.twimg\.com\/tweet_video\/(.*?)\..*",url).group(1) + bfilename = str(id)+".mp4" + furl=f"https://gifs.vxtwitter.com/{bfilename}" #f"https://{bucketname}.s3.amazonaws.com/{bfilename}" + print("get req for: "+url) + try: + s3.head_object(Bucket=bucketname, Key=bfilename) + print("found existing already: "+bfilename) + return redir(furl) + except botocore.exceptions.ClientError: + # Not found + pass + # download video + print("downloading: "+url) + videoLocation = tempfile.mkstemp(suffix=".mp4")[1] + response = requests.get(url, stream=True) + if response.status_code == 200: + with open(videoLocation, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + else: + print("error downloading video") + return redir(url) + + videoLocationLooped = loop_video_until_length(videoLocation, 30) + if videoLocationLooped != videoLocation: + os.remove(videoLocation) + videoLocation = videoLocationLooped + else: + os.remove(videoLocation) + return redir(url) + + if not useBucket: + with open(videoLocation, "rb") as image_file: + encoded_string = base64.b64encode(image_file.read()).decode('ascii') + os.remove(videoLocation) + return { + 'statusCode': 200, + "headers": + { + "Content-Type": "video/mp4" + }, + 'body': encoded_string, + 'isBase64Encoded': True + } + else: + with open(videoLocation, "rb") as image_file: + s3.upload_fileobj(image_file, bucketname, bfilename) + os.remove(videoLocation) + print("converted: "+url+" -> "+furl) + return redir(furl) + except Exception as e: + print("error converting gif: ") + print(e) + return redir(url) \ No newline at end of file diff --git a/gifConvert/conv.sh b/gifConvert/conv.sh deleted file mode 100644 index e24dc04..0000000 --- a/gifConvert/conv.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -e - -usage(){ - echo "Usage: $0 [options] output" - echo "Options:" - echo " --help Show this help" - echo " -u, --url URL of the video" - echo " -w, --max-width Maximum width of the output" - echo " -h, --max-height Maximum height of the output" - echo " -t, --threads Number of threads to use" - exit 1 -} - -URL="" -MAXW=400 -MAXH=267 -THREADS=1 -OUTPUT="out.gif" -FPS=10 - -while [ $# -gt 0 ]; do - case "$1" in - --help) - usage - ;; - -u|--url) - URL="$2" - shift - ;; - -w|--max-width) - MAXW="$2" - shift - ;; - -h|--max-height) - MAXH="$2" - shift - ;; - -t|--threads) - THREADS="$2" - shift - ;; - -f|--fps) - FPS="$2" - shift - ;; - -*) - echo "Unknown option: $1" - usage - ;; - *) - OUTPUT="$1" - ;; - esac - shift -done - -# make unique temp directory -TEMPDIR=$( mktemp -d ) - -./ffmpeg -i "$URL" -vf "scale=if(gte(iw\,ih)\,min($MAXW\,iw)\,-2):if(lt(iw\,ih)\,min($MAXH\,ih)\,-2)" -threads $THREADS "$TEMPDIR/frame%04d.png" -./gifski -o "$TEMPDIR/out.gif" --fast --fps $FPS --quality=90 $TEMPDIR/frame*.png -#./gifsicle -O3 "$TEMPDIR/out.gif" -o "$OUTPUT" -mv "$TEMPDIR/out.gif" "$OUTPUT" -rm -rf "$TEMPDIR" \ No newline at end of file diff --git a/utils.py b/utils.py index 3a305d7..331c9f4 100644 --- a/utils.py +++ b/utils.py @@ -83,9 +83,12 @@ def determineMediaToEmbed(tweetData,embedIndex = -1,convertGif = True): return media elif media['type'] == "video" or media['type'] == "gif": if media['type'] == "gif" and convertGif: - if config['config']['gifConvertAPI'] != "" and config['config']['gifConvertAPI'] != "none": + if config['config']['gifConvertAPI'] != "" and config['config']['gifConvertAPI'] != "none" and config['config']['gifConvertAPI'] != "local": + gcApi = config['config']['gifConvertAPI'] + #if gcApi == "local": # TODO + #gcApi = f"{config['config']['url']}/gifconvert" vurl=media['originalUrl'] if 'originalUrl' in media else media['url'] - media['url'] = config['config']['gifConvertAPI'] + "/convert?url=" + vurl + media['url'] = gcApi + "/convert?url=" + vurl suffix += " • GIF" media["suffix"] = suffix return media \ No newline at end of file From fb8abeb904218bb271ba41e43965a2a530088e38 Mon Sep 17 00:00:00 2001 From: Dylan Date: Sun, 1 Feb 2026 15:57:00 +0000 Subject: [PATCH 02/16] Convert gifv to avif --- gifConvert/Dockerfile | 69 +++++++++++++++++++++++++++----- gifConvert/__init__.py | 91 +++++++++++++++++++----------------------- twitfix.py | 7 +++- utils.py | 2 +- 4 files changed, 107 insertions(+), 62 deletions(-) diff --git a/gifConvert/Dockerfile b/gifConvert/Dockerfile index 20ffa30..2e880dd 100644 --- a/gifConvert/Dockerfile +++ b/gifConvert/Dockerfile @@ -1,21 +1,72 @@ +ARG TARGETARCH FROM public.ecr.aws/lambda/python:3.12 AS builder RUN dnf -y install git cargo && dnf clean all RUN git clone https://github.com/ImageOptim/gifski /gifski WORKDIR /gifski RUN cargo build --release -FROM public.ecr.aws/lambda/python:3.12 AS ffmpeg -RUN dnf -y update -RUN dnf -y install git wget tar.x86_64 xz && dnf clean all -WORKDIR /ffmpeg -RUN wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz -RUN tar -xvf ffmpeg-release-amd64-static.tar.xz +#FROM public.ecr.aws/lambda/python:3.12 AS ffmpeg-linux-amd64 +#RUN dnf -y update +#RUN dnf -y install git wget tar.x86_64 xz && dnf clean all +#WORKDIR /ffmpeg +#RUN wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz +#RUN tar -xvf ffmpeg-release-amd64-static.tar.xz +#FROM public.ecr.aws/lambda/python:3.12 AS ffmpeg-linux-arm64 +#RUN dnf -y update +#RUN dnf -y install git wget tar xz && dnf clean all +#WORKDIR /ffmpeg +#RUN wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-arm64-static.tar.xz +#RUN tar -xvf ffmpeg-release-arm64-static.tar.xz + +#FROM ffmpeg-linux-${TARGETARCH} AS ffmpeg + +FROM public.ecr.aws/lambda/python:3.12 AS ffmpeg +# tl;dr: build ffmpeg from source with svt-av1 support and statically linked binaries + +# Install dependencies +RUN dnf remove -y microdnf-dnf && microdnf install -y dnf +RUN dnf group install "Development Tools" -y + +RUN mkdir /ffmpeg_sources +WORKDIR /ffmpeg_sources +RUN git clone https://git.ffmpeg.org/ffmpeg.git . +RUN git clone --branch stable --depth 1 https://code.videolan.org/videolan/x264.git +RUN git clone https://gitlab.com/AOMediaCodec/SVT-AV1.git + +RUN dnf install curl nasm glibc-static libstdc++-static cmake --allowerasing -y + +# Install x264 +WORKDIR /ffmpeg_sources/x264 +RUN PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" ./configure --prefix="/ffmpeg_build" --bindir="/bin" --enable-static +RUN make -j +RUN make install + +# Install SVT-AV1 +WORKDIR /ffmpeg_sources +WORKDIR /ffmpeg_sources/SVT-AV1/Build + +RUN PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" cmake .. -G"Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DENABLE_STATIC=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_C_FLAGS_INIT="-static" + +RUN make -j +RUN make install + +# Install FFmpeg +WORKDIR /ffmpeg_sources +RUN PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" ./configure --prefix="/ffmpeg_build" \ + --pkg-config-flags="--static" --extra-ldexeflags="-static" --extra-libs="-lpthread -lm" --bindir="/bin" --disable-shared --enable-static --enable-gpl --enable-pthreads \ + --enable-libx264 --enable-libsvtav1 --disable-ffplay + +RUN make -j FROM public.ecr.aws/lambda/python:3.12 -COPY --from=builder /gifski/target/release/gifski ./ -COPY --from=ffmpeg /ffmpeg/ffmpeg-*-amd64-static/ffmpeg /usr/local/bin/ffmpeg -COPY --from=ffmpeg /ffmpeg/ffmpeg-*-amd64-static/ffprobe /usr/local/bin/ffprobe + +#COPY --from=builder /gifski/target/release/gifski /usr/local/bin/gifski +#COPY --from=ffmpeg /ffmpeg/ffmpeg-*-static/ffmpeg /usr/local/bin/ffmpeg +#COPY --from=ffmpeg /ffmpeg/ffmpeg-*-static/ffprobe /usr/local/bin/ffprobe +COPY --from=ffmpeg /ffmpeg_sources/ffmpeg /usr/local/bin/ffmpeg +COPY --from=ffmpeg /ffmpeg_sources/ffprobe /usr/local/bin/ffprobe + RUN pip install requests==2.32.3 # Copy function code diff --git a/gifConvert/__init__.py b/gifConvert/__init__.py index 6dff671..c777bdf 100644 --- a/gifConvert/__init__.py +++ b/gifConvert/__init__.py @@ -21,55 +21,44 @@ else: def extractStatus(url): return "" -def get_video_frame_rate(filename): - result = subprocess.run( - [ - "ffprobe", - "-v", - "error", - "-select_streams", - "v", - "-of", - "default=noprint_wrappers=1:nokey=1", - "-show_entries", - "stream=r_frame_rate", - filename, - ], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - result_string = result.stdout.decode('utf-8').split()[0].split('/') - fps = float(result_string[0])/float(result_string[1]) - return fps +def convert_video_to_gif(filename): + try: + new_filename = tempfile.mkstemp(suffix=".gif")[1] + print("converting gif w gifski") + p_ffmpeg = subprocess.Popen(["ffmpeg", "-i", filename, "-f", "yuv4mpegpipe", "-"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + p_gifski = subprocess.Popen(["gifski","--quality","70","--lossy-quality","30","-o", new_filename, "-"], stdin=p_ffmpeg.stdout, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) -def get_video_length_seconds(filename): - result = subprocess.run( - [ - "ffprobe", - "-v", - "error", - "-show_entries", - "format=duration", - "-of", - "default=noprint_wrappers=1:nokey=1", - filename, - ], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - result_string = result.stdout.decode('utf-8').split()[0] - return float(result_string) - -def loop_video_until_length(filename, length): - # use stream_loop to loop video until it's at least length seconds long - video_length = get_video_length_seconds(filename) - if video_length < length: - loops = int(length/video_length) - new_filename = tempfile.mkstemp(suffix=".mp4")[1] - subprocess.call(["ffmpeg","-stream_loop",str(loops),"-i",filename,"-c","copy","-y",new_filename],stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT) - - return new_filename - else: + p_ffmpeg.stdout.close() + ret_gifski = p_gifski.wait() + ret_ffmpeg = p_ffmpeg.wait() + if ret_gifski != 0: + print("err: gifski exited with code:", ret_gifski) + if ret_ffmpeg != 0: + print("err: ffmpeg exited with code:", ret_ffmpeg) + if os.path.isfile(new_filename) and os.path.getsize(new_filename) > 0: + return new_filename + else: + print("gifski failed to convert gif") + return filename + except Exception as e: + print("error converting gif (convert_video_to_gif):") + print(e) + return filename + +def convert_video_to_avif(filename): + try: + fd,new_filename = tempfile.mkstemp(suffix=".avif") + os.close(fd) + print("converting gif w ffmpeg to avif") + subprocess.call(["ffmpeg","-nostdin","-y", "-i", filename,"-pix_fmt","yuv420p","-an","-c:v","libsvtav1","-crf","30","-b:v","0","-threads","4", new_filename], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + if os.path.isfile(new_filename) and os.path.getsize(new_filename) > 0: + return new_filename + else: + print("ffmpeg failed to convert avif") + return filename + except Exception as e: + print("error converting avif (convert_video_to_avif):") + print(e) return filename def redir(url): @@ -99,7 +88,7 @@ def lambda_handler(event, context): if useBucket: id=re.search(r"https:\/\/video\.twimg\.com\/tweet_video\/(.*?)\..*",url).group(1) - bfilename = str(id)+".mp4" + bfilename = str(id)+".avif" furl=f"https://gifs.vxtwitter.com/{bfilename}" #f"https://{bucketname}.s3.amazonaws.com/{bfilename}" print("get req for: "+url) try: @@ -121,7 +110,7 @@ def lambda_handler(event, context): print("error downloading video") return redir(url) - videoLocationLooped = loop_video_until_length(videoLocation, 30) + videoLocationLooped = convert_video_to_avif(videoLocation) if videoLocationLooped != videoLocation: os.remove(videoLocation) videoLocation = videoLocationLooped @@ -137,7 +126,7 @@ def lambda_handler(event, context): 'statusCode': 200, "headers": { - "Content-Type": "video/mp4" + "Content-Type": "image/avif", }, 'body': encoded_string, 'isBase64Encoded': True diff --git a/twitfix.py b/twitfix.py index da9732e..982a3bd 100644 --- a/twitfix.py +++ b/twitfix.py @@ -439,8 +439,13 @@ def twitfix(sub_path): suffix = media["suffix"] if media['type'] == "image": return Response(renderImageTweetEmbed(tweetData,media['url'] , appnameSuffix=suffix,embedIndex=embedIndex),headers={"Cache-Tag": "embed"}) - elif media['type'] == "video" or media['type'] == "gif": + elif media['type'] == "video": return Response(renderVideoTweetEmbed(tweetData,media,appnameSuffix=suffix,embedIndex=embedIndex),headers={"Cache-Tag": "embed"}) + elif media['type'] == "gif": + if "originalUrl" in media and media["url"] != media["originalUrl"]: + return Response(renderImageTweetEmbed(tweetData,media['url'] , appnameSuffix=suffix,embedIndex=embedIndex),headers={"Cache-Tag": "embed"}) + else: + return Response(renderVideoTweetEmbed(tweetData,media,appnameSuffix=suffix,embedIndex=embedIndex),headers={"Cache-Tag": "embed"}) return message(msgs.failedToScan) diff --git a/utils.py b/utils.py index 331c9f4..037bc04 100644 --- a/utils.py +++ b/utils.py @@ -88,7 +88,7 @@ def determineMediaToEmbed(tweetData,embedIndex = -1,convertGif = True): #if gcApi == "local": # TODO #gcApi = f"{config['config']['url']}/gifconvert" vurl=media['originalUrl'] if 'originalUrl' in media else media['url'] - media['url'] = gcApi + "/convert?url=" + vurl + media['url'] = gcApi + "/convert.avif?url=" + vurl suffix += " • GIF" media["suffix"] = suffix return media \ No newline at end of file From 68cc985b8c69d4d2e3a480d24e43a4da505d2d06 Mon Sep 17 00:00:00 2001 From: Dylan Date: Sun, 1 Feb 2026 16:17:41 +0000 Subject: [PATCH 03/16] Small fix for Activity embeds + non-discord clients --- activity.py | 5 ++++- twitfix.py | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/activity.py b/activity.py index 6780067..bd6264f 100644 --- a/activity.py +++ b/activity.py @@ -35,7 +35,10 @@ def tweetDataToActivity(tweetData,embedIndex = -1): if media is not None: media = deepcopy(media) if media['type'] == "gif": - media['type'] = "gifv" + if "/convert.avif" in media['url']: + media['type'] = "image" + else: + media['type'] = "gifv" if 'thumbnail_url' not in media: media['thumbnail_url'] = media['url'] if media['type'] == "image" and "?" not in media['url']: diff --git a/twitfix.py b/twitfix.py index 982a3bd..f6860c9 100644 --- a/twitfix.py +++ b/twitfix.py @@ -433,7 +433,8 @@ def twitfix(sub_path): elif not embeddingMedia: return Response(renderTextTweetEmbed(tweetData),headers={"Cache-Tag": "embed"}) else: - media = determineMediaToEmbed(embedTweetData,embedIndex) + tryGifConversion = "Discord" in user_agent + media = determineMediaToEmbed(embedTweetData,embedIndex,convertGif=tryGifConversion) suffix="" if "suffix" in media: suffix = media["suffix"] @@ -442,7 +443,7 @@ def twitfix(sub_path): elif media['type'] == "video": return Response(renderVideoTweetEmbed(tweetData,media,appnameSuffix=suffix,embedIndex=embedIndex),headers={"Cache-Tag": "embed"}) elif media['type'] == "gif": - if "originalUrl" in media and media["url"] != media["originalUrl"]: + if "originalUrl" in media and media["url"] != media["originalUrl"] and tryGifConversion: return Response(renderImageTweetEmbed(tweetData,media['url'] , appnameSuffix=suffix,embedIndex=embedIndex),headers={"Cache-Tag": "embed"}) else: return Response(renderVideoTweetEmbed(tweetData,media,appnameSuffix=suffix,embedIndex=embedIndex),headers={"Cache-Tag": "embed"}) From afcf6b10aa23639b063d6c7c5c66a112f123cba7 Mon Sep 17 00:00:00 2001 From: Dylan Date: Sun, 1 Feb 2026 19:12:18 +0000 Subject: [PATCH 04/16] Update extractStatusV2Android --- test_vx_extract.py | 4 ++++ twExtract/__init__.py | 11 +++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/test_vx_extract.py b/test_vx_extract.py index f72afef..bbc8119 100644 --- a/test_vx_extract.py +++ b/test_vx_extract.py @@ -20,6 +20,10 @@ def test_twextract_extractStatusV2Rest(): assert utils.stripEndTCO(tweet["full_text"])[:94]==testMultiMediaTweet_compare['text'][:94] +def test_twextract_v2API_Anon(): + tweet = twExtract.extractStatusV2Rest_Anon(testMediaTweet,workaroundTokens=tokens)['legacy'] + assert utils.stripEndTCO(tweet["full_text"])==testMediaTweet_compare['text'] + def test_twextract_v2API(): tweet = twExtract.extractStatusV2(testMediaTweet,workaroundTokens=tokens)['legacy'] assert utils.stripEndTCO(tweet["full_text"])==testMediaTweet_compare['text'] diff --git a/twExtract/__init__.py b/twExtract/__init__.py index 88f1469..7490cbd 100644 --- a/twExtract/__init__.py +++ b/twExtract/__init__.py @@ -31,8 +31,8 @@ v2AnonFeatures='{"creator_subscriptions_tweet_preview_api_enabled":true,"premium v2AnonGraphql_api="wqi5M7wZ7tW-X9S2t-Mqcg" gt_pattern = r'document\.cookie="gt=([^;]+);' -androidGraphqlFeatures='{"grok_translations_community_note_translation_is_enabled":false,"super_follow_badge_privacy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"super_follow_user_api_enabled":true,"profile_label_improvements_pcf_label_in_profile_enabled":true,"premium_content_api_read_enabled":false,"grok_translations_community_note_auto_translation_is_enabled":false,"android_graphql_skip_api_media_color_palette":true,"tweetypie_unmention_optimization_enabled":true,"longform_notetweets_consumption_enabled":true,"subscriptions_verification_info_enabled":true,"blue_business_profile_image_shape_enabled":true,"super_follow_exclusive_tweet_notifications_enabled":true,"longform_notetweets_inline_media_enabled":true,"grok_android_analyze_trend_fetch_enabled":false,"unified_cards_ad_metadata_container_dynamic_card_content_query_enabled":true,"super_follow_tweet_api_enabled":true,"articles_api_enabled":true,"creator_subscriptions_tweet_preview_api_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"grok_translations_timeline_user_bio_auto_translation_is_enabled":false,"grok_translations_post_auto_translation_is_enabled":false,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"immersive_video_status_linkable_timestamps":true,"profile_label_improvements_pcf_label_in_post_enabled":true}' -androidGraphql_api="k3rtLsS9kG5hI-Jr0dTMCg" +androidGraphqlFeatures='{"grok_translations_community_note_translation_is_enabled":false,"super_follow_badge_privacy_enabled":true,"unified_cards_destination_url_params_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"super_follow_user_api_enabled":true,"profile_label_improvements_pcf_label_in_profile_enabled":true,"premium_content_api_read_enabled":false,"grok_translations_community_note_auto_translation_is_enabled":false,"android_graphql_skip_api_media_color_palette":true,"tweetypie_unmention_optimization_enabled":true,"longform_notetweets_consumption_enabled":true,"subscriptions_verification_info_enabled":true,"blue_business_profile_image_shape_enabled":true,"super_follow_exclusive_tweet_notifications_enabled":true,"longform_notetweets_inline_media_enabled":true,"grok_android_analyze_trend_fetch_enabled":false,"unified_cards_ad_metadata_container_dynamic_card_content_query_enabled":true,"super_follow_tweet_api_enabled":true,"articles_api_enabled":true,"android_ad_formats_media_component_render_overlay_enabled":true,"creator_subscriptions_tweet_preview_api_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"grok_translations_timeline_user_bio_auto_translation_is_enabled":false,"grok_translations_post_auto_translation_is_enabled":false,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"immersive_video_status_linkable_timestamps":true,"profile_label_improvements_pcf_label_in_post_enabled":true}' +androidGraphql_api="Wrspae-uyGj-nWPyUqdUag" tweetDetailGraphqlFeatures='{"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"articles_preview_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"rweb_video_timestamps_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_enhance_cards_enabled":false}' tweetDetailGraphql_api="YVyS4SfwYW7Uw5qwy0mQCA" @@ -113,7 +113,7 @@ def cycleBearerTokenGet(url,headers): return tweet raise TwExtractError(400, "Extract error") -def twitterApiGet(url,btoken=None,authToken=None,guestToken=None): +def twitterApiGet(url,btoken=None,authToken=None,guestToken=None,userAgent=None): if authToken != None and authToken.startswith("oa|"): url = url.replace("https://x.com/i/api/graphql/","https://api.twitter.com/graphql/") @@ -125,6 +125,8 @@ def twitterApiGet(url,btoken=None,authToken=None,guestToken=None): hdr = getAuthHeaders(androidBearer) del hdr["Authorization"] hdr["X-Twitter-Client"] = "TwitterAndroid" + if userAgent is not None: + hdr["User-Agent"] = userAgent uri, headers, body = twt.sign(url, headers=hdr,realm="http://api.twitter.com/") @@ -139,6 +141,7 @@ def twitterApiGet(url,btoken=None,authToken=None,guestToken=None): return response def getAuthHeaders(btoken,authToken=None,guestToken=None): + csrfToken=str(uuid.uuid4()).replace('-', '') headers = {"x-twitter-active-user":"yes","x-twitter-client-language":"en","x-csrf-token":csrfToken,"User-Agent":requestUserAgent} headers['Authorization'] = btoken @@ -332,7 +335,7 @@ def extractStatusV2Android(url,workaroundTokens): try: vars = json.loads('{"referrer":"home","includeTweetImpression":true,"includeHasBirdwatchNotes":false,"isReaderMode":false,"includeEditPerspective":false,"includeEditControl":true,"focalTweetId":0,"includeCommunityTweetRelationship":true,"includeTweetVisibilityNudge":true}') vars['focalTweetId'] = int(twid) - tweet = twitterApiGet(f"https://x.com/i/api/graphql/{androidGraphql_api}/ConversationTimelineV2?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(androidGraphqlFeatures)}", authToken=authToken,btoken=androidBearer) + tweet = twitterApiGet(f"https://x.com/i/api/graphql/{androidGraphql_api}/ConversationTimelineV2?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(androidGraphqlFeatures)}", authToken=authToken,btoken=androidBearer,userAgent="TwitterAndroid/11.61.0-release.0 (311610000-r-0) G011A/9 (google;G011A;google;G011A;0;;1;2016)") try: rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining") print(f"Twitter Android Token Rate limit remaining: {rateLimitRemaining}") From 3d4e42223f1878d950996d6eb4e61e8170ddda28 Mon Sep 17 00:00:00 2001 From: Dylan Date: Sun, 1 Feb 2026 22:39:04 +0000 Subject: [PATCH 05/16] Update api calls --- twExtract/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/twExtract/__init__.py b/twExtract/__init__.py index 7490cbd..cc1e811 100644 --- a/twExtract/__init__.py +++ b/twExtract/__init__.py @@ -27,15 +27,15 @@ userIDregex = r"\/i\/user\/(\d+)" v2Features='{"longform_notetweets_inline_media_enabled":true,"super_follow_badge_privacy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"super_follow_user_api_enabled":true,"super_follow_tweet_api_enabled":true,"android_graphql_skip_api_media_color_palette":true,"creator_subscriptions_tweet_preview_api_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"creator_subscriptions_subscription_count_enabled":true,"tweetypie_unmention_optimization_enabled":true,"longform_notetweets_consumption_enabled":true,"subscriptions_verification_info_enabled":true,"blue_business_profile_image_shape_enabled":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"super_follow_exclusive_tweet_notifications_enabled":true}' v2graphql_api="2OOZWmw8nAtUHVnXXQhgaA" -v2AnonFeatures='{"creator_subscriptions_tweet_preview_api_enabled":true,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":false,"responsive_web_jetfuel_frame":true,"responsive_web_grok_share_attachment_enabled":true,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":false,"responsive_web_grok_analysis_button_from_backend":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"payments_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"verified_phone_label_enabled":false,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_grok_imagine_annotation_enabled":true,"responsive_web_grok_community_note_auto_translation_is_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false}' -v2AnonGraphql_api="wqi5M7wZ7tW-X9S2t-Mqcg" +v2AnonFeatures='{"creator_subscriptions_tweet_preview_api_enabled":true,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":true,"responsive_web_grok_share_attachment_enabled":true,"responsive_web_grok_annotations_enabled":false,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":true,"responsive_web_grok_analysis_button_from_backend":true,"post_ctas_fetch_enabled":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"profile_label_improvements_pcf_label_in_post_enabled":true,"responsive_web_profile_redirect_enabled":false,"rweb_tipjar_consumption_enabled":false,"verified_phone_label_enabled":false,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_grok_imagine_annotation_enabled":true,"responsive_web_grok_community_note_auto_translation_is_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false}' +v2AnonGraphql_api="0aTrQMKgj95K791yXeNDRA" gt_pattern = r'document\.cookie="gt=([^;]+);' androidGraphqlFeatures='{"grok_translations_community_note_translation_is_enabled":false,"super_follow_badge_privacy_enabled":true,"unified_cards_destination_url_params_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"super_follow_user_api_enabled":true,"profile_label_improvements_pcf_label_in_profile_enabled":true,"premium_content_api_read_enabled":false,"grok_translations_community_note_auto_translation_is_enabled":false,"android_graphql_skip_api_media_color_palette":true,"tweetypie_unmention_optimization_enabled":true,"longform_notetweets_consumption_enabled":true,"subscriptions_verification_info_enabled":true,"blue_business_profile_image_shape_enabled":true,"super_follow_exclusive_tweet_notifications_enabled":true,"longform_notetweets_inline_media_enabled":true,"grok_android_analyze_trend_fetch_enabled":false,"unified_cards_ad_metadata_container_dynamic_card_content_query_enabled":true,"super_follow_tweet_api_enabled":true,"articles_api_enabled":true,"android_ad_formats_media_component_render_overlay_enabled":true,"creator_subscriptions_tweet_preview_api_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"grok_translations_timeline_user_bio_auto_translation_is_enabled":false,"grok_translations_post_auto_translation_is_enabled":false,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"immersive_video_status_linkable_timestamps":true,"profile_label_improvements_pcf_label_in_post_enabled":true}' androidGraphql_api="Wrspae-uyGj-nWPyUqdUag" -tweetDetailGraphqlFeatures='{"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"articles_preview_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"rweb_video_timestamps_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_enhance_cards_enabled":false}' -tweetDetailGraphql_api="YVyS4SfwYW7Uw5qwy0mQCA" +tweetDetailGraphqlFeatures='{"rweb_video_screen_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"responsive_web_profile_redirect_enabled":false,"rweb_tipjar_consumption_enabled":false,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":true,"responsive_web_grok_share_attachment_enabled":true,"responsive_web_grok_annotations_enabled":false,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":true,"responsive_web_grok_analysis_button_from_backend":true,"post_ctas_fetch_enabled":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_grok_imagine_annotation_enabled":true,"responsive_web_grok_community_note_auto_translation_is_enabled":false,"responsive_web_enhance_cards_enabled":false}' +tweetDetailGraphql_api="Kzfv17rukSzjT96BerOWZA" # this is for UserTweets endpoint tweetFeedGraphqlFeatures='{"rweb_video_screen_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":false,"responsive_web_grok_share_attachment_enabled":true,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":false,"responsive_web_grok_analysis_button_from_backend":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_enhance_cards_enabled":false}' @@ -454,7 +454,7 @@ def extractStatusV2Rest(url,workaroundTokens): # get tweet try: - vars = json.loads('{"tweetId":"0","withCommunity":false,"includePromotedContent":false,"withVoice":false}') + vars = json.loads('{"tweetId":"0","includePromotedContent":true,"withBirdwatchNotes":true,"withVoice":true,"withCommunity":true}') vars['tweetId'] = str(twid) if workaroundTokens is not None and len(workaroundTokens) > 0: tokens = workaroundTokens @@ -514,7 +514,7 @@ def fixTweetData(tweet): def extractStatus(url,workaroundTokens=None): # TODO: commented out methods are too slow/unreliable at the moment - methods=[extractStatusV2Rest_Anon,extractStatusV2,extractStatusV2Rest,extractStatusV2Android]#,extractStatusV2TweetDetail] + methods=[extractStatusV2Rest_Anon,extractStatusV2,extractStatusV2TweetDetail,extractStatusV2Android]#,extractStatusV2Rest] for method in methods: try: result = method(url,workaroundTokens) From a610cd8e29084665fcaa8753efb43ba79c25a45f Mon Sep 17 00:00:00 2001 From: Dylan Date: Sat, 18 Apr 2026 14:26:29 +0100 Subject: [PATCH 06/16] Add cache headers --- gifConvert/Dockerfile | 2 +- gifConvert/__init__.py | 1 + twExtract/__init__.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/gifConvert/Dockerfile b/gifConvert/Dockerfile index 2e880dd..8e18b61 100644 --- a/gifConvert/Dockerfile +++ b/gifConvert/Dockerfile @@ -57,7 +57,7 @@ RUN PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" ./configure --prefix="/ffmpeg_ --pkg-config-flags="--static" --extra-ldexeflags="-static" --extra-libs="-lpthread -lm" --bindir="/bin" --disable-shared --enable-static --enable-gpl --enable-pthreads \ --enable-libx264 --enable-libsvtav1 --disable-ffplay -RUN make -j +RUN make -j 8 FROM public.ecr.aws/lambda/python:3.12 diff --git a/gifConvert/__init__.py b/gifConvert/__init__.py index c777bdf..f366d58 100644 --- a/gifConvert/__init__.py +++ b/gifConvert/__init__.py @@ -127,6 +127,7 @@ def lambda_handler(event, context): "headers": { "Content-Type": "image/avif", + "Cache-Control": "public, max-age=604800, immutable" }, 'body': encoded_string, 'isBase64Encoded': True diff --git a/twExtract/__init__.py b/twExtract/__init__.py index cc1e811..364f56e 100644 --- a/twExtract/__init__.py +++ b/twExtract/__init__.py @@ -514,7 +514,7 @@ def fixTweetData(tweet): def extractStatus(url,workaroundTokens=None): # TODO: commented out methods are too slow/unreliable at the moment - methods=[extractStatusV2Rest_Anon,extractStatusV2,extractStatusV2TweetDetail,extractStatusV2Android]#,extractStatusV2Rest] + methods=[extractStatusV2Rest_Anon,extractStatusV2,extractStatusV2Rest,extractStatusV2Android]#,extractStatusV2TweetDetail] for method in methods: try: result = method(url,workaroundTokens) From 4794857c0ae32310422714d3037f08c8f3acd6e4 Mon Sep 17 00:00:00 2001 From: Dylan Date: Sat, 18 Apr 2026 14:59:38 +0100 Subject: [PATCH 07/16] Fix feed extract --- twExtract/twUtils.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/twExtract/twUtils.py b/twExtract/twUtils.py index 20f72ae..5b94415 100644 --- a/twExtract/twUtils.py +++ b/twExtract/twUtils.py @@ -3,8 +3,9 @@ import hashlib import base64 import uuid from x_client_transaction import ClientTransaction -from x_client_transaction.utils import handle_x_migration +from x_client_transaction.utils import get_ondemand_file_url, generate_headers import requests +import bs4 digits = "0123456789abcdefghijklmnopqrstuvwxyz" def baseConversion(x, base): @@ -49,6 +50,14 @@ def get_twitter_homepage(headers=None): return response def generate_transaction_id(method: str, path: str,headers=None) -> str: - ct = ClientTransaction(get_twitter_homepage(headers=headers)) + session = requests.Session() + session.headers = generate_headers() + home_page = session.get(url="https://x.com") + home_page_response = bs4.BeautifulSoup(home_page.content, 'html.parser') + ondemand_file_url = get_ondemand_file_url(response=home_page_response) + ondemand_file = session.get(url=ondemand_file_url) + ondemand_file_response = bs4.BeautifulSoup(ondemand_file.content, 'html.parser') + + ct = ClientTransaction(home_page_response=home_page_response, ondemand_file_response=ondemand_file_response) transaction_id = ct.generate_transaction_id(method=method, path=path) return transaction_id \ No newline at end of file From 1a6f424e6993b9f188dbe815831837c42da9b85a Mon Sep 17 00:00:00 2001 From: Dylan Date: Sat, 18 Apr 2026 20:11:03 +0100 Subject: [PATCH 08/16] Initial translation implementation #273 --- activity.py | 13 ++++++-- msgs.py | 15 +++++++-- requirements.txt | 2 +- translation.py | 7 ++++ twExtract/__init__.py | 40 ++++++++++++---------- twitfix.py | 77 ++++++++++++++++++++++++++++++------------- utils.py | 13 +++++++- vxApi.py | 10 ++++++ 8 files changed, 130 insertions(+), 47 deletions(-) create mode 100644 translation.py diff --git a/activity.py b/activity.py index bd6264f..6c52caa 100644 --- a/activity.py +++ b/activity.py @@ -8,11 +8,20 @@ def tweetDataToActivity(tweetData,embedIndex = -1): if tweetData['replyingTo'] is not None: content += f"
↪️ Replying to @{tweetData['replyingTo']}
" - content+=f"

{tweetData['text']}

" + + mainText = tweetData['text'] + if tweetData["translation"] is not None: + content += f"
🌐 {tweetData['translation']['source_language'].upper()}→{tweetData['translation']['destination_language'].upper()}
" + mainText=tweetData['translation']["text"] + + content+=f"

{mainText}

" attachments=[] if tweetData['qrt'] is not None: - content += f"
QRT: {tweetData['qrt']['user_screen_name']}
{tweetData['qrt']['text']}
" + qrtText = tweetData['qrt']['text'] + if "translation" in tweetData['qrt'] and tweetData['qrt']["translation"] is not None: + qrtText = tweetData['qrt']["translation"]["text"] + content += f"
QRT: {tweetData['qrt']['user_screen_name']}
{qrtText}
" if tweetData['pollData'] is not None: content += f"

{msgs.genPollDisplay(tweetData['pollData'])}

" content += "

" diff --git a/msgs.py b/msgs.py index 8bcd17c..07d3d02 100644 --- a/msgs.py +++ b/msgs.py @@ -17,8 +17,11 @@ def genLikesDisplay(vnf): return ("💖 " + numerize.numerize(vnf['likes'])) def genQrtDisplay(qrt): + text = qrt['text'] + if "translation" in qrt: + text = qrt["translation"]["text"] verifiedCheck = "☑️" if ('verified' in qrt and qrt['verified']) else "" - return ("\n\n【QRT of " + qrt['user_name'] + " (@" + qrt['user_screen_name'] + ")"+ verifiedCheck+":】\n\n'" + qrt['text'] + "'") + return ("\n\n【QRT of " + qrt['user_name'] + " (@" + qrt['user_screen_name'] + ")"+ verifiedCheck+":】\n\n'" + text + "'") def genPollDisplay(poll): pctSplit=10 @@ -27,6 +30,9 @@ def genPollDisplay(poll): output+=choice["name"]+"\n"+("█"*int(choice["percent"]/pctSplit)) +" "+str(choice["percent"])+"%\n" return output +def genTranslationDisplay(translation): + return f"【TL "+translation["source_language"]+"→"+translation["destination_language"]+"】\n\n"+translation["text"] + # formats the top text of the embed def formatProvider(base,vnf): finalText = base @@ -38,13 +44,16 @@ def formatProvider(base,vnf): finalText = base return finalText -def formatEmbedDesc(type,body,qrt,pollData): +def formatEmbedDesc(type,body,qrt,pollData,translation): # Trim the embed description to 248 characters, prioritizing poll and likes qrtType=None if qrt!=None: qrtType="Text" + if translation is not None: + body = genTranslationDisplay(translation) + limit = videoDescLimit if type=="Video" or (qrt!=None and (qrtType=="Video")) else tweetDescLimit output = "" @@ -73,6 +82,6 @@ def formatEmbedDesc(type,body,qrt,pollData): diff = len(output)-limit # remove the characters from body, add ellipsis body = body[:-(diff+1)]+"…" - return formatEmbedDesc(type,body,qrt,pollData) + return formatEmbedDesc(type,body,qrt,pollData,None) else: return output diff --git a/requirements.txt b/requirements.txt index ef9f74d..209347e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,4 @@ Werkzeug==2.3.7 numerize==0.12 oauthlib==3.2.2 PyRTF3==0.47.5 -XClientTransaction==0.0.2 \ No newline at end of file +XClientTransaction==1.0.2 \ No newline at end of file diff --git a/translation.py b/translation.py new file mode 100644 index 0000000..7e74ef4 --- /dev/null +++ b/translation.py @@ -0,0 +1,7 @@ +validLanguages = ['AF', 'AN', 'AR', 'AS', 'AY', 'AZ', 'BA', 'BE', 'BG', 'BN', 'BR', 'BS', 'CA', 'CS', 'CY', 'DA', 'DE', 'DE-DE', 'EL', 'EN', 'EO', 'ES', 'ES-419', 'ET', 'EU', 'FA', 'FI', 'FR', 'FR-FR', 'GA', 'GL', 'GN', 'GU', 'HA', 'HE', 'HI', 'HR', 'HT', 'HU', 'HY', 'ID', 'IG', 'IS', 'IT', 'JA', 'JV', 'KA', 'KK', 'KO', 'KY', 'LA', 'LB', 'LN', 'LT', 'LV', 'MG', 'MI', 'MK', 'ML', 'MN', 'MR', 'MS', 'MT', 'MY', 'NB', 'NE', 'NL', 'OC', 'OM', 'PA', 'PL', 'PS', 'PT-BR', 'PT-PT', 'QU', 'RO', 'RU', 'SA', 'SK', 'SL', 'SQ', 'SR', 'ST', 'SU', 'SV', 'SW', 'TA', 'TE', 'TG', 'TH', 'TK', 'TL', 'TN', 'TR', 'TS', 'TT', 'UK', 'UR', 'UZ', 'VI', 'WO', 'XH', 'YI', 'ZH', 'ZH-HANS', 'ZH-HANT', 'ZU'] +languageAliases = { + #"EN":"EN-US", + "JP":"JA", +} + +mergedLangs=(validLanguages+list(languageAliases.keys())) \ No newline at end of file diff --git a/twExtract/__init__.py b/twExtract/__init__.py index 364f56e..793c9af 100644 --- a/twExtract/__init__.py +++ b/twExtract/__init__.py @@ -113,7 +113,7 @@ def cycleBearerTokenGet(url,headers): return tweet raise TwExtractError(400, "Extract error") -def twitterApiGet(url,btoken=None,authToken=None,guestToken=None,userAgent=None): +def twitterApiGet(url,btoken=None,authToken=None,guestToken=None,userAgent=None,language=None): if authToken != None and authToken.startswith("oa|"): url = url.replace("https://x.com/i/api/graphql/","https://api.twitter.com/graphql/") @@ -122,7 +122,7 @@ def twitterApiGet(url,btoken=None,authToken=None,guestToken=None,userAgent=None) secret = authToken.split("|")[1] twt = oauth1.Client(client_key='3nVuSoBZnx6U4vzUxf5w',client_secret='Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys',resource_owner_key=key,resource_owner_secret=secret) - hdr = getAuthHeaders(androidBearer) + hdr = getAuthHeaders(androidBearer,language=language) del hdr["Authorization"] hdr["X-Twitter-Client"] = "TwitterAndroid" if userAgent is not None: @@ -135,12 +135,12 @@ def twitterApiGet(url,btoken=None,authToken=None,guestToken=None,userAgent=None) if btoken is None: btoken = v2bearer #return cycleBearerTokenGet(url,getAuthHeaders(bearer,authToken=authToken,guestToken=guestToken)) - headers = getAuthHeaders(btoken,authToken=authToken,guestToken=guestToken) + headers = getAuthHeaders(btoken,authToken=authToken,guestToken=guestToken,language=language) response = requests.get(url, headers=headers) return response -def getAuthHeaders(btoken,authToken=None,guestToken=None): +def getAuthHeaders(btoken,authToken=None,guestToken=None,language=None): csrfToken=str(uuid.uuid4()).replace('-', '') headers = {"x-twitter-active-user":"yes","x-twitter-client-language":"en","x-csrf-token":csrfToken,"User-Agent":requestUserAgent} @@ -152,6 +152,9 @@ def getAuthHeaders(btoken,authToken=None,guestToken=None): if guestToken is not None: headers["x-guest-token"] = guestToken + if language is not None: + headers["x-twitter-client-language"] = language + return headers def getGuestToken(): @@ -268,7 +271,7 @@ def extractStatus_twExtractProxy(url,workaroundTokens=None): continue return output -def extractStatusV2(url,workaroundTokens): +def extractStatusV2(url,workaroundTokens,tlLanguage=None): # get tweet ID m = re.search(pathregex, url) if m is None: @@ -282,7 +285,7 @@ def extractStatusV2(url,workaroundTokens): def request_with_token(twid, authToken): vars = json.loads('{"includeTweetImpression":true,"includeHasBirdwatchNotes":false,"includeEditPerspective":false,"rest_ids":["x"],"includeEditControl":true,"includeCommunityTweetRelationship":true,"includeTweetVisibilityNudge":true}') vars['rest_ids'][0] = str(twid) - tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2graphql_api}/TweetResultsByIdsQuery?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2Features)}",authToken=authToken) + tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2graphql_api}/TweetResultsByIdsQuery?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2Features)}",authToken=authToken,language=tlLanguage) try: rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining") print(f"Twitter Token Rate limit remaining: {rateLimitRemaining}") @@ -321,7 +324,7 @@ def extractStatusV2(url,workaroundTokens): return tweet return parallel_token_request(twid, tokens, request_with_token) -def extractStatusV2Android(url,workaroundTokens): +def extractStatusV2Android(url,workaroundTokens,tlLanguage=None): # get tweet ID m = re.search(pathregex, url) if m is None: @@ -335,7 +338,7 @@ def extractStatusV2Android(url,workaroundTokens): try: vars = json.loads('{"referrer":"home","includeTweetImpression":true,"includeHasBirdwatchNotes":false,"isReaderMode":false,"includeEditPerspective":false,"includeEditControl":true,"focalTweetId":0,"includeCommunityTweetRelationship":true,"includeTweetVisibilityNudge":true}') vars['focalTweetId'] = int(twid) - tweet = twitterApiGet(f"https://x.com/i/api/graphql/{androidGraphql_api}/ConversationTimelineV2?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(androidGraphqlFeatures)}", authToken=authToken,btoken=androidBearer,userAgent="TwitterAndroid/11.61.0-release.0 (311610000-r-0) G011A/9 (google;G011A;google;G011A;0;;1;2016)") + tweet = twitterApiGet(f"https://x.com/i/api/graphql/{androidGraphql_api}/ConversationTimelineV2?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(androidGraphqlFeatures)}", authToken=authToken,btoken=androidBearer,userAgent="TwitterAndroid/11.61.0-release.0 (311610000-r-0) G011A/9 (google;G011A;google;G011A;0;;1;2016)",language=tlLanguage) try: rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining") print(f"Twitter Android Token Rate limit remaining: {rateLimitRemaining}") @@ -380,7 +383,7 @@ def extractStatusV2Android(url,workaroundTokens): return tweet return parallel_token_request(twid, tokens, request_with_token) -def extractStatusV2TweetDetail(url,workaroundTokens): +def extractStatusV2TweetDetail(url,workaroundTokens,tlLanguage=None): # get tweet ID m = re.search(pathregex, url) if m is None: @@ -395,7 +398,7 @@ def extractStatusV2TweetDetail(url,workaroundTokens): try: vars = json.loads('{"focalTweetId":"0","with_rux_injections":false,"includePromotedContent":true,"withCommunity":true,"withQuickPromoteEligibilityTweetFields":true,"withBirdwatchNotes":true,"withVoice":true,"withV2Timeline":true}') vars['focalTweetId'] = str(twid) - tweet = twitterApiGet(f"https://x.com/i/api/graphql/{tweetDetailGraphql_api}/TweetDetail?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(tweetDetailGraphqlFeatures)}", authToken=authToken,btoken=v2bearer) + tweet = twitterApiGet(f"https://x.com/i/api/graphql/{tweetDetailGraphql_api}/TweetDetail?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(tweetDetailGraphqlFeatures)}", authToken=authToken,btoken=v2bearer,language=tlLanguage) try: rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining") print(f"Twitter Token Rate limit remaining: {rateLimitRemaining}") @@ -440,10 +443,10 @@ def extractStatusV2TweetDetail(url,workaroundTokens): return tweet return parallel_token_request(twid, tokens, request_with_token) -def extractStatusV2Rest_Anon(url,workaroundTokens): - return extractStatusV2Rest(url,None) +def extractStatusV2Rest_Anon(url,workaroundTokens,tlLanguage=None): + return extractStatusV2Rest(url,None,tlLanguage=tlLanguage) -def extractStatusV2Rest(url,workaroundTokens): +def extractStatusV2Rest(url,workaroundTokens,tlLanguage=None): # get tweet ID m = re.search(pathregex, url) if m is None: @@ -461,11 +464,11 @@ def extractStatusV2Rest(url,workaroundTokens): random.shuffle(tokens) for authToken in tokens: try: - tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", btoken=v2bearer,authToken=authToken,guestToken=guestToken) + tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", btoken=v2bearer,authToken=authToken,guestToken=guestToken,language=tlLanguage) except Exception as e: continue else: - tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", btoken=v2bearer,guestToken=guestToken) + tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", btoken=v2bearer,guestToken=guestToken,language=tlLanguage) try: rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining") @@ -512,12 +515,13 @@ def fixTweetData(tweet): pass return tweet -def extractStatus(url,workaroundTokens=None): - # TODO: commented out methods are too slow/unreliable at the moment +def extractStatus(url,workaroundTokens=None,tlLanguage=None): methods=[extractStatusV2Rest_Anon,extractStatusV2,extractStatusV2Rest,extractStatusV2Android]#,extractStatusV2TweetDetail] + if tlLanguage is not None: # prioritize endpoints that return translations + methods = [extractStatusV2Rest,extractStatusV2TweetDetail,extractStatusV2Android,extractStatusV2,extractStatusV2Rest_Anon] for method in methods: try: - result = method(url,workaroundTokens) + result = method(url,workaroundTokens,tlLanguage=tlLanguage) if 'legacy' not in result: print(f"{method.__name__} method failed: Legacy not found for {url}") # try another method diff --git a/twitfix.py b/twitfix.py index f6860c9..13f7a9a 100644 --- a/twitfix.py +++ b/twitfix.py @@ -14,7 +14,7 @@ import msgs import twExtract as twExtract from cache import addVnfToLinkCache,getVnfFromLinkCache import vxlogging as log -from utils import getTweetIdFromUrl, pathregex, determineMediaToEmbed, determineEmbedTweet, BytesIOWrapper, fixMedia +from utils import getTweetIdFromUrl, pathregex, determineMediaToEmbed, determineEmbedTweet, BytesIOWrapper, fixMedia, indexOfAny from vxApi import getApiResponse, getApiUserResponse from urllib.parse import urlparse from PyRTF.Elements import Document @@ -22,12 +22,14 @@ from PyRTF.document.section import Section from PyRTF.document.paragraph import Paragraph from copy import deepcopy import json -import datetime +import translation import activity as activitymg app = Flask(__name__) CORS(app) user_agent="" +activityseparator="zqj" + staticFiles = { # TODO: Use flask static files instead of this "favicon.ico": {"mime": "image/vnd.microsoft.icon","path": "favicon.ico"}, "apple-touch-icon.png": {"mime": "image/png","path": "apple-touch-icon.png"}, @@ -82,7 +84,11 @@ def generateActivityLink(tweetData,media=None,mediatype=None,embedIndex=-1): return None try: embedIndex = embedIndex+1 - return f"{config['config']['url']}/users/{tweetData['user_screen_name']}/statuses/{str(embedIndex)}{tweetData['tweetID']}" + + extras="" + if tweetData["translation"] is not None: + extras+=activityseparator+tweetData["translation"]["destination_language"] + return f"{config['config']['url']}/users/{tweetData['user_screen_name']}/statuses/{str(embedIndex)}{activityseparator}{tweetData['tweetID']}{extras}" except Exception as e: log.error("Error generating activity link: "+str(e)) return None @@ -95,7 +101,7 @@ def getAppName(tweetData,appnameSuffix=""): def renderImageTweetEmbed(tweetData,image,appnameSuffix="",embedIndex=-1): qrt = tweetData['qrt'] - embedDesc = msgs.formatEmbedDesc("Image",tweetData['text'],qrt,tweetData['pollData']) + embedDesc = msgs.formatEmbedDesc("Image",tweetData['text'],qrt,tweetData['pollData'],tweetData["translation"]) if image.startswith("https://pbs.twimg.com") and "?" not in image: image = f"{image}?name=orig" @@ -115,7 +121,7 @@ def renderImageTweetEmbed(tweetData,image,appnameSuffix="",embedIndex=-1): def renderVideoTweetEmbed(tweetData,mediaInfo,appnameSuffix="",embedIndex=-1): qrt = tweetData['qrt'] - embedDesc = msgs.formatEmbedDesc("Video",tweetData['text'],qrt,tweetData['pollData']) + embedDesc = msgs.formatEmbedDesc("Video",tweetData['text'],qrt,tweetData['pollData'],tweetData["translation"]) mediaInfo=fixMedia(mediaInfo) @@ -138,7 +144,7 @@ def renderVideoTweetEmbed(tweetData,mediaInfo,appnameSuffix="",embedIndex=-1): def renderTextTweetEmbed(tweetData,appnameSuffix=""): qrt = tweetData['qrt'] - embedDesc = msgs.formatEmbedDesc("Text",tweetData['text'],qrt,tweetData['pollData']) + embedDesc = msgs.formatEmbedDesc("Text",tweetData['text'],qrt,tweetData['pollData'],tweetData["translation"]) return render_template("text.html", tweet=tweetData, @@ -154,7 +160,7 @@ def renderTextTweetEmbed(tweetData,appnameSuffix=""): def renderArticleTweetEmbed(tweetData,appnameSuffix=""): articlePreview=tweetData['article']["title"]+"\n\n"+tweetData['article']["preview_text"]+"…" - embedDesc = msgs.formatEmbedDesc("Image",articlePreview,None,None) + embedDesc = msgs.formatEmbedDesc("Image",articlePreview,None,None,None) return render_template("image.html", tweet=tweetData, @@ -260,9 +266,9 @@ def userJson(): }, } -def getTweetData(twitter_url,include_txt="false",include_rtf="false"): +def getTweetData(twitter_url,include_txt="false",include_rtf="false",tlLanguage=None): cachedVNF = getVnfFromLinkCache(twitter_url) - if cachedVNF is not None and include_txt == "false" and include_rtf == "false": + if cachedVNF is not None and include_txt == "false" and include_rtf == "false" and tlLanguage is None: return cachedVNF try: @@ -270,7 +276,7 @@ def getTweetData(twitter_url,include_txt="false",include_rtf="false"): workaroundTokens = config['config']['workaroundTokens'].split(",") else: workaroundTokens = None - rawTweetData = twExtract.extractStatus(twitter_url,workaroundTokens=workaroundTokens) + rawTweetData = twExtract.extractStatus(twitter_url,workaroundTokens=workaroundTokens,tlLanguage=tlLanguage) except: rawTweetData = None if rawTweetData == None or 'error' in rawTweetData: @@ -281,7 +287,15 @@ def getTweetData(twitter_url,include_txt="false",include_rtf="false"): tweetData = getApiResponse(rawTweetData,include_txt,include_rtf) if tweetData is None: return None - if include_txt == "false" and include_rtf == "false": + + if tweetData["translation"] is None and tlLanguage is not None and tlLanguage != tweetData["lang"]: + # translation probably failed on twitter's end + pass # TODO: implement DeepL api + + if tweetData["translation"] is not None and tlLanguage is None: + tweetData["translation"] = None # didn't ask for TL + + if include_txt == "false" and include_rtf == "false" and tlLanguage is None: addVnfToLinkCache(twitter_url,tweetData) return tweetData @@ -307,6 +321,7 @@ def getUserData(twitter_url,includeFeed=False): @app.route('/') # Default endpoint used by everything def twitfix(sub_path): + splitPath=sub_path.split("/") global user_agent user_agent = request.headers.get('User-Agent', None) if user_agent is None: @@ -355,24 +370,33 @@ def twitfix(sub_path): include_txt="false" include_rtf="false" + + translationIndex=indexOfAny(splitPath,translation.mergedLangs,caseInsensitive=True) + translationLang = None + if translationIndex != -1: + if splitPath[translationIndex].upper() in translation.languageAliases: + translationLang = translation.languageAliases[splitPath[translationIndex].upper()] + else: + translationLang = splitPath[translationIndex].lower() + if isApiRequest: if "include_txt" in request.args: include_txt = request.args.get("include_txt") if "include_rtf" in request.args: include_rtf = request.args.get("include_rtf") - tweetData = getTweetData(twitter_url,include_txt,include_rtf) + tweetData = getTweetData(twitter_url,include_txt,include_rtf,tlLanguage=translationLang) if tweetData is None: log.error("Tweet Data Get failed for "+twitter_url) return message(msgs.failedToScan) qrt = None if 'qrtURL' in tweetData and tweetData['qrtURL'] is not None: - qrt = getTweetData(tweetData['qrtURL']) + qrt = getTweetData(tweetData['qrtURL'],tlLanguage=translationLang) tweetData['qrt'] = qrt retweet = None if 'retweetURL' in tweetData and tweetData['retweetURL'] is not None: - retweet = getTweetData(tweetData['retweetURL']) + retweet = getTweetData(tweetData['retweetURL'],tlLanguage=translationLang) tweetData['retweet'] = retweet tweetData = deepcopy(tweetData) @@ -403,9 +427,10 @@ def twitfix(sub_path): return send_file(BytesIOWrapper(rtf), mimetype='application/rtf', as_attachment=True, download_name=f'{tweetData["user_screen_name"]}_{tweetData["tweetID"]}.rtf') embedIndex = -1 - # if url ends with /1, /2, /3, or /4, we'll use that as the index - if sub_path[-2:] in ["/1","/2","/3","/4"]: - embedIndex = int(sub_path[-1])-1 + # if url has /1, /2, /3, or /4, we'll use that as the index + numIdx=indexOfAny(splitPath,["1","2","3","4"]); + if numIdx != -1: + embedIndex = int(splitPath[numIdx])-1 sub_path = sub_path[:-2] if isApiRequest: # Directly return the API response if the request is from the API @@ -484,21 +509,29 @@ def rendercombined(): @app.route("/api/v1/statuses/") def api_v1_status(tweet_id): - embedIndex = int(tweet_id[0])-1 - tweet_id = int(tweet_id[1:]) + lang = None + if activityseparator not in tweet_id: #old embeds + embedIndex = int(tweet_id[0])-1 + tweet_id = int(tweet_id[1:]) + else: + args = tweet_id.split(activityseparator) + embedIndex = int(args[0]) + tweet_id = int(args[1]) + if len(args)>2: + lang=args[2] twitter_url=f"https://twitter.com/i/status/{tweet_id}" - tweetData = getTweetData(twitter_url) + tweetData = getTweetData(twitter_url,tlLanguage=lang) if tweetData is None: log.error("Tweet Data Get failed for "+twitter_url) return message(msgs.failedToScan) qrt = None if 'qrtURL' in tweetData and tweetData['qrtURL'] is not None: - qrt = getTweetData(tweetData['qrtURL']) + qrt = getTweetData(tweetData['qrtURL'],tlLanguage=lang) tweetData['qrt'] = qrt retweet = None if 'retweetURL' in tweetData and tweetData['retweetURL'] is not None: - retweet = getTweetData(tweetData['retweetURL']) + retweet = getTweetData(tweetData['retweetURL'],tlLanguage=lang) tweetData['retweet'] = retweet if tweetData is None: diff --git a/utils.py b/utils.py index 037bc04..7dd66b7 100644 --- a/utils.py +++ b/utils.py @@ -91,4 +91,15 @@ def determineMediaToEmbed(tweetData,embedIndex = -1,convertGif = True): media['url'] = gcApi + "/convert.avif?url=" + vurl suffix += " • GIF" media["suffix"] = suffix - return media \ No newline at end of file + return media + +def indexOfAny(list,items,caseInsensitive=False): + if caseInsensitive: + list = [item.upper() for item in list] + items = [item.upper() for item in items] + for item in items: + try: + return list.index(item) + except: + continue + return -1 \ No newline at end of file diff --git a/vxApi.py b/vxApi.py index 69d7ce3..6b2eaad 100644 --- a/vxApi.py +++ b/vxApi.py @@ -77,6 +77,7 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False): oldTweetVersion = False tweetArticle=None lang=None + translation=None if "screen_name" not in userL: userL["screen_name"] = user["core"]["screen_name"] @@ -85,6 +86,14 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False): if "profile_image_url_https" not in userL: userL["profile_image_url_https"] = user["avatar"]["image_url"] + if "grok_translated_post_with_availability" in tweet and tweet["grok_translated_post_with_availability"]["is_available"] and "data" in tweet["grok_translated_post_with_availability"]: + tlData = tweet["grok_translated_post_with_availability"]["data"] + translation = { + "source_language":tlData["source_language"], + "destination_language":tlData["destination_language"], + "text":tlData["translation"] + } + #editedTweet=False try: if "birdwatch_pivot" in tweet: @@ -305,6 +314,7 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False): "replyingToID": replyingToID, "fetched_on": int(datetime.now().timestamp()), "retweetURL":retweetURL, + "translation":translation } try: apiObject["date_epoch"] = int(datetime.strptime(tweetL["created_at"], "%a %b %d %H:%M:%S %z %Y").timestamp()) From 7c7a6e6a5992c0bdf5a9e1c25ae17634477ab48d Mon Sep 17 00:00:00 2001 From: Dylan Date: Sat, 18 Apr 2026 23:50:09 +0100 Subject: [PATCH 09/16] Finish translation implementation --- .github/workflows/deploy.yml | 3 ++- configHandler.py | 6 ++++-- requirements.txt | 3 ++- serverless.yml | 1 + translation.py | 22 +++++++++++++++++++++- twitfix.py | 2 +- 6 files changed, 31 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 695ac90..f353456 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -70,4 +70,5 @@ jobs: VXTWITTER_GIF_CONVERT_API: ${{ secrets.VXTWITTER_GIF_CONVERT_API }} VXTWITTER_WORKAROUND_TOKENS: ${{ secrets.VXTWITTER_WORKAROUND_TOKENS }} VXTWITTER_PROXIES: ${{ secrets.VXTWITTER_PROXIES }} - SERVERLESS_ACCESS_KEY: ${{ secrets.SERVERLESS_ACCESS_KEY }} \ No newline at end of file + SERVERLESS_ACCESS_KEY: ${{ secrets.SERVERLESS_ACCESS_KEY }} + DEEPL_API_KEY: ${{ secrets.DEEPL_API_KEY }} \ No newline at end of file diff --git a/configHandler.py b/configHandler.py index f8e656b..9d89a7c 100644 --- a/configHandler.py +++ b/configHandler.py @@ -13,7 +13,8 @@ if ('RUNNING_SERVERLESS' in os.environ and os.environ['RUNNING_SERVERLESS'] == ' "url": os.getenv("VXTWITTER_URL","https://vxtwitter.com"), "combination_method": os.getenv("VXTWITTER_COMBINATION_METHOD","local"), # can either be 'local' or a URL to a server handling requests in the same format "gifConvertAPI":os.getenv("VXTWITTER_GIF_CONVERT_API",""), - "workaroundTokens":os.getenv("VXTWITTER_WORKAROUND_TOKENS",None) + "workaroundTokens":os.getenv("VXTWITTER_WORKAROUND_TOKENS",None), + "deeplKey":os.getenv("DEEPL_API_KEY",None), } } else: @@ -31,7 +32,8 @@ else: "url": "https://vxtwitter.com", "combination_method": "local", # can either be 'local' or a URL to a server handling requests in the same format "gifConvertAPI":"", - "workaroundTokens":None + "workaroundTokens":None, + "deeplKey":None } } diff --git a/requirements.txt b/requirements.txt index 209347e..b16d9f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,5 @@ Werkzeug==2.3.7 numerize==0.12 oauthlib==3.2.2 PyRTF3==0.47.5 -XClientTransaction==1.0.2 \ No newline at end of file +XClientTransaction==1.0.2 +deepl==1.3.0 \ No newline at end of file diff --git a/serverless.yml b/serverless.yml index 990545a..b5d8ce1 100644 --- a/serverless.yml +++ b/serverless.yml @@ -29,6 +29,7 @@ provider: VXTWITTER_COMBINATION_METHOD: ${env:VXTWITTER_COMBINATION_METHOD, 'local'} VXTWITTER_GIF_CONVERT_API: ${env:VXTWITTER_GIF_CONVERT_API, ''} VXTWITTER_WORKAROUND_TOKENS: ${env:VXTWITTER_WORKAROUND_TOKENS, ''} + DEEPL_API_KEY: ${env:DEEPL_API_KEY, ''} #VXTWITTER_PROXIES: ${env:VXTWITTER_PROXIES, ''} package: diff --git a/translation.py b/translation.py index 7e74ef4..939c236 100644 --- a/translation.py +++ b/translation.py @@ -1,7 +1,27 @@ +import deepl +from configHandler import config + validLanguages = ['AF', 'AN', 'AR', 'AS', 'AY', 'AZ', 'BA', 'BE', 'BG', 'BN', 'BR', 'BS', 'CA', 'CS', 'CY', 'DA', 'DE', 'DE-DE', 'EL', 'EN', 'EO', 'ES', 'ES-419', 'ET', 'EU', 'FA', 'FI', 'FR', 'FR-FR', 'GA', 'GL', 'GN', 'GU', 'HA', 'HE', 'HI', 'HR', 'HT', 'HU', 'HY', 'ID', 'IG', 'IS', 'IT', 'JA', 'JV', 'KA', 'KK', 'KO', 'KY', 'LA', 'LB', 'LN', 'LT', 'LV', 'MG', 'MI', 'MK', 'ML', 'MN', 'MR', 'MS', 'MT', 'MY', 'NB', 'NE', 'NL', 'OC', 'OM', 'PA', 'PL', 'PS', 'PT-BR', 'PT-PT', 'QU', 'RO', 'RU', 'SA', 'SK', 'SL', 'SQ', 'SR', 'ST', 'SU', 'SV', 'SW', 'TA', 'TE', 'TG', 'TH', 'TK', 'TL', 'TN', 'TR', 'TS', 'TT', 'UK', 'UR', 'UZ', 'VI', 'WO', 'XH', 'YI', 'ZH', 'ZH-HANS', 'ZH-HANT', 'ZU'] languageAliases = { #"EN":"EN-US", "JP":"JA", } -mergedLangs=(validLanguages+list(languageAliases.keys())) \ No newline at end of file +mergedLangs=(validLanguages+list(languageAliases.keys())) + +def getDeeplTranslation(text,target): + target = target.upper() + if target == "EN": + target = "EN-US" + try: + if 'deeplKey' not in config["config"] or config["config"]["deeplKey"] == None: + return None + deepl_client = deepl.Translator(config["config"]["deeplKey"]) + result = deepl_client.translate_text(text, target_lang=target) + return { + "text":result.text, + "source_language":result.detected_source_lang, + "destination_language":target + } + except Exception as e: + return None \ No newline at end of file diff --git a/twitfix.py b/twitfix.py index 13f7a9a..e2b16f4 100644 --- a/twitfix.py +++ b/twitfix.py @@ -290,7 +290,7 @@ def getTweetData(twitter_url,include_txt="false",include_rtf="false",tlLanguage= if tweetData["translation"] is None and tlLanguage is not None and tlLanguage != tweetData["lang"]: # translation probably failed on twitter's end - pass # TODO: implement DeepL api + tweetData["translation"] = translation.getDeeplTranslation(tweetData["text"],tlLanguage) if tweetData["translation"] is not None and tlLanguage is None: tweetData["translation"] = None # didn't ask for TL From fb1fdaf4b55208f0b583d304e7731723176e2801 Mon Sep 17 00:00:00 2001 From: Dylan Date: Sun, 19 Apr 2026 01:39:52 +0100 Subject: [PATCH 10/16] Fix reverted change --- msgs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgs.py b/msgs.py index 07d3d02..c65424d 100644 --- a/msgs.py +++ b/msgs.py @@ -18,7 +18,7 @@ def genLikesDisplay(vnf): def genQrtDisplay(qrt): text = qrt['text'] - if "translation" in qrt: + if "translation" in qrt and qrt["translation"] is not None: text = qrt["translation"]["text"] verifiedCheck = "☑️" if ('verified' in qrt and qrt['verified']) else "" return ("\n\n【QRT of " + qrt['user_name'] + " (@" + qrt['user_screen_name'] + ")"+ verifiedCheck+":】\n\n'" + text + "'") From 0e3d24ad5533a924b3f6e3b2523f5caf828cabdb Mon Sep 17 00:00:00 2001 From: Dylan Date: Sun, 19 Apr 2026 13:40:26 +0100 Subject: [PATCH 11/16] Fix for #305 --- twitfix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/twitfix.py b/twitfix.py index e2b16f4..c655f9a 100644 --- a/twitfix.py +++ b/twitfix.py @@ -515,7 +515,7 @@ def api_v1_status(tweet_id): tweet_id = int(tweet_id[1:]) else: args = tweet_id.split(activityseparator) - embedIndex = int(args[0]) + embedIndex = int(args[0])-1 tweet_id = int(args[1]) if len(args)>2: lang=args[2] From 662491f1b488c797231fcd47ab9e44fbec4f6884 Mon Sep 17 00:00:00 2001 From: Dylan Date: Sun, 19 Apr 2026 15:27:34 +0100 Subject: [PATCH 12/16] Add subdomains as input for translation #273 --- test_vx_embeds.py | 18 +++++++++++++++++- twitfix.py | 16 +++++++++++++--- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/test_vx_embeds.py b/test_vx_embeds.py index 56e0718..a28a63b 100644 --- a/test_vx_embeds.py +++ b/test_vx_embeds.py @@ -204,4 +204,20 @@ def test_embed_action(): resp = client.get(testTextTweet.replace("https://twitter.com",""),headers={"User-Agent":"Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)"}) assert resp.status_code==200 assert "application/activity+json" in str(resp.data) - assert "%F0%9F%92%96" not in str(resp.data) # 💖 \ No newline at end of file + assert "%F0%9F%92%96" not in str(resp.data) # 💖 + +def test_embed_translated(): + resp = client.get(testTextTweet.replace("https://twitter.com","")+"/jp",headers={"User-Agent":"test"}) + assert resp.status_code==200 + assert "→" in resp.text + resp = client.get(testTextTweet.replace("https://twitter.com","")+"/ja",headers={"User-Agent":"test"}) + assert resp.status_code==200 + assert "→" in resp.text + +def test_embed_translated_subdomain(): + resp = client.get(testTextTweet.replace("https://twitter.com","https://jp.vxtwitter.com"),headers={"User-Agent":"test"}) + assert resp.status_code==200 + assert "→" in resp.text + resp = client.get(testTextTweet.replace("https://twitter.com","https://ja.vxtwitter.com"),headers={"User-Agent":"test"}) + assert resp.status_code==200 + assert "→" in resp.text \ No newline at end of file diff --git a/twitfix.py b/twitfix.py index c655f9a..1c004d2 100644 --- a/twitfix.py +++ b/twitfix.py @@ -322,13 +322,18 @@ def getUserData(twitter_url,includeFeed=False): @app.route('/') # Default endpoint used by everything def twitfix(sub_path): splitPath=sub_path.split("/") + if "|" in splitPath[-1]: # discord doesn't strip this when a link is spoilered + splitPath[-1] = splitPath[-1].replace("|","") + + subdomain = request.host.rsplit(".",2)[0] + global user_agent user_agent = request.headers.get('User-Agent', None) if user_agent is None: user_agent = "unknown" - isApiRequest=request.url.startswith("https://api.vx") or request.url.startswith("http://api.vx") - if not isApiRequest and (request.url.startswith("https://l.vx") or request.url.startswith("https://old.vx")) and "Discord" in user_agent: + isApiRequest=subdomain=="api" + if not isApiRequest and (subdomain=="l" or subdomain=="old") and "Discord" in user_agent: user_agent = user_agent.replace("Discord","LegacyEmbed") # TODO: Clean up; This is a hacky fix to make the new activity embed not trigger if sub_path in staticFiles: if 'path' not in staticFiles[sub_path] or staticFiles[sub_path]["path"] == None: @@ -378,6 +383,11 @@ def twitfix(sub_path): translationLang = translation.languageAliases[splitPath[translationIndex].upper()] else: translationLang = splitPath[translationIndex].lower() + elif subdomain.upper() in translation.mergedLangs: + if subdomain.upper() in translation.languageAliases: + translationLang = translation.languageAliases[subdomain.upper()] + else: + translationLang = subdomain.lower() if isApiRequest: if "include_txt" in request.args: @@ -407,7 +417,7 @@ def twitfix(sub_path): requestUrlWithoutQuery = request.url directEmbed=False - if requestUrlWithoutQuery.startswith("https://d.vx") or requestUrlWithoutQuery.endswith(".mp4") or requestUrlWithoutQuery.endswith(".png"): + if subdomain=="d" or requestUrlWithoutQuery.endswith(".mp4") or requestUrlWithoutQuery.endswith(".png"): directEmbed = True # remove the .mp4 from the end of the URL if requestUrlWithoutQuery.endswith(".mp4") or requestUrlWithoutQuery.endswith(".png"): From f06ef199bb421b4ece26eb44dbc9e29ab804d673 Mon Sep 17 00:00:00 2001 From: Dylan Date: Tue, 23 Jun 2026 18:19:17 +0100 Subject: [PATCH 13/16] Switch to syndication for anon extract --- twExtract/__init__.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/twExtract/__init__.py b/twExtract/__init__.py index 793c9af..6b5c9c7 100644 --- a/twExtract/__init__.py +++ b/twExtract/__init__.py @@ -14,7 +14,7 @@ bearer="Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DT v2bearer="Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" androidBearer="Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F" -requestUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0" +requestUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0" bearerTokens=[bearer,v2bearer,androidBearer] @@ -161,10 +161,10 @@ def getGuestToken(): global guestToken global guestTokenUses if guestToken is None: - r = requests.get(f"https://{twitterUrl}",headers={"User-Agent":requestUserAgent,"Cookie":"night_mode=2"},allow_redirects=False) + r = requests.get(f"https://{twitterUrl}",headers={"User-Agent":requestUserAgent},allow_redirects=False) m = re.search(gt_pattern, r.text) if m is None: - r = requests.post(f"https://api.{twitterUrl}/1.1/guest/activate.json", headers={"Authorization":bearer}) + r = requests.post(f"https://api.{twitterUrl}/1.1/guest/activate.json", headers={"Authorization":v2bearer}) guestToken = json.loads(r.text)["guest_token"] else: guestToken = m.group(1) @@ -219,7 +219,7 @@ def extractStatus_guestToken(url): raise TwExtractError(error["code"], error["message"]) return output -def extractStatus_syndication(url,workaroundTokens=None): +def extractStatus_syndication(url,workaroundTokens=None,tlLanguage=None): # https://github.com/mikf/gallery-dl/blob/46cae04aa3a113c7b6bbee1bb468669564b14ae8/gallery_dl/extractor/twitter.py#L1784 m = re.search(pathregex, url) if m is None: @@ -516,7 +516,7 @@ def fixTweetData(tweet): return tweet def extractStatus(url,workaroundTokens=None,tlLanguage=None): - methods=[extractStatusV2Rest_Anon,extractStatusV2,extractStatusV2Rest,extractStatusV2Android]#,extractStatusV2TweetDetail] + methods=[extractStatus_syndication,extractStatusV2,extractStatusV2Rest,extractStatusV2Android]#,extractStatusV2TweetDetail] if tlLanguage is not None: # prioritize endpoints that return translations methods = [extractStatusV2Rest,extractStatusV2TweetDetail,extractStatusV2Android,extractStatusV2,extractStatusV2Rest_Anon] for method in methods: From 6ac40e4a52810238ccbb2b8bc3063f708a4166a2 Mon Sep 17 00:00:00 2001 From: Dylan Date: Tue, 23 Jun 2026 22:28:55 +0100 Subject: [PATCH 14/16] Fix guest token fetch and restore original method --- twExtract/__init__.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/twExtract/__init__.py b/twExtract/__init__.py index 6b5c9c7..5122b0b 100644 --- a/twExtract/__init__.py +++ b/twExtract/__init__.py @@ -161,13 +161,8 @@ def getGuestToken(): global guestToken global guestTokenUses if guestToken is None: - r = requests.get(f"https://{twitterUrl}",headers={"User-Agent":requestUserAgent},allow_redirects=False) - m = re.search(gt_pattern, r.text) - if m is None: - r = requests.post(f"https://api.{twitterUrl}/1.1/guest/activate.json", headers={"Authorization":v2bearer}) - guestToken = json.loads(r.text)["guest_token"] - else: - guestToken = m.group(1) + r = requests.post(f"https://api.{twitterUrl}/1.1/guest/activate.json", headers={"Authorization":v2bearer,"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0"}) + guestToken = json.loads(r.text)["guest_token"] guestTokenUses = 0 else: guestTokenUses+=1 @@ -516,7 +511,7 @@ def fixTweetData(tweet): return tweet def extractStatus(url,workaroundTokens=None,tlLanguage=None): - methods=[extractStatus_syndication,extractStatusV2,extractStatusV2Rest,extractStatusV2Android]#,extractStatusV2TweetDetail] + methods=[extractStatusV2Rest_Anon,extractStatusV2,extractStatusV2Rest,extractStatusV2Android]#,extractStatusV2TweetDetail] if tlLanguage is not None: # prioritize endpoints that return translations methods = [extractStatusV2Rest,extractStatusV2TweetDetail,extractStatusV2Android,extractStatusV2,extractStatusV2Rest_Anon] for method in methods: From 161cd4d4e2918ea57745a5963f17f063e6437f9c Mon Sep 17 00:00:00 2001 From: Dylan Date: Wed, 24 Jun 2026 00:23:52 +0100 Subject: [PATCH 15/16] Fix feed endpoints --- twExtract/twUtils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/twExtract/twUtils.py b/twExtract/twUtils.py index 5b94415..82a5b5f 100644 --- a/twExtract/twUtils.py +++ b/twExtract/twUtils.py @@ -52,7 +52,7 @@ def get_twitter_homepage(headers=None): def generate_transaction_id(method: str, path: str,headers=None) -> str: session = requests.Session() session.headers = generate_headers() - home_page = session.get(url="https://x.com") + home_page = session.get(url="https://x.com/home") home_page_response = bs4.BeautifulSoup(home_page.content, 'html.parser') ondemand_file_url = get_ondemand_file_url(response=home_page_response) ondemand_file = session.get(url=ondemand_file_url) From c1d3a7e2ddaf8181978dba449665bc3fba0e7b36 Mon Sep 17 00:00:00 2001 From: Dylan Date: Wed, 15 Jul 2026 14:12:05 +0100 Subject: [PATCH 16/16] Fix HTML escape issues --- activity.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/activity.py b/activity.py index 6c52caa..7eea197 100644 --- a/activity.py +++ b/activity.py @@ -2,6 +2,7 @@ import datetime import msgs from utils import determineEmbedTweet, determineMediaToEmbed from copy import deepcopy +import html def tweetDataToActivity(tweetData,embedIndex = -1): content="" @@ -9,10 +10,10 @@ def tweetDataToActivity(tweetData,embedIndex = -1): if tweetData['replyingTo'] is not None: content += f"
↪️ Replying to @{tweetData['replyingTo']}
" - mainText = tweetData['text'] + mainText = html.escape(tweetData['text']) if tweetData["translation"] is not None: content += f"
🌐 {tweetData['translation']['source_language'].upper()}→{tweetData['translation']['destination_language'].upper()}
" - mainText=tweetData['translation']["text"] + mainText=html.escape(tweetData['translation']["text"]) content+=f"

{mainText}

"