↪️ Replying to @{tweetData['replyingTo']}" - content+=f"
{tweetData['text']}
" + + mainText = html.escape(tweetData['text']) + if tweetData["translation"] is not None: + content += f"🌐 {tweetData['translation']['source_language'].upper()}→{tweetData['translation']['destination_language'].upper()}" + mainText=html.escape(tweetData['translation']["text"]) + + content+=f"
{mainText}
" attachments=[] if tweetData['qrt'] is not None: - content += f"QRT: {tweetData['qrt']['user_screen_name']}" + qrtText = tweetData['qrt']['text'] + if "translation" in tweetData['qrt'] and tweetData['qrt']["translation"] is not None: + qrtText = tweetData['qrt']["translation"]["text"] + content += f"
{tweetData['qrt']['text']}
QRT: {tweetData['qrt']['user_screen_name']}" if tweetData['pollData'] is not None: content += f"
{qrtText}
{msgs.genPollDisplay(tweetData['pollData'])}
" content += "" @@ -35,7 +45,10 @@ def tweetDataToActivity(tweetData,embedIndex = -1): if media is not None: media = deepcopy(media) if media['type'] == "gif": - media['type'] = "gifv" + if "/convert.avif" in media['url']: + media['type'] = "image" + else: + media['type'] = "gifv" if 'thumbnail_url' not in media: media['thumbnail_url'] = media['url'] if media['type'] == "image" and "?" not in media['url']: diff --git a/configHandler.py b/configHandler.py index f8e656b..9d89a7c 100644 --- a/configHandler.py +++ b/configHandler.py @@ -13,7 +13,8 @@ if ('RUNNING_SERVERLESS' in os.environ and os.environ['RUNNING_SERVERLESS'] == ' "url": os.getenv("VXTWITTER_URL","https://vxtwitter.com"), "combination_method": os.getenv("VXTWITTER_COMBINATION_METHOD","local"), # can either be 'local' or a URL to a server handling requests in the same format "gifConvertAPI":os.getenv("VXTWITTER_GIF_CONVERT_API",""), - "workaroundTokens":os.getenv("VXTWITTER_WORKAROUND_TOKENS",None) + "workaroundTokens":os.getenv("VXTWITTER_WORKAROUND_TOKENS",None), + "deeplKey":os.getenv("DEEPL_API_KEY",None), } } else: @@ -31,7 +32,8 @@ else: "url": "https://vxtwitter.com", "combination_method": "local", # can either be 'local' or a URL to a server handling requests in the same format "gifConvertAPI":"", - "workaroundTokens":None + "workaroundTokens":None, + "deeplKey":None } } diff --git a/gifConvert/Dockerfile b/gifConvert/Dockerfile index 8be33eb..8e18b61 100644 --- a/gifConvert/Dockerfile +++ b/gifConvert/Dockerfile @@ -1,30 +1,76 @@ -FROM public.ecr.aws/lambda/python:3.8 AS builder -RUN yum -y install git curl -RUN yum -y groupinstall 'Development Tools' -RUN git clone https://github.com/kohler/gifsicle -WORKDIR gifsicle -RUN autoreconf -i -RUN ./configure --disable-gifview --disable-gifdiff -RUN make -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y -WORKDIR /var/task -RUN git clone https://github.com/ImageOptim/gifski -WORKDIR gifski -RUN /root/.cargo/bin/cargo build --release +ARG TARGETARCH +FROM public.ecr.aws/lambda/python:3.12 AS builder +RUN dnf -y install git cargo && dnf clean all +RUN git clone https://github.com/ImageOptim/gifski /gifski +WORKDIR /gifski +RUN cargo build --release +#FROM public.ecr.aws/lambda/python:3.12 AS ffmpeg-linux-amd64 +#RUN dnf -y update +#RUN dnf -y install git wget tar.x86_64 xz && dnf clean all +#WORKDIR /ffmpeg +#RUN wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz +#RUN tar -xvf ffmpeg-release-amd64-static.tar.xz -FROM public.ecr.aws/lambda/python:3.8 -RUN yum -y update -RUN yum -y install git && yum -y install wget && yum -y install tar.x86_64 && yum -y install xz && yum clean all -RUN wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz -RUN tar -xvf ffmpeg-release-amd64-static.tar.xz -RUN mv ff*/ffmpeg . && mv ff*/ffprobe . && rm *.tar.xz && rm -rf ff*/ -COPY --from=builder /var/task/gifsicle/src/gifsicle ./ -COPY --from=builder /var/task/gifski/target/release/gifski ./ +#FROM public.ecr.aws/lambda/python:3.12 AS ffmpeg-linux-arm64 +#RUN dnf -y update +#RUN dnf -y install git wget tar xz && dnf clean all +#WORKDIR /ffmpeg +#RUN wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-arm64-static.tar.xz +#RUN tar -xvf ffmpeg-release-arm64-static.tar.xz + +#FROM ffmpeg-linux-${TARGETARCH} AS ffmpeg + +FROM public.ecr.aws/lambda/python:3.12 AS ffmpeg +# tl;dr: build ffmpeg from source with svt-av1 support and statically linked binaries + +# Install dependencies +RUN dnf remove -y microdnf-dnf && microdnf install -y dnf +RUN dnf group install "Development Tools" -y + +RUN mkdir /ffmpeg_sources +WORKDIR /ffmpeg_sources +RUN git clone https://git.ffmpeg.org/ffmpeg.git . +RUN git clone --branch stable --depth 1 https://code.videolan.org/videolan/x264.git +RUN git clone https://gitlab.com/AOMediaCodec/SVT-AV1.git + +RUN dnf install curl nasm glibc-static libstdc++-static cmake --allowerasing -y + +# Install x264 +WORKDIR /ffmpeg_sources/x264 +RUN PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" ./configure --prefix="/ffmpeg_build" --bindir="/bin" --enable-static +RUN make -j +RUN make install + +# Install SVT-AV1 +WORKDIR /ffmpeg_sources +WORKDIR /ffmpeg_sources/SVT-AV1/Build + +RUN PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" cmake .. -G"Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DENABLE_STATIC=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_C_FLAGS_INIT="-static" + +RUN make -j +RUN make install + +# Install FFmpeg +WORKDIR /ffmpeg_sources +RUN PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" ./configure --prefix="/ffmpeg_build" \ + --pkg-config-flags="--static" --extra-ldexeflags="-static" --extra-libs="-lpthread -lm" --bindir="/bin" --disable-shared --enable-static --enable-gpl --enable-pthreads \ + --enable-libx264 --enable-libsvtav1 --disable-ffplay + +RUN make -j 8 + +FROM public.ecr.aws/lambda/python:3.12 + +#COPY --from=builder /gifski/target/release/gifski /usr/local/bin/gifski +#COPY --from=ffmpeg /ffmpeg/ffmpeg-*-static/ffmpeg /usr/local/bin/ffmpeg +#COPY --from=ffmpeg /ffmpeg/ffmpeg-*-static/ffprobe /usr/local/bin/ffprobe +COPY --from=ffmpeg /ffmpeg_sources/ffmpeg /usr/local/bin/ffmpeg +COPY --from=ffmpeg /ffmpeg_sources/ffprobe /usr/local/bin/ffprobe + +RUN pip install requests==2.32.3 # Copy function code COPY __init__.py ${LAMBDA_TASK_ROOT}/app.py -COPY conv.sh ${LAMBDA_TASK_ROOT}/conv.sh # Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile) CMD [ "app.lambda_handler" ] \ No newline at end of file diff --git a/gifConvert/__init__.py b/gifConvert/__init__.py index 6d3f19b..f366d58 100644 --- a/gifConvert/__init__.py +++ b/gifConvert/__init__.py @@ -1,100 +1,144 @@ import base64 import os import subprocess -import json -import sys import tempfile +import urllib.request +import re +import botocore +import boto3 +import requests + +useBucket=False +bucketname=os.getenv('CF_BUCKET') +s3=None +if bucketname is None: + useBucket=False +else: + useBucket=True + s3 = boto3.client('s3',endpoint_url=os.getenv('CF_ENDPOINT'),aws_access_key_id=os.getenv('CF_KEY'),aws_secret_access_key=os.getenv('CF_KEY_SECRET')) + def extractStatus(url): return "" -def get_video_frame_rate(filename): - result = subprocess.run( - [ - "./ffprobe", - "-v", - "error", - "-select_streams", - "v", - "-of", - "default=noprint_wrappers=1:nokey=1", - "-show_entries", - "stream=r_frame_rate", - filename, - ], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - result_string = result.stdout.decode('utf-8').split()[0].split('/') - fps = float(result_string[0])/float(result_string[1]) - return fps +def convert_video_to_gif(filename): + try: + new_filename = tempfile.mkstemp(suffix=".gif")[1] + print("converting gif w gifski") + p_ffmpeg = subprocess.Popen(["ffmpeg", "-i", filename, "-f", "yuv4mpegpipe", "-"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + p_gifski = subprocess.Popen(["gifski","--quality","70","--lossy-quality","30","-o", new_filename, "-"], stdin=p_ffmpeg.stdout, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) -def get_video_length_seconds(filename): - result = subprocess.run( - [ - "./ffprobe", - "-v", - "error", - "-show_entries", - "format=duration", - "-of", - "default=noprint_wrappers=1:nokey=1", - filename, - ], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - result_string = result.stdout.decode('utf-8').split()[0] - return float(result_string) - -def calcEdits(vlen,loopTimes): - st="r" - for i in range(loopTimes): - st+=f'e{str((vlen*i))}-9999,0' - return st - -def loop_video_until_length(filename, length): - # use stream_loop to loop video until it's at least length seconds long - video_length = get_video_length_seconds(filename) - if video_length < length: - loops = int(length/video_length) - new_filename = tempfile.mkstemp(suffix=".mp4")[1] - #edits = calcEdits(video_length,loops) - out = subprocess.call(["ffmpeg","-stream_loop",str(loops),"-i",filename,"-c","copy","-y",new_filename],stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT) - #subprocess.run(["./MP4Box", "-add",filename,"-edits",f'1={edits}',new_filename]) - return new_filename - else: + p_ffmpeg.stdout.close() + ret_gifski = p_gifski.wait() + ret_ffmpeg = p_ffmpeg.wait() + if ret_gifski != 0: + print("err: gifski exited with code:", ret_gifski) + if ret_ffmpeg != 0: + print("err: ffmpeg exited with code:", ret_ffmpeg) + if os.path.isfile(new_filename) and os.path.getsize(new_filename) > 0: + return new_filename + else: + print("gifski failed to convert gif") + return filename + except Exception as e: + print("error converting gif (convert_video_to_gif):") + print(e) + return filename + +def convert_video_to_avif(filename): + try: + fd,new_filename = tempfile.mkstemp(suffix=".avif") + os.close(fd) + print("converting gif w ffmpeg to avif") + subprocess.call(["ffmpeg","-nostdin","-y", "-i", filename,"-pix_fmt","yuv420p","-an","-c:v","libsvtav1","-crf","30","-b:v","0","-threads","4", new_filename], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + if os.path.isfile(new_filename) and os.path.getsize(new_filename) > 0: + return new_filename + else: + print("ffmpeg failed to convert avif") + return filename + except Exception as e: + print("error converting avif (convert_video_to_avif):") + print(e) return filename - +def redir(url): + return { + "statusCode": 307, + "headers": { + "Location": url + } + } def lambda_handler(event, context): if ("queryStringParameters" not in event): return { "statusCode": 400, - "body": "Invalid request." + "body": "Invalid request!" } url = event["queryStringParameters"].get("url","") - - # download video - videoLocation = tempfile.mkstemp(suffix=".mp4")[1] - subprocess.call(["wget","-O",videoLocation,url],stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT) - - videoLocationLooped = loop_video_until_length(videoLocation, 30) - if videoLocationLooped != videoLocation: - os.remove(videoLocation) - videoLocation = videoLocationLooped - - with open(videoLocation, "rb") as image_file: - encoded_string = base64.b64encode(image_file.read()).decode('ascii') - os.remove(videoLocation) - return { - 'statusCode': 200, - "headers": - { - "Content-Type": "video/mp4" - }, - 'body': encoded_string, - 'isBase64Encoded': True - } \ No newline at end of file + try: + if url == "": + return { + "statusCode": 400, + "body": "Invalid request!!" + } + if not url.startswith("https://video.twimg.com/tweet_video/"): + return redir(url) + + if useBucket: + id=re.search(r"https:\/\/video\.twimg\.com\/tweet_video\/(.*?)\..*",url).group(1) + bfilename = str(id)+".avif" + furl=f"https://gifs.vxtwitter.com/{bfilename}" #f"https://{bucketname}.s3.amazonaws.com/{bfilename}" + print("get req for: "+url) + try: + s3.head_object(Bucket=bucketname, Key=bfilename) + print("found existing already: "+bfilename) + return redir(furl) + except botocore.exceptions.ClientError: + # Not found + pass + # download video + print("downloading: "+url) + videoLocation = tempfile.mkstemp(suffix=".mp4")[1] + response = requests.get(url, stream=True) + if response.status_code == 200: + with open(videoLocation, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + else: + print("error downloading video") + return redir(url) + + videoLocationLooped = convert_video_to_avif(videoLocation) + if videoLocationLooped != videoLocation: + os.remove(videoLocation) + videoLocation = videoLocationLooped + else: + os.remove(videoLocation) + return redir(url) + + if not useBucket: + with open(videoLocation, "rb") as image_file: + encoded_string = base64.b64encode(image_file.read()).decode('ascii') + os.remove(videoLocation) + return { + 'statusCode': 200, + "headers": + { + "Content-Type": "image/avif", + "Cache-Control": "public, max-age=604800, immutable" + }, + 'body': encoded_string, + 'isBase64Encoded': True + } + else: + with open(videoLocation, "rb") as image_file: + s3.upload_fileobj(image_file, bucketname, bfilename) + os.remove(videoLocation) + print("converted: "+url+" -> "+furl) + return redir(furl) + except Exception as e: + print("error converting gif: ") + print(e) + return redir(url) \ No newline at end of file diff --git a/gifConvert/conv.sh b/gifConvert/conv.sh deleted file mode 100644 index e24dc04..0000000 --- a/gifConvert/conv.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -e - -usage(){ - echo "Usage: $0 [options] output" - echo "Options:" - echo " --help Show this help" - echo " -u, --url URL of the video" - echo " -w, --max-width Maximum width of the output" - echo " -h, --max-height Maximum height of the output" - echo " -t, --threads Number of threads to use" - exit 1 -} - -URL="" -MAXW=400 -MAXH=267 -THREADS=1 -OUTPUT="out.gif" -FPS=10 - -while [ $# -gt 0 ]; do - case "$1" in - --help) - usage - ;; - -u|--url) - URL="$2" - shift - ;; - -w|--max-width) - MAXW="$2" - shift - ;; - -h|--max-height) - MAXH="$2" - shift - ;; - -t|--threads) - THREADS="$2" - shift - ;; - -f|--fps) - FPS="$2" - shift - ;; - -*) - echo "Unknown option: $1" - usage - ;; - *) - OUTPUT="$1" - ;; - esac - shift -done - -# make unique temp directory -TEMPDIR=$( mktemp -d ) - -./ffmpeg -i "$URL" -vf "scale=if(gte(iw\,ih)\,min($MAXW\,iw)\,-2):if(lt(iw\,ih)\,min($MAXH\,ih)\,-2)" -threads $THREADS "$TEMPDIR/frame%04d.png" -./gifski -o "$TEMPDIR/out.gif" --fast --fps $FPS --quality=90 $TEMPDIR/frame*.png -#./gifsicle -O3 "$TEMPDIR/out.gif" -o "$OUTPUT" -mv "$TEMPDIR/out.gif" "$OUTPUT" -rm -rf "$TEMPDIR" \ No newline at end of file diff --git a/msgs.py b/msgs.py index 8bcd17c..c65424d 100644 --- a/msgs.py +++ b/msgs.py @@ -17,8 +17,11 @@ def genLikesDisplay(vnf): return ("💖 " + numerize.numerize(vnf['likes'])) def genQrtDisplay(qrt): + text = qrt['text'] + if "translation" in qrt and qrt["translation"] is not None: + text = qrt["translation"]["text"] verifiedCheck = "☑️" if ('verified' in qrt and qrt['verified']) else "" - return ("\n\n【QRT of " + qrt['user_name'] + " (@" + qrt['user_screen_name'] + ")"+ verifiedCheck+":】\n\n'" + qrt['text'] + "'") + return ("\n\n【QRT of " + qrt['user_name'] + " (@" + qrt['user_screen_name'] + ")"+ verifiedCheck+":】\n\n'" + text + "'") def genPollDisplay(poll): pctSplit=10 @@ -27,6 +30,9 @@ def genPollDisplay(poll): output+=choice["name"]+"\n"+("█"*int(choice["percent"]/pctSplit)) +" "+str(choice["percent"])+"%\n" return output +def genTranslationDisplay(translation): + return f"【TL "+translation["source_language"]+"→"+translation["destination_language"]+"】\n\n"+translation["text"] + # formats the top text of the embed def formatProvider(base,vnf): finalText = base @@ -38,13 +44,16 @@ def formatProvider(base,vnf): finalText = base return finalText -def formatEmbedDesc(type,body,qrt,pollData): +def formatEmbedDesc(type,body,qrt,pollData,translation): # Trim the embed description to 248 characters, prioritizing poll and likes qrtType=None if qrt!=None: qrtType="Text" + if translation is not None: + body = genTranslationDisplay(translation) + limit = videoDescLimit if type=="Video" or (qrt!=None and (qrtType=="Video")) else tweetDescLimit output = "" @@ -73,6 +82,6 @@ def formatEmbedDesc(type,body,qrt,pollData): diff = len(output)-limit # remove the characters from body, add ellipsis body = body[:-(diff+1)]+"…" - return formatEmbedDesc(type,body,qrt,pollData) + return formatEmbedDesc(type,body,qrt,pollData,None) else: return output diff --git a/requirements.txt b/requirements.txt index ef9f74d..b16d9f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,5 @@ Werkzeug==2.3.7 numerize==0.12 oauthlib==3.2.2 PyRTF3==0.47.5 -XClientTransaction==0.0.2 \ No newline at end of file +XClientTransaction==1.0.2 +deepl==1.3.0 \ No newline at end of file diff --git a/serverless.yml b/serverless.yml index 990545a..b5d8ce1 100644 --- a/serverless.yml +++ b/serverless.yml @@ -29,6 +29,7 @@ provider: VXTWITTER_COMBINATION_METHOD: ${env:VXTWITTER_COMBINATION_METHOD, 'local'} VXTWITTER_GIF_CONVERT_API: ${env:VXTWITTER_GIF_CONVERT_API, ''} VXTWITTER_WORKAROUND_TOKENS: ${env:VXTWITTER_WORKAROUND_TOKENS, ''} + DEEPL_API_KEY: ${env:DEEPL_API_KEY, ''} #VXTWITTER_PROXIES: ${env:VXTWITTER_PROXIES, ''} package: diff --git a/test_vx_embeds.py b/test_vx_embeds.py index 56e0718..a28a63b 100644 --- a/test_vx_embeds.py +++ b/test_vx_embeds.py @@ -204,4 +204,20 @@ def test_embed_action(): resp = client.get(testTextTweet.replace("https://twitter.com",""),headers={"User-Agent":"Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)"}) assert resp.status_code==200 assert "application/activity+json" in str(resp.data) - assert "%F0%9F%92%96" not in str(resp.data) # 💖 \ No newline at end of file + assert "%F0%9F%92%96" not in str(resp.data) # 💖 + +def test_embed_translated(): + resp = client.get(testTextTweet.replace("https://twitter.com","")+"/jp",headers={"User-Agent":"test"}) + assert resp.status_code==200 + assert "→" in resp.text + resp = client.get(testTextTweet.replace("https://twitter.com","")+"/ja",headers={"User-Agent":"test"}) + assert resp.status_code==200 + assert "→" in resp.text + +def test_embed_translated_subdomain(): + resp = client.get(testTextTweet.replace("https://twitter.com","https://jp.vxtwitter.com"),headers={"User-Agent":"test"}) + assert resp.status_code==200 + assert "→" in resp.text + resp = client.get(testTextTweet.replace("https://twitter.com","https://ja.vxtwitter.com"),headers={"User-Agent":"test"}) + assert resp.status_code==200 + assert "→" in resp.text \ No newline at end of file diff --git a/test_vx_extract.py b/test_vx_extract.py index f72afef..bbc8119 100644 --- a/test_vx_extract.py +++ b/test_vx_extract.py @@ -20,6 +20,10 @@ def test_twextract_extractStatusV2Rest(): assert utils.stripEndTCO(tweet["full_text"])[:94]==testMultiMediaTweet_compare['text'][:94] +def test_twextract_v2API_Anon(): + tweet = twExtract.extractStatusV2Rest_Anon(testMediaTweet,workaroundTokens=tokens)['legacy'] + assert utils.stripEndTCO(tweet["full_text"])==testMediaTweet_compare['text'] + def test_twextract_v2API(): tweet = twExtract.extractStatusV2(testMediaTweet,workaroundTokens=tokens)['legacy'] assert utils.stripEndTCO(tweet["full_text"])==testMediaTweet_compare['text'] diff --git a/translation.py b/translation.py new file mode 100644 index 0000000..939c236 --- /dev/null +++ b/translation.py @@ -0,0 +1,27 @@ +import deepl +from configHandler import config + +validLanguages = ['AF', 'AN', 'AR', 'AS', 'AY', 'AZ', 'BA', 'BE', 'BG', 'BN', 'BR', 'BS', 'CA', 'CS', 'CY', 'DA', 'DE', 'DE-DE', 'EL', 'EN', 'EO', 'ES', 'ES-419', 'ET', 'EU', 'FA', 'FI', 'FR', 'FR-FR', 'GA', 'GL', 'GN', 'GU', 'HA', 'HE', 'HI', 'HR', 'HT', 'HU', 'HY', 'ID', 'IG', 'IS', 'IT', 'JA', 'JV', 'KA', 'KK', 'KO', 'KY', 'LA', 'LB', 'LN', 'LT', 'LV', 'MG', 'MI', 'MK', 'ML', 'MN', 'MR', 'MS', 'MT', 'MY', 'NB', 'NE', 'NL', 'OC', 'OM', 'PA', 'PL', 'PS', 'PT-BR', 'PT-PT', 'QU', 'RO', 'RU', 'SA', 'SK', 'SL', 'SQ', 'SR', 'ST', 'SU', 'SV', 'SW', 'TA', 'TE', 'TG', 'TH', 'TK', 'TL', 'TN', 'TR', 'TS', 'TT', 'UK', 'UR', 'UZ', 'VI', 'WO', 'XH', 'YI', 'ZH', 'ZH-HANS', 'ZH-HANT', 'ZU'] +languageAliases = { + #"EN":"EN-US", + "JP":"JA", +} + +mergedLangs=(validLanguages+list(languageAliases.keys())) + +def getDeeplTranslation(text,target): + target = target.upper() + if target == "EN": + target = "EN-US" + try: + if 'deeplKey' not in config["config"] or config["config"]["deeplKey"] == None: + return None + deepl_client = deepl.Translator(config["config"]["deeplKey"]) + result = deepl_client.translate_text(text, target_lang=target) + return { + "text":result.text, + "source_language":result.detected_source_lang, + "destination_language":target + } + except Exception as e: + return None \ No newline at end of file diff --git a/twExtract/__init__.py b/twExtract/__init__.py index 88f1469..5122b0b 100644 --- a/twExtract/__init__.py +++ b/twExtract/__init__.py @@ -14,7 +14,7 @@ bearer="Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DT v2bearer="Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" androidBearer="Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F" -requestUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0" +requestUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0" bearerTokens=[bearer,v2bearer,androidBearer] @@ -27,15 +27,15 @@ userIDregex = r"\/i\/user\/(\d+)" v2Features='{"longform_notetweets_inline_media_enabled":true,"super_follow_badge_privacy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"super_follow_user_api_enabled":true,"super_follow_tweet_api_enabled":true,"android_graphql_skip_api_media_color_palette":true,"creator_subscriptions_tweet_preview_api_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"creator_subscriptions_subscription_count_enabled":true,"tweetypie_unmention_optimization_enabled":true,"longform_notetweets_consumption_enabled":true,"subscriptions_verification_info_enabled":true,"blue_business_profile_image_shape_enabled":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"super_follow_exclusive_tweet_notifications_enabled":true}' v2graphql_api="2OOZWmw8nAtUHVnXXQhgaA" -v2AnonFeatures='{"creator_subscriptions_tweet_preview_api_enabled":true,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":false,"responsive_web_jetfuel_frame":true,"responsive_web_grok_share_attachment_enabled":true,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":false,"responsive_web_grok_analysis_button_from_backend":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"payments_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"verified_phone_label_enabled":false,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_grok_imagine_annotation_enabled":true,"responsive_web_grok_community_note_auto_translation_is_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false}' -v2AnonGraphql_api="wqi5M7wZ7tW-X9S2t-Mqcg" +v2AnonFeatures='{"creator_subscriptions_tweet_preview_api_enabled":true,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":true,"responsive_web_grok_share_attachment_enabled":true,"responsive_web_grok_annotations_enabled":false,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":true,"responsive_web_grok_analysis_button_from_backend":true,"post_ctas_fetch_enabled":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"profile_label_improvements_pcf_label_in_post_enabled":true,"responsive_web_profile_redirect_enabled":false,"rweb_tipjar_consumption_enabled":false,"verified_phone_label_enabled":false,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_grok_imagine_annotation_enabled":true,"responsive_web_grok_community_note_auto_translation_is_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false}' +v2AnonGraphql_api="0aTrQMKgj95K791yXeNDRA" gt_pattern = r'document\.cookie="gt=([^;]+);' -androidGraphqlFeatures='{"grok_translations_community_note_translation_is_enabled":false,"super_follow_badge_privacy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"super_follow_user_api_enabled":true,"profile_label_improvements_pcf_label_in_profile_enabled":true,"premium_content_api_read_enabled":false,"grok_translations_community_note_auto_translation_is_enabled":false,"android_graphql_skip_api_media_color_palette":true,"tweetypie_unmention_optimization_enabled":true,"longform_notetweets_consumption_enabled":true,"subscriptions_verification_info_enabled":true,"blue_business_profile_image_shape_enabled":true,"super_follow_exclusive_tweet_notifications_enabled":true,"longform_notetweets_inline_media_enabled":true,"grok_android_analyze_trend_fetch_enabled":false,"unified_cards_ad_metadata_container_dynamic_card_content_query_enabled":true,"super_follow_tweet_api_enabled":true,"articles_api_enabled":true,"creator_subscriptions_tweet_preview_api_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"grok_translations_timeline_user_bio_auto_translation_is_enabled":false,"grok_translations_post_auto_translation_is_enabled":false,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"immersive_video_status_linkable_timestamps":true,"profile_label_improvements_pcf_label_in_post_enabled":true}' -androidGraphql_api="k3rtLsS9kG5hI-Jr0dTMCg" +androidGraphqlFeatures='{"grok_translations_community_note_translation_is_enabled":false,"super_follow_badge_privacy_enabled":true,"unified_cards_destination_url_params_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"super_follow_user_api_enabled":true,"profile_label_improvements_pcf_label_in_profile_enabled":true,"premium_content_api_read_enabled":false,"grok_translations_community_note_auto_translation_is_enabled":false,"android_graphql_skip_api_media_color_palette":true,"tweetypie_unmention_optimization_enabled":true,"longform_notetweets_consumption_enabled":true,"subscriptions_verification_info_enabled":true,"blue_business_profile_image_shape_enabled":true,"super_follow_exclusive_tweet_notifications_enabled":true,"longform_notetweets_inline_media_enabled":true,"grok_android_analyze_trend_fetch_enabled":false,"unified_cards_ad_metadata_container_dynamic_card_content_query_enabled":true,"super_follow_tweet_api_enabled":true,"articles_api_enabled":true,"android_ad_formats_media_component_render_overlay_enabled":true,"creator_subscriptions_tweet_preview_api_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"grok_translations_timeline_user_bio_auto_translation_is_enabled":false,"grok_translations_post_auto_translation_is_enabled":false,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"immersive_video_status_linkable_timestamps":true,"profile_label_improvements_pcf_label_in_post_enabled":true}' +androidGraphql_api="Wrspae-uyGj-nWPyUqdUag" -tweetDetailGraphqlFeatures='{"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"articles_preview_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"rweb_video_timestamps_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_enhance_cards_enabled":false}' -tweetDetailGraphql_api="YVyS4SfwYW7Uw5qwy0mQCA" +tweetDetailGraphqlFeatures='{"rweb_video_screen_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"responsive_web_profile_redirect_enabled":false,"rweb_tipjar_consumption_enabled":false,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":true,"responsive_web_grok_share_attachment_enabled":true,"responsive_web_grok_annotations_enabled":false,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":true,"responsive_web_grok_analysis_button_from_backend":true,"post_ctas_fetch_enabled":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_grok_imagine_annotation_enabled":true,"responsive_web_grok_community_note_auto_translation_is_enabled":false,"responsive_web_enhance_cards_enabled":false}' +tweetDetailGraphql_api="Kzfv17rukSzjT96BerOWZA" # this is for UserTweets endpoint tweetFeedGraphqlFeatures='{"rweb_video_screen_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":false,"responsive_web_grok_share_attachment_enabled":true,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":false,"responsive_web_grok_analysis_button_from_backend":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_enhance_cards_enabled":false}' @@ -113,7 +113,7 @@ def cycleBearerTokenGet(url,headers): return tweet raise TwExtractError(400, "Extract error") -def twitterApiGet(url,btoken=None,authToken=None,guestToken=None): +def twitterApiGet(url,btoken=None,authToken=None,guestToken=None,userAgent=None,language=None): if authToken != None and authToken.startswith("oa|"): url = url.replace("https://x.com/i/api/graphql/","https://api.twitter.com/graphql/") @@ -122,9 +122,11 @@ def twitterApiGet(url,btoken=None,authToken=None,guestToken=None): secret = authToken.split("|")[1] twt = oauth1.Client(client_key='3nVuSoBZnx6U4vzUxf5w',client_secret='Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys',resource_owner_key=key,resource_owner_secret=secret) - hdr = getAuthHeaders(androidBearer) + hdr = getAuthHeaders(androidBearer,language=language) del hdr["Authorization"] hdr["X-Twitter-Client"] = "TwitterAndroid" + if userAgent is not None: + hdr["User-Agent"] = userAgent uri, headers, body = twt.sign(url, headers=hdr,realm="http://api.twitter.com/") @@ -133,12 +135,13 @@ def twitterApiGet(url,btoken=None,authToken=None,guestToken=None): if btoken is None: btoken = v2bearer #return cycleBearerTokenGet(url,getAuthHeaders(bearer,authToken=authToken,guestToken=guestToken)) - headers = getAuthHeaders(btoken,authToken=authToken,guestToken=guestToken) + headers = getAuthHeaders(btoken,authToken=authToken,guestToken=guestToken,language=language) response = requests.get(url, headers=headers) return response -def getAuthHeaders(btoken,authToken=None,guestToken=None): +def getAuthHeaders(btoken,authToken=None,guestToken=None,language=None): + csrfToken=str(uuid.uuid4()).replace('-', '') headers = {"x-twitter-active-user":"yes","x-twitter-client-language":"en","x-csrf-token":csrfToken,"User-Agent":requestUserAgent} headers['Authorization'] = btoken @@ -149,19 +152,17 @@ def getAuthHeaders(btoken,authToken=None,guestToken=None): if guestToken is not None: headers["x-guest-token"] = guestToken + if language is not None: + headers["x-twitter-client-language"] = language + return headers def getGuestToken(): global guestToken global guestTokenUses if guestToken is None: - r = requests.get(f"https://{twitterUrl}",headers={"User-Agent":requestUserAgent,"Cookie":"night_mode=2"},allow_redirects=False) - m = re.search(gt_pattern, r.text) - if m is None: - r = requests.post(f"https://api.{twitterUrl}/1.1/guest/activate.json", headers={"Authorization":bearer}) - guestToken = json.loads(r.text)["guest_token"] - else: - guestToken = m.group(1) + r = requests.post(f"https://api.{twitterUrl}/1.1/guest/activate.json", headers={"Authorization":v2bearer,"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0"}) + guestToken = json.loads(r.text)["guest_token"] guestTokenUses = 0 else: guestTokenUses+=1 @@ -213,7 +214,7 @@ def extractStatus_guestToken(url): raise TwExtractError(error["code"], error["message"]) return output -def extractStatus_syndication(url,workaroundTokens=None): +def extractStatus_syndication(url,workaroundTokens=None,tlLanguage=None): # https://github.com/mikf/gallery-dl/blob/46cae04aa3a113c7b6bbee1bb468669564b14ae8/gallery_dl/extractor/twitter.py#L1784 m = re.search(pathregex, url) if m is None: @@ -265,7 +266,7 @@ def extractStatus_twExtractProxy(url,workaroundTokens=None): continue return output -def extractStatusV2(url,workaroundTokens): +def extractStatusV2(url,workaroundTokens,tlLanguage=None): # get tweet ID m = re.search(pathregex, url) if m is None: @@ -279,7 +280,7 @@ def extractStatusV2(url,workaroundTokens): def request_with_token(twid, authToken): vars = json.loads('{"includeTweetImpression":true,"includeHasBirdwatchNotes":false,"includeEditPerspective":false,"rest_ids":["x"],"includeEditControl":true,"includeCommunityTweetRelationship":true,"includeTweetVisibilityNudge":true}') vars['rest_ids'][0] = str(twid) - tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2graphql_api}/TweetResultsByIdsQuery?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2Features)}",authToken=authToken) + tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2graphql_api}/TweetResultsByIdsQuery?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2Features)}",authToken=authToken,language=tlLanguage) try: rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining") print(f"Twitter Token Rate limit remaining: {rateLimitRemaining}") @@ -318,7 +319,7 @@ def extractStatusV2(url,workaroundTokens): return tweet return parallel_token_request(twid, tokens, request_with_token) -def extractStatusV2Android(url,workaroundTokens): +def extractStatusV2Android(url,workaroundTokens,tlLanguage=None): # get tweet ID m = re.search(pathregex, url) if m is None: @@ -332,7 +333,7 @@ def extractStatusV2Android(url,workaroundTokens): try: vars = json.loads('{"referrer":"home","includeTweetImpression":true,"includeHasBirdwatchNotes":false,"isReaderMode":false,"includeEditPerspective":false,"includeEditControl":true,"focalTweetId":0,"includeCommunityTweetRelationship":true,"includeTweetVisibilityNudge":true}') vars['focalTweetId'] = int(twid) - tweet = twitterApiGet(f"https://x.com/i/api/graphql/{androidGraphql_api}/ConversationTimelineV2?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(androidGraphqlFeatures)}", authToken=authToken,btoken=androidBearer) + tweet = twitterApiGet(f"https://x.com/i/api/graphql/{androidGraphql_api}/ConversationTimelineV2?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(androidGraphqlFeatures)}", authToken=authToken,btoken=androidBearer,userAgent="TwitterAndroid/11.61.0-release.0 (311610000-r-0) G011A/9 (google;G011A;google;G011A;0;;1;2016)",language=tlLanguage) try: rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining") print(f"Twitter Android Token Rate limit remaining: {rateLimitRemaining}") @@ -377,7 +378,7 @@ def extractStatusV2Android(url,workaroundTokens): return tweet return parallel_token_request(twid, tokens, request_with_token) -def extractStatusV2TweetDetail(url,workaroundTokens): +def extractStatusV2TweetDetail(url,workaroundTokens,tlLanguage=None): # get tweet ID m = re.search(pathregex, url) if m is None: @@ -392,7 +393,7 @@ def extractStatusV2TweetDetail(url,workaroundTokens): try: vars = json.loads('{"focalTweetId":"0","with_rux_injections":false,"includePromotedContent":true,"withCommunity":true,"withQuickPromoteEligibilityTweetFields":true,"withBirdwatchNotes":true,"withVoice":true,"withV2Timeline":true}') vars['focalTweetId'] = str(twid) - tweet = twitterApiGet(f"https://x.com/i/api/graphql/{tweetDetailGraphql_api}/TweetDetail?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(tweetDetailGraphqlFeatures)}", authToken=authToken,btoken=v2bearer) + tweet = twitterApiGet(f"https://x.com/i/api/graphql/{tweetDetailGraphql_api}/TweetDetail?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(tweetDetailGraphqlFeatures)}", authToken=authToken,btoken=v2bearer,language=tlLanguage) try: rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining") print(f"Twitter Token Rate limit remaining: {rateLimitRemaining}") @@ -437,10 +438,10 @@ def extractStatusV2TweetDetail(url,workaroundTokens): return tweet return parallel_token_request(twid, tokens, request_with_token) -def extractStatusV2Rest_Anon(url,workaroundTokens): - return extractStatusV2Rest(url,None) +def extractStatusV2Rest_Anon(url,workaroundTokens,tlLanguage=None): + return extractStatusV2Rest(url,None,tlLanguage=tlLanguage) -def extractStatusV2Rest(url,workaroundTokens): +def extractStatusV2Rest(url,workaroundTokens,tlLanguage=None): # get tweet ID m = re.search(pathregex, url) if m is None: @@ -451,18 +452,18 @@ def extractStatusV2Rest(url,workaroundTokens): # get tweet try: - vars = json.loads('{"tweetId":"0","withCommunity":false,"includePromotedContent":false,"withVoice":false}') + vars = json.loads('{"tweetId":"0","includePromotedContent":true,"withBirdwatchNotes":true,"withVoice":true,"withCommunity":true}') vars['tweetId'] = str(twid) if workaroundTokens is not None and len(workaroundTokens) > 0: tokens = workaroundTokens random.shuffle(tokens) for authToken in tokens: try: - tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", btoken=v2bearer,authToken=authToken,guestToken=guestToken) + tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", btoken=v2bearer,authToken=authToken,guestToken=guestToken,language=tlLanguage) except Exception as e: continue else: - tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", btoken=v2bearer,guestToken=guestToken) + tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", btoken=v2bearer,guestToken=guestToken,language=tlLanguage) try: rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining") @@ -509,12 +510,13 @@ def fixTweetData(tweet): pass return tweet -def extractStatus(url,workaroundTokens=None): - # TODO: commented out methods are too slow/unreliable at the moment +def extractStatus(url,workaroundTokens=None,tlLanguage=None): methods=[extractStatusV2Rest_Anon,extractStatusV2,extractStatusV2Rest,extractStatusV2Android]#,extractStatusV2TweetDetail] + if tlLanguage is not None: # prioritize endpoints that return translations + methods = [extractStatusV2Rest,extractStatusV2TweetDetail,extractStatusV2Android,extractStatusV2,extractStatusV2Rest_Anon] for method in methods: try: - result = method(url,workaroundTokens) + result = method(url,workaroundTokens,tlLanguage=tlLanguage) if 'legacy' not in result: print(f"{method.__name__} method failed: Legacy not found for {url}") # try another method diff --git a/twExtract/twUtils.py b/twExtract/twUtils.py index 20f72ae..82a5b5f 100644 --- a/twExtract/twUtils.py +++ b/twExtract/twUtils.py @@ -3,8 +3,9 @@ import hashlib import base64 import uuid from x_client_transaction import ClientTransaction -from x_client_transaction.utils import handle_x_migration +from x_client_transaction.utils import get_ondemand_file_url, generate_headers import requests +import bs4 digits = "0123456789abcdefghijklmnopqrstuvwxyz" def baseConversion(x, base): @@ -49,6 +50,14 @@ def get_twitter_homepage(headers=None): return response def generate_transaction_id(method: str, path: str,headers=None) -> str: - ct = ClientTransaction(get_twitter_homepage(headers=headers)) + session = requests.Session() + session.headers = generate_headers() + home_page = session.get(url="https://x.com/home") + home_page_response = bs4.BeautifulSoup(home_page.content, 'html.parser') + ondemand_file_url = get_ondemand_file_url(response=home_page_response) + ondemand_file = session.get(url=ondemand_file_url) + ondemand_file_response = bs4.BeautifulSoup(ondemand_file.content, 'html.parser') + + ct = ClientTransaction(home_page_response=home_page_response, ondemand_file_response=ondemand_file_response) transaction_id = ct.generate_transaction_id(method=method, path=path) return transaction_id \ No newline at end of file diff --git a/twitfix.py b/twitfix.py index 1bf8204..66e5cb8 100644 --- a/twitfix.py +++ b/twitfix.py @@ -14,7 +14,7 @@ import msgs import twExtract as twExtract from cache import addVnfToLinkCache,getVnfFromLinkCache import vxlogging as log -from utils import getTweetIdFromUrl, pathregex, determineMediaToEmbed, determineEmbedTweet, BytesIOWrapper, fixMedia +from utils import getTweetIdFromUrl, pathregex, determineMediaToEmbed, determineEmbedTweet, BytesIOWrapper, fixMedia, indexOfAny from vxApi import getApiResponse, getApiUserResponse from urllib.parse import urlparse from PyRTF.Elements import Document @@ -22,12 +22,14 @@ from PyRTF.document.section import Section from PyRTF.document.paragraph import Paragraph from copy import deepcopy import json -import datetime +import translation import activity as activitymg app = Flask(__name__) CORS(app) user_agent="" +activityseparator="zqj" + staticFiles = { # TODO: Use flask static files instead of this "favicon.ico": {"mime": "image/vnd.microsoft.icon","path": "favicon.ico"}, "apple-touch-icon.png": {"mime": "image/png","path": "apple-touch-icon.png"}, @@ -82,7 +84,11 @@ def generateActivityLink(tweetData,media=None,mediatype=None,embedIndex=-1): return None try: embedIndex = embedIndex+1 - return f"{config['config']['url']}/users/{tweetData['user_screen_name']}/statuses/{str(embedIndex)}{tweetData['tweetID']}" + + extras="" + if tweetData["translation"] is not None: + extras+=activityseparator+tweetData["translation"]["destination_language"] + return f"{config['config']['url']}/users/{tweetData['user_screen_name']}/statuses/{str(embedIndex)}{activityseparator}{tweetData['tweetID']}{extras}" except Exception as e: log.error("Error generating activity link: "+str(e)) return None @@ -95,7 +101,7 @@ def getAppName(tweetData,appnameSuffix=""): def renderImageTweetEmbed(tweetData,image,appnameSuffix="",embedIndex=-1): qrt = tweetData['qrt'] - embedDesc = msgs.formatEmbedDesc("Image",tweetData['text'],qrt,tweetData['pollData']) + embedDesc = msgs.formatEmbedDesc("Image",tweetData['text'],qrt,tweetData['pollData'],tweetData["translation"]) if image.startswith("https://pbs.twimg.com") and "?" not in image: image = f"{image}?name=orig" @@ -115,7 +121,7 @@ def renderImageTweetEmbed(tweetData,image,appnameSuffix="",embedIndex=-1): def renderVideoTweetEmbed(tweetData,mediaInfo,appnameSuffix="",embedIndex=-1): qrt = tweetData['qrt'] - embedDesc = msgs.formatEmbedDesc("Video",tweetData['text'],qrt,tweetData['pollData']) + embedDesc = msgs.formatEmbedDesc("Video",tweetData['text'],qrt,tweetData['pollData'],tweetData["translation"]) mediaInfo=fixMedia(mediaInfo) @@ -138,7 +144,7 @@ def renderVideoTweetEmbed(tweetData,mediaInfo,appnameSuffix="",embedIndex=-1): def renderTextTweetEmbed(tweetData,appnameSuffix=""): qrt = tweetData['qrt'] - embedDesc = msgs.formatEmbedDesc("Text",tweetData['text'],qrt,tweetData['pollData']) + embedDesc = msgs.formatEmbedDesc("Text",tweetData['text'],qrt,tweetData['pollData'],tweetData["translation"]) return render_template("text.html", tweet=tweetData, @@ -154,7 +160,7 @@ def renderTextTweetEmbed(tweetData,appnameSuffix=""): def renderArticleTweetEmbed(tweetData,appnameSuffix=""): articlePreview=tweetData['article']["title"]+"\n\n"+tweetData['article']["preview_text"]+"…" - embedDesc = msgs.formatEmbedDesc("Image",articlePreview,None,None) + embedDesc = msgs.formatEmbedDesc("Image",articlePreview,None,None,None) return render_template("image.html", tweet=tweetData, @@ -260,9 +266,9 @@ def userJson(): }, } -def getTweetData(twitter_url,include_txt="false",include_rtf="false"): +def getTweetData(twitter_url,include_txt="false",include_rtf="false",tlLanguage=None): cachedVNF = getVnfFromLinkCache(twitter_url) - if cachedVNF is not None and include_txt == "false" and include_rtf == "false": + if cachedVNF is not None and include_txt == "false" and include_rtf == "false" and tlLanguage is None: return cachedVNF try: @@ -270,7 +276,7 @@ def getTweetData(twitter_url,include_txt="false",include_rtf="false"): workaroundTokens = config['config']['workaroundTokens'].split(",") else: workaroundTokens = None - rawTweetData = twExtract.extractStatus(twitter_url,workaroundTokens=workaroundTokens) + rawTweetData = twExtract.extractStatus(twitter_url,workaroundTokens=workaroundTokens,tlLanguage=tlLanguage) except: rawTweetData = None if rawTweetData == None or 'error' in rawTweetData: @@ -281,7 +287,15 @@ def getTweetData(twitter_url,include_txt="false",include_rtf="false"): tweetData = getApiResponse(rawTweetData,include_txt,include_rtf) if tweetData is None: return None - if include_txt == "false" and include_rtf == "false": + + if tweetData["translation"] is None and tlLanguage is not None and tlLanguage != tweetData["lang"]: + # translation probably failed on twitter's end + tweetData["translation"] = translation.getDeeplTranslation(tweetData["text"],tlLanguage) + + if tweetData["translation"] is not None and tlLanguage is None: + tweetData["translation"] = None # didn't ask for TL + + if include_txt == "false" and include_rtf == "false" and tlLanguage is None: addVnfToLinkCache(twitter_url,tweetData) return tweetData @@ -307,13 +321,19 @@ def getUserData(twitter_url,includeFeed=False): @app.route('/