Merge pull request 'YOLO UPDATE' (#3) from update into alterware

Reviewed-on: #3

.github/workflows/deploy.yml

@@ -70,4 +70,5 @@ jobs:
           VXTWITTER_GIF_CONVERT_API: ${{ secrets.VXTWITTER_GIF_CONVERT_API }}
           VXTWITTER_WORKAROUND_TOKENS: ${{ secrets.VXTWITTER_WORKAROUND_TOKENS }}
           VXTWITTER_PROXIES: ${{ secrets.VXTWITTER_PROXIES }}
-          SERVERLESS_ACCESS_KEY: ${{ secrets.SERVERLESS_ACCESS_KEY }}
+          SERVERLESS_ACCESS_KEY: ${{ secrets.SERVERLESS_ACCESS_KEY }}
+          DEEPL_API_KEY: ${{ secrets.DEEPL_API_KEY }}

.github/workflows/python-tests.yml

@@ -6,23 +6,30 @@ jobs:
   build:
     runs-on: ubuntu-latest
     environment: test
-    strategy:
-      matrix:
-        python-version: ["3.10", "3.11", "3.12"]
-
     steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
+      - name: Check out files
+        uses: actions/checkout@v4
+
+      - name: Install Python
+        run: |
+          apt-get update
+          apt-get install -y python3 python3-pip python3-venv
+
+      - name: Set up virtual environment
+        run: |
+          python3 -m venv venv
+          . venv/bin/activate
+          python -m pip install --upgrade pip
+
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          . venv/bin/activate
           pip install pytest pytest-cov pytest-env
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
       - name: Test with pytest
         env:
           VXTWITTER_WORKAROUND_TOKENS: ${{ secrets.VXTWITTER_WORKAROUND_TOKENS }}
         run: |
+          . venv/bin/activate
           pytest --cov-config=.coveragerc --cov=.

activity.py

@@ -2,17 +2,27 @@ import datetime
 import msgs
 from utils import determineEmbedTweet, determineMediaToEmbed
 from copy import deepcopy
+import html
 
 def tweetDataToActivity(tweetData,embedIndex = -1):
     content=""
 
     if tweetData['replyingTo'] is not None:
         content += f"<blockquote>↪️ <i>Replying to @{tweetData['replyingTo']}</i></blockquote>"
-    content+=f"<p>{tweetData['text']}</p>"
+
+    mainText = html.escape(tweetData['text'])
+    if tweetData["translation"] is not None:
+        content += f"<blockquote>🌐 <i>{tweetData['translation']['source_language'].upper()}→{tweetData['translation']['destination_language'].upper()}</i></blockquote>"
+        mainText=html.escape(tweetData['translation']["text"])
+        
+    content+=f"<p>{mainText}</p>"
 
     attachments=[]
     if tweetData['qrt'] is not None:
-        content += f"<blockquote><b>QRT: <a href=\"{tweetData['qrtURL']}\">{tweetData['qrt']['user_screen_name']}</a></b><br>{tweetData['qrt']['text']}</blockquote>"
+        qrtText = tweetData['qrt']['text']
+        if "translation" in tweetData['qrt'] and tweetData['qrt']["translation"] is not None:
+            qrtText = tweetData['qrt']["translation"]["text"]
+        content += f"<blockquote><b>QRT: <a href=\"{tweetData['qrtURL']}\">{tweetData['qrt']['user_screen_name']}</a></b><br>{qrtText}</blockquote>"
     if tweetData['pollData'] is not None:
         content += f"<p>{msgs.genPollDisplay(tweetData['pollData'])}</p>"
         content += "</p>"
@@ -35,7 +45,10 @@ def tweetDataToActivity(tweetData,embedIndex = -1):
     if media is not None:
         media = deepcopy(media)
         if media['type'] == "gif":
-            media['type'] = "gifv"
+            if "/convert.avif" in media['url']:
+                media['type'] = "image"
+            else:
+                media['type'] = "gifv"
         if 'thumbnail_url' not in media:
             media['thumbnail_url'] = media['url']
         if media['type'] == "image" and "?" not in media['url']:

config.json

@@ -2,14 +2,14 @@
     "config": {
         "appname": "vxTwitter",
         "color": "#1DA1F2",
-        "database": "[url to mongo database goes here]",
-        "link_cache": "ram",
+        "database": "mongodb://localhost:27017/bettervxtwitter",
+        "link_cache": "db",
         "method": "hybrid",
-        "repo": "https://github.com/dylanpdx/BetterTwitFix",
-        "table": "[database table here]",
-        "url": "https://vxtwitter.com",
+        "repo": "https://git.alterware.dev/alterware/BetterTwitFix",
+        "table": "links",
+        "url": "https://girlcock.alterware.dev",
         "combination_method": "local",
         "gifConvertAPI": "local",
-        "workaroundTokens":null
+        "workaroundTokens": null
     }
-}
+}

configHandler.py

@@ -13,7 +13,8 @@ if ('RUNNING_SERVERLESS' in os.environ and os.environ['RUNNING_SERVERLESS'] == '
                 "url": os.getenv("VXTWITTER_URL","https://vxtwitter.com"),
                 "combination_method": os.getenv("VXTWITTER_COMBINATION_METHOD","local"), # can either be 'local' or a URL to a server handling requests in the same format
                 "gifConvertAPI":os.getenv("VXTWITTER_GIF_CONVERT_API",""),
-                "workaroundTokens":os.getenv("VXTWITTER_WORKAROUND_TOKENS",None)
+                "workaroundTokens":os.getenv("VXTWITTER_WORKAROUND_TOKENS",None),
+                "deeplKey":os.getenv("DEEPL_API_KEY",None),
             }
         }
 else:
@@ -31,7 +32,8 @@ else:
                     "url": "https://vxtwitter.com",
                     "combination_method": "local", # can either be 'local' or a URL to a server handling requests in the same format
                     "gifConvertAPI":"",
-                    "workaroundTokens":None
+                    "workaroundTokens":None,
+                    "deeplKey":None
                 }
             }
 

gifConvert/Dockerfile

@@ -1,30 +1,76 @@
-FROM public.ecr.aws/lambda/python:3.8 AS builder
-RUN yum -y install git curl
-RUN yum -y groupinstall 'Development Tools'
-RUN git clone https://github.com/kohler/gifsicle
-WORKDIR gifsicle
-RUN autoreconf -i
-RUN ./configure --disable-gifview --disable-gifdiff
-RUN make
-RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
-WORKDIR /var/task
-RUN git clone https://github.com/ImageOptim/gifski
-WORKDIR gifski
-RUN /root/.cargo/bin/cargo build --release
+ARG TARGETARCH
+FROM public.ecr.aws/lambda/python:3.12 AS builder
+RUN dnf -y install git cargo && dnf clean all
+RUN git clone https://github.com/ImageOptim/gifski /gifski
+WORKDIR /gifski
+RUN cargo build --release
 
+#FROM public.ecr.aws/lambda/python:3.12 AS ffmpeg-linux-amd64
+#RUN dnf -y update
+#RUN dnf -y install git wget tar.x86_64 xz && dnf clean all
+#WORKDIR /ffmpeg
+#RUN wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz
+#RUN tar -xvf ffmpeg-release-amd64-static.tar.xz
 
-FROM public.ecr.aws/lambda/python:3.8
-RUN yum -y update
-RUN yum -y install git && yum -y install wget && yum -y install tar.x86_64 && yum -y install xz && yum clean all
-RUN wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz
-RUN tar -xvf ffmpeg-release-amd64-static.tar.xz
-RUN mv ff*/ffmpeg . && mv ff*/ffprobe . && rm *.tar.xz && rm -rf ff*/
-COPY --from=builder /var/task/gifsicle/src/gifsicle ./
-COPY --from=builder /var/task/gifski/target/release/gifski ./
+#FROM public.ecr.aws/lambda/python:3.12 AS ffmpeg-linux-arm64
+#RUN dnf -y update
+#RUN dnf -y install git wget tar xz && dnf clean all
+#WORKDIR /ffmpeg
+#RUN wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-arm64-static.tar.xz
+#RUN tar -xvf ffmpeg-release-arm64-static.tar.xz
+
+#FROM ffmpeg-linux-${TARGETARCH} AS ffmpeg
+
+FROM public.ecr.aws/lambda/python:3.12 AS ffmpeg
+# tl;dr: build ffmpeg from source with svt-av1 support and statically linked binaries
+
+# Install dependencies
+RUN dnf remove -y microdnf-dnf && microdnf install -y dnf
+RUN dnf group install "Development Tools" -y
+
+RUN mkdir /ffmpeg_sources
+WORKDIR /ffmpeg_sources
+RUN git clone https://git.ffmpeg.org/ffmpeg.git .
+RUN git clone --branch stable --depth 1 https://code.videolan.org/videolan/x264.git
+RUN git clone https://gitlab.com/AOMediaCodec/SVT-AV1.git
+
+RUN dnf install curl nasm glibc-static libstdc++-static cmake --allowerasing -y
+
+# Install x264
+WORKDIR /ffmpeg_sources/x264
+RUN PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" ./configure --prefix="/ffmpeg_build" --bindir="/bin" --enable-static
+RUN make -j
+RUN make install
+
+# Install SVT-AV1
+WORKDIR /ffmpeg_sources
+WORKDIR /ffmpeg_sources/SVT-AV1/Build
+
+RUN PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" cmake .. -G"Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DENABLE_STATIC=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_C_FLAGS_INIT="-static"
+
+RUN make -j
+RUN make install
+
+# Install FFmpeg
+WORKDIR /ffmpeg_sources
+RUN PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" ./configure --prefix="/ffmpeg_build" \
+    --pkg-config-flags="--static" --extra-ldexeflags="-static" --extra-libs="-lpthread -lm" --bindir="/bin" --disable-shared --enable-static --enable-gpl --enable-pthreads \
+    --enable-libx264 --enable-libsvtav1 --disable-ffplay
+
+RUN make -j 8
+
+FROM public.ecr.aws/lambda/python:3.12
+
+#COPY --from=builder /gifski/target/release/gifski /usr/local/bin/gifski
+#COPY --from=ffmpeg /ffmpeg/ffmpeg-*-static/ffmpeg /usr/local/bin/ffmpeg
+#COPY --from=ffmpeg /ffmpeg/ffmpeg-*-static/ffprobe /usr/local/bin/ffprobe
+COPY --from=ffmpeg /ffmpeg_sources/ffmpeg /usr/local/bin/ffmpeg
+COPY --from=ffmpeg /ffmpeg_sources/ffprobe /usr/local/bin/ffprobe
+
+RUN pip install requests==2.32.3
 
 # Copy function code
 COPY __init__.py ${LAMBDA_TASK_ROOT}/app.py
-COPY conv.sh ${LAMBDA_TASK_ROOT}/conv.sh
 
 # Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
 CMD [ "app.lambda_handler" ] 

gifConvert/__init__.py

@@ -1,100 +1,144 @@
 import base64
 import os
 import subprocess
-import json
-import sys
 import tempfile
+import urllib.request
+import re
+import botocore
+import boto3
+import requests
+
+useBucket=False
+bucketname=os.getenv('CF_BUCKET')
+s3=None
+if bucketname is None:
+    useBucket=False
+else:
+    useBucket=True
+    s3 = boto3.client('s3',endpoint_url=os.getenv('CF_ENDPOINT'),aws_access_key_id=os.getenv('CF_KEY'),aws_secret_access_key=os.getenv('CF_KEY_SECRET'))
+
 
 def extractStatus(url):
     return ""
 
-def get_video_frame_rate(filename):
-    result = subprocess.run(
-        [
-            "./ffprobe",
-            "-v",
-            "error",
-            "-select_streams",
-            "v",
-            "-of",
-            "default=noprint_wrappers=1:nokey=1",
-            "-show_entries",
-            "stream=r_frame_rate",
-            filename,
-        ],
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-    )
-    result_string = result.stdout.decode('utf-8').split()[0].split('/')
-    fps = float(result_string[0])/float(result_string[1])
-    return fps
+def convert_video_to_gif(filename):
+    try:
+        new_filename = tempfile.mkstemp(suffix=".gif")[1]
+        print("converting gif w gifski")
+        p_ffmpeg = subprocess.Popen(["ffmpeg", "-i", filename, "-f", "yuv4mpegpipe", "-"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+        p_gifski = subprocess.Popen(["gifski","--quality","70","--lossy-quality","30","-o", new_filename, "-"], stdin=p_ffmpeg.stdout, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
 
-def get_video_length_seconds(filename):
-    result = subprocess.run(
-        [
-            "./ffprobe",
-            "-v",
-            "error",
-            "-show_entries",
-            "format=duration",
-            "-of",
-            "default=noprint_wrappers=1:nokey=1",
-            filename,
-        ],
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-    )
-    result_string = result.stdout.decode('utf-8').split()[0]
-    return float(result_string)
-
-def calcEdits(vlen,loopTimes):
-    st="r"
-    for i in range(loopTimes):
-        st+=f'e{str((vlen*i))}-9999,0'
-    return st
-
-def loop_video_until_length(filename, length):
-    # use stream_loop to loop video until it's at least length seconds long
-    video_length = get_video_length_seconds(filename)
-    if video_length < length:
-        loops = int(length/video_length)
-        new_filename = tempfile.mkstemp(suffix=".mp4")[1]
-        #edits = calcEdits(video_length,loops)
-        out = subprocess.call(["ffmpeg","-stream_loop",str(loops),"-i",filename,"-c","copy","-y",new_filename],stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT)
-        #subprocess.run(["./MP4Box", "-add",filename,"-edits",f'1={edits}',new_filename])
-        return new_filename
-    else:
+        p_ffmpeg.stdout.close()
+        ret_gifski = p_gifski.wait()
+        ret_ffmpeg = p_ffmpeg.wait()
+        if ret_gifski != 0:
+            print("err: gifski exited with code:", ret_gifski)
+        if ret_ffmpeg != 0:
+            print("err: ffmpeg exited with code:", ret_ffmpeg)
+        if os.path.isfile(new_filename) and os.path.getsize(new_filename) > 0:
+            return new_filename
+        else:
+            print("gifski failed to convert gif")
+            return filename
+    except Exception as e:
+        print("error converting gif (convert_video_to_gif):")
+        print(e)
+        return filename
+    
+def convert_video_to_avif(filename):
+    try:
+        fd,new_filename = tempfile.mkstemp(suffix=".avif")
+        os.close(fd)
+        print("converting gif w ffmpeg to avif")
+        subprocess.call(["ffmpeg","-nostdin","-y", "-i", filename,"-pix_fmt","yuv420p","-an","-c:v","libsvtav1","-crf","30","-b:v","0","-threads","4", new_filename], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT)
+        if os.path.isfile(new_filename) and os.path.getsize(new_filename) > 0:
+            return new_filename
+        else:
+            print("ffmpeg failed to convert avif")
+            return filename
+    except Exception as e:
+        print("error converting avif (convert_video_to_avif):")
+        print(e)
         return filename
 
-
+def redir(url):
+    return {
+        "statusCode": 307,
+        "headers": {
+            "Location": url
+        }
+    }
 
 def lambda_handler(event, context):
     if ("queryStringParameters" not in event):
         return {
             "statusCode": 400,
-            "body": "Invalid request."
+            "body": "Invalid request!"
         }
     
     url = event["queryStringParameters"].get("url","")
-
-    # download video
-    videoLocation = tempfile.mkstemp(suffix=".mp4")[1]
-    subprocess.call(["wget","-O",videoLocation,url],stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT)
-
-    videoLocationLooped = loop_video_until_length(videoLocation, 30)
-    if videoLocationLooped != videoLocation:
-        os.remove(videoLocation)
-        videoLocation = videoLocationLooped
-
-    with open(videoLocation, "rb") as image_file:
-        encoded_string = base64.b64encode(image_file.read()).decode('ascii')
-    os.remove(videoLocation)
-    return {
-        'statusCode': 200,
-        "headers": 
-        {
-            "Content-Type": "video/mp4"
-        },
-        'body': encoded_string,
-        'isBase64Encoded': True
-    }
+    try:
+        if url == "":
+            return {
+                "statusCode": 400,
+                "body": "Invalid request!!"
+            }
+        if not url.startswith("https://video.twimg.com/tweet_video/"):
+            return redir(url)
+    
+        if useBucket:
+            id=re.search(r"https:\/\/video\.twimg\.com\/tweet_video\/(.*?)\..*",url).group(1)
+            bfilename = str(id)+".avif"
+            furl=f"https://gifs.vxtwitter.com/{bfilename}" #f"https://{bucketname}.s3.amazonaws.com/{bfilename}"
+            print("get req for: "+url)
+            try:
+                s3.head_object(Bucket=bucketname, Key=bfilename)
+                print("found existing already: "+bfilename)
+                return redir(furl)
+            except botocore.exceptions.ClientError:
+                # Not found
+                pass
+        # download video
+        print("downloading: "+url)
+        videoLocation = tempfile.mkstemp(suffix=".mp4")[1]
+        response = requests.get(url, stream=True)
+        if response.status_code == 200:
+            with open(videoLocation, 'wb') as f:
+                for chunk in response.iter_content(chunk_size=8192):
+                    f.write(chunk)
+        else:
+            print("error downloading video")
+            return redir(url)
+    
+        videoLocationLooped = convert_video_to_avif(videoLocation)
+        if videoLocationLooped != videoLocation:
+            os.remove(videoLocation)
+            videoLocation = videoLocationLooped
+        else:
+            os.remove(videoLocation)
+            return redir(url)
+        
+        if not useBucket:
+            with open(videoLocation, "rb") as image_file:
+                encoded_string = base64.b64encode(image_file.read()).decode('ascii')
+                os.remove(videoLocation)
+                return {
+                    'statusCode': 200,
+                    "headers": 
+                    {
+                        "Content-Type": "image/avif",
+                        "Cache-Control": "public, max-age=604800, immutable"
+                    },
+                    'body': encoded_string,
+                    'isBase64Encoded': True
+                }
+        else:
+            with open(videoLocation, "rb") as image_file:
+                s3.upload_fileobj(image_file, bucketname, bfilename)
+            os.remove(videoLocation)
+            print("converted: "+url+" -> "+furl)
+            return redir(furl)
+    except Exception as e:
+        print("error converting gif: ")
+        print(e)
+        return redir(url)

gifConvert/conv.sh

@@ -1,64 +0,0 @@
-#!/bin/bash -e
-
-usage(){
-    echo "Usage: $0 [options] output"
-    echo "Options:"
-    echo "  --help              Show this help"
-    echo "  -u, --url           URL of the video"
-    echo "  -w, --max-width     Maximum width of the output"
-    echo "  -h, --max-height    Maximum height of the output"
-    echo "  -t, --threads       Number of threads to use"
-    exit 1
-}
-
-URL=""
-MAXW=400
-MAXH=267
-THREADS=1
-OUTPUT="out.gif"
-FPS=10
-
-while [ $# -gt 0 ]; do
-    case "$1" in
-        --help)
-            usage
-            ;;
-        -u|--url)
-            URL="$2"
-            shift
-            ;;
-        -w|--max-width)
-            MAXW="$2"
-            shift
-            ;;
-        -h|--max-height)
-            MAXH="$2"
-            shift
-            ;;
-        -t|--threads)
-            THREADS="$2"
-            shift
-            ;;
-        -f|--fps)
-            FPS="$2"
-            shift
-            ;;
-        -*)
-            echo "Unknown option: $1"
-            usage
-            ;;
-        *)
-            OUTPUT="$1"
-            ;;
-    esac
-    shift
-done
-
-# make unique temp directory
-TEMPDIR=$( mktemp -d )
-
-./ffmpeg -i "$URL" -vf "scale=if(gte(iw\,ih)\,min($MAXW\,iw)\,-2):if(lt(iw\,ih)\,min($MAXH\,ih)\,-2)" -threads $THREADS "$TEMPDIR/frame%04d.png"
-./gifski -o "$TEMPDIR/out.gif" --fast --fps $FPS --quality=90 $TEMPDIR/frame*.png
-#./gifsicle -O3 "$TEMPDIR/out.gif" -o "$OUTPUT"
-mv "$TEMPDIR/out.gif" "$OUTPUT"
-rm -rf "$TEMPDIR"

msgs.py

@@ -17,8 +17,11 @@ def genLikesDisplay(vnf):
         return ("💖 " + numerize.numerize(vnf['likes']))
 
 def genQrtDisplay(qrt):
+    text = qrt['text']
+    if "translation" in qrt and qrt["translation"] is not None:
+        text = qrt["translation"]["text"]
     verifiedCheck = "☑️" if ('verified' in qrt and qrt['verified']) else ""
-    return ("\n\n【QRT of " + qrt['user_name'] + " (@" + qrt['user_screen_name'] + ")"+ verifiedCheck+":】\n\n'" + qrt['text'] + "'")
+    return ("\n\n【QRT of " + qrt['user_name'] + " (@" + qrt['user_screen_name'] + ")"+ verifiedCheck+":】\n\n'" + text + "'")
 
 def genPollDisplay(poll):
     pctSplit=10
@@ -27,6 +30,9 @@ def genPollDisplay(poll):
         output+=choice["name"]+"\n"+("█"*int(choice["percent"]/pctSplit)) +" "+str(choice["percent"])+"%\n"
     return output
 
+def genTranslationDisplay(translation):
+    return f"【TL "+translation["source_language"]+"→"+translation["destination_language"]+"】\n\n"+translation["text"]
+
 # formats the top text of the embed
 def formatProvider(base,vnf):
     finalText = base
@@ -38,13 +44,16 @@ def formatProvider(base,vnf):
         finalText = base
     return finalText
 
-def formatEmbedDesc(type,body,qrt,pollData):
+def formatEmbedDesc(type,body,qrt,pollData,translation):
     # Trim the embed description to 248 characters, prioritizing poll and likes
 
     qrtType=None
     if qrt!=None:
         qrtType="Text"
 
+    if translation is not None:
+        body = genTranslationDisplay(translation)
+
     limit = videoDescLimit if type=="Video" or (qrt!=None and (qrtType=="Video")) else tweetDescLimit
 
     output = ""
@@ -73,6 +82,6 @@ def formatEmbedDesc(type,body,qrt,pollData):
         diff = len(output)-limit
         # remove the characters from body, add ellipsis
         body = body[:-(diff+1)]+"…"
-        return formatEmbedDesc(type,body,qrt,pollData)
+        return formatEmbedDesc(type,body,qrt,pollData,None)
     else:
         return output

requirements.txt

@@ -8,4 +8,5 @@ Werkzeug==2.3.7
 numerize==0.12
 oauthlib==3.2.2
 PyRTF3==0.47.5
-XClientTransaction==0.0.2
+XClientTransaction==1.0.2
+deepl==1.3.0

serverless.yml

@@ -29,6 +29,7 @@ provider:
     VXTWITTER_COMBINATION_METHOD: ${env:VXTWITTER_COMBINATION_METHOD, 'local'}
     VXTWITTER_GIF_CONVERT_API: ${env:VXTWITTER_GIF_CONVERT_API, ''}
     VXTWITTER_WORKAROUND_TOKENS: ${env:VXTWITTER_WORKAROUND_TOKENS, ''}
+    DEEPL_API_KEY: ${env:DEEPL_API_KEY, ''}
     #VXTWITTER_PROXIES: ${env:VXTWITTER_PROXIES, ''}
 
 package:

test_vx_embeds.py

@@ -204,4 +204,20 @@ def test_embed_action():
     resp = client.get(testTextTweet.replace("https://twitter.com",""),headers={"User-Agent":"Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)"})
     assert resp.status_code==200
     assert "application/activity+json" in str(resp.data)
-    assert "%F0%9F%92%96" not in str(resp.data) # 💖
+    assert "%F0%9F%92%96" not in str(resp.data) # 💖
+
+def test_embed_translated():
+    resp = client.get(testTextTweet.replace("https://twitter.com","")+"/jp",headers={"User-Agent":"test"})
+    assert resp.status_code==200
+    assert "→" in resp.text
+    resp = client.get(testTextTweet.replace("https://twitter.com","")+"/ja",headers={"User-Agent":"test"})
+    assert resp.status_code==200
+    assert "→" in resp.text
+
+def test_embed_translated_subdomain():
+    resp = client.get(testTextTweet.replace("https://twitter.com","https://jp.vxtwitter.com"),headers={"User-Agent":"test"})
+    assert resp.status_code==200
+    assert "→" in resp.text
+    resp = client.get(testTextTweet.replace("https://twitter.com","https://ja.vxtwitter.com"),headers={"User-Agent":"test"})
+    assert resp.status_code==200
+    assert "→" in resp.text

test_vx_extract.py

@@ -9,17 +9,21 @@ def test_twextract_syndicationAPI():
     tweet = twExtract.extractStatus_syndication(testMediaTweet,workaroundTokens=tokens)
     assert utils.stripEndTCO(utils.stripEndTCO(tweet["full_text"]))==testMediaTweet_compare['text']
 
-def test_twextract_extractStatusV2Anon():
-    tweet = twExtract.extractStatusV2Anon(testTextTweet,None)['legacy']
+def test_twextract_extractStatusV2Rest():
+    tweet = twExtract.extractStatusV2Rest(testTextTweet,None)['legacy']
     assert utils.stripEndTCO(tweet["full_text"])==testTextTweet_compare['text']
-    tweet = twExtract.extractStatusV2Anon(testVideoTweet,None)['legacy']
+    tweet = twExtract.extractStatusV2Rest(testVideoTweet,None)['legacy']
     assert utils.stripEndTCO(tweet["full_text"])==testVideoTweet_compare['text']
-    tweet = twExtract.extractStatusV2Anon(testMediaTweet,None)['legacy']
+    tweet = twExtract.extractStatusV2Rest(testMediaTweet,None)['legacy']
     assert utils.stripEndTCO(tweet["full_text"])==testMediaTweet_compare['text']
-    tweet = twExtract.extractStatusV2Anon(testMultiMediaTweet,None)['legacy']
+    tweet = twExtract.extractStatusV2Rest(testMultiMediaTweet,None)['legacy']
     assert utils.stripEndTCO(tweet["full_text"])[:94]==testMultiMediaTweet_compare['text'][:94]
     
 
+def test_twextract_v2API_Anon():
+    tweet = twExtract.extractStatusV2Rest_Anon(testMediaTweet,workaroundTokens=tokens)['legacy']
+    assert utils.stripEndTCO(tweet["full_text"])==testMediaTweet_compare['text']
+
 def test_twextract_v2API():
     tweet = twExtract.extractStatusV2(testMediaTweet,workaroundTokens=tokens)['legacy']
     assert utils.stripEndTCO(tweet["full_text"])==testMediaTweet_compare['text']
@@ -33,11 +37,6 @@ def test_twextract_extractStatusV2TweetDetails():
     assert utils.stripEndTCO(tweet["full_text"])==testMediaTweet_compare['text']
 
 ## Tweet retrieve tests ##
-def test_twextract_textTweetExtract():
-    tweet = twExtract.extractStatus(testTextTweet,workaroundTokens=tokens)
-    assert utils.stripEndTCO(tweet["legacy"]["full_text"])==testTextTweet_compare['text']
-    assert tweet["user"]["screen_name"]=="jack"
-    assert 'extended_entities' not in tweet
     
 def test_twextract_extractV2():
     tweet = twExtract.extractStatusV2(testTextTweet,workaroundTokens=tokens)

test_vx_misc.py

@@ -1,10 +1,17 @@
-import twitfix, cache, twExtract
+import twitfix, cache, twExtract, utils
 from vx_testdata import *
 from twExtract import twUtils
 
 def test_calcSyndicationToken():
     assert twUtils.calcSyndicationToken("1691389765483200513") == "43lnobuxzql"
 
+def test_stripEndTCO():
+    assert utils.stripEndTCO("Hello World https://t.co/abc123") == "Hello World"
+    assert utils.stripEndTCO("Hello\nWorld https://t.co/abc123") == "Hello\nWorld"
+    assert utils.stripEndTCO("Hello\nWorld\nhttps://t.co/abc123") == "Hello\nWorld"
+    assert utils.stripEndTCO("Hello\nWorld\n https://t.co/abc123") == "Hello\nWorld"
+    assert utils.stripEndTCO("Hello\nWorld \nhttps://t.co/abc123") == "Hello\nWorld"
+
 def test_addToCache():
     cache.clearCache()
     twitfix.getTweetData(testTextTweet)

translation.py

@@ -0,0 +1,27 @@
+import deepl
+from configHandler import config
+
+validLanguages = ['AF', 'AN', 'AR', 'AS', 'AY', 'AZ', 'BA', 'BE', 'BG', 'BN', 'BR', 'BS', 'CA', 'CS', 'CY', 'DA', 'DE', 'DE-DE', 'EL', 'EN', 'EO', 'ES', 'ES-419', 'ET', 'EU', 'FA', 'FI', 'FR', 'FR-FR', 'GA', 'GL', 'GN', 'GU', 'HA', 'HE', 'HI', 'HR', 'HT', 'HU', 'HY', 'ID', 'IG', 'IS', 'IT', 'JA', 'JV', 'KA', 'KK', 'KO', 'KY', 'LA', 'LB', 'LN', 'LT', 'LV', 'MG', 'MI', 'MK', 'ML', 'MN', 'MR', 'MS', 'MT', 'MY', 'NB', 'NE', 'NL', 'OC', 'OM', 'PA', 'PL', 'PS', 'PT-BR', 'PT-PT', 'QU', 'RO', 'RU', 'SA', 'SK', 'SL', 'SQ', 'SR', 'ST', 'SU', 'SV', 'SW', 'TA', 'TE', 'TG', 'TH', 'TK', 'TL', 'TN', 'TR', 'TS', 'TT', 'UK', 'UR', 'UZ', 'VI', 'WO', 'XH', 'YI', 'ZH', 'ZH-HANS', 'ZH-HANT', 'ZU']
+languageAliases = {
+    #"EN":"EN-US",
+    "JP":"JA",
+}
+
+mergedLangs=(validLanguages+list(languageAliases.keys()))
+
+def getDeeplTranslation(text,target):
+    target = target.upper()
+    if target == "EN":
+        target = "EN-US"
+    try:
+        if 'deeplKey' not in config["config"] or config["config"]["deeplKey"] == None:
+            return None
+        deepl_client = deepl.Translator(config["config"]["deeplKey"])
+        result = deepl_client.translate_text(text, target_lang=target)
+        return {
+            "text":result.text,
+            "source_language":result.detected_source_lang,
+            "destination_language":target
+        }
+    except Exception as e:
+        return None

twExtract/__init__.py

@@ -13,11 +13,10 @@ import concurrent.futures
 bearer="Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
 v2bearer="Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
 androidBearer="Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F"
-tweetdeckBearer="Bearer AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF"
 
-requestUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:138.0) Gecko/20100101 Firefox/138.0"
+requestUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0"
 
-bearerTokens=[tweetdeckBearer,bearer,v2bearer,androidBearer]
+bearerTokens=[bearer,v2bearer,androidBearer]
 
 guestToken=None
 guestTokenUses=0
@@ -28,15 +27,15 @@ userIDregex = r"\/i\/user\/(\d+)"
 v2Features='{"longform_notetweets_inline_media_enabled":true,"super_follow_badge_privacy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"super_follow_user_api_enabled":true,"super_follow_tweet_api_enabled":true,"android_graphql_skip_api_media_color_palette":true,"creator_subscriptions_tweet_preview_api_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"creator_subscriptions_subscription_count_enabled":true,"tweetypie_unmention_optimization_enabled":true,"longform_notetweets_consumption_enabled":true,"subscriptions_verification_info_enabled":true,"blue_business_profile_image_shape_enabled":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"super_follow_exclusive_tweet_notifications_enabled":true}'
 v2graphql_api="2OOZWmw8nAtUHVnXXQhgaA"
 
-v2AnonFeatures='{"creator_subscriptions_tweet_preview_api_enabled":true,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":false,"responsive_web_jetfuel_frame":true,"responsive_web_grok_share_attachment_enabled":true,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":false,"responsive_web_grok_analysis_button_from_backend":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"payments_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"verified_phone_label_enabled":false,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_grok_imagine_annotation_enabled":true,"responsive_web_grok_community_note_auto_translation_is_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false}'
-v2AnonGraphql_api="wqi5M7wZ7tW-X9S2t-Mqcg"
+v2AnonFeatures='{"creator_subscriptions_tweet_preview_api_enabled":true,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":true,"responsive_web_grok_share_attachment_enabled":true,"responsive_web_grok_annotations_enabled":false,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":true,"responsive_web_grok_analysis_button_from_backend":true,"post_ctas_fetch_enabled":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"profile_label_improvements_pcf_label_in_post_enabled":true,"responsive_web_profile_redirect_enabled":false,"rweb_tipjar_consumption_enabled":false,"verified_phone_label_enabled":false,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_grok_imagine_annotation_enabled":true,"responsive_web_grok_community_note_auto_translation_is_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false}'
+v2AnonGraphql_api="0aTrQMKgj95K791yXeNDRA"
 gt_pattern = r'document\.cookie="gt=([^;]+);'
 
-androidGraphqlFeatures='{"longform_notetweets_inline_media_enabled":true,"super_follow_badge_privacy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"super_follow_user_api_enabled":true,"unified_cards_ad_metadata_container_dynamic_card_content_query_enabled":true,"super_follow_tweet_api_enabled":true,"articles_api_enabled":true,"android_graphql_skip_api_media_color_palette":true,"creator_subscriptions_tweet_preview_api_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"tweetypie_unmention_optimization_enabled":true,"longform_notetweets_consumption_enabled":true,"subscriptions_verification_info_enabled":true,"blue_business_profile_image_shape_enabled":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"immersive_video_status_linkable_timestamps":true,"super_follow_exclusive_tweet_notifications_enabled":true}'
-androidGraphql_api="llQH5PFIRlenVrlKJU8jNA"
+androidGraphqlFeatures='{"grok_translations_community_note_translation_is_enabled":false,"super_follow_badge_privacy_enabled":true,"unified_cards_destination_url_params_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"super_follow_user_api_enabled":true,"profile_label_improvements_pcf_label_in_profile_enabled":true,"premium_content_api_read_enabled":false,"grok_translations_community_note_auto_translation_is_enabled":false,"android_graphql_skip_api_media_color_palette":true,"tweetypie_unmention_optimization_enabled":true,"longform_notetweets_consumption_enabled":true,"subscriptions_verification_info_enabled":true,"blue_business_profile_image_shape_enabled":true,"super_follow_exclusive_tweet_notifications_enabled":true,"longform_notetweets_inline_media_enabled":true,"grok_android_analyze_trend_fetch_enabled":false,"unified_cards_ad_metadata_container_dynamic_card_content_query_enabled":true,"super_follow_tweet_api_enabled":true,"articles_api_enabled":true,"android_ad_formats_media_component_render_overlay_enabled":true,"creator_subscriptions_tweet_preview_api_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"grok_translations_timeline_user_bio_auto_translation_is_enabled":false,"grok_translations_post_auto_translation_is_enabled":false,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"immersive_video_status_linkable_timestamps":true,"profile_label_improvements_pcf_label_in_post_enabled":true}'
+androidGraphql_api="Wrspae-uyGj-nWPyUqdUag"
 
-tweetDetailGraphqlFeatures='{"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"articles_preview_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"rweb_video_timestamps_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_enhance_cards_enabled":false}'
-tweetDetailGraphql_api="e7RKseIxLu7HgkWNKZ6qnw"
+tweetDetailGraphqlFeatures='{"rweb_video_screen_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"responsive_web_profile_redirect_enabled":false,"rweb_tipjar_consumption_enabled":false,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":true,"responsive_web_grok_share_attachment_enabled":true,"responsive_web_grok_annotations_enabled":false,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":true,"responsive_web_grok_analysis_button_from_backend":true,"post_ctas_fetch_enabled":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_grok_imagine_annotation_enabled":true,"responsive_web_grok_community_note_auto_translation_is_enabled":false,"responsive_web_enhance_cards_enabled":false}'
+tweetDetailGraphql_api="Kzfv17rukSzjT96BerOWZA"
 
 # this is for UserTweets endpoint
 tweetFeedGraphqlFeatures='{"rweb_video_screen_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":false,"responsive_web_grok_share_attachment_enabled":true,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":false,"responsive_web_grok_analysis_button_from_backend":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_enhance_cards_enabled":false}'
@@ -114,31 +113,35 @@ def cycleBearerTokenGet(url,headers):
         return tweet
     raise TwExtractError(400, "Extract error")
 
-def twitterApiGet(url,btoken=None,authToken=None,guestToken=None):
+def twitterApiGet(url,btoken=None,authToken=None,guestToken=None,userAgent=None,language=None):
 
-    if authToken.startswith("oa|"):
+    if authToken != None and authToken.startswith("oa|"):
         url = url.replace("https://x.com/i/api/graphql/","https://api.twitter.com/graphql/")
         authToken = authToken[3:]
         key = authToken.split("|")[0]
         secret = authToken.split("|")[1]
 
         twt = oauth1.Client(client_key='3nVuSoBZnx6U4vzUxf5w',client_secret='Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys',resource_owner_key=key,resource_owner_secret=secret)
-        hdr = getAuthHeaders(androidBearer)
+        hdr = getAuthHeaders(androidBearer,language=language)
         del hdr["Authorization"]
         hdr["X-Twitter-Client"] = "TwitterAndroid"
+        if userAgent is not None:
+            hdr["User-Agent"] = userAgent
 
         uri, headers, body = twt.sign(url, headers=hdr,realm="http://api.twitter.com/")
 
         response = requests.get(url,headers=headers)
     else:
         if btoken is None:
-            return cycleBearerTokenGet(url,getAuthHeaders(bearer,authToken=authToken,guestToken=guestToken))
-        headers = getAuthHeaders(btoken,authToken=authToken,guestToken=guestToken)
+            btoken = v2bearer
+            #return cycleBearerTokenGet(url,getAuthHeaders(bearer,authToken=authToken,guestToken=guestToken))
+        headers = getAuthHeaders(btoken,authToken=authToken,guestToken=guestToken,language=language)
         response = requests.get(url, headers=headers)
 
     return response
 
-def getAuthHeaders(btoken,authToken=None,guestToken=None):
+def getAuthHeaders(btoken,authToken=None,guestToken=None,language=None):
+
     csrfToken=str(uuid.uuid4()).replace('-', '')
     headers = {"x-twitter-active-user":"yes","x-twitter-client-language":"en","x-csrf-token":csrfToken,"User-Agent":requestUserAgent}
     headers['Authorization'] = btoken
@@ -149,19 +152,17 @@ def getAuthHeaders(btoken,authToken=None,guestToken=None):
     if guestToken is not None:
         headers["x-guest-token"] = guestToken
 
+    if language is not None:
+        headers["x-twitter-client-language"] = language
+
     return headers
 
 def getGuestToken():
     global guestToken
     global guestTokenUses
     if guestToken is None:
-        r = requests.get(f"https://{twitterUrl}",headers={"User-Agent":requestUserAgent,"Cookie":"night_mode=2"},allow_redirects=False)
-        m = re.search(gt_pattern, r.text)
-        if m is None:
-            r = requests.post(f"https://api.{twitterUrl}/1.1/guest/activate.json", headers={"Authorization":bearer})
-            guestToken = json.loads(r.text)["guest_token"]
-        else:
-            guestToken = m.group(1)
+        r = requests.post(f"https://api.{twitterUrl}/1.1/guest/activate.json", headers={"Authorization":v2bearer,"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0"})
+        guestToken = json.loads(r.text)["guest_token"]
         guestTokenUses = 0
     else:
         guestTokenUses+=1
@@ -213,7 +214,7 @@ def extractStatus_guestToken(url):
         raise TwExtractError(error["code"], error["message"])
     return output
 
-def extractStatus_syndication(url,workaroundTokens=None):
+def extractStatus_syndication(url,workaroundTokens=None,tlLanguage=None):
     # https://github.com/mikf/gallery-dl/blob/46cae04aa3a113c7b6bbee1bb468669564b14ae8/gallery_dl/extractor/twitter.py#L1784
     m = re.search(pathregex, url)
     if m is None:
@@ -265,7 +266,7 @@ def extractStatus_twExtractProxy(url,workaroundTokens=None):
             continue
         return output
 
-def extractStatusV2(url,workaroundTokens):
+def extractStatusV2(url,workaroundTokens,tlLanguage=None):
     # get tweet ID
     m = re.search(pathregex, url)
     if m is None:
@@ -279,7 +280,7 @@ def extractStatusV2(url,workaroundTokens):
     def request_with_token(twid, authToken):
         vars = json.loads('{"includeTweetImpression":true,"includeHasBirdwatchNotes":false,"includeEditPerspective":false,"rest_ids":["x"],"includeEditControl":true,"includeCommunityTweetRelationship":true,"includeTweetVisibilityNudge":true}')
         vars['rest_ids'][0] = str(twid)
-        tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2graphql_api}/TweetResultsByIdsQuery?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2Features)}",authToken=authToken,btoken=v2bearer)
+        tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2graphql_api}/TweetResultsByIdsQuery?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2Features)}",authToken=authToken,language=tlLanguage)
         try:
             rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining")
             print(f"Twitter Token Rate limit remaining: {rateLimitRemaining}")
@@ -318,7 +319,7 @@ def extractStatusV2(url,workaroundTokens):
         return tweet
     return parallel_token_request(twid, tokens, request_with_token)
 
-def extractStatusV2Android(url,workaroundTokens):
+def extractStatusV2Android(url,workaroundTokens,tlLanguage=None):
     # get tweet ID
     m = re.search(pathregex, url)
     if m is None:
@@ -332,7 +333,7 @@ def extractStatusV2Android(url,workaroundTokens):
         try:
             vars = json.loads('{"referrer":"home","includeTweetImpression":true,"includeHasBirdwatchNotes":false,"isReaderMode":false,"includeEditPerspective":false,"includeEditControl":true,"focalTweetId":0,"includeCommunityTweetRelationship":true,"includeTweetVisibilityNudge":true}')
             vars['focalTweetId'] = int(twid)
-            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{androidGraphql_api}/ConversationTimelineV2?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(androidGraphqlFeatures)}", authToken=authToken,btoken=androidBearer)
+            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{androidGraphql_api}/ConversationTimelineV2?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(androidGraphqlFeatures)}", authToken=authToken,btoken=androidBearer,userAgent="TwitterAndroid/11.61.0-release.0 (311610000-r-0) G011A/9 (google;G011A;google;G011A;0;;1;2016)",language=tlLanguage)
             try:
                 rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining")
                 print(f"Twitter Android Token Rate limit remaining: {rateLimitRemaining}")
@@ -377,7 +378,7 @@ def extractStatusV2Android(url,workaroundTokens):
         return tweet
     return parallel_token_request(twid, tokens, request_with_token)
 
-def extractStatusV2TweetDetail(url,workaroundTokens):
+def extractStatusV2TweetDetail(url,workaroundTokens,tlLanguage=None):
     # get tweet ID
     m = re.search(pathregex, url)
     if m is None:
@@ -392,7 +393,7 @@ def extractStatusV2TweetDetail(url,workaroundTokens):
         try:
             vars = json.loads('{"focalTweetId":"0","with_rux_injections":false,"includePromotedContent":true,"withCommunity":true,"withQuickPromoteEligibilityTweetFields":true,"withBirdwatchNotes":true,"withVoice":true,"withV2Timeline":true}')
             vars['focalTweetId'] = str(twid)
-            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{tweetDetailGraphql_api}/TweetDetail?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(tweetDetailGraphqlFeatures)}", authToken=authToken,btoken=v2bearer)
+            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{tweetDetailGraphql_api}/TweetDetail?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(tweetDetailGraphqlFeatures)}", authToken=authToken,btoken=v2bearer,language=tlLanguage)
             try:
                 rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining")
                 print(f"Twitter Token Rate limit remaining: {rateLimitRemaining}")
@@ -437,7 +438,10 @@ def extractStatusV2TweetDetail(url,workaroundTokens):
         return tweet
     return parallel_token_request(twid, tokens, request_with_token)
 
-def extractStatusV2Anon(url,x):
+def extractStatusV2Rest_Anon(url,workaroundTokens,tlLanguage=None):
+    return extractStatusV2Rest(url,None,tlLanguage=tlLanguage)
+
+def extractStatusV2Rest(url,workaroundTokens,tlLanguage=None):
     # get tweet ID
     m = re.search(pathregex, url)
     if m is None:
@@ -448,9 +452,19 @@ def extractStatusV2Anon(url,x):
 
     # get tweet
     try:
-        vars = json.loads('{"tweetId":"0","withCommunity":false,"includePromotedContent":false,"withVoice":false}')
+        vars = json.loads('{"tweetId":"0","includePromotedContent":true,"withBirdwatchNotes":true,"withVoice":true,"withCommunity":true}')
         vars['tweetId'] = str(twid)
-        tweet = requests.get(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", headers=getAuthHeaders(v2bearer,guestToken=guestToken))
+        if workaroundTokens is not None and len(workaroundTokens) > 0:
+            tokens = workaroundTokens
+            random.shuffle(tokens)
+            for authToken in tokens:
+                try:
+                    tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", btoken=v2bearer,authToken=authToken,guestToken=guestToken,language=tlLanguage)
+                except Exception as e:
+                    continue
+        else:
+            tweet = twitterApiGet(f"https://x.com/i/api/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", btoken=v2bearer,guestToken=guestToken,language=tlLanguage)
+        
         try:
             rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining")
             print(f"Twitter Anon Token Rate limit remaining: {rateLimitRemaining}")
@@ -496,12 +510,13 @@ def fixTweetData(tweet):
         pass
     return tweet
 
-def extractStatus(url,workaroundTokens=None):
-    # TODO: commented out methods are too slow/unreliable at the moment
-    methods=[extractStatusV2Anon,extractStatusV2]#,extractStatusV2Android,extractStatusV2TweetDetail] # 
+def extractStatus(url,workaroundTokens=None,tlLanguage=None):
+    methods=[extractStatusV2Rest_Anon,extractStatusV2,extractStatusV2Rest,extractStatusV2Android]#,extractStatusV2TweetDetail]
+    if tlLanguage is not None: # prioritize endpoints that return translations
+        methods = [extractStatusV2Rest,extractStatusV2TweetDetail,extractStatusV2Android,extractStatusV2,extractStatusV2Rest_Anon]
     for method in methods:
         try:
-            result = method(url,workaroundTokens)
+            result = method(url,workaroundTokens,tlLanguage=tlLanguage)
             if 'legacy' not in result:
                 print(f"{method.__name__} method failed: Legacy not found for {url}")
                 # try another method

twExtract/twUtils.py

@@ -3,8 +3,9 @@ import hashlib
 import base64
 import uuid
 from x_client_transaction import ClientTransaction
-from x_client_transaction.utils import handle_x_migration
+from x_client_transaction.utils import get_ondemand_file_url, generate_headers
 import requests
+import bs4
 digits = "0123456789abcdefghijklmnopqrstuvwxyz"
 
 def baseConversion(x, base):
@@ -49,6 +50,14 @@ def get_twitter_homepage(headers=None):
     return response
 
 def generate_transaction_id(method: str, path: str,headers=None) -> str:
-    ct = ClientTransaction(get_twitter_homepage(headers=headers))
+    session = requests.Session()
+    session.headers = generate_headers()
+    home_page = session.get(url="https://x.com/home")
+    home_page_response = bs4.BeautifulSoup(home_page.content, 'html.parser')
+    ondemand_file_url = get_ondemand_file_url(response=home_page_response)
+    ondemand_file = session.get(url=ondemand_file_url)
+    ondemand_file_response = bs4.BeautifulSoup(ondemand_file.content, 'html.parser')
+
+    ct = ClientTransaction(home_page_response=home_page_response, ondemand_file_response=ondemand_file_response)
     transaction_id = ct.generate_transaction_id(method=method, path=path)
     return transaction_id

twitfix.py

@@ -14,7 +14,7 @@ import msgs
 import twExtract as twExtract
 from cache import addVnfToLinkCache,getVnfFromLinkCache
 import vxlogging as log
-from utils import getTweetIdFromUrl, pathregex, determineMediaToEmbed, determineEmbedTweet, BytesIOWrapper, fixMedia
+from utils import getTweetIdFromUrl, pathregex, determineMediaToEmbed, determineEmbedTweet, BytesIOWrapper, fixMedia, indexOfAny
 from vxApi import getApiResponse, getApiUserResponse
 from urllib.parse import urlparse 
 from PyRTF.Elements import Document
@@ -22,12 +22,14 @@ from PyRTF.document.section import Section
 from PyRTF.document.paragraph import Paragraph
 from copy import deepcopy
 import json
-import datetime
+import translation
 import activity as activitymg
 app = Flask(__name__)
 CORS(app)
 user_agent=""
 
+activityseparator="zqj"
+
 staticFiles = { # TODO: Use flask static files instead of this
     "favicon.ico": {"mime": "image/vnd.microsoft.icon","path": "favicon.ico"},
     "apple-touch-icon.png": {"mime": "image/png","path": "apple-touch-icon.png"},
@@ -82,7 +84,11 @@ def generateActivityLink(tweetData,media=None,mediatype=None,embedIndex=-1):
         return None
     try:
         embedIndex = embedIndex+1
-        return f"{config['config']['url']}/users/{tweetData['user_screen_name']}/statuses/{str(embedIndex)}{tweetData['tweetID']}"
+
+        extras=""
+        if tweetData["translation"] is not None:
+            extras+=activityseparator+tweetData["translation"]["destination_language"]
+        return f"{config['config']['url']}/users/{tweetData['user_screen_name']}/statuses/{str(embedIndex)}{activityseparator}{tweetData['tweetID']}{extras}"
     except Exception as e:
         log.error("Error generating activity link: "+str(e))
         return None
@@ -95,7 +101,7 @@ def getAppName(tweetData,appnameSuffix=""):
 
 def renderImageTweetEmbed(tweetData,image,appnameSuffix="",embedIndex=-1):
     qrt = tweetData['qrt']
-    embedDesc = msgs.formatEmbedDesc("Image",tweetData['text'],qrt,tweetData['pollData'])
+    embedDesc = msgs.formatEmbedDesc("Image",tweetData['text'],qrt,tweetData['pollData'],tweetData["translation"])
 
     if image.startswith("https://pbs.twimg.com") and "?" not in image:
         image = f"{image}?name=orig"
@@ -115,7 +121,7 @@ def renderImageTweetEmbed(tweetData,image,appnameSuffix="",embedIndex=-1):
 
 def renderVideoTweetEmbed(tweetData,mediaInfo,appnameSuffix="",embedIndex=-1):
     qrt = tweetData['qrt']
-    embedDesc = msgs.formatEmbedDesc("Video",tweetData['text'],qrt,tweetData['pollData'])
+    embedDesc = msgs.formatEmbedDesc("Video",tweetData['text'],qrt,tweetData['pollData'],tweetData["translation"])
 
     mediaInfo=fixMedia(mediaInfo)
 
@@ -138,7 +144,7 @@ def renderVideoTweetEmbed(tweetData,mediaInfo,appnameSuffix="",embedIndex=-1):
 
 def renderTextTweetEmbed(tweetData,appnameSuffix=""):
     qrt = tweetData['qrt']
-    embedDesc = msgs.formatEmbedDesc("Text",tweetData['text'],qrt,tweetData['pollData'])
+    embedDesc = msgs.formatEmbedDesc("Text",tweetData['text'],qrt,tweetData['pollData'],tweetData["translation"])
 
     return render_template("text.html",
                     tweet=tweetData,
@@ -154,7 +160,7 @@ def renderTextTweetEmbed(tweetData,appnameSuffix=""):
 
 def renderArticleTweetEmbed(tweetData,appnameSuffix=""):
     articlePreview=tweetData['article']["title"]+"\n\n"+tweetData['article']["preview_text"]+"…"
-    embedDesc = msgs.formatEmbedDesc("Image",articlePreview,None,None)
+    embedDesc = msgs.formatEmbedDesc("Image",articlePreview,None,None,None)
 
     return render_template("image.html",
                     tweet=tweetData,
@@ -260,9 +266,9 @@ def userJson():
         },
     }
 
-def getTweetData(twitter_url,include_txt="false",include_rtf="false"):
+def getTweetData(twitter_url,include_txt="false",include_rtf="false",tlLanguage=None):
     cachedVNF = getVnfFromLinkCache(twitter_url)
-    if cachedVNF is not None and include_txt == "false" and include_rtf == "false":
+    if cachedVNF is not None and include_txt == "false" and include_rtf == "false" and tlLanguage is None:
         return cachedVNF
 
     try:
@@ -270,7 +276,7 @@ def getTweetData(twitter_url,include_txt="false",include_rtf="false"):
             workaroundTokens = config['config']['workaroundTokens'].split(",")
         else:
             workaroundTokens = None
-        rawTweetData = twExtract.extractStatus(twitter_url,workaroundTokens=workaroundTokens)
+        rawTweetData = twExtract.extractStatus(twitter_url,workaroundTokens=workaroundTokens,tlLanguage=tlLanguage)
     except:
         rawTweetData = None
     if rawTweetData == None or 'error' in rawTweetData:
@@ -281,7 +287,15 @@ def getTweetData(twitter_url,include_txt="false",include_rtf="false"):
     tweetData = getApiResponse(rawTweetData,include_txt,include_rtf)
     if tweetData is None:
         return None
-    if include_txt == "false" and include_rtf == "false":
+    
+    if tweetData["translation"] is None and tlLanguage is not None and tlLanguage != tweetData["lang"]:
+        # translation probably failed on twitter's end
+        tweetData["translation"] = translation.getDeeplTranslation(tweetData["text"],tlLanguage)
+
+    if tweetData["translation"] is not None and tlLanguage is None:
+        tweetData["translation"] = None # didn't ask for TL
+
+    if include_txt == "false" and include_rtf == "false" and tlLanguage is None:
         addVnfToLinkCache(twitter_url,tweetData)
     return tweetData
 
@@ -307,13 +321,19 @@ def getUserData(twitter_url,includeFeed=False):
 
 @app.route('/<path:sub_path>') # Default endpoint used by everything
 def twitfix(sub_path):
+    splitPath=sub_path.split("/")
+    if "|" in splitPath[-1]: # discord doesn't strip this when a link is spoilered
+        splitPath[-1] = splitPath[-1].replace("|","")
+
+    subdomain = request.host.rsplit(".",2)[0]
+
     global user_agent
     user_agent = request.headers.get('User-Agent', None)
     if user_agent is None:
         user_agent = "unknown"
 
-    isApiRequest=request.url.startswith("https://api.vx") or request.url.startswith("http://api.vx")
-    if not isApiRequest and (request.url.startswith("https://l.vx") or request.url.startswith("https://old.vx")) and "Discord" in user_agent:
+    isApiRequest=subdomain=="api"
+    if not isApiRequest and (subdomain=="l" or subdomain=="old") and "Discord" in user_agent:
         user_agent = user_agent.replace("Discord","LegacyEmbed") # TODO: Clean up; This is a hacky fix to make the new activity embed not trigger
     if sub_path in staticFiles:
         if 'path' not in staticFiles[sub_path] or staticFiles[sub_path]["path"] == None:
@@ -355,24 +375,38 @@ def twitfix(sub_path):
     include_txt="false"
     include_rtf="false"
 
+    
+    translationIndex=indexOfAny(splitPath,translation.mergedLangs,caseInsensitive=True)
+    translationLang = None
+    if translationIndex != -1:
+        if splitPath[translationIndex].upper() in translation.languageAliases:
+            translationLang = translation.languageAliases[splitPath[translationIndex].upper()]
+        else:
+            translationLang = splitPath[translationIndex].lower()
+    elif subdomain.upper() in translation.mergedLangs:
+        if subdomain.upper() in translation.languageAliases:
+            translationLang = translation.languageAliases[subdomain.upper()]
+        else:
+            translationLang = subdomain.lower()
+
     if isApiRequest:
         if "include_txt" in request.args:
             include_txt = request.args.get("include_txt")
         if "include_rtf" in request.args:
             include_rtf = request.args.get("include_rtf")
 
-    tweetData = getTweetData(twitter_url,include_txt,include_rtf)
+    tweetData = getTweetData(twitter_url,include_txt,include_rtf,tlLanguage=translationLang)
     if tweetData is None:
         log.error("Tweet Data Get failed for "+twitter_url)
         return message(msgs.failedToScan)
     qrt = None
     if 'qrtURL' in tweetData and tweetData['qrtURL'] is not None:
-        qrt = getTweetData(tweetData['qrtURL'])
+        qrt = getTweetData(tweetData['qrtURL'],tlLanguage=translationLang)
     tweetData['qrt'] = qrt
 
     retweet = None
     if 'retweetURL' in tweetData and tweetData['retweetURL'] is not None:
-        retweet = getTweetData(tweetData['retweetURL'])
+        retweet = getTweetData(tweetData['retweetURL'],tlLanguage=translationLang)
     tweetData['retweet'] = retweet
 
     tweetData = deepcopy(tweetData)
@@ -383,7 +417,7 @@ def twitfix(sub_path):
         requestUrlWithoutQuery = request.url
 
     directEmbed=False
-    if requestUrlWithoutQuery.startswith("https://d.vx") or requestUrlWithoutQuery.endswith(".mp4") or requestUrlWithoutQuery.endswith(".png"):
+    if subdomain=="d" or requestUrlWithoutQuery.endswith(".mp4") or requestUrlWithoutQuery.endswith(".png"):
         directEmbed = True
         # remove the .mp4 from the end of the URL
         if requestUrlWithoutQuery.endswith(".mp4") or requestUrlWithoutQuery.endswith(".png"):
@@ -403,9 +437,10 @@ def twitfix(sub_path):
         return send_file(BytesIOWrapper(rtf), mimetype='application/rtf', as_attachment=True, download_name=f'{tweetData["user_screen_name"]}_{tweetData["tweetID"]}.rtf')
 
     embedIndex = -1
-    # if url ends with /1, /2, /3, or /4, we'll use that as the index
-    if sub_path[-2:] in ["/1","/2","/3","/4"]:
-        embedIndex = int(sub_path[-1])-1
+    # if url has /1, /2, /3, or /4, we'll use that as the index
+    numIdx=indexOfAny(splitPath,["1","2","3","4"]);
+    if numIdx != -1:
+        embedIndex = int(splitPath[numIdx])-1
         sub_path = sub_path[:-2]
         
     if isApiRequest: # Directly return the API response if the request is from the API
@@ -433,14 +468,20 @@ def twitfix(sub_path):
         elif not embeddingMedia:
             return Response(renderTextTweetEmbed(tweetData),headers={"Cache-Tag": "embed"})
         else:
-            media = determineMediaToEmbed(embedTweetData,embedIndex)
+            tryGifConversion = "Discord" in user_agent
+            media = determineMediaToEmbed(embedTweetData,embedIndex,convertGif=tryGifConversion)
             suffix=""
             if "suffix" in media:
                 suffix = media["suffix"]
             if media['type'] == "image":
                 return Response(renderImageTweetEmbed(tweetData,media['url'] , appnameSuffix=suffix,embedIndex=embedIndex),headers={"Cache-Tag": "embed"})
-            elif media['type'] == "video" or media['type'] == "gif":
+            elif media['type'] == "video":
                 return Response(renderVideoTweetEmbed(tweetData,media,appnameSuffix=suffix,embedIndex=embedIndex),headers={"Cache-Tag": "embed"})
+            elif media['type'] == "gif":
+                if "originalUrl" in media and media["url"] != media["originalUrl"] and tryGifConversion:
+                    return Response(renderImageTweetEmbed(tweetData,media['url'] , appnameSuffix=suffix,embedIndex=embedIndex),headers={"Cache-Tag": "embed"})
+                else:
+                    return Response(renderVideoTweetEmbed(tweetData,media,appnameSuffix=suffix,embedIndex=embedIndex),headers={"Cache-Tag": "embed"})
 
     return message(msgs.failedToScan)
 
@@ -478,21 +519,29 @@ def rendercombined():
 
 @app.route("/api/v1/statuses/<string:tweet_id>")
 def api_v1_status(tweet_id):
-    embedIndex = int(tweet_id[0])-1
-    tweet_id = int(tweet_id[1:])
+    lang = None
+    if activityseparator not in tweet_id: #old embeds
+        embedIndex = int(tweet_id[0])-1
+        tweet_id = int(tweet_id[1:])
+    else:
+        args = tweet_id.split(activityseparator)
+        embedIndex = int(args[0])-1
+        tweet_id = int(args[1])
+        if len(args)>2:
+            lang=args[2]
     twitter_url=f"https://twitter.com/i/status/{tweet_id}"
-    tweetData = getTweetData(twitter_url)
+    tweetData = getTweetData(twitter_url,tlLanguage=lang)
     if tweetData is None:
         log.error("Tweet Data Get failed for "+twitter_url)
         return message(msgs.failedToScan)
     qrt = None
     if 'qrtURL' in tweetData and tweetData['qrtURL'] is not None:
-        qrt = getTweetData(tweetData['qrtURL'])
+        qrt = getTweetData(tweetData['qrtURL'],tlLanguage=lang)
     tweetData['qrt'] = qrt
 
     retweet = None
     if 'retweetURL' in tweetData and tweetData['retweetURL'] is not None:
-        retweet = getTweetData(tweetData['retweetURL'])
+        retweet = getTweetData(tweetData['retweetURL'],tlLanguage=lang)
     tweetData['retweet'] = retweet
 
     if tweetData is None:
@@ -516,4 +565,4 @@ def oEmbedGen(description, user, video_link, ttype,providerName=None):
 
 if __name__ == "__main__":
     app.config['SERVER_NAME']='localhost:80'
-    app.run(host='0.0.0.0')
+    app.run(host='0.0.0.0', port=8080)

twitfix.service

@@ -3,11 +3,11 @@ Description=Init file for twitfix uwsgi instance
 After=network.target
 
 [Service]
-User=dylan
-Group=dylan
-WorkingDirectory=/home/dylan/BetterTwitFix
-Environment="PATH=/home/dylan/BetterTwitFix/venv/bin"
-ExecStart=/home/dylan/BetterTwitFix/venv/bin/uwsgi --ini twitfix.ini
+User=carbonara
+Group=carbonara
+WorkingDirectory=/home/carbonara/twitter/BetterTwitFix
+Environment="PATH=/home/carbonara/twitter/BetterTwitFix/venv/bin"
+ExecStart=/home/carbonara/twitter/BetterTwitFix/venv/bin/uwsgi --ini twitfix.ini
 Restart=always
 RestartSec=3
 

utils.py

@@ -3,7 +3,7 @@ import io
 from configHandler import config
 
 pathregex = re.compile("\\w{1,15}\\/(status|statuses)\\/(\\d{2,20})")
-endTCOregex = re.compile("(^.*?) +https:\/\/t.co\/.*?$")
+endTCOregex = re.compile("(^.*?)[ \n]+https:\/\/t.co\/.*?$",flags=re.DOTALL)
 
 def getTweetIdFromUrl(url):
     match = pathregex.search(url)
@@ -83,9 +83,23 @@ def determineMediaToEmbed(tweetData,embedIndex = -1,convertGif = True):
             return media
         elif media['type'] == "video" or media['type'] == "gif":
             if media['type'] == "gif" and convertGif:
-                if config['config']['gifConvertAPI'] != "" and config['config']['gifConvertAPI'] != "none":
+                if config['config']['gifConvertAPI'] != "" and config['config']['gifConvertAPI'] != "none" and config['config']['gifConvertAPI'] != "local":
+                    gcApi = config['config']['gifConvertAPI']
+                    #if gcApi == "local": # TODO
+                        #gcApi = f"{config['config']['url']}/gifconvert"
                     vurl=media['originalUrl'] if 'originalUrl' in media else media['url']
-                    media['url'] = config['config']['gifConvertAPI'] + "/convert?url=" + vurl
+                    media['url'] = gcApi + "/convert.avif?url=" + vurl
                     suffix += " • GIF"
                     media["suffix"] = suffix
-        return media
+        return media
+    
+def indexOfAny(list,items,caseInsensitive=False):
+    if caseInsensitive:
+        list = [item.upper() for item in list]
+        items = [item.upper() for item in items]
+    for item in items:
+        try:
+            return list.index(item)
+        except:
+            continue
+    return -1

vxApi.py

@@ -1,5 +1,6 @@
 import html
 from datetime import datetime
+from flask import json
 from configHandler import config
 from utils import stripEndTCO
 
@@ -20,6 +21,48 @@ def getApiUserResponse(user):
         "fetched_on": int(datetime.now().timestamp()),
     }
 
+def getBestMediaUrl(mediaList):
+    # find the highest bitrate
+    best_bitrate = -1
+    besturl=""
+    for j in mediaList:
+        if j['content_type'] == "video/mp4" and '/hevc/' not in j["url"] and j['bitrate'] > best_bitrate:
+            besturl = j["url"]
+            best_bitrate = j['bitrate']
+    if "?tag=" in besturl:
+        besturl = besturl[:besturl.index("?tag=")]
+    return besturl
+
+def getExtendedVideoOrGifInfo(mediaEntry):
+    videoInfo = mediaEntry["video_info"]
+    info = {
+        "url": getBestMediaUrl(videoInfo["variants"]),
+        "type": "gif" if mediaEntry.get("type", "") == "animated_gif" else "video",
+        "size": {
+            "width": mediaEntry['original_info']["width"],
+            "height": mediaEntry['original_info']["height"]
+        },
+        "duration_millis": videoInfo.get("duration_millis", 0),
+        "thumbnail_url": mediaEntry.get("media_url_https", None),
+        "altText": mediaEntry.get("ext_alt_text", None),
+        "id_str": mediaEntry.get("id_str", None)
+    }
+    return info
+
+def getExtendedImageInfo(mediaEntry):
+    info = {
+        "url": mediaEntry.get("media_url_https", None),
+        "type": "image",
+        "size": {
+            "width": mediaEntry["original_info"]["width"],
+            "height": mediaEntry["original_info"]["height"]
+        },
+        "thumbnail_url": mediaEntry.get("media_url_https", None),
+        "altText": mediaEntry.get("ext_alt_text", None),
+        "id_str": mediaEntry.get("id_str", None)
+    }
+    return info
+    
 def getApiResponse(tweet,include_txt=False,include_rtf=False):
     tweetL = tweet["legacy"]
     if "user_result" in tweet["core"]:
@@ -34,6 +77,7 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False):
     oldTweetVersion = False
     tweetArticle=None
     lang=None
+    translation=None
 
     if "screen_name" not in userL:
         userL["screen_name"] = user["core"]["screen_name"]
@@ -42,6 +86,14 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False):
     if "profile_image_url_https" not in userL:
         userL["profile_image_url_https"] = user["avatar"]["image_url"]
 
+    if "grok_translated_post_with_availability" in tweet and tweet["grok_translated_post_with_availability"]["is_available"] and "data" in tweet["grok_translated_post_with_availability"]:
+        tlData = tweet["grok_translated_post_with_availability"]["data"]
+        translation = {
+            "source_language":tlData["source_language"],
+            "destination_language":tlData["destination_language"],
+            "text":tlData["translation"]
+        }
+
     #editedTweet=False
     try:
         if "birdwatch_pivot" in tweet:
@@ -69,63 +121,55 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False):
             for i in tmedia:
                 extendedInfo={}
                 if "video_info" in i:
-                    # find the highest bitrate
-                    best_bitrate = -1
-                    besturl=""
-                    for j in i["video_info"]["variants"]:
-                        if j['content_type'] == "video/mp4" and '/hevc/' not in j["url"] and j['bitrate'] > best_bitrate:
-                            besturl = j['url']
-                            best_bitrate = j['bitrate']
-                    if "?tag=" in besturl:
-                        besturl = besturl[:besturl.index("?tag=")]
-                    media.append(besturl)
-                    extendedInfo["url"] = besturl
-                    extendedInfo["type"] = "video"
-                    if (i["type"] == "animated_gif"):
-                        extendedInfo["type"] = "gif"
-                    altText = None
-                    extendedInfo["size"] = {"width":i["original_info"]["width"],"height":i["original_info"]["height"]}
-                    if "ext_alt_text" in i:
-                        altText=i["ext_alt_text"]
-                    if "duration_millis" in i["video_info"]:
-                        extendedInfo["duration_millis"] = i["video_info"]["duration_millis"]
-                    else:
-                        extendedInfo["duration_millis"] = 0
-                    extendedInfo["thumbnail_url"] = i["media_url_https"]
-                    extendedInfo["altText"] = altText
-                    extendedInfo["id_str"] = i["id_str"]
+                    extendedInfo = getExtendedVideoOrGifInfo(i)
+                    media.append(extendedInfo["url"])
                     media_extended.append(extendedInfo)
                 else:
-                    media.append(i["media_url_https"])
-                    extendedInfo["url"] = i["media_url_https"]
-                    altText=None
-                    if "ext_alt_text" in i:
-                        altText=i["ext_alt_text"]
-                    extendedInfo["altText"] = altText
-                    extendedInfo["type"] = "image"
-                    extendedInfo["size"] = {"width":i["original_info"]["width"],"height":i["original_info"]["height"]}
-                    extendedInfo["thumbnail_url"] = i["media_url_https"]
-                    extendedInfo["id_str"] = i["id_str"]
+                    extendedInfo = getExtendedImageInfo(i)
                     media_extended.append(extendedInfo)
+                    media.append(extendedInfo["url"])
 
         if "hashtags" in tweetL["entities"]:
             for i in tweetL["entities"]["hashtags"]:
                 hashtags.append(i["text"])
-    elif "card" in tweet and 'name' in tweet['card'] and tweet['card']['name'] == "player":
-        width = None
-        height = None
-        vidUrl = None
-        for i in tweet['card']['binding_values']:
-            if i['key'] == 'player_stream_url':
-                vidUrl = i['value']['string_value']
-            elif i['key'] == 'player_width':
-                width = int(i['value']['string_value'])
-            elif i['key'] == 'player_height':
-                height = int(i['value']['string_value'])
-        if vidUrl != None and width != None and height != None:
-            media.append(vidUrl)
-            media_extended.append({"url":vidUrl,"type":"video","size":{"width":width,"height":height}})
-
+    elif "card" in tweet or "tweet_card" in tweet:
+        cardData = tweet["card" if "card" in tweet else "tweet_card"]
+        bindingValues = None
+        if 'binding_values' in cardData:
+            bindingValues = cardData['binding_values']
+        elif 'legacy' in cardData and 'binding_values' in cardData['legacy']:
+            bindingValues = cardData['legacy']['binding_values']
+        if bindingValues != None:
+            if 'name' in cardData and cardData['name'] == "player":
+                width = None
+                height = None
+                vidUrl = None
+                for i in bindingValues:
+                    if i['key'] == 'player_stream_url':
+                        vidUrl = i['value']['string_value']
+                    elif i['key'] == 'player_width':
+                        width = int(i['value']['string_value'])
+                    elif i['key'] == 'player_height':
+                        height = int(i['value']['string_value'])
+                if vidUrl != None and width != None and height != None:
+                    media.append(vidUrl)
+                    media_extended.append({"url":vidUrl,"type":"video","size":{"width":width,"height":height}})
+            else:
+                for i in bindingValues:
+                    if i['key'] == 'unified_card' and 'value' in i and 'string_value' in i['value']:
+                        cardData = json.loads(i['value']['string_value'])
+                        media_key = cardData['component_objects']['media_1']['data']['id']
+                        media_entry = cardData['media_entities'][media_key]
+                        extendedInfo = getExtendedVideoOrGifInfo(media_entry)
+                        media.append(extendedInfo['url'])
+                        media_extended.append(extendedInfo)
+                        break
+                    elif i['key'] == 'photo_image_full_size_large' and 'value' in i and 'image_value' in i['value']:
+                        imgData = i['value']['image_value']
+                        imgurl = imgData['url']
+                        media.append(imgurl)
+                        media_extended.append({"url":imgurl,"type":"image","size":{"width":imgData['width'],"height":imgData['height']}})
+                        break
     if "article" in tweet:
         try:
             result = tweet["article"]["article_results"]["result"]
@@ -270,6 +314,7 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False):
         "replyingToID": replyingToID,
         "fetched_on": int(datetime.now().timestamp()),
         "retweetURL":retweetURL,
+        "translation":translation
     }
     try:
         apiObject["date_epoch"] = int(datetime.strptime(tweetL["created_at"], "%a %b %d %H:%M:%S %z %Y").timestamp())

vx_testdata.py

@@ -15,12 +15,12 @@ testVinePlayerTweet="https://twitter.com/Roblox/status/583302104342638592"
 testRetweetTweet="https://twitter.com/pdxdylan/status/1828570470222045294"
 
 testTextTweet_compare={'text': 'just setting up my twttr', 'date': 'Tue Mar 21 20:50:14 +0000 2006', 'tweetURL': 'https://twitter.com/jack/status/20', 'tweetID': '20', 'conversationID': '20', 'mediaURLs': [], 'media_extended': [], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': False, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1142974214}
-testVideoTweet_compare={'text': 'TikTok embeds on Discord/Telegram bait you with a fake play button, but to see the actual video you have to go to their website.\nAs a request from a friend, I made it so that if you add "vx" before "tiktok" on any link, it fixes that. https://t.co/QYpiVXUIrW', 'date': 'Fri Jun 24 18:17:31 +0000 2022', 'tweetURL': 'https://twitter.com/pdxdylan/status/1540398733669666818', 'tweetID': '1540398733669666818', 'conversationID': '1540398733669666818', 'mediaURLs': ['https://video.twimg.com/ext_tw_video/1540396699037929472/pu/vid/762x528/YxbXbT3X7vq4LWfC.mp4'], 'media_extended': [{'url': 'https://video.twimg.com/ext_tw_video/1540396699037929472/pu/vid/762x528/YxbXbT3X7vq4LWfC.mp4', 'type': 'video', 'size': {'width': 762, 'height': 528}, 'duration_millis': 13650, 'thumbnail_url': 'https://pbs.twimg.com/ext_tw_video_thumb/1540396699037929472/pu/img/l187Z6B9AHHxUKPV.jpg', 'altText': None, 'id_str': '1540396699037929472'}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1656094651}
+testVideoTweet_compare={'text': 'TikTok embeds on Discord/Telegram bait you with a fake play button, but to see the actual video you have to go to their website.\nAs a request from a friend, I made it so that if you add "vx" before "tiktok" on any link, it fixes that.', 'date': 'Fri Jun 24 18:17:31 +0000 2022', 'tweetURL': 'https://twitter.com/pdxdylan/status/1540398733669666818', 'tweetID': '1540398733669666818', 'conversationID': '1540398733669666818', 'mediaURLs': ['https://video.twimg.com/ext_tw_video/1540396699037929472/pu/vid/762x528/YxbXbT3X7vq4LWfC.mp4'], 'media_extended': [{'url': 'https://video.twimg.com/ext_tw_video/1540396699037929472/pu/vid/762x528/YxbXbT3X7vq4LWfC.mp4', 'type': 'video', 'size': {'width': 762, 'height': 528}, 'duration_millis': 13650, 'thumbnail_url': 'https://pbs.twimg.com/ext_tw_video_thumb/1540396699037929472/pu/img/l187Z6B9AHHxUKPV.jpg', 'altText': None, 'id_str': '1540396699037929472'}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1656094651}
 testMediaTweet_compare={'text': 'oh.', 'date': 'Wed Jun 08 23:05:14 +0000 2022', 'tweetURL': 'https://twitter.com/pdxdylan/status/1534672932106035200', 'tweetID': '1534672932106035200', 'conversationID': '1534672673422381057', 'mediaURLs': ['https://pbs.twimg.com/media/FUxAt5LWUAMol0N.png'], 'media_extended': [{'url': 'https://pbs.twimg.com/media/FUxAt5LWUAMol0N.png', 'altText': None, 'type': 'image', 'size': {'width': 927, 'height': 534}, 'thumbnail_url': 'https://pbs.twimg.com/media/FUxAt5LWUAMol0N.png', 'id_str': '1534672730213208067'}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'und', 'replyingTo': 'pdxdylan', 'replyingToID': '1534672673422381057', 'retweetURL': None, 'date_epoch': 1654729514}
-testMultiMediaTweet_compare={'text': 'Released #Retro64 1.0.9. Besides a lot of internal bug-fixes, this adds quicksand blocks, fixes the rendering for the castle stairs block, and adds a new model, Sonic! \nhttps://github.com/Retro64Mod/Retro64Mod/releases/tag/1.18.2-1.0.9 https://t.co/CWZaw4hzyg', 'date': 'Wed Jun 01 14:29:32 +0000 2022', 'tweetURL': 'https://twitter.com/pdxdylan/status/1532006436703715331', 'tweetID': '1532006436703715331', 'conversationID': '1532006436703715331', 'mediaURLs': ['https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png', 'https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png', 'https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png'], 'media_extended': [{'url': 'https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png', 'altText': None, 'type': 'image', 'size': {'width': 507, 'height': 507}, 'thumbnail_url': 'https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png', 'id_str': '1532004485966577667'}, {'url': 'https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png', 'altText': None, 'type': 'image', 'size': {'width': 396, 'height': 431}, 'thumbnail_url': 'https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png', 'id_str': '1532004975269797890'}, {'url': 'https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png', 'altText': None, 'type': 'image', 'size': {'width': 399, 'height': 341}, 'thumbnail_url': 'https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png', 'id_str': '1532005117553164291'}], 'possibly_sensitive': False, 'hashtags': ['Retro64'], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': 'https://vxtwitter.com/rendercombined.jpg?imgs=https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png,https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png,https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png', 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1654093772}
+testMultiMediaTweet_compare={'text': 'Released #Retro64 1.0.9. Besides a lot of internal bug-fixes, this adds quicksand blocks, fixes the rendering for the castle stairs block, and adds a new model, Sonic! \nhttps://github.com/Retro64Mod/Retro64Mod/releases/tag/1.18.2-1.0.9', 'date': 'Wed Jun 01 14:29:32 +0000 2022', 'tweetURL': 'https://twitter.com/pdxdylan/status/1532006436703715331', 'tweetID': '1532006436703715331', 'conversationID': '1532006436703715331', 'mediaURLs': ['https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png', 'https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png', 'https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png'], 'media_extended': [{'url': 'https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png', 'altText': None, 'type': 'image', 'size': {'width': 507, 'height': 507}, 'thumbnail_url': 'https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png', 'id_str': '1532004485966577667'}, {'url': 'https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png', 'altText': None, 'type': 'image', 'size': {'width': 396, 'height': 431}, 'thumbnail_url': 'https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png', 'id_str': '1532004975269797890'}, {'url': 'https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png', 'altText': None, 'type': 'image', 'size': {'width': 399, 'height': 341}, 'thumbnail_url': 'https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png', 'id_str': '1532005117553164291'}], 'possibly_sensitive': False, 'hashtags': ['Retro64'], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': 'https://vxtwitter.com/rendercombined.jpg?imgs=https://pbs.twimg.com/media/FULF9oxXwAMDI-C.png,https://pbs.twimg.com/media/FULGaHkWYAIBV5U.png,https://pbs.twimg.com/media/FULGiZnWQAMBRWl.png', 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1654093772}
 testQRTTweet_compare={'text': "vxTwitter has gotten a *ton* of usage recently, so I'd appreciate a donation to keep things running!\nhttps://x.com/pdxdylan/status/1518309187515781125", 'date': 'Fri Jan 06 21:37:43 +0000 2023', 'tweetURL': 'https://twitter.com/pdxdylan/status/1611477137319514129', 'tweetID': '1611477137319514129', 'conversationID': '1611476665821003776', 'mediaURLs': [], 'media_extended': [], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': 'https://twitter.com/i/status/1518309187515781125', 'allSameType': True, 'hasMedia': False, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': 'pdxdylan', 'replyingToID': '1611476665821003776', 'retweetURL': None, 'date_epoch': 1673041063}
 testQrtVideoTweet_compare={'text': 'good', 'date': 'Thu Jun 29 23:33:29 +0000 2023', 'tweetURL': 'https://twitter.com/pdxdylan/status/1674561759422578690', 'tweetID': '1674561759422578690', 'conversationID': '1674561759422578690', 'mediaURLs': [], 'media_extended': [], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': 'https://twitter.com/i/status/1674197531301904388', 'allSameType': True, 'hasMedia': False, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1688081609}
-testNSFWTweet_compare={'text': "ngl, I'm scared on finding out the cute Sprigatito's final evolution..\n\nso i had a bot generate it for me.... and I'm forever scarred https://t.co/itMay87vcS", 'date': 'Sat Oct 15 07:28:42 +0000 2022', 'tweetURL': 'https://twitter.com/kuyacoy/status/1581185279376838657', 'tweetID': '1581185279376838657', 'conversationID': '1581185279376838657', 'mediaURLs': ['https://pbs.twimg.com/media/FfF_gKwXgAIpnpD.jpg'], 'media_extended': [{'url': 'https://pbs.twimg.com/media/FfF_gKwXgAIpnpD.jpg', 'altText': None, 'type': 'image', 'size': {'width': 760, 'height': 926}, 'thumbnail_url': 'https://pbs.twimg.com/media/FfF_gKwXgAIpnpD.jpg', 'id_str': '1581185134803517442'}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1665818922}
+testNSFWTweet_compare={'text': "ngl, I'm scared on finding out the cute Sprigatito's final evolution..\n\nso i had a bot generate it for me.... and I'm forever scarred", 'date': 'Sat Oct 15 07:28:42 +0000 2022', 'tweetURL': 'https://twitter.com/kuyacoy/status/1581185279376838657', 'tweetID': '1581185279376838657', 'conversationID': '1581185279376838657', 'mediaURLs': ['https://pbs.twimg.com/media/FfF_gKwXgAIpnpD.jpg'], 'media_extended': [{'url': 'https://pbs.twimg.com/media/FfF_gKwXgAIpnpD.jpg', 'altText': None, 'type': 'image', 'size': {'width': 760, 'height': 926}, 'thumbnail_url': 'https://pbs.twimg.com/media/FfF_gKwXgAIpnpD.jpg', 'id_str': '1581185134803517442'}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1665818922}
 testPollTweet_compare={'text': 'I know when that hotline bling, that can only:', 'date': 'Mon Oct 05 22:57:25 +0000 2015', 'tweetURL': 'https://twitter.com/norm/status/651169346518056960', 'tweetID': '651169346518056960', 'conversationID': '651169346518056960', 'mediaURLs': [], 'media_extended': [], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': False, 'combinedMediaUrl': None, 'pollData': {'options': [{'name': 'Mean one thing', 'votes': 124875, 'percent': 78.82}, {'name': 'Mean multiple things', 'votes': 33554, 'percent': 21.18}]}, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1444085845}
 testMixedMediaTweet_compare={'text': 'Some of us here are definitely big nerds about beer, and could talk your ear off about it for days on end, but some of us are just "beer is nice"', 'date': 'Thu Feb 22 12:13:24 +0000 2024', 'tweetURL': 'https://twitter.com/salebeerfest/status/1760638922084741177', 'tweetID': '1760638922084741177', 'conversationID': '1760638922084741177', 'mediaURLs': ['https://pbs.twimg.com/media/GG8LwfuWoAANKhs.jpg', 'https://video.twimg.com/tweet_video/GG8LwqWX0AAZch0.mp4'], 'media_extended': [{'url': 'https://pbs.twimg.com/media/GG8LwfuWoAANKhs.jpg', 'altText': None, 'type': 'image', 'size': {'width': 858, 'height': 960}, 'thumbnail_url': 'https://pbs.twimg.com/media/GG8LwfuWoAANKhs.jpg', 'id_str': '1760638907102699520'}, {'url': 'https://video.twimg.com/tweet_video/GG8LwqWX0AAZch0.mp4', 'type': 'gif', 'size': {'width': 500, 'height': 500}, 'duration_millis': 0, 'thumbnail_url': 'https://pbs.twimg.com/tweet_video_thumb/GG8LwqWX0AAZch0.jpg', 'altText': None, 'id_str': '1760638909954904064'}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': False, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': None, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1708604004}
 testVinePlayerTweet_compare={'text': 'You wanted old ROBLOX back, you got it. Check out our sweet "new" look! #BringBackOldROBLOX https://vine.co/v/OL9VqvM6wJh', 'date': 'Wed Apr 01 16:17:13 +0000 2015', 'tweetURL': 'https://twitter.com/Roblox/status/583302104342638592', 'tweetID': '583302104342638592', 'conversationID': '583302104342638592', 'mediaURLs': ['https://v.cdn.vine.co/r/videos/20A1BE53011195086166081318912_3fe3b526b1a.1.5.3156516531034157495.mp4?versionId=DI1mMu7EI6zcLbvgucyp3GHebdz8.9cQ'], 'media_extended': [{'url': 'https://v.cdn.vine.co/r/videos/20A1BE53011195086166081318912_3fe3b526b1a.1.5.3156516531034157495.mp4?versionId=DI1mMu7EI6zcLbvgucyp3GHebdz8.9cQ', 'type': 'video', 'size': {'width': 435, 'height': 435}}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': {'options': []}, 'article': None, 'lang': 'en', 'replyingTo': None, 'replyingToID': None, 'retweetURL': None, 'date_epoch': 1427905033}