From d262fe10143f44e6c18b9a4c72d42922f2577011 Mon Sep 17 00:00:00 2001 From: Dylan Date: Fri, 30 Jan 2026 22:52:38 +0000 Subject: [PATCH] Update "gif" conversion logic --- gifConvert/Dockerfile | 37 ++++++------ gifConvert/__init__.py | 128 +++++++++++++++++++++++++++++------------ gifConvert/conv.sh | 64 --------------------- utils.py | 7 ++- 4 files changed, 112 insertions(+), 124 deletions(-) delete mode 100644 gifConvert/conv.sh diff --git a/gifConvert/Dockerfile b/gifConvert/Dockerfile index 8be33eb..20ffa30 100644 --- a/gifConvert/Dockerfile +++ b/gifConvert/Dockerfile @@ -1,30 +1,25 @@ -FROM public.ecr.aws/lambda/python:3.8 AS builder -RUN yum -y install git curl -RUN yum -y groupinstall 'Development Tools' -RUN git clone https://github.com/kohler/gifsicle -WORKDIR gifsicle -RUN autoreconf -i -RUN ./configure --disable-gifview --disable-gifdiff -RUN make -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y -WORKDIR /var/task -RUN git clone https://github.com/ImageOptim/gifski -WORKDIR gifski -RUN /root/.cargo/bin/cargo build --release +FROM public.ecr.aws/lambda/python:3.12 AS builder +RUN dnf -y install git cargo && dnf clean all +RUN git clone https://github.com/ImageOptim/gifski /gifski +WORKDIR /gifski +RUN cargo build --release - -FROM public.ecr.aws/lambda/python:3.8 -RUN yum -y update -RUN yum -y install git && yum -y install wget && yum -y install tar.x86_64 && yum -y install xz && yum clean all +FROM public.ecr.aws/lambda/python:3.12 AS ffmpeg +RUN dnf -y update +RUN dnf -y install git wget tar.x86_64 xz && dnf clean all +WORKDIR /ffmpeg RUN wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz RUN tar -xvf ffmpeg-release-amd64-static.tar.xz -RUN mv ff*/ffmpeg . && mv ff*/ffprobe . && rm *.tar.xz && rm -rf ff*/ -COPY --from=builder /var/task/gifsicle/src/gifsicle ./ -COPY --from=builder /var/task/gifski/target/release/gifski ./ + + +FROM public.ecr.aws/lambda/python:3.12 +COPY --from=builder /gifski/target/release/gifski ./ +COPY --from=ffmpeg /ffmpeg/ffmpeg-*-amd64-static/ffmpeg /usr/local/bin/ffmpeg +COPY --from=ffmpeg /ffmpeg/ffmpeg-*-amd64-static/ffprobe /usr/local/bin/ffprobe +RUN pip install requests==2.32.3 # Copy function code COPY __init__.py ${LAMBDA_TASK_ROOT}/app.py -COPY conv.sh ${LAMBDA_TASK_ROOT}/conv.sh # Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile) CMD [ "app.lambda_handler" ] \ No newline at end of file diff --git a/gifConvert/__init__.py b/gifConvert/__init__.py index 6d3f19b..6dff671 100644 --- a/gifConvert/__init__.py +++ b/gifConvert/__init__.py @@ -1,9 +1,22 @@ import base64 import os import subprocess -import json -import sys import tempfile +import urllib.request +import re +import botocore +import boto3 +import requests + +useBucket=False +bucketname=os.getenv('CF_BUCKET') +s3=None +if bucketname is None: + useBucket=False +else: + useBucket=True + s3 = boto3.client('s3',endpoint_url=os.getenv('CF_ENDPOINT'),aws_access_key_id=os.getenv('CF_KEY'),aws_secret_access_key=os.getenv('CF_KEY_SECRET')) + def extractStatus(url): return "" @@ -11,7 +24,7 @@ def extractStatus(url): def get_video_frame_rate(filename): result = subprocess.run( [ - "./ffprobe", + "ffprobe", "-v", "error", "-select_streams", @@ -32,7 +45,7 @@ def get_video_frame_rate(filename): def get_video_length_seconds(filename): result = subprocess.run( [ - "./ffprobe", + "ffprobe", "-v", "error", "-show_entries", @@ -47,54 +60,95 @@ def get_video_length_seconds(filename): result_string = result.stdout.decode('utf-8').split()[0] return float(result_string) -def calcEdits(vlen,loopTimes): - st="r" - for i in range(loopTimes): - st+=f'e{str((vlen*i))}-9999,0' - return st - def loop_video_until_length(filename, length): # use stream_loop to loop video until it's at least length seconds long video_length = get_video_length_seconds(filename) if video_length < length: loops = int(length/video_length) new_filename = tempfile.mkstemp(suffix=".mp4")[1] - #edits = calcEdits(video_length,loops) - out = subprocess.call(["ffmpeg","-stream_loop",str(loops),"-i",filename,"-c","copy","-y",new_filename],stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT) - #subprocess.run(["./MP4Box", "-add",filename,"-edits",f'1={edits}',new_filename]) + subprocess.call(["ffmpeg","-stream_loop",str(loops),"-i",filename,"-c","copy","-y",new_filename],stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT) + return new_filename else: return filename - +def redir(url): + return { + "statusCode": 307, + "headers": { + "Location": url + } + } def lambda_handler(event, context): if ("queryStringParameters" not in event): return { "statusCode": 400, - "body": "Invalid request." + "body": "Invalid request!" } url = event["queryStringParameters"].get("url","") - - # download video - videoLocation = tempfile.mkstemp(suffix=".mp4")[1] - subprocess.call(["wget","-O",videoLocation,url],stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT) - - videoLocationLooped = loop_video_until_length(videoLocation, 30) - if videoLocationLooped != videoLocation: - os.remove(videoLocation) - videoLocation = videoLocationLooped - - with open(videoLocation, "rb") as image_file: - encoded_string = base64.b64encode(image_file.read()).decode('ascii') - os.remove(videoLocation) - return { - 'statusCode': 200, - "headers": - { - "Content-Type": "video/mp4" - }, - 'body': encoded_string, - 'isBase64Encoded': True - } \ No newline at end of file + try: + if url == "": + return { + "statusCode": 400, + "body": "Invalid request!!" + } + if not url.startswith("https://video.twimg.com/tweet_video/"): + return redir(url) + + if useBucket: + id=re.search(r"https:\/\/video\.twimg\.com\/tweet_video\/(.*?)\..*",url).group(1) + bfilename = str(id)+".mp4" + furl=f"https://gifs.vxtwitter.com/{bfilename}" #f"https://{bucketname}.s3.amazonaws.com/{bfilename}" + print("get req for: "+url) + try: + s3.head_object(Bucket=bucketname, Key=bfilename) + print("found existing already: "+bfilename) + return redir(furl) + except botocore.exceptions.ClientError: + # Not found + pass + # download video + print("downloading: "+url) + videoLocation = tempfile.mkstemp(suffix=".mp4")[1] + response = requests.get(url, stream=True) + if response.status_code == 200: + with open(videoLocation, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + else: + print("error downloading video") + return redir(url) + + videoLocationLooped = loop_video_until_length(videoLocation, 30) + if videoLocationLooped != videoLocation: + os.remove(videoLocation) + videoLocation = videoLocationLooped + else: + os.remove(videoLocation) + return redir(url) + + if not useBucket: + with open(videoLocation, "rb") as image_file: + encoded_string = base64.b64encode(image_file.read()).decode('ascii') + os.remove(videoLocation) + return { + 'statusCode': 200, + "headers": + { + "Content-Type": "video/mp4" + }, + 'body': encoded_string, + 'isBase64Encoded': True + } + else: + with open(videoLocation, "rb") as image_file: + s3.upload_fileobj(image_file, bucketname, bfilename) + os.remove(videoLocation) + print("converted: "+url+" -> "+furl) + return redir(furl) + except Exception as e: + print("error converting gif: ") + print(e) + return redir(url) \ No newline at end of file diff --git a/gifConvert/conv.sh b/gifConvert/conv.sh deleted file mode 100644 index e24dc04..0000000 --- a/gifConvert/conv.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -e - -usage(){ - echo "Usage: $0 [options] output" - echo "Options:" - echo " --help Show this help" - echo " -u, --url URL of the video" - echo " -w, --max-width Maximum width of the output" - echo " -h, --max-height Maximum height of the output" - echo " -t, --threads Number of threads to use" - exit 1 -} - -URL="" -MAXW=400 -MAXH=267 -THREADS=1 -OUTPUT="out.gif" -FPS=10 - -while [ $# -gt 0 ]; do - case "$1" in - --help) - usage - ;; - -u|--url) - URL="$2" - shift - ;; - -w|--max-width) - MAXW="$2" - shift - ;; - -h|--max-height) - MAXH="$2" - shift - ;; - -t|--threads) - THREADS="$2" - shift - ;; - -f|--fps) - FPS="$2" - shift - ;; - -*) - echo "Unknown option: $1" - usage - ;; - *) - OUTPUT="$1" - ;; - esac - shift -done - -# make unique temp directory -TEMPDIR=$( mktemp -d ) - -./ffmpeg -i "$URL" -vf "scale=if(gte(iw\,ih)\,min($MAXW\,iw)\,-2):if(lt(iw\,ih)\,min($MAXH\,ih)\,-2)" -threads $THREADS "$TEMPDIR/frame%04d.png" -./gifski -o "$TEMPDIR/out.gif" --fast --fps $FPS --quality=90 $TEMPDIR/frame*.png -#./gifsicle -O3 "$TEMPDIR/out.gif" -o "$OUTPUT" -mv "$TEMPDIR/out.gif" "$OUTPUT" -rm -rf "$TEMPDIR" \ No newline at end of file diff --git a/utils.py b/utils.py index 3a305d7..331c9f4 100644 --- a/utils.py +++ b/utils.py @@ -83,9 +83,12 @@ def determineMediaToEmbed(tweetData,embedIndex = -1,convertGif = True): return media elif media['type'] == "video" or media['type'] == "gif": if media['type'] == "gif" and convertGif: - if config['config']['gifConvertAPI'] != "" and config['config']['gifConvertAPI'] != "none": + if config['config']['gifConvertAPI'] != "" and config['config']['gifConvertAPI'] != "none" and config['config']['gifConvertAPI'] != "local": + gcApi = config['config']['gifConvertAPI'] + #if gcApi == "local": # TODO + #gcApi = f"{config['config']['url']}/gifconvert" vurl=media['originalUrl'] if 'originalUrl' in media else media['url'] - media['url'] = config['config']['gifConvertAPI'] + "/convert?url=" + vurl + media['url'] = gcApi + "/convert?url=" + vurl suffix += " • GIF" media["suffix"] = suffix return media \ No newline at end of file