Update "gif" conversion logic

gifConvert/Dockerfile

@@ -1,30 +1,25 @@
-FROM public.ecr.aws/lambda/python:3.8 AS builder
+FROM public.ecr.aws/lambda/python:3.12 AS builder
-RUN yum -y install git curl
+RUN dnf -y install git cargo && dnf clean all
-RUN yum -y groupinstall 'Development Tools'
+RUN git clone https://github.com/ImageOptim/gifski /gifski
-RUN git clone https://github.com/kohler/gifsicle
+WORKDIR /gifski
-WORKDIR gifsicle
+RUN cargo build --release
-RUN autoreconf -i
-RUN ./configure --disable-gifview --disable-gifdiff
-RUN make
-RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
-WORKDIR /var/task
-RUN git clone https://github.com/ImageOptim/gifski
-WORKDIR gifski
-RUN /root/.cargo/bin/cargo build --release
 
-
+FROM public.ecr.aws/lambda/python:3.12 AS ffmpeg
-FROM public.ecr.aws/lambda/python:3.8
+RUN dnf -y update
-RUN yum -y update
+RUN dnf -y install git wget tar.x86_64 xz && dnf clean all
-RUN yum -y install git && yum -y install wget && yum -y install tar.x86_64 && yum -y install xz && yum clean all
+WORKDIR /ffmpeg
 RUN wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz
 RUN tar -xvf ffmpeg-release-amd64-static.tar.xz
-RUN mv ff*/ffmpeg . && mv ff*/ffprobe . && rm *.tar.xz && rm -rf ff*/
+
-COPY --from=builder /var/task/gifsicle/src/gifsicle ./
+
-COPY --from=builder /var/task/gifski/target/release/gifski ./
+FROM public.ecr.aws/lambda/python:3.12
+COPY --from=builder /gifski/target/release/gifski ./
+COPY --from=ffmpeg /ffmpeg/ffmpeg-*-amd64-static/ffmpeg /usr/local/bin/ffmpeg
+COPY --from=ffmpeg /ffmpeg/ffmpeg-*-amd64-static/ffprobe /usr/local/bin/ffprobe
+RUN pip install requests==2.32.3
 
 # Copy function code
 COPY __init__.py ${LAMBDA_TASK_ROOT}/app.py
-COPY conv.sh ${LAMBDA_TASK_ROOT}/conv.sh
 
 # Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
 CMD [ "app.lambda_handler" ] 

gifConvert/__init__.py

@@ -1,9 +1,22 @@
 import base64
 import os
 import subprocess
-import json
-import sys
 import tempfile
+import urllib.request
+import re
+import botocore
+import boto3
+import requests
+
+useBucket=False
+bucketname=os.getenv('CF_BUCKET')
+s3=None
+if bucketname is None:
+    useBucket=False
+else:
+    useBucket=True
+    s3 = boto3.client('s3',endpoint_url=os.getenv('CF_ENDPOINT'),aws_access_key_id=os.getenv('CF_KEY'),aws_secret_access_key=os.getenv('CF_KEY_SECRET'))
+
 
 def extractStatus(url):
     return ""
@@ -11,7 +24,7 @@ def extractStatus(url):
 def get_video_frame_rate(filename):
     result = subprocess.run(
         [
-            "./ffprobe",
+            "ffprobe",
             "-v",
             "error",
             "-select_streams",
@@ -32,7 +45,7 @@ def get_video_frame_rate(filename):
 def get_video_length_seconds(filename):
     result = subprocess.run(
         [
-            "./ffprobe",
+            "ffprobe",
             "-v",
             "error",
             "-show_entries",
@@ -47,54 +60,95 @@ def get_video_length_seconds(filename):
     result_string = result.stdout.decode('utf-8').split()[0]
     return float(result_string)
 
-def calcEdits(vlen,loopTimes):
-    st="r"
-    for i in range(loopTimes):
-        st+=f'e{str((vlen*i))}-9999,0'
-    return st
-
 def loop_video_until_length(filename, length):
     # use stream_loop to loop video until it's at least length seconds long
     video_length = get_video_length_seconds(filename)
     if video_length < length:
         loops = int(length/video_length)
         new_filename = tempfile.mkstemp(suffix=".mp4")[1]
-        #edits = calcEdits(video_length,loops)
+        subprocess.call(["ffmpeg","-stream_loop",str(loops),"-i",filename,"-c","copy","-y",new_filename],stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT)
-        out = subprocess.call(["ffmpeg","-stream_loop",str(loops),"-i",filename,"-c","copy","-y",new_filename],stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT)
+        
-        #subprocess.run(["./MP4Box", "-add",filename,"-edits",f'1={edits}',new_filename])
         return new_filename
     else:
         return filename
 
-
+def redir(url):
+    return {
+        "statusCode": 307,
+        "headers": {
+            "Location": url
+        }
+    }
 
 def lambda_handler(event, context):
     if ("queryStringParameters" not in event):
         return {
             "statusCode": 400,
-            "body": "Invalid request."
+            "body": "Invalid request!"
         }
     
     url = event["queryStringParameters"].get("url","")
-
+    try:
-    # download video
+        if url == "":
-    videoLocation = tempfile.mkstemp(suffix=".mp4")[1]
+            return {
-    subprocess.call(["wget","-O",videoLocation,url],stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT)
+                "statusCode": 400,
-
+                "body": "Invalid request!!"
-    videoLocationLooped = loop_video_until_length(videoLocation, 30)
+            }
-    if videoLocationLooped != videoLocation:
+        if not url.startswith("https://video.twimg.com/tweet_video/"):
-        os.remove(videoLocation)
+            return redir(url)
-        videoLocation = videoLocationLooped
+    
-
+        if useBucket:
-    with open(videoLocation, "rb") as image_file:
+            id=re.search(r"https:\/\/video\.twimg\.com\/tweet_video\/(.*?)\..*",url).group(1)
-        encoded_string = base64.b64encode(image_file.read()).decode('ascii')
+            bfilename = str(id)+".mp4"
-    os.remove(videoLocation)
+            furl=f"https://gifs.vxtwitter.com/{bfilename}" #f"https://{bucketname}.s3.amazonaws.com/{bfilename}"
-    return {
+            print("get req for: "+url)
-        'statusCode': 200,
+            try:
-        "headers": 
+                s3.head_object(Bucket=bucketname, Key=bfilename)
-        {
+                print("found existing already: "+bfilename)
-            "Content-Type": "video/mp4"
+                return redir(furl)
-        },
+            except botocore.exceptions.ClientError:
-        'body': encoded_string,
+                # Not found
-        'isBase64Encoded': True
+                pass
-    }
+        # download video
+        print("downloading: "+url)
+        videoLocation = tempfile.mkstemp(suffix=".mp4")[1]
+        response = requests.get(url, stream=True)
+        if response.status_code == 200:
+            with open(videoLocation, 'wb') as f:
+                for chunk in response.iter_content(chunk_size=8192):
+                    f.write(chunk)
+        else:
+            print("error downloading video")
+            return redir(url)
+    
+        videoLocationLooped = loop_video_until_length(videoLocation, 30)
+        if videoLocationLooped != videoLocation:
+            os.remove(videoLocation)
+            videoLocation = videoLocationLooped
+        else:
+            os.remove(videoLocation)
+            return redir(url)
+        
+        if not useBucket:
+            with open(videoLocation, "rb") as image_file:
+                encoded_string = base64.b64encode(image_file.read()).decode('ascii')
+                os.remove(videoLocation)
+                return {
+                    'statusCode': 200,
+                    "headers": 
+                    {
+                        "Content-Type": "video/mp4"
+                    },
+                    'body': encoded_string,
+                    'isBase64Encoded': True
+                }
+        else:
+            with open(videoLocation, "rb") as image_file:
+                s3.upload_fileobj(image_file, bucketname, bfilename)
+            os.remove(videoLocation)
+            print("converted: "+url+" -> "+furl)
+            return redir(furl)
+    except Exception as e:
+        print("error converting gif: ")
+        print(e)
+        return redir(url)

gifConvert/conv.sh

@@ -1,64 +0,0 @@
-#!/bin/bash -e
-
-usage(){
-    echo "Usage: $0 [options] output"
-    echo "Options:"
-    echo "  --help              Show this help"
-    echo "  -u, --url           URL of the video"
-    echo "  -w, --max-width     Maximum width of the output"
-    echo "  -h, --max-height    Maximum height of the output"
-    echo "  -t, --threads       Number of threads to use"
-    exit 1
-}
-
-URL=""
-MAXW=400
-MAXH=267
-THREADS=1
-OUTPUT="out.gif"
-FPS=10
-
-while [ $# -gt 0 ]; do
-    case "$1" in
-        --help)
-            usage
-            ;;
-        -u|--url)
-            URL="$2"
-            shift
-            ;;
-        -w|--max-width)
-            MAXW="$2"
-            shift
-            ;;
-        -h|--max-height)
-            MAXH="$2"
-            shift
-            ;;
-        -t|--threads)
-            THREADS="$2"
-            shift
-            ;;
-        -f|--fps)
-            FPS="$2"
-            shift
-            ;;
-        -*)
-            echo "Unknown option: $1"
-            usage
-            ;;
-        *)
-            OUTPUT="$1"
-            ;;
-    esac
-    shift
-done
-
-# make unique temp directory
-TEMPDIR=$( mktemp -d )
-
-./ffmpeg -i "$URL" -vf "scale=if(gte(iw\,ih)\,min($MAXW\,iw)\,-2):if(lt(iw\,ih)\,min($MAXH\,ih)\,-2)" -threads $THREADS "$TEMPDIR/frame%04d.png"
-./gifski -o "$TEMPDIR/out.gif" --fast --fps $FPS --quality=90 $TEMPDIR/frame*.png
-#./gifsicle -O3 "$TEMPDIR/out.gif" -o "$OUTPUT"
-mv "$TEMPDIR/out.gif" "$OUTPUT"
-rm -rf "$TEMPDIR"

utils.py

@@ -83,9 +83,12 @@ def determineMediaToEmbed(tweetData,embedIndex = -1,convertGif = True):
             return media
         elif media['type'] == "video" or media['type'] == "gif":
             if media['type'] == "gif" and convertGif:
-                if config['config']['gifConvertAPI'] != "" and config['config']['gifConvertAPI'] != "none":
+                if config['config']['gifConvertAPI'] != "" and config['config']['gifConvertAPI'] != "none" and config['config']['gifConvertAPI'] != "local":
+                    gcApi = config['config']['gifConvertAPI']
+                    #if gcApi == "local": # TODO
+                        #gcApi = f"{config['config']['url']}/gifconvert"
                     vurl=media['originalUrl'] if 'originalUrl' in media else media['url']
-                    media['url'] = config['config']['gifConvertAPI'] + "/convert?url=" + vurl
+                    media['url'] = gcApi + "/convert?url=" + vurl
                     suffix += " • GIF"
                     media["suffix"] = suffix
         return media