Merge branch 'main' into prod

.github/workflows/deploy.yml

@@ -64,3 +64,4 @@ jobs:
           VXTWITTER_REPO: ${{ secrets.VXTWITTER_REPO }}
           VXTWITTER_URL: ${{ secrets.VXTWITTER_URL }}
           VXTWITTER_COMBINATION_METHOD: ${{ secrets.VXTWITTER_COMBINATION_METHOD }}
+          VXTWITTER_GIF_CONVERT_API: ${{ secrets.VXTWITTER_GIF_CONVERT_API }}

.gitignore

@@ -10,4 +10,6 @@ _meta
 .serverless
 db/
 .coverage
-htmlcov/
+htmlcov/
+template
+build

configHandler.py

@@ -2,7 +2,7 @@ import json
 import os
 
 if ('RUNNING_TESTS' in os.environ):
-    config= {"config":{"link_cache":"ram","database":"","table":"","color":"","appname": "vxTwitter","repo": "https://github.com/dylanpdx/BetterTwitFix","url": "https://vxtwitter.com","combination_method": "local"}}
+    config= {"config":{"link_cache":"ram","database":"","table":"","color":"","appname": "vxTwitter","repo": "https://github.com/dylanpdx/BetterTwitFix","url": "https://vxtwitter.com","combination_method": "local","gifConvertAPI":""}}
 elif ('RUNNING_SERVERLESS' in os.environ and os.environ['RUNNING_SERVERLESS'] == '1'): # pragma: no cover
     config = {
             "config":{
@@ -13,7 +13,8 @@ elif ('RUNNING_SERVERLESS' in os.environ and os.environ['RUNNING_SERVERLESS'] ==
                 "appname": os.environ["VXTWITTER_APP_NAME"], 
                 "repo": os.environ["VXTWITTER_REPO"], 
                 "url": os.environ["VXTWITTER_URL"],
-                "combination_method": os.environ["VXTWITTER_COMBINATION_METHOD"] # can either be 'local' or a URL to a server handling requests in the same format
+                "combination_method": os.environ["VXTWITTER_COMBINATION_METHOD"], # can either be 'local' or a URL to a server handling requests in the same format
+                "gifConvertAPI":os.environ["VXTWITTER_GIF_CONVERT_API"]
             }
         }
 else:
@@ -29,7 +30,8 @@ else:
                     "appname": "vxTwitter", 
                     "repo": "https://github.com/dylanpdx/BetterTwitFix", 
                     "url": "https://vxtwitter.com",
-                    "combination_method": "local" # can either be 'local' or a URL to a server handling requests in the same format
+                    "combination_method": "local", # can either be 'local' or a URL to a server handling requests in the same format
+                    "gifConvertAPI":""
                 }
             }
 

gifConvert/Dockerfile

@@ -0,0 +1,30 @@
+FROM public.ecr.aws/lambda/python:3.8 AS builder
+RUN yum -y install git curl
+RUN yum -y groupinstall 'Development Tools'
+RUN git clone https://github.com/kohler/gifsicle
+WORKDIR gifsicle
+RUN autoreconf -i
+RUN ./configure --disable-gifview --disable-gifdiff
+RUN make
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
+WORKDIR /var/task
+RUN git clone https://github.com/ImageOptim/gifski
+WORKDIR gifski
+RUN /root/.cargo/bin/cargo build --release
+
+
+FROM public.ecr.aws/lambda/python:3.8
+RUN yum -y update
+RUN yum -y install git && yum -y install wget && yum -y install tar.x86_64 && yum -y install xz && yum clean all
+RUN wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz
+RUN tar -xvf ffmpeg-release-amd64-static.tar.xz
+RUN mv ff*/ffmpeg . && mv ff*/ffprobe . && rm *.tar.xz && rm -rf ff*/
+COPY --from=builder /var/task/gifsicle/src/gifsicle ./
+COPY --from=builder /var/task/gifski/target/release/gifski ./
+
+# Copy function code
+COPY __init__.py ${LAMBDA_TASK_ROOT}/app.py
+COPY conv.sh ${LAMBDA_TASK_ROOT}/conv.sh
+
+# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
+CMD [ "app.lambda_handler" ] 

gifConvert/__init__.py

@@ -0,0 +1,92 @@
+import base64
+import os
+import subprocess
+import json
+import sys
+import tempfile
+
+def extractStatus(url):
+    return ""
+
+def get_video_frame_rate(filename):
+    result = subprocess.run(
+        [
+            "./ffprobe",
+            "-v",
+            "error",
+            "-select_streams",
+            "v",
+            "-of",
+            "default=noprint_wrappers=1:nokey=1",
+            "-show_entries",
+            "stream=r_frame_rate",
+            filename,
+        ],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    )
+    result_string = result.stdout.decode('utf-8').split()[0].split('/')
+    fps = float(result_string[0])/float(result_string[1])
+    return fps
+
+def get_video_length_seconds(filename):
+    result = subprocess.run(
+        [
+            "./ffprobe",
+            "-v",
+            "error",
+            "-show_entries",
+            "format=duration",
+            "-of",
+            "default=noprint_wrappers=1:nokey=1",
+            filename,
+        ],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    )
+    result_string = result.stdout.decode('utf-8').split()[0]
+    return float(result_string)
+
+def loop_video_until_length(filename, length):
+    # use stream_loop to loop video until it's at least length seconds long
+    video_length = get_video_length_seconds(filename)
+    if video_length < length:
+        loops = int(length/video_length)
+        new_filename = tempfile.mkstemp(suffix=".mp4")[1]
+        out = subprocess.call(["./ffmpeg","-stream_loop",str(loops),"-i",filename,"-c","copy","-y",new_filename],stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT)
+        return new_filename
+    else:
+        return filename
+
+
+
+def lambda_handler(event, context):
+    if ("queryStringParameters" not in event):
+        return {
+            "statusCode": 400,
+            "body": "Invalid request."
+        }
+    
+    url = event["queryStringParameters"].get("url","")
+
+    # download video
+    videoLocation = tempfile.mkstemp(suffix=".mp4")[1]
+    subprocess.call(["wget","-O",videoLocation,url],stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT)
+
+    videoLocationLooped = loop_video_until_length(videoLocation, 30)
+    if videoLocationLooped != videoLocation:
+        os.remove(videoLocation)
+        videoLocation = videoLocationLooped
+
+    with open(videoLocation, "rb") as image_file:
+        encoded_string = base64.b64encode(image_file.read()).decode('ascii')
+    os.remove(videoLocation)
+    return {
+        'statusCode': 200,
+        "headers": 
+        {
+            "Content-Type": "video/mp4"
+        },
+        'body': encoded_string,
+        'isBase64Encoded': True
+    }

gifConvert/conv.sh

@@ -0,0 +1,64 @@
+#!/bin/bash -e
+
+usage(){
+    echo "Usage: $0 [options] output"
+    echo "Options:"
+    echo "  --help              Show this help"
+    echo "  -u, --url           URL of the video"
+    echo "  -w, --max-width     Maximum width of the output"
+    echo "  -h, --max-height    Maximum height of the output"
+    echo "  -t, --threads       Number of threads to use"
+    exit 1
+}
+
+URL=""
+MAXW=400
+MAXH=267
+THREADS=1
+OUTPUT="out.gif"
+FPS=10
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --help)
+            usage
+            ;;
+        -u|--url)
+            URL="$2"
+            shift
+            ;;
+        -w|--max-width)
+            MAXW="$2"
+            shift
+            ;;
+        -h|--max-height)
+            MAXH="$2"
+            shift
+            ;;
+        -t|--threads)
+            THREADS="$2"
+            shift
+            ;;
+        -f|--fps)
+            FPS="$2"
+            shift
+            ;;
+        -*)
+            echo "Unknown option: $1"
+            usage
+            ;;
+        *)
+            OUTPUT="$1"
+            ;;
+    esac
+    shift
+done
+
+# make unique temp directory
+TEMPDIR=$( mktemp -d )
+
+./ffmpeg -i "$URL" -vf "scale=if(gte(iw\,ih)\,min($MAXW\,iw)\,-2):if(lt(iw\,ih)\,min($MAXH\,ih)\,-2)" -threads $THREADS "$TEMPDIR/frame%04d.png"
+./gifski -o "$TEMPDIR/out.gif" --fast --fps $FPS --quality=90 $TEMPDIR/frame*.png
+#./gifsicle -O3 "$TEMPDIR/out.gif" -o "$OUTPUT"
+mv "$TEMPDIR/out.gif" "$OUTPUT"
+rm -rf "$TEMPDIR"

requirements.txt

@@ -1,5 +1,5 @@
 pymongo==4.3.3
-boto3==1.26.22
+boto3==1.26.37
 requests==2.28.1
 Pillow==9.3.0
 Flask==2.2.2

serverless.yml

@@ -26,6 +26,7 @@ provider:
     VXTWITTER_REPO: ${env:VXTWITTER_REPO, 'https://github.com/dylanpdx/BetterTwitFix'}
     VXTWITTER_URL: ${env:VXTWITTER_URL, 'https://vxtwitter.com'}
     VXTWITTER_COMBINATION_METHOD: ${env:VXTWITTER_COMBINATION_METHOD, 'local'}
+    VXTWITTER_GIF_CONVERT_API: ${env:VXTWITTER_GIF_CONVERT_API, ''}
 
 package:
   patterns:

templates/image.html

@@ -26,5 +26,5 @@
 
 <meta property="og:description" content="{{ desc }}" />
 
-<!--<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user=Twitter&link={{ urlLink }}&ttype=photo" type="application/json+oembed" title="{{ user }}">-->
+<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user=Twitter&link={{ urlUserLink }}&ttype=photo" type="application/json+oembed" title="{{ user }}">
 <meta http-equiv="refresh" content="0; url = {{ tweetLink }}" /> {% endblock %} {% block body %} Redirecting you to the tweet in a moment. <a href="{{ tweetLink }}">Or click here.</a> {% endblock %}

templates/index.html

@@ -26,5 +26,5 @@
 <meta property="og:image" content="{{ pic }}" />
 
 <!--!-->
-<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user={{ urlDesc }}&link={{ urlLink }}" type="application/json+oembed" title="{{ user }}">
+<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user={{ urlDesc }}&link={{ urlUserLink }}" type="application/json+oembed" title="{{ user }}">
 <meta http-equiv="refresh" content="0; url = {{ tweetLink }}" /> {% endblock %} {% block body %} Redirecting you to the tweet in a moment. <a href="{{ tweetLink }}">Or click here.</a> {% endblock %}

templates/text.html

@@ -30,5 +30,5 @@
 
 <meta property="og:description" content="{{ desc }}" />
 
-<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user=Twitter&link={{ urlLink }}&ttype=link" type="application/json+oembed" title="{{ user }}">
+<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user=Twitter&link={{ urlUserLink }}&ttype=link" type="application/json+oembed" title="{{ user }}">
 <meta http-equiv="refresh" content="0; url = {{ tweetLink }}" /> {% endblock %} {% block body %} Redirecting you to the tweet in a moment. <a href="{{ tweetLink }}">Or click here.</a> {% endblock %}

templates/video.html

@@ -21,5 +21,5 @@
 <meta property="og:image" content="{{ pic }}" />
 <meta property="og:description" content="{{ desc }}" />
 
-<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user={{ urlDesc }}&link={{ urlLink }}&ttype=video" type="application/json+oembed" title="{{ user }}">
+<link rel="alternate" href="{{ url }}/oembed.json?desc={{ urlUser }}&user={{ urlDesc }}&link={{ urlUserLink }}&ttype=video" type="application/json+oembed" title="{{ user }}">
 <meta http-equiv="refresh" content="0; url = {{ tweetLink }}" /> {% endblock %} {% block body %} Redirecting you to the tweet in a moment. <a href="{{ tweetLink }}">Or click here.</a> {% endblock %}

twitfix.py

@@ -195,6 +195,8 @@ def upgradeVNF(vnf):
             vnf['qrtURL'] = None
         else: # 
             vnf['qrtURL'] = f"https://twitter.com/{vnf['qrt']['screen_name']}/status/{vnf['qrt']['id']}"
+    if 'isGif' not in vnf:
+        vnf['isGif'] = False
     return vnf
 
 def getDefaultTTL(): # TTL for deleting items from the database
@@ -395,6 +397,7 @@ def getTemplate(template,vnf,desc,image,video_link,color,urlDesc,urlUser,urlLink
         time       = vnf['time'], 
         screenName = vnf['screen_name'], 
         vidlink    = embedVNF['url'], 
+        userLink   = f"https://twitter.com/{vnf['screen_name']}",
         pfp        = vnf['pfp'],  
         vidurl     = embedVNF['url'], 
         desc       = desc,
@@ -408,6 +411,7 @@ def getTemplate(template,vnf,desc,image,video_link,color,urlDesc,urlUser,urlLink
         urlDesc    = urlDesc, 
         urlUser    = urlUser, 
         urlLink    = urlLink,
+        urlUserLink= urllib.parse.quote(f"https://twitter.com/{vnf['screen_name']}"),
         tweetLink  = vnf['tweet'],
         videoSize  = embedVNF['size'] )
 
@@ -452,9 +456,14 @@ def embed(video_link, vnf, image):
             appNamePost = " - Image " + str(image+1) + "/" + str(vnf['images'][4])
         image = vnf['images'][image]
         template = 'image.html'
+
     if vnf['type'] == "Video":
+        if vnf['isGif'] == True and config['config']['gifConvertAPI'] != "" and config['config']['gifConvertAPI'] != "none":
+            vnf['url'] = f"{config['config']['gifConvertAPI']}/convert.mp4?url={vnf['url']}"
+            appNamePost = " - GIF"
         urlDesc = urllib.parse.quote(textwrap.shorten(desc, width=220, placeholder="..."))
         template = 'video.html'
+
     if vnf['type'] == "":
         urlDesc  = urllib.parse.quote(textwrap.shorten(desc, width=220, placeholder="..."))
         template = 'video.html'