From 7ff1031627358178fd02e67a4bf127ea99997dd4 Mon Sep 17 00:00:00 2001 From: Dylan Date: Wed, 28 Dec 2022 18:22:11 +0000 Subject: [PATCH] Squashed commit of the following: commit 4fe0af4e6a4a15ee4796a857e79ad0fe9b72c0f1 Author: Dylan Date: Wed Dec 28 18:09:25 2022 +0000 Misc. changes to MP4 Looping function commit 42ef8845c38c5a74e56da3c5f035d5a6c091c86a Author: Dylan Date: Tue Dec 27 20:00:22 2022 +0000 Abandoning raw gif conversion for now; looping vid instead commit b61938b340a02ac8cfab02048b7e9deaf87edbf5 Author: Dylan Date: Tue Dec 27 19:38:58 2022 +0000 More work on WIP gif generation commit 3bc6e7e0da1ce6ae905c80bbbaaa55a68050de52 Author: Dylan Date: Mon Dec 26 14:46:46 2022 +0000 Experimental gif conversion --- .github/workflows/deploy.yml | 1 + .gitignore | 4 +- configHandler.py | 8 ++-- gifConvert/Dockerfile | 30 ++++++++++++ gifConvert/__init__.py | 92 ++++++++++++++++++++++++++++++++++++ gifConvert/conv.sh | 64 +++++++++++++++++++++++++ serverless.yml | 1 + twitfix.py | 5 ++ 8 files changed, 201 insertions(+), 4 deletions(-) create mode 100644 gifConvert/Dockerfile create mode 100644 gifConvert/__init__.py create mode 100644 gifConvert/conv.sh diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3a10f12..634c4d3 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -64,3 +64,4 @@ jobs: VXTWITTER_REPO: ${{ secrets.VXTWITTER_REPO }} VXTWITTER_URL: ${{ secrets.VXTWITTER_URL }} VXTWITTER_COMBINATION_METHOD: ${{ secrets.VXTWITTER_COMBINATION_METHOD }} + VXTWITTER_GIF_CONVERT_API: ${{ secrets.VXTWITTER_GIF_CONVERT_API }} diff --git a/.gitignore b/.gitignore index 74e9504..e9c5ae8 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,6 @@ _meta .serverless db/ .coverage -htmlcov/ \ No newline at end of file +htmlcov/ +template +build diff --git a/configHandler.py b/configHandler.py index 8fb619d..37f84dc 100644 --- a/configHandler.py +++ b/configHandler.py @@ -2,7 +2,7 @@ import json import os if ('RUNNING_TESTS' in os.environ): - config= {"config":{"link_cache":"ram","database":"","table":"","color":"","appname": "vxTwitter","repo": "https://github.com/dylanpdx/BetterTwitFix","url": "https://vxtwitter.com","combination_method": "local"}} + config= {"config":{"link_cache":"ram","database":"","table":"","color":"","appname": "vxTwitter","repo": "https://github.com/dylanpdx/BetterTwitFix","url": "https://vxtwitter.com","combination_method": "local","gifConvertAPI":""}} elif ('RUNNING_SERVERLESS' in os.environ and os.environ['RUNNING_SERVERLESS'] == '1'): # pragma: no cover config = { "config":{ @@ -13,7 +13,8 @@ elif ('RUNNING_SERVERLESS' in os.environ and os.environ['RUNNING_SERVERLESS'] == "appname": os.environ["VXTWITTER_APP_NAME"], "repo": os.environ["VXTWITTER_REPO"], "url": os.environ["VXTWITTER_URL"], - "combination_method": os.environ["VXTWITTER_COMBINATION_METHOD"] # can either be 'local' or a URL to a server handling requests in the same format + "combination_method": os.environ["VXTWITTER_COMBINATION_METHOD"], # can either be 'local' or a URL to a server handling requests in the same format + "gifConvertAPI":os.environ["VXTWITTER_GIF_CONVERT_API"] } } else: @@ -29,7 +30,8 @@ else: "appname": "vxTwitter", "repo": "https://github.com/dylanpdx/BetterTwitFix", "url": "https://vxtwitter.com", - "combination_method": "local" # can either be 'local' or a URL to a server handling requests in the same format + "combination_method": "local", # can either be 'local' or a URL to a server handling requests in the same format + "gifConvertAPI":"" } } diff --git a/gifConvert/Dockerfile b/gifConvert/Dockerfile new file mode 100644 index 0000000..8be33eb --- /dev/null +++ b/gifConvert/Dockerfile @@ -0,0 +1,30 @@ +FROM public.ecr.aws/lambda/python:3.8 AS builder +RUN yum -y install git curl +RUN yum -y groupinstall 'Development Tools' +RUN git clone https://github.com/kohler/gifsicle +WORKDIR gifsicle +RUN autoreconf -i +RUN ./configure --disable-gifview --disable-gifdiff +RUN make +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y +WORKDIR /var/task +RUN git clone https://github.com/ImageOptim/gifski +WORKDIR gifski +RUN /root/.cargo/bin/cargo build --release + + +FROM public.ecr.aws/lambda/python:3.8 +RUN yum -y update +RUN yum -y install git && yum -y install wget && yum -y install tar.x86_64 && yum -y install xz && yum clean all +RUN wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz +RUN tar -xvf ffmpeg-release-amd64-static.tar.xz +RUN mv ff*/ffmpeg . && mv ff*/ffprobe . && rm *.tar.xz && rm -rf ff*/ +COPY --from=builder /var/task/gifsicle/src/gifsicle ./ +COPY --from=builder /var/task/gifski/target/release/gifski ./ + +# Copy function code +COPY __init__.py ${LAMBDA_TASK_ROOT}/app.py +COPY conv.sh ${LAMBDA_TASK_ROOT}/conv.sh + +# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile) +CMD [ "app.lambda_handler" ] \ No newline at end of file diff --git a/gifConvert/__init__.py b/gifConvert/__init__.py new file mode 100644 index 0000000..6f08ebb --- /dev/null +++ b/gifConvert/__init__.py @@ -0,0 +1,92 @@ +import base64 +import os +import subprocess +import json +import sys +import tempfile + +def extractStatus(url): + return "" + +def get_video_frame_rate(filename): + result = subprocess.run( + [ + "./ffprobe", + "-v", + "error", + "-select_streams", + "v", + "-of", + "default=noprint_wrappers=1:nokey=1", + "-show_entries", + "stream=r_frame_rate", + filename, + ], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + result_string = result.stdout.decode('utf-8').split()[0].split('/') + fps = float(result_string[0])/float(result_string[1]) + return fps + +def get_video_length_seconds(filename): + result = subprocess.run( + [ + "./ffprobe", + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + filename, + ], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + result_string = result.stdout.decode('utf-8').split()[0] + return float(result_string) + +def loop_video_until_length(filename, length): + # use stream_loop to loop video until it's at least length seconds long + video_length = get_video_length_seconds(filename) + if video_length < length: + loops = int(length/video_length) + new_filename = tempfile.mkstemp(suffix=".mp4")[1] + out = subprocess.call(["./ffmpeg","-stream_loop",str(loops),"-i",filename,"-c","copy","-y",new_filename],stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT) + return new_filename + else: + return filename + + + +def lambda_handler(event, context): + if ("queryStringParameters" not in event): + return { + "statusCode": 400, + "body": "Invalid request." + } + + url = event["queryStringParameters"].get("url","") + + # download video + videoLocation = tempfile.mkstemp(suffix=".mp4")[1] + subprocess.call(["wget","-O",videoLocation,url],stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT) + + videoLocationLooped = loop_video_until_length(videoLocation, 30) + if videoLocationLooped != videoLocation: + os.remove(videoLocation) + videoLocation = videoLocationLooped + + with open(videoLocation, "rb") as image_file: + encoded_string = base64.b64encode(image_file.read()).decode('ascii') + os.remove(videoLocation) + return { + 'statusCode': 200, + "headers": + { + "Content-Type": "video/mp4" + }, + 'body': encoded_string, + 'isBase64Encoded': True + } \ No newline at end of file diff --git a/gifConvert/conv.sh b/gifConvert/conv.sh new file mode 100644 index 0000000..e24dc04 --- /dev/null +++ b/gifConvert/conv.sh @@ -0,0 +1,64 @@ +#!/bin/bash -e + +usage(){ + echo "Usage: $0 [options] output" + echo "Options:" + echo " --help Show this help" + echo " -u, --url URL of the video" + echo " -w, --max-width Maximum width of the output" + echo " -h, --max-height Maximum height of the output" + echo " -t, --threads Number of threads to use" + exit 1 +} + +URL="" +MAXW=400 +MAXH=267 +THREADS=1 +OUTPUT="out.gif" +FPS=10 + +while [ $# -gt 0 ]; do + case "$1" in + --help) + usage + ;; + -u|--url) + URL="$2" + shift + ;; + -w|--max-width) + MAXW="$2" + shift + ;; + -h|--max-height) + MAXH="$2" + shift + ;; + -t|--threads) + THREADS="$2" + shift + ;; + -f|--fps) + FPS="$2" + shift + ;; + -*) + echo "Unknown option: $1" + usage + ;; + *) + OUTPUT="$1" + ;; + esac + shift +done + +# make unique temp directory +TEMPDIR=$( mktemp -d ) + +./ffmpeg -i "$URL" -vf "scale=if(gte(iw\,ih)\,min($MAXW\,iw)\,-2):if(lt(iw\,ih)\,min($MAXH\,ih)\,-2)" -threads $THREADS "$TEMPDIR/frame%04d.png" +./gifski -o "$TEMPDIR/out.gif" --fast --fps $FPS --quality=90 $TEMPDIR/frame*.png +#./gifsicle -O3 "$TEMPDIR/out.gif" -o "$OUTPUT" +mv "$TEMPDIR/out.gif" "$OUTPUT" +rm -rf "$TEMPDIR" \ No newline at end of file diff --git a/serverless.yml b/serverless.yml index b5f039d..35bf26f 100644 --- a/serverless.yml +++ b/serverless.yml @@ -26,6 +26,7 @@ provider: VXTWITTER_REPO: ${env:VXTWITTER_REPO, 'https://github.com/dylanpdx/BetterTwitFix'} VXTWITTER_URL: ${env:VXTWITTER_URL, 'https://vxtwitter.com'} VXTWITTER_COMBINATION_METHOD: ${env:VXTWITTER_COMBINATION_METHOD, 'local'} + VXTWITTER_GIF_CONVERT_API: ${env:VXTWITTER_GIF_CONVERT_API, ''} package: patterns: diff --git a/twitfix.py b/twitfix.py index 71eaa98..9c72bf2 100644 --- a/twitfix.py +++ b/twitfix.py @@ -454,9 +454,14 @@ def embed(video_link, vnf, image): appNamePost = " - Image " + str(image+1) + "/" + str(vnf['images'][4]) image = vnf['images'][image] template = 'image.html' + if vnf['type'] == "Video": + if vnf['isGif'] == True and config['config']['gifConvertAPI'] != "" and config['config']['gifConvertAPI'] != "none": + vnf['url'] = f"{config['config']['gifConvertAPI']}/convert.mp4?url={vnf['url']}" + appNamePost = " - GIF" urlDesc = urllib.parse.quote(textwrap.shorten(desc, width=220, placeholder="...")) template = 'video.html' + if vnf['type'] == "": urlDesc = urllib.parse.quote(textwrap.shorten(desc, width=220, placeholder="...")) template = 'video.html'