diff --git a/gifConvert/Dockerfile b/gifConvert/Dockerfile index 20ffa30..2e880dd 100644 --- a/gifConvert/Dockerfile +++ b/gifConvert/Dockerfile @@ -1,21 +1,72 @@ +ARG TARGETARCH FROM public.ecr.aws/lambda/python:3.12 AS builder RUN dnf -y install git cargo && dnf clean all RUN git clone https://github.com/ImageOptim/gifski /gifski WORKDIR /gifski RUN cargo build --release -FROM public.ecr.aws/lambda/python:3.12 AS ffmpeg -RUN dnf -y update -RUN dnf -y install git wget tar.x86_64 xz && dnf clean all -WORKDIR /ffmpeg -RUN wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz -RUN tar -xvf ffmpeg-release-amd64-static.tar.xz +#FROM public.ecr.aws/lambda/python:3.12 AS ffmpeg-linux-amd64 +#RUN dnf -y update +#RUN dnf -y install git wget tar.x86_64 xz && dnf clean all +#WORKDIR /ffmpeg +#RUN wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz +#RUN tar -xvf ffmpeg-release-amd64-static.tar.xz +#FROM public.ecr.aws/lambda/python:3.12 AS ffmpeg-linux-arm64 +#RUN dnf -y update +#RUN dnf -y install git wget tar xz && dnf clean all +#WORKDIR /ffmpeg +#RUN wget https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-arm64-static.tar.xz +#RUN tar -xvf ffmpeg-release-arm64-static.tar.xz + +#FROM ffmpeg-linux-${TARGETARCH} AS ffmpeg + +FROM public.ecr.aws/lambda/python:3.12 AS ffmpeg +# tl;dr: build ffmpeg from source with svt-av1 support and statically linked binaries + +# Install dependencies +RUN dnf remove -y microdnf-dnf && microdnf install -y dnf +RUN dnf group install "Development Tools" -y + +RUN mkdir /ffmpeg_sources +WORKDIR /ffmpeg_sources +RUN git clone https://git.ffmpeg.org/ffmpeg.git . +RUN git clone --branch stable --depth 1 https://code.videolan.org/videolan/x264.git +RUN git clone https://gitlab.com/AOMediaCodec/SVT-AV1.git + +RUN dnf install curl nasm glibc-static libstdc++-static cmake --allowerasing -y + +# Install x264 +WORKDIR /ffmpeg_sources/x264 +RUN PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" ./configure --prefix="/ffmpeg_build" --bindir="/bin" --enable-static +RUN make -j +RUN make install + +# Install SVT-AV1 +WORKDIR /ffmpeg_sources +WORKDIR /ffmpeg_sources/SVT-AV1/Build + +RUN PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" cmake .. -G"Unix Makefiles" -DCMAKE_BUILD_TYPE=Release -DENABLE_STATIC=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_C_FLAGS_INIT="-static" + +RUN make -j +RUN make install + +# Install FFmpeg +WORKDIR /ffmpeg_sources +RUN PKG_CONFIG_PATH="/ffmpeg_build/lib/pkgconfig" ./configure --prefix="/ffmpeg_build" \ + --pkg-config-flags="--static" --extra-ldexeflags="-static" --extra-libs="-lpthread -lm" --bindir="/bin" --disable-shared --enable-static --enable-gpl --enable-pthreads \ + --enable-libx264 --enable-libsvtav1 --disable-ffplay + +RUN make -j FROM public.ecr.aws/lambda/python:3.12 -COPY --from=builder /gifski/target/release/gifski ./ -COPY --from=ffmpeg /ffmpeg/ffmpeg-*-amd64-static/ffmpeg /usr/local/bin/ffmpeg -COPY --from=ffmpeg /ffmpeg/ffmpeg-*-amd64-static/ffprobe /usr/local/bin/ffprobe + +#COPY --from=builder /gifski/target/release/gifski /usr/local/bin/gifski +#COPY --from=ffmpeg /ffmpeg/ffmpeg-*-static/ffmpeg /usr/local/bin/ffmpeg +#COPY --from=ffmpeg /ffmpeg/ffmpeg-*-static/ffprobe /usr/local/bin/ffprobe +COPY --from=ffmpeg /ffmpeg_sources/ffmpeg /usr/local/bin/ffmpeg +COPY --from=ffmpeg /ffmpeg_sources/ffprobe /usr/local/bin/ffprobe + RUN pip install requests==2.32.3 # Copy function code diff --git a/gifConvert/__init__.py b/gifConvert/__init__.py index 6dff671..c777bdf 100644 --- a/gifConvert/__init__.py +++ b/gifConvert/__init__.py @@ -21,55 +21,44 @@ else: def extractStatus(url): return "" -def get_video_frame_rate(filename): - result = subprocess.run( - [ - "ffprobe", - "-v", - "error", - "-select_streams", - "v", - "-of", - "default=noprint_wrappers=1:nokey=1", - "-show_entries", - "stream=r_frame_rate", - filename, - ], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - result_string = result.stdout.decode('utf-8').split()[0].split('/') - fps = float(result_string[0])/float(result_string[1]) - return fps +def convert_video_to_gif(filename): + try: + new_filename = tempfile.mkstemp(suffix=".gif")[1] + print("converting gif w gifski") + p_ffmpeg = subprocess.Popen(["ffmpeg", "-i", filename, "-f", "yuv4mpegpipe", "-"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + p_gifski = subprocess.Popen(["gifski","--quality","70","--lossy-quality","30","-o", new_filename, "-"], stdin=p_ffmpeg.stdout, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) -def get_video_length_seconds(filename): - result = subprocess.run( - [ - "ffprobe", - "-v", - "error", - "-show_entries", - "format=duration", - "-of", - "default=noprint_wrappers=1:nokey=1", - filename, - ], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - result_string = result.stdout.decode('utf-8').split()[0] - return float(result_string) - -def loop_video_until_length(filename, length): - # use stream_loop to loop video until it's at least length seconds long - video_length = get_video_length_seconds(filename) - if video_length < length: - loops = int(length/video_length) - new_filename = tempfile.mkstemp(suffix=".mp4")[1] - subprocess.call(["ffmpeg","-stream_loop",str(loops),"-i",filename,"-c","copy","-y",new_filename],stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT) - - return new_filename - else: + p_ffmpeg.stdout.close() + ret_gifski = p_gifski.wait() + ret_ffmpeg = p_ffmpeg.wait() + if ret_gifski != 0: + print("err: gifski exited with code:", ret_gifski) + if ret_ffmpeg != 0: + print("err: ffmpeg exited with code:", ret_ffmpeg) + if os.path.isfile(new_filename) and os.path.getsize(new_filename) > 0: + return new_filename + else: + print("gifski failed to convert gif") + return filename + except Exception as e: + print("error converting gif (convert_video_to_gif):") + print(e) + return filename + +def convert_video_to_avif(filename): + try: + fd,new_filename = tempfile.mkstemp(suffix=".avif") + os.close(fd) + print("converting gif w ffmpeg to avif") + subprocess.call(["ffmpeg","-nostdin","-y", "-i", filename,"-pix_fmt","yuv420p","-an","-c:v","libsvtav1","-crf","30","-b:v","0","-threads","4", new_filename], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) + if os.path.isfile(new_filename) and os.path.getsize(new_filename) > 0: + return new_filename + else: + print("ffmpeg failed to convert avif") + return filename + except Exception as e: + print("error converting avif (convert_video_to_avif):") + print(e) return filename def redir(url): @@ -99,7 +88,7 @@ def lambda_handler(event, context): if useBucket: id=re.search(r"https:\/\/video\.twimg\.com\/tweet_video\/(.*?)\..*",url).group(1) - bfilename = str(id)+".mp4" + bfilename = str(id)+".avif" furl=f"https://gifs.vxtwitter.com/{bfilename}" #f"https://{bucketname}.s3.amazonaws.com/{bfilename}" print("get req for: "+url) try: @@ -121,7 +110,7 @@ def lambda_handler(event, context): print("error downloading video") return redir(url) - videoLocationLooped = loop_video_until_length(videoLocation, 30) + videoLocationLooped = convert_video_to_avif(videoLocation) if videoLocationLooped != videoLocation: os.remove(videoLocation) videoLocation = videoLocationLooped @@ -137,7 +126,7 @@ def lambda_handler(event, context): 'statusCode': 200, "headers": { - "Content-Type": "video/mp4" + "Content-Type": "image/avif", }, 'body': encoded_string, 'isBase64Encoded': True diff --git a/twitfix.py b/twitfix.py index da9732e..982a3bd 100644 --- a/twitfix.py +++ b/twitfix.py @@ -439,8 +439,13 @@ def twitfix(sub_path): suffix = media["suffix"] if media['type'] == "image": return Response(renderImageTweetEmbed(tweetData,media['url'] , appnameSuffix=suffix,embedIndex=embedIndex),headers={"Cache-Tag": "embed"}) - elif media['type'] == "video" or media['type'] == "gif": + elif media['type'] == "video": return Response(renderVideoTweetEmbed(tweetData,media,appnameSuffix=suffix,embedIndex=embedIndex),headers={"Cache-Tag": "embed"}) + elif media['type'] == "gif": + if "originalUrl" in media and media["url"] != media["originalUrl"]: + return Response(renderImageTweetEmbed(tweetData,media['url'] , appnameSuffix=suffix,embedIndex=embedIndex),headers={"Cache-Tag": "embed"}) + else: + return Response(renderVideoTweetEmbed(tweetData,media,appnameSuffix=suffix,embedIndex=embedIndex),headers={"Cache-Tag": "embed"}) return message(msgs.failedToScan) diff --git a/utils.py b/utils.py index 331c9f4..037bc04 100644 --- a/utils.py +++ b/utils.py @@ -88,7 +88,7 @@ def determineMediaToEmbed(tweetData,embedIndex = -1,convertGif = True): #if gcApi == "local": # TODO #gcApi = f"{config['config']['url']}/gifconvert" vurl=media['originalUrl'] if 'originalUrl' in media else media['url'] - media['url'] = gcApi + "/convert?url=" + vurl + media['url'] = gcApi + "/convert.avif?url=" + vurl suffix += " • GIF" media["suffix"] = suffix return media \ No newline at end of file