Merge branch 'main' into tests

configHandler.py

@@ -11,8 +11,7 @@ if ('RUNNING_SERVERLESS' in os.environ and os.environ['RUNNING_SERVERLESS'] == '
                 "appname": os.environ["VXTWITTER_APP_NAME"], 
                 "repo": os.environ["VXTWITTER_REPO"], 
                 "url": os.environ["VXTWITTER_URL"],
-                "combination_method": os.environ["VXTWITTER_COMBINATION_METHOD"], # can either be 'local' or a URL to a server handling requests in the same format
+                "combination_method": os.environ["VXTWITTER_COMBINATION_METHOD"] # can either be 'local' or a URL to a server handling requests in the same format
-                "apiMirrors":[]
             }
         }
 else:
@@ -28,8 +27,7 @@ else:
                     "appname": "vxTwitter", 
                     "repo": "https://github.com/dylanpdx/BetterTwitFix", 
                     "url": "https://vxtwitter.com",
-                    "combination_method": "local", # can either be 'local' or a URL to a server handling requests in the same format
+                    "combination_method": "local" # can either be 'local' or a URL to a server handling requests in the same format
-                    "apiMirrors":[]
                 }
             }
 

msgs.py

@@ -1,5 +1,5 @@
-failedToScan="Failed to scan your link! This may be due to an incorrect link, private account, or the twitter API itself might be having issues (Check here: https://api.twitterstat.us/)\nIt's also possible that Twitter is API limiting me, in which case I can't do anything about it."
+failedToScan="Failed to scan your link! This may be due to an incorrect link, private/suspended account, deleted tweet, or Twitter itself might be having issues (Check here: https://api.twitterstat.us/)"
-
+failedToScanExtra = "\n\nTwitter gave me this error: "
 
 def genLikesDisplay(vnf):
     return ("\n\n💖 " + str(vnf['likes']) + " 🔁 " + str(vnf['rts']) + "\n")

readme.md

@@ -41,15 +41,9 @@ vxTwitter generates a config.json in its root directory the first time you run i
 
 - **db**: Caches all links to a mongoDB database. This should be used it you are using uWSGI and are not just running the script on its own as one worker
 - **json**: This saves cached links to a local **links.json** file
+- **dynamodb**: Saves cached links to a DynamoDB database - set `table` to the table name to cache links to.
 - **none**: Does not cache requests. Not reccomended as you can easily use up your Twitter API credits with this. Intended for use with another cache system (i.e NGINX uwsgi_cache)
 
-**method** - ( Options: **youtube-dl**, **api**, **hybrid** ) 
-
-- **youtube-dl**: the original method for grabbing twitter video links, this uses a guest token provided via youtube-dl and should work well for individual instances, but may not scale up to a very large amount of usage
-
-- **api**: this directly uses the twitter API to grab tweet info, limited to 900 calls per 15m
-- **hybrid**: This will start off by using the twitter API to grab tweet info, but if the rate limit is reached or the api fails for any other reason it will switch over to youtube-dl to avoid downtime
-
 **color** - Accepts a hex formatted color code, can change the embed color
 
 **appname** - Can change the app name easily wherever it's shown
@@ -60,13 +54,8 @@ vxTwitter generates a config.json in its root directory the first time you run i
 
 **combination_method** - using c.vxtwitter as the url causes vxTwitter to combine all images in the post into one. This is CPU intensive, so you might not want it running on the same machine that's serving requests. When `combination_method` is set to `local`, it will use the local machine to combine the images. This requires pillow to be installed. If you want to use another server, replace `local` with the URL to the endpoint which combines images. Both methods use the code in the `combineImg` module. Inside, there's also a `Dockerfile` intended to be deployed as a combination endpoint on an [AWS Lambda function](https://docs.aws.amazon.com/lambda/latest/dg/images-create.html).
 
-**apiMirrors** - During an influx of traffic (i.e when Twitter embeds break!) it's very likely that Twitter will begin to block you for sending too many requests.
-This is an array of replacement Twitter API URLs which can be called upon when requesting tweet info locally fails. An example of an implementation of an API Mirror is in the twExtract directory, and is intended to be hosted on AWS Lambda. If multiple mirror URLs are specified, one will be selected at random.
-
-This project is licensed under the **Do What The Fuck You Want Public License**
-
-
-
 ## Other stuff
 
 We check for t.co links in non video tweets, and if one is found, we direct the discord useragent to embed that link directly, this means that twitter links containing youtube / vimeo links will automatically embed those as if you had just directly linked to that content
+
+This project is licensed under the **Do What The Fuck You Want Public License**

serverless.yml

@@ -47,8 +47,8 @@ functions:
   vxTwitterApp:
     handler: wsgi_handler.handler
     url: true
-    timeout: 15
+    timeout: 5
-    MemorySize: 1024
+    memorySize: 150
     layers:
       - Ref: PythonRequirementsLambdaLayer
 

templates/rawvideo.html

@@ -0,0 +1,16 @@
+{% extends 'base.html' %} {% block head %}
+<meta content='text/html; charset=UTF-8' http-equiv='Content-Type' />
+
+<meta name="twitter:player:stream" content="{{ vidurl }}" />
+<meta name="twitter:player:stream:content_type" content="video/mp4" />
+<meta name="twitter:player:width" content="{{ videoSize['width'] }}" />
+<meta name="twitter:player:height" content="{{ videoSize['height'] }}" />
+<meta property="og:url" content="{{ vidurl }}" />
+<meta property="og:video" content="{{ vidurl }}" />
+<meta property="og:video:secure_url" content="{{ vidurl }}" />
+<meta property="og:video:type" content="video/mp4" />
+<meta property="og:video:width" content="{{ videoSize['width'] }}" />
+<meta property="og:video:height" content="{{ videoSize['height'] }}" />
+<meta name="twitter:card" content="player" />
+
+<meta http-equiv="refresh" content="0; url = {{ vidurl }}" /> {% endblock %} {% block body %} Redirecting you to the video in a moment. <a href="{{ vidurl }}">Or click here.</a> {% endblock %}

templates/video.html

@@ -6,8 +6,8 @@
 <meta name="twitter:card" content="player" />
 <meta name="twitter:title" content="{{ user }} (@{{ screenName }}) {% if verified %}☑️{% else %}{% endif %}" />
 <meta name="twitter:image" content="{{ pic }}" />
-<meta name="twitter:player:width" content="720" />
+<meta name="twitter:player:width" content="{{ videoSize['width'] }}" />
-<meta name="twitter:player:height" content="480" />
+<meta name="twitter:player:height" content="{{ videoSize['height'] }}" />
 <meta name="twitter:player:stream" content="{{ vidurl }}" />
 <meta name="twitter:player:stream:content_type" content="video/mp4" />
 
@@ -15,8 +15,8 @@
 <meta property="og:video" content="{{ vidurl }}" />
 <meta property="og:video:secure_url" content="{{ vidurl }}" />
 <meta property="og:video:type" content="video/mp4" />
-<meta property="og:video:width" content="720" />
+<meta property="og:video:width" content="{{ videoSize['width'] }}" />
-<meta property="og:video:height" content="480" />
+<meta property="og:video:height" content="{{ videoSize['height'] }}" />
 <meta name="twitter:title" content="{{ user }} (@{{ screenName }}) {% if verified %}☑️{% else %}{% endif %}" />
 <meta property="og:image" content="{{ pic }}" />
 <meta property="og:description" content="{{ desc }}" />

twitfix.py

@@ -1,8 +1,10 @@
+from email import utils
 from random import Random, random
 from weakref import finalize
 from flask import Flask, render_template, request, redirect, abort, Response, send_from_directory, url_for, send_file, make_response, jsonify
 from flask_cors import CORS
 import textwrap
+from pkg_resources import ExtractionError
 import requests
 import re
 import os
@@ -16,6 +18,7 @@ import twExtract as twExtract
 from configHandler import config
 from cache import addVnfToLinkCache,getVnfFromLinkCache
 import random
+from yt_dlp.utils import ExtractorError
 app = Flask(__name__)
 CORS(app)
 
@@ -55,7 +58,6 @@ def oembedend():
 def twitfix(sub_path):
     user_agent = request.headers.get('user-agent')
     match = pathregex.search(sub_path)
-    print(request.url)
 
     if request.url.startswith("https://d.vx"): # Matches d.fx? Try to give the user a direct link
         if match.start() == 0:
@@ -63,13 +65,18 @@ def twitfix(sub_path):
         if user_agent in generate_embed_user_agents:
             print( " ➤ [ D ] d.vx link shown to discord user-agent!")
             if request.url.endswith(".mp4") and "?" not in request.url:
-                # TODO: Cache this, but not for too long as disk space can fill up
+                
                 if "?" not in request.url:
                     clean = twitter_url[:-4]
                 else:
                     clean = twitter_url
-                vid = requests.get(direct_video_link(clean))
+
-                return Response(vid.content,mimetype="video/mp4")
+                vnf,e = vnfFromCacheOrDL(clean)
+                if vnf == None:
+                    if e is not None:
+                        return message(msgs.failedToScan+msgs.failedToScanExtra+e)
+                    return message(msgs.failedToScan)
+                return getTemplate("rawvideo.html",vnf,"","",clean,"","","","")
             else:
                 return message("To use a direct MP4 link in discord, remove anything past '?' and put '.mp4' at the end")
         else:
@@ -93,9 +100,13 @@ def twitfix(sub_path):
             clean = twitter_url[:-4]
         else:
             clean = twitter_url
-        # TODO: Cache this, but not for too long as disk space can fill up
+            
-        vid = requests.get(direct_video_link(clean))
+        vnf,e = vnfFromCacheOrDL(clean)
-        return Response(vid.content,mimetype="video/mp4")
+        if vnf is None:
+            if e is not None:
+                return message(msgs.failedToScan+msgs.failedToScanExtra+e)
+            return message(msgs.failedToScan)
+        return getTemplate("rawvideo.html",vnf,"","",clean,"","","","")
 
     elif request.url.endswith("/1") or request.url.endswith("/2") or request.url.endswith("/3") or request.url.endswith("/4") or request.url.endswith("%2F1") or request.url.endswith("%2F2") or request.url.endswith("%2F3") or request.url.endswith("%2F4"):
         twitter_url = "https://twitter.com/" + sub_path
@@ -178,58 +189,66 @@ def upgradeVNF(vnf):
     # Makes sure any VNF object passed through this has proper fields if they're added in later versions
     if 'verified' not in vnf:
         vnf['verified']=False
+    if 'size' not in vnf:
+        if vnf['type'] == 'Video':
+            vnf['size']={'width':720,'height':480}
+        else:
+            vnf['size']={}
     return vnf
 
 def getDefaultTTL(): # TTL for deleting items from the database
     return datetime.today().replace(microsecond=0) + timedelta(days=1)
 
-def direct_video(video_link): # Just get a redirect to a MP4 link from any tweet link
+def vnfFromCacheOrDL(video_link):
     cached_vnf = getVnfFromLinkCache(video_link)
     if cached_vnf == None:
         try:
             vnf = link_to_vnf(video_link)
             addVnfToLinkCache(video_link, vnf)
-            return redirect(vnf['url'], 301)
+            return vnf,None
-            print(" ➤ [ D ] Redirecting to direct URL: " + vnf['url'])
+        except ExtractorError as exErr:
+            if 'HTTP Error 404' in exErr.msg:
+                exErr.msg="Tweet not found."
+            elif 'suspended' in exErr.msg:
+                exErr.msg="This Tweet is from a suspended account."
+            else:
+                exErr.msg=None
+            return None,exErr.msg
         except Exception as e:
             print(e)
-            return message(msgs.failedToScan)
+            return None,None
     else:
-        return redirect(cached_vnf['url'], 301)
+        return upgradeVNF(cached_vnf),None
-        print(" ➤ [ D ] Redirecting to direct URL: " + vnf['url'])
+
+def direct_video(video_link): # Just get a redirect to a MP4 link from any tweet link
+    vnf,e = vnfFromCacheOrDL(video_link)
+    if vnf != None:
+        return redirect(vnf['url'], 301)
+    else:
+        if e is not None:
+            return message(msgs.failedToScan+msgs.failedToScanExtra+e)
+        return message(msgs.failedToScan)
 
 def direct_video_link(video_link): # Just get a redirect to a MP4 link from any tweet link
-    cached_vnf = getVnfFromLinkCache(video_link)
+    vnf,e = vnfFromCacheOrDL(video_link)
-    if cached_vnf == None:
+    if vnf != None:
-        try:
+        return vnf['url']
-            vnf = link_to_vnf(video_link)
-            addVnfToLinkCache(video_link, vnf)
-            return vnf['url']
-            print(" ➤ [ D ] Redirecting to direct URL: " + vnf['url'])
-        except Exception as e:
-            print(e)
-            return message(msgs.failedToScan)
     else:
-        return cached_vnf['url']
+        if e is not None:
-        print(" ➤ [ D ] Redirecting to direct URL: " + vnf['url'])
+            return message(msgs.failedToScan+msgs.failedToScanExtra+e)
+        return message(msgs.failedToScan)
 
 def embed_video(video_link, image=0): # Return Embed from any tweet link
-    cached_vnf = getVnfFromLinkCache(video_link)
+    vnf,e = vnfFromCacheOrDL(video_link)
 
-    if cached_vnf == None:
+    if vnf != None:
-        try:
+        return embed(video_link, vnf, image)
-            vnf = link_to_vnf(video_link)
-            addVnfToLinkCache(video_link, vnf)
-            return embed(video_link, vnf, image)
-
-        except Exception as e:
-            print(e)
-            return message(msgs.failedToScan)
     else:
-        vnf = upgradeVNF(vnf)
+        if e is not None:
-        return embed(video_link, cached_vnf, image)
+            return message(msgs.failedToScan+msgs.failedToScanExtra+e)
+        return message(msgs.failedToScan)
 
-def tweetInfo(url, tweet="", desc="", thumb="", uploader="", screen_name="", pfp="", tweetType="", images="", hits=0, likes=0, rts=0, time="", qrt={}, nsfw=False,ttl=None,verified=False): # Return a dict of video info with default values
+def tweetInfo(url, tweet="", desc="", thumb="", uploader="", screen_name="", pfp="", tweetType="", images="", hits=0, likes=0, rts=0, time="", qrt={}, nsfw=False,ttl=None,verified=False,size={}): # Return a dict of video info with default values
     if (ttl==None):
         ttl = getDefaultTTL()
     vnf = {
@@ -249,7 +268,8 @@ def tweetInfo(url, tweet="", desc="", thumb="", uploader="", screen_name="", pfp
         "qrt"           : qrt,
         "nsfw"          : nsfw,
         "ttl"           : ttl,
-        "verified"      : verified
+        "verified"      : verified,
+        "size"          : size
     }
     return vnf
 
@@ -268,6 +288,7 @@ def link_to_vnf_from_tweet_data(tweet,video_link):
         if tweet['extended_entities']['media'][0]['video_info']['variants']:
             best_bitrate = -1
             thumb = tweet['extended_entities']['media'][0]['media_url']
+            size=tweet['extended_entities']['media'][0]["original_info"]
             for video in tweet['extended_entities']['media'][0]['video_info']['variants']:
                 if video['content_type'] == "video/mp4" and video['bitrate'] > best_bitrate:
                     url = video['url']
@@ -275,6 +296,7 @@ def link_to_vnf_from_tweet_data(tweet,video_link):
     elif tweetType(tweet) == "Text":
         url   = ""
         thumb = ""
+        size  = {}
     else:
         imgs = ["","","","", ""]
         i = 0
@@ -287,6 +309,7 @@ def link_to_vnf_from_tweet_data(tweet,video_link):
         url   = ""
         images= imgs
         thumb = tweet['extended_entities']['media'][0]['media_url_https']
+        size  = {}
 
     qrt = {}
 
@@ -321,22 +344,17 @@ def link_to_vnf_from_tweet_data(tweet,video_link):
         qrt=qrt, 
         images=imgs,
         nsfw=nsfw,
-        verified=tweet['user']['verified']
+        verified=tweet['user']['verified'],
+        size=size
         )
         
     return vnf
 
 
 def link_to_vnf_from_unofficial_api(video_link):
+    tweet=None
     print(" ➤ [ + ] Attempting to download tweet info from UNOFFICIAL Twitter API")
-    try:
+    tweet = twExtract.extractStatus(video_link)
-        tweet = twExtract.extractStatus(video_link)
-    except Exception as e:
-        print(' ➤ [ !!! ] Local UNOFFICIAL API Failed')
-        if ('apiMirrors' in config['config'] and len(config['config']['apiMirrors']) > 0):
-            mirror = random.choice(config['config']['apiMirrors'])
-            print(" ➤ [ + ] Using API Mirror: "+mirror)
-            tweet = requests.get(mirror+"?url="+video_link).json()
     print (" ➤ [ ✔ ] Unofficial API Success")
     return link_to_vnf_from_tweet_data(tweet,video_link)
 
@@ -346,11 +364,7 @@ def link_to_vnf_from_api(video_link):
     return link_to_vnf_from_tweet_data(tweet,video_link)
 
 def link_to_vnf(video_link): # Return a VideoInfo object or die trying
-    try:
+    return link_to_vnf_from_unofficial_api(video_link)
-        return link_to_vnf_from_unofficial_api(video_link)
-    except Exception as e:
-        print(" ➤ [ !!! ] Unofficial Twitter API Failed")
-        print(e)
 
 def message(text):
     return render_template(
@@ -362,6 +376,9 @@ def message(text):
         url     = config['config']['url'] )
 
 def getTemplate(template,vnf,desc,image,video_link,color,urlDesc,urlUser,urlLink,appNameSuffix=""):
+    if ('width' in vnf['size'] and 'height' in vnf['size']):
+        vnf['size']['width'] = min(vnf['size']['width'],2000)
+        vnf['size']['height'] = min(vnf['size']['height'],2000)
     return render_template(
         template, 
         likes      = vnf['likes'], 
@@ -382,7 +399,8 @@ def getTemplate(template,vnf,desc,image,video_link,color,urlDesc,urlUser,urlLink
         urlDesc    = urlDesc, 
         urlUser    = urlUser, 
         urlLink    = urlLink,
-        tweetLink  = vnf['tweet'] )
+        tweetLink  = vnf['tweet'],
+        videoSize  = vnf['size'] )
 
 def embed(video_link, vnf, image):
     print(" ➤ [ E ] Embedding " + vnf['type'] + ": " + vnf['url'])
@@ -430,19 +448,14 @@ def embed(video_link, vnf, image):
 
 
 def embedCombined(video_link):
-    cached_vnf = getVnfFromLinkCache(video_link)
+    vnf,e = vnfFromCacheOrDL(video_link)
 
-    if cached_vnf == None:
+    if vnf != None:
-        try:
+        return embedCombinedVnf(video_link, vnf)
-            vnf = link_to_vnf(video_link)
-            addVnfToLinkCache(video_link, vnf)
-            return embedCombinedVnf(video_link, vnf)
-
-        except Exception as e:
-            print(e)
-            return message(msgs.failedToScan)
     else:
-        return embedCombinedVnf(video_link, cached_vnf)
+        if e is not None:
+            return message(msgs.failedToScan+msgs.failedToScanExtra+e)
+        return message(msgs.failedToScan)
 
 def embedCombinedVnf(video_link,vnf):
     if vnf['type'] != "Image" or vnf['images'][4] == "1":