Merge branch 'main' into tests
This commit is contained in:
Dylan
commit
ac9415a8b3
@ -11,8 +11,7 @@ if ('RUNNING_SERVERLESS' in os.environ and os.environ['RUNNING_SERVERLESS'] == '
|
||||
"appname": os.environ["VXTWITTER_APP_NAME"],
|
||||
"repo": os.environ["VXTWITTER_REPO"],
|
||||
"url": os.environ["VXTWITTER_URL"],
|
||||
"combination_method": os.environ["VXTWITTER_COMBINATION_METHOD"], # can either be 'local' or a URL to a server handling requests in the same format
|
||||
"apiMirrors":[]
|
||||
"combination_method": os.environ["VXTWITTER_COMBINATION_METHOD"] # can either be 'local' or a URL to a server handling requests in the same format
|
||||
}
|
||||
}
|
||||
else:
|
||||
@ -28,8 +27,7 @@ else:
|
||||
"appname": "vxTwitter",
|
||||
"repo": "https://github.com/dylanpdx/BetterTwitFix",
|
||||
"url": "https://vxtwitter.com",
|
||||
"combination_method": "local", # can either be 'local' or a URL to a server handling requests in the same format
|
||||
"apiMirrors":[]
|
||||
"combination_method": "local" # can either be 'local' or a URL to a server handling requests in the same format
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
4
msgs.py
4
msgs.py
@ -1,5 +1,5 @@
|
||||
failedToScan="Failed to scan your link! This may be due to an incorrect link, private account, or the twitter API itself might be having issues (Check here: https://api.twitterstat.us/)\nIt's also possible that Twitter is API limiting me, in which case I can't do anything about it."
|
||||
|
||||
failedToScan="Failed to scan your link! This may be due to an incorrect link, private/suspended account, deleted tweet, or Twitter itself might be having issues (Check here: https://api.twitterstat.us/)"
|
||||
failedToScanExtra = "\n\nTwitter gave me this error: "
|
||||
|
||||
def genLikesDisplay(vnf):
|
||||
return ("\n\n💖 " + str(vnf['likes']) + " 🔁 " + str(vnf['rts']) + "\n")
|
||||
|
||||
17
readme.md
17
readme.md
@ -41,15 +41,9 @@ vxTwitter generates a config.json in its root directory the first time you run i
|
||||
|
||||
- **db**: Caches all links to a mongoDB database. This should be used it you are using uWSGI and are not just running the script on its own as one worker
|
||||
- **json**: This saves cached links to a local **links.json** file
|
||||
- **dynamodb**: Saves cached links to a DynamoDB database - set `table` to the table name to cache links to.
|
||||
- **none**: Does not cache requests. Not reccomended as you can easily use up your Twitter API credits with this. Intended for use with another cache system (i.e NGINX uwsgi_cache)
|
||||
|
||||
**method** - ( Options: **youtube-dl**, **api**, **hybrid** )
|
||||
|
||||
- **youtube-dl**: the original method for grabbing twitter video links, this uses a guest token provided via youtube-dl and should work well for individual instances, but may not scale up to a very large amount of usage
|
||||
|
||||
- **api**: this directly uses the twitter API to grab tweet info, limited to 900 calls per 15m
|
||||
- **hybrid**: This will start off by using the twitter API to grab tweet info, but if the rate limit is reached or the api fails for any other reason it will switch over to youtube-dl to avoid downtime
|
||||
|
||||
**color** - Accepts a hex formatted color code, can change the embed color
|
||||
|
||||
**appname** - Can change the app name easily wherever it's shown
|
||||
@ -60,13 +54,8 @@ vxTwitter generates a config.json in its root directory the first time you run i
|
||||
|
||||
**combination_method** - using c.vxtwitter as the url causes vxTwitter to combine all images in the post into one. This is CPU intensive, so you might not want it running on the same machine that's serving requests. When `combination_method` is set to `local`, it will use the local machine to combine the images. This requires pillow to be installed. If you want to use another server, replace `local` with the URL to the endpoint which combines images. Both methods use the code in the `combineImg` module. Inside, there's also a `Dockerfile` intended to be deployed as a combination endpoint on an [AWS Lambda function](https://docs.aws.amazon.com/lambda/latest/dg/images-create.html).
|
||||
|
||||
**apiMirrors** - During an influx of traffic (i.e when Twitter embeds break!) it's very likely that Twitter will begin to block you for sending too many requests.
|
||||
This is an array of replacement Twitter API URLs which can be called upon when requesting tweet info locally fails. An example of an implementation of an API Mirror is in the twExtract directory, and is intended to be hosted on AWS Lambda. If multiple mirror URLs are specified, one will be selected at random.
|
||||
|
||||
This project is licensed under the **Do What The Fuck You Want Public License**
|
||||
|
||||
|
||||
|
||||
## Other stuff
|
||||
|
||||
We check for t.co links in non video tweets, and if one is found, we direct the discord useragent to embed that link directly, this means that twitter links containing youtube / vimeo links will automatically embed those as if you had just directly linked to that content
|
||||
|
||||
This project is licensed under the **Do What The Fuck You Want Public License**
|
||||
@ -47,8 +47,8 @@ functions:
|
||||
vxTwitterApp:
|
||||
handler: wsgi_handler.handler
|
||||
url: true
|
||||
timeout: 15
|
||||
MemorySize: 1024
|
||||
timeout: 5
|
||||
memorySize: 150
|
||||
layers:
|
||||
- Ref: PythonRequirementsLambdaLayer
|
||||
|
||||
|
||||
16
templates/rawvideo.html
Normal file
16
templates/rawvideo.html
Normal file
@ -0,0 +1,16 @@
|
||||
{% extends 'base.html' %} {% block head %}
|
||||
<meta content='text/html; charset=UTF-8' http-equiv='Content-Type' />
|
||||
|
||||
<meta name="twitter:player:stream" content="{{ vidurl }}" />
|
||||
<meta name="twitter:player:stream:content_type" content="video/mp4" />
|
||||
<meta name="twitter:player:width" content="{{ videoSize['width'] }}" />
|
||||
<meta name="twitter:player:height" content="{{ videoSize['height'] }}" />
|
||||
<meta property="og:url" content="{{ vidurl }}" />
|
||||
<meta property="og:video" content="{{ vidurl }}" />
|
||||
<meta property="og:video:secure_url" content="{{ vidurl }}" />
|
||||
<meta property="og:video:type" content="video/mp4" />
|
||||
<meta property="og:video:width" content="{{ videoSize['width'] }}" />
|
||||
<meta property="og:video:height" content="{{ videoSize['height'] }}" />
|
||||
<meta name="twitter:card" content="player" />
|
||||
|
||||
<meta http-equiv="refresh" content="0; url = {{ vidurl }}" /> {% endblock %} {% block body %} Redirecting you to the video in a moment. <a href="{{ vidurl }}">Or click here.</a> {% endblock %}
|
||||
@ -6,8 +6,8 @@
|
||||
<meta name="twitter:card" content="player" />
|
||||
<meta name="twitter:title" content="{{ user }} (@{{ screenName }}) {% if verified %}☑️{% else %}{% endif %}" />
|
||||
<meta name="twitter:image" content="{{ pic }}" />
|
||||
<meta name="twitter:player:width" content="720" />
|
||||
<meta name="twitter:player:height" content="480" />
|
||||
<meta name="twitter:player:width" content="{{ videoSize['width'] }}" />
|
||||
<meta name="twitter:player:height" content="{{ videoSize['height'] }}" />
|
||||
<meta name="twitter:player:stream" content="{{ vidurl }}" />
|
||||
<meta name="twitter:player:stream:content_type" content="video/mp4" />
|
||||
|
||||
@ -15,8 +15,8 @@
|
||||
<meta property="og:video" content="{{ vidurl }}" />
|
||||
<meta property="og:video:secure_url" content="{{ vidurl }}" />
|
||||
<meta property="og:video:type" content="video/mp4" />
|
||||
<meta property="og:video:width" content="720" />
|
||||
<meta property="og:video:height" content="480" />
|
||||
<meta property="og:video:width" content="{{ videoSize['width'] }}" />
|
||||
<meta property="og:video:height" content="{{ videoSize['height'] }}" />
|
||||
<meta name="twitter:title" content="{{ user }} (@{{ screenName }}) {% if verified %}☑️{% else %}{% endif %}" />
|
||||
<meta property="og:image" content="{{ pic }}" />
|
||||
<meta property="og:description" content="{{ desc }}" />
|
||||
|
||||
143
twitfix.py
143
twitfix.py
@ -1,8 +1,10 @@
|
||||
from email import utils
|
||||
from random import Random, random
|
||||
from weakref import finalize
|
||||
from flask import Flask, render_template, request, redirect, abort, Response, send_from_directory, url_for, send_file, make_response, jsonify
|
||||
from flask_cors import CORS
|
||||
import textwrap
|
||||
from pkg_resources import ExtractionError
|
||||
import requests
|
||||
import re
|
||||
import os
|
||||
@ -16,6 +18,7 @@ import twExtract as twExtract
|
||||
from configHandler import config
|
||||
from cache import addVnfToLinkCache,getVnfFromLinkCache
|
||||
import random
|
||||
from yt_dlp.utils import ExtractorError
|
||||
app = Flask(__name__)
|
||||
CORS(app)
|
||||
|
||||
@ -55,7 +58,6 @@ def oembedend():
|
||||
def twitfix(sub_path):
|
||||
user_agent = request.headers.get('user-agent')
|
||||
match = pathregex.search(sub_path)
|
||||
print(request.url)
|
||||
|
||||
if request.url.startswith("https://d.vx"): # Matches d.fx? Try to give the user a direct link
|
||||
if match.start() == 0:
|
||||
@ -63,13 +65,18 @@ def twitfix(sub_path):
|
||||
if user_agent in generate_embed_user_agents:
|
||||
print( " ➤ [ D ] d.vx link shown to discord user-agent!")
|
||||
if request.url.endswith(".mp4") and "?" not in request.url:
|
||||
# TODO: Cache this, but not for too long as disk space can fill up
|
||||
|
||||
if "?" not in request.url:
|
||||
clean = twitter_url[:-4]
|
||||
else:
|
||||
clean = twitter_url
|
||||
vid = requests.get(direct_video_link(clean))
|
||||
return Response(vid.content,mimetype="video/mp4")
|
||||
|
||||
vnf,e = vnfFromCacheOrDL(clean)
|
||||
if vnf == None:
|
||||
if e is not None:
|
||||
return message(msgs.failedToScan+msgs.failedToScanExtra+e)
|
||||
return message(msgs.failedToScan)
|
||||
return getTemplate("rawvideo.html",vnf,"","",clean,"","","","")
|
||||
else:
|
||||
return message("To use a direct MP4 link in discord, remove anything past '?' and put '.mp4' at the end")
|
||||
else:
|
||||
@ -93,9 +100,13 @@ def twitfix(sub_path):
|
||||
clean = twitter_url[:-4]
|
||||
else:
|
||||
clean = twitter_url
|
||||
# TODO: Cache this, but not for too long as disk space can fill up
|
||||
vid = requests.get(direct_video_link(clean))
|
||||
return Response(vid.content,mimetype="video/mp4")
|
||||
|
||||
vnf,e = vnfFromCacheOrDL(clean)
|
||||
if vnf is None:
|
||||
if e is not None:
|
||||
return message(msgs.failedToScan+msgs.failedToScanExtra+e)
|
||||
return message(msgs.failedToScan)
|
||||
return getTemplate("rawvideo.html",vnf,"","",clean,"","","","")
|
||||
|
||||
elif request.url.endswith("/1") or request.url.endswith("/2") or request.url.endswith("/3") or request.url.endswith("/4") or request.url.endswith("%2F1") or request.url.endswith("%2F2") or request.url.endswith("%2F3") or request.url.endswith("%2F4"):
|
||||
twitter_url = "https://twitter.com/" + sub_path
|
||||
@ -178,58 +189,66 @@ def upgradeVNF(vnf):
|
||||
# Makes sure any VNF object passed through this has proper fields if they're added in later versions
|
||||
if 'verified' not in vnf:
|
||||
vnf['verified']=False
|
||||
if 'size' not in vnf:
|
||||
if vnf['type'] == 'Video':
|
||||
vnf['size']={'width':720,'height':480}
|
||||
else:
|
||||
vnf['size']={}
|
||||
return vnf
|
||||
|
||||
def getDefaultTTL(): # TTL for deleting items from the database
|
||||
return datetime.today().replace(microsecond=0) + timedelta(days=1)
|
||||
|
||||
def direct_video(video_link): # Just get a redirect to a MP4 link from any tweet link
|
||||
def vnfFromCacheOrDL(video_link):
|
||||
cached_vnf = getVnfFromLinkCache(video_link)
|
||||
if cached_vnf == None:
|
||||
try:
|
||||
vnf = link_to_vnf(video_link)
|
||||
addVnfToLinkCache(video_link, vnf)
|
||||
return redirect(vnf['url'], 301)
|
||||
print(" ➤ [ D ] Redirecting to direct URL: " + vnf['url'])
|
||||
return vnf,None
|
||||
except ExtractorError as exErr:
|
||||
if 'HTTP Error 404' in exErr.msg:
|
||||
exErr.msg="Tweet not found."
|
||||
elif 'suspended' in exErr.msg:
|
||||
exErr.msg="This Tweet is from a suspended account."
|
||||
else:
|
||||
exErr.msg=None
|
||||
return None,exErr.msg
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return message(msgs.failedToScan)
|
||||
return None,None
|
||||
else:
|
||||
return redirect(cached_vnf['url'], 301)
|
||||
print(" ➤ [ D ] Redirecting to direct URL: " + vnf['url'])
|
||||
return upgradeVNF(cached_vnf),None
|
||||
|
||||
def direct_video(video_link): # Just get a redirect to a MP4 link from any tweet link
|
||||
vnf,e = vnfFromCacheOrDL(video_link)
|
||||
if vnf != None:
|
||||
return redirect(vnf['url'], 301)
|
||||
else:
|
||||
if e is not None:
|
||||
return message(msgs.failedToScan+msgs.failedToScanExtra+e)
|
||||
return message(msgs.failedToScan)
|
||||
|
||||
def direct_video_link(video_link): # Just get a redirect to a MP4 link from any tweet link
|
||||
cached_vnf = getVnfFromLinkCache(video_link)
|
||||
if cached_vnf == None:
|
||||
try:
|
||||
vnf = link_to_vnf(video_link)
|
||||
addVnfToLinkCache(video_link, vnf)
|
||||
return vnf['url']
|
||||
print(" ➤ [ D ] Redirecting to direct URL: " + vnf['url'])
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return message(msgs.failedToScan)
|
||||
vnf,e = vnfFromCacheOrDL(video_link)
|
||||
if vnf != None:
|
||||
return vnf['url']
|
||||
else:
|
||||
return cached_vnf['url']
|
||||
print(" ➤ [ D ] Redirecting to direct URL: " + vnf['url'])
|
||||
if e is not None:
|
||||
return message(msgs.failedToScan+msgs.failedToScanExtra+e)
|
||||
return message(msgs.failedToScan)
|
||||
|
||||
def embed_video(video_link, image=0): # Return Embed from any tweet link
|
||||
cached_vnf = getVnfFromLinkCache(video_link)
|
||||
vnf,e = vnfFromCacheOrDL(video_link)
|
||||
|
||||
if cached_vnf == None:
|
||||
try:
|
||||
vnf = link_to_vnf(video_link)
|
||||
addVnfToLinkCache(video_link, vnf)
|
||||
return embed(video_link, vnf, image)
|
||||
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return message(msgs.failedToScan)
|
||||
if vnf != None:
|
||||
return embed(video_link, vnf, image)
|
||||
else:
|
||||
vnf = upgradeVNF(vnf)
|
||||
return embed(video_link, cached_vnf, image)
|
||||
if e is not None:
|
||||
return message(msgs.failedToScan+msgs.failedToScanExtra+e)
|
||||
return message(msgs.failedToScan)
|
||||
|
||||
def tweetInfo(url, tweet="", desc="", thumb="", uploader="", screen_name="", pfp="", tweetType="", images="", hits=0, likes=0, rts=0, time="", qrt={}, nsfw=False,ttl=None,verified=False): # Return a dict of video info with default values
|
||||
def tweetInfo(url, tweet="", desc="", thumb="", uploader="", screen_name="", pfp="", tweetType="", images="", hits=0, likes=0, rts=0, time="", qrt={}, nsfw=False,ttl=None,verified=False,size={}): # Return a dict of video info with default values
|
||||
if (ttl==None):
|
||||
ttl = getDefaultTTL()
|
||||
vnf = {
|
||||
@ -249,7 +268,8 @@ def tweetInfo(url, tweet="", desc="", thumb="", uploader="", screen_name="", pfp
|
||||
"qrt" : qrt,
|
||||
"nsfw" : nsfw,
|
||||
"ttl" : ttl,
|
||||
"verified" : verified
|
||||
"verified" : verified,
|
||||
"size" : size
|
||||
}
|
||||
return vnf
|
||||
|
||||
@ -268,6 +288,7 @@ def link_to_vnf_from_tweet_data(tweet,video_link):
|
||||
if tweet['extended_entities']['media'][0]['video_info']['variants']:
|
||||
best_bitrate = -1
|
||||
thumb = tweet['extended_entities']['media'][0]['media_url']
|
||||
size=tweet['extended_entities']['media'][0]["original_info"]
|
||||
for video in tweet['extended_entities']['media'][0]['video_info']['variants']:
|
||||
if video['content_type'] == "video/mp4" and video['bitrate'] > best_bitrate:
|
||||
url = video['url']
|
||||
@ -275,6 +296,7 @@ def link_to_vnf_from_tweet_data(tweet,video_link):
|
||||
elif tweetType(tweet) == "Text":
|
||||
url = ""
|
||||
thumb = ""
|
||||
size = {}
|
||||
else:
|
||||
imgs = ["","","","", ""]
|
||||
i = 0
|
||||
@ -287,6 +309,7 @@ def link_to_vnf_from_tweet_data(tweet,video_link):
|
||||
url = ""
|
||||
images= imgs
|
||||
thumb = tweet['extended_entities']['media'][0]['media_url_https']
|
||||
size = {}
|
||||
|
||||
qrt = {}
|
||||
|
||||
@ -321,22 +344,17 @@ def link_to_vnf_from_tweet_data(tweet,video_link):
|
||||
qrt=qrt,
|
||||
images=imgs,
|
||||
nsfw=nsfw,
|
||||
verified=tweet['user']['verified']
|
||||
verified=tweet['user']['verified'],
|
||||
size=size
|
||||
)
|
||||
|
||||
return vnf
|
||||
|
||||
|
||||
def link_to_vnf_from_unofficial_api(video_link):
|
||||
tweet=None
|
||||
print(" ➤ [ + ] Attempting to download tweet info from UNOFFICIAL Twitter API")
|
||||
try:
|
||||
tweet = twExtract.extractStatus(video_link)
|
||||
except Exception as e:
|
||||
print(' ➤ [ !!! ] Local UNOFFICIAL API Failed')
|
||||
if ('apiMirrors' in config['config'] and len(config['config']['apiMirrors']) > 0):
|
||||
mirror = random.choice(config['config']['apiMirrors'])
|
||||
print(" ➤ [ + ] Using API Mirror: "+mirror)
|
||||
tweet = requests.get(mirror+"?url="+video_link).json()
|
||||
tweet = twExtract.extractStatus(video_link)
|
||||
print (" ➤ [ ✔ ] Unofficial API Success")
|
||||
return link_to_vnf_from_tweet_data(tweet,video_link)
|
||||
|
||||
@ -346,11 +364,7 @@ def link_to_vnf_from_api(video_link):
|
||||
return link_to_vnf_from_tweet_data(tweet,video_link)
|
||||
|
||||
def link_to_vnf(video_link): # Return a VideoInfo object or die trying
|
||||
try:
|
||||
return link_to_vnf_from_unofficial_api(video_link)
|
||||
except Exception as e:
|
||||
print(" ➤ [ !!! ] Unofficial Twitter API Failed")
|
||||
print(e)
|
||||
return link_to_vnf_from_unofficial_api(video_link)
|
||||
|
||||
def message(text):
|
||||
return render_template(
|
||||
@ -362,6 +376,9 @@ def message(text):
|
||||
url = config['config']['url'] )
|
||||
|
||||
def getTemplate(template,vnf,desc,image,video_link,color,urlDesc,urlUser,urlLink,appNameSuffix=""):
|
||||
if ('width' in vnf['size'] and 'height' in vnf['size']):
|
||||
vnf['size']['width'] = min(vnf['size']['width'],2000)
|
||||
vnf['size']['height'] = min(vnf['size']['height'],2000)
|
||||
return render_template(
|
||||
template,
|
||||
likes = vnf['likes'],
|
||||
@ -382,7 +399,8 @@ def getTemplate(template,vnf,desc,image,video_link,color,urlDesc,urlUser,urlLink
|
||||
urlDesc = urlDesc,
|
||||
urlUser = urlUser,
|
||||
urlLink = urlLink,
|
||||
tweetLink = vnf['tweet'] )
|
||||
tweetLink = vnf['tweet'],
|
||||
videoSize = vnf['size'] )
|
||||
|
||||
def embed(video_link, vnf, image):
|
||||
print(" ➤ [ E ] Embedding " + vnf['type'] + ": " + vnf['url'])
|
||||
@ -430,19 +448,14 @@ def embed(video_link, vnf, image):
|
||||
|
||||
|
||||
def embedCombined(video_link):
|
||||
cached_vnf = getVnfFromLinkCache(video_link)
|
||||
vnf,e = vnfFromCacheOrDL(video_link)
|
||||
|
||||
if cached_vnf == None:
|
||||
try:
|
||||
vnf = link_to_vnf(video_link)
|
||||
addVnfToLinkCache(video_link, vnf)
|
||||
return embedCombinedVnf(video_link, vnf)
|
||||
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return message(msgs.failedToScan)
|
||||
if vnf != None:
|
||||
return embedCombinedVnf(video_link, vnf)
|
||||
else:
|
||||
return embedCombinedVnf(video_link, cached_vnf)
|
||||
if e is not None:
|
||||
return message(msgs.failedToScan+msgs.failedToScanExtra+e)
|
||||
return message(msgs.failedToScan)
|
||||
|
||||
def embedCombinedVnf(video_link,vnf):
|
||||
if vnf['type'] != "Image" or vnf['images'][4] == "1":
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user