Changes QRT storage; Embeds qrt videos, closes #19
This commit is contained in:
Dylan
parent
acd24a08c8
commit
2b3af43098
10
cache.py
10
cache.py
@ -44,19 +44,22 @@ def serializeUnknown(obj):
|
||||
raise TypeError ("Type %s not serializable" % type(obj))
|
||||
|
||||
def addVnfToLinkCache(video_link, vnf):
|
||||
video_link = video_link.lower()
|
||||
global link_cache
|
||||
try:
|
||||
if link_cache_system == "db":
|
||||
out = db.linkCache.insert_one(vnf)
|
||||
out = db.linkCache.update_one(vnf)
|
||||
print(" ➤ [ + ] Link added to DB cache ")
|
||||
return True
|
||||
elif link_cache_system == "json":
|
||||
link_cache[video_link] = vnf
|
||||
with open("links.json", "w") as outfile:
|
||||
json.dump(link_cache, outfile, indent=4, sort_keys=True, default=serializeUnknown)
|
||||
return None
|
||||
print(" ➤ [ + ] Link added to JSON cache ")
|
||||
return True
|
||||
elif link_cache_system == "ram": # FOR TESTS ONLY
|
||||
link_cache[video_link] = vnf
|
||||
print(" ➤ [ + ] Link added to RAM cache ")
|
||||
elif link_cache_system == "dynamodb": # pragma: no cover
|
||||
vnf["ttl"] = int(vnf["ttl"].strftime('%s'))
|
||||
table = client.Table(DYNAMO_CACHE_TBL)
|
||||
@ -72,9 +75,10 @@ def addVnfToLinkCache(video_link, vnf):
|
||||
except Exception as e:
|
||||
print(" ➤ [ X ] Failed to add link to DB cache")
|
||||
print(e)
|
||||
return None
|
||||
return False
|
||||
|
||||
def getVnfFromLinkCache(video_link):
|
||||
video_link = video_link.lower()
|
||||
global link_cache
|
||||
if link_cache_system == "db":
|
||||
collection = db.linkCache
|
||||
|
||||
4
msgs.py
4
msgs.py
@ -4,11 +4,11 @@ tweetNotFound="Tweet not found."
|
||||
tweetSuspended="This Tweet is from a suspended account."
|
||||
|
||||
def genLikesDisplay(vnf):
|
||||
return ("\n\n💖 " + str(vnf['likes']) + " 🔁 " + str(vnf['rts']) + "\n")
|
||||
return ("\n\n💖 " + str(vnf['likes']) + " 🔁 " + str(vnf['rts']))
|
||||
|
||||
def genQrtDisplay(qrt):
|
||||
verifiedCheck = "☑️" if ('verified' in qrt and qrt['verified']) else ""
|
||||
return ("\n─────────────\n ➤ QRT of " + qrt['handle'] + " (@" + qrt['screen_name'] + ")"+ verifiedCheck+":\n─────────────\n'" + qrt['desc'] + "'")
|
||||
return ("\n─────────────\n ➤ QRT of " + qrt['uploader'] + " (@" + qrt['screen_name'] + ")"+ verifiedCheck+":\n─────────────\n'" + qrt['description'] + "'")
|
||||
|
||||
def genPollDisplay(poll):
|
||||
pctSplit=10
|
||||
|
||||
37
test_vx.py
37
test_vx.py
@ -13,11 +13,13 @@ testMediaTweet="https://twitter.com/Twitter/status/1118295916874739714"
|
||||
testMultiMediaTweet="https://twitter.com/Twitter/status/1293239745695211520"
|
||||
testPollTweet="https://twitter.com/norm/status/651169346518056960"
|
||||
testQRTTweet="https://twitter.com/Twitter/status/1232823570046255104"
|
||||
testQrtCeptionTweet="https://twitter.com/CatherineShu/status/585253766271672320"
|
||||
testQrtVideoTweet="https://twitter.com/Twitter/status/1494436688554344449"
|
||||
|
||||
textVNF_compare = {'tweet': 'https://twitter.com/jack/status/20', 'url': '', 'description': 'just setting up my twttr', 'screen_name': 'jack', 'type': 'Text', 'images': ['', '', '', '', ''], 'time': 'Tue Mar 21 20:50:14 +0000 2006', 'qrt': {}, 'nsfw': False}
|
||||
videoVNF_compare={'tweet': 'https://twitter.com/Twitter/status/1263145271946551300', 'url': 'https://video.twimg.com/amplify_video/1263145212760805376/vid/1280x720/9jous8HM0_duxL0w.mp4?tag=13', 'description': 'Testing, testing...\n\nA new way to have a convo with exactly who you want. We’re starting with a small % globally, so keep your 👀 out to see it in action. https://t.co/pV53mvjAVT', 'thumbnail': 'http://pbs.twimg.com/media/EYeX7akWsAIP1_1.jpg', 'screen_name': 'Twitter', 'type': 'Video', 'images': ['', '', '', '', ''], 'time': 'Wed May 20 16:31:15 +0000 2020', 'qrt': {}, 'nsfw': False,'verified': True, 'size': {'width': 1920, 'height': 1080}}
|
||||
testMedia_compare={'tweet': 'https://twitter.com/Twitter/status/1118295916874739714', 'url': '', 'description': 'On profile pages, we used to only show someone’s replies, not the original Tweet 🙄 Now we’re showing both so you can follow the conversation more easily! https://t.co/LSBEZYFqmY', 'thumbnail': 'https://pbs.twimg.com/media/D4TS4xeX4AA02DI.jpg', 'screen_name': 'Twitter', 'type': 'Image', 'images': ['https://pbs.twimg.com/media/D4TS4xeX4AA02DI.jpg', '', '', '', '1'], 'time': 'Tue Apr 16 23:31:38 +0000 2019', 'qrt': {}, 'nsfw': False, 'size': {}}
|
||||
testMultiMedia_compare={'tweet': 'https://twitter.com/Twitter/status/1293239745695211520', 'url': '', 'description': 'We tested, you Tweeted, and now we’re rolling it out to everyone! https://t.co/w6Q3Q6DiKz', 'thumbnail': 'https://pbs.twimg.com/media/EfJ-C-JU0AAQL_C.jpg', 'screen_name': 'Twitter', 'type': 'Image', 'images': ['https://pbs.twimg.com/media/EfJ-C-JU0AAQL_C.jpg', 'https://pbs.twimg.com/media/EfJ-aHlU0AAU1kq.jpg', '', '', '2'], 'time': 'Tue Aug 11 17:35:57 +0000 2020', 'qrt': {}, 'nsfw': False, 'verified': True, 'size': {}}
|
||||
textVNF_compare = {'tweet': 'https://twitter.com/jack/status/20', 'url': '', 'description': 'just setting up my twttr', 'screen_name': 'jack', 'type': 'Text', 'images': ['', '', '', '', ''], 'time': 'Tue Mar 21 20:50:14 +0000 2006', 'qrtURL': None, 'nsfw': False}
|
||||
videoVNF_compare={'tweet': 'https://twitter.com/Twitter/status/1263145271946551300', 'url': 'https://video.twimg.com/amplify_video/1263145212760805376/vid/1280x720/9jous8HM0_duxL0w.mp4?tag=13', 'description': 'Testing, testing...\n\nA new way to have a convo with exactly who you want. We’re starting with a small % globally, so keep your 👀 out to see it in action. https://t.co/pV53mvjAVT', 'thumbnail': 'http://pbs.twimg.com/media/EYeX7akWsAIP1_1.jpg', 'screen_name': 'Twitter', 'type': 'Video', 'images': ['', '', '', '', ''], 'time': 'Wed May 20 16:31:15 +0000 2020', 'qrtURL': None, 'nsfw': False,'verified': True, 'size': {'width': 1920, 'height': 1080}}
|
||||
testMedia_compare={'tweet': 'https://twitter.com/Twitter/status/1118295916874739714', 'url': '', 'description': 'On profile pages, we used to only show someone’s replies, not the original Tweet 🙄 Now we’re showing both so you can follow the conversation more easily! https://t.co/LSBEZYFqmY', 'thumbnail': 'https://pbs.twimg.com/media/D4TS4xeX4AA02DI.jpg', 'screen_name': 'Twitter', 'type': 'Image', 'images': ['https://pbs.twimg.com/media/D4TS4xeX4AA02DI.jpg', '', '', '', '1'], 'time': 'Tue Apr 16 23:31:38 +0000 2019', 'qrtURL': None, 'nsfw': False, 'size': {}}
|
||||
testMultiMedia_compare={'tweet': 'https://twitter.com/Twitter/status/1293239745695211520', 'url': '', 'description': 'We tested, you Tweeted, and now we’re rolling it out to everyone! https://t.co/w6Q3Q6DiKz', 'thumbnail': 'https://pbs.twimg.com/media/EfJ-C-JU0AAQL_C.jpg', 'screen_name': 'Twitter', 'type': 'Image', 'images': ['https://pbs.twimg.com/media/EfJ-C-JU0AAQL_C.jpg', 'https://pbs.twimg.com/media/EfJ-aHlU0AAU1kq.jpg', '', '', '2'], 'time': 'Tue Aug 11 17:35:57 +0000 2020', 'qrtURL': None, 'nsfw': False, 'verified': True, 'size': {}}
|
||||
|
||||
testPoll_comparePoll={"name":"poll2choice_text_only","binding_values":{"choice1_label":{"type":"STRING","string_value":"Mean one thing"},"choice2_label":{"type":"STRING","string_value":"Mean multiple things"},"end_datetime_utc":{"type":"STRING","string_value":"2015-10-06T22:57:24Z"},"counts_are_final":{"type":"BOOLEAN","boolean_value":True},"choice2_count":{"type":"STRING","string_value":"33554"},"choice1_count":{"type":"STRING","string_value":"124875"},"last_updated_datetime_utc":{"type":"STRING","string_value":"2015-10-06T22:57:31Z"},"duration_minutes":{"type":"STRING","string_value":"1440"}}}
|
||||
testPoll_comparePollVNF={'total_votes': 158429, 'choices': [{'text': 'Mean one thing', 'votes': 124875, 'percent': 78.8}, {'text': 'Mean multiple things', 'votes': 33554, 'percent': 21.2}]}
|
||||
@ -101,11 +103,30 @@ def test_pollTweetVNF():
|
||||
vnf = twitfix.link_to_vnf_from_unofficial_api(testPollTweet)
|
||||
compareDict(testPoll_comparePollVNF,vnf['poll'])
|
||||
|
||||
def test_qrtTweetVNF():
|
||||
def test_qrtTweet():
|
||||
cache.clearCache()
|
||||
# this is an incredibly lazy test, todo: improve it in the future
|
||||
vnf = twitfix.link_to_vnf_from_unofficial_api(testQRTTweet)
|
||||
assert 'qrt' in vnf
|
||||
assert vnf['qrt']['desc'].startswith("Twitter says I have 382 followers")
|
||||
resp = client.get(testQRTTweet.replace("https://twitter.com",""),headers={"User-Agent":"test"})
|
||||
assert resp.status_code==200
|
||||
assert "Twitter says I have 382 followers" in str(resp.data)
|
||||
# test qrt-ception
|
||||
resp = client.get(testQrtCeptionTweet.replace("https://twitter.com",""),headers={"User-Agent":"test"}) # get top level tweet
|
||||
assert resp.status_code==200
|
||||
assert "Please retweet this to spread awareness for retweets" in str(resp.data)
|
||||
qtd_tweet=cache.getVnfFromLinkCache("https://twitter.com/EliLanger/status/585253161260216320") # check that the quoted tweet for the top level tweet is cached
|
||||
assert qtd_tweet is not None
|
||||
assert qtd_tweet["qrtURL"] is not None # check that the quoted tweet for the top level tweet has a qrtURL
|
||||
assert cache.getVnfFromLinkCache("https://twitter.com/EliLanger/status/313143446842007553") is None # check that the bottom level tweet has NOT been cached
|
||||
resp = client.get("/EliLanger/status/585253161260216320",headers={"User-Agent":"test"}) # get mid level tweet
|
||||
assert resp.status_code==200
|
||||
assert cache.getVnfFromLinkCache("https://twitter.com/EliLanger/status/313143446842007553") is not None # check that the bottom level tweet has been cached now
|
||||
|
||||
def test_qrtVideoTweet():
|
||||
cache.clearCache()
|
||||
# this is an incredibly lazy test, todo: improve it in the future
|
||||
resp = client.get(testQrtVideoTweet.replace("https://twitter.com",""),headers={"User-Agent":"test"})
|
||||
assert resp.status_code==200
|
||||
assert "twitter:player:stream\" content=\"https://video.twimg.com/tweet_video/FL0gdK8WUAIHHKa.mp4" in str(resp.data)
|
||||
|
||||
## Test adding to cache ; cache should be empty ##
|
||||
def test_addToCache():
|
||||
|
||||
80
twitfix.py
80
twitfix.py
@ -189,6 +189,11 @@ def upgradeVNF(vnf):
|
||||
vnf['size']={'width':720,'height':480}
|
||||
else:
|
||||
vnf['size']={}
|
||||
if 'qrtURL' not in vnf:
|
||||
if vnf['qrt'] == {}:
|
||||
vnf['qrtURL'] = None
|
||||
else: #
|
||||
vnf['qrtURL'] = f"https://twitter.com/{vnf['qrt']['handle']}/status/{vnf['qrt']['id']}"
|
||||
return vnf
|
||||
|
||||
def getDefaultTTL(): # TTL for deleting items from the database
|
||||
@ -243,7 +248,7 @@ def embed_video(video_link, image=0): # Return Embed from any tweet link
|
||||
return message(msgs.failedToScan+msgs.failedToScanExtra+e)
|
||||
return message(msgs.failedToScan)
|
||||
|
||||
def tweetInfo(url, tweet="", desc="", thumb="", uploader="", screen_name="", pfp="", tweetType="", images="", hits=0, likes=0, rts=0, time="", qrt={}, nsfw=False,ttl=None,verified=False,size={},poll=None): # Return a dict of video info with default values
|
||||
def tweetInfo(url, tweet="", desc="", thumb="", uploader="", screen_name="", pfp="", tweetType="", images="", hits=0, likes=0, rts=0, time="", qrtURL="", nsfw=False,ttl=None,verified=False,size={},poll=None): # Return a dict of video info with default values
|
||||
if (ttl==None):
|
||||
ttl = getDefaultTTL()
|
||||
vnf = {
|
||||
@ -260,7 +265,7 @@ def tweetInfo(url, tweet="", desc="", thumb="", uploader="", screen_name="", pfp
|
||||
"likes" : likes,
|
||||
"rts" : rts,
|
||||
"time" : time,
|
||||
"qrt" : qrt,
|
||||
"qrtURL" : qrtURL,
|
||||
"nsfw" : nsfw,
|
||||
"ttl" : ttl,
|
||||
"verified" : verified,
|
||||
@ -302,14 +307,9 @@ def link_to_vnf_from_tweet_data(tweet,video_link):
|
||||
thumb = tweet['extended_entities']['media'][0]['media_url_https']
|
||||
size = {}
|
||||
|
||||
qrt = {}
|
||||
|
||||
if 'quoted_status' in tweet:
|
||||
qrt['desc'] = tweet['quoted_status']['full_text']
|
||||
qrt['handle'] = tweet['quoted_status']['user']['name']
|
||||
qrt['screen_name'] = tweet['quoted_status']['user']['screen_name']
|
||||
qrt['verified'] = tweet['quoted_status']['user']['verified']
|
||||
qrt['id'] = tweet['quoted_status']['id_str']
|
||||
qrtURL = None
|
||||
if 'quoted_status' in tweet and 'quoted_status_permalink' in tweet:
|
||||
qrtURL = tweet['quoted_status_permalink']['expanded']
|
||||
|
||||
text = tweet['full_text']
|
||||
|
||||
@ -320,8 +320,10 @@ def link_to_vnf_from_tweet_data(tweet,video_link):
|
||||
|
||||
if 'entities' in tweet and 'urls' in tweet['entities']:
|
||||
for eurl in tweet['entities']['urls']:
|
||||
text = text.replace(eurl["url"],eurl["expanded_url"])
|
||||
|
||||
if "/status/" in eurl["expanded_url"] and eurl["expanded_url"].startswith("https://twitter.com/"):
|
||||
text = text.replace(eurl["url"], "")
|
||||
else:
|
||||
text = text.replace(eurl["url"],eurl["expanded_url"])
|
||||
ttl = None #default
|
||||
|
||||
if 'card' in tweet and tweet['card']['name'].startswith('poll'):
|
||||
@ -342,7 +344,7 @@ def link_to_vnf_from_tweet_data(tweet,video_link):
|
||||
likes=tweet['favorite_count'],
|
||||
rts=tweet['retweet_count'],
|
||||
time=tweet['created_at'],
|
||||
qrt=qrt,
|
||||
qrtURL=qrtURL,
|
||||
images=imgs,
|
||||
nsfw=nsfw,
|
||||
verified=tweet['user']['verified'],
|
||||
@ -361,11 +363,6 @@ def link_to_vnf_from_unofficial_api(video_link):
|
||||
print (" ➤ [ ✔ ] Unofficial API Success")
|
||||
return link_to_vnf_from_tweet_data(tweet,video_link)
|
||||
|
||||
|
||||
def link_to_vnf_from_api(video_link):
|
||||
tweet = get_tweet_data_from_api(video_link)
|
||||
return link_to_vnf_from_tweet_data(tweet,video_link)
|
||||
|
||||
def link_to_vnf(video_link): # Return a VideoInfo object or die trying
|
||||
return link_to_vnf_from_unofficial_api(video_link)
|
||||
|
||||
@ -378,23 +375,25 @@ def message(text):
|
||||
repo = config['config']['repo'],
|
||||
url = config['config']['url'] )
|
||||
|
||||
def getTemplate(template,vnf,desc,image,video_link,color,urlDesc,urlUser,urlLink,appNameSuffix=""):
|
||||
if ('width' in vnf['size'] and 'height' in vnf['size']):
|
||||
vnf['size']['width'] = min(vnf['size']['width'],2000)
|
||||
vnf['size']['height'] = min(vnf['size']['height'],2000)
|
||||
def getTemplate(template,vnf,desc,image,video_link,color,urlDesc,urlUser,urlLink,appNameSuffix="",embedVNF=None):
|
||||
if (embedVNF is None):
|
||||
embedVNF = vnf
|
||||
if ('width' in embedVNF['size'] and 'height' in embedVNF['size']):
|
||||
embedVNF['size']['width'] = min(embedVNF['size']['width'],2000)
|
||||
embedVNF['size']['height'] = min(embedVNF['size']['height'],2000)
|
||||
return render_template(
|
||||
template,
|
||||
likes = vnf['likes'],
|
||||
rts = vnf['rts'],
|
||||
time = vnf['time'],
|
||||
screenName = vnf['screen_name'],
|
||||
vidlink = vnf['url'],
|
||||
vidlink = embedVNF['url'],
|
||||
pfp = vnf['pfp'],
|
||||
vidurl = vnf['url'],
|
||||
vidurl = embedVNF['url'],
|
||||
desc = desc,
|
||||
pic = image,
|
||||
user = vnf['uploader'],
|
||||
video_link = video_link,
|
||||
video_link = vnf,
|
||||
color = color,
|
||||
appname = config['config']['appname'] + appNameSuffix,
|
||||
repo = config['config']['repo'],
|
||||
@ -403,10 +402,10 @@ def getTemplate(template,vnf,desc,image,video_link,color,urlDesc,urlUser,urlLink
|
||||
urlUser = urlUser,
|
||||
urlLink = urlLink,
|
||||
tweetLink = vnf['tweet'],
|
||||
videoSize = vnf['size'] )
|
||||
videoSize = embedVNF['size'] )
|
||||
|
||||
def embed(video_link, vnf, image):
|
||||
print(" ➤ [ E ] Embedding " + vnf['type'] + ": " + vnf['url'])
|
||||
print(" ➤ [ E ] Embedding " + vnf['type'] + ": " + video_link)
|
||||
|
||||
desc = re.sub(r' http.*t\.co\S+', '', vnf['description'])
|
||||
urlUser = urllib.parse.quote(vnf['uploader'])
|
||||
@ -419,11 +418,26 @@ def embed(video_link, vnf, image):
|
||||
else:
|
||||
pollDisplay=""
|
||||
|
||||
desc=msgs.formatEmbedDesc(vnf['type'],desc,vnf['qrt'],pollDisplay,likeDisplay)
|
||||
|
||||
qrt=None
|
||||
if vnf['qrtURL'] is not None:
|
||||
qrt,e=vnfFromCacheOrDL(vnf['qrtURL'])
|
||||
if qrt is not None:
|
||||
desc=msgs.formatEmbedDesc(vnf['type'],desc,qrt,pollDisplay,likeDisplay)
|
||||
embedVNF=None
|
||||
appNamePost = ""
|
||||
if vnf['type'] == "Text": # Change the template based on tweet type
|
||||
template = 'text.html'
|
||||
if qrt is not None and qrt['type'] != "Text":
|
||||
embedVNF=qrt
|
||||
if qrt['type'] == "Image":
|
||||
if embedVNF['images'][4]!="1":
|
||||
appNamePost = " - Image " + str(image+1) + "/" + str(vnf['images'][4])
|
||||
image = embedVNF['images'][image]
|
||||
template = 'image.html'
|
||||
elif qrt['type'] == "Video" or qrt['type'] == "":
|
||||
urlDesc = urllib.parse.quote(textwrap.shorten(desc, width=220, placeholder="..."))
|
||||
template = 'video.html'
|
||||
|
||||
if vnf['type'] == "Image":
|
||||
if vnf['images'][4]!="1":
|
||||
appNamePost = " - Image " + str(image+1) + "/" + str(vnf['images'][4])
|
||||
@ -441,7 +455,7 @@ def embed(video_link, vnf, image):
|
||||
if vnf['nsfw'] == True:
|
||||
color = "#800020" # Red
|
||||
|
||||
return getTemplate(template,vnf,desc,image,video_link,color,urlDesc,urlUser,urlLink)
|
||||
return getTemplate(template,vnf,desc,image,video_link,color,urlDesc,urlUser,urlLink,appNamePost,embedVNF)
|
||||
|
||||
|
||||
def embedCombined(video_link):
|
||||
@ -468,7 +482,11 @@ def embedCombinedVnf(video_link,vnf):
|
||||
else:
|
||||
pollDisplay=""
|
||||
|
||||
desc=msgs.formatEmbedDesc(vnf['type'],desc,vnf['qrt'],pollDisplay,likeDisplay)
|
||||
qrt=None
|
||||
if vnf['qrtURL'] is not None:
|
||||
qrt,e=vnfFromCacheOrDL(vnf['qrtURL'])
|
||||
if qrt is not None:
|
||||
desc=msgs.formatEmbedDesc(vnf['type'],desc,qrt,pollDisplay,likeDisplay)
|
||||
|
||||
|
||||
image = "https://vxtwitter.com/rendercombined.jpg?imgs="
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user