diff --git a/cache.py b/cache.py index 705b8df..8258f9f 100644 --- a/cache.py +++ b/cache.py @@ -44,19 +44,22 @@ def serializeUnknown(obj): raise TypeError ("Type %s not serializable" % type(obj)) def addVnfToLinkCache(video_link, vnf): + video_link = video_link.lower() global link_cache try: if link_cache_system == "db": - out = db.linkCache.insert_one(vnf) + out = db.linkCache.update_one(vnf) print(" ➀ [ + ] Link added to DB cache ") return True elif link_cache_system == "json": link_cache[video_link] = vnf with open("links.json", "w") as outfile: json.dump(link_cache, outfile, indent=4, sort_keys=True, default=serializeUnknown) - return None + print(" ➀ [ + ] Link added to JSON cache ") + return True elif link_cache_system == "ram": # FOR TESTS ONLY link_cache[video_link] = vnf + print(" ➀ [ + ] Link added to RAM cache ") elif link_cache_system == "dynamodb": # pragma: no cover vnf["ttl"] = int(vnf["ttl"].strftime('%s')) table = client.Table(DYNAMO_CACHE_TBL) @@ -72,9 +75,10 @@ def addVnfToLinkCache(video_link, vnf): except Exception as e: print(" ➀ [ X ] Failed to add link to DB cache") print(e) - return None + return False def getVnfFromLinkCache(video_link): + video_link = video_link.lower() global link_cache if link_cache_system == "db": collection = db.linkCache diff --git a/msgs.py b/msgs.py index 2bd125d..330a420 100644 --- a/msgs.py +++ b/msgs.py @@ -4,11 +4,11 @@ tweetNotFound="Tweet not found." tweetSuspended="This Tweet is from a suspended account." def genLikesDisplay(vnf): - return ("\n\nπŸ’– " + str(vnf['likes']) + " πŸ” " + str(vnf['rts']) + "\n") + return ("\n\nπŸ’– " + str(vnf['likes']) + " πŸ” " + str(vnf['rts'])) def genQrtDisplay(qrt): verifiedCheck = "β˜‘οΈ" if ('verified' in qrt and qrt['verified']) else "" - return ("\n─────────────\n ➀ QRT of " + qrt['handle'] + " (@" + qrt['screen_name'] + ")"+ verifiedCheck+":\n─────────────\n'" + qrt['desc'] + "'") + return ("\n─────────────\n ➀ QRT of " + qrt['uploader'] + " (@" + qrt['screen_name'] + ")"+ verifiedCheck+":\n─────────────\n'" + qrt['description'] + "'") def genPollDisplay(poll): pctSplit=10 diff --git a/test_vx.py b/test_vx.py index 5bfef9e..c9fdd67 100644 --- a/test_vx.py +++ b/test_vx.py @@ -13,11 +13,13 @@ testMediaTweet="https://twitter.com/Twitter/status/1118295916874739714" testMultiMediaTweet="https://twitter.com/Twitter/status/1293239745695211520" testPollTweet="https://twitter.com/norm/status/651169346518056960" testQRTTweet="https://twitter.com/Twitter/status/1232823570046255104" +testQrtCeptionTweet="https://twitter.com/CatherineShu/status/585253766271672320" +testQrtVideoTweet="https://twitter.com/Twitter/status/1494436688554344449" -textVNF_compare = {'tweet': 'https://twitter.com/jack/status/20', 'url': '', 'description': 'just setting up my twttr', 'screen_name': 'jack', 'type': 'Text', 'images': ['', '', '', '', ''], 'time': 'Tue Mar 21 20:50:14 +0000 2006', 'qrt': {}, 'nsfw': False} -videoVNF_compare={'tweet': 'https://twitter.com/Twitter/status/1263145271946551300', 'url': 'https://video.twimg.com/amplify_video/1263145212760805376/vid/1280x720/9jous8HM0_duxL0w.mp4?tag=13', 'description': 'Testing, testing...\n\nA new way to have a convo with exactly who you want. We’re starting with a small % globally, so keep your πŸ‘€ out to see it in action. https://t.co/pV53mvjAVT', 'thumbnail': 'http://pbs.twimg.com/media/EYeX7akWsAIP1_1.jpg', 'screen_name': 'Twitter', 'type': 'Video', 'images': ['', '', '', '', ''], 'time': 'Wed May 20 16:31:15 +0000 2020', 'qrt': {}, 'nsfw': False,'verified': True, 'size': {'width': 1920, 'height': 1080}} -testMedia_compare={'tweet': 'https://twitter.com/Twitter/status/1118295916874739714', 'url': '', 'description': 'On profile pages, we used to only show someone’s replies, not the original Tweet πŸ™„ Now we’re showing both so you can follow the conversation more easily! https://t.co/LSBEZYFqmY', 'thumbnail': 'https://pbs.twimg.com/media/D4TS4xeX4AA02DI.jpg', 'screen_name': 'Twitter', 'type': 'Image', 'images': ['https://pbs.twimg.com/media/D4TS4xeX4AA02DI.jpg', '', '', '', '1'], 'time': 'Tue Apr 16 23:31:38 +0000 2019', 'qrt': {}, 'nsfw': False, 'size': {}} -testMultiMedia_compare={'tweet': 'https://twitter.com/Twitter/status/1293239745695211520', 'url': '', 'description': 'We tested, you Tweeted, and now we’re rolling it out to everyone! https://t.co/w6Q3Q6DiKz', 'thumbnail': 'https://pbs.twimg.com/media/EfJ-C-JU0AAQL_C.jpg', 'screen_name': 'Twitter', 'type': 'Image', 'images': ['https://pbs.twimg.com/media/EfJ-C-JU0AAQL_C.jpg', 'https://pbs.twimg.com/media/EfJ-aHlU0AAU1kq.jpg', '', '', '2'], 'time': 'Tue Aug 11 17:35:57 +0000 2020', 'qrt': {}, 'nsfw': False, 'verified': True, 'size': {}} +textVNF_compare = {'tweet': 'https://twitter.com/jack/status/20', 'url': '', 'description': 'just setting up my twttr', 'screen_name': 'jack', 'type': 'Text', 'images': ['', '', '', '', ''], 'time': 'Tue Mar 21 20:50:14 +0000 2006', 'qrtURL': None, 'nsfw': False} +videoVNF_compare={'tweet': 'https://twitter.com/Twitter/status/1263145271946551300', 'url': 'https://video.twimg.com/amplify_video/1263145212760805376/vid/1280x720/9jous8HM0_duxL0w.mp4?tag=13', 'description': 'Testing, testing...\n\nA new way to have a convo with exactly who you want. We’re starting with a small % globally, so keep your πŸ‘€ out to see it in action. https://t.co/pV53mvjAVT', 'thumbnail': 'http://pbs.twimg.com/media/EYeX7akWsAIP1_1.jpg', 'screen_name': 'Twitter', 'type': 'Video', 'images': ['', '', '', '', ''], 'time': 'Wed May 20 16:31:15 +0000 2020', 'qrtURL': None, 'nsfw': False,'verified': True, 'size': {'width': 1920, 'height': 1080}} +testMedia_compare={'tweet': 'https://twitter.com/Twitter/status/1118295916874739714', 'url': '', 'description': 'On profile pages, we used to only show someone’s replies, not the original Tweet πŸ™„ Now we’re showing both so you can follow the conversation more easily! https://t.co/LSBEZYFqmY', 'thumbnail': 'https://pbs.twimg.com/media/D4TS4xeX4AA02DI.jpg', 'screen_name': 'Twitter', 'type': 'Image', 'images': ['https://pbs.twimg.com/media/D4TS4xeX4AA02DI.jpg', '', '', '', '1'], 'time': 'Tue Apr 16 23:31:38 +0000 2019', 'qrtURL': None, 'nsfw': False, 'size': {}} +testMultiMedia_compare={'tweet': 'https://twitter.com/Twitter/status/1293239745695211520', 'url': '', 'description': 'We tested, you Tweeted, and now we’re rolling it out to everyone! https://t.co/w6Q3Q6DiKz', 'thumbnail': 'https://pbs.twimg.com/media/EfJ-C-JU0AAQL_C.jpg', 'screen_name': 'Twitter', 'type': 'Image', 'images': ['https://pbs.twimg.com/media/EfJ-C-JU0AAQL_C.jpg', 'https://pbs.twimg.com/media/EfJ-aHlU0AAU1kq.jpg', '', '', '2'], 'time': 'Tue Aug 11 17:35:57 +0000 2020', 'qrtURL': None, 'nsfw': False, 'verified': True, 'size': {}} testPoll_comparePoll={"name":"poll2choice_text_only","binding_values":{"choice1_label":{"type":"STRING","string_value":"Mean one thing"},"choice2_label":{"type":"STRING","string_value":"Mean multiple things"},"end_datetime_utc":{"type":"STRING","string_value":"2015-10-06T22:57:24Z"},"counts_are_final":{"type":"BOOLEAN","boolean_value":True},"choice2_count":{"type":"STRING","string_value":"33554"},"choice1_count":{"type":"STRING","string_value":"124875"},"last_updated_datetime_utc":{"type":"STRING","string_value":"2015-10-06T22:57:31Z"},"duration_minutes":{"type":"STRING","string_value":"1440"}}} testPoll_comparePollVNF={'total_votes': 158429, 'choices': [{'text': 'Mean one thing', 'votes': 124875, 'percent': 78.8}, {'text': 'Mean multiple things', 'votes': 33554, 'percent': 21.2}]} @@ -101,11 +103,30 @@ def test_pollTweetVNF(): vnf = twitfix.link_to_vnf_from_unofficial_api(testPollTweet) compareDict(testPoll_comparePollVNF,vnf['poll']) -def test_qrtTweetVNF(): +def test_qrtTweet(): + cache.clearCache() # this is an incredibly lazy test, todo: improve it in the future - vnf = twitfix.link_to_vnf_from_unofficial_api(testQRTTweet) - assert 'qrt' in vnf - assert vnf['qrt']['desc'].startswith("Twitter says I have 382 followers") + resp = client.get(testQRTTweet.replace("https://twitter.com",""),headers={"User-Agent":"test"}) + assert resp.status_code==200 + assert "Twitter says I have 382 followers" in str(resp.data) + # test qrt-ception + resp = client.get(testQrtCeptionTweet.replace("https://twitter.com",""),headers={"User-Agent":"test"}) # get top level tweet + assert resp.status_code==200 + assert "Please retweet this to spread awareness for retweets" in str(resp.data) + qtd_tweet=cache.getVnfFromLinkCache("https://twitter.com/EliLanger/status/585253161260216320") # check that the quoted tweet for the top level tweet is cached + assert qtd_tweet is not None + assert qtd_tweet["qrtURL"] is not None # check that the quoted tweet for the top level tweet has a qrtURL + assert cache.getVnfFromLinkCache("https://twitter.com/EliLanger/status/313143446842007553") is None # check that the bottom level tweet has NOT been cached + resp = client.get("/EliLanger/status/585253161260216320",headers={"User-Agent":"test"}) # get mid level tweet + assert resp.status_code==200 + assert cache.getVnfFromLinkCache("https://twitter.com/EliLanger/status/313143446842007553") is not None # check that the bottom level tweet has been cached now + +def test_qrtVideoTweet(): + cache.clearCache() + # this is an incredibly lazy test, todo: improve it in the future + resp = client.get(testQrtVideoTweet.replace("https://twitter.com",""),headers={"User-Agent":"test"}) + assert resp.status_code==200 + assert "twitter:player:stream\" content=\"https://video.twimg.com/tweet_video/FL0gdK8WUAIHHKa.mp4" in str(resp.data) ## Test adding to cache ; cache should be empty ## def test_addToCache(): diff --git a/twitfix.py b/twitfix.py index eb2a672..e25393c 100644 --- a/twitfix.py +++ b/twitfix.py @@ -189,6 +189,11 @@ def upgradeVNF(vnf): vnf['size']={'width':720,'height':480} else: vnf['size']={} + if 'qrtURL' not in vnf: + if vnf['qrt'] == {}: + vnf['qrtURL'] = None + else: # + vnf['qrtURL'] = f"https://twitter.com/{vnf['qrt']['handle']}/status/{vnf['qrt']['id']}" return vnf def getDefaultTTL(): # TTL for deleting items from the database @@ -243,7 +248,7 @@ def embed_video(video_link, image=0): # Return Embed from any tweet link return message(msgs.failedToScan+msgs.failedToScanExtra+e) return message(msgs.failedToScan) -def tweetInfo(url, tweet="", desc="", thumb="", uploader="", screen_name="", pfp="", tweetType="", images="", hits=0, likes=0, rts=0, time="", qrt={}, nsfw=False,ttl=None,verified=False,size={},poll=None): # Return a dict of video info with default values +def tweetInfo(url, tweet="", desc="", thumb="", uploader="", screen_name="", pfp="", tweetType="", images="", hits=0, likes=0, rts=0, time="", qrtURL="", nsfw=False,ttl=None,verified=False,size={},poll=None): # Return a dict of video info with default values if (ttl==None): ttl = getDefaultTTL() vnf = { @@ -260,7 +265,7 @@ def tweetInfo(url, tweet="", desc="", thumb="", uploader="", screen_name="", pfp "likes" : likes, "rts" : rts, "time" : time, - "qrt" : qrt, + "qrtURL" : qrtURL, "nsfw" : nsfw, "ttl" : ttl, "verified" : verified, @@ -302,14 +307,9 @@ def link_to_vnf_from_tweet_data(tweet,video_link): thumb = tweet['extended_entities']['media'][0]['media_url_https'] size = {} - qrt = {} - - if 'quoted_status' in tweet: - qrt['desc'] = tweet['quoted_status']['full_text'] - qrt['handle'] = tweet['quoted_status']['user']['name'] - qrt['screen_name'] = tweet['quoted_status']['user']['screen_name'] - qrt['verified'] = tweet['quoted_status']['user']['verified'] - qrt['id'] = tweet['quoted_status']['id_str'] + qrtURL = None + if 'quoted_status' in tweet and 'quoted_status_permalink' in tweet: + qrtURL = tweet['quoted_status_permalink']['expanded'] text = tweet['full_text'] @@ -320,8 +320,10 @@ def link_to_vnf_from_tweet_data(tweet,video_link): if 'entities' in tweet and 'urls' in tweet['entities']: for eurl in tweet['entities']['urls']: - text = text.replace(eurl["url"],eurl["expanded_url"]) - + if "/status/" in eurl["expanded_url"] and eurl["expanded_url"].startswith("https://twitter.com/"): + text = text.replace(eurl["url"], "") + else: + text = text.replace(eurl["url"],eurl["expanded_url"]) ttl = None #default if 'card' in tweet and tweet['card']['name'].startswith('poll'): @@ -342,7 +344,7 @@ def link_to_vnf_from_tweet_data(tweet,video_link): likes=tweet['favorite_count'], rts=tweet['retweet_count'], time=tweet['created_at'], - qrt=qrt, + qrtURL=qrtURL, images=imgs, nsfw=nsfw, verified=tweet['user']['verified'], @@ -361,11 +363,6 @@ def link_to_vnf_from_unofficial_api(video_link): print (" ➀ [ βœ” ] Unofficial API Success") return link_to_vnf_from_tweet_data(tweet,video_link) - -def link_to_vnf_from_api(video_link): - tweet = get_tweet_data_from_api(video_link) - return link_to_vnf_from_tweet_data(tweet,video_link) - def link_to_vnf(video_link): # Return a VideoInfo object or die trying return link_to_vnf_from_unofficial_api(video_link) @@ -378,23 +375,25 @@ def message(text): repo = config['config']['repo'], url = config['config']['url'] ) -def getTemplate(template,vnf,desc,image,video_link,color,urlDesc,urlUser,urlLink,appNameSuffix=""): - if ('width' in vnf['size'] and 'height' in vnf['size']): - vnf['size']['width'] = min(vnf['size']['width'],2000) - vnf['size']['height'] = min(vnf['size']['height'],2000) +def getTemplate(template,vnf,desc,image,video_link,color,urlDesc,urlUser,urlLink,appNameSuffix="",embedVNF=None): + if (embedVNF is None): + embedVNF = vnf + if ('width' in embedVNF['size'] and 'height' in embedVNF['size']): + embedVNF['size']['width'] = min(embedVNF['size']['width'],2000) + embedVNF['size']['height'] = min(embedVNF['size']['height'],2000) return render_template( template, likes = vnf['likes'], rts = vnf['rts'], time = vnf['time'], screenName = vnf['screen_name'], - vidlink = vnf['url'], + vidlink = embedVNF['url'], pfp = vnf['pfp'], - vidurl = vnf['url'], + vidurl = embedVNF['url'], desc = desc, pic = image, user = vnf['uploader'], - video_link = video_link, + video_link = vnf, color = color, appname = config['config']['appname'] + appNameSuffix, repo = config['config']['repo'], @@ -403,10 +402,10 @@ def getTemplate(template,vnf,desc,image,video_link,color,urlDesc,urlUser,urlLink urlUser = urlUser, urlLink = urlLink, tweetLink = vnf['tweet'], - videoSize = vnf['size'] ) + videoSize = embedVNF['size'] ) def embed(video_link, vnf, image): - print(" ➀ [ E ] Embedding " + vnf['type'] + ": " + vnf['url']) + print(" ➀ [ E ] Embedding " + vnf['type'] + ": " + video_link) desc = re.sub(r' http.*t\.co\S+', '', vnf['description']) urlUser = urllib.parse.quote(vnf['uploader']) @@ -419,11 +418,26 @@ def embed(video_link, vnf, image): else: pollDisplay="" - desc=msgs.formatEmbedDesc(vnf['type'],desc,vnf['qrt'],pollDisplay,likeDisplay) - + qrt=None + if vnf['qrtURL'] is not None: + qrt,e=vnfFromCacheOrDL(vnf['qrtURL']) + if qrt is not None: + desc=msgs.formatEmbedDesc(vnf['type'],desc,qrt,pollDisplay,likeDisplay) + embedVNF=None appNamePost = "" if vnf['type'] == "Text": # Change the template based on tweet type template = 'text.html' + if qrt is not None and qrt['type'] != "Text": + embedVNF=qrt + if qrt['type'] == "Image": + if embedVNF['images'][4]!="1": + appNamePost = " - Image " + str(image+1) + "/" + str(vnf['images'][4]) + image = embedVNF['images'][image] + template = 'image.html' + elif qrt['type'] == "Video" or qrt['type'] == "": + urlDesc = urllib.parse.quote(textwrap.shorten(desc, width=220, placeholder="...")) + template = 'video.html' + if vnf['type'] == "Image": if vnf['images'][4]!="1": appNamePost = " - Image " + str(image+1) + "/" + str(vnf['images'][4]) @@ -441,7 +455,7 @@ def embed(video_link, vnf, image): if vnf['nsfw'] == True: color = "#800020" # Red - return getTemplate(template,vnf,desc,image,video_link,color,urlDesc,urlUser,urlLink) + return getTemplate(template,vnf,desc,image,video_link,color,urlDesc,urlUser,urlLink,appNamePost,embedVNF) def embedCombined(video_link): @@ -468,7 +482,11 @@ def embedCombinedVnf(video_link,vnf): else: pollDisplay="" - desc=msgs.formatEmbedDesc(vnf['type'],desc,vnf['qrt'],pollDisplay,likeDisplay) + qrt=None + if vnf['qrtURL'] is not None: + qrt,e=vnfFromCacheOrDL(vnf['qrtURL']) + if qrt is not None: + desc=msgs.formatEmbedDesc(vnf['type'],desc,qrt,pollDisplay,likeDisplay) image = "https://vxtwitter.com/rendercombined.jpg?imgs="