From 1e9265ed2654bc84b19748bcce08c69afafc9c21 Mon Sep 17 00:00:00 2001 From: Dylan Date: Fri, 17 May 2024 23:43:18 +0100 Subject: [PATCH] Fix tests & poll extract --- test_vx_extract.py | 20 ++++++++++---------- twExtract/__init__.py | 24 +++++++++++++++++++++++- vxApi.py | 10 ++++++++-- 3 files changed, 41 insertions(+), 13 deletions(-) diff --git a/test_vx_extract.py b/test_vx_extract.py index 8575272..75eb7b7 100644 --- a/test_vx_extract.py +++ b/test_vx_extract.py @@ -9,28 +9,28 @@ def test_twextract_syndicationAPI(): assert utils.stripEndTCO(utils.stripEndTCO(tweet["full_text"]))==testMediaTweet_compare['text'] def test_twextract_extractStatusV2Anon(): - tweet = twExtract.extractStatusV2AnonLegacy(testTextTweet,None) + tweet = twExtract.extractStatusV2Anon(testTextTweet,None)['legacy'] assert utils.stripEndTCO(tweet["full_text"])==testTextTweet_compare['text'] - tweet = twExtract.extractStatusV2AnonLegacy(testVideoTweet,None) + tweet = twExtract.extractStatusV2Anon(testVideoTweet,None)['legacy'] assert utils.stripEndTCO(tweet["full_text"])==testVideoTweet_compare['text'] - tweet = twExtract.extractStatusV2AnonLegacy(testMediaTweet,None) + tweet = twExtract.extractStatusV2Anon(testMediaTweet,None)['legacy'] assert utils.stripEndTCO(tweet["full_text"])==testMediaTweet_compare['text'] - tweet = twExtract.extractStatusV2AnonLegacy(testMultiMediaTweet,None) + tweet = twExtract.extractStatusV2Anon(testMultiMediaTweet,None)['legacy'] assert utils.stripEndTCO(tweet["full_text"])[:94]==testMultiMediaTweet_compare['text'][:94] def test_twextract_v2API(): - tweet = twExtract.extractStatusV2Legacy(testMediaTweet,workaroundTokens=tokens) + tweet = twExtract.extractStatusV2(testMediaTweet,workaroundTokens=tokens)['legacy'] assert utils.stripEndTCO(tweet["full_text"])==testMediaTweet_compare['text'] def test_twextract_v2AndroidAPI(): - tweet = twExtract.extractStatusV2AndroidLegacy(testMediaTweet,workaroundTokens=tokens) + tweet = twExtract.extractStatusV2Android(testMediaTweet,workaroundTokens=tokens)['legacy'] assert utils.stripEndTCO(tweet["full_text"])==testMediaTweet_compare['text'] ## Tweet retrieve tests ## def test_twextract_textTweetExtract(): tweet = twExtract.extractStatus(testTextTweet,workaroundTokens=tokens) - assert utils.stripEndTCO(tweet["full_text"])==testTextTweet_compare['text'] + assert utils.stripEndTCO(tweet["legacy"]["full_text"])==testTextTweet_compare['text'] assert tweet["user"]["screen_name"]=="jack" assert 'extended_entities' not in tweet @@ -58,7 +58,7 @@ def test_twextract_UserExtractWeirdURLs(): def test_twextract_videoTweetExtract(): tweet = twExtract.extractStatus(testVideoTweet,workaroundTokens=tokens) - assert utils.stripEndTCO(tweet["full_text"])==testVideoTweet_compare['text'] + assert utils.stripEndTCO(tweet["legacy"]["full_text"])==testVideoTweet_compare['text'] assert 'extended_entities' in tweet assert len(tweet['extended_entities']["media"])==1 video = tweet['extended_entities']["media"][0] @@ -68,7 +68,7 @@ def test_twextract_videoTweetExtract(): def test_twextract_mediaTweetExtract(): tweet = twExtract.extractStatus(testMediaTweet,workaroundTokens=tokens) - assert utils.stripEndTCO(tweet["full_text"])==testMediaTweet_compare['text'] + assert utils.stripEndTCO(tweet['legacy']["full_text"])==testMediaTweet_compare['text'] assert 'extended_entities' in tweet assert len(tweet['extended_entities']["media"])==1 video = tweet['extended_entities']["media"][0] @@ -79,7 +79,7 @@ def test_twextract_mediaTweetExtract(): def test_twextract_multimediaTweetExtract(): tweet = twExtract.extractStatus(testMultiMediaTweet,workaroundTokens=tokens) - assert utils.stripEndTCO(tweet["full_text"])[:94]==testMultiMediaTweet_compare['text'][:94] + assert utils.stripEndTCO(tweet['legacy']["full_text"])[:94]==testMultiMediaTweet_compare['text'][:94] assert 'extended_entities' in tweet assert len(tweet['extended_entities']["media"])==3 video = tweet['extended_entities']["media"][0] diff --git a/twExtract/__init__.py b/twExtract/__init__.py index 34c542a..0a31856 100644 --- a/twExtract/__init__.py +++ b/twExtract/__init__.py @@ -321,11 +321,33 @@ def extractStatusV2Anon(url,x): result = entry['result'] if '__typename' in result and result['__typename'] == 'TweetWithVisibilityResults': result=result['tweet'] + elif '__typename' in result and result['__typename'] == 'TweetUnavailable': + if 'reason' in result: + raise TwExtractError(400, "Extract error: "+result['reason']) + raise TwExtractError(400, "Extract error") if 'rest_id' in result and result['rest_id'] == twid: tweetEntry=result tweet=tweetEntry except Exception as e: raise TwExtractError(400, "Extract error") + if 'card' in tweet and 'legacy' in tweet['card']: + tweet['card'] = tweet['card']['legacy'] + return tweet + +def fixTweetData(tweet): + try: + if 'user' not in tweet: + tweet['user'] = tweet['core']['user_results']['result']['legacy'] + except: + print("fixTweetData error: No user") + pass + + try: + if 'extended_entities' not in tweet and 'extended_entities' in tweet['legacy']: + tweet['extended_entities'] = tweet['legacy']['extended_entities'] + except: + print("fixTweetData error: extended_entities") + pass return tweet def extractStatus(url,workaroundTokens=None): @@ -337,7 +359,7 @@ def extractStatus(url,workaroundTokens=None): print(f"{method.__name__} method failed: Legacy not found for {url}") # try another method continue - return result + return fixTweetData(result) except Exception as e: print(f"{method.__name__} method failed: {str(e)} for {url}") continue diff --git a/vxApi.py b/vxApi.py index 4b8b724..33ac506 100644 --- a/vxApi.py +++ b/vxApi.py @@ -112,12 +112,18 @@ def getApiResponse(tweet,include_txt=False,include_zip=False): combinedMediaUrl = combinedMediaUrl[:-1] pollData = None + card = None if 'card' in tweet and 'legacy' in tweet['card'] and tweet['card']['legacy']['name'].startswith("poll"): - cardName = tweet['card']['legacy']['name'] + card = tweet['card']['legacy'] + elif 'card' in tweet and 'binding_values' in tweet['card']: + card = tweet['card'] + + if card != None: + cardName = card['name'] pollData={} # format: {"options":["name":"Option 1 Name","votes":5,"percent":50]} pollData["options"] = [] totalVotes = 0 - bindingValues = tweet['card']['legacy']['binding_values'] + bindingValues = card['binding_values'] pollValues = {} for i in bindingValues: key = i["key"]