Fix tests & poll extract

test_vx_extract.py

@@ -9,28 +9,28 @@ def test_twextract_syndicationAPI():
     assert utils.stripEndTCO(utils.stripEndTCO(tweet["full_text"]))==testMediaTweet_compare['text']
 
 def test_twextract_extractStatusV2Anon():
-    tweet = twExtract.extractStatusV2AnonLegacy(testTextTweet,None)
+    tweet = twExtract.extractStatusV2Anon(testTextTweet,None)['legacy']
     assert utils.stripEndTCO(tweet["full_text"])==testTextTweet_compare['text']
-    tweet = twExtract.extractStatusV2AnonLegacy(testVideoTweet,None)
+    tweet = twExtract.extractStatusV2Anon(testVideoTweet,None)['legacy']
     assert utils.stripEndTCO(tweet["full_text"])==testVideoTweet_compare['text']
-    tweet = twExtract.extractStatusV2AnonLegacy(testMediaTweet,None)
+    tweet = twExtract.extractStatusV2Anon(testMediaTweet,None)['legacy']
     assert utils.stripEndTCO(tweet["full_text"])==testMediaTweet_compare['text']
-    tweet = twExtract.extractStatusV2AnonLegacy(testMultiMediaTweet,None)
+    tweet = twExtract.extractStatusV2Anon(testMultiMediaTweet,None)['legacy']
     assert utils.stripEndTCO(tweet["full_text"])[:94]==testMultiMediaTweet_compare['text'][:94]
     
 
 def test_twextract_v2API():
-    tweet = twExtract.extractStatusV2Legacy(testMediaTweet,workaroundTokens=tokens)
+    tweet = twExtract.extractStatusV2(testMediaTweet,workaroundTokens=tokens)['legacy']
     assert utils.stripEndTCO(tweet["full_text"])==testMediaTweet_compare['text']
 
 def test_twextract_v2AndroidAPI():
-    tweet = twExtract.extractStatusV2AndroidLegacy(testMediaTweet,workaroundTokens=tokens)
+    tweet = twExtract.extractStatusV2Android(testMediaTweet,workaroundTokens=tokens)['legacy']
     assert utils.stripEndTCO(tweet["full_text"])==testMediaTweet_compare['text']
 
 ## Tweet retrieve tests ##
 def test_twextract_textTweetExtract():
     tweet = twExtract.extractStatus(testTextTweet,workaroundTokens=tokens)
-    assert utils.stripEndTCO(tweet["full_text"])==testTextTweet_compare['text']
+    assert utils.stripEndTCO(tweet["legacy"]["full_text"])==testTextTweet_compare['text']
     assert tweet["user"]["screen_name"]=="jack"
     assert 'extended_entities' not in tweet
     
@@ -58,7 +58,7 @@ def test_twextract_UserExtractWeirdURLs():
 
 def test_twextract_videoTweetExtract():
     tweet = twExtract.extractStatus(testVideoTweet,workaroundTokens=tokens)
-    assert utils.stripEndTCO(tweet["full_text"])==testVideoTweet_compare['text']
+    assert utils.stripEndTCO(tweet["legacy"]["full_text"])==testVideoTweet_compare['text']
     assert 'extended_entities' in tweet
     assert len(tweet['extended_entities']["media"])==1
     video = tweet['extended_entities']["media"][0]
@@ -68,7 +68,7 @@ def test_twextract_videoTweetExtract():
 
 def test_twextract_mediaTweetExtract():
     tweet = twExtract.extractStatus(testMediaTweet,workaroundTokens=tokens)
-    assert utils.stripEndTCO(tweet["full_text"])==testMediaTweet_compare['text']
+    assert utils.stripEndTCO(tweet['legacy']["full_text"])==testMediaTweet_compare['text']
     assert 'extended_entities' in tweet
     assert len(tweet['extended_entities']["media"])==1
     video = tweet['extended_entities']["media"][0]
@@ -79,7 +79,7 @@ def test_twextract_mediaTweetExtract():
 
 def test_twextract_multimediaTweetExtract():
     tweet = twExtract.extractStatus(testMultiMediaTweet,workaroundTokens=tokens)
-    assert utils.stripEndTCO(tweet["full_text"])[:94]==testMultiMediaTweet_compare['text'][:94]
+    assert utils.stripEndTCO(tweet['legacy']["full_text"])[:94]==testMultiMediaTweet_compare['text'][:94]
     assert 'extended_entities' in tweet
     assert len(tweet['extended_entities']["media"])==3
     video = tweet['extended_entities']["media"][0]

twExtract/__init__.py

@@ -321,11 +321,33 @@ def extractStatusV2Anon(url,x):
         result = entry['result']
         if '__typename' in result and result['__typename'] == 'TweetWithVisibilityResults':
             result=result['tweet']
+        elif '__typename' in result and result['__typename'] == 'TweetUnavailable':
+            if 'reason' in result:
+                raise TwExtractError(400, "Extract error: "+result['reason'])
+            raise TwExtractError(400, "Extract error")
         if 'rest_id' in result and result['rest_id'] == twid:
             tweetEntry=result
         tweet=tweetEntry
     except Exception as e:
         raise TwExtractError(400, "Extract error")
+    if 'card' in tweet and 'legacy' in tweet['card']:
+        tweet['card'] = tweet['card']['legacy']
+    return tweet
+
+def fixTweetData(tweet):
+    try:
+        if 'user' not in tweet:
+            tweet['user'] = tweet['core']['user_results']['result']['legacy']
+    except:
+        print("fixTweetData error: No user")
+        pass
+
+    try:
+        if 'extended_entities' not in tweet and 'extended_entities' in tweet['legacy']:
+            tweet['extended_entities'] = tweet['legacy']['extended_entities']
+    except:
+        print("fixTweetData error: extended_entities")
+        pass
     return tweet
 
 def extractStatus(url,workaroundTokens=None):
@@ -337,7 +359,7 @@ def extractStatus(url,workaroundTokens=None):
                 print(f"{method.__name__} method failed: Legacy not found for {url}")
                 # try another method
                 continue
-            return result
+            return fixTweetData(result)
         except Exception as e:
             print(f"{method.__name__} method failed: {str(e)} for {url}")
             continue

vxApi.py

@@ -112,12 +112,18 @@ def getApiResponse(tweet,include_txt=False,include_zip=False):
         combinedMediaUrl = combinedMediaUrl[:-1]
 
     pollData = None
+    card = None
     if 'card' in tweet and 'legacy' in tweet['card'] and tweet['card']['legacy']['name'].startswith("poll"):
-        cardName = tweet['card']['legacy']['name']
+        card = tweet['card']['legacy']
+    elif 'card' in tweet and 'binding_values' in tweet['card']:
+        card = tweet['card']
+
+    if card != None:
+        cardName = card['name']
         pollData={} # format: {"options":["name":"Option 1 Name","votes":5,"percent":50]}
         pollData["options"] = []
         totalVotes = 0
-        bindingValues = tweet['card']['legacy']['binding_values']
+        bindingValues = card['binding_values']
         pollValues = {}
         for i in bindingValues:
             key = i["key"]