Provide better responses for errors (#196); Fix some extract methods

test_api.py

@@ -54,6 +54,23 @@ def test_api_user():
     assert resp.status_code==200
     assert jData["screen_name"]=="jack"
 
+def test_api_user_suspended():
+    resp = client.get(testUserSuspended.replace("https://twitter.com","https://api.vxtwitter.com"),headers={"User-Agent":"test"})
+    jData = resp.get_json()
+    assert resp.status_code==500
+    assert 'suspended' in jData["error"]
+
+def test_api_user_private():
+    resp = client.get(testUserPrivate.replace("https://twitter.com","https://api.vxtwitter.com")+"?with_tweets=true",headers={"User-Agent":"test"})
+    jData = resp.get_json()
+    assert jData['protected'] == True
+    assert len(jData["latest_tweets"])==0
+
+def test_api_user_invalid():
+    resp = client.get(testUserInvalid.replace("https://twitter.com","https://api.vxtwitter.com")+"?with_tweets=true",headers={"User-Agent":"test"})
+    jData = resp.get_json()
+    assert resp.status_code==404
+
 def test_api_user_feed():
     resp = client.get(testUser.replace("https://twitter.com","https://api.vxtwitter.com")+"?with_tweets=true",headers={"User-Agent":"test"})
     jData = resp.get_json()

twExtract/__init__.py

@@ -315,7 +315,11 @@ def extractStatusV2Android(url,workaroundTokens):
                 print(f"Error in output: {json.dumps(output['errors'])}")
                 # try another token
                 continue
-            entries=output['data']['timeline_response']['instructions'][0]['entries']
+            entries = None
+            for instruction in output['data']['timeline_response']['instructions']:
+                if instruction["__typename"] == "TimelineAddEntries":
+                    entries = instruction['entries']
+                    break
             tweetEntry=None
             for entry in entries:
                 if 'content' not in entry:
@@ -372,7 +376,11 @@ def extractStatusV2TweetDetail(url,workaroundTokens):
                 print(f"Error in output: {json.dumps(output['errors'])}")
                 # try another token
                 continue
-            entries=output['data']['threaded_conversation_with_injections_v2']['instructions'][0]['entries']
+            entries = None
+            for instruction in output['data']['threaded_conversation_with_injections_v2']['instructions']:
+                if instruction["type"] == "TimelineAddEntries":
+                    entries = instruction['entries']
+                    break
             tweetEntry=None
             for entry in entries:
                 if 'content' not in entry:
@@ -501,6 +509,8 @@ def extractUser(url,workaroundTokens):
                 raise TwExtractError(error["code"], error["message"])
             return output
         except Exception as e:
+            if hasattr(e,"msg") and (e.msg == 'User has been suspended.' or e.msg == 'User not found.'):
+                raise e
             continue
     raise TwExtractError(400, "Extract error")
 

twitfix.py

@@ -295,6 +295,9 @@ def getUserData(twitter_url,includeFeed=False):
     userData = getApiUserResponse(rawUserData)
 
     if includeFeed:
+        if userData['protected']:
+            userData['latest_tweets']=[]
+        else:
             feed = twExtract.extractUserFeedFromId(userData['id'],workaroundTokens=config['config']['workaroundTokens'].split(','))
             apiFeed = []
             for tweet in feed:
@@ -330,7 +333,15 @@ def twitfix(sub_path):
             username=sub_path.split("/")[0]
             extra = sub_path.split("/")[1]
         if extra in [None,"with_replies","media","likes","highlights","superfollows","media",''] and username != "" and username != None:
+            try:
                 userData = getUserData(f"https://twitter.com/{username}","with_tweets" in request.args)
+            except twExtract.TwExtractError as e:
+                if isApiRequest:
+                    status=500
+                    if 'not found' in e.msg:
+                        status=404
+                    return Response(json.dumps({"error": e.msg}), status=status,mimetype='application/json')
+                return message("Error getting user data: "+str(e.msg))
             if isApiRequest:
                 if userData is None:
                     abort(404)

vxApi.py

@@ -217,7 +217,7 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False):
                 totalVotes += option["votes"]
                 pollData["options"].append(option)
         for i in pollData["options"]:
-            i["percent"] = round((i["votes"]/totalVotes)*100,2)
+            i["percent"] = round((i["votes"]/totalVotes)*100,2) if totalVotes > 0 else 0
         
     if 'lang' in tweetL:
         lang = tweetL['lang']

vx_testdata.py

@@ -27,6 +27,9 @@ testMixedMediaTweet_compare={'text': 'Some of us here are definitely big nerds a
 testVinePlayerTweet_compare={'text': 'You wanted old ROBLOX back, you got it. Check out our sweet "new" look! #BringBackOldROBLOX https://vine.co/v/OL9VqvM6wJh', 'date': 'Wed Apr 01 16:17:13 +0000 2015', 'tweetURL': 'https://twitter.com/Roblox/status/583302104342638592', 'tweetID': '583302104342638592', 'conversationID': '583302104342638592', 'mediaURLs': ['https://v.cdn.vine.co/r/videos/20A1BE53011195086166081318912_3fe3b526b1a.1.5.3156516531034157495.mp4?versionId=DI1mMu7EI6zcLbvgucyp3GHebdz8.9cQ'], 'media_extended': [{'url': 'https://v.cdn.vine.co/r/videos/20A1BE53011195086166081318912_3fe3b526b1a.1.5.3156516531034157495.mp4?versionId=DI1mMu7EI6zcLbvgucyp3GHebdz8.9cQ', 'type': 'video', 'size': {'width': 435, 'height': 435}}], 'possibly_sensitive': False, 'hashtags': [], 'qrtURL': None, 'allSameType': True, 'hasMedia': True, 'combinedMediaUrl': None, 'pollData': {'options': []}, 'article': None, 'date_epoch': 1427905033}
 
 testUser="https://twitter.com/jack"
+testUserSuspended="https://twitter.com/twitter"
+testUserPrivate="https://twitter.com/PrestigeIsKey"
+testUserInvalid="https://twitter.com/.a"
 testUserID=12 # could also be 170824883
 testUserIDUrl = "https://twitter.com/i/user/"+str(testUserID)
 testUserWeirdURLs=["https://twitter.com/jack?lang=en","https://twitter.com/jack/with_replies","https://twitter.com/jack/media","https://twitter.com/jack/likes","https://twitter.com/jack/with_replies?lang=en","https://twitter.com/jack/media?lang=en","https://twitter.com/jack/likes?lang=en","https://twitter.com/jack/"]