From d24903e383e0caa4d5f8ecd01e088ef201514b43 Mon Sep 17 00:00:00 2001
From: Dylan <dylanpdx@gmail.com>
Date: Tue, 15 Aug 2023 02:00:49 +0100
Subject: [PATCH] Misc fixes for ratelimiting

---
 twExtract/__init__.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/twExtract/__init__.py b/twExtract/__init__.py
index c137b7a..cdd0c91 100644
--- a/twExtract/__init__.py
+++ b/twExtract/__init__.py
@@ -17,6 +17,7 @@ userIDregex = r"\/i\/user\/(\d+)"
 
 v2Features='{"longform_notetweets_inline_media_enabled":true,"super_follow_badge_privacy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"super_follow_user_api_enabled":true,"super_follow_tweet_api_enabled":true,"android_graphql_skip_api_media_color_palette":true,"creator_subscriptions_tweet_preview_api_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"creator_subscriptions_subscription_count_enabled":true,"tweetypie_unmention_optimization_enabled":true,"longform_notetweets_consumption_enabled":true,"subscriptions_verification_info_enabled":true,"blue_business_profile_image_shape_enabled":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"super_follow_exclusive_tweet_notifications_enabled":true}'
 v2graphql_api="2OOZWmw8nAtUHVnXXQhgaA"
+usedTokens=[]
 
 def getGuestToken():
     global guestToken
@@ -26,6 +27,7 @@ def getGuestToken():
     return guestToken
 
 def extractStatus_token(url):
+    global usedTokens
     # get tweet ID
     m = re.search(pathregex, url)
     if m is None:
@@ -35,8 +37,13 @@ def extractStatus_token(url):
         raise twExtractError.TwExtractError(400, "Extract error (no tokens defined)")
     # get tweet
     tokens = config["config"]["workaroundTokens"].split(",")
+    tokens = [i for i in tokens if i not in usedTokens]
+    if len(tokens) == 0:
+        tokens = config["config"]["workaroundTokens"].split(",")
+        usedTokens.clear()
     random.shuffle(tokens)
     for authToken in tokens:
+        usedTokens.append(authToken)
         try:
             csrfToken=str(uuid.uuid4()).replace('-', '')
             tweet = requests.get("https://api.twitter.com/1.1/statuses/show/" + twid + ".json?tweet_mode=extended&cards_platform=Web-12&include_cards=1&include_reply_count=1&include_user_entities=0", headers={"Authorization":bearer,"Cookie":f"auth_token={authToken}; ct0={csrfToken}; ","x-twitter-active-user":"yes","x-twitter-auth-type":"OAuth2Session","x-twitter-client-language":"en","x-csrf-token":csrfToken,"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0"})
@@ -102,6 +109,7 @@ def extractStatus_syndication(url):
     return output
 
 def extractStatusV2(url):
+    global usedTokens
     # get tweet ID
     m = re.search(pathregex, url)
     if m is None:
@@ -111,8 +119,14 @@ def extractStatusV2(url):
         raise twExtractError.TwExtractError(400, "Extract error (no tokens defined)")
     # get tweet
     tokens = config["config"]["workaroundTokens"].split(",")
+    print("Number of tokens: "+str(len(tokens)))
+    tokens = [i for i in tokens if i not in usedTokens]
+    if len(tokens) == 0:
+        tokens = config["config"]["workaroundTokens"].split(",")
+        usedTokens.clear()
     random.shuffle(tokens)
     for authToken in tokens:
+        usedTokens.append(authToken)
         try:
             csrfToken=str(uuid.uuid4()).replace('-', '')
             vars = json.loads('{"includeTweetImpression":true,"includeHasBirdwatchNotes":false,"includeEditPerspective":false,"rest_ids":["x"],"includeEditControl":true,"includeCommunityTweetRelationship":true,"includeTweetVisibilityNudge":true}')
@@ -176,6 +190,7 @@ def extractStatus(url):
     raise twExtractError.TwExtractError(400, "Extract error")
 
 def extractUser(url):
+    global usedTokens
     useId=True
     m = re.search(userIDregex, url)
     if m is None:
@@ -187,8 +202,13 @@ def extractUser(url):
     screen_name = m.group(1)
     # get user
     tokens = config["config"]["workaroundTokens"].split(",")
+    tokens = [i for i in tokens if i not in usedTokens]
+    if len(tokens) == 0:
+        tokens = config["config"]["workaroundTokens"].split(",")
+        usedTokens.clear()
     random.shuffle(tokens)
     for authToken in tokens:
+        usedTokens.append(authToken)
         try:
             csrfToken=str(uuid.uuid4()).replace('-', '')
             reqHeaders = {"Authorization":bearer,"Cookie":f"auth_token={authToken}; ct0={csrfToken}; ","x-twitter-active-user":"yes","x-twitter-auth-type":"OAuth2Session","x-twitter-client-language":"en","x-csrf-token":csrfToken,"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0"}