From 25ebf7d9d3c69b1c64dd23d753befbcb39ed1701 Mon Sep 17 00:00:00 2001 From: Dylan Date: Tue, 15 Aug 2023 02:13:53 +0100 Subject: [PATCH] Changes to used token logic --- twExtract/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/twExtract/__init__.py b/twExtract/__init__.py index 90ef98b..a2ecc17 100644 --- a/twExtract/__init__.py +++ b/twExtract/__init__.py @@ -43,7 +43,6 @@ def extractStatus_token(url): usedTokens.clear() random.shuffle(tokens) for authToken in tokens: - usedTokens.append(authToken) try: csrfToken=str(uuid.uuid4()).replace('-', '') tweet = requests.get("https://api.twitter.com/1.1/statuses/show/" + twid + ".json?tweet_mode=extended&cards_platform=Web-12&include_cards=1&include_reply_count=1&include_user_entities=0", headers={"Authorization":bearer,"Cookie":f"auth_token={authToken}; ct0={csrfToken}; ","x-twitter-active-user":"yes","x-twitter-auth-type":"OAuth2Session","x-twitter-client-language":"en","x-csrf-token":csrfToken,"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0"}) @@ -126,7 +125,6 @@ def extractStatusV2(url): usedTokens.clear() random.shuffle(tokens) for authToken in tokens: - usedTokens.append(authToken) try: csrfToken=str(uuid.uuid4()).replace('-', '') vars = json.loads('{"includeTweetImpression":true,"includeHasBirdwatchNotes":false,"includeEditPerspective":false,"rest_ids":["x"],"includeEditControl":true,"includeCommunityTweetRelationship":true,"includeTweetVisibilityNudge":true}') @@ -135,10 +133,14 @@ def extractStatusV2(url): try: rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining") print(f"Twitter Token Rate limit remaining: {rateLimitRemaining}") + if (rateLimitRemaining == "0"): + usedTokens.append(authToken) + continue except: # for some reason the header is not always present pass if tweet.status_code == 429: # try another token + usedTokens.append(authToken) continue output = tweet.json() @@ -208,7 +210,6 @@ def extractUser(url): usedTokens.clear() random.shuffle(tokens) for authToken in tokens: - usedTokens.append(authToken) try: csrfToken=str(uuid.uuid4()).replace('-', '') reqHeaders = {"Authorization":bearer,"Cookie":f"auth_token={authToken}; ct0={csrfToken}; ","x-twitter-active-user":"yes","x-twitter-auth-type":"OAuth2Session","x-twitter-client-language":"en","x-csrf-token":csrfToken,"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0"}