From bc1d6bbf3eb258e89a08e789ce0f704e6d9fe0be Mon Sep 17 00:00:00 2001 From: Dylan Date: Tue, 7 Nov 2023 17:24:30 +0000 Subject: [PATCH] New possible extract method (V2 Anon) --- twExtract/__init__.py | 61 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/twExtract/__init__.py b/twExtract/__init__.py index 203fe26..2548b59 100644 --- a/twExtract/__init__.py +++ b/twExtract/__init__.py @@ -16,6 +16,9 @@ userIDregex = r"\/i\/user\/(\d+)" v2Features='{"longform_notetweets_inline_media_enabled":true,"super_follow_badge_privacy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"super_follow_user_api_enabled":true,"super_follow_tweet_api_enabled":true,"android_graphql_skip_api_media_color_palette":true,"creator_subscriptions_tweet_preview_api_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"creator_subscriptions_subscription_count_enabled":true,"tweetypie_unmention_optimization_enabled":true,"longform_notetweets_consumption_enabled":true,"subscriptions_verification_info_enabled":true,"blue_business_profile_image_shape_enabled":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"super_follow_exclusive_tweet_notifications_enabled":true}' v2graphql_api="2OOZWmw8nAtUHVnXXQhgaA" +v2AnonFeatures='{"creator_subscriptions_tweet_preview_api_enabled":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":false,"tweet_awards_web_tipping_enabled":false,"responsive_web_home_pinned_timelines_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"responsive_web_media_download_video_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false}' +v2AnonGraphql_api="5GOHgZe-8U2j5sVHQzEm9A" + class TwExtractError(Exception): def __init__(self, code, message): self.code = code @@ -27,7 +30,7 @@ class TwExtractError(Exception): def getGuestToken(): global guestToken if guestToken is None: - r = requests.post("https://api.twitter.com/1.1/guest/activate.json", headers={"Authorization":bearer}) + r = requests.post("https://api.twitter.com/1.1/guest/activate.json", headers={"Authorization":v2Bearer}) guestToken = json.loads(r.text)["guest_token"] return guestToken @@ -197,6 +200,43 @@ def extractStatusV2(url,workaroundTokens): return tweet raise TwExtractError(400, "Extract error") +def extractStatusV2Anon(url): + # get tweet ID + m = re.search(pathregex, url) + if m is None: + raise TwExtractError(400, "Extract error") + twid = m.group(2) + + guestToken = getGuestToken() + + # get tweet + try: + vars = json.loads('{"tweetId":"0","withCommunity":false,"includePromotedContent":false,"withVoice":false}') + vars['tweetId'] = str(twid) + tweet = requests.get(f"https://api.twitter.com/graphql/{v2AnonGraphql_api}/TweetResultByRestId?variables={urllib.parse.quote(json.dumps(vars))}&features={urllib.parse.quote(v2AnonFeatures)}", headers={"Authorization":v2Bearer,"x-twitter-active-user":"yes","x-guest-token":guestToken,"x-twitter-client-language":"en","User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0"}) + try: + rateLimitRemaining = tweet.headers.get("x-rate-limit-remaining") + print(f"Twitter Anon Token Rate limit remaining: {rateLimitRemaining}") + except: # for some reason the header is not always present + pass + if tweet.status_code == 429: + raise TwExtractError(400, "Extract error") + output = tweet.json() + + if "errors" in output: + raise TwExtractError(400, "Extract error") + entry=output['data']['tweetResult'] + tweetEntry=None + result = entry['result'] + if '__typename' in result and result['__typename'] == 'TweetWithVisibilityResults': + result=result['tweet'] + if 'rest_id' in result and result['rest_id'] == twid: + tweetEntry=result + tweet=tweetEntry + except Exception as e: + raise TwExtractError(400, "Extract error") + return tweet + def extractStatusV2Legacy(url,workaroundTokens): tweet = extractStatusV2(url,workaroundTokens) if 'errors' in tweet or 'legacy' not in tweet: @@ -216,6 +256,25 @@ def extractStatusV2Legacy(url,workaroundTokens): tweet['legacy']['card'] = tweet['tweet_card']['legacy'] return tweet['legacy'] +def extractStatusV2AnonLegacy(url,workaroundTokens): + tweet = extractStatusV2Anon(url) + if 'errors' in tweet or 'legacy' not in tweet: + if 'errors' in tweet: + raise TwExtractError(400, "Extract error: "+json.dumps(tweet['errors'])) + else: + raise TwExtractError(400, "Extract error (no legacy data)") + tweet['legacy']['user'] = tweet["core"]["user_results"]["result"]["legacy"] + tweet['legacy']['user']['profile_image_url'] = tweet['legacy']['user']['profile_image_url_https'] + if 'card' in tweet: + tweet['legacy']['card'] = tweet['card']['legacy'] + if 'extended_entities' in tweet['legacy']: + tweet['legacy']['extended_entities'] = {'media':tweet['legacy']['extended_entities']['media']} + for media in tweet['legacy']['extended_entities']['media']: + media['media_url'] = media['media_url_https'] + if 'tweet_card' in tweet: + tweet['legacy']['card'] = tweet['tweet_card']['legacy'] + return tweet['legacy'] + def extractStatus(url,workaroundTokens=None): methods=[extractStatus_syndication,extractStatusV2Legacy,extractStatus_twExtractProxy] for method in methods: