Merge branch 'main' into alterware

.github/workflows/python-tests.yml

@@ -6,23 +6,30 @@ jobs:
   build:
     runs-on: ubuntu-latest
     environment: test
-    strategy:
-      matrix:
-        python-version: ["3.10", "3.11", "3.12"]
-
     steps:
-      - uses: actions/checkout@v3
+      - name: Check out files
-      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/checkout@v4
-        uses: actions/setup-python@v4
+
-        with:
+      - name: Install Python
-          python-version: ${{ matrix.python-version }}
+        run: |
+          apt-get update
+          apt-get install -y python3 python3-pip python3-venv
+
+      - name: Set up virtual environment
+        run: |
+          python3 -m venv venv
+          . venv/bin/activate
+          python -m pip install --upgrade pip
+
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          . venv/bin/activate
           pip install pytest pytest-cov pytest-env
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
       - name: Test with pytest
         env:
           VXTWITTER_WORKAROUND_TOKENS: ${{ secrets.VXTWITTER_WORKAROUND_TOKENS }}
         run: |
+          . venv/bin/activate
           pytest --cov-config=.coveragerc --cov=.

config.json

@@ -2,14 +2,14 @@
     "config": {
         "appname": "vxTwitter",
         "color": "#1DA1F2",
-        "database": "[url to mongo database goes here]",
+        "database": "mongodb://localhost:27017/bettervxtwitter",
-        "link_cache": "ram",
+        "link_cache": "db",
         "method": "hybrid",
-        "repo": "https://github.com/dylanpdx/BetterTwitFix",
+        "repo": "https://git.alterware.dev/alterware/BetterTwitFix",
-        "table": "[database table here]",
+        "table": "links",
-        "url": "https://vxtwitter.com",
+        "url": "https://girlcock.alterware.dev",
         "combination_method": "local",
         "gifConvertAPI": "local",
-        "workaroundTokens":null
+        "workaroundTokens": null
     }
 }

twExtract/__init__.py

@@ -38,8 +38,8 @@ tweetDetailGraphqlFeatures='{"rweb_tipjar_consumption_enabled":true,"responsive_
 tweetDetailGraphql_api="e7RKseIxLu7HgkWNKZ6qnw"
 
 # this is for UserTweets endpoint
-tweetFeedGraphqlFeatures='{"rweb_video_screen_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":false,"responsive_web_grok_share_attachment_enabled":true,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":false,"responsive_web_grok_analysis_button_from_backend":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_enhance_cards_enabled":false}'
+tweetFeedGraphqlFeatures='{"rweb_video_screen_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":false,"responsive_web_jetfuel_frame":false,"responsive_web_grok_share_attachment_enabled":true,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":false,"responsive_web_grok_analysis_button_from_backend":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_enhance_cards_enabled":false}'
-tweetFeedGraphql_api="OAx9yEcW3JA9bPo63pcYlA"
+tweetFeedGraphql_api="Li2XXGESVev94TzFtntrgA"
 
 twitterUrl = "x.com" # doubt this will change but just in case
 class TwExtractError(Exception):
@@ -109,6 +109,7 @@ def getAuthHeaders(btoken,authToken=None,guestToken=None):
         headers["x-twitter-auth-type"] = "OAuth2Session"
     if guestToken is not None:
         headers["x-guest-token"] = guestToken
+        headers["Cookie"] = f"gt={guestToken}; ct0={csrfToken}; guest_id=v1:174804309415864668;"
 
     return headers
 
@@ -522,11 +523,11 @@ def extractUserFeedFromId(userId,workaroundTokens):
             # TODO: https://api.twitter.com/graphql/x31u1gdnjcqtiVZFc1zWnQ/UserWithProfileTweetsQueryV2?variables={"cursor":"?","includeTweetImpression":true,"includeHasBirdwatchNotes":false,"includeEditPerspective":false,"includeEditControl":true,"count":40,"rest_id":"12","includeTweetVisibilityNudge":true,"autoplay_enabled":true}&features={"longform_notetweets_inline_media_enabled":true,"super_follow_badge_privacy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"super_follow_user_api_enabled":true,"unified_cards_ad_metadata_container_dynamic_card_content_query_enabled":true,"super_follow_tweet_api_enabled":true,"articles_api_enabled":true,"android_graphql_skip_api_media_color_palette":true,"creator_subscriptions_tweet_preview_api_enabled":true,"freedom_of_speech_not_reach_fetch_enabled":true,"tweetypie_unmention_optimization_enabled":true,"longform_notetweets_consumption_enabled":true,"subscriptions_verification_info_enabled":true,"blue_business_profile_image_shape_enabled":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"immersive_video_status_linkable_timestamps":false,"super_follow_exclusive_tweet_notifications_enabled":true}
             continue
         try:
-            vars = json.loads('{"userId":"0","count":20,"includePromotedContent":true,"withCommunity":true,"withVoice":true}')
+            vars = json.loads('{"userId":"0","count":20,"includePromotedContent":true,"withQuickPromoteEligibilityTweetFields":true,"withVoice":true}')
             vars['userId'] = str(userId)
             vars['includePromotedContent'] = False # idk if this works
-            reqHeaders = getAuthHeaders(bearer,authToken=authToken)
+            reqHeaders = getAuthHeaders(v2bearer,guestToken=getGuestToken())
-            endpoint=f"/i/api/graphql/{tweetFeedGraphql_api}/UserTweetsAndReplies"
+            endpoint=f"/i/api/graphql/{tweetFeedGraphql_api}/UserTweets"
             reqHeaders["x-client-transaction-id"] = twUtils.generate_transaction_id("GET",endpoint)
             feed = requests.get(f"https://{twitterUrl}{endpoint}", {'variables':json.dumps(vars),'features':tweetFeedGraphqlFeatures,'fieldToggles':'{"withArticlePlainText":false}'},headers=reqHeaders)
             if feed.status_code == 403:
@@ -553,6 +554,7 @@ def extractUserFeedFromId(userId,workaroundTokens):
                             tweets.append(tweet)
             return tweets
         except Exception as e:
+            print(f"Exception in extractUserFeedFromId: {str(e)}")
             continue
     raise TwExtractError(400, "Extract error")
 

twitfix.py

@@ -517,4 +517,4 @@ def oEmbedGen(description, user, video_link, ttype,providerName=None):
 
 if __name__ == "__main__":
     app.config['SERVER_NAME']='localhost:80'
-    app.run(host='0.0.0.0')
+    app.run(host='0.0.0.0', port=8080)

twitfix.service

@@ -3,11 +3,11 @@ Description=Init file for twitfix uwsgi instance
 After=network.target
 
 [Service]
-User=dylan
+User=carbonara
-Group=dylan
+Group=carbonara
-WorkingDirectory=/home/dylan/BetterTwitFix
+WorkingDirectory=/home/carbonara/twitter/BetterTwitFix
-Environment="PATH=/home/dylan/BetterTwitFix/venv/bin"
+Environment="PATH=/home/carbonara/twitter/BetterTwitFix/venv/bin"
-ExecStart=/home/dylan/BetterTwitFix/venv/bin/uwsgi --ini twitfix.ini
+ExecStart=/home/carbonara/twitter/BetterTwitFix/venv/bin/uwsgi --ini twitfix.ini
 Restart=always
 RestartSec=3
 

vxApi.py

@@ -230,6 +230,14 @@ def getApiResponse(tweet,include_txt=False,include_rtf=False):
     if 'in_reply_to_status_id_str' in tweetL and tweetL['in_reply_to_status_id_str'] != None:
         replyingToID = tweetL['in_reply_to_status_id_str']
 
+    if 'screen_name' not in userL and 'core' in tweet["core"]["user_results"]["result"]:
+        userL['screen_name'] = tweet["core"]["user_results"]["result"]["core"]["screen_name"]
+        if 'name' not in userL:
+            userL['name'] = tweet["core"]["user_results"]["result"]["core"]["name"]
+    
+    if 'profile_image_url_https' not in userL and 'avatar' in tweet["core"]["user_results"]["result"]:
+        userL['profile_image_url_https'] = tweet["core"]["user_results"]["result"]["avatar"]["image_url"]
+
     apiObject = {
         "text": twText,
         "likes": tweetL["favorite_count"],