Refactor more; fix Activity url for videos

activity.py

@@ -0,0 +1,75 @@
+import datetime
+import msgs
+from utils import determineEmbedTweet, determineMediaToEmbed
+from copy import deepcopy
+
+def tweetDataToActivity(tweetData):
+    content=""
+
+    if tweetData['replyingTo'] is not None:
+        content += f"<blockquote>↪️ <i>Replying to @{tweetData['replyingTo']}</i></blockquote>"
+    content+=f"<p>{tweetData['text']}</p>"
+
+    attachments=[]
+    if tweetData['qrt'] is not None:
+        content += f"<blockquote><b>QRT: <a href=\"{tweetData['qrtURL']}\">{tweetData['qrt']['user_screen_name']}</a></b><br>{tweetData['qrt']['text']}</blockquote>"
+    if tweetData['pollData'] is not None:
+        content += f"<p>{msgs.genPollDisplay(tweetData['pollData'])}</p>"
+        content += "</p>"
+    content = content.replace("\n","<br>")
+    #if media is not None:
+    #    attachments.append({"type":mediatype,"url":media})
+    likes = tweetData['likes']
+    retweets = tweetData['retweets']
+
+    # convert date epoch to iso format
+    date = tweetData['date_epoch']
+    date = datetime.datetime.fromtimestamp(date).isoformat() + "Z"
+
+    embedTweetData = determineEmbedTweet(tweetData)
+    embeddingMedia = embedTweetData['hasMedia']
+    media = None
+    if embeddingMedia:
+        media = determineMediaToEmbed(embedTweetData,-1)
+
+    if media is not None:
+        media = deepcopy(media)
+        if media['type'] == "gif":
+            media['type'] = "gifv"
+        attachments.append({
+            "id": "114163769487684704",
+            "type": media['type'],
+            "url": media['url'],
+            "preview_url": media['thumbnail_url'],
+        })
+
+    # https://docs.joinmastodon.org/methods/statuses/
+    return {
+	"id": tweetData['tweetID'],
+	"url": f"https://x.com/{tweetData['user_screen_name']}/status/{tweetData['tweetID']}",
+	"uri": f"https://x.com/{tweetData['user_screen_name']}/status/{tweetData['tweetID']}",
+	"created_at": date,
+	"edited_at": None,
+	"reblog": None,
+	"in_reply_to_account_id": None,
+	"language": "en",
+	"content": content,
+	"spoiler_text": "",
+	"visibility": "public",
+	"application": {
+		"website": None
+	},
+	"media_attachments": attachments,
+	"account": {
+		"display_name": tweetData['user_name'],
+		"username": tweetData['user_screen_name'],
+		"acct": tweetData['user_screen_name'],
+		"url": f"https://x.com/{tweetData['user_screen_name']}/status/{tweetData['tweetID']}",
+		"uri": f"https://x.com/{tweetData['user_screen_name']}/status/{tweetData['tweetID']}",
+		"locked": False,
+		"avatar": tweetData['user_profile_image_url'],
+		"avatar_static": tweetData['user_profile_image_url'],
+		"hide_collections": False,
+		"noindex": False,
+	},
+}

twitfix.py

@@ -1,5 +1,4 @@
-from weakref import finalize
+from flask import Flask, render_template, request, redirect, abort, Response, send_from_directory, send_file
-from flask import Flask, render_template, request, redirect, abort, Response, send_from_directory, url_for, send_file, make_response, jsonify
 
 from configHandler import config
 remoteCombine='combination_method' in config['config'] and config['config']['combination_method'] != "local"
@@ -15,16 +14,16 @@ import msgs
 import twExtract as twExtract
 from cache import addVnfToLinkCache,getVnfFromLinkCache
 import vxlogging as log
-from utils import getTweetIdFromUrl, pathregex
+from utils import getTweetIdFromUrl, pathregex, determineMediaToEmbed, determineEmbedTweet, BytesIOWrapper, fixMedia
 from vxApi import getApiResponse, getApiUserResponse
 from urllib.parse import urlparse 
 from PyRTF.Elements import Document
 from PyRTF.document.section import Section
 from PyRTF.document.paragraph import Paragraph
-from utils import BytesIOWrapper
 from copy import deepcopy
 import json
 import datetime
+import activity as activitymg
 app = Flask(__name__)
 CORS(app)
 user_agent=""
@@ -67,13 +66,6 @@ def isValidUserAgent(user_agent):
         return True
     return False
 
-def fixMedia(mediaInfo):
-    # This is for the iOS Discord app, which has issues when serving URLs ending in .mp4 (https://github.com/dylanpdx/BetterTwitFix/issues/210)
-    if 'video.twimg.com' not in mediaInfo['url'] or 'convert?url=' in mediaInfo['url']:
-        return mediaInfo
-    mediaInfo['url'] = mediaInfo['url'].replace("https://video.twimg.com",f"{config['config']['url']}/tvid").replace(".mp4","")
-    return mediaInfo
-
 def message(text):
     return render_template(
         'default.html', 
@@ -109,7 +101,7 @@ def generateActivityLink(tweetData,media=None,mediatype=None):
 
         attributedTo = f"{config['config']['url']}/user.json?name={urllib.parse.quote(tweetData['user_name'])}&screen_name={urllib.parse.quote(tweetData['user_screen_name'])}&pfp={urllib.parse.quote(tweetData['user_profile_image_url'])}"
 
-        return f"{config['config']['url']}/activity.json?id={tweetData['tweetID']}&content={urllib.parse.quote(content)}&attachments={urllib.parse.quote(json.dumps(attachments))}&likes={likes}&retweets={retweets}&published={urllib.parse.quote(date)}&user={urllib.parse.quote(attributedTo)}"
+        return f"{config['config']['url']}/users/{tweetData['user_screen_name']}/statuses/{tweetData['tweetID']}"
     except Exception as e:
         log.error("Error generating activity link: "+str(e))
         return None
@@ -147,8 +139,8 @@ def renderVideoTweetEmbed(tweetData,mediaInfo,appnameSuffix=""):
     mediaInfo=fixMedia(mediaInfo)
 
     appName = config['config']['appname']+appnameSuffix
-    #if 'Discord' not in user_agent:
+    if 'Discord' not in user_agent:
-    appName = msgs.formatProvider(config['config']['appname']+appnameSuffix,tweetData)
+        appName = msgs.formatProvider(config['config']['appname']+appnameSuffix,tweetData)
 
     return render_template("video.html",
                     tweet=tweetData,
@@ -160,7 +152,7 @@ def renderVideoTweetEmbed(tweetData,mediaInfo,appnameSuffix=""):
                     appname=appName,
                     color=config['config']['color'],
                     sicon="video",
-                    #activityLink=generateActivityLink(tweetData,mediaInfo['url'],"video/mp4") # this is broken on Discord's end
+                    activityLink=generateActivityLink(tweetData,mediaInfo['url'],"video/mp4") # this is broken on Discord's end
                     )
 
 def renderTextTweetEmbed(tweetData,appnameSuffix=""):
@@ -240,7 +232,8 @@ def activity():
         attachmentsRaw.append({
             "type": "Document",
             "mediaType": attachment["type"],
-            "url": attachment["url"]
+            "url": attachment["url"],
+            "preview_url": "https://pbs.twimg.com/ext_tw_video_thumb/1906073839441735680/pu/img/2xqg6tlK9mK0mSOR.jpg",
     })
 
     return {
@@ -271,7 +264,7 @@ def userJson():
     pfp = request.args.get("pfp", None)
 
     return {
-        "id": "https://x.com/"+screen_name,
+        "id": screen_name,
         "type": "Person",
         "preferredUsername": screen_name,
         "name": name,
@@ -322,19 +315,6 @@ def getUserData(twitter_url):
     userData = getApiUserResponse(rawUserData)
     return userData
 
-def determineEmbedTweet(tweetData):
-    # Determine which tweet, i.e main or QRT, to embed the media from.
-    # if there is no QRT, return the main tweet => default behavior
-    # if both don't have media, return the main tweet => embedding qrt text will be handled in the embed description
-    # if both have media, return the main tweet => priority is given to the main tweet's media
-    # if only the QRT has media, return the QRT => show the QRT's media, not the main tweet's
-    # if only the main tweet has media, return the main tweet => show the main tweet's media, embedding QRT text will be handled in the embed description
-    if tweetData['qrt'] is None:
-        return tweetData
-    if tweetData['qrt']['hasMedia'] and not tweetData['hasMedia']:
-        return tweetData['qrt']
-    return tweetData
-
 @app.route('/<path:sub_path>') # Default endpoint used by everything
 def twitfix(sub_path):
     global user_agent
@@ -425,47 +405,35 @@ def twitfix(sub_path):
     if isApiRequest: # Directly return the API response if the request is from the API
         return tweetData
     elif directEmbed: # direct embed
+        embeddingMedia = tweetData['hasMedia']
+        renderMedia = None
+        if embeddingMedia:
+            renderMedia = determineMediaToEmbed(tweetData,embedIndex)
         # direct embeds should always prioritize the main tweet, so don't check for qrt
         # determine what type of media we're dealing with
-        if not tweetData['hasMedia'] and qrt is None:
+        if not embeddingMedia and qrt is None:
             return renderTextTweetEmbed(tweetData)
-        elif tweetData['allSameType'] and tweetData['media_extended'][0]['type'] == "image" and embedIndex == -1 and tweetData['combinedMediaUrl'] != None:
-            return render_template("rawimage.html",media={"url":tweetData['combinedMediaUrl']})
         else:
-            # this means we have mixed media or video, and we're only going to embed one
+            if renderMedia['type'] == "image":
-            if embedIndex == -1: # if the user didn't specify an index, we'll just use the first one
+                return render_template("rawimage.html",media=renderMedia)
-                embedIndex = 0
+            elif renderMedia['type'] == "video" or renderMedia['type'] == "gif":
-            media = tweetData['media_extended'][embedIndex]
+                return render_template("rawvideo.html",media=renderMedia)
-            media=fixMedia(media)
-            if media['type'] == "image":
-                return render_template("rawimage.html",media=media)
-            elif media['type'] == "video" or media['type'] == "gif":
-                return render_template("rawvideo.html",media=media)
     else: # full embed
         embedTweetData = determineEmbedTweet(tweetData)
+        embeddingMedia = embedTweetData['hasMedia']
+        
         if "article" in embedTweetData and embedTweetData["article"] is not None:
             return renderArticleTweetEmbed(tweetData," • See original tweet for full article")
-        elif not embedTweetData['hasMedia']:
+        elif not embeddingMedia:
             return renderTextTweetEmbed(tweetData)
-        elif embedTweetData['allSameType'] and embedTweetData['media_extended'][0]['type'] == "image" and embedIndex == -1 and embedTweetData['combinedMediaUrl'] != None:
-            return renderImageTweetEmbed(tweetData,embedTweetData['combinedMediaUrl'],appnameSuffix=" • See original tweet for full quality")
         else:
-            # this means we have mixed media or video, and we're only going to embed one
+            media = determineMediaToEmbed(embedTweetData,embedIndex)
-            if embedIndex == -1: # if the user didn't specify an index, we'll just use the first one
+            suffix=""
-                embedIndex = 0
+            if "suffix" in media:
-            media = embedTweetData['media_extended'][embedIndex]
+                suffix = media["suffix"]
-            if len(embedTweetData["media_extended"]) > 1:
-                suffix = f' • Media {embedIndex+1}/{len(embedTweetData["media_extended"])}'
-            else:
-                suffix = ''
             if media['type'] == "image":
                 return renderImageTweetEmbed(tweetData,media['url'] , appnameSuffix=suffix)
             elif media['type'] == "video" or media['type'] == "gif":
-                if media['type'] == "gif":
-                    if config['config']['gifConvertAPI'] != "" and config['config']['gifConvertAPI'] != "none":
-                        vurl=media['originalUrl'] if 'originalUrl' in media else media['url']
-                        media['url'] = config['config']['gifConvertAPI'] + "/convert?url=" + vurl
-                        suffix += " • GIF"
                 return renderVideoTweetEmbed(tweetData,media,appnameSuffix=suffix)
 
     return message(msgs.failedToScan)
@@ -502,6 +470,13 @@ def rendercombined():
     imgIo.seek(0)
     return send_file(imgIo, mimetype='image/jpeg',max_age=86400)
 
+@app.route("/api/v1/statuses/<int:tweet_id>")
+def api_v1_status(tweet_id):
+    tweetData = getTweetData(f"https://twitter.com/i/status/{tweet_id}")
+    if tweetData is None:
+        abort(500) # this should cause Discord to fall back to the default embed
+    return activitymg.tweetDataToActivity(tweetData)
+
 def oEmbedGen(description, user, video_link, ttype,providerName=None):
     if providerName == None:
         providerName = config['config']['appname']

utils.py

@@ -1,5 +1,6 @@
 import re
 import io
+from configHandler import config
 
 pathregex = re.compile("\\w{1,15}\\/(status|statuses)\\/(\\d{2,20})")
 endTCOregex = re.compile("(^.*?) +https:\/\/t.co\/.*?$")
@@ -40,4 +41,51 @@ class BytesIOWrapper(io.BufferedReader):
         return self._encoding_call('read1', size)
 
     def peek(self, size=-1):
-        return self._encoding_call('peek', size)
+        return self._encoding_call('peek', size)
+    
+def fixMedia(mediaInfo):
+    # This is for the iOS Discord app, which has issues when serving URLs ending in .mp4 (https://github.com/dylanpdx/BetterTwitFix/issues/210)
+    if 'video.twimg.com' not in mediaInfo['url'] or 'convert?url=' in mediaInfo['url'] or 'originalUrl' in mediaInfo:
+        return mediaInfo
+    mediaInfo["originalUrl"] = mediaInfo['url']
+    mediaInfo['url'] = mediaInfo['url'].replace("https://video.twimg.com",f"{config['config']['url']}/tvid").replace(".mp4","")
+    return mediaInfo
+
+def determineEmbedTweet(tweetData):
+    # Determine which tweet, i.e main or QRT, to embed the media from.
+    # if there is no QRT, return the main tweet => default behavior
+    # if both don't have media, return the main tweet => embedding qrt text will be handled in the embed description
+    # if both have media, return the main tweet => priority is given to the main tweet's media
+    # if only the QRT has media, return the QRT => show the QRT's media, not the main tweet's
+    # if only the main tweet has media, return the main tweet => show the main tweet's media, embedding QRT text will be handled in the embed description
+    if tweetData['qrt'] is None:
+        return tweetData
+    if tweetData['qrt']['hasMedia'] and not tweetData['hasMedia']:
+        return tweetData['qrt']
+    return tweetData
+
+def determineMediaToEmbed(tweetData,embedIndex = -1):
+    if tweetData['allSameType'] and tweetData['media_extended'][0]['type'] == "image" and embedIndex == -1 and tweetData['combinedMediaUrl'] != None:
+        return {"url":tweetData['combinedMediaUrl'],"type":"image"}
+    else:
+        # this means we have mixed media or video, and we're only going to embed one
+        if embedIndex == -1: # if the user didn't specify an index, we'll just use the first one
+            embedIndex = 0
+        media = tweetData['media_extended'][embedIndex]
+        media=fixMedia(media)
+        suffix=""
+        if len(tweetData["media_extended"]) > 1:
+            suffix = f' • Media {embedIndex+1}/{len(tweetData["media_extended"])}'
+        else:
+            suffix = ''
+        media["suffix"] = suffix
+        if media['type'] == "image":
+            return media
+        elif media['type'] == "video" or media['type'] == "gif":
+            if media['type'] == "gif":
+                if config['config']['gifConvertAPI'] != "" and config['config']['gifConvertAPI'] != "none":
+                    vurl=media['originalUrl'] if 'originalUrl' in media else media['url']
+                    media['url'] = config['config']['gifConvertAPI'] + "/convert?url=" + vurl
+                    suffix += " • GIF"
+                    media["suffix"] = suffix
+        return media