From 3840e9051897baa9437a62cceca1d85b3d824a86 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Feb 2023 02:55:23 +0000 Subject: [PATCH 01/12] Bump flask from 2.2.2 to 2.2.3 Bumps [flask](https://github.com/pallets/flask) from 2.2.2 to 2.2.3. - [Release notes](https://github.com/pallets/flask/releases) - [Changelog](https://github.com/pallets/flask/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/flask/compare/2.2.2...2.2.3) --- updated-dependencies: - dependency-name: flask dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 97306a6..0024fff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,6 @@ boto3==1.26.69 requests==2.28.2 Pillow==9.4.0 -Flask==2.2.2 +Flask==2.2.3 Flask-Cors==3.0.10 yt-dlp==2022.7.18 \ No newline at end of file From b841a11662237f8a147c205d3317cb662a18f129 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Feb 2023 02:48:32 +0000 Subject: [PATCH 02/12] Bump boto3 from 1.26.69 to 1.26.79 Bumps [boto3](https://github.com/boto/boto3) from 1.26.69 to 1.26.79. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.26.69...1.26.79) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 97306a6..6ae60e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ pymongo==4.3.3 -boto3==1.26.69 +boto3==1.26.79 requests==2.28.2 Pillow==9.4.0 Flask==2.2.2 From adc85e2e5171a9a539d7916d45a102d7db2058fa Mon Sep 17 00:00:00 2001 From: Dylan Date: Sun, 5 Mar 2023 21:23:27 +0000 Subject: [PATCH 03/12] Added function & tests for getting user profile (#65) --- test_vx.py | 7 +++++++ twExtract/__init__.py | 17 ++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/test_vx.py b/test_vx.py index 1a51140..61e09bd 100644 --- a/test_vx.py +++ b/test_vx.py @@ -7,6 +7,7 @@ import msgs from flask.testing import FlaskClient client = FlaskClient(twitfix.app) +testUser="https://twitter.com/jack" testTextTweet="https://twitter.com/jack/status/20" testVideoTweet="https://twitter.com/Twitter/status/1263145271946551300" testMediaTweet="https://twitter.com/Twitter/status/1118295916874739714" @@ -40,6 +41,12 @@ def test_textTweetExtract(): assert 'extended_entities' not in tweet assert tweet["is_quote_status"]==False +def test_textUserExtract(): + user = twExtract.extractUser(testUser) + assert user["screen_name"]=="jack" + assert user["id"]==12 + assert user["created_at"] == "Tue Mar 21 20:50:14 +0000 2006" + def test_videoTweetExtract(): tweet = twExtract.extractStatus(testVideoTweet) assert tweet["full_text"]==videoVNF_compare['description'] diff --git a/twExtract/__init__.py b/twExtract/__init__.py index bb3f940..a5d548e 100644 --- a/twExtract/__init__.py +++ b/twExtract/__init__.py @@ -7,7 +7,7 @@ from . import twExtractError guestToken=None pathregex = r"\w{1,15}\/(status|statuses)\/(\d{2,20})" - +userregex = r"^https?:\/\/(?:www\.)?twitter\.com\/(?:#!\/)?@?([^/?#]*)(?:[?#].*)?$" def getGuestToken(): global guestToken if guestToken is None: @@ -50,6 +50,21 @@ def extractStatus(url): except Exception as e: return extractStatus_fallback(url) +def extractUser(url): + m = re.search(userregex, url) + if m is None: + raise twExtractError.TwExtractError(400, "Invalid URL") + screen_name = m.group(1) + # get guest token + guestToken = getGuestToken() + # get user + user = requests.get(f"https://api.twitter.com/1.1/users/show.json?screen_name={screen_name}",headers={"Authorization":"Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw", "x-guest-token":guestToken}) + output = user.json() + if "errors" in output: + # pick the first error and create a twExtractError + error = output["errors"][0] + raise twExtractError.TwExtractError(error["code"], error["message"]) + return output def lambda_handler(event, context): if ("queryStringParameters" not in event): From b06f84449451299f2680be660ff0acc8decfecb4 Mon Sep 17 00:00:00 2001 From: Dylan Date: Sun, 5 Mar 2023 21:31:35 +0000 Subject: [PATCH 04/12] Support for odd user URLs --- test_vx.py | 10 +++++++++- twExtract/__init__.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/test_vx.py b/test_vx.py index 61e09bd..409b97f 100644 --- a/test_vx.py +++ b/test_vx.py @@ -8,6 +8,7 @@ from flask.testing import FlaskClient client = FlaskClient(twitfix.app) testUser="https://twitter.com/jack" +testUserWeirdURLs=["https://twitter.com/jack?lang=en","https://twitter.com/jack/with_replies","https://twitter.com/jack/media","https://twitter.com/jack/likes","https://twitter.com/jack/with_replies?lang=en","https://twitter.com/jack/media?lang=en","https://twitter.com/jack/likes?lang=en","https://twitter.com/jack/"] testTextTweet="https://twitter.com/jack/status/20" testVideoTweet="https://twitter.com/Twitter/status/1263145271946551300" testMediaTweet="https://twitter.com/Twitter/status/1118295916874739714" @@ -41,12 +42,19 @@ def test_textTweetExtract(): assert 'extended_entities' not in tweet assert tweet["is_quote_status"]==False -def test_textUserExtract(): +def test_UserExtract(): user = twExtract.extractUser(testUser) assert user["screen_name"]=="jack" assert user["id"]==12 assert user["created_at"] == "Tue Mar 21 20:50:14 +0000 2006" +def test_UserExtractWeirdURLs(): + for url in testUserWeirdURLs: + user = twExtract.extractUser(url) + assert user["screen_name"]=="jack" + assert user["id"]==12 + assert user["created_at"] == "Tue Mar 21 20:50:14 +0000 2006" + def test_videoTweetExtract(): tweet = twExtract.extractStatus(testVideoTweet) assert tweet["full_text"]==videoVNF_compare['description'] diff --git a/twExtract/__init__.py b/twExtract/__init__.py index a5d548e..645b075 100644 --- a/twExtract/__init__.py +++ b/twExtract/__init__.py @@ -7,7 +7,7 @@ from . import twExtractError guestToken=None pathregex = r"\w{1,15}\/(status|statuses)\/(\d{2,20})" -userregex = r"^https?:\/\/(?:www\.)?twitter\.com\/(?:#!\/)?@?([^/?#]*)(?:[?#].*)?$" +userregex = r"^https?:\/\/(?:www\.)?twitter\.com\/(?:#!\/)?@?([^/?#]*)(?:[?#/].*)?$" def getGuestToken(): global guestToken if guestToken is None: From 223d40eb9bc222c9e938b00a528927d965f7ea10 Mon Sep 17 00:00:00 2001 From: Dylan Date: Fri, 10 Mar 2023 18:32:51 +0000 Subject: [PATCH 05/12] Better image combination --- combineImg/__init__.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/combineImg/__init__.py b/combineImg/__init__.py index 2784b9b..e3ee318 100644 --- a/combineImg/__init__.py +++ b/combineImg/__init__.py @@ -30,9 +30,10 @@ def scaleImageIterable(args): targetWidth = args[1] targetHeight = args[2] pad=args[3] + image = image.convert('RGBA') + image = ImageOps.expand(image,20) if pad: - image = image.convert('RGBA') - newImg = ImageOps.pad(image, (targetWidth, targetHeight),color=(0, 0, 0, 0)) + newImg = ImageOps.contain(image, (targetWidth, targetHeight)) else: newImg = ImageOps.fit(image, (targetWidth, targetHeight)) # scale + crop return newImg @@ -91,11 +92,12 @@ def saveImage(image, name): def genImage(imageArray): totalSize=getTotalImgSize(imageArray) combined = combineImages(imageArray, *totalSize) - combinedBG = combineImages(imageArray, *totalSize,False) - combinedBG = blurImage(combinedBG,50) - finalImg = Image.alpha_composite(combinedBG,combined) - #finalImg = ImageOps.pad(finalImg, findImageWithMostPixels(imageArray).size,color=(0, 0, 0, 0)) - finalImg = finalImg.convert('RGB') + + finalImg = combined.convert('RGB') + + bbox = finalImg.getbbox() + finalImg = finalImg.crop(bbox) + return finalImg def downloadImage(url): From 4b5b1936579cc0c7c87d5dfe523de30277c56101 Mon Sep 17 00:00:00 2001 From: Dylan Date: Fri, 10 Mar 2023 18:50:42 +0000 Subject: [PATCH 06/12] Quick fix for small images --- combineImg/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/combineImg/__init__.py b/combineImg/__init__.py index e3ee318..82dfb2e 100644 --- a/combineImg/__init__.py +++ b/combineImg/__init__.py @@ -69,7 +69,8 @@ def combineImages(imageArray, totalWidth, totalHeight,pad=True): x += image.size[0] y += imageArray[0].size[1] x = 0 - newImage.paste(imageArray[2], (x, y)) + # paste the final image so that it's centered + newImage.paste(imageArray[2], (int((totalWidth - imageArray[2].size[0]) / 2), y)) elif (len(imageArray) == 4): # if there are four images, combine the first two horizontally, then combine the last two vertically for image in imageArray[0:2]: newImage.paste(image, (x, y)) @@ -92,7 +93,7 @@ def saveImage(image, name): def genImage(imageArray): totalSize=getTotalImgSize(imageArray) combined = combineImages(imageArray, *totalSize) - + finalImg = combined.convert('RGB') bbox = finalImg.getbbox() From a8ac35122cc237afcc069e2f580b8f0c0db108bb Mon Sep 17 00:00:00 2001 From: Dylan Date: Sun, 12 Mar 2023 16:00:09 +0000 Subject: [PATCH 07/12] Updated dockerfile to Python 3.10 --- Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9db5f69..457372c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,19 @@ -FROM python:3.6-alpine AS build +FROM python:3.10-alpine AS build RUN apk add build-base python3-dev linux-headers pcre-dev jpeg-dev zlib-dev RUN pip install --upgrade pip RUN pip install yt-dlp pillow uwsgi -FROM python:3.6-alpine AS deps +FROM python:3.10-alpine AS deps WORKDIR /twitfix COPY requirements.txt requirements.txt -COPY --from=build /usr/local/lib/python3.6/site-packages /usr/local/lib/python3.6/site-packages +COPY --from=build /usr/local/lib/python3.10/site-packages /usr/local/lib/python3.10/site-packages RUN pip install -r requirements.txt -FROM python:3.6-alpine AS runner +FROM python:3.10-alpine AS runner EXPOSE 9000 RUN apk add pcre-dev jpeg-dev zlib-dev WORKDIR /twitfix CMD ["uwsgi", "twitfix.ini"] COPY --from=build /usr/local/bin/uwsgi /usr/local/bin/uwsgi -COPY --from=deps /usr/local/lib/python3.6/site-packages /usr/local/lib/python3.6/site-packages +COPY --from=deps /usr/local/lib/python3.10/site-packages /usr/local/lib/python3.10/site-packages COPY . . From 1a4bfab98fc8516395ae429dfdf38021b796b399 Mon Sep 17 00:00:00 2001 From: Dylan Date: Sat, 25 Mar 2023 02:44:55 +0000 Subject: [PATCH 08/12] Extract user by ID --- test_vx.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test_vx.py b/test_vx.py index 409b97f..b6a3e0e 100644 --- a/test_vx.py +++ b/test_vx.py @@ -8,6 +8,7 @@ from flask.testing import FlaskClient client = FlaskClient(twitfix.app) testUser="https://twitter.com/jack" +testUserID = "https://twitter.com/i/user/12" testUserWeirdURLs=["https://twitter.com/jack?lang=en","https://twitter.com/jack/with_replies","https://twitter.com/jack/media","https://twitter.com/jack/likes","https://twitter.com/jack/with_replies?lang=en","https://twitter.com/jack/media?lang=en","https://twitter.com/jack/likes?lang=en","https://twitter.com/jack/"] testTextTweet="https://twitter.com/jack/status/20" testVideoTweet="https://twitter.com/Twitter/status/1263145271946551300" @@ -48,6 +49,12 @@ def test_UserExtract(): assert user["id"]==12 assert user["created_at"] == "Tue Mar 21 20:50:14 +0000 2006" +def test_UserExtractID(): + user = twExtract.extractUser(testUserID) + assert user["screen_name"]=="jack" + assert user["id"]==12 + assert user["created_at"] == "Tue Mar 21 20:50:14 +0000 2006" + def test_UserExtractWeirdURLs(): for url in testUserWeirdURLs: user = twExtract.extractUser(url) From b0bf814a88b43a89b9161cc40c1d7a2e526c9770 Mon Sep 17 00:00:00 2001 From: Dylan Date: Sat, 25 Mar 2023 02:54:07 +0000 Subject: [PATCH 09/12] twExtract changes for user getting --- twExtract/__init__.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/twExtract/__init__.py b/twExtract/__init__.py index 645b075..82c6227 100644 --- a/twExtract/__init__.py +++ b/twExtract/__init__.py @@ -4,14 +4,15 @@ import json import requests import re from . import twExtractError - +bearer="Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw" guestToken=None pathregex = r"\w{1,15}\/(status|statuses)\/(\d{2,20})" userregex = r"^https?:\/\/(?:www\.)?twitter\.com\/(?:#!\/)?@?([^/?#]*)(?:[?#/].*)?$" +userIDregex = r"\/i\/user\/(\d+)" def getGuestToken(): global guestToken if guestToken is None: - r = requests.post("https://api.twitter.com/1.1/guest/activate.json", headers={"Authorization":"Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"}) + r = requests.post("https://api.twitter.com/1.1/guest/activate.json", headers={"Authorization":bearer}) guestToken = json.loads(r.text)["guest_token"] return guestToken @@ -40,7 +41,7 @@ def extractStatus(url): # get guest token guestToken = getGuestToken() # get tweet - tweet = requests.get("https://api.twitter.com/1.1/statuses/show/" + twid + ".json?tweet_mode=extended&cards_platform=Web-12&include_cards=1&include_reply_count=1&include_user_entities=0", headers={"Authorization":"Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw", "x-guest-token":guestToken}) + tweet = requests.get("https://api.twitter.com/1.1/statuses/show/" + twid + ".json?tweet_mode=extended&cards_platform=Web-12&include_cards=1&include_reply_count=1&include_user_entities=0", headers={"Authorization":bearer, "x-guest-token":guestToken}) output = tweet.json() if "errors" in output: # pick the first error and create a twExtractError @@ -51,14 +52,22 @@ def extractStatus(url): return extractStatus_fallback(url) def extractUser(url): - m = re.search(userregex, url) + useId=True + m = re.search(userIDregex, url) if m is None: - raise twExtractError.TwExtractError(400, "Invalid URL") + m = re.search(userregex, url) + if m is None: + raise twExtractError.TwExtractError(400, "Invalid URL") + else: + useId=False screen_name = m.group(1) # get guest token guestToken = getGuestToken() # get user - user = requests.get(f"https://api.twitter.com/1.1/users/show.json?screen_name={screen_name}",headers={"Authorization":"Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw", "x-guest-token":guestToken}) + if not useId: + user = requests.get(f"https://api.twitter.com/1.1/users/show.json?screen_name={screen_name}",headers={"Authorization":bearer, "x-guest-token":guestToken}) + else: + user = requests.get(f"https://api.twitter.com/1.1/users/show.json?user_id={screen_name}",headers={"Authorization":bearer, "x-guest-token":guestToken}) output = user.json() if "errors" in output: # pick the first error and create a twExtractError @@ -66,6 +75,9 @@ def extractUser(url): raise twExtractError.TwExtractError(error["code"], error["message"]) return output +#def extractUserByID(id): + + def lambda_handler(event, context): if ("queryStringParameters" not in event): return { From 18674b4f3fd6a27e6a2d95d9dd5c8342f973a80e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Apr 2023 02:00:36 +0000 Subject: [PATCH 10/12] Bump boto3 from 1.26.79 to 1.26.104 Bumps [boto3](https://github.com/boto/boto3) from 1.26.79 to 1.26.104. - [Release notes](https://github.com/boto/boto3/releases) - [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst) - [Commits](https://github.com/boto/boto3/compare/1.26.79...1.26.104) --- updated-dependencies: - dependency-name: boto3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1d57ed1..3f58c01 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -pymongo==4.3.3 -boto3==1.26.79 +pymongo==4.3.3 +boto3==1.26.104 requests==2.28.2 Pillow==9.4.0 Flask==2.2.3 From 114099670c8e64dea1224da61d8b0aedd842ef5d Mon Sep 17 00:00:00 2001 From: Nekromateion <43814053+Nekromateion@users.noreply.github.com> Date: Mon, 3 Apr 2023 13:09:21 +0200 Subject: [PATCH 11/12] Fix typo in readme.md --- readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index a566fdf..5eb7e6b 100644 --- a/readme.md +++ b/readme.md @@ -39,7 +39,7 @@ vxTwitter generates a config.json in its root directory the first time you run i **link_cache** - (Options: **db**, **json**) -- **db**: Caches all links to a mongoDB database. This should be used it you are using uWSGI and are not just running the script on its own as one worker +- **db**: Caches all links to a mongoDB database. This should be used if you are using uWSGI and are not just running the script on its own as one worker - **json**: This saves cached links to a local **links.json** file - **dynamodb**: Saves cached links to a DynamoDB database - set `table` to the table name to cache links to. - **none**: Does not cache requests. Not reccomended as you can easily use up your Twitter API credits with this. Intended for use with another cache system (i.e NGINX uwsgi_cache) @@ -58,4 +58,4 @@ vxTwitter generates a config.json in its root directory the first time you run i We check for t.co links in non video tweets, and if one is found, we direct the discord useragent to embed that link directly, this means that twitter links containing youtube / vimeo links will automatically embed those as if you had just directly linked to that content -This project is licensed under the **Do What The Fuck You Want Public License** \ No newline at end of file +This project is licensed under the **Do What The Fuck You Want Public License** From 39490a49b98ba6e56a109958dab4cb953ede4fe8 Mon Sep 17 00:00:00 2001 From: Dylan Date: Sat, 8 Apr 2023 16:28:35 +0100 Subject: [PATCH 12/12] Add info message for #74 --- msgs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msgs.py b/msgs.py index b9cffdc..770df48 100644 --- a/msgs.py +++ b/msgs.py @@ -1,6 +1,6 @@ failedToScan="Failed to scan your link! This may be due to an incorrect link, private/suspended account, deleted tweet, or Twitter itself might be having issues (Check here: https://api.twitterstat.us/)" failedToScanExtra = "\n\nTwitter gave me this error: " -tweetNotFound="Tweet not found." +tweetNotFound="Tweet not found. Note that this may be a result of Twitter blocking some tweets from being viewed as of April 8 2023." tweetSuspended="This Tweet is from a suspended account." tweetDescLimit=340