From 7c25a2d364ee8b315d70862f8b677789944d757d Mon Sep 17 00:00:00 2001 From: diamante0018 Date: Thu, 5 Sep 2024 13:47:20 +0200 Subject: [PATCH] fix: resume progress because my friend has too many favorites and got api rate limited --- scraper.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/scraper.py b/scraper.py index be36d4f..84e8d1e 100644 --- a/scraper.py +++ b/scraper.py @@ -135,19 +135,24 @@ def get_torrents(): print("Please edit set.yaml") exit() + output_dir = "torrents" + os.makedirs(output_dir, exist_ok=True) # Create the directory if it doesn't exist + for url in url_list: torrent_url = url - # Call wtfcloudflare to download the torrent file - response = wtfcloudflare_t(torrent_url, useragent=useragent, cookie=cookie) - + torrent_id = url.split('/')[4] # The ID is in the 4th segment of the URL + torrent_path = os.path.join(output_dir, f"{torrent_id}.torrent") + # Extract the ID from the URL for naming the file torrent_id = url.split('/')[4] # The ID is in the 4th segment of the URL - - # Define the output directory and file name - output_dir = "torrents" - os.makedirs(output_dir, exist_ok=True) # Create the directory if it doesn't exist - torrent_path = os.path.join(output_dir, f"{torrent_id}.torrent") + + # Skip downloading if the torrent file already exists + if os.path.exists(torrent_path): + print(f"Torrent file already exists: {torrent_path}") + continue + + response = wtfcloudflare_t(torrent_url, useragent=useragent, cookie=cookie) # Save the torrent file to disk with open(torrent_path, 'wb') as torrent_file: