diff --git a/src/byuser.py b/src/byuser.py index 215ee62..16dc5a1 100644 --- a/src/byuser.py +++ b/src/byuser.py @@ -25,20 +25,14 @@ def proxitok_scraper(username: str) -> list[str]: session = requests.Session() direct_links = [] next_href = "" - rate_limit = 0 while True: url = f"{OPTIONS['proxitok_instance']}/@{username}{next_href}" response = session.get(url) log(f"Scraping {url}") - if response.status_code == 429 or response.status_code == 403: - # may want to adjust this ratio - rate_limit += 1 - sleep_time = 30 * rate_limit - print(f"{response.status_code} {response.reason} sleeping for {sleep_time}") - log(f"\n{response.status_code} {response.reason} sleeping for {sleep_time}") - time.sleep(sleep_time) - continue + if OPTIONS["ratelimit"] != 0: + log(f'Sleeping for {OPTIONS["ratelimit"]}s') + time.sleep(OPTIONS["ratelimit"]) if not response.ok: error_msg = f"{response.status_code} {response.reason} getting {url}" diff --git a/src/constants.py b/src/constants.py index 6c53f18..8c9419a 100644 --- a/src/constants.py +++ b/src/constants.py @@ -1,16 +1,20 @@ # Do not change anything here APP = { "name": "CLI TikTok", - "version": 0.85 + "version": 0.86 } # Here are the app settings. You are free to configure this field OPTIONS = { # This controls the Proxitok instance that will be scraped to obtain the URLs. - "proxitok_instance": "https://proxitok.pabloferreiro.es", + "proxitok_instance": "https://proxitok.esmailelbob.xyz", # This handles the command used to playback videos. It's heavily recommended to use MPV # Make sure your player can launch through the CLI and exits after playback - "player_command": "mpv" + "player_command": "mpv", + + # This is the wait time in seconds. It helps Tiktok and Proxitok don't ratelimit the user + # It applies to the scraping and downloading + "ratelimit": 5 } \ No newline at end of file diff --git a/src/downloader.py b/src/downloader.py index 2995ec4..6224d43 100644 --- a/src/downloader.py +++ b/src/downloader.py @@ -1,11 +1,12 @@ import os +import time from yt_dlp import YoutubeDL from yt_dlp.utils import DownloadError from log import logtofile as log from src.functions import url_redirection - +from src.constants import OPTIONS def downloader(url): ydl_opts = { @@ -38,6 +39,10 @@ def downloadtiktoks(urls): randomvideo = index = index + 1 url = url_redirection(urls[randomvideo]) + if OPTIONS["ratelimit"] != 0: + log(f'Sleeping for {OPTIONS["ratelimit"]}s') + time.sleep(OPTIONS["ratelimit"]) + try: downloader(url) log(f"Video {url} was downloaded") diff --git a/src/trending.py b/src/trending.py index 6003f81..9205eda 100644 --- a/src/trending.py +++ b/src/trending.py @@ -4,7 +4,7 @@ import requests from bs4 import BeautifulSoup from log import logtofile as log - +from src.constants import OPTIONS def streamtrending(amount:int = 24): links = proxitok_trending(amount) @@ -28,7 +28,6 @@ def proxitok_trending(amount: int = 24) -> list[str]: session = requests.Session() direct_links = [] next_href = "" - rate_limit = 0 while True: # The "next" page url is always the same but loads different trending videos each time url = f"{OPTIONS['proxitok_instance']}/trending{next_href}" @@ -36,14 +35,9 @@ def proxitok_trending(amount: int = 24) -> list[str]: response = session.get(url) log(f"Scraping {url}") - if response.status_code == 429 or response.status_code == 403: - # may want to adjust this ratio - rate_limit += 1 - sleep_time = 30 * rate_limit - print(f"{response.status_code} {response.reason} sleeping for {sleep_time}") - log(f"\n{response.status_code} {response.reason} sleeping for {sleep_time}") - time.sleep(sleep_time) - continue + if OPTIONS["ratelimit"] != 0: + log(f'Sleeping for {OPTIONS["ratelimit"]}s') + time.sleep(OPTIONS["ratelimit"]) if not response.ok: error_msg = f"{response.status_code} {response.reason} getting {url}"