From 85c68547d6f08525857722754df56e7170a5aae0 Mon Sep 17 00:00:00 2001 From: Nikhil Badyal <59223300+nikhilbadyal@users.noreply.github.com> Date: Mon, 21 Aug 2023 13:30:36 +0530 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=A8=20Moved=20apk=20sources=20out=20of?= =?UTF-8?q?=20config.=20(#294)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build-artifact.yml | 2 +- src/config.py | 57 +++------------------------- src/downloader/apkmirror.py | 28 ++++++++------ src/downloader/apkpure.py | 5 ++- src/downloader/apksos.py | 3 +- src/downloader/factory.py | 19 +++++++--- src/downloader/sources.py | 47 +++++++++++++++++++++++ src/downloader/uptodown.py | 9 ++--- 8 files changed, 93 insertions(+), 77 deletions(-) create mode 100644 src/downloader/sources.py diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index 6bc056d..49c11d0 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -34,7 +34,7 @@ jobs: build-apk: name: APK Build runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 60 steps: - name: Check out Git repository diff --git a/src/config.py b/src/config.py index fbd31f8..b85d103 100644 --- a/src/config.py +++ b/src/config.py @@ -5,6 +5,8 @@ from typing import List from environs import Env from requests import Session +from src.downloader.sources import apk_sources + default_cli = "https://github.com/revanced/revanced-cli/releases/latest" default_patches = "https://github.com/revanced/revanced-patches/releases/latest" default_patches_json = default_patches @@ -12,8 +14,6 @@ default_integrations = ( "https://github.com/revanced/revanced-integrations/releases/latest" ) -APK_MIRROR_BASE_URL = "https://www.apkmirror.com" - class RevancedConfig(object): """Revanced Configurations.""" @@ -25,57 +25,12 @@ class RevancedConfig(object): self.temp_folder = Path("apks") self.session = Session() self.session.headers["User-Agent"] = "anything" - self.apk_mirror = APK_MIRROR_BASE_URL - self.upto_down = { - "spotify": "spotify", - "nyx-music-player": "nyx-music-player", - "my-expenses": "my-expenses", - "backdrops": "backdrops", - "twitch": "twitch", - "irplus": "irplus", - "meme-generator-free": "meme-generator-free", - "yuka": "yuka", - "grecorder": "opnemer", - } - self.apk_pure = ["hex-editor", "androidtwelvewidgets"] - self.apk_sos = ["expensemanager", "candyvpn"] self.ci_test = env.bool("CI_TEST", False) - self.apps = env.list("PATCH_APPS", default_build) + self.apps = env.list( + "PATCH_APPS", + list(apk_sources.keys()) if self.ci_test else default_build, + ) self.rip_libs_apps: List[str] = [] - self.apk_mirror_urls = { - "reddit": f"{self.apk_mirror}/apk/redditinc/reddit/", - "twitter": f"{self.apk_mirror}/apk/x-corp/twitter/", - "tiktok": f"{self.apk_mirror}/apk/tiktok-pte-ltd/tik-tok-including-musical-ly/", - "warnwetter": f"{self.apk_mirror}/apk/deutscher-wetterdienst/warnwetter/", - "youtube": f"{self.apk_mirror}/apk/google-inc/youtube/", - "youtube_music": f"{self.apk_mirror}/apk/google-inc/youtube-music/", - "ticktick": f"{self.apk_mirror}/apk/appest-inc/ticktick-to-do-list-with-reminder-day-planner/", - "icon_pack_studio": f"{self.apk_mirror}/apk/smart-launcher-team/icon-pack-studio/", - "twitch": f"{self.apk_mirror}/apk/twitch-interactive-inc/twitch/", - "windy": f"{self.apk_mirror}/apk/windy-weather-world-inc/windy-wind-weather-forecast/", - "tasker": f"{self.apk_mirror}/apk/joaomgcd/tasker-crafty-apps-eu/", - "vsco": f"{self.apk_mirror}/apk/vsco/vsco-cam/", - "nova_launcher": f"{self.apk_mirror}/apk/teslacoil-software/nova-launcher/", - "netguard": f"{self.apk_mirror}/apk/marcel-bokhorst/netguard-no-root-firewall/", - "instagram": f"{self.apk_mirror}/apk/instagram/instagram-instagram/", - "inshorts": f"{self.apk_mirror}/apk/inshorts-formerly-news-in-shorts/", - "messenger": f"{self.apk_mirror}/apk/facebook-2/messenger/", - "trakt": f"{self.apk_mirror}/apk/trakt/trakt/", - "candyvpn": f"{self.apk_mirror}/apk/liondev-io/candylink-vpn/", - "sonyheadphone": f"{self.apk_mirror}/apk/sony-corporation/sony-headphones-connect/", - "relay": f"{self.apk_mirror}/apk/dbrady/relay-for-reddit-2/", - "boost": f"{self.apk_mirror}/apk/ruben-mayayo/boost-for-reddit/", - "rif": f"{self.apk_mirror}/apk/talklittle/reddit-is-fun/", - "sync": f"{self.apk_mirror}/apk/red-apps-ltd/sync-for-reddit/", - "infinity": f"{self.apk_mirror}/apk/docile-alligator/infinity-for-reddit/", - "slide": f"{self.apk_mirror}/apk/haptic-apps/slide-for-reddit/", - "bacon": f"{self.apk_mirror}/apk/onelouder-apps/baconreader-for-reddit/", - "pixiv": f"{self.apk_mirror}/apk/pixiv-inc/pixiv/", - } - self.apk_mirror_version_urls = { - key: value + value.split("/")[-2] - for key, value in self.apk_mirror_urls.items() - } self.existing_downloaded_apks = env.list("EXISTING_DOWNLOADED_APKS", []) self.personal_access_token = env.str("PERSONAL_ACCESS_TOKEN", None) self.dry_run = env.bool("DRY_RUN", False) diff --git a/src/downloader/apkmirror.py b/src/downloader/apkmirror.py index 97b2947..cb8daa1 100644 --- a/src/downloader/apkmirror.py +++ b/src/downloader/apkmirror.py @@ -7,6 +7,7 @@ from loguru import logger from scripts.status_check import headers from src.downloader.download import Downloader +from src.downloader.sources import APK_MIRROR_BASE_URL, apk_sources from src.exceptions import APKMirrorAPKDownloadFailure from src.utils import bs4_parser @@ -23,7 +24,7 @@ class ApkMirror(Downloader): "href" ): return self._download( - self.config.apk_mirror + possible_link["href"], f"{app}.apk" + APK_MIRROR_BASE_URL + possible_link["href"], f"{app}.apk" ) raise APKMirrorAPKDownloadFailure( f"Unable to extract force download for {app}", url=link @@ -48,7 +49,7 @@ class ApkMirror(Downloader): None, ): self._extract_force_download_link( - self.config.apk_mirror + final_download_link, app + APK_MIRROR_BASE_URL + final_download_link, app ) else: raise APKMirrorAPKDownloadFailure( @@ -68,7 +69,9 @@ class ApkMirror(Downloader): if row.find(class_="accent_color"): apk_type = row.find(class_="apkm-badge").get_text() if apk_type == "APK" and ( - "arm64-v8a" in row.text.strip() or "universal" in row.text.strip() + "arm64-v8a" in row.text.strip() + or "universal" in row.text.strip() + or "noarch" in row.text.strip() ): sub_url = row.find(class_="accent_color")["href"] break @@ -76,12 +79,12 @@ class ApkMirror(Downloader): raise APKMirrorAPKDownloadFailure( "Unable to extract download page", url=main_page ) - return f"{self.config.apk_mirror}{sub_url}" + return f"{APK_MIRROR_BASE_URL}{sub_url}" @staticmethod def _extracted_search_div(url: str, search_class: str) -> Tag: """Extract search div.""" - r = requests.get(url, headers=headers) + r = requests.get(url, headers=headers, timeout=10) if r.status_code != 200: raise APKMirrorAPKDownloadFailure( f"Unable to connect with {url} on ApkMirror. Are you blocked by APKMirror or abused apkmirror " @@ -101,9 +104,9 @@ class ApkMirror(Downloader): """ if not main_page: version = version.replace(".", "-") - main_page = ( - f"{self.config.apk_mirror_version_urls.get(app)}-{version}-release/" - ) + apk_main_page = apk_sources[app] + version_page = apk_main_page + apk_main_page.split("/")[-2] + main_page = f"{version_page}-{version}-release/" download_page = self.get_download_page(main_page) self.extract_download_link(download_page, app) @@ -115,14 +118,17 @@ class ApkMirror(Downloader): :return: Version of downloaded apk """ - app_main_page = self.config.apk_mirror_urls[app] + app_main_page = apk_sources[app] versions_div = self._extracted_search_div( app_main_page, "listWidget p-relative" ) app_rows = versions_div.find_all(class_="appRow") version_urls = [ - app_row.find(class_="downloadLink")["href"] for app_row in app_rows + app_row.find(class_="downloadLink")["href"] + for app_row in app_rows + if "beta" not in app_row.find(class_="appRowTitle").get_text().lower() + and "alpha" not in app_row.find(class_="appRowTitle").get_text().lower() ] return self.specific_version( - app, "latest", self.config.apk_mirror + max(version_urls) + app, "latest", APK_MIRROR_BASE_URL + max(version_urls) ) diff --git a/src/downloader/apkpure.py b/src/downloader/apkpure.py index b771613..50c221f 100644 --- a/src/downloader/apkpure.py +++ b/src/downloader/apkpure.py @@ -4,6 +4,7 @@ from typing import Any from loguru import logger from src.downloader.download import Downloader +from src.downloader.sources import apk_sources from src.patches import Patches @@ -18,6 +19,6 @@ class ApkPure(Downloader): :return: Version of downloaded apk """ package_name = Patches.get_package_name(app) - download_url = f"https://d.apkpure.com/b/APK/{package_name}?version=latest" + download_url = apk_sources[app].format(package_name) self._download(download_url, f"{app}.apk") - logger.debug(f"Downloaded {app} apk from apk_pure_downloader in rt") + logger.debug(f"Downloaded {app} apk from apk_pure_downloader") diff --git a/src/downloader/apksos.py b/src/downloader/apksos.py index 465b463..177ae27 100644 --- a/src/downloader/apksos.py +++ b/src/downloader/apksos.py @@ -6,6 +6,7 @@ from bs4 import BeautifulSoup from scripts.status_check import headers from src.downloader.download import Downloader +from src.downloader.sources import apk_sources from src.exceptions import APKSosAPKDownloadFailure from src.utils import bs4_parser @@ -36,5 +37,5 @@ class ApkSos(Downloader): :return: Version of downloaded apk """ package_name = self.patcher.get_package_name(app) - download_url = f"https://apksos.com/download-app/{package_name}" + download_url = apk_sources[app].format(package_name) self.extract_download_link(download_url, app) diff --git a/src/downloader/factory.py b/src/downloader/factory.py index 7b236e8..ba9d4ef 100644 --- a/src/downloader/factory.py +++ b/src/downloader/factory.py @@ -5,7 +5,15 @@ from src.downloader.apkpure import ApkPure from src.downloader.apksos import ApkSos from src.downloader.download import Downloader from src.downloader.github import Github +from src.downloader.sources import ( + APK_MIRROR_BASE_URL, + APK_PURE_URL, + APK_SOS_URL, + GITHUB_BASE_URL, + apk_sources, +) from src.downloader.uptodown import UptoDown +from src.exceptions import DownloadFailure from src.patches import Patches @@ -24,13 +32,14 @@ class DownloaderFactory(object): patcher : Patcher config : Config """ - if app in {"patches", "microg"}: + if apk_sources[app].startswith(GITHUB_BASE_URL): return Github(patcher, config) - if app in config.apk_pure: + if apk_sources[app].startswith(APK_PURE_URL): return ApkPure(patcher, config) - elif app in config.apk_sos: + elif apk_sources[app].startswith(APK_SOS_URL): return ApkSos(patcher, config) - elif app in config.upto_down: + elif apk_sources[app].endswith("en.uptodown.com/android"): return UptoDown(patcher, config) - else: + elif apk_sources[app].startswith(APK_MIRROR_BASE_URL): return ApkMirror(patcher, config) + raise DownloadFailure(f"No download factory found for {app}") diff --git a/src/downloader/sources.py b/src/downloader/sources.py new file mode 100644 index 0000000..9812fd1 --- /dev/null +++ b/src/downloader/sources.py @@ -0,0 +1,47 @@ +APK_MIRROR_BASE_URL = "https://www.apkmirror.com" +APK_MIRROR_BASE_APK_URL = f"{APK_MIRROR_BASE_URL}/apk" +UPTODOWN_BASE_URL = "https://{}.en.uptodown.com/android" +APK_PURE_URL = "https://d.apkpure.com/b/APK/{}?version=latest" +APK_SOS_URL = "https://apksos.com/download-app/{}" +GITHUB_BASE_URL = "https://github.com" +apk_sources = { + "reddit": f"{APK_MIRROR_BASE_APK_URL}/redditinc/reddit/", + "twitter": f"{APK_MIRROR_BASE_APK_URL}/x-corp/twitter/", + "tiktok": f"{APK_MIRROR_BASE_APK_URL}/tiktok-pte-ltd/tik-tok-including-musical-ly/", + "warnwetter": f"{APK_MIRROR_BASE_APK_URL}/deutscher-wetterdienst/warnwetter/", + "youtube": f"{APK_MIRROR_BASE_APK_URL}/google-inc/youtube/", + "youtube_music": f"{APK_MIRROR_BASE_APK_URL}/google-inc/youtube-music/", + "ticktick": f"{APK_MIRROR_BASE_APK_URL}/appest-inc/ticktick-to-do-list-with-reminder-day-planner/", + "icon_pack_studio": f"{APK_MIRROR_BASE_APK_URL}/smart-launcher-team/icon-pack-studio/", + "windy": f"{APK_MIRROR_BASE_APK_URL}/windy-weather-world-inc/windy-wind-weather-forecast/", + "tasker": f"{APK_MIRROR_BASE_APK_URL}/joaomgcd/tasker-crafty-apps-eu/", + "vsco": f"{APK_MIRROR_BASE_APK_URL}/vsco/vsco-cam/", + "nova_launcher": f"{APK_MIRROR_BASE_APK_URL}/teslacoil-software/nova-launcher/", + "netguard": f"{APK_MIRROR_BASE_APK_URL}/marcel-bokhorst/netguard-no-root-firewall/", + "instagram": f"{APK_MIRROR_BASE_APK_URL}/instagram/instagram-instagram/", + "inshorts": f"{APK_MIRROR_BASE_APK_URL}/inshorts-formerly-news-in-shorts/", + "messenger": f"{APK_MIRROR_BASE_APK_URL}/facebook-2/messenger/", + "trakt": f"{APK_MIRROR_BASE_APK_URL}/trakt/trakt/", + "candyvpn": f"{APK_MIRROR_BASE_APK_URL}/liondev-io/candylink-vpn/", + "sonyheadphone": f"{APK_MIRROR_BASE_APK_URL}/sony-corporation/sony-headphones-connect/", + "boost": f"{APK_MIRROR_BASE_APK_URL}/ruben-mayayo/boost-for-reddit/", + "rif": f"{APK_MIRROR_BASE_APK_URL}/talklittle/reddit-is-fun/", + "sync": f"{APK_MIRROR_BASE_APK_URL}/red-apps-ltd/sync-for-reddit/", + "infinity": f"{APK_MIRROR_BASE_APK_URL}/docile-alligator/infinity-for-reddit/", + "slide": f"{APK_MIRROR_BASE_APK_URL}/haptic-apps/slide-for-reddit/", + "bacon": f"{APK_MIRROR_BASE_APK_URL}/onelouder-apps/baconreader-for-reddit/", + "pixiv": f"{APK_MIRROR_BASE_APK_URL}/pixiv-inc/pixiv/", + "spotify": UPTODOWN_BASE_URL.format("spotify"), + "nyx-music-player": UPTODOWN_BASE_URL.format("nyx-music-player"), + "my-expenses": UPTODOWN_BASE_URL.format("my-expenses"), + "backdrops": UPTODOWN_BASE_URL.format("backdrops"), + "twitch": UPTODOWN_BASE_URL.format("twitch"), + "irplus": UPTODOWN_BASE_URL.format("irplus"), + "meme-generator-free": UPTODOWN_BASE_URL.format("meme-generator-free"), + "yuka": UPTODOWN_BASE_URL.format("yuka"), + "relay": UPTODOWN_BASE_URL.format("relay"), + "grecorder": UPTODOWN_BASE_URL.format("opnemer"), + "hex-editor": APK_PURE_URL, + "androidtwelvewidgets": APK_PURE_URL, + "expensemanager": APK_SOS_URL, +} diff --git a/src/downloader/uptodown.py b/src/downloader/uptodown.py index 9587fd4..5d45e68 100644 --- a/src/downloader/uptodown.py +++ b/src/downloader/uptodown.py @@ -7,6 +7,7 @@ from loguru import logger from scripts.status_check import headers from src.downloader.download import Downloader +from src.downloader.sources import apk_sources from src.exceptions import UptoDownAPKDownloadFailure from src.utils import bs4_parser @@ -34,9 +35,7 @@ class UptoDown(Downloader): :return: Version of downloaded apk """ logger.debug("downloading specified version of app from uptodown.") - url = ( - f"https://{self.config.upto_down.get(app)}.en.uptodown.com/android/versions" - ) + url = f"{apk_sources[app]}/versions" html = self.config.session.get(url).text soup = BeautifulSoup(html, bs4_parser) versions_list = soup.find("section", {"id": "versions"}) @@ -54,7 +53,5 @@ class UptoDown(Downloader): logger.debug(f"Downloaded {app} apk from upto_down_downloader in rt") def latest_version(self, app: str, **kwargs: Any) -> None: - page = ( - f"https://{self.config.upto_down.get(app)}.en.uptodown.com/android/download" - ) + page = f"{apk_sources[app]}/download" self.extract_download_link(page, app)