🎨 Moved apk sources out of config. (#294)

.github/workflows/build-artifact.yml

@@ -34,7 +34,7 @@ jobs:
   build-apk:
     name: APK Build
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 60
 
     steps:
       - name: Check out Git repository

src/config.py

@@ -5,6 +5,8 @@ from typing import List
 from environs import Env
 from requests import Session
 
+from src.downloader.sources import apk_sources
+
 default_cli = "https://github.com/revanced/revanced-cli/releases/latest"
 default_patches = "https://github.com/revanced/revanced-patches/releases/latest"
 default_patches_json = default_patches
@@ -12,8 +14,6 @@ default_integrations = (
     "https://github.com/revanced/revanced-integrations/releases/latest"
 )
 
-APK_MIRROR_BASE_URL = "https://www.apkmirror.com"
-
 
 class RevancedConfig(object):
     """Revanced Configurations."""
@@ -25,57 +25,12 @@ class RevancedConfig(object):
         self.temp_folder = Path("apks")
         self.session = Session()
         self.session.headers["User-Agent"] = "anything"
-        self.apk_mirror = APK_MIRROR_BASE_URL
-        self.upto_down = {
-            "spotify": "spotify",
-            "nyx-music-player": "nyx-music-player",
-            "my-expenses": "my-expenses",
-            "backdrops": "backdrops",
-            "twitch": "twitch",
-            "irplus": "irplus",
-            "meme-generator-free": "meme-generator-free",
-            "yuka": "yuka",
-            "grecorder": "opnemer",
-        }
-        self.apk_pure = ["hex-editor", "androidtwelvewidgets"]
-        self.apk_sos = ["expensemanager", "candyvpn"]
         self.ci_test = env.bool("CI_TEST", False)
-        self.apps = env.list("PATCH_APPS", default_build)
+        self.apps = env.list(
+            "PATCH_APPS",
+            list(apk_sources.keys()) if self.ci_test else default_build,
+        )
         self.rip_libs_apps: List[str] = []
-        self.apk_mirror_urls = {
-            "reddit": f"{self.apk_mirror}/apk/redditinc/reddit/",
-            "twitter": f"{self.apk_mirror}/apk/x-corp/twitter/",
-            "tiktok": f"{self.apk_mirror}/apk/tiktok-pte-ltd/tik-tok-including-musical-ly/",
-            "warnwetter": f"{self.apk_mirror}/apk/deutscher-wetterdienst/warnwetter/",
-            "youtube": f"{self.apk_mirror}/apk/google-inc/youtube/",
-            "youtube_music": f"{self.apk_mirror}/apk/google-inc/youtube-music/",
-            "ticktick": f"{self.apk_mirror}/apk/appest-inc/ticktick-to-do-list-with-reminder-day-planner/",
-            "icon_pack_studio": f"{self.apk_mirror}/apk/smart-launcher-team/icon-pack-studio/",
-            "twitch": f"{self.apk_mirror}/apk/twitch-interactive-inc/twitch/",
-            "windy": f"{self.apk_mirror}/apk/windy-weather-world-inc/windy-wind-weather-forecast/",
-            "tasker": f"{self.apk_mirror}/apk/joaomgcd/tasker-crafty-apps-eu/",
-            "vsco": f"{self.apk_mirror}/apk/vsco/vsco-cam/",
-            "nova_launcher": f"{self.apk_mirror}/apk/teslacoil-software/nova-launcher/",
-            "netguard": f"{self.apk_mirror}/apk/marcel-bokhorst/netguard-no-root-firewall/",
-            "instagram": f"{self.apk_mirror}/apk/instagram/instagram-instagram/",
-            "inshorts": f"{self.apk_mirror}/apk/inshorts-formerly-news-in-shorts/",
-            "messenger": f"{self.apk_mirror}/apk/facebook-2/messenger/",
-            "trakt": f"{self.apk_mirror}/apk/trakt/trakt/",
-            "candyvpn": f"{self.apk_mirror}/apk/liondev-io/candylink-vpn/",
-            "sonyheadphone": f"{self.apk_mirror}/apk/sony-corporation/sony-headphones-connect/",
-            "relay": f"{self.apk_mirror}/apk/dbrady/relay-for-reddit-2/",
-            "boost": f"{self.apk_mirror}/apk/ruben-mayayo/boost-for-reddit/",
-            "rif": f"{self.apk_mirror}/apk/talklittle/reddit-is-fun/",
-            "sync": f"{self.apk_mirror}/apk/red-apps-ltd/sync-for-reddit/",
-            "infinity": f"{self.apk_mirror}/apk/docile-alligator/infinity-for-reddit/",
-            "slide": f"{self.apk_mirror}/apk/haptic-apps/slide-for-reddit/",
-            "bacon": f"{self.apk_mirror}/apk/onelouder-apps/baconreader-for-reddit/",
-            "pixiv": f"{self.apk_mirror}/apk/pixiv-inc/pixiv/",
-        }
-        self.apk_mirror_version_urls = {
-            key: value + value.split("/")[-2]
-            for key, value in self.apk_mirror_urls.items()
-        }
         self.existing_downloaded_apks = env.list("EXISTING_DOWNLOADED_APKS", [])
         self.personal_access_token = env.str("PERSONAL_ACCESS_TOKEN", None)
         self.dry_run = env.bool("DRY_RUN", False)

src/downloader/apkmirror.py

@@ -7,6 +7,7 @@ from loguru import logger
 
 from scripts.status_check import headers
 from src.downloader.download import Downloader
+from src.downloader.sources import APK_MIRROR_BASE_URL, apk_sources
 from src.exceptions import APKMirrorAPKDownloadFailure
 from src.utils import bs4_parser
 
@@ -23,7 +24,7 @@ class ApkMirror(Downloader):
                 "href"
             ):
                 return self._download(
-                    self.config.apk_mirror + possible_link["href"], f"{app}.apk"
+                    APK_MIRROR_BASE_URL + possible_link["href"], f"{app}.apk"
                 )
         raise APKMirrorAPKDownloadFailure(
             f"Unable to extract force download for {app}", url=link
@@ -48,7 +49,7 @@ class ApkMirror(Downloader):
             None,
         ):
             self._extract_force_download_link(
-                self.config.apk_mirror + final_download_link, app
+                APK_MIRROR_BASE_URL + final_download_link, app
             )
         else:
             raise APKMirrorAPKDownloadFailure(
@@ -68,7 +69,9 @@ class ApkMirror(Downloader):
             if row.find(class_="accent_color"):
                 apk_type = row.find(class_="apkm-badge").get_text()
                 if apk_type == "APK" and (
-                    "arm64-v8a" in row.text.strip() or "universal" in row.text.strip()
+                    "arm64-v8a" in row.text.strip()
+                    or "universal" in row.text.strip()
+                    or "noarch" in row.text.strip()
                 ):
                     sub_url = row.find(class_="accent_color")["href"]
                     break
@@ -76,12 +79,12 @@ class ApkMirror(Downloader):
             raise APKMirrorAPKDownloadFailure(
                 "Unable to extract download page", url=main_page
             )
-        return f"{self.config.apk_mirror}{sub_url}"
+        return f"{APK_MIRROR_BASE_URL}{sub_url}"
 
     @staticmethod
     def _extracted_search_div(url: str, search_class: str) -> Tag:
         """Extract search div."""
-        r = requests.get(url, headers=headers)
+        r = requests.get(url, headers=headers, timeout=10)
         if r.status_code != 200:
             raise APKMirrorAPKDownloadFailure(
                 f"Unable to connect with {url} on ApkMirror. Are you blocked by APKMirror or abused apkmirror "
@@ -101,9 +104,9 @@ class ApkMirror(Downloader):
         """
         if not main_page:
             version = version.replace(".", "-")
-            main_page = (
+            apk_main_page = apk_sources[app]
-                f"{self.config.apk_mirror_version_urls.get(app)}-{version}-release/"
+            version_page = apk_main_page + apk_main_page.split("/")[-2]
-            )
+            main_page = f"{version_page}-{version}-release/"
         download_page = self.get_download_page(main_page)
         self.extract_download_link(download_page, app)
 
@@ -115,14 +118,17 @@ class ApkMirror(Downloader):
         :return: Version of downloaded apk
         """
 
-        app_main_page = self.config.apk_mirror_urls[app]
+        app_main_page = apk_sources[app]
         versions_div = self._extracted_search_div(
             app_main_page, "listWidget p-relative"
         )
         app_rows = versions_div.find_all(class_="appRow")
         version_urls = [
-            app_row.find(class_="downloadLink")["href"] for app_row in app_rows
+            app_row.find(class_="downloadLink")["href"]
+            for app_row in app_rows
+            if "beta" not in app_row.find(class_="appRowTitle").get_text().lower()
+            and "alpha" not in app_row.find(class_="appRowTitle").get_text().lower()
         ]
         return self.specific_version(
-            app, "latest", self.config.apk_mirror + max(version_urls)
+            app, "latest", APK_MIRROR_BASE_URL + max(version_urls)
         )

src/downloader/apkpure.py

@@ -4,6 +4,7 @@ from typing import Any
 from loguru import logger
 
 from src.downloader.download import Downloader
+from src.downloader.sources import apk_sources
 from src.patches import Patches
 
 
@@ -18,6 +19,6 @@ class ApkPure(Downloader):
         :return: Version of downloaded apk
         """
         package_name = Patches.get_package_name(app)
-        download_url = f"https://d.apkpure.com/b/APK/{package_name}?version=latest"
+        download_url = apk_sources[app].format(package_name)
         self._download(download_url, f"{app}.apk")
-        logger.debug(f"Downloaded {app} apk from apk_pure_downloader in rt")
+        logger.debug(f"Downloaded {app} apk from apk_pure_downloader")

src/downloader/apksos.py

@@ -6,6 +6,7 @@ from bs4 import BeautifulSoup
 
 from scripts.status_check import headers
 from src.downloader.download import Downloader
+from src.downloader.sources import apk_sources
 from src.exceptions import APKSosAPKDownloadFailure
 from src.utils import bs4_parser
 
@@ -36,5 +37,5 @@ class ApkSos(Downloader):
         :return: Version of downloaded apk
         """
         package_name = self.patcher.get_package_name(app)
-        download_url = f"https://apksos.com/download-app/{package_name}"
+        download_url = apk_sources[app].format(package_name)
         self.extract_download_link(download_url, app)

src/downloader/factory.py

@@ -5,7 +5,15 @@ from src.downloader.apkpure import ApkPure
 from src.downloader.apksos import ApkSos
 from src.downloader.download import Downloader
 from src.downloader.github import Github
+from src.downloader.sources import (
+    APK_MIRROR_BASE_URL,
+    APK_PURE_URL,
+    APK_SOS_URL,
+    GITHUB_BASE_URL,
+    apk_sources,
+)
 from src.downloader.uptodown import UptoDown
+from src.exceptions import DownloadFailure
 from src.patches import Patches
 
 
@@ -24,13 +32,14 @@ class DownloaderFactory(object):
         patcher : Patcher
         config : Config
         """
-        if app in {"patches", "microg"}:
+        if apk_sources[app].startswith(GITHUB_BASE_URL):
             return Github(patcher, config)
-        if app in config.apk_pure:
+        if apk_sources[app].startswith(APK_PURE_URL):
             return ApkPure(patcher, config)
-        elif app in config.apk_sos:
+        elif apk_sources[app].startswith(APK_SOS_URL):
             return ApkSos(patcher, config)
-        elif app in config.upto_down:
+        elif apk_sources[app].endswith("en.uptodown.com/android"):
             return UptoDown(patcher, config)
-        else:
+        elif apk_sources[app].startswith(APK_MIRROR_BASE_URL):
             return ApkMirror(patcher, config)
+        raise DownloadFailure(f"No download factory found for {app}")

src/downloader/sources.py

@@ -0,0 +1,47 @@
+APK_MIRROR_BASE_URL = "https://www.apkmirror.com"
+APK_MIRROR_BASE_APK_URL = f"{APK_MIRROR_BASE_URL}/apk"
+UPTODOWN_BASE_URL = "https://{}.en.uptodown.com/android"
+APK_PURE_URL = "https://d.apkpure.com/b/APK/{}?version=latest"
+APK_SOS_URL = "https://apksos.com/download-app/{}"
+GITHUB_BASE_URL = "https://github.com"
+apk_sources = {
+    "reddit": f"{APK_MIRROR_BASE_APK_URL}/redditinc/reddit/",
+    "twitter": f"{APK_MIRROR_BASE_APK_URL}/x-corp/twitter/",
+    "tiktok": f"{APK_MIRROR_BASE_APK_URL}/tiktok-pte-ltd/tik-tok-including-musical-ly/",
+    "warnwetter": f"{APK_MIRROR_BASE_APK_URL}/deutscher-wetterdienst/warnwetter/",
+    "youtube": f"{APK_MIRROR_BASE_APK_URL}/google-inc/youtube/",
+    "youtube_music": f"{APK_MIRROR_BASE_APK_URL}/google-inc/youtube-music/",
+    "ticktick": f"{APK_MIRROR_BASE_APK_URL}/appest-inc/ticktick-to-do-list-with-reminder-day-planner/",
+    "icon_pack_studio": f"{APK_MIRROR_BASE_APK_URL}/smart-launcher-team/icon-pack-studio/",
+    "windy": f"{APK_MIRROR_BASE_APK_URL}/windy-weather-world-inc/windy-wind-weather-forecast/",
+    "tasker": f"{APK_MIRROR_BASE_APK_URL}/joaomgcd/tasker-crafty-apps-eu/",
+    "vsco": f"{APK_MIRROR_BASE_APK_URL}/vsco/vsco-cam/",
+    "nova_launcher": f"{APK_MIRROR_BASE_APK_URL}/teslacoil-software/nova-launcher/",
+    "netguard": f"{APK_MIRROR_BASE_APK_URL}/marcel-bokhorst/netguard-no-root-firewall/",
+    "instagram": f"{APK_MIRROR_BASE_APK_URL}/instagram/instagram-instagram/",
+    "inshorts": f"{APK_MIRROR_BASE_APK_URL}/inshorts-formerly-news-in-shorts/",
+    "messenger": f"{APK_MIRROR_BASE_APK_URL}/facebook-2/messenger/",
+    "trakt": f"{APK_MIRROR_BASE_APK_URL}/trakt/trakt/",
+    "candyvpn": f"{APK_MIRROR_BASE_APK_URL}/liondev-io/candylink-vpn/",
+    "sonyheadphone": f"{APK_MIRROR_BASE_APK_URL}/sony-corporation/sony-headphones-connect/",
+    "boost": f"{APK_MIRROR_BASE_APK_URL}/ruben-mayayo/boost-for-reddit/",
+    "rif": f"{APK_MIRROR_BASE_APK_URL}/talklittle/reddit-is-fun/",
+    "sync": f"{APK_MIRROR_BASE_APK_URL}/red-apps-ltd/sync-for-reddit/",
+    "infinity": f"{APK_MIRROR_BASE_APK_URL}/docile-alligator/infinity-for-reddit/",
+    "slide": f"{APK_MIRROR_BASE_APK_URL}/haptic-apps/slide-for-reddit/",
+    "bacon": f"{APK_MIRROR_BASE_APK_URL}/onelouder-apps/baconreader-for-reddit/",
+    "pixiv": f"{APK_MIRROR_BASE_APK_URL}/pixiv-inc/pixiv/",
+    "spotify": UPTODOWN_BASE_URL.format("spotify"),
+    "nyx-music-player": UPTODOWN_BASE_URL.format("nyx-music-player"),
+    "my-expenses": UPTODOWN_BASE_URL.format("my-expenses"),
+    "backdrops": UPTODOWN_BASE_URL.format("backdrops"),
+    "twitch": UPTODOWN_BASE_URL.format("twitch"),
+    "irplus": UPTODOWN_BASE_URL.format("irplus"),
+    "meme-generator-free": UPTODOWN_BASE_URL.format("meme-generator-free"),
+    "yuka": UPTODOWN_BASE_URL.format("yuka"),
+    "relay": UPTODOWN_BASE_URL.format("relay"),
+    "grecorder": UPTODOWN_BASE_URL.format("opnemer"),
+    "hex-editor": APK_PURE_URL,
+    "androidtwelvewidgets": APK_PURE_URL,
+    "expensemanager": APK_SOS_URL,
+}

src/downloader/uptodown.py

@@ -7,6 +7,7 @@ from loguru import logger
 
 from scripts.status_check import headers
 from src.downloader.download import Downloader
+from src.downloader.sources import apk_sources
 from src.exceptions import UptoDownAPKDownloadFailure
 from src.utils import bs4_parser
 
@@ -34,9 +35,7 @@ class UptoDown(Downloader):
         :return: Version of downloaded apk
         """
         logger.debug("downloading specified version of app from uptodown.")
-        url = (
+        url = f"{apk_sources[app]}/versions"
-            f"https://{self.config.upto_down.get(app)}.en.uptodown.com/android/versions"
-        )
         html = self.config.session.get(url).text
         soup = BeautifulSoup(html, bs4_parser)
         versions_list = soup.find("section", {"id": "versions"})
@@ -54,7 +53,5 @@ class UptoDown(Downloader):
         logger.debug(f"Downloaded {app} apk from upto_down_downloader in rt")
 
     def latest_version(self, app: str, **kwargs: Any) -> None:
-        page = (
+        page = f"{apk_sources[app]}/download"
-            f"https://{self.config.upto_down.get(app)}.en.uptodown.com/android/download"
-        )
         self.extract_download_link(page, app)