diff --git a/.gitignore b/.gitignore index 5ef80a1..aba434d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,4 @@ venv /revanced-cache/ changelog.md .idea -*patches.json +*.json diff --git a/README.md b/README.md index 3a5b90d..7d135f1 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,11 @@ # 🤓Docker-Py-ReVanced -A little python script that will help you in building Revanced and Revanced-Extended [apps](#note). +A little python script that will help you in building Revanced [apps](#patch-apps). **`Note`** - If you are a root user and want magisk module (Extended). Get them [here](https://github.com/nikhilbadyal/revanced-magisk-module) This is just a builder for revanced and not a revanced support. Please be understanding and refrain from asking -about revanced features/bugs. Discuss those on proper relevant forums(on Revanced GitHub , Discord) - -**`Note`** - I prefer [Revanced Extended](https://github.com/inotia00/revanced-patches/tree/revanced-extended) more -(for YouTube & YouTube Music) hence the YouTube and YouTube Music builds in this repo are from -Revanced Extended. +about revanced features/bugs. Discuss those on proper relevant forums. ## Pre-Built APKs @@ -19,7 +15,7 @@ You can get pre-built apks [here](https://revanced_apkss.t.me/) You can use any of the following methods to build. -- 🚀In GitHub (**_`Recommended`_**) +- 🚀 **_GitHub**_ (**_`Recommended`_**) 1. Click Star to support the project.

@@ -51,8 +47,8 @@ You can use any of the following methods to build. 5. If the building process is successful, you’ll get your APKs in the
-- 🐳With Docker Compose - Windows/Mac users simply install Docker Desktop. If using Linux see below +- 🐳 **_Docker Compose_**
+ Windows/Mac users simply install Docker Desktop. If using Linux see below 1. Install Docker(Skip if already installed) ```bash @@ -80,7 +76,7 @@ You can use any of the following methods to build. 6. Update `.env` file if you want some customization(See notes) 7. Run script with ```shell - docker-compose up + docker-compose up --build ``` - 🐳With Docker @@ -99,7 +95,7 @@ You can use any of the following methods to build. - 🫠Without Docker - 1. Install Java17 (zulu preferred) + 1. Install Java >= 17 2. Install Python 3. Create virtual environment ``` @@ -115,15 +111,53 @@ You can use any of the following methods to build. ``` 6. Run the script with ``` - python python main.py + python main.py ``` +## Configurations + +### Global Config + +| **Env Name** | **Description** | **Default** | +|:---------------------------------------------------------|:-------------------------------------------------:|:---------------------------------------------------------------------------------------------------------| +| [PATCH_APPS](#patch-apps) | Apps to patch/build | youtube | +| [EXISTING_DOWNLOADED_APKS ](#existing-downloaded-apks) | Already downloaded clean apks | [] | +| [PERSONAL_ACCESS_TOKEN](#personal-access-token) | Github Token to be used | None | +| DRY_RUN | Do a dry run | False | +| [GLOBAL_CLI_DL*](#global-resources) | DL for CLI to be used for patching apps. | [Revanced CLI](https://github.com/revanced/revanced-cli) | +| [GLOBAL_PATCHES_DL*](#global-resources) | DL for Patches to be used for patching apps. | [Revanced Patches](https://github.com/revanced/revanced-patches) | +| [GLOBAL_PATCHES_JSON_DL*](#global-resources) | DL for Patches Json to be used for patching apps. | [Revanced Patches](https://github.com/revanced/revanced-patches) | +| [GLOBAL_INTEGRATIONS_DL*](#global-resources) | DL for Integrations to be used for patching apps. | [Revanced CLI](https://github.com/revanced/revanced-integrations) | +| [GLOBAL_KEYSTORE_FILE_NAME*](#global-keystore-file-name) | Key file to be used for signing apps | [Builder's own key](https://github.com/nikhilbadyal/docker-py-revanced/blob/main/apks/revanced.keystore) | +| [GLOBAL_ARCHS_TO_BUILD*](#global-archs-to-build) | Arch to keep in the patched apk. | All | +| REDDIT_CLIENT_ID | Reddit Client ID to patch reddit apps | None | +| VT_API_KEY | Virus Total Key to scan APKs | None | +| [TELEGRAM_CHAT_ID](#telegram-support) | Receiver in Telegram upload | None | +| [TELEGRAM_BOT_TOKEN](#telegram-support) | APKs Sender for Telegram upload | None | +| [TELEGRAM_API_ID](#telegram-support) | Used for telegram Authentication | None | +| [TELEGRAM_API_HASH](#telegram-support) | Used for telegram Authentication | None | + +`*` - Can be overridden for individual app. +### App Level Config + +| Env Name | Description | Default | +|:------------------------------------------------------------|:---------------------------------------------------------:|:-------------------------------| +| [*APP_NAME*_CLI_DL](#global-resources) | DL for CLI to be used for patching **APP_NAME**. | GLOBAL_CLI_DL | +| [*APP_NAME*_PATCHES_DL](#global-resources) | DL for Patches to be used for patching **APP_NAME**. | GLOBAL_PATCHES_DL | +| [*APP_NAME*_PATCHES_JSON_DL](#global-resources) | DL for Patches Json to be used for patching **APP_NAME**. | GLOBAL_PATCHES_JSON_DL | +| [*APP_NAME*_INTEGRATIONS_DL](#global-resources) | DL for Integrations to be used for patching **APP_NAME**. | GLOBAL_INTEGRATIONS_DL | +| [*APP_NAME*_KEYSTORE_FILE_NAME](#global-keystore-file-name) | Key file to be used for signing **APP_NAME**. | GLOBAL_KEYSTORE_FILE_NAME | +| [*APP_NAME*_ARCHS_TO_BUILD](#global-archs-to-build) | Arch to keep in the patched **APP_NAME**. | GLOBAL_ARCHS_TO_BUILD | +| [*APP_NAME*_EXCLUDE_PATCH**](#custom-exclude-patching) | Patches to exclude while patching **APP_NAME**. | [] | +| [*APP_NAME*_INCLUDE_PATCH**](#custom-include-patching) | Patches to include while patching **APP_NAME**. | [] | +| [*APP_NAME*_VERSION**](#app-version) | Version to use for download for patching. | Recommended by patch resources | + +`**` - By default all patches for a given app are included.
+`**` - Can be used to included universal patch. + ## Note -(Pay attention to 3,4)
-By default, script build the version as recommended by Revanced team. - -1. Supported values for **REVANCED_APPS_NAME** are : +1. Supported values for **APP_NAME** are : 1. [youtube](https://www.apkmirror.com/apk/google-inc/youtube/) 2. [youtube_music](https://www.apkmirror.com/apk/google-inc/youtube-music/) @@ -165,156 +199,145 @@ By default, script build the version as recommended by Revanced team. 38. [bacon](https://www.apkmirror.com/apk/onelouder-apps/baconreader-for-reddit/) 39. [microg](https://github.com/inotia00/mMicroG/releases) -
Please verify the source of original APKs yourself with links provided. I'm not responsible for any damaged caused. - If you know any better/safe source to download clean. Please raise a PR. - -2. Remember to download the **_Microg_**. Otherwise, you will not be able to open YouTube. -3. By default, it will build only `youtube`. To build other apps supported by revanced team. - Add the apps you want to build in `.env` file or in `ENVS` in - `GitHub secrets` in the format +
Please verify the source of original APKs yourself with links provided. I'm not responsible for any damage + caused.If you know any better/safe source to download clean. Open a discussion. +2. By default, script build the latest version as recommended by `patches.json` team. +3. Remember to download the **_Microg_**. Otherwise, you may not be able to open YouTube/YouTube Music. +4. By default, tool will build only `youtube`. To build other apps supported by patching + resources.Add the apps you want to build in `.env` file or in `ENVS` in `GitHub secrets` in the format ```ini - PATCH_APPS= + PATCH_APPS= ``` Example: ```ini PATCH_APPS=youtube,twitter,reddit ``` -4. If you want to exclude any patch. Set comma separated patch in `.env` file or in `ENVS` in `GitHub secrets` - (Recommended) in the format - ```ini - EXCLUDE_PATCH_= +5. If APKMirror or other apk sources are blocked in your region or script + somehow is unable to download from apkmirror. You can download apk manually from any source. Place them in + `/apks` directory and provide environment variable in `.env` file or in `ENVS` in `GitHub secrets`(Recommended) + in the format. + ```dotenv + EXISTING_DOWNLOADED_APKS= ``` Example: ```dotenv - EXCLUDE_PATCH_YOUTUBE=custom-branding,hide-get-premium - EXCLUDE_PATCH_YOUTUBE_MUSIC=yt-music-is-shit + EXISTING_DOWNLOADED_APKS=youtube,youtube_music ``` - If you are using `Revanced Extended.` Add `_EXTENDED` in exclude options. + If you add above. Script will not download the `youtube` & `youtube_music`apk from internet and expects an apk in + `/apks` folder with **same** name. +6. If you run script again & again. You might hit GitHub API limit. In that case + you can provide your Personal GitHub Access Token in `.env` file or in `ENVS` in `GitHub secrets` (Recommended) + in the format - + ```dotenv + PERSONAL_ACCESS_TOKEN= + ``` +7. You can provide Direct download to the resource to used for patching apps `.env` file + or in `ENVS` in `GitHub secrets` (Recommended) in the format - + ```dotenv + GLOBAL_CLI_DL=https://github.com/revanced/revanced-cli + GLOBAL_PATCHES_DL=https://github.com/revanced/revanced-patches + GLOBAL_PATCHES_JSON_DL=https://github.com/revanced/revanced-patches + GLOBAL_INTEGRATIONS_DL=https://github.com/revanced/revanced-integrations + ``` + Resources downloaded from envs and will be used for patching for any **APP_NAME**. + Unless provided different resource for the individual app.

+ Tool also support resource config at app level. You can patch A app with X resources while patching B with Y + resources. + This can be done by providing Direct download link for resources for app.
Example: ```dotenv - EXCLUDE_PATCH_YOUTUBE_EXTENDED=custom-branding-red,custom-branding-blue,materialyou - EXCLUDE_PATCH_YOUTUBE_MUSIC_EXTENDED=custom-branding-music + YOUTUBE_CLI_DL=https://github.com/inotia00/revanced-cli + YOUTUBE_PATCHES_DL=https://github.com/inotia00/revanced-patches + YOUTUBE_PATCHES_JSON_DL=https://github.com/inotia00/revanced-patches + YOUTUBE_INTEGRATIONS_DL=https://github.com/inotia00/revanced-integrations ``` - **_All the patches for an app are included by default._**.

If you want to apply a universal patch. You can - include it - manually. See below for more information.
- If you want to include any universal patch. Set comma separated patch in `.env` file or in `ENVS` in `GitHub - secrets` - (Recommended) in the format - ```ini - INCLUDE_PATCH_= - ``` - Example: + With the config tool will try to patch youtube with resources from inotia00 while other global resource will used + for patching other apps. + *Note* - The link provided must be DLs. Unless they are from GitHub. +8. If you don't want to use default keystore. You can provide your own by + placing it inside `apks` folder. And adding the name of `keystore-file` in `.env` file or in `ENVS` in `GitHub + secrets` (Recommended) in the format ```dotenv - INCLUDE_PATCH_YOUTUBE=remove-screenshot-restriction + GLOBAL_KEYSTORE_FILE_NAME=revanced.keystore ``` - If you are using `Revanced Extended.` Add `_EXTENDED` in exclude options. - Example: + Tool also support providing secret key at app level. You can sign A app with X key while signing B with Y + key.
+ Example: ```dotenv - INCLUDE_PATCH_YOUTUBE_EXTENDED=remove-screenshot-restriction + YOUTUBE_KEYSTORE_FILE_NAME=youtube.keystore ``` - **_Remember_** - Revanced patches are provided space separated, make sure you type those **-** separated here. It means a - patch named _**Hey There**_ will be entered as **_hey-there_** in the above example. -5. If you want to build a specific version . Add `version` in `.env` file or in `ENVS` in `GitHub secrets` (Recommended) - in the format - ```ini - _VERSION= - ``` - Example: - ```ini - YOUTUBE_VERSION=17.31.36 - YOUTUBE_MUSIC_VERSION=X.X.X - TWITTER_VERSION=X.X.X - REDDIT_VERSION=X.X.X - TIKTOK_VERSION=X.X.X - WARNWETTER_VERSION=X.X.X - ``` -6. If you want to build `latest` version, whatever latest is available(including - beta) . - Add `latest` in `.env` file or in `ENVS` in `GitHub secrets` (Recommended) in the format - - ```ini - _VERSION=latest +9. You can build only for a particular arch in order to get smaller apk files.This + can be done with by adding comma separated `ARCHS_TO_BUILD` in `ENVS` in `GitHub secrets` (Recommended) in the + format. + ```dotenv + GLOABAL_ARCHS_TO_BUILD=arm64-v8a,armeabi-v7a ``` + Tool also support configuring at app level.
Example: - - ```ini - YOUTUBE_VERSION=latest - YOUTUBE_MUSIC_VERSION=latest - TWITTER_VERSION=latest - REDDIT_VERSION=latest - TIKTOK_VERSION=latest - WARNWETTER_VERSION=latest - ``` - -7. If you don't want to use default keystore. You can provide your own by placing it - inside `apks` folder. And adding the name of `keystore-file` in `.env` file or in `ENVS` in `GitHub secrets` - (Recommended) in the format ```dotenv - KEYSTORE_FILE_NAME=revanced.keystore + YOUTUBE_ARCHS_TO_BUILD=arm64-v8a,armeabi-v7a ``` -8. If you want to use Revanced-Extended for YouTube and YouTube Music. Add the following adding - in `.env` file or in `ENVS` in `GitHub secrets` (Recommended) in the format - ```dotenv - BUILD_EXTENDED=True - ``` - or disable it with (default) - ```dotenv - BUILD_EXTENDED=False - ``` -9. For Telegram Upload. - 1. Set up a telegram channel, send a message to it and forward the message to - this telegram [bot](https://t.me/username_to_id_bot) - 2. Copy `id` and save it to `TELEGRAM_CHAT_ID`
-
- 3. `TELEGRAM_BOT_TOKEN` - Telegram provides BOT_TOKEN. It works as sender. Open [bot](https://t.me/BotFather) and - create one copy api key
-
- 4. `TELEGRAM_API_ID` - Telegram API_ID is provided by telegram [here](https://my.telegram.org/apps)
-
- 5. `TELEGRAM_API_HASH` - Telegram API_HASH is provided by telegram [here](https://my.telegram.org/apps)
-
- 6. After Everything done successfully the actions secrets of the repository will look something like
- -10. You can build only for a particular arch in order to get smaller apk files.This can be done with by adding comma - separated `ARCHS_TO_BUILD` in `ENVS` in `GitHub secrets` (Recommended) in the format. - ```dotenv - ARCHS_TO_BUILD=arm64-v8a,armeabi-v7a - ``` - Possible values for `ARCHS_TO_BUILD` are: `armeabi-v7a`,`x86`,`x86_64`,`arm64-v8a` - Make sure you are using `revanced-extended` as `revanced` doesn't support this. -11. You can scan your built apks files with VirusTotal. For that, Add `VT_API_KEY` in `GitHub secrets`. -12. Configuration defined in `ENVS` in `GitHub secrets` will override the configuration in `.env` file. You can use this - fact to define your normal configurations in `.env` file and sometimes if you want to build something different just - once. Add it in `GitHub secrets`.
- Or you can ignore what I said above and always use `GitHub secrets`. -13. If APKMirror or other apk source is blocked in your region or script somehow is unable to download from apkmirror. - You can download apk manually from any source. Place them in `/apks` directory and provide environment variable - in `.env` file or in `ENVS` in `GitHub secrets`(Recommended) in the format. - ```dotenv - EXISTING_DOWNLOADED_APKS= + *Note* - + 1. Possible values are: `armeabi-v7a`,`x86`,`x86_64`,`arm64-v8a` + 2. Make sure the patching resource(CLI) support this feature. +10. If you want to exclude any patch. Set comma separated patch in `.env` file + or in `ENVS` in `GitHub secrets` (Recommended) in the format + ```ini + _EXCLUDE_PATCH= ``` Example: ```dotenv - EXISTING_DOWNLOADED_APKS=youtube,youtube_music + YOUTUBE_EXCLUDE_PATCH=custom-branding,hide-get-premium + YOUTUBE_MUSIC_EXCLUDE_PATCH=yt-music-is-shit ``` - If you add above. Script will not download the `Youtube` & `youtube music`apk from internet and expects an apk in - `/apks` folder. - - Name of the downloaded apk must match with the available app choices found [here.](#note) -14. If you run script again & again. You might hit GitHub API limit. In that case you can provide your Personal - GitHub Access Token in `.env` file or in `ENVS` in `GitHub secrets` (Recommended) in the format - + Note - + 1. **All** the patches for an app are **included** by default.
+ 2. Revanced patches are provided as space separated, make sure you type those **-** separated here. + It means a patch named _**Hey There**_ must be entered as **_hey-there_** in the above example. +11. If you want to include any universal patch. Set comma separated patch in `.env` + file or in `ENVS` in `GitHub secrets` (Recommended) in the format + ```ini + _INCLUDE_PATCH= + ``` + Example: ```dotenv - PERSONAL_ACCESS_TOKEN= + YOUTUBE_INCLUDE_PATCH=remove-screenshot-restriction ``` + Note - + 1. Revanced patches are provided as space separated, make sure you type those **-** separated here. + It means a patch named _**Hey There**_ must be entered as **_hey-there_** in the above example. +12. If you want to build a specific version or latest version. Add `version` in `.env` file + or in `ENVS` in `GitHub secrets` (Recommended) in the format + ```ini + _VERSION= + ``` + Example: + ```ini + YOUTUBE_VERSION=17.31.36 + YOUTUBE_MUSIC_VERSION=X.X.X + TWITTER_VERSION=latest + ``` +13. For Telegram Upload. + 1. Set up a telegram channel, send a message to it and forward the message to + this telegram [bot](https://t.me/username_to_id_bot) + 2. Copy `id` and save it to `TELEGRAM_CHAT_ID`
+
+ 3. `TELEGRAM_BOT_TOKEN` - Telegram provides BOT_TOKEN. It works as sender. Open [bot](https://t.me/BotFather) and + create one copy api key
+
+ 4. `TELEGRAM_API_ID` - Telegram API_ID is provided by telegram [here](https://my.telegram.org/apps)
+
+ 5. `TELEGRAM_API_HASH` - Telegram API_HASH is provided by telegram [here](https://my.telegram.org/apps)
+
+ 6. After Everything done successfully a part of the actions secrets of the repository may look like
+ +14. Configuration defined in `ENVS` in `GitHub secrets` will override the configuration in `.env` file. You can use this + fact to define your normal configurations in `.env` file and sometimes if you want to build something different just + once. Add it in `GitHub secrets`.
15. Sample Envs
- -16. Make your Action has write access. If not click + +16. Make sure your Action has write access. If not click [here](https://github.com/nikhilbadyal/docker-py-revanced/settings/actions). In the bottom give read and write access to Actions. -17. If you want to patch reddit apps using your own Client ID. You can provide your Client ID - as secret `REDDIT_CLIENT_ID` in `GitHub secrets`. - -Thanks to [@aliharslan0](https://github.com/aliharslan0/pyrevanced) for his work. diff --git a/main.py b/main.py index b1b4869..dfe9594 100644 --- a/main.py +++ b/main.py @@ -6,86 +6,39 @@ from loguru import logger from src.config import RevancedConfig from src.downloader.factory import DownloaderFactory -from src.downloader.utils import download_revanced from src.parser import Parser from src.patches import Patches -from src.utils import AppNotFound, PatcherDownloadFailed +from src.utils import AppNotFound, PatchesJsonFailed, check_java def main() -> None: """Entry point.""" + from src.app import APP + env = Env() config = RevancedConfig(env) + check_java(config.dry_run) - patcher = Patches(config) - try: - download_revanced(config, patcher) - except PatcherDownloadFailed as e: - logger.error(f"Failed to download {e}") - sys.exit(1) - - logger.info(f"Will Patch only {patcher.config.apps}") - for app in patcher.config.apps: + logger.info(f"Will Patch only {config.apps}") + for app in config.apps: + logger.info(f"Trying to build {app}") try: - logger.info("Trying to build %s" % app) + app = APP(app_name=app, config=config) + patcher = Patches(config, app) parser = Parser(patcher, config) - app_all_patches, version, is_experimental = patcher.get_app_configs(app) + app_all_patches = patcher.get_app_configs(app) patcher.include_exclude_patch(app, parser, app_all_patches) downloader = DownloaderFactory.create_downloader( - app=app, patcher=patcher, config=config + app=app.app_name, patcher=patcher, config=config ) - downloader.download(version, app) - config.app_versions[app] = version - logger.info(f"Downloaded {app}, version {version}") - parser.patch_app(app=app, version=version, is_experimental=is_experimental) + downloader.download(app.app_version, app.app_name) + parser.patch_app(app) except AppNotFound as e: logger.info(f"Invalid app requested to build {e}") + except PatchesJsonFailed: + logger.exception("Patches.json not found") except Exception as e: logger.exception(f"Failed to build {app} because of {e}") - if len(config.alternative_youtube_patches) and "youtube" in config.apps: - for alternative_patch in config.alternative_youtube_patches: - parser = Parser(patcher, config) - app_all_patches, version, is_experimental = patcher.get_app_configs( - "youtube" - ) - patcher.include_exclude_patch("youtube", parser, app_all_patches) - was_inverted = parser.invert_patch(alternative_patch) - if was_inverted: - logger.info( - f"Rebuilding youtube with inverted {alternative_patch} patch." - ) - parser.patch_app( - app="youtube", - version=config.app_versions.get("youtube", "latest"), - is_experimental=is_experimental, - output_prefix="-" + alternative_patch + "-", - ) - else: - logger.info( - f"Skipping Rebuilding youtube as {alternative_patch} patch was not found." - ) - if len(config.alternative_youtube_music_patches) and "youtube_music" in config.apps: - for alternative_patch in config.alternative_youtube_music_patches: - parser = Parser(patcher, config) - app_all_patches, version, is_experimental = patcher.get_app_configs( - "youtube_music" - ) - patcher.include_exclude_patch("youtube_music", parser, app_all_patches) - was_inverted = parser.invert_patch(alternative_patch) - if was_inverted: - logger.info( - f"Rebuilding youtube music with inverted {alternative_patch} patch." - ) - parser.patch_app( - app="youtube_music", - version=config.app_versions.get("youtube_music", "latest"), - is_experimental=is_experimental, - output_prefix="-" + alternative_patch + "-", - ) - else: - logger.info( - f"Skipping Rebuilding youtube music as {alternative_patch} patch was not found." - ) if __name__ == "__main__": diff --git a/src/app.py b/src/app.py new file mode 100644 index 0000000..9ae881c --- /dev/null +++ b/src/app.py @@ -0,0 +1,103 @@ +"""Class to represent apk to be patched.""" +import concurrent +import hashlib +import pathlib +from concurrent.futures import ThreadPoolExecutor +from typing import Dict + +from loguru import logger + +from src.config import RevancedConfig +from src.utils import PatcherDownloadFailed, slugify + + +class APP(object): + """Patched APK.""" + + def __init__(self, app_name: str, config: RevancedConfig): + self.app_name = app_name + self.app_version = config.env.str(f"{app_name}_VERSION".upper(), None) + self.experiment = False + self.cli_dl = config.env.str(f"{app_name}_CLI_DL".upper(), config.global_cli_dl) + self.patches_dl = config.env.str( + f"{app_name}_PATCHES_DL".upper(), config.global_patches_dl + ) + self.integrations_dl = config.env.str( + f"{app_name}_INTEGRATION_DL".upper(), config.global_integrations_dl + ) + self.patches_json_dl = config.env.str( + f"{app_name}_PATCHES_JSON_DL".upper(), config.global_patches_json_dl + ) + self.exclude_request = config.env.list(f"{app_name}_EXCLUDE_PATCH".upper(), []) + self.include_request = config.env.list(f"{app_name}_INCLUDE_PATCH".upper(), []) + self.resource: Dict[str, str] = {} + self.no_of_patches = 0 + self.keystore_name = config.env.str( + f"{app_name}_KEYSTORE_FILE_NAME".upper(), config.global_keystore_name + ) + self.archs_to_build = config.env.list( + f"{app_name}_ARCHS_TO_BUILD".upper(), config.global_archs_to_build + ) + self.download_patch_resources(config) + + def get_output_file_name(self) -> str: + """Get output file appended with version.""" + return f"Re-{self.app_name}-{slugify(self.app_version)}-output.apk" + + def set_recommended_version(self, version: str, exp: bool = False) -> None: + """Update if cooking non-recommended.""" + self.app_version = version + self.experiment = exp + + def __str__(self: "APP") -> str: + attrs = vars(self) + return ", ".join([f"{key}: {value}" for key, value in attrs.items()]) + + @staticmethod + def download(url: str, config: RevancedConfig, assets_filter: str) -> str: + """Downloader.""" + from src.downloader.download import Downloader + + url = url.strip() + if url.startswith("https://github"): + from src.downloader.github import Github + + url = Github.patch_resource(url, assets_filter)[0] + extension = pathlib.Path(url).suffix + file_name = APP.generate_filename(url) + extension + Downloader(None, config).direct_download(url, file_name) # type: ignore + return file_name + + def download_patch_resources(self, config: RevancedConfig) -> None: + """Download resource for patching.""" + logger.info("Downloading resources for patching.") + # Create a list of resource download tasks + download_tasks = [ + ("cli", self.cli_dl, config, ".*jar"), + ("integrations", self.integrations_dl, config, ".*apk"), + ("patches", self.patches_dl, config, ".*jar"), + ("patches_json", self.patches_json_dl, config, ".*json"), + ] + + # Using a ThreadPoolExecutor for parallelism + with ThreadPoolExecutor(4) as executor: + futures = { + resource_name: executor.submit(self.download, *args) + for resource_name, *args in download_tasks + } + + # Wait for all tasks to complete + concurrent.futures.wait(futures.values()) + + # Retrieve results from completed tasks + for resource_name, future in futures.items(): + try: + self.resource[resource_name] = future.result() + except Exception as e: + raise PatcherDownloadFailed(f"An exception occurred: {e}") + + @staticmethod + def generate_filename(url: str) -> str: + """Get file name from url.""" + encoded_url: str = hashlib.sha256(url.encode()).hexdigest() + return encoded_url diff --git a/src/config.py b/src/config.py index d7b3471..c3baf9a 100644 --- a/src/config.py +++ b/src/config.py @@ -1,23 +1,28 @@ """Revanced Configurations.""" from pathlib import Path -from typing import Dict, List +from typing import List from environs import Env from requests import Session from src.utils import default_build +default_cli = "https://github.com/revanced/revanced-cli/releases/latest" +default_patches = "https://github.com/revanced/revanced-patches/releases/latest" +default_patches_json = default_patches +default_integrations = ( + "https://github.com/revanced/revanced-integrations/releases/latest" +) + class RevancedConfig(object): """Revanced Configurations.""" def __init__(self, env: Env) -> None: - self.app_versions: Dict[str, str] = {} self.env = env self.temp_folder = Path("apks") self.session = Session() self.session.headers["User-Agent"] = "anything" - self.build_extended = env.bool("BUILD_EXTENDED", False) self.apk_mirror = "https://www.apkmirror.com" self.upto_down = [ "spotify", @@ -31,30 +36,9 @@ class RevancedConfig(object): ] self.apk_pure = ["hex-editor", "androidtwelvewidgets"] self.apk_sos = ["expensemanager", "candyvpn"] - self.keystore_name = env.str("KEYSTORE_FILE_NAME", "revanced.keystore") self.ci_test = env.bool("CI_TEST", False) self.apps = env.list("PATCH_APPS", default_build) - self.extended_apps: List[str] = ["youtube", "youtube_music", "microg", "reddit"] - self.rip_libs_apps: List[str] = ["youtube"] - self.normal_cli_jar = "revanced-cli.jar" - self.normal_patches_jar = "revanced-patches.jar" - self.normal_integrations_apk = "revanced-integrations.apk" - self.normal_options_json = "options.json" - self.cli_jar = ( - f"inotia00-{self.normal_cli_jar}" - if self.build_extended - else self.normal_cli_jar - ) - self.patches_jar = ( - f"inotia00-{self.normal_patches_jar}" - if self.build_extended - else self.normal_patches_jar - ) - self.integrations_apk = ( - f"inotia00-{self.normal_integrations_apk}" - if self.build_extended - else self.normal_integrations_apk - ) + self.rip_libs_apps: List[str] = [] self.apk_mirror_urls = { "reddit": f"{self.apk_mirror}/apk/redditinc/reddit/", "twitter": f"{self.apk_mirror}/apk/x-corp/twitter/", @@ -89,11 +73,18 @@ class RevancedConfig(object): key: value + value.split("/")[-2] for key, value in self.apk_mirror_urls.items() } - self.archs_to_build = env.list("ARCHS_TO_BUILD", []) - self.alternative_youtube_patches = env.list("ALTERNATIVE_YOUTUBE_PATCHES", []) - self.alternative_youtube_music_patches = env.list( - "ALTERNATIVE_YOUTUBE_MUSIC_PATCHES", [] - ) self.existing_downloaded_apks = env.list("EXISTING_DOWNLOADED_APKS", []) self.personal_access_token = env.str("PERSONAL_ACCESS_TOKEN", None) self.dry_run = env.bool("DRY_RUN", False) + self.global_cli_dl = env.str("GLOBAL_CLI_DL", default_cli) + self.global_patches_dl = env.str("GLOBAL_PATCHES_DL", default_patches) + self.global_patches_json_dl = env.str( + "GLOBAL_PATCHES_JSON_DL", default_patches_json + ) + self.global_integrations_dl = env.str( + "GLOBAL_INTEGRATIONS_DL", default_integrations + ) + self.global_keystore_name = env.str( + "GLOBAL_KEYSTORE_FILE_NAME", "revanced.keystore" + ) + self.global_archs_to_build = env.list("GLOBAL_ARCHS_TO_BUILD", []) diff --git a/src/downloader/apkmirror.py b/src/downloader/apkmirror.py index 4a80527..d0975af 100644 --- a/src/downloader/apkmirror.py +++ b/src/downloader/apkmirror.py @@ -30,7 +30,6 @@ class ApkMirror(Downloader): "p.notes:nth-child(3) > span:nth-child(1) > a:nth-child(1)" ).attributes["href"] self._download(self.config.apk_mirror + href, f"{app}.apk") - logger.debug("Finished Extracting link and downloading") def get_download_page(self, parser: LexborHTMLParser, main_page: str) -> str: """Function to get the download page in apk_mirror. @@ -67,7 +66,6 @@ class ApkMirror(Downloader): :param version: Version of the application to download :return: Version of downloaded apk """ - logger.debug(f"Trying to download {app},specific version {version}") version = version.replace(".", "-") main_page = f"{self.config.apk_mirror_version_urls.get(app)}-{version}-release/" parser = LexborHTMLParser( @@ -75,7 +73,6 @@ class ApkMirror(Downloader): ) download_page = self.get_download_page(parser, main_page) self.extract_download_link(download_page, app) - logger.debug(f"Downloaded {app} apk from apkmirror_specific_version") def latest_version(self, app: str, **kwargs: Any) -> None: """Function to download whatever the latest version of app from @@ -105,4 +102,3 @@ class ApkMirror(Downloader): parser = LexborHTMLParser(self.config.session.get(main_page).text) download_page = self.get_download_page(parser, main_page) self.extract_download_link(download_page, app) - logger.debug(f"Downloaded {app} apk from apkmirror_specific_version in rt") diff --git a/src/downloader/download.py b/src/downloader/download.py index 0dbe4e3..6960048 100644 --- a/src/downloader/download.py +++ b/src/downloader/download.py @@ -1,5 +1,6 @@ """Downloader Class.""" import os +from pathlib import Path from queue import PriorityQueue from time import perf_counter from typing import Any, Tuple @@ -23,14 +24,20 @@ class Downloader(object): self.config = config self.patcher = patcher - def _download(self, url: str, file_name: str) -> None: - if ( - os.path.exists(self.config.temp_folder.joinpath(file_name)) - or self.config.dry_run - ): + @staticmethod + def file_status_check(file_name: Path, dry_run: bool, url: str) -> bool: + """Check if file already exists.""" + if os.path.exists(file_name) or dry_run: logger.debug( - f"Skipping download of {file_name}. File already exists or dry running." + f"Skipping download of {file_name} from {url}. File already exists or dry running." ) + return True + return False + + def _download(self, url: str, file_name: str) -> None: + if self.file_status_check( + self.config.temp_folder.joinpath(file_name), self.config.dry_run, url + ): return logger.info(f"Trying to download {file_name} from {url}") self._QUEUE_LENGTH += 1 @@ -99,3 +106,7 @@ class Downloader(object): self.specific_version(app, version) else: self.latest_version(app, **kwargs) + + def direct_download(self, dl: str, file_name: str) -> None: + """Download from DL.""" + self._download(dl, file_name) diff --git a/src/downloader/github.py b/src/downloader/github.py index 91046c2..8121959 100644 --- a/src/downloader/github.py +++ b/src/downloader/github.py @@ -1,7 +1,8 @@ """Github Downloader.""" -from typing import Dict +from typing import Dict, List import requests +from lastversion import latest from loguru import logger from src.downloader.download import Downloader @@ -41,3 +42,11 @@ class Github(Downloader): download_url = response.json()["assets"][0]["browser_download_url"] update_changelog(f"{owner}/{repo_name}", response.json()) self._download(download_url, file_name=app) + + @staticmethod + def patch_resource(repo_url: str, assets_filter: str) -> list[str]: + """Fetch patch resource from repo url.""" + latest_resource_version: List[str] = latest( + repo_url, assets_filter=assets_filter, output_format="assets" + ) + return latest_resource_version diff --git a/src/downloader/utils.py b/src/downloader/utils.py index 6a62436..667199b 100644 --- a/src/downloader/utils.py +++ b/src/downloader/utils.py @@ -1,63 +1,4 @@ """Utility class.""" -import os -from concurrent.futures import ThreadPoolExecutor, as_completed -from loguru import logger - -from src.config import RevancedConfig -from src.patches import Patches -from src.utils import PatcherDownloadFailed implement_method = "Please implement the method" - - -def download_revanced(config: RevancedConfig, patcher: Patches) -> None: - """Download Revanced and Extended Patches, Integration and CLI.""" - from src.downloader.factory import DownloaderFactory - - if os.path.exists("changelog.md") and not config.dry_run: - logger.debug("Deleting old changelog.md") - os.remove("changelog.md") - assets = [ - ["revanced", "revanced-cli", config.normal_cli_jar], - ["revanced", "revanced-integrations", config.normal_integrations_apk], - ["revanced", "revanced-patches", config.normal_patches_jar], - ] - if config.build_extended: - assets += [ - ["inotia00", "revanced-cli", config.cli_jar], - ["inotia00", "revanced-integrations", config.integrations_apk], - ["inotia00", "revanced-patches", config.patches_jar], - ] - if ( - "youtube" in config.apps - or "youtube_music" in config.apps - or "microg" in config.apps - ): - if config.build_extended and "microg" in config.apps: - assets += [ - ["inotia00", "mMicroG", "microg.apk"], - ] - else: - assets += [ - ["inotia00", "mMicroG", "microg-output.apk"], - ] - downloader = DownloaderFactory.create_downloader( - app="patches", patcher=patcher, config=config - ) - with ThreadPoolExecutor(7) as executor: - futures = [ - executor.submit( - downloader.download, - version="latest", - app=repo[2], - **{"owner": repo[0], "name": repo[1]}, - ) - for repo in assets - ] - for future in as_completed(futures): - try: - future.result() - except Exception as e: - raise PatcherDownloadFailed(f"An exception occurred: {e}") - logger.info("Downloaded revanced microG ,cli, integrations and patches.") diff --git a/src/parser.py b/src/parser.py index e5694c4..473edd8 100644 --- a/src/parser.py +++ b/src/parser.py @@ -6,9 +6,10 @@ from typing import List from loguru import logger +from src.app import APP from src.config import RevancedConfig from src.patches import Patches -from src.utils import possible_archs, slugify +from src.utils import possible_archs class Parser(object): @@ -70,63 +71,45 @@ class Parser(object): # noinspection IncorrectFormatting def patch_app( self, - app: str, - version: str, - is_experimental: bool = False, - output_prefix: str = "-", + app: APP, ) -> None: """Revanced APP Patcher. :param app: Name of the app - :param version: Version of the application - :param is_experimental: Whether to enable experimental support - :param output_prefix: Prefix to add to the output apks file name """ - cli = self.config.normal_cli_jar - patches = self.config.normal_patches_jar - integrations = self.config.normal_integrations_apk - options = self.config.normal_options_json - if self.config.build_extended and app in self.config.extended_apps: - cli = self.config.cli_jar - patches = self.config.patches_jar - integrations = self.config.integrations_apk args = [ "-jar", - cli, + app.resource["cli"], "-a", - app + ".apk", + app.app_name + ".apk", "-b", - patches, + app.resource["patches"], "-m", - integrations, + app.resource["integrations"], "-o", - f"Re-{app}-{slugify(version)}{output_prefix}output.apk", + app.get_output_file_name(), "--keystore", - self.config.keystore_name, + app.keystore_name, "--options", - options, + "options.json", ] - if is_experimental: + if app.experiment: logger.debug("Using experimental features") args.append("--experimental") - args[1::2] = map(lambda i: self.config.temp_folder.joinpath(i), args[1::2]) + args[1::2] = map(self.config.temp_folder.joinpath, args[1::2]) if self.config.ci_test: self.exclude_all_patches() if self._PATCHES: args.extend(self._PATCHES) - if ( - self.config.build_extended - and len(self.config.archs_to_build) > 0 - and app in self.config.rip_libs_apps - ): - excluded = set(possible_archs) - set(self.config.archs_to_build) + if app.app_name in self.config.rip_libs_apps: + excluded = set(possible_archs) - set(app.archs_to_build) for arch in excluded: args.append("--rip-lib") args.append(arch) start = perf_counter() logger.debug( - f"Sending request to revanced cli for building {app} revanced with args java {args}" + f"Sending request to revanced cli for building with args java {args}" ) process = Popen(["java", *args], stdout=PIPE) output = process.stdout diff --git a/src/patches.py b/src/patches.py index e168b81..35217bb 100644 --- a/src/patches.py +++ b/src/patches.py @@ -1,13 +1,13 @@ """Revanced Patches.""" import json -import subprocess +import os from typing import Any, Dict, List, Tuple from loguru import logger -from requests import Session +from src.app import APP from src.config import RevancedConfig -from src.utils import AppNotFound, handle_response +from src.utils import AppNotFound, PatchesJsonFailed class Patches(object): @@ -50,18 +50,12 @@ class Patches(object): "ml.docilealligator.infinityforreddit": "infinity", "me.ccrama.redditslide": "slide", "com.onelouder.baconreader": "bacon", - } - revanced_app_ids = { - key: (value, "_" + value) for key, value in _revanced_app_ids.items() - } - _revanced_extended_app_ids = { "com.google.android.youtube": "youtube", "com.google.android.apps.youtube.music": "youtube_music", "com.mgoogle.android.gms": "microg", - "com.reddit.frontpage": "reddit", } - revanced_extended_app_ids = { - key: (value, "_" + value) for key, value in _revanced_extended_app_ids.items() + revanced_app_ids = { + key: (value, "_" + value) for key, value in _revanced_app_ids.items() } @staticmethod @@ -69,40 +63,20 @@ class Patches(object): """Return supported apps.""" return Patches._revanced_app_ids - @staticmethod - def check_java(dry_run: bool) -> None: - """Check if Java17 is installed.""" - try: - if dry_run: - return - jd = subprocess.check_output( - ["java", "-version"], stderr=subprocess.STDOUT - ).decode("utf-8") - jd = jd[1:-1] - if "Runtime Environment" not in jd: - raise subprocess.CalledProcessError(-1, "java -version") - if "17" not in jd and "20" not in jd: - raise subprocess.CalledProcessError(-1, "java -version") - logger.debug("Cool!! Java is available") - except subprocess.CalledProcessError: - logger.debug("Java>= 17 Must be installed") - exit(-1) + def scrap_patches(self, file_name: str) -> Any: + """Scrap Patches.""" + if os.path.exists(file_name): + with open(file_name) as f: + patches = json.load(f) + return patches + raise PatchesJsonFailed() # noinspection DuplicatedCode - def fetch_patches(self) -> None: + def fetch_patches(self, config: RevancedConfig, app: APP) -> None: """Function to fetch all patches.""" - session = Session() - if self.config.dry_run: - logger.debug("fetching all patches from local file") - with open("patches.json") as f: - patches = json.load(f) - else: - url = "https://raw.githubusercontent.com/revanced/revanced-patches/main/patches.json" - logger.debug(f"fetching all patches from {url}") - response = session.get(url) - handle_response(response) - patches = response.json() - + patches = self.scrap_patches( + f'{config.temp_folder}/{app.resource["patches_json"]}' + ) for app_name in (self.revanced_app_ids[x][1] for x in self.revanced_app_ids): setattr(self, app_name, []) setattr(self, "universal_patch", []) @@ -122,47 +96,11 @@ class Patches(object): p["app"] = compatible_package p["version"] = version[-1] if version else "all" getattr(self, app_name).append(p) - if self.config.dry_run: - extended_patches = patches - else: - if self.config.build_extended: - url = "https://raw.githubusercontent.com/inotia00/revanced-patches/revanced-extended/patches.json" - else: - url = "https://raw.githubusercontent.com/revanced/revanced-patches/main/patches.json" - response = session.get(url) - handle_response(response) - extended_patches = response.json() - for app_name in ( - self.revanced_extended_app_ids[x][1] for x in self.revanced_extended_app_ids - ): - setattr(self, app_name, []) + n_patches = len(getattr(self, f"_{app.app_name}")) + app.no_of_patches = n_patches - for patch in extended_patches: - for compatible_package, version in [ - (x["name"], x["versions"]) for x in patch["compatiblePackages"] - ]: - if compatible_package in self.revanced_extended_app_ids: - app_name = self.revanced_extended_app_ids[compatible_package][1] - p = {x: patch[x] for x in ["name", "description"]} - p["app"] = compatible_package - p["version"] = version[-1] if version else "all" - getattr(self, app_name).append(p) - - for app_name, app_id in self.revanced_extended_app_ids.values(): - n_patches = len(getattr(self, app_id)) - logger.debug(f"Total patches in {app_name} are {n_patches}") - for app_name, app_id in self.revanced_app_ids.values(): - n_patches = len(getattr(self, app_id)) - logger.debug(f"Total patches in {app_name} are {n_patches}") - n_patches = len(getattr(self, "universal_patch")) - logger.debug(f"Total universal patches are {n_patches}") - - def __init__(self, config: RevancedConfig) -> None: - self.config = config - self.check_java(self.config.dry_run) - self.fetch_patches() - if self.config.dry_run: - self.config.apps = list(self._revanced_app_ids.values()) + def __init__(self, config: RevancedConfig, app: APP) -> None: + self.fetch_patches(config, app) def get(self, app: str) -> Tuple[List[Dict[str, str]], str]: """Get all patches for the given app. @@ -170,11 +108,7 @@ class Patches(object): :param app: Name of the application :return: Patches """ - logger.debug("Getting patches for %s" % app) app_names = {value[0]: value[1] for value in self.revanced_app_ids.values()} - app_names.update( - {value[0]: value[1] for value in self.revanced_extended_app_ids.values()} - ) if not (app_name := app_names.get(app)): raise AppNotFound(app) @@ -182,14 +116,13 @@ class Patches(object): version = "latest" try: version = next(i["version"] for i in patches if i["version"] != "all") - logger.debug(f"Recommended Version for patching {app} is {version}") except StopIteration: pass return patches, version # noinspection IncorrectFormatting def include_exclude_patch( - self, app: str, parser: Any, patches: List[Dict[str, str]] + self, app: APP, parser: Any, patches: List[Dict[str, str]] ) -> None: """Include and exclude patches for a given app. @@ -197,34 +130,20 @@ class Patches(object): :param parser: Parser Obj :param patches: All the patches of a given app """ - if self.config.build_extended and app in self.config.extended_apps: - excluded_patches = self.config.env.list( - f"EXCLUDE_PATCH_{app}_EXTENDED".upper(), [] - ) - included_patches = self.config.env.list( - f"INCLUDE_PATCH_{app}_EXTENDED".upper(), [] - ) - else: - excluded_patches = self.config.env.list(f"EXCLUDE_PATCH_{app}".upper(), []) - included_patches = self.config.env.list(f"INCLUDE_PATCH_{app}".upper(), []) for patch in patches: normalized_patch = patch["name"].lower().replace(" ", "-") parser.include( normalized_patch - ) if normalized_patch not in excluded_patches else parser.exclude( + ) if normalized_patch not in app.exclude_request else parser.exclude( normalized_patch ) - for normalized_patch in included_patches: + for normalized_patch in app.include_request: parser.include(normalized_patch) if normalized_patch not in getattr( self, "universal_patch", [] ) else () - excluded = parser.get_excluded_patches() - if excluded: - logger.debug(f"Excluded patches {excluded} for {app}") - else: - logger.debug(f"No excluded patches for {app}") + logger.info(app) - def get_app_configs(self, app: str) -> Tuple[List[Dict[str, str]], str, bool]: + def get_app_configs(self, app: "APP") -> List[Dict[str, str]]: """Get Configurations for a given app. :param app: Name of the application @@ -232,15 +151,15 @@ class Patches(object): experimental """ experiment = False - total_patches, recommended_version = self.get(app=app) - env_version = self.config.env.str(f"{app}_VERSION".upper(), None) - if env_version: - logger.debug(f"Picked {app} version {env_version} from env.") + total_patches, recommended_version = self.get(app=app.app_name) + if app.app_version: + logger.debug(f"Picked {app} version {app.app_version:} from env.") if ( - env_version == "latest" - or env_version > recommended_version - or env_version < recommended_version + app.app_version == "latest" + or app.app_version > recommended_version + or app.app_version < recommended_version ): experiment = True - recommended_version = env_version - return total_patches, recommended_version, experiment + recommended_version = app.app_version + app.set_recommended_version(recommended_version, experiment) + return total_patches diff --git a/src/utils.py b/src/utils.py index 1cb7ad0..9e0ed76 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,5 +1,6 @@ """Utilities.""" import re +import subprocess from typing import Dict from loguru import logger @@ -50,6 +51,12 @@ class PatcherDownloadFailed(Exception): pass +class PatchesJsonFailed(ValueError): + """Patches failed.""" + + pass + + def handle_response(response: Response) -> None: """Handle Get Request Response.""" response_code = response.status_code @@ -76,3 +83,22 @@ def slugify(string: str) -> str: string = string.strip("-") return string + + +def check_java(dry_run: bool) -> None: + """Check if Java>=17 is installed.""" + try: + if dry_run: + return + jd = subprocess.check_output( + ["java", "-version"], stderr=subprocess.STDOUT + ).decode("utf-8") + jd = jd[1:-1] + if "Runtime Environment" not in jd: + raise subprocess.CalledProcessError(-1, "java -version") + if "17" not in jd and "20" not in jd: + raise subprocess.CalledProcessError(-1, "java -version") + logger.debug("Cool!! Java is available") + except subprocess.CalledProcessError: + logger.debug("Java>= 17 Must be installed") + exit(-1)