From 1ea1de8086c05cc4e6350d23628d034ff43c92d2 Mon Sep 17 00:00:00 2001 From: Nikhil Badyal Date: Sun, 25 Dec 2022 12:48:43 +0530 Subject: [PATCH 01/11] =?UTF-8?q?=E2=9C=A8=20Ability=20to=20provide=20GitH?= =?UTF-8?q?ub=20Access=20Token=20to=20bypass=20GitHub=20Rate=20Limit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 7 ++++++- src/config.py | 1 + src/downloader.py | 36 ++++++++++++++++++++++++++---------- src/patches.py | 12 +++++++----- src/utils.py | 10 ++++++++++ 5 files changed, 50 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index f3f8706..3a45a96 100644 --- a/README.md +++ b/README.md @@ -257,7 +257,12 @@ By default, script build the version as recommended by Revanced team. `/apks` folder. Name of the downloaded apk must match with the available app choices found [here.](#note) -14. Sample Envs
+14. If you run script again & again. You might hit GitHub API limit. In that case you can provide your Personal + GitHub Access Token in `.env` file or in `ENVS` in `GitHub secrets` (Recommended) in the format - + ```dotenv + PERSONAL_ACCESS_TOKEN= + ``` +15. Sample Envs
Thanks to [@aliharslan0](https://github.com/aliharslan0/pyrevanced) for his work. diff --git a/src/config.py b/src/config.py index f6a24fc..95a7fc5 100644 --- a/src/config.py +++ b/src/config.py @@ -84,3 +84,4 @@ class RevancedConfig: "ALTERNATIVE_YOUTUBE_MUSIC_PATCHES", [] ) self.existing_downloaded_apks = env.list("EXISTING_DOWNLOADED_APKS", []) + self.personal_access_token = env.str("PERSONAL_ACCESS_TOKEN", None) diff --git a/src/downloader.py b/src/downloader.py index 809f11d..d63f232 100644 --- a/src/downloader.py +++ b/src/downloader.py @@ -14,7 +14,7 @@ from tqdm import tqdm from src.config import RevancedConfig from src.patches import Patches -from src.utils import AppNotFound, update_changelog +from src.utils import AppNotFound, handle_response, update_changelog class Downloader(object): @@ -32,8 +32,18 @@ class Downloader(object): logger.debug(f"Trying to download {file_name} from {url}") self._QUEUE_LENGTH += 1 start = perf_counter() - resp = self.config.session.get(url, stream=True) - total = int(resp.headers.get("content-length", 0)) + headers = {} + if self.config.personal_access_token: + headers.update( + {"Authorization": "token " + self.config.personal_access_token} + ) + response = self.config.session.get( + url, + stream=True, + headers=headers, + ) + handle_response(response) + total = int(response.headers.get("content-length", 0)) bar = tqdm( desc=file_name, total=total, @@ -43,7 +53,7 @@ class Downloader(object): colour="green", ) with self.config.temp_folder.joinpath(file_name).open("wb") as dl_file, bar: - for chunk in resp.iter_content(self._CHUNK_SIZE): + for chunk in response.iter_content(self._CHUNK_SIZE): size = dl_file.write(chunk) bar.update(size) self._QUEUE.put((perf_counter() - start, file_name)) @@ -197,14 +207,20 @@ class Downloader(object): """ logger.debug(f"Trying to download {name} from github") repo_url = f"https://api.github.com/repos/{owner}/{name}/releases/latest" - r = requests.get( - repo_url, headers={"Content-Type": "application/vnd.github.v3+json"} - ) + headers = { + "Content-Type": "application/vnd.github.v3+json", + } + if self.config.personal_access_token: + headers.update( + {"Authorization": "token " + self.config.personal_access_token} + ) + response = requests.get(repo_url, headers=headers) + handle_response(response) if name == "revanced-patches": - download_url = r.json()["assets"][1]["browser_download_url"] + download_url = response.json()["assets"][1]["browser_download_url"] else: - download_url = r.json()["assets"][0]["browser_download_url"] - update_changelog(f"{owner}/{name}", r.json()) + download_url = response.json()["assets"][0]["browser_download_url"] + update_changelog(f"{owner}/{name}", response.json()) self._download(download_url, file_name=file_name) def download_revanced(self) -> None: diff --git a/src/patches.py b/src/patches.py index 48b5edb..eaa6b84 100644 --- a/src/patches.py +++ b/src/patches.py @@ -6,7 +6,7 @@ from loguru import logger from requests import Session from src.config import RevancedConfig -from src.utils import AppNotFound +from src.utils import AppNotFound, handle_response class Patches(object): @@ -60,10 +60,11 @@ class Patches(object): session = Session() logger.debug("fetching all patches") - resp = session.get( + response = session.get( "https://raw.githubusercontent.com/revanced/revanced-patches/main/patches.json" ) - patches = resp.json() + handle_response(response) + patches = response.json() for app_name in (self.revanced_app_ids[x][1] for x in self.revanced_app_ids): setattr(self, app_name, []) @@ -83,8 +84,9 @@ class Patches(object): else: url = "https://raw.githubusercontent.com/revanced/revanced-patches/main/patches.json" - resp_extended = session.get(url) - extended_patches = resp_extended.json() + response = session.get(url) + handle_response(response) + extended_patches = response.json() for app_name in ( self.revanced_extended_app_ids[x][1] for x in self.revanced_extended_app_ids ): diff --git a/src/utils.py b/src/utils.py index e66c109..56e6dde 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,6 +1,9 @@ """Utilities.""" from typing import Dict +from loguru import logger +from requests import Response + default_build = [ "youtube", ] @@ -55,3 +58,10 @@ class AppNotFound(ValueError): """Not a valid Revanced App.""" pass + + +def handle_response(response: Response) -> None: + """Handle Get Request Response.""" + response_code = response.status_code + if response_code != 200: + logger.info(response.text) From e0afe1596caf405f3271ec8e829fdee9e26ad665 Mon Sep 17 00:00:00 2001 From: Nikhil Badyal Date: Sun, 25 Dec 2022 12:55:41 +0530 Subject: [PATCH 02/11] =?UTF-8?q?=E2=9C=A8=20Ability=20debug=20log?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/downloader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/downloader.py b/src/downloader.py index d63f232..5216ded 100644 --- a/src/downloader.py +++ b/src/downloader.py @@ -34,6 +34,7 @@ class Downloader(object): start = perf_counter() headers = {} if self.config.personal_access_token: + logger.debug("Using personal access token") headers.update( {"Authorization": "token " + self.config.personal_access_token} ) @@ -211,6 +212,7 @@ class Downloader(object): "Content-Type": "application/vnd.github.v3+json", } if self.config.personal_access_token: + logger.debug("Using personal access token") headers.update( {"Authorization": "token " + self.config.personal_access_token} ) From 60b6f0d82f45a87e72e2b7b0f7b02c886b6eab7a Mon Sep 17 00:00:00 2001 From: Nikhil Badyal Date: Sun, 25 Dec 2022 13:06:54 +0530 Subject: [PATCH 03/11] =?UTF-8?q?=E2=9C=A8=20Raise=20Error=20instead=20of?= =?UTF-8?q?=20exiting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/downloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/downloader.py b/src/downloader.py index 5216ded..5845ab8 100644 --- a/src/downloader.py +++ b/src/downloader.py @@ -187,7 +187,7 @@ class Downloader(object): match = re.search(r"\d", main_page) if not match: logger.error("Cannot find app main page") - sys.exit(-1) + raise AppNotFound() int_version = match.start() extra_release = main_page.rfind("release") - 1 version: str = main_page[int_version:extra_release] From cf4bfe63534cb1154847c38995e20e5da65bb9ec Mon Sep 17 00:00:00 2001 From: Nikhil Badyal Date: Sun, 25 Dec 2022 20:10:42 +0530 Subject: [PATCH 04/11] =?UTF-8?q?=F0=9F=99=88=20Updated=20gitignore?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 22f64d6..d7bb86a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ venv */__pycache__* *.pyc /revanced-cache/ +changelog.md From 7f945c4d5fd82403c8da9130a9783f4b041463c6 Mon Sep 17 00:00:00 2001 From: Nikhil Badyal Date: Sun, 25 Dec 2022 20:12:13 +0530 Subject: [PATCH 05/11] =?UTF-8?q?=F0=9F=91=B7=20Added=20Tmate=20Debug=20se?= =?UTF-8?q?ssion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build-apk.yml | 6 ++++++ .github/workflows/build-artifact.yml | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/.github/workflows/build-apk.yml b/.github/workflows/build-apk.yml index a4ebfbe..5effabc 100644 --- a/.github/workflows/build-apk.yml +++ b/.github/workflows/build-apk.yml @@ -36,6 +36,11 @@ on: type: boolean required: false default: true + DEBUG_ENABLED: + type: boolean + description: 'Run the build with tmate debugging enabled.' + required: false + default: false concurrency: group: ${{ github.head_ref || github.run_id }} @@ -45,6 +50,7 @@ jobs: uses: ./.github/workflows/build-artifact.yml with: COMMIT_CHANGELOG: ${{ inputs.COMMIT_CHANGELOG }} + DEBUG_ENABLED: ${{ inputs.DEBUG_ENABLED }} secrets: ENVS: ${{ secrets.ENVS }} diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index 9b252f9..a8a1ecc 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -22,6 +22,11 @@ on: type: boolean required: false default: false + DEBUG_ENABLED: + type: boolean + description: 'Run the build with tmate debugging enabled.' + required: false + default: false jobs: build-apk: @@ -37,6 +42,10 @@ jobs: run: | echo "${{ secrets.ENVS }}" >> .env + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + if: ${{ github.event_name == 'workflow_dispatch' && inputs.DEBUG_ENABLED }} + - name: Build Revanced APKs run: | docker-compose up From 69a9a2e13ed28aaa873aa049814e0adc98ef60b1 Mon Sep 17 00:00:00 2001 From: Nikhil Badyal Date: Sun, 25 Dec 2022 20:23:19 +0530 Subject: [PATCH 06/11] =?UTF-8?q?=F0=9F=90=9B=20Use=20PAT=20only=20for=20G?= =?UTF-8?q?itHub?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/downloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/downloader.py b/src/downloader.py index 5845ab8..649125a 100644 --- a/src/downloader.py +++ b/src/downloader.py @@ -33,7 +33,7 @@ class Downloader(object): self._QUEUE_LENGTH += 1 start = perf_counter() headers = {} - if self.config.personal_access_token: + if self.config.personal_access_token and "github" in url: logger.debug("Using personal access token") headers.update( {"Authorization": "token " + self.config.personal_access_token} From fd0df79d12d3cdfdf1349dd06ad3d1fad776a2c9 Mon Sep 17 00:00:00 2001 From: Nikhil Badyal Date: Sun, 25 Dec 2022 20:23:38 +0530 Subject: [PATCH 07/11] =?UTF-8?q?=F0=9F=90=9B=20Run=20Local=20Docker-compo?= =?UTF-8?q?se=20in=20case=20of=20PR?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build-artifact.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index a8a1ecc..eec751e 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -48,7 +48,13 @@ jobs: - name: Build Revanced APKs run: | - docker-compose up + if [ "${{ inputs.DEBUG_ENABLED }}" == "-1" ]; then + echo "Normal Build" + docker-compose up + else + echo "Local Build" + docker-compose -f docker-compose-local.yml up + fi - name: Upload Build APKS uses: nikhilbadyal/upload-artifact@count-check From 37ae1fc11d5e2908c97f2936892b7204c095b8c7 Mon Sep 17 00:00:00 2001 From: Nikhil Badyal Date: Sun, 25 Dec 2022 20:26:39 +0530 Subject: [PATCH 08/11] =?UTF-8?q?=F0=9F=91=B7=20Pass=20GitHub=20Token=20fo?= =?UTF-8?q?r=20PR?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pr-checks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 611e549..3e765e8 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -20,3 +20,4 @@ jobs: ENVS: | CI_TEST=True ES_JAVA_OPTS: '-Xms4g -Xmx4g' + PERSONAL_ACCESS_TOKEN=${{ secrets.GITHUB_TOKEN }} From ff0b1c95c3d0dac8200555bca463d1a7862571af Mon Sep 17 00:00:00 2001 From: Nikhil Badyal Date: Sun, 25 Dec 2022 20:28:36 +0530 Subject: [PATCH 09/11] =?UTF-8?q?=F0=9F=91=B7=20Always=20Build=20Image=20i?= =?UTF-8?q?n=20PR=20checks=20[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build-artifact.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index eec751e..1786c4d 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -53,7 +53,7 @@ jobs: docker-compose up else echo "Local Build" - docker-compose -f docker-compose-local.yml up + docker-compose -f docker-compose-local.yml up --build fi - name: Upload Build APKS From c6636f4e7df89f0549cd8eb883bc2c4d6e43ee53 Mon Sep 17 00:00:00 2001 From: Nikhil Badyal Date: Sun, 25 Dec 2022 20:30:49 +0530 Subject: [PATCH 10/11] =?UTF-8?q?=F0=9F=92=9A=20Fixed=20variable=20name=20?= =?UTF-8?q?[skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build-artifact.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index 1786c4d..9501c3a 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -48,7 +48,7 @@ jobs: - name: Build Revanced APKs run: | - if [ "${{ inputs.DEBUG_ENABLED }}" == "-1" ]; then + if [ "${{ inputs.FILES_TO_EXPECT }}" == "-1" ]; then echo "Normal Build" docker-compose up else From 8b9ac5d444b3d119fca53646566432dd04757b9d Mon Sep 17 00:00:00 2001 From: Nikhil Badyal Date: Sun, 25 Dec 2022 20:41:48 +0530 Subject: [PATCH 11/11] =?UTF-8?q?=F0=9F=92=9A=20Always=20build=20image=20[?= =?UTF-8?q?skip=20ci]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build-artifact.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index 9501c3a..ff0c05e 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -48,13 +48,7 @@ jobs: - name: Build Revanced APKs run: | - if [ "${{ inputs.FILES_TO_EXPECT }}" == "-1" ]; then - echo "Normal Build" - docker-compose up - else - echo "Local Build" - docker-compose -f docker-compose-local.yml up --build - fi + docker-compose up --build - name: Upload Build APKS uses: nikhilbadyal/upload-artifact@count-check