Multi Source patch

This commit is contained in:
Nikhil Badyal
2025-06-19 22:31:46 +05:30
committed by Nikhil Badyal
parent a2ba0b89d1
commit 4283925f3e
6 changed files with 177 additions and 74 deletions
+74 -24
View File
@@ -31,10 +31,17 @@ class APP(object):
self.app_version = config.env.str(f"{app_name}_VERSION".upper(), None)
self.experiment = False
self.cli_dl = config.env.str(f"{app_name}_CLI_DL".upper(), config.global_cli_dl)
self.patches_dl = config.env.str(f"{app_name}_PATCHES_DL".upper(), config.global_patches_dl)
# Support multiple patch bundles via comma-separated URLs
patches_dl_raw = config.env.str(f"{app_name}_PATCHES_DL".upper(), config.global_patches_dl)
self.patches_dl_list = [url.strip() for url in patches_dl_raw.split(",") if url.strip()]
# Keep backward compatibility
self.patches_dl = patches_dl_raw
self.exclude_request: list[str] = config.env.list(f"{app_name}_EXCLUDE_PATCH".upper(), [])
self.include_request: list[str] = config.env.list(f"{app_name}_INCLUDE_PATCH".upper(), [])
self.resource: dict[str, dict[str, str]] = {}
self.patch_bundles: list[dict[str, str]] = [] # Store multiple patch bundles
self.no_of_patches: int = 0
self.keystore_name = config.env.str(f"{app_name}_KEYSTORE_FILE_NAME".upper(), config.global_keystore_name)
self.archs_to_build = config.env.list(f"{app_name}_ARCHS_TO_BUILD".upper(), config.global_archs_to_build)
@@ -113,18 +120,18 @@ class APP(object):
----------
url : str
The `url` parameter is a string that represents the URL of the resource you want to download.
It can be a URL from GitHub or a local file URL.
It can be a URL from GitHub or a local file URL.
config : RevancedConfig
The `config` parameter is an instance of the `RevancedConfig` class. It is used to provide
configuration settings for the download process.
configuration settings for the download process.
assets_filter : str
The `assets_filter` parameter is a string that is used to filter the assets to be downloaded
from a GitHub repository. It is used when the `url` parameter starts with "https://github". The
`assets_filter` string is matched against the names of the assets in the repository, and only
file_name : str
from a GitHub repository. It is used when the `url` parameter starts with "https://github". The
`assets_filter` string is matched against the names of the assets in the repository, and only
file_name : str
The `file_name` parameter is a string that represents the name of the file that will be
downloaded. If no value is provided for `file_name`, the function will generate a filename based
on the URL of the file being downloaded.
downloaded. If no value is provided for `file_name`, the function will generate a filename based
on the URL of the file being downloaded.
Returns
-------
@@ -148,6 +155,58 @@ class APP(object):
Downloader(config).direct_download(url, file_name)
return tag, file_name
def _setup_download_tasks(self: Self) -> list[tuple[str, str, None, str]]:
"""Setup download tasks for CLI and patch bundles."""
download_tasks = [
("cli", self.cli_dl, None, ".*jar"),
]
# Download multiple patch bundles
for i, patches_url in enumerate(self.patches_dl_list):
bundle_name = f"patches_{i}" if len(self.patches_dl_list) > 1 else "patches"
download_tasks.append((bundle_name, patches_url, None, ".*rvp"))
return download_tasks
def _handle_cached_resource(self: Self, resource_name: str, tag: str, file_name: str) -> None:
"""Handle cached resource and update appropriate data structures."""
if resource_name.startswith("patches"):
self.patch_bundles.append(
{
"name": resource_name,
"file_name": file_name,
"version": tag,
},
)
# Keep backward compatibility for single bundle
if resource_name == "patches" or len(self.patches_dl_list) == 1:
self.resource["patches"] = {
"file_name": file_name,
"version": tag,
}
else:
self.resource[resource_name] = {
"file_name": file_name,
"version": tag,
}
def _handle_downloaded_resource(
self: Self,
resource_name: str,
tag: str,
file_name: str,
download_tasks: list[tuple[str, str, RevancedConfig, str]],
resource_cache: dict[str, tuple[str, str]],
) -> None:
"""Handle newly downloaded resource and update cache."""
self._handle_cached_resource(resource_name, tag, file_name)
# Update cache for the corresponding URL
for task_name, task_url, _, _ in download_tasks:
if task_name == resource_name:
resource_cache[task_url.strip()] = (tag, file_name)
break
def download_patch_resources(
self: Self,
config: RevancedConfig,
@@ -159,14 +218,15 @@ class APP(object):
----------
config : RevancedConfig
The `config` parameter is an instance of the `RevancedConfig` class. It is used to provide
configuration settings for the resource download tasks.
configuration settings for the resource download tasks.
resource_cache: dict[str, tuple[str, str]]
"""
logger.info("Downloading resources for patching.")
download_tasks = [
("cli", self.cli_dl, config, ".*jar"),
("patches", self.patches_dl, config, ".*rvp"),
base_tasks = self._setup_download_tasks()
# Update download tasks with config
download_tasks: list[tuple[str, str, RevancedConfig, str]] = [
(name, url, config, filter_pattern) for name, url, _, filter_pattern in base_tasks
]
with ThreadPoolExecutor(1) as executor:
@@ -177,10 +237,7 @@ class APP(object):
if url in resource_cache:
logger.info(f"Skipping {resource_name} download, using cached resource: {url}")
tag, file_name = resource_cache[url]
self.resource[resource_name] = {
"file_name": file_name,
"version": tag,
}
self._handle_cached_resource(resource_name, tag, file_name)
continue
futures[resource_name] = executor.submit(self.download, url, cfg, assets_filter)
@@ -190,14 +247,7 @@ class APP(object):
for resource_name, future in futures.items():
try:
tag, file_name = future.result()
self.resource[resource_name] = {
"file_name": file_name,
"version": tag,
}
resource_cache[download_tasks[["cli", "patches"].index(resource_name)][1].strip()] = (
tag,
file_name,
)
self._handle_downloaded_resource(resource_name, tag, file_name, download_tasks, resource_cache)
except BuilderError as e:
msg = f"Failed to download {resource_name} resource."
raise PatchingFailedError(msg) from e