From d35f93ef77f5c839fc725901d5470aafd8d2796a Mon Sep 17 00:00:00 2001 From: Nikhil Badyal <59223300+nikhilbadyal@users.noreply.github.com> Date: Tue, 15 Aug 2023 16:26:08 +0530 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=A8=20Extracted=20common=20parser=20st?= =?UTF-8?q?ring=20(#285)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/status_check.py | 5 +++-- src/downloader/uptodown.py | 3 ++- src/utils.py | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/status_check.py b/scripts/status_check.py index a82e836..8ee771c 100644 --- a/scripts/status_check.py +++ b/scripts/status_check.py @@ -12,6 +12,7 @@ from src.patches import Patches from src.utils import ( apk_mirror_base_url, apkmirror_status_check, + bs4_parser, handle_github_response, ) @@ -28,7 +29,7 @@ def apkcombo_scrapper(package_name: str) -> str: try: apkcombo_url = f"https://apkcombo.com/genericApp/{package_name}" r = requests.get(apkcombo_url, headers=headers, allow_redirects=True) - soup = BeautifulSoup(r.text, "html.parser") + soup = BeautifulSoup(r.text, bs4_parser) url = soup.select_one("div.avatar > img")["data-src"] return re.sub(r"=.*$", "", url) except Exception: @@ -41,7 +42,7 @@ def apkmirror_scrapper(package_name: str) -> str: if response["data"][0]["exists"]: search_url = f"{apk_mirror_base_url}/?s={package_name}" r = requests.get(search_url, headers=headers) - soup = BeautifulSoup(r.text, "html.parser") + soup = BeautifulSoup(r.text, bs4_parser) sub_url = soup.select_one("div.bubble-wrap > img")["src"] new_width = 500 new_height = 500 diff --git a/src/downloader/uptodown.py b/src/downloader/uptodown.py index 15c4e1b..b24ee79 100644 --- a/src/downloader/uptodown.py +++ b/src/downloader/uptodown.py @@ -7,6 +7,7 @@ from selectolax.lexbor import LexborHTMLParser from src.downloader.download import Downloader from src.exceptions import AppNotFound +from src.utils import bs4_parser class UptoDown(Downloader): @@ -31,7 +32,7 @@ class UptoDown(Downloader): f"https://{self.config.upto_down.get(app)}.en.uptodown.com/android/versions" ) html = self.config.session.get(url).text - soup = BeautifulSoup(html, "html.parser") + soup = BeautifulSoup(html, bs4_parser) versions_list = soup.find("section", {"id": "versions"}) download_url = None for version_item in versions_list.find_all("div", {"data-url": True}): diff --git a/src/utils.py b/src/utils.py index 88106ed..1a0c6fa 100644 --- a/src/utils.py +++ b/src/utils.py @@ -23,6 +23,7 @@ apk_mirror_header = { "Authorization": "Basic YXBpLWFwa3VwZGF0ZXI6cm01cmNmcnVVakt5MDRzTXB5TVBKWFc4", "Content-Type": "application/json", } +bs4_parser = "html.parser" def update_changelog(name: str, response: Dict[str, str]) -> None: