add support for Back to Basics scraping

Author: noisecode3

CMakeLists.txt

@@ -155,7 +155,9 @@ if(NOT TEST)
         ${CMAKE_SOURCE_DIR}/database/data_factory.py
         ${CMAKE_SOURCE_DIR}/database/get_leaf_cert.py
         ${CMAKE_SOURCE_DIR}/database/https.py
-        ${CMAKE_SOURCE_DIR}/database/scrape.py
+        ${CMAKE_SOURCE_DIR}/database/scrape_common.py
+        ${CMAKE_SOURCE_DIR}/database/scrape_trle.py
+        ${CMAKE_SOURCE_DIR}/database/scrape_trle_download.py
         ${CMAKE_SOURCE_DIR}/database/tombll_add_data.py
         ${CMAKE_SOURCE_DIR}/database/tombll_get_data.py
         ${CMAKE_SOURCE_DIR}/database/tombll_get_list.py

README.md

@@ -30,7 +30,9 @@ sudo pacman -S qt5-wayland qt5-webengine qt5-imageformats boost
 
 ### Build
 ```shell
-cmake -DCMAKE_INSTALL_PREFIX=~/.local .
+mkdir build
+cd build
+cmake -DCMAKE_INSTALL_PREFIX=/home/$USER/.local ..
 make install -j$(nproc)
 ```
 
@@ -85,17 +87,20 @@ sudo apk add py3-pycurl py3-tqdm py3-cryptography py3-beautifulsoup4 py3-pillow
 ```
 
 Some levels wont be added because they use external or different download URL's
-You can add maps to the database if you cd into where you installed you're database.
+You can add levels to the database if you cd into where you installed you're database.
+From you're browser copy the number from level page https://www.trle.net/sc/levelfeatures.php?lid=3684
 
 If you did just follow the command above you can use:
 
 
 ```shell
-python3 tombll_get_data.py https://www.trle.net/sc/levelfeatures.php?lid=3684
+python3 tombll_get_data.py 3684
 
 ```
 Now that you have an data.json file you get a chance to edit it.
 Sometimes you need or want to edit those but right now I allow only trle.net
+You can add you're own local file with its md5sum and it should not try to download it.
+But it has to be a zip file at this point.
 
 ```text
   "zipFileName": "",

database/https.py

@@ -74,7 +74,8 @@ def validate_url(self, url):
             "https://www.trle.net/",
             "https://trle.net/",
             "https://trcustoms.org/",
-            "https://data.trcustoms.org/"
+            "https://data.trcustoms.org/",
+            "https://www.trlevel.de/"
         )
 
         if any(url.startswith(domain) for domain in allowed_domains):
@@ -160,6 +161,9 @@ def get_leaf(self, url):
 
     def setup_before_get_response(self, url, content_type):
         """Validate known URL and content type."""
+        if not url:
+            logging.error("Faild setup_before_get_response, url is None. Exiting.")
+            sys.exit(1)
         self.validate_url(url)
         self.validate_data_type(content_type)
 
@@ -170,6 +174,12 @@ def get_response(self, url, content_type):
         """Handle all https requests."""
         self.setup_before_get_response(url, content_type)
 
+        if url.startswith("https://trcustoms.org"):
+            self.misconfigured_server = False
+
+        if url.startswith("https://www.trlevel.de"):
+            self.misconfigured_server = False
+
         if content_type == 'application/zip':
             return DOWNLOADER.download_file(url)
 
@@ -321,7 +331,7 @@ def progress_callback(self, total_to_download, downloaded, total_to_upload, uplo
             self.progress_bar.total = total_to_download
         return 0  # Returning 0 means to continue
 
-    def download_file_trle(self, url):
+    def download_file(self, url):
         """
         Download a file from the trle URL and stores its contents in a buffer.
 
@@ -430,147 +440,7 @@ def download_file_trle(self, url):
 
         return zip_file  # Always return the zip_file dictionary
 
-    def download_file_trcustoms(self, url):
-        """
-        Download a file from the trcustoms URL and stores its contents in a buffer.
-
-        This method utilizes the `pycurl` library to perform the download, providing
-        a progress bar for user feedback. It handles server JSON API, follows redirects,
-        and calculates the MD5 checksum of the downloaded file.
-
-        Parameters:
-        ----------
-        url : str
-            The URL of the file to download. Must be a valid URL.
-
-        Raises:
-        -------
-        SystemExit
-            Exits the program if the server is misconfigured and no leaf certificate is available.
-
-        Exceptions:
-        ------------
-        pycurl.error
-            Raised if an error occurs during the download process.
 
-        Returns:
-        --------
-        dict
-            Returns a dictionary containing details of the downloaded file, including:
-            - 'size': Size of the file in MiB (mebibytes).
-            - 'url': The effective URL from which the file was downloaded.
-            - 'name': The name of the file.
-            - 'md5': The MD5 checksum of the downloaded content.
-
-        Notes:
-        ------
-        - The progress bar is displayed using the `tqdm` library to indicate the download status.
-        - The method checks the HTTP response code after the download to ensure success (HTTP 200).
-        - Temporary files created for certificate handling are cleaned up after the download.
-        """
-        curl = pycurl.Curl()
-        zip_file = data_factory.make_zip_file()  # Initialize the zip_file dictionary
-
-        REQUEST_HANDLER.misconfigured_server = False
-        level_id = url.split("https://trcustoms.org/levels/")[1]
-        print(f'https://trcustoms.org/api/levels/{level_id}/')
-        level_json = get(f'https://trcustoms.org/api/levels/{level_id}/', 'application/json')
-        if not isinstance(level_json, dict):
-            sys.exit(1)
-        level_files = level_json.get('files', [])
-        level_last_file = level_files[-1]
-        url = level_last_file.get('url', '')
-
-        try:
-            # Get header info
-            total_size = level_last_file.get('size', '')
-            zip_file['size'] = round(total_size / (1024 * 1024), 2)  # Size in MiB
-            zip_file['url'] = level_last_file.get('url', '')
-            level_version = level_last_file.get('version', '')
-            zip_file['name'] = f"{level_id}-title-V{level_version}-authors.zip"
-
-            # Set up for download
-            curl.setopt(pycurl.URL, url)
-            curl.setopt(pycurl.FOLLOWLOCATION, True)
-            curl.setopt(pycurl.WRITEFUNCTION, self.write_callback)
-            curl.setopt(pycurl.WRITEDATA, self.buffer)
-
-            # Enable progress meter
-            self.progress_bar = tqdm(total=total_size,
-                                     unit='B',
-                                     unit_scale=True,
-                                     unit_divisor=1024,
-                                     desc="Downloading")
-
-            curl.setopt(pycurl.NOPROGRESS, False)
-            curl.setopt(pycurl.XFERINFOFUNCTION, self.progress_callback)
-
-            # Perform the download
-            curl.perform()
-
-            # Check for errors
-            http_code = curl.getinfo(pycurl.RESPONSE_CODE)
-            if http_code != 200:
-                self.status = 1
-                print(f"Error: HTTP response code {http_code}")
-                return {}  # Return an empty dict on error
-
-            self.status = 0
-
-            # Finalize MD5 checksum
-            md5_hash = hashlib.md5(usedforsecurity=False)
-            self.buffer.seek(0)  # Reset buffer pointer
-            md5_hash.update(self.buffer.getvalue())
-            zip_file['md5'] = md5_hash.hexdigest()
-
-        except pycurl.error as e:
-            self.status = 1
-            print(f"Error: {e}")
-            return {}  # Return an empty dict on error
-
-        finally:
-            if self.progress_bar:
-                self.progress_bar.close()
-            curl.close()
-            REQUEST_HANDLER.misconfigured_server = True
-
-        return zip_file  # Always return the zip_file dictionary
-
-    def download_file(self, url):
-        """
-        Branch the download from an trle.net redirect link to trcustoms or trle.
-
-        This method checks the head for the domain the redirection leads to.
-
-        Parameters:
-        ----------
-        url : str
-            The URL of the file to download. Must be a valid URL.
-
-        Returns:
-        --------
-        dict
-            Returns a dictionary containing details of the downloaded file, including:
-            - 'size': Size of the file in MiB (mebibytes).
-            - 'url': The effective URL from which the file was downloaded.
-            - 'name': The name of the file.
-            - 'md5': The MD5 checksum of the downloaded content.
-
-        """
-        head_string = REQUEST_HANDLER.head(url)
-        redirect_url = head_string.partition("Location: ")[2].split("\r\n", 1)[0]
-
-        if redirect_url.startswith("https://trcustoms.org/levels/"):
-            print("trcustoms.org")
-            return self.download_file_trcustoms(redirect_url)
-        if redirect_url.startswith("https://www.trle.net/levels/"):
-            print("trle.net")
-            return self.download_file_trle(url)
-        if redirect_url.startswith("https://trle.net/levels/"):
-            print("trle.net")
-            return self.download_file_trle(url)
-
-        return data_factory.make_zip_file()
 
 
 ACQUIRE_LOCK = AcquireLock()

database/ideas.txt

@@ -288,7 +288,7 @@ we can get those also so there will be 3 hosts but they use rar
 still the app should support rar also at some point
 
 we need to support this kind of download link also
-https://www.trle.net/levels/levels/2020/0620/BtB
+https://www.trle.net/levels/levels/2020/0620/BtB/Web/index.html
 
 we have typechecking lint, so we need to install like this:
 pip install types-tqdm