Added key to async and fixed requests

Author: ScrappyCocco

.vscode/settings.json

@@ -4,4 +4,4 @@
     ],
     "python.testing.unittestEnabled": false,
     "python.testing.pytestEnabled": true
-}
+}

howlongtobeatpy/howlongtobeatpy/HTMLRequests.py

@@ -116,8 +116,9 @@ async def send_async_web_request(game_name: str, search_modifiers: SearchModifie
         headers = HTMLRequests.get_search_request_headers()
         payload = HTMLRequests.get_search_request_data(game_name, search_modifiers, page)
         # Make the post request and return the result if is valid
+        search_url_with_key = HTMLRequests.SEARCH_URL + "/" + HTMLRequests.SEARCH_API_KEY
         async with aiohttp.ClientSession() as session:
-            async with session.post(HTMLRequests.SEARCH_URL, headers=headers, data=payload) as resp:
+            async with session.post(search_url_with_key, headers=headers, data=payload) as resp:
                 if resp is not None and str(resp.status) == "200":
                     return await resp.text()
                 return None
@@ -226,3 +227,38 @@ def send_website_request_getcode(parse_all_scripts: bool):
                         for match in matches:
                             return match
         return None
+    
+    @staticmethod
+    async def async_send_website_request_getcode(parse_all_scripts: bool):
+        """
+        Function that send a request to howlongtobeat to scrape the /api/search key
+        @return: The string key to use on /api/search
+        """
+        # Make the post request and return the result if is valid
+        headers = HTMLRequests.get_title_request_headers()
+        async with aiohttp.ClientSession() as session:
+            async with session.get(HTMLRequests.BASE_URL, headers=headers) as resp:
+                if resp is not None and str(resp.status) == "200":
+                    resp_text = await resp.text()
+                    # Parse the HTML content using BeautifulSoup
+                    soup = BeautifulSoup(resp_text, 'html.parser')
+                    # Find all <script> tags with a src attribute containing the substring
+                    scripts = soup.find_all('script', src=True)
+                    if parse_all_scripts:
+                        matching_scripts = [script['src'] for script in scripts]
+                    else:
+                        matching_scripts = [script['src'] for script in scripts if '_app-' in script['src']]
+                    for script_url in matching_scripts:
+                        script_url = HTMLRequests.BASE_URL + script_url
+                        async with aiohttp.ClientSession() as session:
+                            async with session.get(script_url, headers=headers) as script_resp:
+                                if script_resp is not None and str(resp.status) == "200":
+                                    script_resp_text = await script_resp.text()
+                                    pattern = r'"/api/search/".concat\("([a-zA-Z0-9]+)"\)'
+                                    matches = re.findall(pattern, script_resp_text)
+                                    for match in matches:
+                                        return match
+                                else:
+                                    return None        
+                else:
+                    return None                

howlongtobeatpy/howlongtobeatpy/HowLongToBeat.py

@@ -44,6 +44,17 @@ async def async_search(self, game_name: str, search_modifiers: SearchModifiers =
         """
         if game_name is None or len(game_name) == 0:
             return None
+        # Fetch the API Key
+        if HTMLRequests.SEARCH_API_KEY is None:
+            api_key_result = await HTMLRequests.async_send_website_request_getcode(False)
+            if api_key_result is None:
+                api_key_result = await HTMLRequests.async_send_website_request_getcode(True)
+        if api_key_result is not None:
+            # Set it for Caching
+            HTMLRequests.SEARCH_API_KEY = api_key_result
+        else:
+            return None
+        # Fetch the other data
         html_result = await HTMLRequests.send_async_web_request(game_name, search_modifiers)
         if html_result is not None:
             return self.__parse_web_result(game_name, html_result, None, similarity_case_sensitive)
@@ -60,6 +71,7 @@ def search(self, game_name: str, search_modifiers: SearchModifiers = SearchModif
         """
         if game_name is None or len(game_name) == 0:
             return None
+        # Fetch the API Key
         if HTMLRequests.SEARCH_API_KEY is None:
             api_key_result = HTMLRequests.send_website_request_getcode(False)
             if api_key_result is None:
@@ -68,7 +80,8 @@ def search(self, game_name: str, search_modifiers: SearchModifiers = SearchModif
             # Set it for Caching
             HTMLRequests.SEARCH_API_KEY = api_key_result
         else:
-            return None         
+            return None
+        # Fetch the other data
         html_result = HTMLRequests.send_web_request(game_name, search_modifiers)
         if html_result is not None:
             return self.__parse_web_result(game_name, html_result, None, similarity_case_sensitive)
@@ -88,6 +101,17 @@ async def async_search_from_id(self, game_id: int):
         """
         if game_id is None or game_id == 0:
             return None
+        # Fetch the API Key
+        if HTMLRequests.SEARCH_API_KEY is None:
+            api_key_result = await HTMLRequests.async_send_website_request_getcode(False)
+            if api_key_result is None:
+                api_key_result = await HTMLRequests.async_send_website_request_getcode(True)
+        if api_key_result is not None:
+            # Set it for Caching
+            HTMLRequests.SEARCH_API_KEY = api_key_result
+        else:
+            return None
+        # Fetch the other data
         game_title = await HTMLRequests.async_get_game_title(game_id)
         if game_title is not None:
             html_result = await HTMLRequests.send_async_web_request(game_title)
@@ -108,6 +132,17 @@ def search_from_id(self, game_id: int):
         """
         if game_id is None or game_id == 0:
             return None
+        # Fetch the API Key
+        if HTMLRequests.SEARCH_API_KEY is None:
+            api_key_result = HTMLRequests.send_website_request_getcode(False)
+            if api_key_result is None:
+                api_key_result = HTMLRequests.send_website_request_getcode(True)
+        if api_key_result is not None:
+            # Set it for Caching
+            HTMLRequests.SEARCH_API_KEY = api_key_result
+        else:
+            return None
+        # Fetch the other data
         game_title = HTMLRequests.get_game_title(game_id)
         if game_title is not None:
             html_result = HTMLRequests.send_web_request(game_title)

howlongtobeatpy/setup.py

@@ -4,7 +4,7 @@
     long_description = fh.read()
 
 setup(name='howlongtobeatpy',
-      version='1.0.6',
+      version='1.0.7',
       packages=find_packages(exclude=['tests']),
       description='A Python API for How Long to Beat',
       long_description=long_description,
@@ -14,9 +14,10 @@
       license='MIT',
       keywords='howlongtobeat gaming steam uplay origin time length how long to beat',
       install_requires=[
-          'aiohttp>=3.8.3',
-          'requests>=2.28.1',
+          'aiohttp>=3.10.1',
+          'requests>=2.32.3',
           'aiounittest>=1.4.2',
-          'fake_useragent>=1.1.0'
+          'fake_useragent>=1.5.1',
+          'beautifulsoup4>=4.12.3'
       ],
       zip_safe=False)

sonar-project.properties

@@ -2,7 +2,7 @@ sonar.organization=scrappycocco-github
 sonar.projectKey=ScrappyCocco_HowLongToBeat-PythonAPI
 
 sonar.projectName=HowLongToBeat-PythonAPI
-sonar.projectVersion=1.0.6
+sonar.projectVersion=1.0.7
 sonar.python.version=3.9
 
 # Path is relative to the sonar-project.properties file. Replace "\" by "/" on Windows.