Merge pull request #26 from ScrappyCocco/FetchApiKey

Fetch api key from website

Author: ScrappyCocco

.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+    "python.testing.pytestArgs": [
+        "howlongtobeatpy"
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true
+}

howlongtobeatpy/howlongtobeatpy/HTMLRequests.py

@@ -4,6 +4,7 @@
 import re
 import html
 import json
+from bs4 import BeautifulSoup
 from enum import Enum
 import aiohttp
 import requests
@@ -23,7 +24,7 @@ class SearchModifiers(Enum):
 class HTMLRequests:
     BASE_URL = 'https://howlongtobeat.com/'
     REFERER_HEADER = BASE_URL
-    SEARCH_URL = BASE_URL + "api/search" + "/4b4cbe570602c88660f7df8ea0cb6b6e"
+    SEARCH_URL = BASE_URL + "api/search"
     GAME_URL = BASE_URL + "game"
 
     @staticmethod
@@ -94,8 +95,12 @@ def send_web_request(game_name: str, search_modifiers: SearchModifiers = SearchM
         """
         headers = HTMLRequests.get_search_request_headers()
         payload = HTMLRequests.get_search_request_data(game_name, search_modifiers, page)
+        api_key_result = HTMLRequests.send_website_request_getcode(False)
+        if api_key_result is None:
+            api_key_result = HTMLRequests.send_website_request_getcode(True)
         # Make the post request and return the result if is valid
-        resp = requests.post(HTMLRequests.SEARCH_URL, headers=headers, data=payload)
+        search_url_with_key = HTMLRequests.SEARCH_URL + "/" + api_key_result
+        resp = requests.post(search_url_with_key, headers=headers, data=payload)
         if resp.status_code == 200:
             return resp.text
         return None
@@ -112,9 +117,13 @@ async def send_async_web_request(game_name: str, search_modifiers: SearchModifie
         """
         headers = HTMLRequests.get_search_request_headers()
         payload = HTMLRequests.get_search_request_data(game_name, search_modifiers, page)
+        api_key_result = await HTMLRequests.async_send_website_request_getcode(False)
+        if api_key_result is None:
+            api_key_result = await HTMLRequests.async_send_website_request_getcode(True)
         # Make the post request and return the result if is valid
+        search_url_with_key = HTMLRequests.SEARCH_URL + "/" + api_key_result
         async with aiohttp.ClientSession() as session:
-            async with session.post(HTMLRequests.SEARCH_URL, headers=headers, data=payload) as resp:
+            async with session.post(search_url_with_key, headers=headers, data=payload) as resp:
                 if resp is not None and str(resp.status) == "200":
                     return await resp.text()
                 return None
@@ -195,3 +204,66 @@ async def async_get_game_title(game_id: int):
                     text = await resp.text()
                     return HTMLRequests.__cut_game_title(text)
                 return None
+            
+    @staticmethod
+    def send_website_request_getcode(parse_all_scripts: bool):
+        """
+        Function that send a request to howlongtobeat to scrape the /api/search key
+        @return: The string key to use on /api/search
+        """
+        # Make the post request and return the result if is valid
+        headers = HTMLRequests.get_title_request_headers()
+        resp = requests.get(HTMLRequests.BASE_URL, headers=headers)
+        if resp.status_code == 200 and resp.text is not None:
+                # Parse the HTML content using BeautifulSoup
+                soup = BeautifulSoup(resp.text, 'html.parser')
+                # Find all <script> tags with a src attribute containing the substring
+                scripts = soup.find_all('script', src=True)
+                if parse_all_scripts:
+                    matching_scripts = [script['src'] for script in scripts]
+                else:
+                    matching_scripts = [script['src'] for script in scripts if '_app-' in script['src']]
+                for script_url in matching_scripts:
+                    script_url = HTMLRequests.BASE_URL + script_url
+                    script_resp = requests.get(script_url, headers=headers)
+                    if script_resp.status_code == 200 and script_resp.text is not None:
+                        pattern = r'"/api/search/".concat\("([a-zA-Z0-9]+)"\)'
+                        matches = re.findall(pattern, script_resp.text)
+                        for match in matches:
+                            return match
+        return None
+    
+    @staticmethod
+    async def async_send_website_request_getcode(parse_all_scripts: bool):
+        """
+        Function that send a request to howlongtobeat to scrape the /api/search key
+        @return: The string key to use on /api/search
+        """
+        # Make the post request and return the result if is valid
+        headers = HTMLRequests.get_title_request_headers()
+        async with aiohttp.ClientSession() as session:
+            async with session.get(HTMLRequests.BASE_URL, headers=headers) as resp:
+                if resp is not None and str(resp.status) == "200":
+                    resp_text = await resp.text()
+                    # Parse the HTML content using BeautifulSoup
+                    soup = BeautifulSoup(resp_text, 'html.parser')
+                    # Find all <script> tags with a src attribute containing the substring
+                    scripts = soup.find_all('script', src=True)
+                    if parse_all_scripts:
+                        matching_scripts = [script['src'] for script in scripts]
+                    else:
+                        matching_scripts = [script['src'] for script in scripts if '_app-' in script['src']]
+                    for script_url in matching_scripts:
+                        script_url = HTMLRequests.BASE_URL + script_url
+                        async with aiohttp.ClientSession() as session:
+                            async with session.get(script_url, headers=headers) as script_resp:
+                                if script_resp is not None and str(resp.status) == "200":
+                                    script_resp_text = await script_resp.text()
+                                    pattern = r'"/api/search/".concat\("([a-zA-Z0-9]+)"\)'
+                                    matches = re.findall(pattern, script_resp_text)
+                                    for match in matches:
+                                        return match
+                                else:
+                                    return None        
+                else:
+                    return None                

howlongtobeatpy/setup.py

@@ -4,7 +4,7 @@
     long_description = fh.read()
 
 setup(name='howlongtobeatpy',
-      version='1.0.6',
+      version='1.0.7',
       packages=find_packages(exclude=['tests']),
       description='A Python API for How Long to Beat',
       long_description=long_description,
@@ -14,9 +14,10 @@
       license='MIT',
       keywords='howlongtobeat gaming steam uplay origin time length how long to beat',
       install_requires=[
-          'aiohttp>=3.8.3',
-          'requests>=2.28.1',
+          'aiohttp>=3.10.1',
+          'requests>=2.32.3',
           'aiounittest>=1.4.2',
-          'fake_useragent>=1.1.0'
+          'fake_useragent>=1.5.1',
+          'beautifulsoup4>=4.12.3'
       ],
       zip_safe=False)

sonar-project.properties

@@ -2,7 +2,7 @@ sonar.organization=scrappycocco-github
 sonar.projectKey=ScrappyCocco_HowLongToBeat-PythonAPI
 
 sonar.projectName=HowLongToBeat-PythonAPI
-sonar.projectVersion=1.0.6
+sonar.projectVersion=1.0.7
 sonar.python.version=3.9
 
 # Path is relative to the sonar-project.properties file. Replace "\" by "/" on Windows.