Release v1.0.3: Enhanced Search Pagination

Author: Mohammedcha

CHANGELOG.md

@@ -2,6 +2,21 @@
 
 All notable changes to this project will be documented in this file.
 
+## [1.0.3] - 2025-10-15
+
+### New Features
+
+- **Enhanced Search Pagination**: Now able to fetch unlimited search results (300+) with automatic pagination, not limited to 50 results anymore
+- **Improved Search Performance**: Optimized search result fetching with better token handling and batch processing
+
+### Bug Fixes & Code Quality Improvements
+
+- **Code Review**: Addressed security vulnerabilities and code quality issues
+- **Error Handling**: Improved error handling patterns across all modules
+- **Performance**: Optimized JSON parsing and HTTP client fallback logic
+- **Security**: Fixed potential SSRF and injection vulnerabilities
+- **Maintainability**: Enhanced code readability and documentation
+
 ## [1.0.2] - 2025-01-15
 
 ### Major Release - Complete Library Redesign 🚀

README.md

@@ -48,7 +48,7 @@
 - Localized pricing and availability
 - Competitive analysis and benchmarking
 
-## 🆕 **What's New in v1.0.2** 
+## 🆕 **What's New in v1.0.3** 
 
 **✅ 7 Method Types:**
 - **App Methods** - Extract 65+ data fields from any app (ratings, installs, pricing, permissions, etc.)

docs/changelog.rst

@@ -3,7 +3,23 @@ Changelog
 
 All notable changes to this project will be documented in this file.
 
-Version 1.0.2 (2025-01-15)
+Version 1.0.3 (2025-10-15)
+---------------------------
+
+**New Features**
+
+- **Enhanced Search Pagination**: Now able to fetch unlimited search results (300+) with automatic pagination, not limited to 50 results anymore
+- **Improved Search Performance**: Optimized search result fetching with better token handling and batch processing
+
+**Bug Fixes & Code Quality Improvements**
+
+- **Code Review**: Addressed security vulnerabilities and code quality issues
+- **Error Handling**: Improved error handling patterns across all modules
+- **Performance**: Optimized JSON parsing and HTTP client fallback logic
+- **Security**: Fixed potential SSRF and injection vulnerabilities
+- **Maintainability**: Enhanced code readability and documentation
+
+Version 1.0.2 (2025-10-15)
 ---------------------------
 
 **Major Release - Complete Library Redesign** 🚀

docs/conf.py

@@ -8,7 +8,7 @@
 project = 'GPlay Scraper'
 copyright = '2025, Mohammed Cha'
 author = 'Mohammed Cha'
-release = '1.0.2'
+release = '1.0.3'
 
 # Extensions
 extensions = [

gplay_scraper/__init__.py

@@ -35,7 +35,7 @@
 logging.getLogger(__name__).addHandler(logging.NullHandler())
 
 # Package metadata
-__version__ = "1.0.2"
+__version__ = "1.0.3"
 
 # Public API exports
 __all__ = [

gplay_scraper/config.py

@@ -78,7 +78,8 @@ class Config:
         "SUGGEST_NOT_FOUND": "Suggestions not found for: {term}",
         "NO_DS3_DATA": "No data found in dataset",
         "DS3_NOT_FOUND": "Could not find data",
-        "DS3_JSON_PARSE_FAILED": "Failed to parse JSON: {error}"
+        "DS3_JSON_PARSE_FAILED": "Failed to parse JSON: {error}",
+        "SEARCH_PAGINATION_FAILED": "Failed to fetch paginated search results: {error}"
     }
 
     @classmethod

gplay_scraper/core/gplay_methods.py

@@ -142,7 +142,7 @@ def __init__(self, http_client: str = None):
         self.parser = SearchParser()
 
     def search_analyze(self, query: str, count: int = Config.DEFAULT_SEARCH_COUNT, lang: str = Config.DEFAULT_LANGUAGE, country: str = Config.DEFAULT_COUNTRY) -> List[Dict]:
-        """Search for apps and get complete results.
+        """Search for apps and get complete results with pagination support.
         
         Args:
             query: Search query string
@@ -158,11 +158,11 @@ def search_analyze(self, query: str, count: int = Config.DEFAULT_SEARCH_COUNT, l
         """
         if not query or not isinstance(query, str):
             raise InvalidAppIdError(Config.ERROR_MESSAGES["INVALID_QUERY"])
-            
-        html_content = self.scraper.fetch_playstore_search(query, count, lang, country)
-        dataset = self.scraper.scrape_play_store_data(html_content)
-        raw_results = self.parser.parse_search_results(dataset, count)
 
+        # scrape_play_store_data now handles pagination automatically
+        dataset = self.scraper.scrape_play_store_data(query, count, lang, country)
+        
+        raw_results = self.parser.parse_search_results(dataset, count)
         return [self.parser.format_search_result(result) for result in raw_results]
 
     def search_get_field(self, query: str, field: str, count: int = Config.DEFAULT_SEARCH_COUNT, lang: str = Config.DEFAULT_LANGUAGE, country: str = Config.DEFAULT_COUNTRY) -> List[Any]:

gplay_scraper/core/gplay_parser.py

@@ -8,7 +8,7 @@
 import re
 from datetime import datetime, timezone
 from typing import Dict, Any, List, Optional, Tuple
-from ..models.element_specs import ElementSpecs
+from ..models.element_specs import ElementSpecs, nested_lookup
 from ..utils.helpers import clean_json_string, alternative_json_clean, calculate_app_age, calculate_daily_installs, calculate_monthly_installs
 from ..config import Config
 from ..exceptions import DataParsingError
@@ -155,7 +155,6 @@ def parse_search_results(self, dataset: Dict, count: int) -> List[Dict]:
         if "ds:1" not in dataset:
             return []
 
-        from ..models.element_specs import nested_lookup
         search_data = nested_lookup(dataset.get("ds:1", {}), [0, 1, 0, 0, 0])
 
         if not search_data:
@@ -209,6 +208,67 @@ def format_search_result(self, result: dict) -> dict:
             "free": result.get("free"),
             "url": result.get("url"),
         }
+    
+    def extract_pagination_token(self, dataset: Dict) -> str:
+        """Extract pagination token from search dataset.
+        
+        Args:
+            dataset: Search dataset
+            
+        Returns:
+            Pagination token or None
+        """
+        sections = nested_lookup(dataset.get("ds:1", {}), [0, 1, 0, 0])
+        
+        if not sections:
+            return None
+            
+        for section in sections:
+            if isinstance(section, list) and len(section) > 1:
+                potential_token = nested_lookup(section, [1])
+                if isinstance(potential_token, str):
+                    return potential_token
+        return None
+    
+    def parse_html_content(self, html_content: str) -> Dict:
+        """Extract datasets from search page HTML.
+        
+        Args:
+            html_content: HTML content of search page
+            
+        Returns:
+            Dictionary containing all datasets
+            
+        Raises:
+            DataParsingError: If no datasets found
+        """
+        import re
+        script_regex = re.compile(r"AF_initDataCallback[\s\S]*?</script")
+        key_regex = re.compile(r"(ds:.*?)'")
+        value_regex = re.compile(r"data:([\s\S]*?), sideChannel: \{\}\}\);</")
+        
+        matches = script_regex.findall(html_content)
+        dataset = {}
+        
+        for match in matches:
+            key_match = key_regex.findall(match)
+            value_match = value_regex.findall(match)
+            
+            if key_match and value_match:
+                key = key_match[0]
+                try:
+                    import json
+                    value = json.loads(value_match[0])
+                    dataset[key] = value
+                except json.JSONDecodeError:
+                    continue
+        
+        if not dataset:
+            from ..exceptions import DataParsingError
+            from ..config import Config
+            raise DataParsingError("No search data found in HTML")
+        
+        return dataset
 
 
 class ReviewsParser:
@@ -365,7 +425,6 @@ def parse_developer_data(self, dataset: Dict, dev_id: str) -> List[Dict]:
         else:
             apps_path = [0, 1, 0, 22, 0]
 
-        from ..models.element_specs import nested_lookup
         apps_data = nested_lookup(data.get("data", data), apps_path)
         if not apps_data:
             return []
@@ -438,7 +497,6 @@ def parse_similar_data(self, dataset: Dict) -> List[Dict]:
             except Exception:
                 return []
 
-        from ..models.element_specs import nested_lookup
         apps_data = nested_lookup(data.get("data", data), [0, 1, 0, 21, 0])
         if not apps_data:
             return []
@@ -501,7 +559,6 @@ def parse_list_data(self, dataset: Dict, count: int) -> List[Dict]:
         if not collection_data:
             return []
 
-        from ..models.element_specs import nested_lookup
         apps_data = nested_lookup(collection_data, [0, 1, 0, 28, 0])
         if not apps_data:
             return []

gplay_scraper/core/gplay_scraper.py

@@ -96,43 +96,88 @@ def fetch_playstore_search(self, query: str, count: int, lang: str = Config.DEFA
         if count <= 0:
             return ""
 
-        return self.http_client.fetch_search_page(query, lang, country)
+        return self.http_client.fetch_search_page(query=query, lang=lang, country=country)
 
-    def scrape_play_store_data(self, html_content: str) -> Dict:
-        """Extract datasets from search page HTML.
+    def scrape_play_store_data(self, query: str, count: int = Config.DEFAULT_SEARCH_COUNT, lang: str = Config.DEFAULT_LANGUAGE, country: str = Config.DEFAULT_COUNTRY) -> Dict:
+        """Scrape search results with automatic pagination support.
         
         Args:
-            html_content: HTML content of search page
+            query: Search query string
+            count: Total number of results to fetch
+            lang: Language code
+            country: Country code
             
         Returns:
-            Dictionary containing all datasets
+            Dictionary containing all search results
             
         Raises:
-            DataParsingError: If no datasets found
+            DataParsingError: If parsing fails
         """
-        script_regex = re.compile(r"AF_initDataCallback[\s\S]*?</script")
-        key_regex = re.compile(r"(ds:.*?)'")
-        value_regex = re.compile(r"data:([\s\S]*?), sideChannel: {}}\);</")
+        # Get initial search page
+        html_content = self.fetch_playstore_search(query, count, lang, country)
 
-        matches = script_regex.findall(html_content)
-        dataset = {}
+        # Use parser for all parsing operations
+        from .gplay_parser import SearchParser
+        parser = SearchParser()
+        dataset = parser.parse_html_content(html_content)
 
-        for match in matches:
-            key_match = key_regex.findall(match)
-            value_match = value_regex.findall(match)
-            
-            if key_match and value_match:
-                key = key_match[0]
-                try:
-                    value = json.loads(value_match[0])
-                    dataset[key] = value
-                except json.JSONDecodeError:
-                    continue
+        # If count <= 20, return initial results
+        if count <= 20:
+            return dataset
+        
+        # Extract pagination token
+        token = parser.extract_pagination_token(dataset)
+        
+        # Collect all results
+        all_results = []
+        initial_results = self._get_nested_value(dataset.get("ds:1", []), [0, 1, 0, 0, 0], [])
+        all_results.extend(initial_results)
+        
+        # Paginate if needed and token exists
+        while len(all_results) < count and token:
+            needed = min(100, count - len(all_results))
+            try:
+                response_text = self.http_client.fetch_search_page(token=token, needed=needed, lang=lang, country=country)
+                data = json.loads(response_text[5:])
+                parsed_data = json.loads(data[0][2])
+                
+                if parsed_data:
+                    paginated_results = self._get_nested_value(parsed_data, [0, 0, 0], [])
+                    all_results.extend(paginated_results)
+                    # Update token for next iteration
+                    token = self._get_nested_value(parsed_data, [0, 0, 7, 1])
+                else:
+                    break
+            except (json.JSONDecodeError, IndexError, KeyError, Exception):
+                break
 
-        if not dataset:
-            raise DataParsingError(Config.ERROR_MESSAGES["NO_DS5_DATA"])
+        # Update dataset with all results
+        if "ds:1" in dataset:
+            dataset["ds:1"][0][1][0][0][0] = all_results[:count]
 
         return dataset
+    
+
+    
+
+    
+    def _get_nested_value(self, data, path, default=None):
+        """Safely get nested value from data structure.
+        
+        Args:
+            data: Data structure to traverse
+            path: List of keys/indices to follow
+            default: Default value if path not found
+            
+        Returns:
+            Value at path or default
+        """
+        try:
+            for key in path:
+                data = data[key]
+            return data
+        except (KeyError, IndexError, TypeError):
+            return default
 
 
 class ReviewsScraper:

gplay_scraper/utils/helpers.py

@@ -230,4 +230,4 @@ def calculate_monthly_installs(install_count, release_date_str: Optional[str], c
         return 0
 
     months_since_release = days_since_release / 30.44
-    return int(install_count / months_since_release)
+    return int(install_count / months_since_release)