Address code review feedback

- Add version constraint to scrapegraph-py dependency
- Improve error handling with specific exceptions in close() method
- Add response validation in scrape_product() and scrape_search_results()
- Use safe dictionary access with isinstance checks
- Add logging for exception types in error handlers

Co-authored-by: lurenss <[email protected]>

Author: Copilot

requirements.txt

@@ -1,5 +1,5 @@
 # ScrapeGraphAI SDK (API-based)
-scrapegraph-py
+scrapegraph-py>=1.0.0
 
 # Elasticsearch
 elasticsearch>=8.0.0

src/scrapegraph_demo/scraper.py

@@ -95,10 +95,15 @@ def scrape_product(self, url: str, marketplace: str) -> Optional[Product]:
                 output_schema=output_schema
             )
 
-            # Parse SDK response
-            if response and 'result' in response:
-                product = self._parse_scraped_data(response['result'], url, marketplace)
-                return product
+            # Parse SDK response with validation
+            if response and isinstance(response, dict) and 'result' in response:
+                result_data = response['result']
+                if isinstance(result_data, dict):
+                    product = self._parse_scraped_data(result_data, url, marketplace)
+                    return product
+                else:
+                    print(f"Warning: Invalid result structure in SDK response for {url}")
+                    return self._mock_scrape_product(url, marketplace)
             else:
                 print(f"Warning: No valid response from SDK for {url}")
                 return self._mock_scrape_product(url, marketplace)
@@ -173,30 +178,37 @@ def scrape_search_results(
                 output_schema=output_schema
             )
 
-            # Parse response
+            # Parse response with validation
             products = []
-            if response and 'result' in response and 'products' in response['result']:
-                for i, product_data in enumerate(response['result']['products'][:max_results]):
-                    product_url = product_data.get('product_url', f"https://{marketplace.lower()}.com/product/{i}")
-                    
-                    # Create Product object from search result data
-                    product = Product(
-                        product_id=self._extract_product_id(product_url),
-                        name=product_data.get('name', f"{search_query} Product {i+1}"),
-                        price=self._extract_price(str(product_data.get('price', 0))),
-                        currency=product_data.get('currency', 'USD'),
-                        url=product_url,
-                        marketplace=marketplace,
-                        rating=product_data.get('rating'),
-                        review_count=product_data.get('review_count'),
-                        scraped_at=datetime.utcnow()
-                    )
-                    products.append(product)
-                
-                return products
-            else:
-                print(f"Warning: No valid search results from SDK, using mock data")
-                return self._get_mock_search_results(search_query, marketplace, max_results)
+            if response and isinstance(response, dict) and 'result' in response:
+                result_data = response.get('result', {})
+                if isinstance(result_data, dict) and 'products' in result_data:
+                    products_list = result_data.get('products', [])
+                    if isinstance(products_list, list):
+                        for i, product_data in enumerate(products_list[:max_results]):
+                            if not isinstance(product_data, dict):
+                                continue
+                            product_url = product_data.get('product_url', f"https://{marketplace.lower()}.com/product/{i}")
+                            
+                            # Create Product object from search result data
+                            product = Product(
+                                product_id=self._extract_product_id(product_url),
+                                name=product_data.get('name', f"{search_query} Product {i+1}"),
+                                price=self._extract_price(str(product_data.get('price', 0))),
+                                currency=product_data.get('currency', 'USD'),
+                                url=product_url,
+                                marketplace=marketplace,
+                                rating=product_data.get('rating'),
+                                review_count=product_data.get('review_count'),
+                                scraped_at=datetime.utcnow()
+                            )
+                            products.append(product)
+                        
+                        return products
+            
+            # If we get here, no valid results were found
+            print(f"Warning: No valid search results from SDK, using mock data")
+            return self._get_mock_search_results(search_query, marketplace, max_results)
 
         except Exception as e:
             print(f"Error scraping search results: {str(e)}")
@@ -342,5 +354,10 @@ def close(self):
         if self.client is not None:
             try:
                 self.client.close()
+            except (AttributeError, RuntimeError) as e:
+                # AttributeError: if close method doesn't exist
+                # RuntimeError: if connection already closed or other runtime errors
+                print(f"Warning: Error closing SDK client ({type(e).__name__}): {e}")
             except Exception as e:
-                print(f"Warning: Error closing SDK client: {e}")
+                # Catch any other unexpected exceptions but log the type
+                print(f"Warning: Unexpected error closing SDK client ({type(e).__name__}): {e}")