fix: data menu data formatting

Author: shreyash776

backend/app/api/routes/scrapper.py

@@ -15,6 +15,7 @@
     MenuSubCategoryCreate,
     MenuItemCreate
 )
+from app.api.routes.utils import transform_restaurant_data
 
 logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
 logger = logging.getLogger(__name__)
@@ -26,10 +27,9 @@ class ZomatoUrl(BaseModel):
 
 def fetch_zomato_data(url: str) -> str:
     headers = {
-        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
         'Accept-Language': 'en-US,en;q=0.5',
-        'Connection': 'keep-alive',
     }
 
     try:
@@ -43,33 +43,25 @@ def fetch_zomato_data(url: str) -> str:
 
 def parse_zomato_page(html_content: str) -> dict:
     try:
-        logger.debug("Starting page parsing...")
         soup = BeautifulSoup(html_content, 'html.parser')
-        
         scripts = soup.find_all('script')
-        target_script = None
 
         for script in scripts:
             if script.string and 'window.__PRELOADED_STATE__' in script.string:
-                target_script = script
-                break
-        
-        if not target_script:
-            raise ValueError("Could not find PRELOADED_STATE in page")
-        
-        json_str = target_script.string.split('window.__PRELOADED_STATE__ = JSON.parse(')[1]
-        json_str = json_str.split(');')[0].strip()
-        json_str = json_str.strip('"').replace('\\"', '"').replace('\\\\', '\\').replace('\\n', '')
-        
-        return json.loads(json_str)
-        
+                json_str = script.string.split('window.__PRELOADED_STATE__ = JSON.parse(')[1]
+                json_str = json_str.split(');')[0].strip()
+                json_str = json_str.strip('"').replace('\\"', '"').replace('\\\\', '\\')
+                return json.loads(json_str)
+                
+        raise ValueError("Could not find PRELOADED_STATE in page")
     except Exception as e:
         logger.error(f"Error parsing page: {str(e)}")
         raise
 
 def extract_menu_data(json_data: dict) -> dict:
     try:
-        logger.debug("Starting menu data extraction...")
+        with open('data.json', 'w') as f:
+            json.dump(json_data, f, indent=4)
         restaurant_data = json_data.get('pages', {}).get('current', {})
         restaurant_details = json_data.get('pages', {}).get('restaurant', {})
 
@@ -79,56 +71,99 @@ def extract_menu_data(json_data: dict) -> dict:
 
         res_info = restaurant_details[restaurant_id].get('sections', {})
         basic_info = res_info.get('SECTION_BASIC_INFO', {})
-        
+        menu_widget = res_info.get('SECTION_MENU_WIDGET', {})
+
+
+        # Validate required name field
+        if not basic_info.get('name'):
+            raise ValueError("Restaurant name is required")
+
         restaurant_info = {
-            'name': basic_info.get('name'),
-            'cuisines': basic_info.get('cuisine_string'),
-            'rating': {
-                'aggregate_rating': basic_info.get('rating', {}).get('aggregate_rating'),
-                'votes': basic_info.get('rating', {}).get('votes'),
-                'rating_text': basic_info.get('rating', {}).get('rating_text')
-            },
-            'location': {
-                'locality': restaurant_data.get('pageDescription', ''),
-                'url': basic_info.get('resUrl')
-            },
-            'timing': {
-                'description': basic_info.get('timing', {}).get('timing_desc'),
-                'hours': basic_info.get('timing', {}).get('customised_timings', {}).get('opening_hours', [])
+            'cuisine_type': basic_info.get('cuisine_string', ''),
+            'venue': {
+                'name': basic_info.get('name', ''),
+                'address': basic_info.get('address', ''),
+                'locality': basic_info.get('locality_verbose', ''),
+                'city': basic_info.get('city', ''),
+                'latitude': basic_info.get('latitude', '0'),
+                'longitude': basic_info.get('longitude', '0'),
+                'zipcode': basic_info.get('zipcode', ''),
+                'rating': basic_info.get('rating', {}).get('aggregate_rating', '0'),
+                'timing': basic_info.get('timing', {}).get('timing', ''),
+                'avg_cost_for_two': basic_info.get('average_cost_for_two', 0)
             }
         }
-        
+
+
         menu_categories = []
-        menu_data = res_info.get('SECTION_MENU_WIDGET', {})
-        
-        for category in menu_data.get('categories', []):
-            category_items = {
-                'category': category.get('name', ''),
-                'items': []
+        print("Catorgies", menu_widget.get('menu', {}).get('categories', []))
+        for category in menu_widget.get('menu', {}).get('categories', []):
+            print("category", category)
+            category_data = {
+                'name': category.get('name', ''),
+                'description': category.get('description', ''),
+                'subcategories': []
             }
 
+            # Group items by subcategory
+            subcategories = {}
             for item in category.get('items', []):
+                subcategory_name = item.get('category', 'Other')
+                
+                if subcategory_name not in subcategories:
+                    subcategories[subcategory_name] = {
+                        'name': subcategory_name,
+                        'description': '',
+                        'items': []
+                    }
+                
                 menu_item = {
-                    'name': item.get('name'),
-                    'description': item.get('description', ''),
-                    'price': float(item.get('price', 0)),
-                    'image_url': item.get('imageUrl', ''),
+                    'name': item.get('name', ''),
+                    'description': item.get('desc', ''),
                     'is_veg': item.get('isVeg', True),
-                    'spice_level': item.get('spiceLevel', 'None')
+                    'image_url': item.get('itemImage', ''),
+                    'variants': []
                 }
-                category_items['items'].append(menu_item)
+                
+                # Handle variants
+                if item.get('variantsV2'):
+                    for variant in item['variantsV2']:
+                        menu_item['variants'].append({
+                            'name': variant.get('variantName', ''),
+                            'price': float(variant.get('price', 0)) / 100,
+                            'is_default': variant.get('isDefault', False)
+                        })
+                else:
+                    menu_item['variants'].append({
+                        'name': 'Regular',
+                        'price': float(item.get('defaultPrice', 0)) / 100,
+                        'is_default': True
+                    })
+                
+                subcategories[subcategory_name]['items'].append(menu_item)
 
-            if category_items['items']:
-                menu_categories.append(category_items)
-        
+            # Add non-empty subcategories to category
+            category_data['subcategories'] = [
+                subcat for subcat in subcategories.values()
+                if subcat['items']
+            ]
+            
+            if category_data['subcategories']:
+                menu_categories.append(category_data)
+
         return {
+            'status': 'success',
             'restaurant_info': restaurant_info,
             'menu': menu_categories
         }
-
+        
     except Exception as e:
         logger.error(f"Error extracting menu data: {str(e)}")
-        raise
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to extract menu data: {str(e)}"
+        )
+
 
 async def create_restaurant(client: httpx.AsyncClient, restaurant_data: RestaurantCreate):
     response = await client.post("/venue/restaurants/", json=restaurant_data.dict())
@@ -161,18 +196,21 @@ async def scrape_and_create_menu(
     current_user: UserBusiness = Depends(get_business_user)
 ):
     try:
-        # 1. Scrape data
+        # 1. Scrape and log data
         html_content = fetch_zomato_data(str(request.url))
         json_data = parse_zomato_page(html_content)
         scraped_data = extract_menu_data(json_data)
+        
+        logger.info("Scraped Restaurant Info:")
+        logger.info(f"Name: {scraped_data['restaurant_info']['name']}")
+        logger.info(f"Cuisines: {scraped_data['restaurant_info']['cuisines']}")
 
         async with httpx.AsyncClient() as client:
             # 2. Create Restaurant
             venue_data = VenueCreate(
                 name=scraped_data['restaurant_info']['name'],
                 description="Restaurant imported from Zomato",
-                opening_time=scraped_data['restaurant_info']['timing']['opening_time'],
-                avg_expense_for_two=float(scraped_data['restaurant_info']['avg_cost_for_two']),
+                avg_expense_for_two=scraped_data['restaurant_info']['avg_cost_for_two'],
                 zomato_link=str(request.url)
             )
 
@@ -196,15 +234,13 @@ async def scrape_and_create_menu(
 
             # 4. Create Categories, Subcategories, and Items sequentially
             for category in scraped_data['menu']:
-                # Create Category
                 category_data = MenuCategoryCreate(
                     name=category['category'],
                     menu_id=menu_id
                 )
                 category_id = await create_category(client, category_data)
                 logger.info(f"Created category: {category['category']}")
 
-                # Create default subcategory
                 subcategory_data = MenuSubCategoryCreate(
                     name=f"{category['category']} Items",
                     category_id=category_id,
@@ -213,7 +249,6 @@ async def scrape_and_create_menu(
                 subcategory_id = await create_subcategory(client, subcategory_data)
                 logger.info(f"Created subcategory for {category['category']}")
 
-                # Create items one by one
                 for item in category['items']:
                     item_data = MenuItemCreate(
                         name=item['name'],
@@ -230,9 +265,45 @@ async def scrape_and_create_menu(
                 "message": "Menu successfully created",
                 "venue_id": str(venue_id),
                 "menu_id": str(menu_id),
-                "restaurant_name": scraped_data['restaurant_info']['name']
+                "restaurant_name": scraped_data['restaurant_info']['name'],
+                "scraped_data": scraped_data
             }
 
     except Exception as e:
         logger.error(f"Menu creation failed: {str(e)}")
         raise HTTPException(status_code=500, detail=f"Failed to create menu: {str(e)}")
+
+
+@router.get("/menu/scrape")
+async def get_scraped_menu(url: str):
+    try:
+        # Validate URL
+        zomato_url = ZomatoUrl(url=url)
+        
+        # Scrape data using existing functions
+        html_content = fetch_zomato_data(str(zomato_url.url))
+        json_data = parse_zomato_page(html_content)
+        # print(json_data)
+        # scraped_data = extract_menu_data(json_data)
+        print("==== cleaning the data ====")
+        scraped_data = transform_restaurant_data(json_data)
+        print(scraped_data)
+        
+        return {
+            "status": "success",
+            "restaurant_info": scraped_data,
+            "menu": scraped_data['menu']
+        }
+        
+    except ValueError as ve:
+        logger.error(f"Invalid URL format: {str(ve)}")
+        raise HTTPException(
+            status_code=400,
+            detail=f"Invalid Zomato URL: {str(ve)}"
+        )
+    except Exception as e:
+        logger.error(f"Scraping failed: {str(e)}")
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to scrape menu data: {str(e)}"
+        )