Test incremental update

Author: AnushaFF

.github/workflows/update-docs-dev.yml

@@ -2,7 +2,7 @@ name: Deploy to Dev Firestore
 
 on:
   push:
-    branches: [ main]
+    branches: [ anusha/pinecone-github-action ]
 
 jobs:
   deploy-dev:
@@ -28,5 +28,5 @@ jobs:
         FIRESTORE_CREDENTIALS_JSON: ${{ secrets.DEV_FIRESTORE_CREDENTIALS_JSON }}
         OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       run: |
-        python update_docs.py --mode full
+        python update_docs.py --mode incremental
       

.github/workflows/update-docs-prod.yml

@@ -29,23 +29,4 @@ jobs:
         FIRESTORE_CREDENTIALS_JSON: ${{ secrets.PROD_FIRESTORE_CREDENTIALS_JSON }}
         OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       run: |
-        python update_docs.py --mode full
-        
-    - name: Verify deployment
-      env:
-        FIRESTORE_PROJECT_ID: ${{ secrets.PROD_FIRESTORE_PROJECT_ID }}
-        FIRESTORE_CREDENTIALS_JSON: ${{ secrets.PROD_FIRESTORE_CREDENTIALS_JSON }}
-      run: |
-        python -c "
-        from google.cloud import firestore
-        from google.oauth2 import service_account
-        import json
-        
-        credentials_info = json.loads('$FIRESTORE_CREDENTIALS_JSON')
-        credentials = service_account.Credentials.from_service_account_info(credentials_info)
-        db = firestore.Client(project='$FIRESTORE_PROJECT_ID', credentials=credentials)
-        
-        docs = db.collection('knowledge_base').stream()
-        count = sum(1 for _ in docs)
-        print(f'✅ Prod Firestore contains {count} documents')
-        "
+        python update_docs.py --mode incremental

update_docs.py

@@ -14,7 +14,6 @@
 class Config:
     """Configuration constants for the documentation updater."""
     REPO_FOLDER = "./docs"
-    OUTPUT_FILE = "documentation_rag_chunks.json"
     STATE_FILE = "docs_state.json"
     MAX_CHUNK_SIZE = 500
     BATCH_SIZE = 50
@@ -55,6 +54,7 @@ def save_file_state(state: Dict[str, Dict[str, str]]) -> None:
     """Save current file states."""
     with open(Config.STATE_FILE, 'w') as f:
         json.dump(state, f, indent=2)
+        
 
 def get_changed_files() -> Tuple[List[str], List[str], Dict[str, str]]:
     """Detect which files have changed since last run."""
@@ -385,15 +385,6 @@ def incremental_update() -> None:
     print("✅ Incremental update completed!")
 
 
-def save_to_json(chunks: List[Dict[str, any]]) -> None:
-    """Save processed chunks to JSON file for inspection."""
-    try:
-        with open(Config.OUTPUT_FILE, 'w', encoding='utf-8') as f:
-            json.dump(chunks, f, indent=2, ensure_ascii=False)
-        print(f"📄 Saved {len(chunks)} chunks to {Config.OUTPUT_FILE}")
-    except Exception as e:
-        print(f"⚠️ Error saving chunks to JSON: {e}")
-
 def full_refresh() -> None:
     """Perform full refresh - delete all and regenerate."""
     print("🔄 Starting full refresh...")