RFP deployment update

Author: blessing-sanusi

data/datasets/RFP_dataset/20251020_090020_RFP_Security_Risk_Guidelines.pdf

@@ -491,4 +491,4 @@ To debug the python server in the frontend directory (frontend_server.py) and re
       "args": ["frontend_server:app", "--port", "3000", "--reload"],
       "jinja": true
     }
-```
+```

data/datasets/RFP_dataset/20251020_090032_RFP_Compliance_Guidelines.pdf

@@ -5,6 +5,49 @@
 from azure.storage.blob import BlobServiceClient
 import sys
 
+# PDF text extraction function
+def extract_pdf_text(pdf_bytes):
+    """Extract text content from PDF bytes using PyPDF2"""
+    try:
+        import PyPDF2
+        import io
+        
+        pdf_file = io.BytesIO(pdf_bytes)
+        pdf_reader = PyPDF2.PdfReader(pdf_file)
+        
+        # Check if PDF is encrypted/protected
+        if pdf_reader.is_encrypted:
+            return "PDF_PROTECTED: This PDF document is password-protected or encrypted and cannot be processed."
+        
+        text_content = []
+        for page in pdf_reader.pages:
+            try:
+                page_text = page.extract_text()
+                if page_text and page_text.strip():
+                    text_content.append(page_text)
+            except Exception:
+                continue
+        
+        full_text = "\n".join(text_content).strip()
+        
+        # Check for protection messages
+        protection_indicators = [
+            "protected by Microsoft Office",
+            "You'll need a different reader",
+            "Download a compatible PDF reader",
+            "This PDF Document has been protected"
+        ]
+        
+        if any(indicator.lower() in full_text.lower() for indicator in protection_indicators):
+            return "PDF_PROTECTED: This PDF document appears to be protected or encrypted."
+        
+        return full_text if full_text else "PDF_NO_TEXT: No readable text content found in PDF."
+        
+    except ImportError:
+        return "PDF_ERROR: PyPDF2 library not available. Install with: pip install PyPDF2"
+    except Exception as e:
+        return f"PDF_ERROR: Error reading PDF content: {str(e)}"
+
 if len(sys.argv) < 4:
     print("Usage: python index_datasets.py <storage_account_name> <blob_container_name> <ai_search_endpoint> [<ai_search_index_name>]")
     sys.exit(1)
@@ -51,11 +94,19 @@
     #if blob.name.endswith(".csv"):
     title = blob.name.replace(".csv", "")
     title = blob.name.replace(".json", "")
+    title = blob.name.replace(".pdf", "")  # Also handle PDF extension
     data = container_client.download_blob(blob.name).readall()
 
     try:
         print(f"Reading data from blob: {blob.name}...")
-        text = data.decode('utf-8')
+        
+        # Check if this is a PDF file and process accordingly
+        if blob.name.lower().endswith('.pdf'):
+            text = extract_pdf_text(data)
+        else:
+            # Original processing for non-PDF files
+            text = data.decode('utf-8')
+            
         data_list.append({
             "content": text,
             "id": str(idx),

data/datasets/RFP_dataset/20251020_090046_Woodgrove_Bank_RFP_Response_Contoso_Ltd.pdf

@@ -123,13 +123,14 @@ fi
 
 
 #Upload sample CSV files to blob storage
-echo "Uploading CSV sample files to blob storage..."
+echo "Uploading CSV and JSON sample files to blob storage..."
 az storage blob upload-batch --account-name "$storageAccount" --destination "$blobContainer" --source "data/datasets" --auth-mode login --pattern '*.csv' --overwrite --output none
+az storage blob upload-batch --account-name "$storageAccount" --destination "$blobContainer" --source "data/datasets" --auth-mode login --pattern '*.json' --overwrite --output none
 if [ $? -ne 0 ]; then
-    echo "Error: Failed to upload CSV files to blob storage."
+    echo "Error: Failed to upload CSV and JSON files to blob storage."
     exit 1
 fi
-echo "CSV files uploaded successfully to blob storage."
+echo "CSV and JSON files uploaded successfully to blob storage."
 
 #Upload PDF files from RFP_dataset to blob storage
 echo "Uploading PDF files from RFP_dataset to blob storage..."
@@ -194,14 +195,14 @@ if [ "$has_csv" = true ]; then
     fi
 fi
 
-if [ "$has_pdf" = true ]; then
-    echo "Running the python script to index PDF data"
-    $PYTHON_CMD infra/scripts/index_rfp_data.py "$storageAccount" "$blobContainer" "$aiSearch" "$aiSearchIndex"
-    if [ $? -ne 0 ]; then
-        echo "Error: PDF indexing python script execution failed."
-        exit 1
-    fi
-fi
+# if [ "$has_pdf" = true ]; then
+#     echo "Running the python script to index PDF data"
+#     $PYTHON_CMD infra/scripts/index_rfp_data.py "$storageAccount" "$blobContainer" "$aiSearch" "$aiSearchIndex"
+#     if [ $? -ne 0 ]; then
+#         echo "Error: PDF indexing python script execution failed."
+#         exit 1
+#     fi
+# fi
 
 if [ "$has_csv" = false ] && [ "$has_pdf" = false ]; then
     echo "No CSV or PDF files found to index."
@@ -228,4 +229,4 @@ if [ "$srchIsPublicAccessDisabled" = true ]; then
     echo "Public access disabled for search service: $aiSearch"
 fi
 
-echo "Script executed successfully. Sample Data Processed Successfully."
+echo "Script executed successfully. Sample Data Processed Successfully."