Cdifonzo/multi index testing (#162)

Author: dorbaker

SECURITY.md

@@ -14,7 +14,7 @@ Instead, please report them to the Microsoft Security Response Center (MSRC) at
 
 If you prefer to submit without logging in, send email to [[email protected]](mailto:[email protected]).  If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/security.md/msrc/pgp).
 
-You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 
+You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc).
 
 Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
 

backend/manage-indexing-jobs.py

@@ -95,14 +95,12 @@ def main():
             )
             exit()
         if item["status"] == PipelineJobState.SCHEDULED.value:
-            job_metadata.append(
-                {
-                    "human_readable_index_name": item["human_readable_index_name"],
-                    "epoch_request_time": item["epoch_request_time"],
-                    "status": item["status"],
-                    "percent_complete": item["percent_complete"],
-                }
-            )
+            job_metadata.append({
+                "human_readable_index_name": item["human_readable_index_name"],
+                "epoch_request_time": item["epoch_request_time"],
+                "status": item["status"],
+                "percent_complete": item["percent_complete"],
+            })
     # exit if no jobs found
     if not job_metadata:
         print("No jobs found")

backend/src/api/common.py

@@ -3,12 +3,8 @@
 
 import hashlib
 import re
-from typing import Annotated
 
-from fastapi import (
-    Header,
-    HTTPException,
-)
+from fastapi import HTTPException
 
 from src.api.azure_clients import (
     AzureStorageClientManager,
@@ -188,14 +184,3 @@ def retrieve_original_entity_config_name(sanitized_name: str) -> str | None:
             status_code=500, detail="Error retrieving original entity config name."
         )
     return None
-
-
-async def verify_subscription_key_exist(
-    Ocp_Apim_Subscription_Key: Annotated[str, Header()],
-):
-    # a function that will be injected as a dependency to API routes - it will be called to verify the Ocp_Apim_Subscription_Key is present
-    if not Ocp_Apim_Subscription_Key:
-        raise HTTPException(
-            status_code=400, detail="Ocp-Apim-Subscription-Key required"
-        )
-    return Ocp_Apim_Subscription_Key

backend/src/api/data.py

@@ -2,15 +2,13 @@
 # Licensed under the MIT License.
 
 import asyncio
-import os
 import re
 from math import ceil
 from typing import List
 
 from azure.storage.blob import ContainerClient
 from fastapi import (
     APIRouter,
-    Depends,
     HTTPException,
     UploadFile,
 )
@@ -23,7 +21,6 @@
     delete_blob_container,
     sanitize_name,
     validate_blob_container_name,
-    verify_subscription_key_exist,
 )
 from src.models import (
     BaseResponse,
@@ -38,9 +35,6 @@
     tags=["Data Management"],
 )
 
-if os.getenv("KUBERNETES_SERVICE_HOST"):
-    data_route.dependencies.append(Depends(verify_subscription_key_exist))
-
 
 @data_route.get(
     "",
@@ -165,7 +159,6 @@ async def upload_files(
         batches = ceil(len(files) / batch_size)
         for i in range(batches):
             batch_files = files[i * batch_size : (i + 1) * batch_size]
-            print(f"Uploading batch {i+1} of {batches}...")
             tasks = [
                 upload_file_async(file, container_client, overwrite)
                 for file in batch_files
@@ -177,13 +170,11 @@ async def upload_files(
                 database_name="graphrag", container_name="container-store"
             )
         )
-        container_store_client.upsert_item(
-            {
-                "id": sanitized_storage_name,
-                "human_readable_name": storage_name,
-                "type": "data",
-            }
-        )
+        container_store_client.upsert_item({
+            "id": sanitized_storage_name,
+            "human_readable_name": storage_name,
+            "type": "data",
+        })
         return BaseResponse(status="File upload successful.")
     except Exception:
         reporter.on_error("Error uploading files.", details={"files": files})

backend/src/api/experimental.py

@@ -1,13 +1,11 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
-import os
 from io import BytesIO
 
 import networkx as nx
 from fastapi import (
     APIRouter,
-    Depends,
     HTTPException,
 )
 from fastapi.responses import StreamingResponse
@@ -16,7 +14,6 @@
 from src.api.common import (
     sanitize_name,
     validate_index_file_exist,
-    verify_subscription_key_exist,
 )
 from src.models import GraphDataResponse
 from src.reporting import ReporterSingleton
@@ -29,9 +26,6 @@
     tags=["Graph Operations"],
 )
 
-if os.getenv("KUBERNETES_SERVICE_HOST"):
-    graph_route.dependencies.append(Depends(verify_subscription_key_exist))
-
 
 @graph_route.get(
     "/graphml/{index_name}",

backend/src/api/graph.py

@@ -14,7 +14,6 @@
 from datashaper import WorkflowCallbacksManager
 from fastapi import (
     APIRouter,
-    Depends,
     HTTPException,
     UploadFile,
 )
@@ -36,7 +35,6 @@
     delete_blob_container,
     sanitize_name,
     validate_blob_container_name,
-    verify_subscription_key_exist,
 )
 from src.models import (
     BaseResponse,
@@ -63,9 +61,6 @@
     tags=["Index Operations"],
 )
 
-if os.getenv("KUBERNETES_SERVICE_HOST"):
-    index_route.dependencies.append(Depends(verify_subscription_key_exist))
-
 
 @index_route.post(
     "",
@@ -172,13 +167,11 @@ async def _start_indexing_pipeline(index_name: str):
     container_store_client = get_database_container_client(
         database_name="graphrag", container_name="container-store"
     )
-    container_store_client.upsert_item(
-        {
-            "id": sanitized_index_name,
-            "human_readable_name": index_name,
-            "type": "index",
-        }
-    )
+    container_store_client.upsert_item({
+        "id": sanitized_index_name,
+        "human_readable_name": index_name,
+        "type": "index",
+    })
 
     reporter = ReporterSingleton().get_instance()
     pipelinejob = PipelineJob()