Adds storageURL field to track file location (#1535)

* Adds storageURL field to track file location

* Fix tests, update mocks

* Adding script

* Test adls2 strategy

* Fixes to manageacl script to let you call update_storage_urls

* Add PNG for document intelligence

* Set facetable to false

Author: pamelafox

app/backend/app.py

@@ -260,8 +260,8 @@ async def upload(auth_claims: dict[str, Any]):
     file_io = io.BufferedReader(file_io)
     await file_client.upload_data(file_io, overwrite=True, metadata={"UploadedBy": user_oid})
     file_io.seek(0)
-    ingester = current_app.config[CONFIG_INGESTER]
-    await ingester.add_file(File(content=file_io, acls={"oids": [user_oid]}))
+    ingester: UploadUserFileStrategy = current_app.config[CONFIG_INGESTER]
+    await ingester.add_file(File(content=file_io, acls={"oids": [user_oid]}, url=file_client.url))
     return jsonify({"message": "File uploaded successfully"}), 200
 
 

app/backend/prepdocslib/blobmanager.py

@@ -53,10 +53,12 @@ async def upload_blob(self, file: File) -> Optional[List[str]]:
                 await container_client.create_container()
 
             # Re-open and upload the original file
-            with open(file.content.name, "rb") as reopened_file:
-                blob_name = BlobManager.blob_name_from_file_name(file.content.name)
-                logger.info("Uploading blob for whole file -> %s", blob_name)
-                await container_client.upload_blob(blob_name, reopened_file, overwrite=True)
+            if file.url is None:
+                with open(file.content.name, "rb") as reopened_file:
+                    blob_name = BlobManager.blob_name_from_file_name(file.content.name)
+                    logger.info("Uploading blob for whole file -> %s", blob_name)
+                    blob_client = await container_client.upload_blob(blob_name, reopened_file, overwrite=True)
+                    file.url = blob_client.url
 
             if self.store_page_images:
                 if os.path.splitext(file.content.name)[1].lower() == ".pdf":

app/backend/prepdocslib/filestrategy.py

@@ -87,7 +87,7 @@ async def run(self):
                         blob_image_embeddings: Optional[List[List[float]]] = None
                         if self.image_embeddings and blob_sas_uris:
                             blob_image_embeddings = await self.image_embeddings.create_embeddings(blob_sas_uris)
-                        await search_manager.update_content(sections, blob_image_embeddings)
+                        await search_manager.update_content(sections, blob_image_embeddings, url=file.url)
                 finally:
                     if file:
                         file.close()
@@ -124,7 +124,7 @@ async def add_file(self, file: File):
             logging.warning("Image embeddings are not currently supported for the user upload feature")
         sections = await parse_file(file, self.file_processors)
         if sections:
-            await self.search_manager.update_content(sections)
+            await self.search_manager.update_content(sections, url=file.url)
 
     async def remove_file(self, filename: str, oid: str):
         if filename is None or filename == "":

app/backend/prepdocslib/listfilestrategy.py

@@ -22,9 +22,10 @@ class File:
     This file might contain access control information about which users or groups can access it
     """
 
-    def __init__(self, content: IO, acls: Optional[dict[str, list]] = None):
+    def __init__(self, content: IO, acls: Optional[dict[str, list]] = None, url: Optional[str] = None):
         self.content = content
         self.acls = acls or {}
+        self.url = url
 
     def filename(self):
         return os.path.basename(self.content.name)
@@ -167,7 +168,7 @@ async def list(self) -> AsyncGenerator[File, None]:
                             acls["oids"].append(acl_parts[1])
                         if acl_parts[0] == "group" and "r" in acl_parts[2]:
                             acls["groups"].append(acl_parts[1])
-                    yield File(content=open(temp_file_path, "rb"), acls=acls)
+                    yield File(content=open(temp_file_path, "rb"), acls=acls, url=file_client.url)
                 except Exception as data_lake_exception:
                     logger.error(f"\tGot an error while reading {path} -> {data_lake_exception} --> skipping file")
                     try:

app/backend/prepdocslib/searchmanager.py

@@ -111,6 +111,12 @@ async def create_index(self, vectorizers: Optional[List[VectorSearchVectorizer]]
                     filterable=True,
                     facetable=True,
                 ),
+                SimpleField(
+                    name="storageUrl",
+                    type="Edm.String",
+                    filterable=True,
+                    facetable=False,
+                ),
             ]
             if self.use_acls:
                 fields.append(
@@ -182,7 +188,9 @@ async def create_index(self, vectorizers: Optional[List[VectorSearchVectorizer]]
             else:
                 logger.info("Search index %s already exists", self.search_info.index_name)
 
-    async def update_content(self, sections: List[Section], image_embeddings: Optional[List[List[float]]] = None):
+    async def update_content(
+        self, sections: List[Section], image_embeddings: Optional[List[List[float]]] = None, url: Optional[str] = None
+    ):
         MAX_BATCH_SIZE = 1000
         section_batches = [sections[i : i + MAX_BATCH_SIZE] for i in range(0, len(sections), MAX_BATCH_SIZE)]
 
@@ -209,6 +217,9 @@ async def update_content(self, sections: List[Section], image_embeddings: Option
                     }
                     for section_index, section in enumerate(batch)
                 ]
+                if url:
+                    for document in documents:
+                        document["storageUrl"] = url
                 if self.embeddings:
                     embeddings = await self.embeddings.create_embeddings(
                         texts=[section.split_page.text for section in batch]

app/frontend/src/components/UploadFile/UploadFile.tsx

@@ -120,7 +120,7 @@ export const UploadFile: React.FC<Props> = ({ className, disabled }: Props) => {
                             <div>
                                 <Label>Upload file:</Label>
                                 <input
-                                    accept=".txt, .md, .json, .jpg, .jpeg, .bmp, .heic, .tiff, .pdf, .docx, .xlsx, .pptx, .html"
+                                    accept=".txt, .md, .json, .png, .jpg, .jpeg, .bmp, .heic, .tiff, .pdf, .docx, .xlsx, .pptx, .html"
                                     className={styles.chooseFiles}
                                     type="file"
                                     onChange={handleUploadFile}

docs/deploy_features.md

@@ -136,6 +136,21 @@ Then you'll need to run `azd up` to provision an Azure Data Lake Storage Gen2 ac
 When the user uploads a document, it will be stored in a directory in that account with the same name as the user's Entra object id,
 and will have ACLs associated with that directory. When the ingester runs, it will also set the `oids` of the indexed chunks to the user's Entra object id.
 
+If you are enabling this feature on an existing index, you should also update your index to have the new `storageUrl` field:
+
+```shell
+./scripts/manageacl.ps1  -v --acl-action enable_acls
+```
+
+And then update existing search documents with the storage URL of the main Blob container:
+
+```shell
+./scripts/manageacl.ps1  -v --acl-action update_storage_urls --url <https://YOUR-MAIN-STORAGE-ACCOUNT.blob.core.windows.net/content/>
+```
+
+Going forward, all uploaded documents will have their `storageUrl` set in the search index.
+This is necessary to disambiguate user-uploaded documents from admin-uploaded documents.
+
 
 ## Enabling CORS for an alternate frontend
 

docs/login_and_acl.md

@@ -150,17 +150,46 @@ Manually enable document level access control on a search index and manually set
 
 Run `azd up` or use `azd env set` to manually set `AZURE_SEARCH_SERVICE` and `AZURE_SEARCH_INDEX` environment variables prior to running the script.
 
-The script supports the following commands. Note that the syntax is the same regardless of whether [manageacl.ps1](../scripts/manageacl.ps1) or [manageacl.sh](../scripts/manageacl.sh) is used.
-* `./scripts/manageacl.ps1 --acl-action enable_acls`: Creates the required `oids` (User ID) and `groups` (Group IDs) [security filter](https://learn.microsoft.com/azure/search/search-security-trimming-for-azure-search) fields for document level access control on your index. Does nothing if these fields already exist.
-  * Example usage: `./scripts/manageacl.ps1 --acl-action enable_acls`
-* `./scripts/manageacl.ps1 --document [name-of-pdf.pdf] --acl-type [oids or groups]--acl-action view`: Prints access control values associated with either User IDs or Group IDs for a specific document.
-  * Example to view all Group IDs from the Benefit_Options PDF: `./scripts/manageacl.ps1 --document Benefit_Options.pdf --acl-type oids --acl-action view`.
-* `./scripts/manageacl.ps1 --document [name-of-pdf.pdf] --acl-type [oids or groups]--acl-action add --acl [ID of group or user]`: Adds an access control value associated with either User IDs or Group IDs for a specific document.
-  * Example to add a Group ID to the Benefit_Options PDF: `./scripts/manageacl.ps1 --document Benefit_Options.pdf --acl-type groups --acl-action add --acl xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`.
-* `./scripts/manageacl.ps1 --document [name-of-pdf.pdf] --acl-type [oids or groups]--acl-action remove_all`: Removes all access control values associated with either User IDs or Group IDs for a specific document.
-  * Example to remove all Group IDs from the Benefit_Options PDF: `./scripts/manageacl.ps1 --document Benefit_Options.pdf --acl-type groups --acl-action remove_all`.
-* `./scripts/manageacl.ps1 --document [name-of-pdf.pdf] --acl-type [oids or groups]--acl-action remove --acl [ID of group or user]`: Removes an access control value associated with either User IDs or Group IDs for a specific document.
-  * Example to remove a specific User ID from the Benefit_Options PDF: `./scripts/manageacl.ps1 --document Benefit_Options.pdf --acl-type oids --acl-action remove --acl xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`.
+The script supports the following commands. Note that the syntax is the same regardless of whether [manageacl.ps1](../scripts/manageacl.ps1) or [manageacl.sh](../scripts/manageacl.sh) is used. All commands support `-v` for verbose logging.
+* `./scripts/manageacl.ps1 --acl-action enable_acls`: Creates the required `oids` (User ID) and `groups` (Group IDs) [security filter](https://learn.microsoft.com/azure/search/search-security-trimming-for-azure-search) fields for document level access control on your index, as well as the `storageUrl` field for storing the Blob storage URL. Does nothing if these fields already exist.
+
+  Example usage:
+
+  ```shell
+  ./scripts/manageacl.ps1  -v --acl-action enable_acls
+  ```
+
+* `./scripts/manageacl.ps1 --acl-type [oids or groups]--acl-action view --url [https://url.pdf]`: Prints access control values associated with either User IDs or Group IDs for the document at the specified URL.
+
+  Example to view all Group IDs:
+
+  ```shell
+  ./scripts/manageacl.ps1 -v --acl-type groups --acl-action view --url https://st12345.blob.core.windows.net/content/Benefit_Options.pdf
+  ```
+
+* `./scripts/manageacl.ps1 --url [https://url.pdf] --acl-type [oids or groups]--acl-action add --acl [ID of group or user]`: Adds an access control value associated with either User IDs or Group IDs for the document at the specified URL.
+
+  Example to add a Group ID:
+
+  ```shell
+  ./scripts/manageacl.ps1 -v --acl-type groups --acl-action add --acl xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --url https://st12345.blob.core.windows.net/content/Benefit_Options.pdf
+  ```
+
+* `./scripts/manageacl.ps1 --url [https://url.pdf] --acl-type [oids or groups]--acl-action remove_all`: Removes all access control values associated with either User IDs or Group IDs for a specific document.
+
+  Example to remove all Group IDs:
+
+  ```shell
+  ./scripts/manageacl.ps1 -v --acl-type groups --acl-action remove_all --url https://st12345.blob.core.windows.net/content/Benefit_Options.pdf
+  ```
+
+* `./scripts/manageacl.ps1 --url [https://url.pdf] --acl-type [oids or groups]--acl-action remove --acl [ID of group or user]`: Removes an access control value associated with either User IDs or Group IDs for a specific document.
+
+  Example to remove a specific User ID:
+
+  ```shell
+  ./scripts/manageacl.ps1 -v --acl-type oids --acl-action remove --acl xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --url https://st12345.blob.core.windows.net/content/Benefit_Options.pdf
+  ```
 
 ### Azure Data Lake Storage Gen2 Setup