Updates Deid tests (#42029)

* Adding text encoding doc, fixing test file upload path

* directory separator

* updating samples

* property name

---------

Co-authored-by: Josiah Vinson <[email protected]>

Author: jovinson-ms

sdk/healthdataaiservices/azure-health-deidentification/README.md

@@ -48,10 +48,10 @@ Here's an example of setting an environment variable in Bash using Azure CLI:
 
 ```bash
 # Get the service URL for the resource
-export AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT=$(az deidservice show --name "<resource-name>" --resource-group "<resource-group-name>" --query "properties.serviceUrl")
+export HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT=$(az deidservice show --name "<resource-name>" --resource-group "<resource-group-name>" --query "properties.serviceUrl")
 ```
 
-Optionally, save the service URL as an environment variable named `AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT` for the sample client initialization code.
+Optionally, save the service URL as an environment variable named `HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT` for the sample client initialization code.
 
 Create a client with the endpoint and credential:
 <!-- SNIPPET: examples.create_client -->
@@ -62,7 +62,7 @@ from azure.identity import DefaultAzureCredential
 import os
 
 
-endpoint = os.environ["AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT"]
+endpoint = os.environ["HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT"]
 credential = DefaultAzureCredential()
 client = DeidentificationClient(endpoint, credential)
 ```
@@ -77,6 +77,12 @@ Given an input text, the de-identification service can perform three main operat
 - `Redact` returns output text where detected PHI entities are replaced with placeholder text. For example `John` replaced with `[name]`.
 - `Surrogate` returns output text where detected PHI entities are replaced with realistic replacement values. For example, `My name is John Smith` could become `My name is Tom Jones`.
 
+### String Encoding
+When using the `Tag` operation, the service will return the locations of PHI entities in the input text. These locations will be represented as offsets and lengths, each of which is a [StringIndex][string_index] containing
+three properties corresponding to three different text encodings. **Python applications should use the `code_point` property.**
+
+For more on text encoding, see [Character encoding in .NET][character_encoding].
+
 ### Available endpoints
 There are two ways to interact with the de-identification service. You can send text directly, or you can create jobs 
 to de-identify documents in Azure Storage.
@@ -94,7 +100,7 @@ from azure.health.deidentification.models import (
 from azure.identity import DefaultAzureCredential
 import os
 
-endpoint = os.environ["AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT"]
+endpoint = os.environ["HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT"]
 credential = DefaultAzureCredential()
 client = DeidentificationClient(endpoint, credential)
 
@@ -136,7 +142,7 @@ Your target Azure Storage account and container where documents will be written
 
 Set the following environment variables, updating the storage account and container with real values:
 ```bash
-export AZURE_STORAGE_ACCOUNT_LOCATION="https://<storageaccount>.blob.core.windows.net/<container>"
+export HEALTHDATAAISERVICES_STORAGE_ACCOUNT_LOCATION="https://<storageaccount>.blob.core.windows.net/<container>"
 export INPUT_PREFIX="example_patient_1"
 export OUTPUT_PREFIX="_output"
 ```
@@ -156,9 +162,9 @@ from azure.identity import DefaultAzureCredential
 import os
 import uuid
 
-endpoint = os.environ["AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT"]
-storage_location = os.environ["AZURE_STORAGE_ACCOUNT_LOCATION"]
-inputPrefix = os.environ["INPUT_PREFIX"]
+endpoint = os.environ["HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT"]
+storage_location = os.environ["HEALTHDATAAISERVICES_STORAGE_ACCOUNT_LOCATION"]
+inputPrefix = os.environ.get("INPUT_PREFIX", "example_patient_1")
 outputPrefix = os.environ.get("OUTPUT_PREFIX", "_output")
 
 credential = DefaultAzureCredential()
@@ -208,7 +214,7 @@ from azure.health.deidentification.models import (
 from azure.identity import DefaultAzureCredential
 import os
 
-endpoint = os.environ["AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT"]
+endpoint = os.environ["HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT"]
 credential = DefaultAzureCredential()
 client = DeidentificationClient(endpoint, credential)
 
@@ -244,7 +250,7 @@ from azure.health.deidentification.models import (
 from azure.identity import DefaultAzureCredential
 import os
 
-endpoint = os.environ["AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT"]
+endpoint = os.environ["HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT"]
 credential = DefaultAzureCredential()
 client = DeidentificationClient(endpoint, credential)
 
@@ -272,7 +278,7 @@ from azure.health.deidentification.models import (
 from azure.identity import DefaultAzureCredential
 import os
 
-endpoint = os.environ["AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT"]
+endpoint = os.environ["HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT"]
 credential = DefaultAzureCredential()
 client = DeidentificationClient(endpoint, credential)
 
@@ -349,6 +355,8 @@ additional questions or comments.
 [pip]: https://pypi.org/project/pip/
 [azure_sub]: https://azure.microsoft.com/free/
 [deid_quickstart]: https://learn.microsoft.com/azure/healthcare-apis/deidentification/quickstart
+[string_index]: https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/healthdataaiservices/azure-health-deidentification/azure/health/deidentification/models/_models.py#L548
+[character_encoding]: https://learn.microsoft.com/dotnet/standard/base-types/character-encoding-introduction
 [deid_redact]: https://learn.microsoft.com/azure/healthcare-apis/deidentification/redaction-format
 [deid_rbac]: https://learn.microsoft.com/azure/healthcare-apis/deidentification/manage-access-rbac
 [deid_managed_identity]: https://learn.microsoft.com/azure/healthcare-apis/deidentification/managed-identities

sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/deidentify_documents_async.py

@@ -15,8 +15,8 @@
     python deidentify_documents_async.py
 
     Set the environment variables with your own values before running the sample:
-    1) AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT - the service URL endpoint for a de-identification service.
-    2) AZURE_STORAGE_ACCOUNT_LOCATION - an Azure Storage container endpoint, like "https://<storageaccount>.blob.core.windows.net/<container>".
+    1) HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT - the service URL endpoint for a de-identification service.
+    2) HEALTHDATAAISERVICES_STORAGE_ACCOUNT_LOCATION - an Azure Storage container endpoint, like "https://<storageaccount>.blob.core.windows.net/<container>".
     3) INPUT_PREFIX - the prefix of the input document name(s) in the container.
         For example, providing "folder1" would create a job that would process documents like "https://<storageaccount>.blob.core.windows.net/<container>/folder1/document1.txt".
 """
@@ -36,9 +36,9 @@
 
 
 async def deidentify_documents_async():
-    endpoint = os.environ["AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT"]
-    storage_location = os.environ["AZURE_STORAGE_ACCOUNT_LOCATION"]
-    inputPrefix = os.environ["INPUT_PREFIX"]
+    endpoint = os.environ["HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT"]
+    storage_location = os.environ["HEALTHDATAAISERVICES_STORAGE_ACCOUNT_LOCATION"]
+    inputPrefix = os.environ.get("INPUT_PREFIX", "example_patient_1")
     outputPrefix = "_output"
 
     credential = DefaultAzureCredential()

sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/deidentify_text_redact_async.py

@@ -14,7 +14,7 @@
     python deidentify_text_redact_async.py
 
     Set the environment variables with your own values before running the sample:
-    1) AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT - the service URL endpoint for a de-identification service.
+    1) HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT - the service URL endpoint for a de-identification service.
 """
 
 
@@ -30,7 +30,7 @@
 
 
 async def deidentify_text_redact_async():
-    endpoint = os.environ["AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT"]
+    endpoint = os.environ["HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT"]
     credential = DefaultAzureCredential()
     client = DeidentificationClient(endpoint, credential)
 

sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/deidentify_text_surrogate_async.py

@@ -14,7 +14,7 @@
     python deidentify_text_surrogate_async.py
 
     Set the environment variables with your own values before running the sample:
-    1) AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT - the service URL endpoint for a de-identification service.
+    1) HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT - the service URL endpoint for a de-identification service.
 """
 
 
@@ -30,7 +30,7 @@
 
 
 async def deidentify_text_surrogate_async():
-    endpoint = os.environ["AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT"]
+    endpoint = os.environ["HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT"]
     credential = DefaultAzureCredential()
     client = DeidentificationClient(endpoint, credential)
 

sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/deidentify_text_tag_async.py

@@ -14,7 +14,7 @@
     python deidentify_text_tag_async.py
 
     Set the environment variables with your own values before running the sample:
-    1) AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT - the service URL endpoint for a de-identification service.
+    1) HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT - the service URL endpoint for a de-identification service.
 """
 
 
@@ -30,7 +30,7 @@
 
 
 async def deidentify_text_tag_async():
-    endpoint = os.environ["AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT"]
+    endpoint = os.environ["HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT"]
     credential = DefaultAzureCredential()
     client = DeidentificationClient(endpoint, credential)
 

sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/list_job_documents_async.py

@@ -14,8 +14,8 @@
     python list_job_documents_async.py
 
     Set the environment variables with your own values before running the sample:
-    1) AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT - the service URL endpoint for a de-identification service.
-    2) AZURE_STORAGE_ACCOUNT_LOCATION - an Azure Storage container endpoint, like "https://<storageaccount>.blob.core.windows.net/<container>".
+    1) HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT - the service URL endpoint for a de-identification service.
+    2) HEALTHDATAAISERVICES_STORAGE_ACCOUNT_LOCATION - an Azure Storage container endpoint, like "https://<storageaccount>.blob.core.windows.net/<container>".
     3) INPUT_PREFIX - the prefix of the input document name(s) in the container.
         For example, providing "folder1" would create a job that would process documents like "https://<storageaccount>.blob.core.windows.net/<container>/folder1/document1.txt".
 """
@@ -35,9 +35,9 @@
 
 
 async def list_job_documents_async():
-    endpoint = os.environ["AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT"]
-    storage_location = os.environ["AZURE_STORAGE_ACCOUNT_LOCATION"]
-    inputPrefix = os.environ["INPUT_PREFIX"]
+    endpoint = os.environ["HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT"]
+    storage_location = os.environ["HEALTHDATAAISERVICES_STORAGE_ACCOUNT_LOCATION"]
+    inputPrefix = os.environ.get("INPUT_PREFIX", "example_patient_1")
     outputPrefix = "_output"
 
     credential = DefaultAzureCredential()

sdk/healthdataaiservices/azure-health-deidentification/samples/async_samples/list_jobs_async.py

@@ -13,7 +13,7 @@
     python list_jobs_async.py
 
     Set the environment variables with your own values before running the sample:
-    1) AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT - the service URL endpoint for a de-identification service.
+    1) HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT - the service URL endpoint for a de-identification service.
 """
 
 
@@ -24,7 +24,7 @@
 
 
 async def list_jobs_async():
-    endpoint = os.environ["AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT"]
+    endpoint = os.environ["HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT"]
     credential = DefaultAzureCredential()
     client = DeidentificationClient(endpoint, credential)
 

sdk/healthdataaiservices/azure-health-deidentification/samples/deidentify_documents.py

@@ -15,8 +15,8 @@
     python deidentify_documents.py
 
     Set the environment variables with your own values before running the sample:
-    1) AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT - the service URL endpoint for a de-identification service.
-    2) AZURE_STORAGE_ACCOUNT_LOCATION - an Azure Storage container endpoint, like "https://<storageaccount>.blob.core.windows.net/<container>".
+    1) HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT - the service URL endpoint for a de-identification service.
+    2) HEALTHDATAAISERVICES_STORAGE_ACCOUNT_LOCATION - an Azure Storage container endpoint, like "https://<storageaccount>.blob.core.windows.net/<container>".
     3) INPUT_PREFIX - the prefix of the input document name(s) in the container. 
         For example, providing "folder1" would create a job that would process documents like "https://<storageaccount>.blob.core.windows.net/<container>/folder1/document1.txt".
     4) OUTPUT_PREFIX - the prefix of the output document name(s) in the container. This will appear as a folder which will be created if it does not exist, and defaults to "_output" if not provided.
@@ -37,9 +37,9 @@ def deidentify_documents():
     import os
     import uuid
 
-    endpoint = os.environ["AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT"]
-    storage_location = os.environ["AZURE_STORAGE_ACCOUNT_LOCATION"]
-    inputPrefix = os.environ["INPUT_PREFIX"]
+    endpoint = os.environ["HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT"]
+    storage_location = os.environ["HEALTHDATAAISERVICES_STORAGE_ACCOUNT_LOCATION"]
+    inputPrefix = os.environ.get("INPUT_PREFIX", "example_patient_1")
     outputPrefix = os.environ.get("OUTPUT_PREFIX", "_output")
 
     credential = DefaultAzureCredential()

sdk/healthdataaiservices/azure-health-deidentification/samples/deidentify_text_redact.py

@@ -14,7 +14,7 @@
     python deidentify_text_redact.py
 
     Set the environment variables with your own values before running the sample:
-    1) AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT - the service URL endpoint for a de-identification service.
+    1) HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT - the service URL endpoint for a de-identification service.
 """
 
 
@@ -29,7 +29,7 @@ def deidentify_text_redact():
     from azure.identity import DefaultAzureCredential
     import os
 
-    endpoint = os.environ["AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT"]
+    endpoint = os.environ["HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT"]
     credential = DefaultAzureCredential()
     client = DeidentificationClient(endpoint, credential)
 

sdk/healthdataaiservices/azure-health-deidentification/samples/deidentify_text_surrogate.py

@@ -14,7 +14,7 @@
     python deidentify_text_surrogate.py
 
     Set the environment variables with your own values before running the sample:
-    1) AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT - the service URL endpoint for a de-identification service.
+    1) HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT - the service URL endpoint for a de-identification service.
 """
 
 
@@ -29,7 +29,7 @@ def deidentify_text_surrogate():
     from azure.identity import DefaultAzureCredential
     import os
 
-    endpoint = os.environ["AZURE_HEALTH_DEIDENTIFICATION_ENDPOINT"]
+    endpoint = os.environ["HEALTHDATAAISERVICES_DEID_SERVICE_ENDPOINT"]
     credential = DefaultAzureCredential()
     client = DeidentificationClient(endpoint, credential)