Skip to content

Commit 697a9cb

Browse files
mattgotteinermattgotteiner
committed
sample indexer run
1 parent 2ff326d commit 697a9cb

File tree

2 files changed

+151
-12
lines changed

2 files changed

+151
-12
lines changed

‎Quickstart-Document-Permissions/document-permissions.ipynb‎

Lines changed: 144 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,21 +5,156 @@
55
"id": "aba4346f",
66
"metadata": {},
77
"source": [
8-
"## Document Permissions in Azure AI Search\n",
8+
"## Document Permissions in Azure AI Search"
9+
]
10+
},
11+
{
12+
"cell_type": "code",
13+
"execution_count": 10,
14+
"id": "0b40bb5b",
15+
"metadata": {},
16+
"outputs": [],
17+
"source": [
18+
"from dotenv import load_dotenv\n",
19+
"from azure.identity import DefaultAzureCredential, get_bearer_token_provider\n",
20+
"import os\n",
21+
"\n",
22+
"load_dotenv(override=True) # take environment variables from .env.\n",
23+
"\n",
24+
"# The following variables from your .env file are used in this notebook\n",
25+
"endpoint = os.environ[\"AZURE_SEARCH_ENDPOINT\"]\n",
26+
"credential = DefaultAzureCredential()\n",
27+
"index_name = os.getenv(\"AZURE_SEARCH_INDEX\", \"document-permissions-sample\")\n",
28+
"indexer_name = os.getenv(\"AZURE_SEARCH_INDEXER\", \"document-permissions-sample-indexer\")\n",
29+
"datasource_name = os.getenv(\"AZURE_SEARCH_DATASOURCE\", \"document-permissions-sample-datasource\")\n",
30+
"adls_gen2_account_name = os.getenv(\"AZURE_STORAGE_ACCOUNT_NAME\", \"documentpermissionssample\")\n",
31+
"adls_gen2_container_name = os.getenv(\"AZURE_STORAGE_CONTAINER_NAME\", \"documentpermissionssample\")\n",
32+
"adls_gen2_connection_string = os.environ[\"AZURE_STORAGE_CONNECTION_STRING\"]\n",
33+
"token_provider = get_bearer_token_provider(credential, \"https://search.azure.com/.default\")"
34+
]
35+
},
36+
{
37+
"cell_type": "code",
38+
"execution_count": 2,
39+
"id": "2f981cad",
40+
"metadata": {},
41+
"outputs": [
42+
{
43+
"name": "stdout",
44+
"output_type": "stream",
45+
"text": [
46+
"Index 'document-permissions-sample' created with permission filter option enabled.\n"
47+
]
48+
}
49+
],
50+
"source": [
51+
"from azure.search.documents.indexes.models import SearchField, SearchIndex, PermissionFilter, SearchIndexPermissionFilterOption\n",
52+
"from azure.search.documents.indexes import SearchIndexClient\n",
53+
"\n",
54+
"index_client = SearchIndexClient(endpoint=endpoint, credential=credential)\n",
55+
"index = SearchIndex(\n",
56+
" name=index_name,\n",
57+
" fields=[\n",
58+
" SearchField(name=\"id\", type=\"Edm.String\", key=True, filterable=True, sortable=True),\n",
59+
" SearchField(name=\"oid\", type=\"Collection(Edm.String)\", filterable=True, permission_filter=PermissionFilter.USER_IDS),\n",
60+
" SearchField(name=\"group\", type=\"Collection(Edm.String)\", filterable=True, permission_filter=PermissionFilter.GROUP_IDS),\n",
61+
" SearchField(name=\"metadata_storage_path\", type=\"Edm.String\", searchable=True),\n",
62+
" SearchField(name=\"metadata_storage_name\", type=\"Edm.String\", searchable=True)\n",
63+
" ],\n",
64+
" permission_filter_option=SearchIndexPermissionFilterOption.ENABLED\n",
65+
")\n",
66+
"\n",
67+
"index_client.create_or_update_index(index=index)\n",
68+
"print(f\"Index '{index_name}' created with permission filter option enabled.\")"
69+
]
70+
},
71+
{
72+
"cell_type": "code",
73+
"execution_count": 11,
74+
"id": "b25aaf7b",
75+
"metadata": {},
76+
"outputs": [
77+
{
78+
"name": "stdout",
79+
"output_type": "stream",
80+
"text": [
81+
"Datasource 'document-permissions-sample-datasource' created with permission filter option enabled.\n"
82+
]
83+
}
84+
],
85+
"source": [
86+
"from azure.search.documents.indexes.models import SearchIndexerDataSourceConnection, SearchIndexerDataSourceType, IndexerPermissionOption, SearchIndexerDataContainer, DataSourceCredentials\n",
87+
"from azure.search.documents.indexes import SearchIndexerClient\n",
88+
"indexer_client = SearchIndexerClient(endpoint=endpoint, credential=credential)\n",
89+
"datasource = SearchIndexerDataSourceConnection(\n",
90+
" name=datasource_name,\n",
91+
" type=SearchIndexerDataSourceType.ADLS_GEN2,\n",
92+
" connection_string=adls_gen2_connection_string,\n",
93+
" container=SearchIndexerDataContainer(name=adls_gen2_container_name),\n",
94+
" indexer_permission_options=[IndexerPermissionOption.GROUP_IDS]\n",
95+
")\n",
96+
"\n",
97+
"indexer_client.create_or_update_data_source_connection(datasource)\n",
98+
"print(f\"Datasource '{datasource_name}' created with permission filter option enabled.\")"
99+
]
100+
},
101+
{
102+
"cell_type": "code",
103+
"execution_count": null,
104+
"id": "2ce7eb5e",
105+
"metadata": {},
106+
"outputs": [
107+
{
108+
"ename": "HttpResponseError",
109+
"evalue": "() Field mapping specifies target field 'metadata_group_ids' that is not present in the target index\nCode: \nMessage: Field mapping specifies target field 'metadata_group_ids' that is not present in the target index",
110+
"output_type": "error",
111+
"traceback": [
112+
"\u001b[31m---------------------------------------------------------------------------\u001b[39m",
113+
"\u001b[31mHttpResponseError\u001b[39m Traceback (most recent call last)",
114+
"\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[14]\u001b[39m\u001b[32m, line 13\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mazure\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01msearch\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mdocuments\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mindexes\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mmodels\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m SearchIndexer, FieldMapping\n\u001b[32m 3\u001b[39m indexer = SearchIndexer(\n\u001b[32m 4\u001b[39m name=indexer_name,\n\u001b[32m 5\u001b[39m target_index_name=index_name,\n\u001b[32m (...)\u001b[39m\u001b[32m 10\u001b[39m ]\n\u001b[32m 11\u001b[39m )\n\u001b[32m---> \u001b[39m\u001b[32m13\u001b[39m \u001b[43mindexer_client\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcreate_or_update_indexer\u001b[49m\u001b[43m(\u001b[49m\u001b[43mindexer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 14\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mIndexer \u001b[39m\u001b[33m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mindexer_name\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m created\u001b[39m\u001b[33m\"\u001b[39m)\n",
115+
"\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\magottei\\source\\azure-search-python-samples-pr\\.venv\\Lib\\site-packages\\azure\\core\\tracing\\decorator.py:119\u001b[39m, in \u001b[36mdistributed_trace.<locals>.decorator.<locals>.wrapper_use_tracer\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m 117\u001b[39m \u001b[38;5;66;03m# If tracing is disabled globally and user didn't explicitly enable it, don't trace.\u001b[39;00m\n\u001b[32m 118\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m user_enabled \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m (\u001b[38;5;129;01mnot\u001b[39;00m tracing_enabled \u001b[38;5;129;01mand\u001b[39;00m user_enabled \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[32m--> \u001b[39m\u001b[32m119\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 121\u001b[39m \u001b[38;5;66;03m# Merge span is parameter is set, but only if no explicit parent are passed\u001b[39;00m\n\u001b[32m 122\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m merge_span \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m passed_in_parent:\n",
116+
"\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\magottei\\source\\azure-search-python-samples-pr\\.venv\\Lib\\site-packages\\azure\\search\\documents\\indexes\\_search_indexer_client.py:140\u001b[39m, in \u001b[36mSearchIndexerClient.create_or_update_indexer\u001b[39m\u001b[34m(self, indexer, match_condition, skip_indexer_reset_requirement_for_cache, disable_cache_reprocessing_change_detection, **kwargs)\u001b[39m\n\u001b[32m 138\u001b[39m name = indexer.name\n\u001b[32m 139\u001b[39m patched_indexer = indexer._to_generated() \u001b[38;5;66;03m# pylint:disable=protected-access\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m140\u001b[39m result = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_client\u001b[49m\u001b[43m.\u001b[49m\u001b[43mindexers\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcreate_or_update\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 141\u001b[39m \u001b[43m \u001b[49m\u001b[43mindexer_name\u001b[49m\u001b[43m=\u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 142\u001b[39m \u001b[43m \u001b[49m\u001b[43mindexer\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpatched_indexer\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 143\u001b[39m \u001b[43m \u001b[49m\u001b[43mprefer\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mreturn=representation\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[32m 144\u001b[39m \u001b[43m \u001b[49m\u001b[43merror_map\u001b[49m\u001b[43m=\u001b[49m\u001b[43merror_map\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 145\u001b[39m \u001b[43m \u001b[49m\u001b[43mskip_indexer_reset_requirement_for_cache\u001b[49m\u001b[43m=\u001b[49m\u001b[43mskip_indexer_reset_requirement_for_cache\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 146\u001b[39m \u001b[43m \u001b[49m\u001b[43mdisable_cache_reprocessing_change_detection\u001b[49m\u001b[43m=\u001b[49m\u001b[43mdisable_cache_reprocessing_change_detection\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 147\u001b[39m \u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\n\u001b[32m 148\u001b[39m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 149\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m cast(SearchIndexer, SearchIndexer._from_generated(result))\n",
117+
"\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\magottei\\source\\azure-search-python-samples-pr\\.venv\\Lib\\site-packages\\azure\\core\\tracing\\decorator.py:119\u001b[39m, in \u001b[36mdistributed_trace.<locals>.decorator.<locals>.wrapper_use_tracer\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m 117\u001b[39m \u001b[38;5;66;03m# If tracing is disabled globally and user didn't explicitly enable it, don't trace.\u001b[39;00m\n\u001b[32m 118\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m user_enabled \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m (\u001b[38;5;129;01mnot\u001b[39;00m tracing_enabled \u001b[38;5;129;01mand\u001b[39;00m user_enabled \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[32m--> \u001b[39m\u001b[32m119\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 121\u001b[39m \u001b[38;5;66;03m# Merge span is parameter is set, but only if no explicit parent are passed\u001b[39;00m\n\u001b[32m 122\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m merge_span \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m passed_in_parent:\n",
118+
"\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\magottei\\source\\azure-search-python-samples-pr\\.venv\\Lib\\site-packages\\azure\\search\\documents\\indexes\\_generated\\operations\\_indexers_operations.py:981\u001b[39m, in \u001b[36mIndexersOperations.create_or_update\u001b[39m\u001b[34m(self, indexer_name, prefer, indexer, if_match, if_none_match, skip_indexer_reset_requirement_for_cache, disable_cache_reprocessing_change_detection, request_options, **kwargs)\u001b[39m\n\u001b[32m 979\u001b[39m map_error(status_code=response.status_code, response=response, error_map=error_map)\n\u001b[32m 980\u001b[39m error = \u001b[38;5;28mself\u001b[39m._deserialize.failsafe_deserialize(_models.ErrorResponse, pipeline_response)\n\u001b[32m--> \u001b[39m\u001b[32m981\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m HttpResponseError(response=response, model=error)\n\u001b[32m 983\u001b[39m deserialized = \u001b[38;5;28mself\u001b[39m._deserialize(\u001b[33m\"\u001b[39m\u001b[33mSearchIndexer\u001b[39m\u001b[33m\"\u001b[39m, pipeline_response.http_response)\n\u001b[32m 985\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mcls\u001b[39m:\n",
119+
"\u001b[31mHttpResponseError\u001b[39m: () Field mapping specifies target field 'metadata_group_ids' that is not present in the target index\nCode: \nMessage: Field mapping specifies target field 'metadata_group_ids' that is not present in the target index"
120+
]
121+
}
122+
],
123+
"source": [
124+
"from azure.search.documents.indexes.models import SearchIndexer, FieldMapping\n",
125+
"\n",
126+
"indexer = SearchIndexer(\n",
127+
" name=indexer_name,\n",
128+
" target_index_name=index_name,\n",
129+
" data_source_name=datasource_name,\n",
130+
" field_mappings=[\n",
131+
" FieldMapping(source_field_name=\"metadata_group_ids\", target_field_name=\"group\"),\n",
132+
" FieldMapping(source_field_name=\"metadata_user_ids\", target_field_name=\"oid\"),\n",
133+
" ]\n",
134+
")\n",
9135
"\n",
10-
"1. Setup index w/ permissions\n",
11-
"1. Setup data source w/ permissions\n",
12-
"1. Setup indexer\n",
13-
"1. Connect to App\n",
14-
"1. Setup authentication\n",
15-
"1. Show how to query\n",
16-
"1. Show pushing data with permissions into index"
136+
"indexer_client.create_or_update_indexer(indexer)\n",
137+
"print(f\"Indexer '{indexer_name}' created\")\n"
17138
]
18139
}
19140
],
20141
"metadata": {
142+
"kernelspec": {
143+
"display_name": ".venv",
144+
"language": "python",
145+
"name": "python3"
146+
},
21147
"language_info": {
22-
"name": "python"
148+
"codemirror_mode": {
149+
"name": "ipython",
150+
"version": 3
151+
},
152+
"file_extension": ".py",
153+
"mimetype": "text/x-python",
154+
"name": "python",
155+
"nbconvert_exporter": "python",
156+
"pygments_lexer": "ipython3",
157+
"version": "3.12.10"
23158
}
24159
},
25160
"nbformat": 4,

‎Quickstart-Document-Permissions/requirements.txt‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
1-
azure-search-documents==11.6.0b11
21
azure-identity
3-
openai
42
aiohttp
5-
ipykernel
3+
ipykernel
4+
dotenv
5+
requests
6+
msgraph-sdk
7+
8+
--extra-index-url https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/
9+
azure-search-documents==11.6.0a20250505003

0 commit comments

Comments
 (0)