Skip to content

Commit e3602f9

Browse files
nick863nick863
authored
Add enterprise file search (Azure#47614)
* Add enterprise and batch file search. * Fix
1 parent 9c6f9c6 commit e3602f9

13 files changed

+772
-3
lines changed

‎sdk/ai/Azure.AI.Projects/README.md‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ Use the AI Projects client library to:
2525
- [Create and execute run](#create-and-execute-run)
2626
- [Retrieve messages](#retrieve-messages)
2727
- [File search](#file-search)
28+
- [Enterprise File Search](#create-agent-with-enterprise-file-search)
29+
- [Code interpreter attachment](#create-message-with-code-interpreter-attachment)
2830
- [Function call](#function-call)
2931
- [Azure function call](#azure-function-call)
3032
- [Azure Function Call](#create-agent-with-azure-function-call)
@@ -210,6 +212,108 @@ Agent agent = agentResponse.Value;
210212
With a file ID association and a supported tool enabled, the agent will then be able to consume the associated
211213
data when running threads.
212214

215+
#### Create Agent with Enterprise File Search
216+
217+
We can upload file to Azure as it is shown in the example, or use the existing Azure blob storage. In the code below we demonstrate how this can be achieved. First we upload file to azure and create `VectorStoreDataSource`, which then is used to create vector store. This vector store is then given to the `FileSearchTool` constructor.
218+
219+
```C# Snippet:CreateVectorStoreBlob
220+
var ds = new VectorStoreDataSource(
221+
assetIdentifier: blobURI,
222+
assetType: VectorStoreDataSourceAssetType.UriAsset
223+
);
224+
var vectorStoreTask = await client.CreateVectorStoreAsync(
225+
name: "sample_vector_store",
226+
storeConfiguration: new VectorStoreConfiguration(
227+
dataSources: new List<VectorStoreDataSource> { ds }
228+
)
229+
);
230+
var vectorStore = vectorStoreTask.Value;
231+
232+
FileSearchToolResource fileSearchResource = new([vectorStore.Id], null);
233+
234+
List<ToolDefinition> tools = [new FileSearchToolDefinition()];
235+
Response<Agent> agentResponse = await client.CreateAgentAsync(
236+
model: modelName,
237+
name: "my-assistant",
238+
instructions: "You are helpful assistant.",
239+
tools: tools,
240+
toolResources: new ToolResources() { FileSearch = fileSearchResource }
241+
);
242+
```
243+
244+
We also can attach files to the existing vector store. In the code snippet below, we first create an empty vector store and add file to it.
245+
246+
```C# Snippet:BatchFileAttachment
247+
var ds = new VectorStoreDataSource(
248+
assetIdentifier: blobURI,
249+
assetType: VectorStoreDataSourceAssetType.UriAsset
250+
);
251+
var vectorStoreTask = await client.CreateVectorStoreAsync(
252+
name: "sample_vector_store"
253+
);
254+
var vectorStore = vectorStoreTask.Value;
255+
256+
var uploadTask = await client.CreateVectorStoreFileBatchAsync(
257+
vectorStoreId: vectorStore.Id,
258+
dataSources: new List<VectorStoreDataSource> { ds }
259+
);
260+
Console.WriteLine($"Created vector store file batch, vector store file batch ID: {uploadTask.Value.Id}");
261+
262+
FileSearchToolResource fileSearchResource = new([vectorStore.Id], null);
263+
```
264+
265+
#### Create Message with Code Interpreter Attachment
266+
267+
To attach a file with the context to the message, use the `MessageAttachment` class. To be able to process the attached file contents we need to provide the `List` with the single element `CodeInterpreterToolDefinition` as a `tools` parameter to both `CreateAgent` method and `MessageAttachment` class constructor.
268+
269+
Here is an example to pass `CodeInterpreterTool` as tool:
270+
271+
```C# Snippet:CreateAgentWithInterpreterTool
272+
AgentsClient client = new AgentsClient(connectionString, new DefaultAzureCredential());
273+
274+
List<ToolDefinition> tools = [ new CodeInterpreterToolDefinition() ];
275+
Response<Agent> agentResponse = await client.CreateAgentAsync(
276+
model: modelName,
277+
name: "my-assistant",
278+
instructions: "You are helpful assistant.",
279+
tools: tools
280+
);
281+
Agent agent = agentResponse.Value;
282+
283+
var fileResponse = await client.UploadFileAsync(filePath, AgentFilePurpose.Agents);
284+
var fileId = fileResponse.Value.Id;
285+
286+
var attachment = new MessageAttachment(
287+
fileId: fileId,
288+
tools: tools
289+
);
290+
291+
Response<AgentThread> threadResponse = await client.CreateThreadAsync();
292+
AgentThread thread = threadResponse.Value;
293+
294+
Response<ThreadMessage> messageResponse = await client.CreateMessageAsync(
295+
threadId: thread.Id,
296+
role: MessageRole.User,
297+
content: "What does the attachment say?",
298+
attachments: new List< MessageAttachment > { attachment}
299+
);
300+
ThreadMessage message = messageResponse.Value;
301+
```
302+
303+
Azure blob storage can be used as a message attachment. In this case, use `VectorStoreDataSource` as a data source:
304+
305+
```C# Snippet:CreateMessageAttachmentWithBlobStore
306+
var ds = new VectorStoreDataSource(
307+
assetIdentifier: blobURI,
308+
assetType: VectorStoreDataSourceAssetType.UriAsset
309+
);
310+
311+
var attachment = new MessageAttachment(
312+
ds: ds,
313+
tools: tools
314+
);
315+
```
316+
213317
#### Function call
214318

215319
Tools that reference caller-defined capabilities as functions can be provided to an agent to allow it to

‎sdk/ai/Azure.AI.Projects/api/Azure.AI.Projects.net8.0.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -968,6 +968,7 @@ protected virtual void JsonModelWriteCore(System.Text.Json.Utf8JsonWriter writer
968968
public partial class FileSearchToolResource : System.ClientModel.Primitives.IJsonModel<Azure.AI.Projects.FileSearchToolResource>, System.ClientModel.Primitives.IPersistableModel<Azure.AI.Projects.FileSearchToolResource>
969969
{
970970
public FileSearchToolResource() { }
971+
public FileSearchToolResource(System.Collections.Generic.IList<string> vectorStoreIds, System.Collections.Generic.IList<Azure.AI.Projects.VectorStoreConfigurations> vectorStores) { }
971972
public System.Collections.Generic.IList<string> VectorStoreIds { get { throw null; } }
972973
public System.Collections.Generic.IList<Azure.AI.Projects.VectorStoreConfigurations> VectorStores { get { throw null; } }
973974
protected virtual void JsonModelWriteCore(System.Text.Json.Utf8JsonWriter writer, System.ClientModel.Primitives.ModelReaderWriterOptions options) { }
@@ -1151,7 +1152,9 @@ protected virtual void JsonModelWriteCore(System.Text.Json.Utf8JsonWriter writer
11511152
}
11521153
public partial class MessageAttachment : System.ClientModel.Primitives.IJsonModel<Azure.AI.Projects.MessageAttachment>, System.ClientModel.Primitives.IPersistableModel<Azure.AI.Projects.MessageAttachment>
11531154
{
1155+
public MessageAttachment(Azure.AI.Projects.VectorStoreDataSource ds, System.Collections.Generic.List<Azure.AI.Projects.ToolDefinition> tools) { }
11541156
public MessageAttachment(System.Collections.Generic.IEnumerable<System.BinaryData> tools) { }
1157+
public MessageAttachment(string fileId, System.Collections.Generic.List<Azure.AI.Projects.ToolDefinition> tools) { }
11551158
public Azure.AI.Projects.VectorStoreDataSource DataSource { get { throw null; } set { } }
11561159
public string FileId { get { throw null; } set { } }
11571160
public System.Collections.Generic.IList<System.BinaryData> Tools { get { throw null; } }

‎sdk/ai/Azure.AI.Projects/api/Azure.AI.Projects.netstandard2.0.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -968,6 +968,7 @@ protected virtual void JsonModelWriteCore(System.Text.Json.Utf8JsonWriter writer
968968
public partial class FileSearchToolResource : System.ClientModel.Primitives.IJsonModel<Azure.AI.Projects.FileSearchToolResource>, System.ClientModel.Primitives.IPersistableModel<Azure.AI.Projects.FileSearchToolResource>
969969
{
970970
public FileSearchToolResource() { }
971+
public FileSearchToolResource(System.Collections.Generic.IList<string> vectorStoreIds, System.Collections.Generic.IList<Azure.AI.Projects.VectorStoreConfigurations> vectorStores) { }
971972
public System.Collections.Generic.IList<string> VectorStoreIds { get { throw null; } }
972973
public System.Collections.Generic.IList<Azure.AI.Projects.VectorStoreConfigurations> VectorStores { get { throw null; } }
973974
protected virtual void JsonModelWriteCore(System.Text.Json.Utf8JsonWriter writer, System.ClientModel.Primitives.ModelReaderWriterOptions options) { }
@@ -1151,7 +1152,9 @@ protected virtual void JsonModelWriteCore(System.Text.Json.Utf8JsonWriter writer
11511152
}
11521153
public partial class MessageAttachment : System.ClientModel.Primitives.IJsonModel<Azure.AI.Projects.MessageAttachment>, System.ClientModel.Primitives.IPersistableModel<Azure.AI.Projects.MessageAttachment>
11531154
{
1155+
public MessageAttachment(Azure.AI.Projects.VectorStoreDataSource ds, System.Collections.Generic.List<Azure.AI.Projects.ToolDefinition> tools) { }
11541156
public MessageAttachment(System.Collections.Generic.IEnumerable<System.BinaryData> tools) { }
1157+
public MessageAttachment(string fileId, System.Collections.Generic.List<Azure.AI.Projects.ToolDefinition> tools) { }
11551158
public Azure.AI.Projects.VectorStoreDataSource DataSource { get { throw null; } set { } }
11561159
public string FileId { get { throw null; } set { } }
11571160
public System.Collections.Generic.IList<System.BinaryData> Tools { get { throw null; } }

‎sdk/ai/Azure.AI.Projects/src/Custom/Agent/FileSearchToolResource.cs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System.Collections.Generic;
5+
6+
namespace Azure.AI.Projects
7+
{
8+
public partial class FileSearchToolResource
9+
{
10+
public FileSearchToolResource(
11+
IList<string> vectorStoreIds,
12+
IList<VectorStoreConfigurations> vectorStores
13+
)
14+
{
15+
VectorStoreIds = vectorStoreIds;
16+
if (vectorStores == null)
17+
VectorStores = new ChangeTrackingList<VectorStoreConfigurations>();
18+
else
19+
VectorStores = vectorStores;
20+
}
21+
}
22+
}

‎sdk/ai/Azure.AI.Projects/src/Custom/Agent/MessageAttachment.cs‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System;
5+
using System.ClientModel.Primitives;
6+
using System.Collections.Generic;
7+
using System.IO;
8+
using System.Text;
9+
using System.Text.Json;
10+
namespace Azure.AI.Projects;
11+
12+
public partial class MessageAttachment
13+
{
14+
public MessageAttachment(VectorStoreDataSource ds, List<ToolDefinition> tools)
15+
{
16+
FileId = null;
17+
DataSource = ds;
18+
Tools = serializeJson(tools);
19+
_serializedAdditionalRawData = null;
20+
}
21+
22+
public MessageAttachment(string fileId, List<ToolDefinition> tools)
23+
{
24+
FileId = fileId;
25+
DataSource = null;
26+
Tools = serializeJson(tools);
27+
_serializedAdditionalRawData = null;
28+
}
29+
30+
private static List<BinaryData> serializeJson<T>(List<T> definitions) where T: IJsonModel<T>
31+
{
32+
List<BinaryData> serializedDefinitions = new();
33+
foreach (IJsonModel<T> definition in definitions)
34+
{
35+
var stream = new MemoryStream();
36+
var writer = new Utf8JsonWriter(stream);
37+
definition.Write(writer, ModelReaderWriterOptions.Json);
38+
writer.Flush();
39+
string json = Encoding.UTF8.GetString(stream.ToArray());
40+
serializedDefinitions.Add(new BinaryData(json));
41+
}
42+
return serializedDefinitions;
43+
}
44+
}

‎sdk/ai/Azure.AI.Projects/tests/AIProjectsTestEnvironment.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,5 +11,6 @@ public class AIProjectsTestEnvironment : TestEnvironment
1111
public string BINGCONNECTIONNAME => GetRecordedVariable("BING_CONNECTION_NAME");
1212
public string MODELDEPLOYMENTNAME => GetRecordedVariable("MODEL_DEPLOYMENT_NAME");
1313
public string STORAGE_QUEUE_URI => GetRecordedVariable("STORAGE_QUEUE_URI");
14+
public string AZURE_BLOB_URI => GetRecordedVariable("AZURE_BLOB_URI");
1415
}
1516
}

‎sdk/ai/Azure.AI.Projects/tests/Samples/Agent/Sample_Agent_Enterprise_File_Search.cs‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.Net.Mail;
7+
using System.Threading;
8+
using System.Threading.Tasks;
9+
using Azure.Core.TestFramework;
10+
using NUnit.Framework;
11+
using NUnit.Framework.Internal.Execution;
12+
13+
namespace Azure.AI.Projects.Tests;
14+
15+
public partial class Sample_Agent_Enterprise_File_Search : SamplesBase<AIProjectsTestEnvironment>
16+
{
17+
[Test]
18+
public async Task EnterpriseFileSearch()
19+
{
20+
var connectionString = TestEnvironment.AzureAICONNECTIONSTRING;
21+
// For now we will take the File URI from the environment variables.
22+
// In future we may want to upload file to Azure here.
23+
var blobURI = TestEnvironment.AZURE_BLOB_URI;
24+
var modelName = TestEnvironment.MODELDEPLOYMENTNAME;
25+
AgentsClient client = new AgentsClient(connectionString, new DefaultAzureCredential());
26+
27+
#region Snippet:CreateVectorStoreBlob
28+
var ds = new VectorStoreDataSource(
29+
assetIdentifier: blobURI,
30+
assetType: VectorStoreDataSourceAssetType.UriAsset
31+
);
32+
var vectorStoreTask = await client.CreateVectorStoreAsync(
33+
name: "sample_vector_store",
34+
storeConfiguration: new VectorStoreConfiguration(
35+
dataSources: new List<VectorStoreDataSource> { ds }
36+
)
37+
);
38+
var vectorStore = vectorStoreTask.Value;
39+
40+
FileSearchToolResource fileSearchResource = new([vectorStore.Id], null);
41+
42+
List<ToolDefinition> tools = [new FileSearchToolDefinition()];
43+
Response<Agent> agentResponse = await client.CreateAgentAsync(
44+
model: modelName,
45+
name: "my-assistant",
46+
instructions: "You are helpful assistant.",
47+
tools: tools,
48+
toolResources: new ToolResources() { FileSearch = fileSearchResource }
49+
);
50+
#endregion
51+
Response<AgentThread> threadResponse = await client.CreateThreadAsync();
52+
AgentThread thread = threadResponse.Value;
53+
54+
Response<ThreadMessage> messageResponse = await client.CreateMessageAsync(
55+
threadId: thread.Id,
56+
role: MessageRole.User,
57+
content: "What feature does Smart Eyewear offer?"
58+
);
59+
ThreadMessage message = messageResponse.Value;
60+
61+
Response<ThreadRun> runResponse = await client.CreateRunAsync(
62+
thread.Id,
63+
agentResponse.Value.Id
64+
);
65+
ThreadRun run = runResponse.Value;
66+
67+
do
68+
{
69+
await Task.Delay(TimeSpan.FromMilliseconds(500));
70+
runResponse = await client.GetRunAsync(thread.Id, runResponse.Value.Id);
71+
}
72+
while (runResponse.Value.Status == RunStatus.Queued
73+
|| runResponse.Value.Status == RunStatus.InProgress);
74+
75+
Response<PageableList<ThreadMessage>> afterRunMessagesResponse
76+
= await client.GetMessagesAsync(thread.Id);
77+
IReadOnlyList<ThreadMessage> messages = afterRunMessagesResponse.Value.Data;
78+
WriteMessages(messages);
79+
80+
var delTask = await client.DeleteVectorStoreAsync(vectorStore.Id);
81+
if (delTask.Value.Deleted)
82+
{
83+
Console.WriteLine($"Deleted vector store {vectorStore.Id}");
84+
}
85+
else
86+
{
87+
Console.WriteLine($"Unable to delete vector store {vectorStore.Id}");
88+
}
89+
await client.DeleteAgentAsync(agentResponse.Value.Id);
90+
}
91+
92+
private void WriteMessages(IEnumerable<ThreadMessage> messages)
93+
{
94+
foreach (ThreadMessage threadMessage in messages)
95+
{
96+
Console.Write($"{threadMessage.CreatedAt:yyyy-MM-dd HH:mm:ss} - {threadMessage.Role,10}: ");
97+
foreach (MessageContent contentItem in threadMessage.ContentItems)
98+
{
99+
if (contentItem is MessageTextContent textItem)
100+
{
101+
Console.Write(textItem.Text);
102+
}
103+
else if (contentItem is MessageImageFileContent imageFileItem)
104+
{
105+
Console.Write($"<image from ID: {imageFileItem.FileId}");
106+
}
107+
Console.WriteLine();
108+
}
109+
}
110+
}
111+
}

‎sdk/ai/Azure.AI.Projects/tests/Samples/Agent/Sample_Agent_OpenAPI.cs‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,9 @@
77
using System.Collections.Generic;
88
using System.IO;
99
using System.Runtime.CompilerServices;
10-
using System.Text.Json.Serialization;
11-
using System.Text.Json;
1210
using System.Threading.Tasks;
1311
using Azure.Core.TestFramework;
1412
using NUnit.Framework;
15-
using Newtonsoft.Json.Linq;
1613

1714
namespace Azure.AI.Projects.Tests;
1815

0 commit comments

Comments
 (0)