Skip to content

Commit 072d98f

Browse files
nick863nick863
committed
Fixes for indexes
1 parent 4936c93 commit 072d98f

File tree

6 files changed

+53
-983
lines changed

6 files changed

+53
-983
lines changed

‎src/api/data/embeddings.csv

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
token,embedding,document_reference
1+
token,embedding,title
22
"# Information about product item_number: 1 TrailMaster X4 Tent, price $250, ## Brand OutdoorLiving","[0.07389036, -0.058623426, -0.03301825, 0.18448547, -0.13143188, -0.07857863, -0.03488154, 0.0669982, 0.20179601, -0.11420148, -0.044678822, -0.0114502, 0.09072005, -0.056259252, 0.11019441, 0.024122557, -0.0044253073, 0.12902763, -0.22119023, 0.20083432, 0.00994755, -0.03466115, -0.17743303, -0.26013893, 0.0009172432, 0.04033115, 0.01455568, -0.055578053, 0.066196784, -0.055738334, -0.003926928, -0.12221561, 0.062069505, -0.20916902, 0.05365466, -0.03233705, 0.10971356, -0.0036364153, 0.08358747, -0.037125498, 0.10017674, 0.07320916, 0.014976421, 0.076414816, 0.09047963, -0.0039845295, 0.05657982, -0.035743058, -0.110274546, -0.06667764, 0.17374653, -0.037706524, 0.03367942, 0.004590599, -0.04924688, -0.11973123, 0.03061401, -0.005700056, 0.29155436, 0.13279428, -0.040611647, -0.12486028, 0.08166407, 0.06327163, 0.17599049, -0.031675883, -0.051009994, -0.063992895, -0.007573361, -0.010999406, -0.070925124, -0.023361214, 0.024383018, -0.06775954, 0.081824355, 0.19089678, -0.16212602, -0.039890375, -0.010328221, -0.009461693, -0.1121178, 0.04628165, -0.17438766, -0.042635217, -0.06775954, -0.011620501, -0.0397902, -0.01597819, 0.034460798, -0.08975835, -0.15611543, 0.06423332, 0.11067525, -0.11764756, 0.055818476, -0.14729987, -0.15042539, -0.08110309, -0.08719383, -0.09769235]",product_info_1.md
33
Main Category: CAMPING & HIKING Sub Category: TENTS & SHELTERS Product Type: BACKPACKING TENTS ## Features,"[0.018007893, -0.01959113, 0.08387737, 0.06556194, -0.13659118, -0.08260167, 0.1815141, 0.09768228, 0.12565659, -0.047793243, -0.14324304, -0.12073603, -0.03849885, 0.040685765, 0.054126192, 0.047884364, -0.07809115, -0.006196264, -0.0772255, 0.25222442, 0.059274558, 0.03868109, 0.022461457, -0.018634353, -0.085289754, 0.057361003, 0.015262857, 0.111897245, 0.0005517124, 0.02442057, -0.026402464, -0.114904255, 0.0127570145, -0.17021503, 0.11335519, -0.03496789, 0.10606547, 0.04455843, 0.08155377, 0.06137035, 0.27846742, -0.03852163, 0.043009363, -0.06738437, 0.0656075, 0.079321295, 0.022996796, 0.00040897907, -0.022917066, 0.034307256, 0.085426435, -0.08820564, 0.049205627, 0.069343485, -0.02854382, -0.09959584, -0.058500025, -0.014101057, 0.21450008, 0.2416543, -0.036425833, -0.06606311, -0.036175247, 0.029591717, 0.13012154, 0.023042358, -0.030639615, -0.117911264, -0.04519628, -0.002273767, -0.015843756, 0.031254686, 0.015513441, -0.1861613, 0.016572729, 0.13321967, -0.13923371, 0.031482488, 0.0054302737, 0.036676418, -0.1421496, 0.25040197, -0.1794183, 0.05767993, 0.072942786, -0.063375026, -0.30106553, -0.07440073, -0.21741597, -0.045492426, 0.028794404, 0.040708546, -0.010330903, -0.05294161, -0.031528052, 0.05184815, -0.015046443, -0.0441256, -0.15846035, 0.07157596]",product_info_1.md
44
- Polyester material for durability - Spacious interior to accommodate multiple people - Easy setup with included instructions - Water-resistant construction to withstand light rain,"[-0.001641822, 0.01622261, -0.007005455, 0.098566696, -0.08483747, -0.10649543, -0.036472183, 0.01352058, 0.214827, -0.04166759, -0.061635483, 0.015888767, 0.10549391, 0.06726906, 0.07411281, -0.046362236, 0.07373724, -0.030212654, -0.081499055, 0.11651068, 0.057462465, 0.14296761, -0.06447314, -0.31264254, 0.17810443, -0.0068228855, 0.07394589, 0.027270677, -0.061718944, 0.00878942, 0.040979043, -0.108581945, -0.08058099, -0.20731556, 0.011089797, -0.14680679, 0.04694646, 0.06050877, -0.062094517, 0.041125096, 0.13812691, 0.048824318, 0.0539154, -0.20965245, 0.07440492, 0.09197333, 0.04110423, 0.27708843, -0.1375427, 0.16274773, 0.054499622, 0.050660446, -0.0016822481, -0.16199659, -0.05621056, -0.04294036, -0.10173819, -0.08262577, 0.0432742, 0.06071742, 0.090971805, -0.022847276, 0.021532776, 0.0049945815, 0.008142603, -0.04536071, -0.06034185, -0.020510387, -0.033384148, -0.05821361, 0.20063873, 0.02679078, -0.10716312, -0.18010747, 0.043107282, -0.009665755, 0.090971805, -0.29962274, -0.06977287, 0.14597219, 0.08166597, -0.021678831, -0.09297486, 0.009764863, 0.018340416, 0.09965169, -0.05846399, -0.03367626, 0.0004101295, 0.0861311, 0.006447314, 0.03597142, -0.07740949, 0.0084451465, 0.0070367525, -0.21382546, 0.017286729, -0.043649774, -0.10883232, 0.049784113]",product_info_1.md

‎src/api/routes.py

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ async def on_thread_message(self, message: ThreadMessage) -> Optional[str]:
6565

6666
logger.info("MyEventHandler: Received completed message")
6767
annotations = []
68+
# Get file annotations for the file search.
6869
for annotation in (a.as_dict() for a in message.file_citation_annotations):
6970
file_id = annotation["file_citation"]["file_id"]
7071
logger.info(f"Fetching file with ID for annotation {file_id}")
@@ -73,14 +74,21 @@ async def on_thread_message(self, message: ThreadMessage) -> Optional[str]:
7374
logger.info(f"File name for annotation: {annotation['file_name']}")
7475
annotations.append(annotation)
7576

77+
# Get url annotation for the index search.
78+
for url_annotation in message.url_citation_annotations:
79+
annotation = url_annotation.as_dict()
80+
annotation["file_name"] = annotation['url_citation']['title']
81+
logger.info(f"File name for annotation: {annotation['file_name']}")
82+
annotations.append(annotation)
83+
7684
stream_data = {
7785
'content': message.text_messages[0].text.value,
7886
'annotations': annotations,
7987
'type': "completed_message"
8088
}
8189
return serialize_sse_event(stream_data)
8290
except Exception as e:
83-
logger.error("Error in event handler for thread message", exc_info=True)
91+
logger.error(f"Error in event handler for thread message: {e}", exc_info=True)
8492
return None
8593

8694
async def on_thread_run(self, run: ThreadRun) -> Optional[str]:
@@ -130,7 +138,7 @@ async def get_result(thread_id: str, agent_id: str, ai_client : AIProjectClient)
130138
) as stream:
131139
logger.info("Successfully created stream; starting to process events")
132140
async for event in stream:
133-
_, event_data_obj, event_func_return_val = event
141+
_, _, event_func_return_val = event
134142
logger.debug(f"Received event: {event}")
135143
if event_func_return_val:
136144
logger.info(f"Yielding event: {event_func_return_val}")

‎src/api/search_index_manager.py

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ async def upload_documents(self, embeddings_file: str) -> None:
104104
'embedId': str(index),
105105
'token': row['token'],
106106
'embedding': json.loads(row['embedding']),
107-
'document_reference': row['document_reference']
107+
'title': row['title']
108108
}
109109
)
110110
index += 1
@@ -153,7 +153,7 @@ async def _format_search_results(self, response: AsyncSearchItemPaged[Dict]) ->
153153
:param response: The search results.
154154
:return: The formatted response string.
155155
"""
156-
results = [f"{result['token']}, source: {result['document_reference']}" async for result in response]
156+
results = [f"{result['token']}, source: {result['title']}" async for result in response]
157157
return "\n------\n".join(results)
158158

159159
async def semantic_search(self, message: str) -> str:
@@ -166,8 +166,8 @@ async def semantic_search(self, message: str) -> str:
166166
self._raise_if_no_index()
167167
response = await self._get_client().search(
168168
search_text=message,
169-
query_type="semantic",
170-
search_fields=['token', 'document_reference'],
169+
query_type="full",
170+
search_fields=['token', 'title'],
171171
semantic_configuration_name=SearchIndexManager._SEMANTIC_CONFIG,
172172
)
173173
return await self._format_search_results(response)
@@ -188,7 +188,7 @@ async def search(self, message: str) -> str:
188188
)
189189
response = await self._get_client().search(
190190
vector_queries=[vector_query],
191-
select=['token', 'document_reference'],
191+
select=['token', 'title'],
192192
)
193193
# This lag is necessary, despite it is not described in documentation.
194194
time.sleep(1)
@@ -249,7 +249,7 @@ async def _index_create(self, vector_index_dimensions: int) -> SearchIndex:
249249
vector_search_profile_name=SearchIndexManager._EMBEDDING_CONFIG
250250
),
251251
SearchField(name="token", searchable=True, type=SearchFieldDataType.String, hidden=False),
252-
SearchField(name="document_reference", type=SearchFieldDataType.String, hidden=False),
252+
SearchField(name="title", type=SearchFieldDataType.String, hidden=False),
253253
]
254254
vector_search = VectorSearch(
255255
profiles=[
@@ -278,7 +278,7 @@ async def _index_create(self, vector_index_dimensions: int) -> SearchIndex:
278278
SemanticConfiguration(
279279
name=SearchIndexManager._SEMANTIC_CONFIG,
280280
prioritized_fields=SemanticPrioritizedFields(
281-
title_field=SemanticField(field_name="document_reference"),
281+
title_field=SemanticField(field_name="title"),
282282
content_fields=[
283283
SemanticField(field_name="token"),
284284
]
@@ -344,7 +344,7 @@ async def build_embeddings_file(
344344
# For each token build the embedding, which will be used in the search.
345345
batch_size = 2000
346346
with open(output_file, 'w') as fp:
347-
writer = csv.DictWriter(fp, fieldnames=['token', 'embedding', 'document_reference'])
347+
writer = csv.DictWriter(fp, fieldnames=['token', 'embedding', 'title'])
348348
writer.writeheader()
349349
for i in range(0, len(sentence_tokens), batch_size):
350350
emedding = (await self._embedding_client.embed(
@@ -356,7 +356,7 @@ async def build_embeddings_file(
356356
writer.writerow({
357357
'token': token,
358358
'embedding': json.dumps(float_data['embedding']),
359-
'document_reference': reference})
359+
'title': reference})
360360

361361
async def close(self):
362362
"""Close the closeable resources, associated with SearchIndexManager."""

0 commit comments

Comments
 (0)