Skip to content

Commit bc36e0f

Browse files
phrockerphrocker
committed
fix old data source lookup
1 parent dd9ee06 commit bc36e0f

File tree

7 files changed

+528
-12
lines changed

7 files changed

+528
-12
lines changed

‎.azure.env‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
SENTRIUS_VERSION=1.1.104
1+
SENTRIUS_VERSION=1.1.105
22
SENTRIUS_SSH_VERSION=1.1.12
33
SENTRIUS_KEYCLOAK_VERSION=1.1.15
44
SENTRIUS_AGENT_VERSION=1.1.24

‎api/src/main/resources/templates/sso/agents/memory_search.html‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -291,13 +291,14 @@ <h2 class="toast-header">Agent Memory Search</h2>
291291
}
292292

293293
function displayResults(data) {
294-
totalPages = data.totalPages;
294+
totalPages = data.totalPages || 0;
295295

296296
// Update stats
297-
searchStats.textContent = `Found ${data.totalElements} memories (Page ${data.number + 1} of ${data.totalPages})`;
297+
searchStats.textContent = `Found ${data.totalElements || 0} memories (Page ${(data.number || 0) + 1} of ${data.totalPages || 0})`;
298298
searchStats.style.display = 'block';
299299

300-
if (data.content && data.content.length > 0) {
300+
// Validate that data.content is an array with items
301+
if (data.content && Array.isArray(data.content) && data.content.length > 0) {
301302
// Display memory items
302303
memoryResults.innerHTML = data.content.map(memory => `
303304
<div class="memory-item">
@@ -318,6 +319,7 @@ <h2 class="toast-header">Agent Memory Search</h2>
318319
paginationContainer.style.display = 'flex';
319320
noResults.style.display = 'none';
320321
} else {
322+
// No results - clear and show message
321323
memoryResults.innerHTML = '';
322324
paginationContainer.style.display = 'none';
323325
noResults.style.display = 'block';

‎api/src/main/resources/templates/sso/data/sources.html‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,12 @@ <h2 class="page-header">
106106
<button class="btn btn-success" id="retrieveExternalBtn">
107107
<i class="fas fa-download"></i> Retrieve from External Source
108108
</button>
109+
<div class="form-check form-check-inline ms-3">
110+
<input class="form-check-input" type="checkbox" id="useSemanticSearch" checked>
111+
<label class="form-check-label" for="useSemanticSearch">
112+
Use AI-powered search (slower but finds similar content)
113+
</label>
114+
</div>
109115
</div>
110116
</div>
111117
</div>
@@ -268,11 +274,13 @@ <h5 class="modal-title" id="retrieveExternalModalLabel">Retrieve External Data S
268274
const query = document.getElementById('searchQuery').value;
269275
const sourceType = document.getElementById('sourceTypeFilter').value;
270276
const markings = document.getElementById('markingsFilter').value;
277+
const useSemanticSearch = document.getElementById('useSemanticSearch').checked;
271278

272279
const searchRequest = {
273280
query: query || null,
274281
documentType: sourceType || null,
275-
markings: markings || null
282+
markings: markings || null,
283+
useSemanticSearch: useSemanticSearch
276284
};
277285

278286
fetch('/api/v1/documents/search', {
@@ -283,9 +291,20 @@ <h5 class="modal-title" id="retrieveExternalModalLabel">Retrieve External Data S
283291
},
284292
body: JSON.stringify(searchRequest)
285293
})
286-
.then(response => response.json())
294+
.then(response => {
295+
if (!response.ok) {
296+
throw new Error('Search request failed');
297+
}
298+
return response.json();
299+
})
287300
.then(data => {
288-
displaySources(data);
301+
// Ensure data is an array, even if API returns unexpected format
302+
if (Array.isArray(data)) {
303+
displaySources(data);
304+
} else {
305+
console.error('API returned unexpected format:', data);
306+
displaySources([]);
307+
}
289308
})
290309
.catch(err => {
291310
console.error('Failed to search data sources:', err);
@@ -297,16 +316,20 @@ <h5 class="modal-title" id="retrieveExternalModalLabel">Retrieve External Data S
297316
const sourceList = document.getElementById('sourceList');
298317
const noResults = document.getElementById('noResults');
299318

319+
// Always clear the source list first
300320
sourceList.innerHTML = '';
301321

302-
if (!sources || sources.length === 0) {
322+
// Validate that sources is an array and has items
323+
if (!sources || !Array.isArray(sources) || sources.length === 0) {
303324
showNoResults();
304325
return;
305326
}
306327

328+
// Hide no results message and show the grid
307329
noResults.style.display = 'none';
308330
sourceList.style.display = 'grid';
309331

332+
// Display each source card
310333
sources.forEach(source => {
311334
const card = createSourceCard(source);
312335
sourceList.appendChild(card);

‎api/src/test/java/io/sentrius/sso/controllers/api/documents/DocumentSearchIntegrationTest.java‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
package io.sentrius.sso.controllers.api.documents;
2+
3+
import io.sentrius.sso.core.config.SystemOptions;
4+
import io.sentrius.sso.core.dto.documents.DocumentSearchDTO;
5+
import io.sentrius.sso.core.model.documents.Document;
6+
import io.sentrius.sso.core.model.users.User;
7+
import io.sentrius.sso.core.services.ErrorOutputService;
8+
import io.sentrius.sso.core.services.UserService;
9+
import io.sentrius.sso.core.services.documents.DocumentService;
10+
import org.junit.jupiter.api.Test;
11+
import org.springframework.beans.factory.annotation.Autowired;
12+
import org.springframework.boot.test.context.SpringBootTest;
13+
import org.springframework.boot.test.mock.mockito.MockBean;
14+
15+
import java.util.Collections;
16+
import java.util.List;
17+
18+
import static org.junit.jupiter.api.Assertions.*;
19+
import static org.mockito.ArgumentMatchers.any;
20+
import static org.mockito.Mockito.when;
21+
22+
/**
23+
* Integration test to verify document search returns empty results when no matches found.
24+
*/
25+
@SpringBootTest
26+
class DocumentSearchIntegrationTest {
27+
28+
@MockBean
29+
private DocumentService documentService;
30+
31+
@MockBean
32+
private UserService userService;
33+
34+
@MockBean
35+
private SystemOptions systemOptions;
36+
37+
@MockBean
38+
private ErrorOutputService errorOutputService;
39+
40+
@Autowired
41+
private DocumentController documentController;
42+
43+
@Test
44+
void testSearchDocuments_NoMatchesReturnsEmptyList() {
45+
// Setup: Mock user service
46+
User mockUser = new User();
47+
mockUser.setUserId("test-user");
48+
when(userService.getOperatingUser(any(), any(), any())).thenReturn(mockUser);
49+
50+
// Setup: Search for non-existent content
51+
DocumentSearchDTO searchDTO = DocumentSearchDTO.builder()
52+
.query("XYZNONEXISTENTQUERY123456")
53+
.build();
54+
55+
// Setup: Mock service to return empty list (simulating no matches)
56+
when(documentService.searchDocuments(any(DocumentSearchDTO.class)))
57+
.thenReturn(Collections.emptyList());
58+
59+
// Execute search
60+
var response = documentController.searchDocuments(searchDTO, null, null);
61+
62+
// Verify: Should return empty list, not all documents
63+
assertNotNull(response);
64+
assertNotNull(response.getBody());
65+
assertTrue(response.getBody().isEmpty(),
66+
"Search with no matches should return empty list, not all documents");
67+
}
68+
69+
@Test
70+
void testSearchDocuments_WithFiltersButNoMatches() {
71+
// Setup: Mock user service
72+
User mockUser = new User();
73+
mockUser.setUserId("test-user");
74+
when(userService.getOperatingUser(any(), any(), any())).thenReturn(mockUser);
75+
76+
// Setup: Search with filters that don't match any documents
77+
DocumentSearchDTO searchDTO = DocumentSearchDTO.builder()
78+
.query("test query")
79+
.documentType("NONEXISTENT_TYPE")
80+
.markings("NONEXISTENT_MARKINGS")
81+
.build();
82+
83+
// Setup: Mock service to return empty list
84+
when(documentService.searchDocuments(any(DocumentSearchDTO.class)))
85+
.thenReturn(Collections.emptyList());
86+
87+
// Execute search
88+
var response = documentController.searchDocuments(searchDTO, null, null);
89+
90+
// Verify: Should return empty list
91+
assertNotNull(response);
92+
assertNotNull(response.getBody());
93+
assertTrue(response.getBody().isEmpty(),
94+
"Search with non-matching filters should return empty list");
95+
}
96+
}

‎dataplane/src/main/java/io/sentrius/sso/core/services/documents/DocumentService.java‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -126,17 +126,21 @@ public Optional<Document> getDocumentByName(String documentName) {
126126
* Search documents using hybrid text and vector search
127127
*/
128128
public List<Document> searchDocuments(DocumentSearchDTO searchDTO) {
129-
log.info("Searching documents with query: {}, type: {}, markings: {}",
130-
searchDTO.getQuery(), searchDTO.getDocumentType(), searchDTO.getMarkings());
129+
log.info("Searching documents with query: '{}', type: {}, markings: {}, useSemanticSearch: {}",
130+
searchDTO.getQuery(), searchDTO.getDocumentType(), searchDTO.getMarkings(),
131+
searchDTO.isUseSemanticSearch());
131132

132133
if (searchDTO.getQuery() == null || searchDTO.getQuery().trim().isEmpty()) {
134+
log.info("Query is null or empty, returning all documents with filters");
133135
return getAllDocuments(searchDTO);
134136
}
135137

136138
if (!searchDTO.isUseSemanticSearch() || embeddingService == null || !embeddingService.isAvailable()) {
139+
log.info("Using text search (semantic search disabled or unavailable)");
137140
return textSearchDocuments(searchDTO);
138141
}
139142

143+
log.info("Using hybrid search (semantic + text)");
140144
return hybridSearchDocuments(searchDTO);
141145
}
142146

@@ -331,16 +335,20 @@ private String buildTextForEmbedding(Document document) {
331335
}
332336

333337
private List<Document> textSearchDocuments(DocumentSearchDTO searchDTO) {
338+
log.debug("Performing text search with query: '{}'", searchDTO.getQuery());
334339
List<Document> results = documentRepository.searchByContent(searchDTO.getQuery());
340+
log.debug("Text search returned {} results before filtering", results.size());
335341

336342
// Apply filters
337343
results = applySearchFilters(results, searchDTO);
344+
log.debug("After filtering: {} results", results.size());
338345

339346
// Apply limit
340347
if (searchDTO.getLimit() != null && searchDTO.getLimit() > 0) {
341348
results = results.stream().limit(searchDTO.getLimit()).collect(Collectors.toList());
342349
}
343350

351+
log.info("Text search final result count: {}", results.size());
344352
return results;
345353
}
346354

@@ -396,13 +404,17 @@ private List<Document> hybridSearchDocuments(DocumentSearchDTO searchDTO) {
396404

397405
// Merge and sort by score
398406
Set<Long> seenIds = new HashSet<>();
399-
return Stream.concat(textResults.stream(), vectorResults.stream())
400-
.filter(doc -> seenIds.add(doc.getId()))
407+
List<Document> finalResults = Stream.concat(textResults.stream(), vectorResults.stream())
408+
.filter(doc -> seenIds.add(doc.getId())) // Deduplicate
409+
.filter(doc -> scores.containsKey(doc.getId())) // Only include docs with scores
401410
.sorted((a, b) -> Double.compare(
402411
scores.getOrDefault(b.getId(), 0.0),
403412
scores.getOrDefault(a.getId(), 0.0)))
404413
.limit(limit)
405414
.collect(Collectors.toList());
415+
416+
log.info("Hybrid search final result count: {}", finalResults.size());
417+
return finalResults;
406418

407419
} catch (Exception e) {
408420
log.error("Error in hybrid search, falling back to text search", e);

0 commit comments

Comments
 (0)