Skip to content

Commit 9ebe2dc

Browse files
mohityadav766mohityadav766
committed
Merge remote-tracking branch 'origin/fix-indexing' into fix-indexing
2 parents 52acc7a + 2e55e54 commit 9ebe2dc

File tree

22 files changed

+1014
-103
lines changed

22 files changed

+1014
-103
lines changed

‎openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/IncidentPaginationIT.java‎

Lines changed: 234 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,234 @@
1+
package org.openmetadata.it.tests;
2+
3+
import static org.awaitility.Awaitility.await;
4+
import static org.junit.jupiter.api.Assertions.assertEquals;
5+
import static org.junit.jupiter.api.Assertions.assertNotNull;
6+
import static org.junit.jupiter.api.Assertions.assertTrue;
7+
8+
import java.time.Duration;
9+
import java.util.ArrayList;
10+
import java.util.List;
11+
import org.junit.jupiter.api.BeforeAll;
12+
import org.junit.jupiter.api.Test;
13+
import org.junit.jupiter.api.TestInstance;
14+
import org.openmetadata.it.bootstrap.SharedEntities;
15+
import org.openmetadata.it.util.SdkClients;
16+
import org.openmetadata.schema.api.data.CreateDatabase;
17+
import org.openmetadata.schema.api.data.CreateDatabaseSchema;
18+
import org.openmetadata.schema.api.data.CreateTable;
19+
import org.openmetadata.schema.api.tests.CreateTestCase;
20+
import org.openmetadata.schema.api.tests.CreateTestCaseResolutionStatus;
21+
import org.openmetadata.schema.entity.data.Database;
22+
import org.openmetadata.schema.entity.data.Table;
23+
import org.openmetadata.schema.tests.TestCase;
24+
import org.openmetadata.schema.tests.type.Severity;
25+
import org.openmetadata.schema.tests.type.TestCaseResolutionStatus;
26+
import org.openmetadata.schema.tests.type.TestCaseResolutionStatusTypes;
27+
import org.openmetadata.schema.type.Column;
28+
import org.openmetadata.schema.type.ColumnDataType;
29+
import org.openmetadata.sdk.client.OpenMetadataClient;
30+
import org.openmetadata.sdk.models.ListParams;
31+
import org.openmetadata.sdk.models.ListResponse;
32+
33+
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
34+
public class IncidentPaginationIT {
35+
36+
private static final int TEST_DATA_SIZE = 11;
37+
private static final int PAGE_SIZE = 5;
38+
private OpenMetadataClient client;
39+
private List<TestCase> testCases;
40+
private String databaseSchemaFqn;
41+
42+
@BeforeAll
43+
public void setup() throws Exception {
44+
client = SdkClients.adminClient();
45+
testCases = new ArrayList<>();
46+
47+
long ts = System.currentTimeMillis();
48+
Database database =
49+
client
50+
.databases()
51+
.create(
52+
new CreateDatabase()
53+
.withName("pagination_test_db_" + ts)
54+
.withService(SharedEntities.get().MYSQL_SERVICE.getFullyQualifiedName()));
55+
databaseSchemaFqn =
56+
client
57+
.databaseSchemas()
58+
.create(
59+
new CreateDatabaseSchema()
60+
.withName("pagination_test_schema_" + ts)
61+
.withDatabase(database.getFullyQualifiedName()))
62+
.getFullyQualifiedName();
63+
64+
Table table = createTestTable();
65+
String testDefFqn =
66+
client
67+
.testDefinitions()
68+
.list(new ListParams().withLimit(1))
69+
.getData()
70+
.get(0)
71+
.getFullyQualifiedName();
72+
73+
for (int i = 0; i < TEST_DATA_SIZE; i++) {
74+
TestCase testCase = createTestCase(table, i, testDefFqn);
75+
testCases.add(testCase);
76+
createIncidentStatus(testCase);
77+
}
78+
79+
await()
80+
.atMost(Duration.ofMinutes(3))
81+
.pollInterval(Duration.ofSeconds(5))
82+
.until(
83+
() -> {
84+
try {
85+
ListParams params =
86+
new ListParams().withLimit(TEST_DATA_SIZE + 10).withLatest(true);
87+
ListResponse<TestCaseResolutionStatus> response =
88+
client.testCaseResolutionStatuses().searchList(params);
89+
return response.getPaging().getTotal() >= TEST_DATA_SIZE;
90+
} catch (Exception e) {
91+
return false;
92+
}
93+
});
94+
}
95+
96+
@Test
97+
public void testPaginationFirstPage() throws Exception {
98+
ListParams params = new ListParams().withLimit(PAGE_SIZE).withOffset(0).withLatest(true);
99+
100+
ListResponse<TestCaseResolutionStatus> response =
101+
client.testCaseResolutionStatuses().searchList(params);
102+
103+
assertNotNull(response);
104+
assertEquals(
105+
PAGE_SIZE, response.getData().size(), "First page should return " + PAGE_SIZE + " results");
106+
assertTrue(
107+
response.getPaging().getTotal() >= TEST_DATA_SIZE,
108+
"Total should be at least " + TEST_DATA_SIZE);
109+
}
110+
111+
@Test
112+
public void testPaginationSecondPage() throws Exception {
113+
ListParams firstPageParams =
114+
new ListParams().withLimit(PAGE_SIZE).withOffset(0).withLatest(true);
115+
ListResponse<TestCaseResolutionStatus> firstPage =
116+
client.testCaseResolutionStatuses().searchList(firstPageParams);
117+
118+
ListParams secondPageParams =
119+
new ListParams().withLimit(PAGE_SIZE).withOffset(PAGE_SIZE).withLatest(true);
120+
ListResponse<TestCaseResolutionStatus> secondPage =
121+
client.testCaseResolutionStatuses().searchList(secondPageParams);
122+
123+
assertNotNull(secondPage);
124+
assertEquals(
125+
PAGE_SIZE,
126+
secondPage.getData().size(),
127+
"Second page should return " + PAGE_SIZE + " results");
128+
assertEquals(
129+
firstPage.getPaging().getTotal(),
130+
secondPage.getPaging().getTotal(),
131+
"Total count should be consistent across pages");
132+
133+
if (!firstPage.getData().isEmpty() && !secondPage.getData().isEmpty()) {
134+
Object firstItem = firstPage.getData().get(0);
135+
Object secondItem = secondPage.getData().get(0);
136+
assertTrue(!firstItem.equals(secondItem), "Pages should contain different data");
137+
}
138+
}
139+
140+
@Test
141+
public void testPaginationLastPage() throws Exception {
142+
ListParams params = new ListParams().withLimit(PAGE_SIZE).withOffset(0).withLatest(true);
143+
ListResponse<TestCaseResolutionStatus> firstPage =
144+
client.testCaseResolutionStatuses().searchList(params);
145+
int total = firstPage.getPaging().getTotal();
146+
int lastPageOffset = ((total - 1) / PAGE_SIZE) * PAGE_SIZE;
147+
148+
ListParams lastPageParams =
149+
new ListParams().withLimit(PAGE_SIZE).withOffset(lastPageOffset).withLatest(true);
150+
ListResponse<TestCaseResolutionStatus> lastPage =
151+
client.testCaseResolutionStatuses().searchList(lastPageParams);
152+
153+
assertNotNull(lastPage);
154+
assertTrue(
155+
lastPage.getData().size() > 0 && lastPage.getData().size() <= PAGE_SIZE,
156+
"Last page should have between 1 and " + PAGE_SIZE + " results");
157+
}
158+
159+
@Test
160+
public void testBackwardsCompatibilityNoParams() throws Exception {
161+
ListParams params = new ListParams().withLatest(true);
162+
163+
ListResponse<TestCaseResolutionStatus> response =
164+
client.testCaseResolutionStatuses().searchList(params);
165+
166+
assertNotNull(response);
167+
assertTrue(
168+
response.getPaging().getTotal() >= TEST_DATA_SIZE,
169+
"Total should be at least " + TEST_DATA_SIZE + " even without explicit pagination params");
170+
}
171+
172+
@Test
173+
public void testOffsetBeyondResults() throws Exception {
174+
ListParams params = new ListParams().withLimit(PAGE_SIZE).withOffset(10000).withLatest(true);
175+
176+
ListResponse<TestCaseResolutionStatus> response =
177+
client.testCaseResolutionStatuses().searchList(params);
178+
179+
assertNotNull(response);
180+
assertEquals(0, response.getData().size(), "Offset beyond results should return empty list");
181+
assertTrue(response.getPaging().getTotal() > 0, "Total should still be accurate");
182+
}
183+
184+
@Test
185+
public void testFilteredTotalCountIsExact() throws Exception {
186+
String targetFqn = testCases.get(0).getFullyQualifiedName();
187+
ListParams params =
188+
new ListParams()
189+
.withLimit(PAGE_SIZE)
190+
.withOffset(0)
191+
.withLatest(true)
192+
.addFilter("testCaseFQN", targetFqn);
193+
194+
ListResponse<TestCaseResolutionStatus> response =
195+
client.testCaseResolutionStatuses().searchList(params);
196+
197+
assertNotNull(response);
198+
assertEquals(1, response.getData().size(), "Filter should return exactly 1 incident");
199+
assertEquals(
200+
1,
201+
response.getPaging().getTotal(),
202+
"Total count must reflect only the filtered group, not all groups");
203+
}
204+
205+
private Table createTestTable() throws Exception {
206+
CreateTable createTable =
207+
new CreateTable()
208+
.withName("pagination_test_table_" + System.currentTimeMillis())
209+
.withDatabaseSchema(databaseSchemaFqn)
210+
.withColumns(List.of(new Column().withName("id").withDataType(ColumnDataType.BIGINT)));
211+
212+
return client.tables().create(createTable);
213+
}
214+
215+
private TestCase createTestCase(Table table, int index, String testDefFqn) throws Exception {
216+
CreateTestCase createTestCase =
217+
new CreateTestCase()
218+
.withName("pagination_test_case_" + index)
219+
.withEntityLink("<#E::table::" + table.getFullyQualifiedName() + "::columns::id>")
220+
.withTestDefinition(testDefFqn);
221+
222+
return client.testCases().create(createTestCase);
223+
}
224+
225+
private void createIncidentStatus(TestCase testCase) throws Exception {
226+
CreateTestCaseResolutionStatus createStatus =
227+
new CreateTestCaseResolutionStatus()
228+
.withTestCaseResolutionStatusType(TestCaseResolutionStatusTypes.New)
229+
.withTestCaseReference(testCase.getFullyQualifiedName())
230+
.withSeverity(Severity.Severity2);
231+
232+
client.testCaseResolutionStatuses().create(createStatus);
233+
}
234+
}

‎openmetadata-mcp/src/main/java/org/openmetadata/mcp/tools/RootCauseAnalysisTool.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,10 @@ private Map<String, Object> addTestCaseResultForTestSuite(Map<String, Object> no
195195
testResultTimeSeriesRepository.getFields("testCaseStatus,result,testResultValue"),
196196
searchListFilter,
197197
"testCaseFQN.keyword",
198+
null,
199+
null,
200+
null,
201+
null,
198202
null);
199203
if (testCaseResults.getData() != null && !testCaseResults.getData().isEmpty()) {
200204
testCaseResult.put("testCaseResults", testCaseResults.getData());

‎openmetadata-sdk/src/main/java/org/openmetadata/sdk/models/ListParams.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
*/
99
public class ListParams {
1010
private Integer limit;
11+
private Integer offset;
12+
private Boolean latest;
1113
private String after;
1214
private String before;
1315
private String fields;
@@ -142,6 +144,32 @@ public ListParams withPipelineType(String pipelineType) {
142144
return setPipelineType(pipelineType);
143145
}
144146

147+
public Integer getOffset() {
148+
return offset;
149+
}
150+
151+
public ListParams setOffset(Integer offset) {
152+
this.offset = offset;
153+
return this;
154+
}
155+
156+
public ListParams withOffset(Integer offset) {
157+
return setOffset(offset);
158+
}
159+
160+
public Boolean getLatest() {
161+
return latest;
162+
}
163+
164+
public ListParams setLatest(Boolean latest) {
165+
this.latest = latest;
166+
return this;
167+
}
168+
169+
public ListParams withLatest(boolean latest) {
170+
return setLatest(latest);
171+
}
172+
145173
public ListParams withLimit(Integer limit) {
146174
return setLimit(limit);
147175
}
@@ -182,6 +210,12 @@ public Map<String, String> toQueryParams() {
182210
if (limit != null) {
183211
params.put("limit", limit.toString());
184212
}
213+
if (offset != null) {
214+
params.put("offset", offset.toString());
215+
}
216+
if (latest != null) {
217+
params.put("latest", latest.toString());
218+
}
185219
if (after != null) {
186220
params.put("after", after);
187221
}
@@ -198,6 +232,8 @@ public Map<String, String> toQueryParams() {
198232
public ListParams copy() {
199233
ListParams copy = new ListParams();
200234
copy.limit = this.limit;
235+
copy.offset = this.offset;
236+
copy.latest = this.latest;
201237
copy.after = this.after;
202238
copy.before = this.before;
203239
copy.fields = this.fields;

‎openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/DistributedSearchIndexExecutor.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -649,11 +649,6 @@ private void runWorkerLoop(
649649
partition.getEntityType(),
650650
e.getMessage(),
651651
e);
652-
try {
653-
coordinator.failPartition(partition.getId(), e.getMessage());
654-
} catch (Exception fe) {
655-
LOG.error("Failed to mark partition {} as failed", partition.getId(), fe);
656-
}
657652
} finally {
658653
activePartitions.remove(partition.getId());
659654
}

‎openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/distributed/EntityCompletionTracker.java‎

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -108,28 +108,6 @@ public void recordPartitionComplete(String entityType, boolean partitionFailed)
108108
}
109109
}
110110

111-
private void promoteIfReady(String entityType) {
112-
if (promotedEntities.add(entityType)) {
113-
AtomicInteger failed = failedPartitions.get(entityType);
114-
boolean success = (failed == null || failed.get() == 0);
115-
116-
LOG.info(
117-
"Entity '{}' all partitions complete (success={}, failedPartitions={}, job {})",
118-
entityType,
119-
success,
120-
failed != null ? failed.get() : 0,
121-
jobId);
122-
123-
if (onEntityComplete != null) {
124-
try {
125-
onEntityComplete.accept(entityType, success);
126-
} catch (Exception e) {
127-
LOG.error("Error in entity completion callback for '{}' (job {})", entityType, jobId, e);
128-
}
129-
}
130-
}
131-
}
132-
133111
/**
134112
* Check if an entity has already been promoted.
135113
*

0 commit comments

Comments
 (0)