Skip to content

Commit 6f57765

Browse files
harshachharshach
authored
Fix integration tests (#25753)
* Fix - disk space in github workflows * Fix - disk space in github workflows * Fix - disk space in github workflows * Fix running tests with bulk apis * Fix running tests with bulk apis * Address comments; make awaitability for tests * Address comments
1 parent e4dd825 commit 6f57765

File tree

18 files changed

+278
-148
lines changed

18 files changed

+278
-148
lines changed

‎.github/workflows/maven-postgres-tests-build.yml‎

Lines changed: 3 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -10,38 +10,11 @@
1010
# limitations under the License.
1111

1212
name: Maven Postgres Tests CI
13+
# Disabled: Integration tests (openmetadata-integration-tests) now cover the same ground
14+
# with full Testcontainers-based PostgreSQL testing.
1315

1416
on:
15-
push:
16-
branches:
17-
- main
18-
paths:
19-
- "openmetadata-service/**"
20-
- "openmetadata-ui/**"
21-
- "!openmetadata-ui/src/main/resources/ui/playwright/doc-generator/**"
22-
- "!openmetadata-ui/src/main/resources/ui/playwright/docs/**"
23-
- "openmetadata-spec/src/main/resources/json/schema/**"
24-
- "openmetadata-dist/**"
25-
- "openmetadata-clients/**"
26-
- "openmetadata-sdk/**"
27-
- "common/**"
28-
- "pom.xml"
29-
- "yarn.lock"
30-
- "Makefile"
31-
- "bootstrap/**"
32-
pull_request_target:
33-
types: [labeled, opened, synchronize, reopened, ready_for_review]
34-
paths:
35-
- "openmetadata-service/**"
36-
- "openmetadata-spec/src/main/resources/json/schema/**"
37-
- "openmetadata-dist/**"
38-
- "openmetadata-clients/**"
39-
- "openmetadata-sdk/**"
40-
- "common/**"
41-
- "pom.xml"
42-
- "yarn.lock"
43-
- "Makefile"
44-
- "bootstrap/**"
17+
workflow_dispatch:
4518

4619
permissions:
4720
contents: read

‎bootstrap/sql/migrations/native/1.12.0/mysql/schemaChanges.sql‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,11 @@ CREATE TABLE IF NOT EXISTS learning_resource_entity (
246246
UNIQUE KEY fqnHash (fqnHash)
247247
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
248248

249+
-- Widen entityLink generated column from VARCHAR(255) to TEXT
250+
-- The entity link from workflow variables can exceed 255 characters for deeply nested entities
251+
ALTER TABLE workflow_instance_time_series
252+
MODIFY COLUMN entityLink TEXT GENERATED ALWAYS AS (json ->> '$.variables.global_relatedEntity');
253+
249254
-- Add process and vector stage columns to search_index_server_stats table
250255
-- These columns support the 4-stage pipeline model (Reader, Process, Sink, Vector) for search indexing stats
251256

‎bootstrap/sql/migrations/native/1.12.0/postgres/schemaChanges.sql‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,10 @@ CREATE TABLE IF NOT EXISTS learning_resource_entity (
262262
UNIQUE (fqnhash)
263263
);
264264

265+
-- Widen entityLink generated column from VARCHAR(255) to TEXT
266+
-- The entity link from workflow variables can exceed 255 characters for deeply nested entities
267+
ALTER TABLE workflow_instance_time_series ALTER COLUMN entityLink TYPE TEXT;
268+
265269
-- Add process and vector stage columns to search_index_server_stats table
266270
-- These columns support the 4-stage pipeline model (Reader, Process, Sink, Vector) for search indexing stats
267271

‎openmetadata-integration-tests/src/test/java/org/openmetadata/it/bootstrap/TestSuiteBootstrap.java‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,7 @@ private void startDatabase() {
203203
mysql.withPassword("test");
204204
mysql.withStartupTimeoutSeconds(240);
205205
mysql.withConnectTimeoutSeconds(240);
206+
mysql.withTmpFs(java.util.Map.of("/var/lib/mysql", "rw,size=2g"));
206207
mysql.withCreateContainerCmdModifier(
207208
cmd ->
208209
cmd.getHostConfig()
@@ -223,6 +224,29 @@ private void startDatabase() {
223224
postgres.withPassword("test");
224225
postgres.withStartupTimeoutSeconds(240);
225226
postgres.withConnectTimeoutSeconds(240);
227+
postgres.withCommand(
228+
"postgres",
229+
"-c",
230+
"max_wal_size=512MB",
231+
"-c",
232+
"min_wal_size=64MB",
233+
"-c",
234+
"wal_level=minimal",
235+
"-c",
236+
"max_wal_senders=0",
237+
"-c",
238+
"checkpoint_completion_target=0.5",
239+
"-c",
240+
"checkpoint_timeout=30s",
241+
"-c",
242+
"shared_buffers=128MB",
243+
"-c",
244+
"fsync=off",
245+
"-c",
246+
"synchronous_commit=off",
247+
"-c",
248+
"full_page_writes=off");
249+
postgres.withTmpFs(java.util.Map.of("/var/lib/postgresql/data", "rw,size=2g"));
226250
postgres.withCreateContainerCmdModifier(
227251
cmd ->
228252
cmd.getHostConfig()
@@ -250,6 +274,7 @@ private void startSearch() {
250274
opensearch.withEnv("DISABLE_SECURITY_PLUGIN", "true");
251275
opensearch.withEnv("OPENSEARCH_JAVA_OPTS", "-Xms1g -Xmx1g");
252276
opensearch.withStartupAttempts(3);
277+
opensearch.withTmpFs(java.util.Map.of("/usr/share/opensearch/data", "rw,size=1g"));
253278
opensearch.withCreateContainerCmdModifier(
254279
cmd ->
255280
cmd.getHostConfig()
@@ -274,6 +299,7 @@ private void startSearch() {
274299
elasticsearch.withEnv("xpack.security.enabled", "false");
275300
elasticsearch.withEnv("ES_JAVA_OPTS", "-Xms1g -Xmx1g");
276301
elasticsearch.withStartupAttempts(3);
302+
elasticsearch.withTmpFs(java.util.Map.of("/usr/share/elasticsearch/data", "rw,size=1g"));
277303
elasticsearch.setWaitStrategy(
278304
new LogMessageWaitStrategy()
279305
.withRegEx(".*(\"message\":\\s?\"started[\\s?|\"].*|] started\n$)")
@@ -300,6 +326,7 @@ private void startFuseki() {
300326
.withExposedPorts(FUSEKI_PORT)
301327
.withEnv("ADMIN_PASSWORD", FUSEKI_ADMIN_PASSWORD)
302328
.withEnv("FUSEKI_DATASET_1", FUSEKI_DATASET)
329+
.withTmpFs(java.util.Map.of("/fuseki/databases", "rw,size=256m,uid=100,gid=101"))
303330
.waitingFor(
304331
Wait.forHttp("/$/ping")
305332
.forPort(FUSEKI_PORT)
@@ -555,7 +582,7 @@ private void configureRdf(OpenMetadataApplicationConfig config) {
555582
config.setRdfConfiguration(rdfConfig);
556583
}
557584

558-
rdfConfig.setEnabled(true);
585+
rdfConfig.setEnabled(false);
559586
rdfConfig.setBaseUri(java.net.URI.create("https://open-metadata.org/"));
560587
rdfConfig.setStorageType(RdfConfiguration.StorageType.FUSEKI);
561588
rdfConfig.setRemoteEndpoint(java.net.URI.create(fusekiEndpoint));

‎openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/AppsResourceIT.java‎

Lines changed: 13 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,10 @@
2121
import static org.junit.jupiter.api.Assumptions.assumeFalse;
2222

2323
import com.fasterxml.jackson.core.type.TypeReference;
24+
import java.time.Duration;
2425
import java.util.HashMap;
2526
import java.util.Map;
27+
import org.awaitility.Awaitility;
2628
import org.junit.jupiter.api.BeforeAll;
2729
import org.junit.jupiter.api.Test;
2830
import org.junit.jupiter.api.extension.ExtendWith;
@@ -371,29 +373,17 @@ void test_triggerApp_200(TestNamespace ns) throws Exception {
371373

372374
waitForAppJobCompletion(appName);
373375

374-
// Retry trigger with backoff in case a previous job is still finishing up
375-
int maxTriggerRetries = 10;
376-
boolean triggered = false;
377-
Exception lastException = null;
378-
379-
for (int i = 0; i < maxTriggerRetries && !triggered; i++) {
380-
try {
381-
Apps.trigger(appName).run();
382-
triggered = true;
383-
} catch (Exception e) {
384-
lastException = e;
385-
if (e.getMessage() != null && e.getMessage().contains("already running")) {
386-
// Job is still running from previous test, wait and retry
387-
Thread.sleep(2000);
388-
} else {
389-
throw e; // Re-throw if it's a different error
390-
}
391-
}
392-
}
393-
394-
if (!triggered && lastException != null) {
395-
throw lastException;
396-
}
376+
// Wait for any in-flight job to finish, then trigger
377+
Awaitility.await("Trigger " + appName)
378+
.atMost(Duration.ofMinutes(2))
379+
.pollInterval(Duration.ofSeconds(3))
380+
.ignoreExceptionsMatching(
381+
e -> e.getMessage() != null && e.getMessage().contains("already running"))
382+
.until(
383+
() -> {
384+
Apps.trigger(appName).run();
385+
return true;
386+
});
397387

398388
Thread.sleep(2000);
399389

‎openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/ColumnBulkUpdateIT.java‎

Lines changed: 25 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -430,21 +430,28 @@ void test_bulkUpdateColumns_withGlossaryTerms(TestNamespace ns) throws Exception
430430
&& t.getSource() == TagLabel.TagSource.GLOSSARY);
431431
});
432432

433-
Table updatedTable2 = getTableWithColumns(table2.getId().toString());
434-
Column col2 =
435-
updatedTable2.getColumns().stream()
436-
.filter(c -> c.getName().equals(columnName))
437-
.findFirst()
438-
.orElseThrow();
433+
Awaitility.await()
434+
.pollInterval(1, TimeUnit.SECONDS)
435+
.atMost(30, TimeUnit.SECONDS)
436+
.until(
437+
() -> {
438+
Table updatedTable2 = getTableWithColumns(table2.getId().toString());
439+
Column col2 =
440+
updatedTable2.getColumns().stream()
441+
.filter(c -> c.getName().equals(columnName))
442+
.findFirst()
443+
.orElse(null);
439444

440-
assertNotNull(col2.getTags(), "Table2 column should have glossary term tags");
441-
assertTrue(
442-
col2.getTags().stream()
443-
.anyMatch(
444-
t ->
445-
t.getTagFQN().equals(personalInfoTerm.getFullyQualifiedName())
446-
&& t.getSource() == TagLabel.TagSource.GLOSSARY),
447-
"Table2 column should have the PersonalInfo glossary term");
445+
if (col2 == null || col2.getTags() == null || col2.getTags().isEmpty()) {
446+
return false;
447+
}
448+
449+
return col2.getTags().stream()
450+
.anyMatch(
451+
t ->
452+
t.getTagFQN().equals(personalInfoTerm.getFullyQualifiedName())
453+
&& t.getSource() == TagLabel.TagSource.GLOSSARY);
454+
});
448455
}
449456

450457
@Test
@@ -509,7 +516,10 @@ void test_bulkUpdateColumns_mixedTagsAndGlossaryTerms(TestNamespace ns) throws E
509516
.findFirst()
510517
.orElse(null);
511518

512-
if (col == null || col.getTags() == null || col.getTags().size() < 2) {
519+
if (col == null
520+
|| col.getTags() == null
521+
|| col.getTags().size() < 2
522+
|| col.getDisplayName() == null) {
513523
return false;
514524
}
515525

‎openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/ColumnGridResourceIT.java‎

Lines changed: 32 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -100,35 +100,40 @@ void test_getColumnGrid_aggregatesColumnsAcrossEntityTypes(TestNamespace ns) thr
100100

101101
waitForSearchIndexRefresh();
102102

103-
// Query both entity types - columns should be aggregated
104-
// Use just "shared_across" as pattern (not ns.prefix) to find the column name
105-
ColumnGridResponse response =
106-
getColumnGrid(
107-
client, "entityTypes=table,dashboardDataModel&columnNamePattern=shared_across");
108-
109-
assertNotNull(response, "Response should not be null");
110-
assertNotNull(response.getColumns(), "Columns should not be null");
111-
assertFalse(response.getColumns().isEmpty(), "Should find the shared column");
112-
113-
// Find the shared column
114-
ColumnGridItem sharedItem = null;
115-
for (ColumnGridItem item : response.getColumns()) {
116-
if (item.getColumnName().equals(sharedColName)) {
117-
sharedItem = item;
118-
break;
119-
}
120-
}
103+
// Poll until both entities are indexed and the column grid shows 2 occurrences
104+
await("Wait for column grid to show both occurrences")
105+
.atMost(Duration.ofSeconds(30))
106+
.pollInterval(Duration.ofSeconds(2))
107+
.untilAsserted(
108+
() -> {
109+
ColumnGridResponse response =
110+
getColumnGrid(
111+
client,
112+
"entityTypes=table,dashboardDataModel&columnNamePattern=shared_across");
113+
114+
assertNotNull(response, "Response should not be null");
115+
assertNotNull(response.getColumns(), "Columns should not be null");
116+
assertFalse(response.getColumns().isEmpty(), "Should find the shared column");
117+
118+
ColumnGridItem sharedItem = null;
119+
for (ColumnGridItem item : response.getColumns()) {
120+
if (item.getColumnName().equals(sharedColName)) {
121+
sharedItem = item;
122+
break;
123+
}
124+
}
121125

122-
assertNotNull(sharedItem, "Column '" + sharedColName + "' should be in aggregated results");
123-
assertEquals(
124-
2,
125-
sharedItem.getTotalOccurrences(),
126-
"Column should have 2 occurrences (one from table, one from dashboardDataModel)");
126+
assertNotNull(
127+
sharedItem, "Column '" + sharedColName + "' should be in aggregated results");
128+
assertEquals(
129+
2,
130+
sharedItem.getTotalOccurrences(),
131+
"Column should have 2 occurrences (one from table, one from dashboardDataModel)");
127132

128-
// Since descriptions are different, it should have variations
129-
assertTrue(
130-
sharedItem.getHasVariations(),
131-
"Column should have variations since descriptions differ across entity types");
133+
assertTrue(
134+
sharedItem.getHasVariations(),
135+
"Column should have variations since descriptions differ across entity types");
136+
});
132137
}
133138

134139
@Test

‎openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataProductResourceIT.java‎

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1062,20 +1062,36 @@ void test_multipleRenamesWithUpdatesPreservesAssets(TestNamespace ns) throws Exc
10621062
dataProduct = patchEntity(dataProduct.getId().toString(), dataProduct);
10631063
assertEquals(newName, dataProduct.getName());
10641064

1065-
ResultList<EntityReference> assets = getAssets(dataProduct.getId(), 10, 0);
1066-
assertEquals(
1067-
1,
1068-
assets.getPaging().getTotal(),
1069-
"Assets should be preserved immediately after rename " + (i + 1));
1070-
1071-
dataProduct.setDescription("Description after rename " + (i + 1));
1065+
final UUID dpId = dataProduct.getId();
1066+
final int iteration = i + 1;
1067+
Awaitility.await("Wait for assets after rename " + iteration)
1068+
.pollDelay(Duration.ofMillis(100))
1069+
.pollInterval(Duration.ofMillis(500))
1070+
.atMost(Duration.ofSeconds(30))
1071+
.untilAsserted(
1072+
() -> {
1073+
ResultList<EntityReference> a = getAssets(dpId, 10, 0);
1074+
assertEquals(
1075+
1,
1076+
a.getPaging().getTotal(),
1077+
"Assets should be preserved immediately after rename " + iteration);
1078+
});
1079+
1080+
dataProduct.setDescription("Description after rename " + iteration);
10721081
dataProduct = patchEntity(dataProduct.getId().toString(), dataProduct);
10731082

1074-
assets = getAssets(dataProduct.getId(), 10, 0);
1075-
assertEquals(
1076-
1,
1077-
assets.getPaging().getTotal(),
1078-
"Assets should be preserved after rename + update iteration " + (i + 1));
1083+
Awaitility.await("Wait for assets after rename + update " + iteration)
1084+
.pollDelay(Duration.ofMillis(100))
1085+
.pollInterval(Duration.ofMillis(500))
1086+
.atMost(Duration.ofSeconds(30))
1087+
.untilAsserted(
1088+
() -> {
1089+
ResultList<EntityReference> a = getAssets(dpId, 10, 0);
1090+
assertEquals(
1091+
1,
1092+
a.getPaging().getTotal(),
1093+
"Assets should be preserved after rename + update iteration " + iteration);
1094+
});
10791095
}
10801096

10811097
Table tableWithDataProducts =

‎openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryResourceIT.java‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -962,16 +962,14 @@ void test_bulkImportGlossaryTermsIncrementsVersion(TestNamespace ns) {
962962

963963
// Step 9: Verify glossary terms were actually created
964964
try {
965-
// Get all glossary terms and filter by glossary
966-
List<org.openmetadata.schema.entity.data.GlossaryTerm> allTerms =
967-
client.glossaryTerms().list().getData();
968-
969-
// Filter terms belonging to this glossary
970965
List<org.openmetadata.schema.entity.data.GlossaryTerm> terms =
971-
allTerms.stream()
972-
.filter(
973-
t -> t.getGlossary() != null && glossary.getId().equals(t.getGlossary().getId()))
974-
.toList();
966+
client
967+
.glossaryTerms()
968+
.list(
969+
new org.openmetadata.sdk.models.ListParams()
970+
.addFilter("glossary", glossary.getId().toString())
971+
.setLimit(100))
972+
.getData();
975973

976974
assertNotNull(terms, "Glossary terms should be returned");
977975
assertEquals(3, terms.size(), "Should have imported 3 glossary terms");

0 commit comments

Comments
 (0)