Skip to content

Commit 7b7f280

Browse files
ajantha-bhatajantha-bhat
authored
Add polaris catalog migrator tests (#32)
Added Integration tests with Polaris docker image to migrate tables from Nessie, Hadoop, Hive to polaris.
1 parent a47c962 commit 7b7f280

File tree

10 files changed

+430
-15
lines changed

10 files changed

+430
-15
lines changed

‎iceberg-catalog-migrator/api-test/src/main/java/org/apache/polaris/iceberg/catalog/migrator/api/test/AbstractTest.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
*/
1919
package org.apache.polaris.iceberg.catalog.migrator.api.test;
2020

21+
import java.io.File;
2122
import java.nio.file.Path;
2223
import java.util.Arrays;
2324
import java.util.HashMap;
@@ -55,8 +56,8 @@ public abstract class AbstractTest {
5556
protected static final Namespace NS_A_B_C_D = Namespace.of("a", "b", "c", "d");
5657
protected static final Namespace NS_A_B_C_D_E = Namespace.of("a", "b", "c", "d", "e");
5758

58-
private static String sourceCatalogWarehouse;
59-
private static String targetCatalogWarehouse;
59+
protected static String sourceCatalogWarehouse;
60+
protected static String targetCatalogWarehouse;
6061

6162
protected static Catalog sourceCatalog;
6263
protected static Catalog targetCatalog;
@@ -73,7 +74,9 @@ public abstract class AbstractTest {
7374
protected static void initLogDir() {
7475
System.setProperty("catalog.migration.log.dir", logDir.toAbsolutePath().toString());
7576
sourceCatalogWarehouse = tempDir.resolve("sourceCatalogWarehouse").toAbsolutePath().toString();
77+
ensureDirectoryExists(sourceCatalogWarehouse);
7678
targetCatalogWarehouse = tempDir.resolve("targetCatalogWarehouse").toAbsolutePath().toString();
79+
ensureDirectoryExists(targetCatalogWarehouse);
7780
}
7881

7982
@AfterAll
@@ -156,4 +159,13 @@ protected static Map<String, String> hiveCatalogProperties(
156159
properties.putAll(dynamicProperties);
157160
return properties;
158161
}
162+
163+
private static void ensureDirectoryExists(String path) {
164+
File dir = new File(path);
165+
if (!dir.exists()) {
166+
if (!dir.mkdirs()) {
167+
throw new RuntimeException("Unable to create directory: " + path);
168+
}
169+
}
170+
}
159171
}

‎iceberg-catalog-migrator/cli/build.gradle.kts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,8 @@ dependencies {
128128
}
129129
testImplementation("org.apache.hadoop:hadoop-mapreduce-client-core:${libs.versions.hadoop.get()}")
130130

131+
testImplementation("org.testcontainers:testcontainers:${libs.versions.testcontainers.get()}")
132+
131133
nessieQuarkusServer(
132134
"org.projectnessie.nessie:nessie-quarkus:${libs.versions.nessie.get()}:runner"
133135
)

‎iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/AbstractCLIMigrationTest.java‎

Lines changed: 3 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,11 @@
3232
import nl.altindag.log.LogCaptor;
3333
import nl.altindag.log.model.LogEvent;
3434
import org.apache.iceberg.catalog.Catalog;
35-
import org.apache.iceberg.catalog.TableIdentifier;
3635
import org.apache.iceberg.exceptions.NoSuchTableException;
3736
import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
3837
import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrator;
3938
import org.apache.polaris.iceberg.catalog.migrator.api.test.AbstractTest;
4039
import org.assertj.core.api.Assertions;
41-
import org.junit.jupiter.api.AfterAll;
4240
import org.junit.jupiter.api.AfterEach;
4341
import org.junit.jupiter.api.BeforeEach;
4442
import org.junit.jupiter.api.io.TempDir;
@@ -83,6 +81,9 @@ private static void initializeCatalog(
8381
case HIVE:
8482
properties = hiveCatalogProperties(isSourceCatalog, additionalProp);
8583
break;
84+
case REST:
85+
properties = additionalProp;
86+
break;
8687
default:
8788
throw new UnsupportedOperationException(
8889
String.format("Unsupported for catalog type: %s", catalogType));
@@ -106,24 +107,13 @@ private static void initializeCatalog(
106107
}
107108
}
108109

109-
@AfterAll
110-
protected static void tearDown() throws Exception {
111-
dropNamespaces();
112-
}
113-
114110
@BeforeEach
115111
protected void beforeEach() {
116112
createTables();
117113
}
118114

119115
@AfterEach
120116
protected void afterEach() {
121-
// manually refreshing catalog due to missing refresh in Nessie catalog
122-
// https://github.com/apache/iceberg/pull/6789
123-
// create table will call refresh internally.
124-
sourceCatalog.createTable(TableIdentifier.of(BAR, "tblx"), schema).refresh();
125-
targetCatalog.createTable(TableIdentifier.of(BAR, "tblx"), schema).refresh();
126-
127117
dropTables();
128118
}
129119

‎iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/HadoopCLIMigrationTest.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
import java.util.Collections;
2222
import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
23+
import org.junit.jupiter.api.AfterAll;
2324
import org.junit.jupiter.api.BeforeAll;
2425

2526
public class HadoopCLIMigrationTest extends AbstractCLIMigrationTest {
@@ -29,4 +30,9 @@ protected static void setup() {
2930
initializeSourceCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
3031
initializeTargetCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
3132
}
33+
34+
@AfterAll
35+
protected static void tearDown() throws Exception {
36+
dropNamespaces();
37+
}
3238
}

‎iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHadoopToNessieCLIMigrationTest.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.iceberg.catalog.TableIdentifier;
2828
import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
2929
import org.assertj.core.api.Assertions;
30+
import org.junit.jupiter.api.AfterAll;
3031
import org.junit.jupiter.api.BeforeAll;
3132
import org.junit.jupiter.api.Test;
3233

@@ -38,6 +39,11 @@ protected static void setup() {
3839
initializeTargetCatalog(CatalogMigrationUtil.CatalogType.NESSIE, Collections.emptyMap());
3940
}
4041

42+
@AfterAll
43+
protected static void tearDown() throws Exception {
44+
dropNamespaces();
45+
}
46+
4147
@Test
4248
public void testRegisterLargeNumberOfTablesWithNestedNamespaces() throws Exception {
4349
List<Namespace> namespaceList =

‎iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHadoopToPolarisCLIMigrationTest.java‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.polaris.iceberg.catalog.migrator.cli;
20+
21+
import static org.assertj.core.api.Assertions.assertThat;
22+
23+
import java.util.Collections;
24+
import java.util.Map;
25+
import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
26+
import org.junit.jupiter.api.AfterAll;
27+
import org.junit.jupiter.api.BeforeAll;
28+
29+
public class ITHadoopToPolarisCLIMigrationTest extends AbstractCLIMigrationTest {
30+
31+
private static PolarisContainer polarisContainer;
32+
33+
@BeforeAll
34+
protected static void setup() throws Exception {
35+
polarisContainer = new PolarisContainer(sourceCatalogWarehouse);
36+
polarisContainer.start();
37+
38+
assertThat(polarisContainer.httpGet("/api/management/v1/catalogs"))
39+
.contains(PolarisContainer.CATALOG_NAME);
40+
41+
initializeSourceCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
42+
43+
initializeTargetCatalog(
44+
CatalogMigrationUtil.CatalogType.REST,
45+
Map.of(
46+
"uri",
47+
polarisContainer.getIcebergApiEndpoint(),
48+
"warehouse",
49+
PolarisContainer.CATALOG_NAME,
50+
"token",
51+
polarisContainer.getAccessToken(
52+
polarisContainer.getClientId(), polarisContainer.getClientSecret())));
53+
}
54+
55+
@AfterAll
56+
protected static void tearDown() throws Exception {
57+
dropNamespaces();
58+
if (polarisContainer != null) {
59+
polarisContainer.stop();
60+
}
61+
}
62+
}

‎iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITHiveToPolarisCLIMigrationTest.java‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.polaris.iceberg.catalog.migrator.cli;
20+
21+
import static org.assertj.core.api.Assertions.assertThat;
22+
23+
import java.util.Collections;
24+
import java.util.Map;
25+
import org.apache.iceberg.hive.HiveMetastoreExtension;
26+
import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
27+
import org.junit.jupiter.api.AfterAll;
28+
import org.junit.jupiter.api.BeforeAll;
29+
import org.junit.jupiter.api.Disabled;
30+
import org.junit.jupiter.api.extension.RegisterExtension;
31+
32+
@Disabled("Because of https://github.com/apache/polaris/issues/2756")
33+
public class ITHiveToPolarisCLIMigrationTest extends AbstractCLIMigrationTest {
34+
35+
@RegisterExtension
36+
public static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION =
37+
HiveMetastoreExtension.builder().build();
38+
39+
private static PolarisContainer polarisContainer;
40+
41+
@BeforeAll
42+
protected static void setup() throws Exception {
43+
polarisContainer = new PolarisContainer(sourceCatalogWarehouse);
44+
polarisContainer.start();
45+
assertThat(polarisContainer.httpGet("/api/management/v1/catalogs"))
46+
.contains(PolarisContainer.CATALOG_NAME);
47+
48+
initializeSourceCatalog(
49+
CatalogMigrationUtil.CatalogType.HIVE,
50+
Collections.singletonMap(
51+
"uri", HIVE_METASTORE_EXTENSION.hiveConf().get("hive.metastore.uris")));
52+
53+
initializeTargetCatalog(
54+
CatalogMigrationUtil.CatalogType.REST,
55+
Map.of(
56+
"uri",
57+
polarisContainer.getIcebergApiEndpoint(),
58+
"warehouse",
59+
PolarisContainer.CATALOG_NAME,
60+
"token",
61+
polarisContainer.getAccessToken(
62+
polarisContainer.getClientId(), polarisContainer.getClientSecret())));
63+
}
64+
65+
@AfterAll
66+
protected static void tearDown() throws Exception {
67+
dropNamespaces();
68+
if (polarisContainer != null) {
69+
polarisContainer.stop();
70+
}
71+
}
72+
}

‎iceberg-catalog-migrator/cli/src/test/java/org/apache/polaris/iceberg/catalog/migrator/cli/ITNessieToPolarisCLIMigrationTest.java‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.polaris.iceberg.catalog.migrator.cli;
20+
21+
import static org.assertj.core.api.Assertions.assertThat;
22+
23+
import java.util.Collections;
24+
import java.util.Map;
25+
import org.apache.polaris.iceberg.catalog.migrator.api.CatalogMigrationUtil;
26+
import org.junit.jupiter.api.AfterAll;
27+
import org.junit.jupiter.api.BeforeAll;
28+
29+
public class ITNessieToPolarisCLIMigrationTest extends AbstractCLIMigrationTest {
30+
31+
private static PolarisContainer polarisContainer;
32+
33+
@BeforeAll
34+
protected static void setup() throws Exception {
35+
polarisContainer = new PolarisContainer(sourceCatalogWarehouse);
36+
polarisContainer.start();
37+
assertThat(polarisContainer.httpGet("/api/management/v1/catalogs"))
38+
.contains(PolarisContainer.CATALOG_NAME);
39+
40+
initializeSourceCatalog(CatalogMigrationUtil.CatalogType.NESSIE, Collections.emptyMap());
41+
42+
initializeTargetCatalog(
43+
CatalogMigrationUtil.CatalogType.REST,
44+
Map.of(
45+
"uri",
46+
polarisContainer.getIcebergApiEndpoint(),
47+
"warehouse",
48+
PolarisContainer.CATALOG_NAME,
49+
"token",
50+
polarisContainer.getAccessToken(
51+
polarisContainer.getClientId(), polarisContainer.getClientSecret())));
52+
}
53+
54+
@AfterAll
55+
protected static void tearDown() throws Exception {
56+
dropNamespaces();
57+
58+
if (polarisContainer != null) {
59+
polarisContainer.stop();
60+
}
61+
}
62+
}

0 commit comments

Comments
 (0)