Skip to content

Commit f6e342b

Browse files
daindain
committed
Run Iceberg connector tests on v2 and v3
1 parent c9d7fe6 commit f6e342b

11 files changed

+178
-22
lines changed

plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergAvroConnectorTest.java renamed to plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergAvroConnectorTest.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,12 @@
1818
import static io.trino.plugin.iceberg.IcebergFileFormat.AVRO;
1919
import static org.junit.jupiter.api.Assumptions.abort;
2020

21-
public class TestIcebergAvroConnectorTest
21+
public abstract class BaseIcebergAvroConnectorTest
2222
extends BaseIcebergConnectorTest
2323
{
24-
public TestIcebergAvroConnectorTest()
24+
public BaseIcebergAvroConnectorTest(int formatVersion)
2525
{
26-
super(AVRO);
26+
super(AVRO, formatVersion);
2727
}
2828

2929
@Override

‎plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java‎

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -196,13 +196,15 @@ public abstract class BaseIcebergConnectorTest
196196
private static final Pattern WITH_CLAUSE_EXTRACTOR = Pattern.compile(".*(WITH\\s*\\([^)]*\\))\\s*$", Pattern.DOTALL);
197197

198198
protected final IcebergFileFormat format;
199+
protected final int formatVersion;
199200

200201
protected TrinoFileSystem fileSystem;
201202
protected TimeUnit storageTimePrecision;
202203

203-
protected BaseIcebergConnectorTest(IcebergFileFormat format)
204+
protected BaseIcebergConnectorTest(IcebergFileFormat format, int formatVersion)
204205
{
205206
this.format = requireNonNull(format, "format is null");
207+
this.formatVersion = formatVersion;
206208
}
207209

208210
@Override
@@ -218,6 +220,7 @@ protected IcebergQueryRunner.Builder createQueryRunnerBuilder()
218220
return IcebergQueryRunner.builder()
219221
.setIcebergProperties(ImmutableMap.<String, String>builder()
220222
.put("iceberg.file-format", format.name())
223+
.put("iceberg.format-version", String.valueOf(formatVersion))
221224
// Only allow some extra properties. Add "sorted_by" so that we can test that the property is disallowed by the connector explicitly.
222225
.put("iceberg.allowed-extra-properties", "extra.property.one,extra.property.two,extra.property.three,sorted_by")
223226
// Allows testing the sorting writer flushing to the file system with smaller tables
@@ -277,22 +280,28 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior)
277280
case SUPPORTS_CREATE_OR_REPLACE_TABLE,
278281
SUPPORTS_REPORTING_WRITTEN_BYTES -> true;
279282
case SUPPORTS_ADD_COLUMN_NOT_NULL_CONSTRAINT,
280-
SUPPORTS_DEFAULT_COLUMN_VALUE,
281283
SUPPORTS_LIMIT_PUSHDOWN,
282284
SUPPORTS_REFRESH_VIEW,
283285
SUPPORTS_RENAME_MATERIALIZED_VIEW_ACROSS_SCHEMAS,
284286
SUPPORTS_TOPN_PUSHDOWN -> false;
287+
case SUPPORTS_DEFAULT_COLUMN_VALUE -> formatVersion >= 3;
285288
default -> super.hasBehavior(connectorBehavior);
286289
};
287290
}
288291

289-
@Override
290292
@Test
293+
@Override
291294
public void testCreateTableWithDefaultColumn()
292295
{
293-
String tableName = "test_default_value_" + randomNameSuffix();
294-
assertThatThrownBy(() -> assertUpdate("CREATE TABLE " + tableName + " (x int DEFAULT 1)"))
295-
.hasMessageContaining("Default column values are not supported for Iceberg table format version < 3");
296+
skipTestUnless(hasBehavior(TestingConnectorBehavior.SUPPORTS_CREATE_TABLE) && hasBehavior(TestingConnectorBehavior.SUPPORTS_INSERT));
297+
298+
if (formatVersion < 3) {
299+
String tableName = "test_default_value_" + randomNameSuffix();
300+
assertQueryFails("CREATE TABLE " + tableName + " (x int DEFAULT 1)", "Default column values are not supported for Iceberg table format version < 3");
301+
return;
302+
}
303+
304+
super.testCreateTableWithDefaultColumn();
296305
}
297306

298307
@Test
@@ -424,7 +433,7 @@ public void testShowCreateTable()
424433
")\n" +
425434
"WITH (\n" +
426435
" format = '" + format.name() + "',\n" +
427-
" format_version = 2,\n" +
436+
" format_version = " + formatVersion + ",\n" +
428437
" location = '\\E.*/tpch/orders-.*\\Q'\n" +
429438
")\\E");
430439
}
@@ -2064,11 +2073,12 @@ private void testCreateTableLikeForFormat(IcebergFileFormat otherFormat)
20642073
"""
20652074
WITH (
20662075
format = '%s',
2067-
format_version = 2,
2076+
format_version = %s,
20682077
location = '%s',
20692078
partitioning = ARRAY['adate']
20702079
)""",
20712080
format,
2081+
formatVersion,
20722082
tempDirPath));
20732083

20742084
assertUpdate("CREATE TABLE test_create_table_like_copy0 (LIKE test_create_table_like_original, col2 INTEGER)");
@@ -2080,21 +2090,23 @@ private void testCreateTableLikeForFormat(IcebergFileFormat otherFormat)
20802090
"""
20812091
WITH (
20822092
format = '%s',
2083-
format_version = 2,
2093+
format_version = %s,
20842094
location = '%s'
20852095
)""",
20862096
format,
2097+
formatVersion,
20872098
getTableLocation("test_create_table_like_copy1")));
20882099

20892100
assertUpdate("CREATE TABLE test_create_table_like_copy2 (LIKE test_create_table_like_original EXCLUDING PROPERTIES)");
20902101
assertThat(getTablePropertiesString("test_create_table_like_copy2")).isEqualTo(format(
20912102
"""
20922103
WITH (
20932104
format = '%s',
2094-
format_version = 2,
2105+
format_version = %s,
20952106
location = '%s'
20962107
)""",
20972108
format,
2109+
formatVersion,
20982110
getTableLocation("test_create_table_like_copy2")));
20992111
assertUpdate("DROP TABLE test_create_table_like_copy2");
21002112

@@ -5463,7 +5475,7 @@ public void testOptimize()
54635475
.hasSizeGreaterThan(workerCount);
54645476

54655477
// For optimize we need to set task_min_writer_count to 1, otherwise it will create more than one file.
5466-
computeActual(withSingleWriterPerTask(getSession()), "ALTER TABLE " + tableName + " EXECUTE OPTIMIZE");
5478+
computeActual(withSingleWriterPerTask(getSession()), "ALTER TABLE " + tableName + " EXECUTE OPTIMIZE");
54675479
assertThat(query("SELECT sum(key), listagg(value, ' ') WITHIN GROUP (ORDER BY key) FROM " + tableName))
54685480
.matches("VALUES (BIGINT '65', VARCHAR 'eleven zwölf trzynaście quatorze пʼятнадцять')");
54695481
List<String> updatedFiles = getActiveFiles(tableName);

plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMinioOrcConnectorTest.java renamed to plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMinioOrcConnectorTest.java

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,14 +43,14 @@
4343
* Iceberg connector test ORC and with S3-compatible storage (but without real metastore).
4444
*/
4545
@Execution(SAME_THREAD)
46-
public class TestIcebergMinioOrcConnectorTest
46+
public abstract class BaseIcebergMinioOrcConnectorTest
4747
extends BaseIcebergConnectorTest
4848
{
4949
private final String bucketName = "test-iceberg-orc-" + randomNameSuffix();
5050

51-
public TestIcebergMinioOrcConnectorTest()
51+
public BaseIcebergMinioOrcConnectorTest(int formatVersion)
5252
{
53-
super(ORC);
53+
super(ORC, formatVersion);
5454
}
5555

5656
@Override
@@ -65,6 +65,7 @@ protected QueryRunner createQueryRunner()
6565
.setIcebergProperties(
6666
ImmutableMap.<String, String>builder()
6767
.put("iceberg.file-format", format.name())
68+
.put("iceberg.format-version", String.valueOf(formatVersion))
6869
.put("fs.hadoop.enabled", "true")
6970
.put("fs.native-s3.enabled", "true")
7071
.put("s3.aws-access-key", MINIO_ROOT_USER)

plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergParquetConnectorTest.java renamed to plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergParquetConnectorTest.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,12 @@
3939
import static java.time.ZoneOffset.UTC;
4040
import static org.assertj.core.api.Assertions.assertThat;
4141

42-
public class TestIcebergParquetConnectorTest
42+
public abstract class BaseIcebergParquetConnectorTest
4343
extends BaseIcebergConnectorTest
4444
{
45-
public TestIcebergParquetConnectorTest()
45+
public BaseIcebergParquetConnectorTest(int formatVersion)
4646
{
47-
super(PARQUET);
47+
super(PARQUET, formatVersion);
4848
}
4949

5050
@Override

‎plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2AvroConnectorTest.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.plugin.iceberg;
15+
16+
class TestIcebergV2AvroConnectorTest
17+
extends BaseIcebergAvroConnectorTest
18+
{
19+
TestIcebergV2AvroConnectorTest()
20+
{
21+
super(2);
22+
}
23+
}

‎plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2MinioOrcConnectorTest.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.plugin.iceberg;
15+
16+
class TestIcebergV2MinioOrcConnectorTest
17+
extends BaseIcebergMinioOrcConnectorTest
18+
{
19+
TestIcebergV2MinioOrcConnectorTest()
20+
{
21+
super(2);
22+
}
23+
}

‎plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2ParquetConnectorTest.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.plugin.iceberg;
15+
16+
class TestIcebergV2ParquetConnectorTest
17+
extends BaseIcebergParquetConnectorTest
18+
{
19+
TestIcebergV2ParquetConnectorTest()
20+
{
21+
super(2);
22+
}
23+
}

‎plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV3AvroConnectorTest.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.plugin.iceberg;
15+
16+
class TestIcebergV3AvroConnectorTest
17+
extends BaseIcebergAvroConnectorTest
18+
{
19+
TestIcebergV3AvroConnectorTest()
20+
{
21+
super(3);
22+
}
23+
}

‎plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV3MinioOrcConnectorTest.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.plugin.iceberg;
15+
16+
class TestIcebergV3MinioOrcConnectorTest
17+
extends BaseIcebergMinioOrcConnectorTest
18+
{
19+
TestIcebergV3MinioOrcConnectorTest()
20+
{
21+
super(3);
22+
}
23+
}

‎plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV3ParquetConnectorTest.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.plugin.iceberg;
15+
16+
class TestIcebergV3ParquetConnectorTest
17+
extends BaseIcebergParquetConnectorTest
18+
{
19+
TestIcebergV3ParquetConnectorTest()
20+
{
21+
super(3);
22+
}
23+
}

0 commit comments

Comments
 (0)