diff --git a/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsStorageFactory.java b/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsStorageFactory.java index aff17b743dd2..eef133a2ac32 100644 --- a/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsStorageFactory.java +++ b/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsStorageFactory.java @@ -14,6 +14,9 @@ package io.trino.filesystem.gcs; import com.google.api.gax.retrying.RetrySettings; +import com.google.auth.oauth2.AccessToken; +import com.google.auth.oauth2.GoogleCredentials; +import com.google.cloud.NoCredentials; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; import com.google.inject.Inject; @@ -22,11 +25,18 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.time.Duration; +import java.util.Date; import java.util.Map; import java.util.Optional; import static com.google.cloud.storage.StorageRetryStrategy.getUniformStorageRetryStrategy; import static com.google.common.net.HttpHeaders.USER_AGENT; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_NO_AUTH_PROPERTY; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_EXPIRES_AT_PROPERTY; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_PROPERTY; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_PROJECT_ID_PROPERTY; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_SERVICE_HOST_PROPERTY; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_USER_PROJECT_PROPERTY; import static java.util.Objects.requireNonNull; public class GcsStorageFactory @@ -59,14 +69,45 @@ public GcsStorageFactory(GcsFileSystemConfig config, GcsAuth gcsAuth) public Storage create(ConnectorIdentity identity) { try { + Map extraCredentials = identity.getExtraCredentials(); + boolean noAuth = Boolean.parseBoolean(extraCredentials.getOrDefault(EXTRA_CREDENTIALS_GCS_NO_AUTH_PROPERTY, "false")); + String vendedOAuthToken = extraCredentials.get(EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_PROPERTY); + StorageOptions.Builder storageOptionsBuilder = StorageOptions.newBuilder(); - if (projectId != null) { - storageOptionsBuilder.setProjectId(projectId); + + String effectiveProjectId = extraCredentials.getOrDefault(EXTRA_CREDENTIALS_GCS_PROJECT_ID_PROPERTY, projectId); + if (effectiveProjectId != null) { + storageOptionsBuilder.setProjectId(effectiveProjectId); } - gcsAuth.setAuth(storageOptionsBuilder, identity); + if (noAuth) { + storageOptionsBuilder.setCredentials(NoCredentials.getInstance()); + } + else if (vendedOAuthToken != null) { + Date expirationTime = null; + String expiresAt = extraCredentials.get(EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_EXPIRES_AT_PROPERTY); + if (expiresAt != null) { + expirationTime = new Date(Long.parseLong(expiresAt)); + } + AccessToken accessToken = new AccessToken(vendedOAuthToken, expirationTime); + storageOptionsBuilder.setCredentials(GoogleCredentials.create(accessToken)); + } + else { + gcsAuth.setAuth(storageOptionsBuilder, identity); + } - endpoint.ifPresent(storageOptionsBuilder::setHost); + String vendedServiceHost = extraCredentials.get(EXTRA_CREDENTIALS_GCS_SERVICE_HOST_PROPERTY); + if (vendedServiceHost != null) { + storageOptionsBuilder.setHost(vendedServiceHost); + } + else { + endpoint.ifPresent(storageOptionsBuilder::setHost); + } + + String vendedUserProject = extraCredentials.get(EXTRA_CREDENTIALS_GCS_USER_PROJECT_PROPERTY); + if (vendedUserProject != null) { + storageOptionsBuilder.setQuotaProjectId(vendedUserProject); + } // Note: without uniform strategy we cannot retry idempotent operations. // The trino-filesystem api does not violate the conditions for idempotency, see https://cloud.google.com/storage/docs/retry-strategy#java for details. diff --git a/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsStorageFactory.java b/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsStorageFactory.java index 5649bdf7a4d3..7a64f5925f9b 100644 --- a/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsStorageFactory.java +++ b/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsStorageFactory.java @@ -14,12 +14,21 @@ package io.trino.filesystem.gcs; import com.google.auth.Credentials; +import com.google.auth.oauth2.AccessToken; +import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.NoCredentials; import com.google.cloud.storage.Storage; +import com.google.common.collect.ImmutableMap; import io.trino.spi.security.ConnectorIdentity; import org.junit.jupiter.api.Test; import static io.trino.filesystem.gcs.GcsFileSystemConfig.AuthType; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_NO_AUTH_PROPERTY; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_EXPIRES_AT_PROPERTY; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_PROPERTY; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_PROJECT_ID_PROPERTY; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_SERVICE_HOST_PROPERTY; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_USER_PROJECT_PROPERTY; import static org.assertj.core.api.Assertions.assertThat; final class TestGcsStorageFactory @@ -38,4 +47,192 @@ void testApplicationDefaultCredentials() assertThat(actualCredentials).isEqualTo(NoCredentials.getInstance()); } + + @Test + void testVendedOAuthToken() + throws Exception + { + GcsFileSystemConfig config = new GcsFileSystemConfig().setAuthType(AuthType.APPLICATION_DEFAULT); + GcsStorageFactory storageFactory = new GcsStorageFactory(config, new ApplicationDefaultAuth()); + + ConnectorIdentity identity = ConnectorIdentity.forUser("test") + .withExtraCredentials(ImmutableMap.of( + EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_PROPERTY, "ya29.test-token")) + .build(); + + try (Storage storage = storageFactory.create(identity)) { + Credentials credentials = storage.getOptions().getCredentials(); + assertThat(credentials).isInstanceOf(GoogleCredentials.class); + GoogleCredentials googleCredentials = (GoogleCredentials) credentials; + AccessToken accessToken = googleCredentials.getAccessToken(); + assertThat(accessToken).isNotNull(); + assertThat(accessToken.getTokenValue()).isEqualTo("ya29.test-token"); + } + } + + @Test + void testVendedOAuthTokenWithExpiration() + throws Exception + { + GcsFileSystemConfig config = new GcsFileSystemConfig().setAuthType(AuthType.APPLICATION_DEFAULT); + GcsStorageFactory storageFactory = new GcsStorageFactory(config, new ApplicationDefaultAuth()); + + ConnectorIdentity identity = ConnectorIdentity.forUser("test") + .withExtraCredentials(ImmutableMap.of( + EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_PROPERTY, "ya29.test-token", + EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_EXPIRES_AT_PROPERTY, "1700000000000")) + .build(); + + try (Storage storage = storageFactory.create(identity)) { + Credentials credentials = storage.getOptions().getCredentials(); + assertThat(credentials).isInstanceOf(GoogleCredentials.class); + GoogleCredentials googleCredentials = (GoogleCredentials) credentials; + AccessToken accessToken = googleCredentials.getAccessToken(); + assertThat(accessToken).isNotNull(); + assertThat(accessToken.getTokenValue()).isEqualTo("ya29.test-token"); + assertThat(accessToken.getExpirationTime()).isNotNull(); + assertThat(accessToken.getExpirationTime().getTime()).isEqualTo(1700000000000L); + } + } + + @Test + void testVendedProjectId() + throws Exception + { + GcsFileSystemConfig config = new GcsFileSystemConfig() + .setAuthType(AuthType.APPLICATION_DEFAULT) + .setProjectId("static-project"); + GcsStorageFactory storageFactory = new GcsStorageFactory(config, new ApplicationDefaultAuth()); + + ConnectorIdentity identity = ConnectorIdentity.forUser("test") + .withExtraCredentials(ImmutableMap.of( + EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_PROPERTY, "ya29.test-token", + EXTRA_CREDENTIALS_GCS_PROJECT_ID_PROPERTY, "vended-project")) + .build(); + + try (Storage storage = storageFactory.create(identity)) { + assertThat(storage.getOptions().getProjectId()).isEqualTo("vended-project"); + } + } + + @Test + void testVendedServiceHost() + throws Exception + { + GcsFileSystemConfig config = new GcsFileSystemConfig() + .setAuthType(AuthType.APPLICATION_DEFAULT); + GcsStorageFactory storageFactory = new GcsStorageFactory(config, new ApplicationDefaultAuth()); + + ConnectorIdentity identity = ConnectorIdentity.forUser("test") + .withExtraCredentials(ImmutableMap.of( + EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_PROPERTY, "ya29.test-token", + EXTRA_CREDENTIALS_GCS_SERVICE_HOST_PROPERTY, "https://custom-storage.googleapis.com")) + .build(); + + try (Storage storage = storageFactory.create(identity)) { + assertThat(storage.getOptions().getHost()).isEqualTo("https://custom-storage.googleapis.com"); + } + } + + @Test + void testVendedNoAuth() + throws Exception + { + GcsFileSystemConfig config = new GcsFileSystemConfig().setAuthType(AuthType.APPLICATION_DEFAULT); + GcsStorageFactory storageFactory = new GcsStorageFactory(config, new ApplicationDefaultAuth()); + + ConnectorIdentity identity = ConnectorIdentity.forUser("test") + .withExtraCredentials(ImmutableMap.of( + EXTRA_CREDENTIALS_GCS_NO_AUTH_PROPERTY, "true")) + .build(); + + try (Storage storage = storageFactory.create(identity)) { + assertThat(storage.getOptions().getCredentials()).isEqualTo(NoCredentials.getInstance()); + } + } + + @Test + void testNoAuthTakesPriorityOverOAuthToken() + throws Exception + { + GcsFileSystemConfig config = new GcsFileSystemConfig().setAuthType(AuthType.APPLICATION_DEFAULT); + GcsStorageFactory storageFactory = new GcsStorageFactory(config, new ApplicationDefaultAuth()); + + ConnectorIdentity identity = ConnectorIdentity.forUser("test") + .withExtraCredentials(ImmutableMap.of( + EXTRA_CREDENTIALS_GCS_NO_AUTH_PROPERTY, "true", + EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_PROPERTY, "ya29.test-token")) + .build(); + + try (Storage storage = storageFactory.create(identity)) { + assertThat(storage.getOptions().getCredentials()).isEqualTo(NoCredentials.getInstance()); + } + } + + @Test + void testVendedUserProject() + throws Exception + { + GcsFileSystemConfig config = new GcsFileSystemConfig() + .setAuthType(AuthType.APPLICATION_DEFAULT); + GcsStorageFactory storageFactory = new GcsStorageFactory(config, new ApplicationDefaultAuth()); + + ConnectorIdentity identity = ConnectorIdentity.forUser("test") + .withExtraCredentials(ImmutableMap.of( + EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_PROPERTY, "ya29.test-token", + EXTRA_CREDENTIALS_GCS_USER_PROJECT_PROPERTY, "billing-project")) + .build(); + + try (Storage storage = storageFactory.create(identity)) { + assertThat(storage.getOptions().getQuotaProjectId()).isEqualTo("billing-project"); + } + } + + @Test + void testNoAuthFalseDoesNotSkipAuth() + throws Exception + { + GcsFileSystemConfig config = new GcsFileSystemConfig().setAuthType(AuthType.APPLICATION_DEFAULT); + GcsStorageFactory storageFactory = new GcsStorageFactory(config, new ApplicationDefaultAuth()); + + ConnectorIdentity identity = ConnectorIdentity.forUser("test") + .withExtraCredentials(ImmutableMap.of( + EXTRA_CREDENTIALS_GCS_NO_AUTH_PROPERTY, "false", + EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_PROPERTY, "ya29.test-token")) + .build(); + + try (Storage storage = storageFactory.create(identity)) { + Credentials credentials = storage.getOptions().getCredentials(); + assertThat(credentials).isInstanceOf(GoogleCredentials.class); + assertThat(((GoogleCredentials) credentials).getAccessToken().getTokenValue()).isEqualTo("ya29.test-token"); + } + } + + @Test + void testUserProjectNotSetWithoutVendedCredentials() + throws Exception + { + GcsFileSystemConfig config = new GcsFileSystemConfig() + .setAuthType(AuthType.APPLICATION_DEFAULT); + GcsStorageFactory storageFactory = new GcsStorageFactory(config, new ApplicationDefaultAuth()); + + try (Storage storage = storageFactory.create(ConnectorIdentity.ofUser("test"))) { + assertThat(storage.getOptions().getQuotaProjectId()).isNull(); + } + } + + @Test + void testStaticConfigUsedWithoutVendedCredentials() + throws Exception + { + GcsFileSystemConfig config = new GcsFileSystemConfig() + .setAuthType(AuthType.APPLICATION_DEFAULT) + .setProjectId("static-project"); + GcsStorageFactory storageFactory = new GcsStorageFactory(config, new ApplicationDefaultAuth()); + + try (Storage storage = storageFactory.create(ConnectorIdentity.ofUser("test"))) { + assertThat(storage.getOptions().getProjectId()).isEqualTo("static-project"); + assertThat(storage.getOptions().getCredentials()).isEqualTo(NoCredentials.getInstance()); + } + } } diff --git a/lib/trino-filesystem/src/main/java/io/trino/filesystem/gcs/GcsFileSystemConstants.java b/lib/trino-filesystem/src/main/java/io/trino/filesystem/gcs/GcsFileSystemConstants.java new file mode 100644 index 000000000000..8f8e4906d608 --- /dev/null +++ b/lib/trino-filesystem/src/main/java/io/trino/filesystem/gcs/GcsFileSystemConstants.java @@ -0,0 +1,26 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.filesystem.gcs; + +public final class GcsFileSystemConstants +{ + public static final String EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_PROPERTY = "internal$gcs_oauth2_token"; + public static final String EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_EXPIRES_AT_PROPERTY = "internal$gcs_oauth2_token_expires_at"; + public static final String EXTRA_CREDENTIALS_GCS_PROJECT_ID_PROPERTY = "internal$gcs_project_id"; + public static final String EXTRA_CREDENTIALS_GCS_SERVICE_HOST_PROPERTY = "internal$gcs_service_host"; + public static final String EXTRA_CREDENTIALS_GCS_NO_AUTH_PROPERTY = "internal$gcs_no_auth"; + public static final String EXTRA_CREDENTIALS_GCS_USER_PROJECT_PROPERTY = "internal$gcs_user_project"; + + private GcsFileSystemConstants() {} +} diff --git a/plugin/trino-iceberg/pom.xml b/plugin/trino-iceberg/pom.xml index 6246d51ffb8d..4d938a8a3729 100644 --- a/plugin/trino-iceberg/pom.xml +++ b/plugin/trino-iceberg/pom.xml @@ -778,6 +778,7 @@ **/TestIcebergS3TablesConnectorSmokeTest.java **/TestIcebergBigLakeMetastoreConnectorSmokeTest.java **/TestIcebergGcsConnectorSmokeTest.java + **/TestIcebergGcsVendingRestCatalogConnectorSmokeTest.java **/TestIcebergAbfsConnectorSmokeTest.java **/Test*FailureRecoveryTest.java **/TestIcebergSnowflakeCatalogConnectorSmokeTest.java @@ -845,6 +846,7 @@ **/TestIcebergS3TablesConnectorSmokeTest.java **/TestIcebergBigLakeMetastoreConnectorSmokeTest.java **/TestIcebergGcsConnectorSmokeTest.java + **/TestIcebergGcsVendingRestCatalogConnectorSmokeTest.java **/TestIcebergAbfsConnectorSmokeTest.java **/TestIcebergSnowflakeCatalogConnectorSmokeTest.java **/TestTrinoSnowflakeCatalog.java diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/IcebergRestCatalogFileSystemFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/IcebergRestCatalogFileSystemFactory.java index b5e777c7ee15..34d8ed2227e4 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/IcebergRestCatalogFileSystemFactory.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/IcebergRestCatalogFileSystemFactory.java @@ -22,6 +22,12 @@ import java.util.Map; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_NO_AUTH_PROPERTY; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_EXPIRES_AT_PROPERTY; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_PROPERTY; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_PROJECT_ID_PROPERTY; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_SERVICE_HOST_PROPERTY; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_USER_PROJECT_PROPERTY; import static io.trino.filesystem.s3.S3FileSystemConstants.EXTRA_CREDENTIALS_ACCESS_KEY_PROPERTY; import static io.trino.filesystem.s3.S3FileSystemConstants.EXTRA_CREDENTIALS_SECRET_KEY_PROPERTY; import static io.trino.filesystem.s3.S3FileSystemConstants.EXTRA_CREDENTIALS_SESSION_TOKEN_PROPERTY; @@ -34,6 +40,13 @@ public class IcebergRestCatalogFileSystemFactory private static final String VENDED_S3_SECRET_KEY = "s3.secret-access-key"; private static final String VENDED_S3_SESSION_TOKEN = "s3.session-token"; + private static final String VENDED_GCS_OAUTH_TOKEN = "gcs.oauth2.token"; + private static final String VENDED_GCS_OAUTH_TOKEN_EXPIRES_AT = "gcs.oauth2.token-expires-at"; + private static final String VENDED_GCS_PROJECT_ID = "gcs.project-id"; + private static final String VENDED_GCS_SERVICE_HOST = "gcs.service.host"; + private static final String VENDED_GCS_NO_AUTH = "gcs.no-auth"; + private static final String VENDED_GCS_USER_PROJECT = "gcs.user-project"; + private final TrinoFileSystemFactory fileSystemFactory; private final boolean vendedCredentialsEnabled; @@ -66,6 +79,46 @@ public TrinoFileSystem create(ConnectorIdentity identity, Map fi return fileSystemFactory.create(identityWithExtraCredentials); } + if (vendedCredentialsEnabled && hasAnyGcsVendedProperty(fileIoProperties)) { + ImmutableMap.Builder extraCredentialsBuilder = ImmutableMap.builder(); + + addOptionalProperty(extraCredentialsBuilder, fileIoProperties, VENDED_GCS_OAUTH_TOKEN, EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_PROPERTY); + addOptionalProperty(extraCredentialsBuilder, fileIoProperties, VENDED_GCS_OAUTH_TOKEN_EXPIRES_AT, EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_EXPIRES_AT_PROPERTY); + addOptionalProperty(extraCredentialsBuilder, fileIoProperties, VENDED_GCS_PROJECT_ID, EXTRA_CREDENTIALS_GCS_PROJECT_ID_PROPERTY); + addOptionalProperty(extraCredentialsBuilder, fileIoProperties, VENDED_GCS_SERVICE_HOST, EXTRA_CREDENTIALS_GCS_SERVICE_HOST_PROPERTY); + addOptionalProperty(extraCredentialsBuilder, fileIoProperties, VENDED_GCS_NO_AUTH, EXTRA_CREDENTIALS_GCS_NO_AUTH_PROPERTY); + addOptionalProperty(extraCredentialsBuilder, fileIoProperties, VENDED_GCS_USER_PROJECT, EXTRA_CREDENTIALS_GCS_USER_PROJECT_PROPERTY); + ConnectorIdentity identityWithExtraCredentials = ConnectorIdentity.forUser(identity.getUser()) + .withGroups(identity.getGroups()) + .withPrincipal(identity.getPrincipal()) + .withEnabledSystemRoles(identity.getEnabledSystemRoles()) + .withConnectorRole(identity.getConnectorRole()) + .withExtraCredentials(extraCredentialsBuilder.buildOrThrow()) + .build(); + return fileSystemFactory.create(identityWithExtraCredentials); + } + return fileSystemFactory.create(identity); } + + private static boolean hasAnyGcsVendedProperty(Map fileIoProperties) + { + return fileIoProperties.containsKey(VENDED_GCS_OAUTH_TOKEN) || + fileIoProperties.containsKey(VENDED_GCS_NO_AUTH) || + fileIoProperties.containsKey(VENDED_GCS_PROJECT_ID) || + fileIoProperties.containsKey(VENDED_GCS_SERVICE_HOST) || + fileIoProperties.containsKey(VENDED_GCS_USER_PROJECT); + } + + private static void addOptionalProperty( + ImmutableMap.Builder builder, + Map fileIoProperties, + String vendedKey, + String extraCredentialKey) + { + String value = fileIoProperties.get(vendedKey); + if (value != null) { + builder.put(extraCredentialKey, value); + } + } } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergGcsVendingRestCatalogConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergGcsVendingRestCatalogConnectorSmokeTest.java new file mode 100644 index 000000000000..41bf71e89f89 --- /dev/null +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergGcsVendingRestCatalogConnectorSmokeTest.java @@ -0,0 +1,330 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.iceberg.catalog.rest; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; +import io.airlift.log.Logger; +import io.trino.filesystem.Location; +import io.trino.plugin.iceberg.BaseIcebergConnectorSmokeTest; +import io.trino.plugin.iceberg.IcebergConfig; +import io.trino.plugin.iceberg.IcebergQueryRunner; +import io.trino.testing.QueryFailedException; +import io.trino.testing.QueryRunner; +import io.trino.testing.TestingConnectorBehavior; +import io.trino.testing.containers.IcebergGcsRestCatalogBackendContainer; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.catalog.SessionCatalog; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.rest.RESTSessionCatalog; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Base64; +import java.util.Optional; + +import static io.trino.plugin.iceberg.IcebergTestUtils.checkOrcFileSorting; +import static io.trino.plugin.iceberg.IcebergTestUtils.checkParquetFileSorting; +import static io.trino.testing.TestingConnectorSession.SESSION; +import static io.trino.testing.TestingProperties.requiredNonEmptySystemProperty; +import static java.lang.String.format; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.iceberg.FileFormat.PARQUET; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestIcebergGcsVendingRestCatalogConnectorSmokeTest + extends BaseIcebergConnectorSmokeTest +{ + private static final Logger LOG = Logger.get(TestIcebergGcsVendingRestCatalogConnectorSmokeTest.class); + + private final String gcpStorageBucket; + private final String gcpCredentialKey; + private String warehouseLocation; + private IcebergGcsRestCatalogBackendContainer restCatalogBackendContainer; + + public TestIcebergGcsVendingRestCatalogConnectorSmokeTest() + { + super(new IcebergConfig().getFileFormat().toIceberg()); + this.gcpStorageBucket = requiredNonEmptySystemProperty("testing.gcp-storage-bucket"); + this.gcpCredentialKey = requiredNonEmptySystemProperty("testing.gcp-credentials-key"); + } + + @Override + protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) + { + return switch (connectorBehavior) { + case SUPPORTS_CREATE_MATERIALIZED_VIEW, + SUPPORTS_RENAME_MATERIALIZED_VIEW, + SUPPORTS_RENAME_SCHEMA -> false; + default -> super.hasBehavior(connectorBehavior); + }; + } + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + byte[] jsonKeyBytes = Base64.getDecoder().decode(gcpCredentialKey); + Path gcpCredentialsFile = Files.createTempFile("gcp-credentials", ".json"); + gcpCredentialsFile.toFile().deleteOnExit(); + Files.write(gcpCredentialsFile, jsonKeyBytes); + String gcpCredentials = new String(jsonKeyBytes, UTF_8); + + ObjectMapper mapper = new ObjectMapper(); + JsonNode jsonKey = mapper.readTree(gcpCredentials); + String gcpProjectId = jsonKey.get("project_id").asText(); + + this.warehouseLocation = "gs://%s/gcs-vending-rest-test/".formatted(gcpStorageBucket); + + restCatalogBackendContainer = closeAfterClass(new IcebergGcsRestCatalogBackendContainer( + Optional.empty(), + warehouseLocation, + gcpCredentialsFile.toAbsolutePath().toString(), + gcpProjectId)); + restCatalogBackendContainer.start(); + + return IcebergQueryRunner.builder() + .setIcebergProperties( + ImmutableMap.builder() + .put("iceberg.file-format", format.name()) + .put("iceberg.catalog.type", "rest") + .put("iceberg.rest-catalog.uri", "http://" + restCatalogBackendContainer.getRestCatalogEndpoint()) + .put("iceberg.rest-catalog.vended-credentials-enabled", "true") + .put("iceberg.writer-sort-buffer-size", "1MB") + .put("fs.native-gcs.enabled", "true") + .put("gcs.project-id", gcpProjectId) + .put("gcs.json-key", gcpCredentials) + .buildOrThrow()) + .setInitialTables(REQUIRED_TPCH_TABLES) + .build(); + } + + @AfterAll + public void removeTestData() + { + if (fileSystem == null) { + return; + } + try { + fileSystem.deleteDirectory(Location.of(warehouseLocation)); + } + catch (IOException e) { + // The GCS bucket should be configured to expire objects automatically. Clean up issues do not need to fail the test. + LOG.warn(e, "Failed to clean up GCS test directory: %s", warehouseLocation); + } + } + + @Test + @Override + public void testMaterializedView() + { + assertThatThrownBy(super::testMaterializedView) + .hasMessageContaining("createMaterializedView is not supported for Iceberg REST catalog"); + } + + @Test + @Override + public void testRenameSchema() + { + assertThatThrownBy(super::testRenameSchema) + .hasMessageContaining("renameNamespace is not supported for Iceberg REST catalog"); + } + + @Override + protected void dropTableFromCatalog(String tableName) + { + // TODO: Get register table tests working + } + + @Override + protected String getMetadataLocation(String tableName) + { + try (RESTSessionCatalog catalog = new RESTSessionCatalog()) { + catalog.initialize("rest-catalog", ImmutableMap.of(CatalogProperties.URI, "http://" + restCatalogBackendContainer.getRestCatalogEndpoint())); + SessionCatalog.SessionContext context = new SessionCatalog.SessionContext( + "user-default", + "user", + ImmutableMap.of(), + ImmutableMap.of(), + SESSION.getIdentity()); + return ((BaseTable) catalog.loadTable(context, toIdentifier(tableName))).operations().current().metadataFileLocation(); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + protected String schemaPath() + { + return format("%s%s", warehouseLocation, getSession().getSchema().orElseThrow()); + } + + @Override + protected boolean locationExists(String location) + { + try { + return fileSystem.newInputFile(Location.of(location)).exists(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Test + @Override + public void testRegisterTableWithTableLocation() + { + assertThatThrownBy(super::testRegisterTableWithTableLocation) + .hasMessageContaining("register_table procedure is disabled"); + } + + @Test + @Override + public void testRegisterTableWithComments() + { + assertThatThrownBy(super::testRegisterTableWithComments) + .hasMessageContaining("register_table procedure is disabled"); + } + + @Test + @Override + public void testRegisterTableWithShowCreateTable() + { + assertThatThrownBy(super::testRegisterTableWithShowCreateTable) + .hasMessageContaining("register_table procedure is disabled"); + } + + @Test + @Override + public void testRegisterTableWithReInsert() + { + assertThatThrownBy(super::testRegisterTableWithReInsert) + .hasMessageContaining("register_table procedure is disabled"); + } + + @Test + @Override + public void testRegisterTableWithDroppedTable() + { + assertThatThrownBy(super::testRegisterTableWithDroppedTable) + .hasMessageContaining("register_table procedure is disabled"); + } + + @Test + @Override + public void testRegisterTableWithDifferentTableName() + { + assertThatThrownBy(super::testRegisterTableWithDifferentTableName) + .hasMessageContaining("register_table procedure is disabled"); + } + + @Test + @Override + public void testRegisterTableWithMetadataFile() + { + assertThatThrownBy(super::testRegisterTableWithMetadataFile) + .hasMessageContaining("register_table procedure is disabled"); + } + + @Test + @Override + public void testRegisterTableWithTrailingSpaceInLocation() + { + assertThatThrownBy(super::testRegisterTableWithTrailingSpaceInLocation) + .hasMessageContaining("register_table procedure is disabled"); + } + + @Test + @Override + public void testUnregisterTable() + { + assertThatThrownBy(super::testUnregisterTable) + .hasMessageContaining("register_table procedure is disabled"); + } + + @Test + @Override + public void testRepeatUnregisterTable() + { + assertThatThrownBy(super::testRepeatUnregisterTable) + .hasMessageContaining("register_table procedure is disabled"); + } + + @Test + @Override + public void testDropTableWithMissingMetadataFile() + { + assertThatThrownBy(super::testDropTableWithMissingMetadataFile) + .hasMessageMatching("Failed to load table: (.*)"); + } + + @Test + @Override + public void testDropTableWithMissingSnapshotFile() + { + assertThatThrownBy(super::testDropTableWithMissingSnapshotFile) + .isInstanceOf(QueryFailedException.class) + .cause() + .hasMessageContaining("Failed to drop table") + .hasNoCause(); + } + + @Test + @Override + public void testDropTableWithMissingManifestListFile() + { + assertThatThrownBy(super::testDropTableWithMissingManifestListFile) + .hasMessageContaining("Table location should not exist"); + } + + @Test + @Override + public void testDropTableWithNonExistentTableLocation() + { + assertThatThrownBy(super::testDropTableWithNonExistentTableLocation) + .hasMessageMatching("Failed to load table: (.*)"); + } + + @Override + protected boolean isFileSorted(Location path, String sortColumnName) + { + if (format == PARQUET) { + return checkParquetFileSorting(fileSystem.newInputFile(path), sortColumnName); + } + return checkOrcFileSorting(fileSystem, path, sortColumnName); + } + + @Override + protected void deleteDirectory(String location) + { + try { + fileSystem.deleteDirectory(Location.of(location)); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private TableIdentifier toIdentifier(String tableName) + { + return TableIdentifier.of(getSession().getSchema().orElseThrow(), tableName); + } +} diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergRestCatalogFileSystemFactory.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergRestCatalogFileSystemFactory.java new file mode 100644 index 000000000000..bbb656c91098 --- /dev/null +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergRestCatalogFileSystemFactory.java @@ -0,0 +1,229 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.iceberg.catalog.rest; + +import com.google.common.collect.ImmutableMap; +import io.trino.filesystem.TrinoFileSystemFactory; +import io.trino.spi.security.ConnectorIdentity; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_NO_AUTH_PROPERTY; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_EXPIRES_AT_PROPERTY; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_PROPERTY; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_PROJECT_ID_PROPERTY; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_SERVICE_HOST_PROPERTY; +import static io.trino.filesystem.gcs.GcsFileSystemConstants.EXTRA_CREDENTIALS_GCS_USER_PROJECT_PROPERTY; +import static io.trino.filesystem.s3.S3FileSystemConstants.EXTRA_CREDENTIALS_ACCESS_KEY_PROPERTY; +import static io.trino.filesystem.s3.S3FileSystemConstants.EXTRA_CREDENTIALS_SECRET_KEY_PROPERTY; +import static io.trino.filesystem.s3.S3FileSystemConstants.EXTRA_CREDENTIALS_SESSION_TOKEN_PROPERTY; +import static org.assertj.core.api.Assertions.assertThat; + +final class TestIcebergRestCatalogFileSystemFactory +{ + @Test + void testS3VendedCredentials() + { + AtomicReference capturedIdentity = new AtomicReference<>(); + TrinoFileSystemFactory delegate = identity -> { + capturedIdentity.set(identity); + return null; + }; + + IcebergRestCatalogConfig config = new IcebergRestCatalogConfig() + .setBaseUri("http://localhost") + .setVendedCredentialsEnabled(true); + IcebergRestCatalogFileSystemFactory factory = new IcebergRestCatalogFileSystemFactory(delegate, config); + + Map fileIoProperties = ImmutableMap.of( + "s3.access-key-id", "test-access-key", + "s3.secret-access-key", "test-secret-key", + "s3.session-token", "test-session-token"); + + factory.create(ConnectorIdentity.ofUser("test"), fileIoProperties); + + ConnectorIdentity identity = capturedIdentity.get(); + assertThat(identity).isNotNull(); + assertThat(identity.getExtraCredentials()) + .containsEntry(EXTRA_CREDENTIALS_ACCESS_KEY_PROPERTY, "test-access-key") + .containsEntry(EXTRA_CREDENTIALS_SECRET_KEY_PROPERTY, "test-secret-key") + .containsEntry(EXTRA_CREDENTIALS_SESSION_TOKEN_PROPERTY, "test-session-token"); + } + + @Test + void testGcsVendedCredentialsWithOAuthTokenOnly() + { + AtomicReference capturedIdentity = new AtomicReference<>(); + TrinoFileSystemFactory delegate = identity -> { + capturedIdentity.set(identity); + return null; + }; + + IcebergRestCatalogConfig config = new IcebergRestCatalogConfig() + .setBaseUri("http://localhost") + .setVendedCredentialsEnabled(true); + IcebergRestCatalogFileSystemFactory factory = new IcebergRestCatalogFileSystemFactory(delegate, config); + + Map fileIoProperties = ImmutableMap.of( + "gcs.oauth2.token", "ya29.test-token"); + + factory.create(ConnectorIdentity.ofUser("test"), fileIoProperties); + + ConnectorIdentity identity = capturedIdentity.get(); + assertThat(identity).isNotNull(); + assertThat(identity.getExtraCredentials()) + .containsEntry(EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_PROPERTY, "ya29.test-token") + .doesNotContainKey(EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_EXPIRES_AT_PROPERTY) + .doesNotContainKey(EXTRA_CREDENTIALS_GCS_PROJECT_ID_PROPERTY) + .doesNotContainKey(EXTRA_CREDENTIALS_GCS_SERVICE_HOST_PROPERTY); + } + + @Test + void testGcsVendedCredentialsWithAllProperties() + { + AtomicReference capturedIdentity = new AtomicReference<>(); + TrinoFileSystemFactory delegate = identity -> { + capturedIdentity.set(identity); + return null; + }; + + IcebergRestCatalogConfig config = new IcebergRestCatalogConfig() + .setBaseUri("http://localhost") + .setVendedCredentialsEnabled(true); + IcebergRestCatalogFileSystemFactory factory = new IcebergRestCatalogFileSystemFactory(delegate, config); + + Map fileIoProperties = ImmutableMap.builder() + .put("gcs.oauth2.token", "ya29.test-token") + .put("gcs.oauth2.token-expires-at", "1700000000000") + .put("gcs.project-id", "my-gcp-project") + .put("gcs.service.host", "https://custom-storage.googleapis.com") + .put("gcs.user-project", "billing-project") + .buildOrThrow(); + + factory.create(ConnectorIdentity.ofUser("test"), fileIoProperties); + + ConnectorIdentity identity = capturedIdentity.get(); + assertThat(identity).isNotNull(); + assertThat(identity.getExtraCredentials()) + .containsEntry(EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_PROPERTY, "ya29.test-token") + .containsEntry(EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_EXPIRES_AT_PROPERTY, "1700000000000") + .containsEntry(EXTRA_CREDENTIALS_GCS_PROJECT_ID_PROPERTY, "my-gcp-project") + .containsEntry(EXTRA_CREDENTIALS_GCS_SERVICE_HOST_PROPERTY, "https://custom-storage.googleapis.com") + .containsEntry(EXTRA_CREDENTIALS_GCS_USER_PROJECT_PROPERTY, "billing-project"); + } + + @Test + void testGcsVendedNoAuth() + { + AtomicReference capturedIdentity = new AtomicReference<>(); + TrinoFileSystemFactory delegate = identity -> { + capturedIdentity.set(identity); + return null; + }; + + IcebergRestCatalogConfig config = new IcebergRestCatalogConfig() + .setBaseUri("http://localhost") + .setVendedCredentialsEnabled(true); + IcebergRestCatalogFileSystemFactory factory = new IcebergRestCatalogFileSystemFactory(delegate, config); + + Map fileIoProperties = ImmutableMap.of( + "gcs.no-auth", "true", + "gcs.project-id", "public-project"); + + factory.create(ConnectorIdentity.ofUser("test"), fileIoProperties); + + ConnectorIdentity identity = capturedIdentity.get(); + assertThat(identity).isNotNull(); + assertThat(identity.getExtraCredentials()) + .containsEntry(EXTRA_CREDENTIALS_GCS_NO_AUTH_PROPERTY, "true") + .containsEntry(EXTRA_CREDENTIALS_GCS_PROJECT_ID_PROPERTY, "public-project") + .doesNotContainKey(EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_PROPERTY); + } + + @Test + void testGcsVendedCredentialsDisabled() + { + AtomicReference capturedIdentity = new AtomicReference<>(); + TrinoFileSystemFactory delegate = identity -> { + capturedIdentity.set(identity); + return null; + }; + + IcebergRestCatalogConfig config = new IcebergRestCatalogConfig() + .setBaseUri("http://localhost") + .setVendedCredentialsEnabled(false); + IcebergRestCatalogFileSystemFactory factory = new IcebergRestCatalogFileSystemFactory(delegate, config); + + Map fileIoProperties = ImmutableMap.of( + "gcs.oauth2.token", "ya29.test-token", + "gcs.project-id", "my-gcp-project"); + + ConnectorIdentity originalIdentity = ConnectorIdentity.ofUser("test"); + factory.create(originalIdentity, fileIoProperties); + + ConnectorIdentity identity = capturedIdentity.get(); + assertThat(identity).isNotNull(); + assertThat(identity.getExtraCredentials()).isEmpty(); + } + + @Test + void testGcsVendedUserProjectOnly() + { + AtomicReference capturedIdentity = new AtomicReference<>(); + TrinoFileSystemFactory delegate = identity -> { + capturedIdentity.set(identity); + return null; + }; + + IcebergRestCatalogConfig config = new IcebergRestCatalogConfig() + .setBaseUri("http://localhost") + .setVendedCredentialsEnabled(true); + IcebergRestCatalogFileSystemFactory factory = new IcebergRestCatalogFileSystemFactory(delegate, config); + + Map fileIoProperties = ImmutableMap.of( + "gcs.user-project", "billing-project"); + + factory.create(ConnectorIdentity.ofUser("test"), fileIoProperties); + + ConnectorIdentity identity = capturedIdentity.get(); + assertThat(identity).isNotNull(); + assertThat(identity.getExtraCredentials()) + .containsEntry(EXTRA_CREDENTIALS_GCS_USER_PROJECT_PROPERTY, "billing-project") + .doesNotContainKey(EXTRA_CREDENTIALS_GCS_OAUTH_TOKEN_PROPERTY) + .doesNotContainKey(EXTRA_CREDENTIALS_GCS_NO_AUTH_PROPERTY); + } + + @Test + void testNoVendedCredentialsInProperties() + { + AtomicReference capturedIdentity = new AtomicReference<>(); + TrinoFileSystemFactory delegate = identity -> { + capturedIdentity.set(identity); + return null; + }; + + IcebergRestCatalogConfig config = new IcebergRestCatalogConfig() + .setBaseUri("http://localhost") + .setVendedCredentialsEnabled(true); + IcebergRestCatalogFileSystemFactory factory = new IcebergRestCatalogFileSystemFactory(delegate, config); + + ConnectorIdentity originalIdentity = ConnectorIdentity.ofUser("test"); + factory.create(originalIdentity, ImmutableMap.of()); + + ConnectorIdentity identity = capturedIdentity.get(); + assertThat(identity).isSameAs(originalIdentity); + } +} diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergVendingRestCatalogConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergVendingRestCatalogConnectorSmokeTest.java index c1c78a5959c8..d76a1a38a8a2 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergVendingRestCatalogConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergVendingRestCatalogConnectorSmokeTest.java @@ -25,7 +25,7 @@ import io.trino.testing.QueryFailedException; import io.trino.testing.QueryRunner; import io.trino.testing.TestingConnectorBehavior; -import io.trino.testing.containers.IcebergRestCatalogBackendContainer; +import io.trino.testing.containers.IcebergS3RestCatalogBackendContainer; import io.trino.testing.containers.Minio; import io.trino.testing.minio.MinioClient; import org.apache.iceberg.BaseTable; @@ -67,7 +67,7 @@ public class TestIcebergVendingRestCatalogConnectorSmokeTest { private final String bucketName; private String warehouseLocation; - private IcebergRestCatalogBackendContainer restCatalogBackendContainer; + private IcebergS3RestCatalogBackendContainer restCatalogBackendContainer; private Minio minio; public TestIcebergVendingRestCatalogConnectorSmokeTest() @@ -106,7 +106,7 @@ protected QueryRunner createQueryRunner() .build(); AssumeRoleResponse assumeRoleResponse = stsClient.assumeRole(AssumeRoleRequest.builder().build()); - restCatalogBackendContainer = closeAfterClass(new IcebergRestCatalogBackendContainer( + restCatalogBackendContainer = closeAfterClass(new IcebergS3RestCatalogBackendContainer( Optional.of(network), warehouseLocation, assumeRoleResponse.credentials().accessKeyId(), diff --git a/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/IcebergGcsRestCatalogBackendContainer.java b/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/IcebergGcsRestCatalogBackendContainer.java new file mode 100644 index 000000000000..93197cf40894 --- /dev/null +++ b/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/IcebergGcsRestCatalogBackendContainer.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.testing.containers; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.testcontainers.containers.Network; + +import java.util.Optional; + +public class IcebergGcsRestCatalogBackendContainer + extends BaseTestContainer +{ + public IcebergGcsRestCatalogBackendContainer( + Optional network, + String warehouseLocation, + String gcpCredentialsFilePath, + String gcpProjectId) + { + super( + "apache/iceberg-rest-fixture:1.10.1", + "iceberg-rest", + ImmutableSet.of(8181), + ImmutableMap.of("/gcs-credentials.json", gcpCredentialsFilePath), + ImmutableMap.of( + "CATALOG_INCLUDE__CREDENTIALS", "true", + "CATALOG_WAREHOUSE", warehouseLocation, + "CATALOG_IO__IMPL", "org.apache.iceberg.gcp.gcs.GCSFileIO", + "CATALOG_GCS_PROJECT__ID", gcpProjectId, + "GOOGLE_APPLICATION_CREDENTIALS", "/gcs-credentials.json"), + network, + 5); + } + + public String getRestCatalogEndpoint() + { + return getMappedHostAndPortForExposedPort(8181).toString(); + } +} diff --git a/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/IcebergRestCatalogBackendContainer.java b/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/IcebergS3RestCatalogBackendContainer.java similarity index 95% rename from testing/trino-testing-containers/src/main/java/io/trino/testing/containers/IcebergRestCatalogBackendContainer.java rename to testing/trino-testing-containers/src/main/java/io/trino/testing/containers/IcebergS3RestCatalogBackendContainer.java index d4fb6064dca0..0fa48b239d8e 100644 --- a/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/IcebergRestCatalogBackendContainer.java +++ b/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/IcebergS3RestCatalogBackendContainer.java @@ -21,10 +21,10 @@ import static io.trino.testing.containers.Minio.MINIO_REGION; -public class IcebergRestCatalogBackendContainer +public class IcebergS3RestCatalogBackendContainer extends BaseTestContainer { - public IcebergRestCatalogBackendContainer( + public IcebergS3RestCatalogBackendContainer( Optional network, String warehouseLocation, String minioAccessKey,