diff --git a/.github/workflows/permission-check.yaml b/.github/workflows/permission-check.yaml index 5740a084d4..bd56dbeaa3 100644 --- a/.github/workflows/permission-check.yaml +++ b/.github/workflows/permission-check.yaml @@ -7,6 +7,8 @@ env: TERM: dumb JAVA_VERSION: '8' JAVA_VENDOR: 'temurin' + DYNAMO_ACCESS_KEY_ID: ${{ secrets.DYNAMO_ACCESS_KEY }} + DYNAMO_SECRET_ACCESS_KEY: ${{ secrets.DYNAMO_SECRET_ACCESS_KEY }} jobs: integration-test-permission-cassandra-3-0: @@ -86,3 +88,29 @@ jobs: with: name: cassandra_3.11_permission_integration_test_reports path: core/build/reports/tests/integrationTestCassandraPermission + + integration-test-permission-dynamo: + name: DynamoDB Permission Integration Test + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK ${{ env.JAVA_VERSION }} (${{ env.JAVA_VENDOR }}) + uses: actions/setup-java@v4 + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: ${{ env.JAVA_VENDOR }} + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 + + - name: Execute Gradle 'integrationTestDynamoPermission' task + run: ./gradlew integrationTestDynamoPermission -Dscalardb.dynamo.emulator_used=false -Dscalardb.dynamo.region=ap-northeast-1 -Dscalardb.dynamo.access_key_id=${{ env.DYNAMO_ACCESS_KEY_ID }} -Dscalardb.dynamo.secret_access_key=${{ env.DYNAMO_SECRET_ACCESS_KEY }} + + - name: Upload Gradle test reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: dynamo_permission_integration_test_reports + path: core/build/reports/tests/integrationTestDynamoPermission diff --git a/core/build.gradle b/core/build.gradle index 74b77f9fb8..2ecde2ce19 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -45,6 +45,9 @@ sourceSets { srcDir file('src/integration-test/java') include '**/com/scalar/db/common/*.java' include '**/com/scalar/db/storage/dynamo/*.java' + exclude '**/com/scalar/db/storage/dynamo/DynamoPermissionTestUtils.java' + exclude '**/com/scalar/db/storage/dynamo/DynamoPermissionIntegrationTest.java' + exclude '**/com/scalar/db/storage/dynamo/DynamoAdminPermissionIntegrationTest.java' } resources.srcDir file('src/integration-test/resources') } @@ -84,6 +87,20 @@ sourceSets { } resources.srcDir file('src/integration-test/resources') } + integrationTestDynamoPermission { + java { + compileClasspath += main.output + test.output + runtimeClasspath += main.output + test.output + srcDir file('src/integration-test/java') + include '**/com/scalar/db/common/*.java' + include '**/com/scalar/db/storage/dynamo/DynamoPermissionTestUtils.java' + include '**/com/scalar/db/storage/dynamo/DynamoAdminTestUtils.java' + include '**/com/scalar/db/storage/dynamo/DynamoEnv.java' + include '**/com/scalar/db/storage/dynamo/DynamoPermissionIntegrationTest.java' + include '**/com/scalar/db/storage/dynamo/DynamoAdminPermissionIntegrationTest.java' + } + resources.srcDir file('src/integration-test/resources') + } } configurations { @@ -108,6 +125,9 @@ configurations { integrationTestCassandraPermissionImplementation.extendsFrom testImplementation integrationTestCassandraPermissionRuntimeOnly.extendsFrom testRuntimeOnly integrationTestCassandraPermissionCompileOnly.extendsFrom testCompileOnly + integrationTestDynamoPermissionImplementation.extendsFrom testImplementation + integrationTestDynamoPermissionRuntimeOnly.extendsFrom testRuntimeOnly + integrationTestDynamoPermissionCompileOnly.extendsFrom testCompileOnly } dependencies { @@ -120,6 +140,8 @@ dependencies { implementation platform("software.amazon.awssdk:bom:${awssdkVersion}") implementation 'software.amazon.awssdk:applicationautoscaling' implementation 'software.amazon.awssdk:dynamodb' + testImplementation 'software.amazon.awssdk:iam' + testImplementation 'software.amazon.awssdk:iam-policy-builder' implementation "org.apache.commons:commons-dbcp2:${commonsDbcp2Version}" implementation "com.mysql:mysql-connector-j:${mysqlDriverVersion}" implementation "org.postgresql:postgresql:${postgresqlDriverVersion}" @@ -227,6 +249,17 @@ task integrationTestCassandraPermission(type: Test) { } } +task integrationTestDynamoPermission(type: Test) { + description = 'Runs the integration tests for DynamoDB permissions.' + group = 'verification' + testClassesDirs = sourceSets.integrationTestDynamoPermission.output.classesDirs + classpath = sourceSets.integrationTestDynamoPermission.runtimeClasspath + outputs.upToDateWhen { false } // ensures integration tests are run every time when called + options { + systemProperties(System.getProperties().findAll { it.key.toString().startsWith("scalardb") }) + } +} + spotless { java { target 'src/*/java/**/*.java' diff --git a/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoAdminPermissionIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoAdminPermissionIntegrationTest.java new file mode 100644 index 0000000000..b2fcd6aa5e --- /dev/null +++ b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoAdminPermissionIntegrationTest.java @@ -0,0 +1,62 @@ +package com.scalar.db.storage.dynamo; + +import static com.scalar.db.storage.dynamo.DynamoPermissionTestUtils.SLEEP_BETWEEN_TESTS_SECONDS; + +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.Uninterruptibles; +import com.scalar.db.api.DistributedStorageAdminPermissionIntegrationTestBase; +import com.scalar.db.util.AdminTestUtils; +import com.scalar.db.util.PermissionTestUtils; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +public class DynamoAdminPermissionIntegrationTest + extends DistributedStorageAdminPermissionIntegrationTestBase { + @Override + protected Properties getProperties(String testName) { + return DynamoEnv.getProperties(testName); + } + + @Override + protected Properties getPropertiesForNormalUser(String testName) { + return DynamoEnv.getProperties(testName); + } + + @Override + protected Map getCreationOptions() { + return ImmutableMap.of(DynamoAdmin.NO_SCALING, "false", DynamoAdmin.NO_BACKUP, "false"); + } + + @Override + protected AdminTestUtils getAdminTestUtils(String testName) { + return new DynamoAdminTestUtils(getProperties(testName)); + } + + @Override + protected PermissionTestUtils getPermissionTestUtils(String testName) { + return new DynamoPermissionTestUtils(getProperties(testName)); + } + + @Override + protected void sleepBetweenTests() { + Uninterruptibles.sleepUninterruptibly(SLEEP_BETWEEN_TESTS_SECONDS, TimeUnit.SECONDS); + } + + @Test + @Override + @Disabled("Import-related functionality is not supported in DynamoDB") + public void getImportTableMetadata_WithSufficientPermission_ShouldSucceed() {} + + @Test + @Override + @Disabled("Import-related functionality is not supported in DynamoDB") + public void addRawColumnToTable_WithSufficientPermission_ShouldSucceed() {} + + @Test + @Override + @Disabled("Import-related functionality is not supported in DynamoDB") + public void importTable_WithSufficientPermission_ShouldSucceed() {} +} diff --git a/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoEnv.java b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoEnv.java index 25af1476a3..d7df4fb67f 100644 --- a/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoEnv.java +++ b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoEnv.java @@ -10,12 +10,14 @@ public final class DynamoEnv { private static final String PROP_DYNAMO_REGION = "scalardb.dynamo.region"; private static final String PROP_DYNAMO_ACCESS_KEY_ID = "scalardb.dynamo.access_key_id"; private static final String PROP_DYNAMO_SECRET_ACCESS_KEY = "scalardb.dynamo.secret_access_key"; + private static final String PROP_DYNAMO_EMULATOR_USED = "scalardb.dynamo.emulator_used"; private static final String PROP_DYNAMO_CREATE_OPTIONS = "scalardb.dynamo.create_options"; private static final String DEFAULT_DYNAMO_ENDPOINT_OVERRIDE = "http://localhost:8000"; private static final String DEFAULT_DYNAMO_REGION = "us-west-2"; private static final String DEFAULT_DYNAMO_ACCESS_KEY_ID = "fakeMyKeyId"; private static final String DEFAULT_DYNAMO_SECRET_ACCESS_KEY = "fakeSecretAccessKey"; + private static final String DEFAULT_DYNAMO_EMULATOR_USED = "true"; private static final ImmutableMap DEFAULT_DYNAMO_CREATE_OPTIONS = ImmutableMap.of(DynamoAdmin.NO_SCALING, "true", DynamoAdmin.NO_BACKUP, "true"); @@ -30,25 +32,27 @@ public static Properties getProperties(String testName) { System.getProperty(PROP_DYNAMO_ACCESS_KEY_ID, DEFAULT_DYNAMO_ACCESS_KEY_ID); String secretAccessKey = System.getProperty(PROP_DYNAMO_SECRET_ACCESS_KEY, DEFAULT_DYNAMO_SECRET_ACCESS_KEY); + String isEmulatorUsed = + System.getProperty(PROP_DYNAMO_EMULATOR_USED, DEFAULT_DYNAMO_EMULATOR_USED); - Properties props = new Properties(); - if (endpointOverride != null) { - props.setProperty(DynamoConfig.ENDPOINT_OVERRIDE, endpointOverride); + Properties properties = new Properties(); + if (Boolean.parseBoolean(isEmulatorUsed) && endpointOverride != null) { + properties.setProperty(DynamoConfig.ENDPOINT_OVERRIDE, endpointOverride); } - props.setProperty(DatabaseConfig.CONTACT_POINTS, region); - props.setProperty(DatabaseConfig.USERNAME, accessKeyId); - props.setProperty(DatabaseConfig.PASSWORD, secretAccessKey); - props.setProperty(DatabaseConfig.STORAGE, "dynamo"); - props.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN, "true"); - props.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN_FILTERING, "true"); - props.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN_ORDERING, "false"); + properties.setProperty(DatabaseConfig.CONTACT_POINTS, region); + properties.setProperty(DatabaseConfig.USERNAME, accessKeyId); + properties.setProperty(DatabaseConfig.PASSWORD, secretAccessKey); + properties.setProperty(DatabaseConfig.STORAGE, "dynamo"); + properties.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN, "true"); + properties.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN_FILTERING, "true"); + properties.setProperty(DatabaseConfig.CROSS_PARTITION_SCAN_ORDERING, "false"); // Add testName as a metadata namespace suffix - props.setProperty( + properties.setProperty( DynamoConfig.TABLE_METADATA_NAMESPACE, DatabaseConfig.DEFAULT_SYSTEM_NAMESPACE_NAME + "_" + testName); - return props; + return properties; } public static Map getCreationOptions() { diff --git a/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoPermissionIntegrationTest.java b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoPermissionIntegrationTest.java new file mode 100644 index 0000000000..beb5816f5b --- /dev/null +++ b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoPermissionIntegrationTest.java @@ -0,0 +1,36 @@ +package com.scalar.db.storage.dynamo; + +import com.google.common.collect.ImmutableMap; +import com.scalar.db.api.DistributedStoragePermissionIntegrationTestBase; +import com.scalar.db.util.AdminTestUtils; +import com.scalar.db.util.PermissionTestUtils; +import java.util.Map; +import java.util.Properties; + +public class DynamoPermissionIntegrationTest + extends DistributedStoragePermissionIntegrationTestBase { + @Override + protected Properties getProperties(String testName) { + return DynamoEnv.getProperties(testName); + } + + @Override + protected Properties getPropertiesForNormalUser(String testName) { + return DynamoEnv.getProperties(testName); + } + + @Override + protected Map getCreationOptions() { + return ImmutableMap.of(DynamoAdmin.NO_SCALING, "false", DynamoAdmin.NO_BACKUP, "false"); + } + + @Override + protected PermissionTestUtils getPermissionTestUtils(String testName) { + return new DynamoPermissionTestUtils(getProperties(testName)); + } + + @Override + protected AdminTestUtils getAdminTestUtils(String testName) { + return new DynamoAdminTestUtils(getProperties(testName)); + } +} diff --git a/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoPermissionTestUtils.java b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoPermissionTestUtils.java new file mode 100644 index 0000000000..05f2549475 --- /dev/null +++ b/core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoPermissionTestUtils.java @@ -0,0 +1,166 @@ +package com.scalar.db.storage.dynamo; + +import com.scalar.db.config.DatabaseConfig; +import com.scalar.db.util.PermissionTestUtils; +import java.util.Optional; +import java.util.Properties; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.policybuilder.iam.IamEffect; +import software.amazon.awssdk.policybuilder.iam.IamPolicy; +import software.amazon.awssdk.policybuilder.iam.IamResource; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.iam.IamClient; +import software.amazon.awssdk.services.iam.model.AttachUserPolicyRequest; +import software.amazon.awssdk.services.iam.model.AttachedPolicy; +import software.amazon.awssdk.services.iam.model.CreatePolicyRequest; +import software.amazon.awssdk.services.iam.model.CreatePolicyVersionRequest; +import software.amazon.awssdk.services.iam.model.DeletePolicyVersionRequest; +import software.amazon.awssdk.services.iam.model.ListAttachedUserPoliciesRequest; +import software.amazon.awssdk.services.iam.model.ListPolicyVersionsRequest; +import software.amazon.awssdk.services.iam.model.User; + +public class DynamoPermissionTestUtils implements PermissionTestUtils { + public static final int SLEEP_BETWEEN_TESTS_SECONDS = 10; + private static final String IAM_POLICY_NAME = "test-dynamodb-permissions"; + private static final IamPolicy POLICY = + IamPolicy.builder() + .addStatement( + s -> + s.effect(IamEffect.ALLOW) + .addAction("dynamodb:ConditionCheckItem") + .addAction("dynamodb:PutItem") + .addAction("dynamodb:ListTables") + .addAction("dynamodb:DeleteItem") + .addAction("dynamodb:Scan") + .addAction("dynamodb:Query") + .addAction("dynamodb:UpdateItem") + .addAction("dynamodb:DeleteTable") + .addAction("dynamodb:UpdateContinuousBackups") + .addAction("dynamodb:CreateTable") + .addAction("dynamodb:DescribeTable") + .addAction("dynamodb:GetItem") + .addAction("dynamodb:DescribeContinuousBackups") + .addAction("dynamodb:UpdateTable") + .addAction("application-autoscaling:RegisterScalableTarget") + .addAction("application-autoscaling:DeleteScalingPolicy") + .addAction("application-autoscaling:PutScalingPolicy") + .addAction("application-autoscaling:DeregisterScalableTarget") + .addAction("application-autoscaling:TagResource") + .addResource(IamResource.ALL)) + .build(); + private final IamClient client; + + public DynamoPermissionTestUtils(Properties properties) { + DynamoConfig config = new DynamoConfig(new DatabaseConfig(properties)); + this.client = + IamClient.builder() + .region(Region.of(config.getRegion())) + .credentialsProvider( + StaticCredentialsProvider.create( + AwsBasicCredentials.create( + config.getAccessKeyId(), config.getSecretAccessKey()))) + .build(); + } + + @Override + public void createNormalUser(String userName, String password) { + // Do nothing for DynamoDB. + } + + @Override + public void dropNormalUser(String userName) { + // Do nothing for DynamoDB. + } + + @Override + public void grantRequiredPermission(String userName) { + try { + User user = client.getUser().user(); + Optional attachedPolicyArn = getAttachedPolicyArn(user.userName()); + if (attachedPolicyArn.isPresent()) { + String policyArn = attachedPolicyArn.get(); + try { + deleteStalePolicyVersions(policyArn); + createNewPolicyVersion(policyArn); + } catch (SdkException e) { + throw new RuntimeException( + String.format( + "Failed to update policy for user: %s, policyArn: %s", userName, policyArn), + e); + } + } else { + String policyArn = createNewPolicy(); + try { + client.attachUserPolicy( + AttachUserPolicyRequest.builder() + .userName(user.userName()) + .policyArn(policyArn) + .build()); + } catch (SdkException e) { + throw new RuntimeException( + String.format( + "Failed to attach new policy for user: %s, policyArn: %s", userName, policyArn), + e); + } + } + } catch (SdkException e) { + throw new RuntimeException( + String.format( + "Failed to grant required permissions for user: %s, error: %s", + userName, e.getMessage()), + e); + } + } + + @Override + public void close() { + client.close(); + } + + private Optional getAttachedPolicyArn(String userName) { + AttachedPolicy attachedPolicy = + client + .listAttachedUserPolicies( + ListAttachedUserPoliciesRequest.builder().userName(userName).build()) + .attachedPolicies().stream() + .filter(policy -> policy.policyName().equals(DynamoPermissionTestUtils.IAM_POLICY_NAME)) + .findFirst() + .orElse(null); + return Optional.ofNullable(attachedPolicy).map(AttachedPolicy::policyArn); + } + + private String createNewPolicy() { + return client + .createPolicy( + CreatePolicyRequest.builder() + .policyName(IAM_POLICY_NAME) + .policyDocument(POLICY.toJson()) + .build()) + .policy() + .arn(); + } + + private void deleteStalePolicyVersions(String policyArn) { + client.listPolicyVersions(ListPolicyVersionsRequest.builder().policyArn(policyArn).build()) + .versions().stream() + .filter(version -> !version.isDefaultVersion()) + .forEach( + version -> + client.deletePolicyVersion( + DeletePolicyVersionRequest.builder() + .policyArn(policyArn) + .versionId(version.versionId()) + .build())); + } + + private void createNewPolicyVersion(String policyArn) { + client.createPolicyVersion( + CreatePolicyVersionRequest.builder() + .policyArn(policyArn) + .policyDocument(POLICY.toJson()) + .setAsDefault(true) + .build()); + } +} diff --git a/integration-test/src/main/java/com/scalar/db/api/DistributedStorageAdminPermissionIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/api/DistributedStorageAdminPermissionIntegrationTestBase.java index 69b97d6d2b..438f0cd4aa 100644 --- a/integration-test/src/main/java/com/scalar/db/api/DistributedStorageAdminPermissionIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/api/DistributedStorageAdminPermissionIntegrationTestBase.java @@ -55,19 +55,24 @@ public void beforeAll() throws Exception { Properties propertiesForRootUser = getProperties(TEST_NAME); Properties propertiesForNormalUser = getPropertiesForNormalUser(TEST_NAME); - // Initialize the admin for root user - StorageFactory factoryForRootUser = StorageFactory.create(propertiesForRootUser); - adminForRootUser = factoryForRootUser.getStorageAdmin(); - - // Create normal user and give permissions - DatabaseConfig config = new DatabaseConfig(propertiesForNormalUser); - normalUserName = getUserNameFromConfig(config); - normalUserPassword = getPasswordFromConfig(config); - setUpNormalUser(); - - // Initialize the admin for normal user - StorageFactory factoryForNormalUser = StorageFactory.create(propertiesForNormalUser); - adminForNormalUser = factoryForNormalUser.getStorageAdmin(); + try { + // Initialize the admin for root user + StorageFactory factoryForRootUser = StorageFactory.create(propertiesForRootUser); + adminForRootUser = factoryForRootUser.getStorageAdmin(); + + // Create normal user and give permissions + DatabaseConfig config = new DatabaseConfig(propertiesForNormalUser); + normalUserName = getUserNameFromConfig(config); + normalUserPassword = getPasswordFromConfig(config); + setUpNormalUser(); + + // Initialize the admin for normal user + StorageFactory factoryForNormalUser = StorageFactory.create(propertiesForNormalUser); + adminForNormalUser = factoryForNormalUser.getStorageAdmin(); + } catch (Exception e) { + logger.error("Failed to set up the test environment", e); + throw e; + } } @AfterAll diff --git a/integration-test/src/main/java/com/scalar/db/api/DistributedStoragePermissionIntegrationTestBase.java b/integration-test/src/main/java/com/scalar/db/api/DistributedStoragePermissionIntegrationTestBase.java index f28e41a60c..f4be283209 100644 --- a/integration-test/src/main/java/com/scalar/db/api/DistributedStoragePermissionIntegrationTestBase.java +++ b/integration-test/src/main/java/com/scalar/db/api/DistributedStoragePermissionIntegrationTestBase.java @@ -51,23 +51,28 @@ public void beforeAll() throws Exception { Properties propertiesForRootUser = getProperties(TEST_NAME); Properties propertiesForNormalUser = getPropertiesForNormalUser(TEST_NAME); - // Create admin for root user - StorageFactory factoryForRootUser = StorageFactory.create(propertiesForRootUser); - adminForRootUser = factoryForRootUser.getStorageAdmin(); - - // Create normal user and give permissions - DatabaseConfig config = new DatabaseConfig(propertiesForNormalUser); - normalUserName = getUserNameFromConfig(config); - normalUserPassword = getPasswordFromConfig(config); - setUpNormalUser(); - - // Create storage for normal user - StorageFactory factoryForNormalUser = StorageFactory.create(propertiesForNormalUser); - storageForNormalUser = factoryForNormalUser.getStorage(); - - namespace = getNamespace(); - createTable(); - waitForTableCreation(); + try { + // Create admin for root user + StorageFactory factoryForRootUser = StorageFactory.create(propertiesForRootUser); + adminForRootUser = factoryForRootUser.getStorageAdmin(); + + // Create normal user and give permissions + DatabaseConfig config = new DatabaseConfig(propertiesForNormalUser); + normalUserName = getUserNameFromConfig(config); + normalUserPassword = getPasswordFromConfig(config); + setUpNormalUser(); + + // Create storage for normal user + StorageFactory factoryForNormalUser = StorageFactory.create(propertiesForNormalUser); + storageForNormalUser = factoryForNormalUser.getStorage(); + + namespace = getNamespace(); + createTable(); + waitForTableCreation(); + } catch (Exception e) { + logger.error("Failed to set up the test environment", e); + throw e; + } } @BeforeEach