Merge branch 'master' into permission-list

Author: KodaiD

.github/dependabot.yml

@@ -36,49 +36,49 @@ updates:
       interval: "weekly"
 
   - package-ecosystem: "github-actions"
-    target-branch: "3.15"
+    target-branch: "3.16"
     groups:
-      actions on branch 3.15:
+      actions on branch 3.16:
         patterns:
           - "*"
     directory: "/"
     schedule:
       interval: "weekly"
 
   - package-ecosystem: "github-actions"
-    target-branch: "3.14"
+    target-branch: "3.15"
     groups:
-      actions on branch 3.14:
+      actions on branch 3.15:
         patterns:
           - "*"
     directory: "/"
     schedule:
       interval: "weekly"
 
   - package-ecosystem: "github-actions"
-    target-branch: "3.13"
+    target-branch: "3.14"
     groups:
-      actions on branch 3.13:
+      actions on branch 3.14:
         patterns:
           - "*"
     directory: "/"
     schedule:
       interval: "weekly"
 
   - package-ecosystem: "github-actions"
-    target-branch: "3.12"
+    target-branch: "3.13"
     groups:
-      actions on branch 3.12:
+      actions on branch 3.13:
         patterns:
           - "*"
     directory: "/"
     schedule:
       interval: "weekly"
 
   - package-ecosystem: "github-actions"
-    target-branch: "3.11"
+    target-branch: "3.12"
     groups:
-      actions on branch 3.11:
+      actions on branch 3.12:
         patterns:
           - "*"
     directory: "/"

.github/workflows/scheduled-vuln-check.yaml

@@ -16,15 +16,6 @@ jobs:
       CR_PAT: ${{ secrets.CR_PAT }}
       SLACK_SECURITY_WEBHOOK_URL: ${{ secrets.SLACK_SECURITY_WEBHOOK_URL }}
 
-  call-vuln-check-for-v3_11:
-    uses: ./.github/workflows/vuln-check.yaml
-    with:
-      target-ref: v3.11
-      find-latest-release: true
-    secrets:
-      CR_PAT: ${{ secrets.CR_PAT }}
-      SLACK_SECURITY_WEBHOOK_URL: ${{ secrets.SLACK_SECURITY_WEBHOOK_URL }}
-
   call-vuln-check-for-v3_12:
     uses: ./.github/workflows/vuln-check.yaml
     with:
@@ -60,3 +51,12 @@ jobs:
     secrets:
       CR_PAT: ${{ secrets.CR_PAT }}
       SLACK_SECURITY_WEBHOOK_URL: ${{ secrets.SLACK_SECURITY_WEBHOOK_URL }}
+
+  call-vuln-check-for-v3_16:
+    uses: ./.github/workflows/vuln-check.yaml
+    with:
+      target-ref: v3.16
+      find-latest-release: true
+    secrets:
+      CR_PAT: ${{ secrets.CR_PAT }}
+      SLACK_SECURITY_WEBHOOK_URL: ${{ secrets.SLACK_SECURITY_WEBHOOK_URL }}

README.md

@@ -37,7 +37,7 @@ You can install it in your application using your build tool such as Gradle and
 To add a dependency on ScalarDB using Gradle, use the following:
 ```gradle
 dependencies {
-    implementation 'com.scalar-labs:scalardb:3.15.2'
+    implementation 'com.scalar-labs:scalardb:3.16.0'
 }
 ```
 
@@ -46,7 +46,7 @@ To add a dependency using Maven:
 <dependency>
   <groupId>com.scalar-labs</groupId>
   <artifactId>scalardb</artifactId>
-  <version>3.15.2</version>
+  <version>3.16.0</version>
 </dependency>
 ```
 

core/src/main/java/com/scalar/db/storage/cosmos/CosmosAdmin.java

@@ -588,6 +588,7 @@ public void repairTable(
       throws ExecutionException {
     try {
       createTableInternal(namespace, table, metadata, true);
+      updateIndexingPolicy(namespace, table, metadata);
     } catch (IllegalArgumentException e) {
       throw e;
     } catch (Exception e) {

core/src/main/java/com/scalar/db/storage/dynamo/DynamoAdmin.java

@@ -1032,6 +1032,46 @@ columnName, getFullTableName(namespace, table)),
         TableMetadata.newBuilder(tableMetadata).addSecondaryIndex(columnName).build());
   }
 
+  private boolean rawIndexExists(String nonPrefixedNamespace, String table, String indexName) {
+    Namespace namespace = Namespace.of(namespacePrefix, nonPrefixedNamespace);
+    String globalIndexName =
+        String.join(
+            ".",
+            getFullTableName(namespace, table),
+            DynamoAdmin.GLOBAL_INDEX_NAME_PREFIX,
+            indexName);
+    int retryCount = 0;
+    try {
+      while (true) {
+        DescribeTableResponse response =
+            client.describeTable(
+                DescribeTableRequest.builder()
+                    .tableName(getFullTableName(namespace, table))
+                    .build());
+        GlobalSecondaryIndexDescription description =
+            response.table().globalSecondaryIndexes().stream()
+                .filter(d -> d.indexName().equals(globalIndexName))
+                .findFirst()
+                .orElse(null);
+        if (description == null) {
+          return false;
+        }
+        if (description.indexStatus() == IndexStatus.ACTIVE) {
+          return true;
+        }
+        if (retryCount++ >= MAX_RETRY_COUNT) {
+          throw new IllegalStateException(
+              String.format(
+                  "Waiting for the secondary index %s on the %s table to be active failed",
+                  indexName, getFullTableName(namespace, table)));
+        }
+        Uninterruptibles.sleepUninterruptibly(waitingDurationSecs, TimeUnit.SECONDS);
+      }
+    } catch (ResourceNotFoundException e) {
+      return false;
+    }
+  }
+
   private void waitForIndexCreation(Namespace namespace, String table, String columnName)
       throws ExecutionException {
     try {
@@ -1366,6 +1406,11 @@ public void repairTable(
       throws ExecutionException {
     try {
       createTableInternal(nonPrefixedNamespace, table, metadata, true, options);
+      for (String indexColumnName : metadata.getSecondaryIndexNames()) {
+        if (!rawIndexExists(nonPrefixedNamespace, table, indexColumnName)) {
+          createIndex(nonPrefixedNamespace, table, indexColumnName, options);
+        }
+      }
     } catch (RuntimeException e) {
       throw new ExecutionException(
           String.format(

core/src/main/java/com/scalar/db/storage/jdbc/JdbcAdmin.java

@@ -889,7 +889,8 @@ columnName, getFullTableName(namespace, table)),
     }
   }
 
-  private void createIndex(
+  @VisibleForTesting
+  void createIndex(
       Connection connection, String schema, String table, String indexedColumn, boolean ifNotExists)
       throws SQLException {
     String indexName = getIndexName(schema, table, indexedColumn);

core/src/test/java/com/scalar/db/storage/cassandra/CassandraAdminTest.java

@@ -7,6 +7,7 @@
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -797,6 +798,43 @@ public void repairTable_ShouldCreateTableAndIndexesIfTheyDoNotExists() throws Ex
     }
   }
 
+  @Test
+  public void repairTable_WhenTableAlreadyExistsWithoutIndex_ShouldCreateIndex()
+      throws ExecutionException {
+    // Arrange
+    String namespace = "sample_ns";
+    String table = "tbl";
+    TableMetadata tableMetadata =
+        TableMetadata.newBuilder()
+            .addPartitionKey("c1")
+            .addClusteringKey("c4")
+            .addColumn("c1", DataType.INT)
+            .addColumn("c2", DataType.TEXT)
+            .addColumn("c3", DataType.BLOB)
+            .addColumn("c4", DataType.INT)
+            .addColumn("c5", DataType.BOOLEAN)
+            .addSecondaryIndex("c2")
+            .addSecondaryIndex("c4")
+            .build();
+    // The table already exists
+    when(clusterManager.getMetadata(namespace, table))
+        .thenReturn(mock(com.datastax.driver.core.TableMetadata.class));
+    when(cassandraSession.getCluster()).thenReturn(cluster);
+    when(cluster.getMetadata()).thenReturn(metadata);
+    when(metadata.getKeyspace(any())).thenReturn(keyspaceMetadata);
+    when(keyspaceMetadata.getTable(table))
+        .thenReturn(mock(com.datastax.driver.core.TableMetadata.class));
+
+    CassandraAdmin adminSpy = spy(cassandraAdmin);
+
+    // Act
+    adminSpy.repairTable(namespace, table, tableMetadata, Collections.emptyMap());
+
+    // Assert
+    verify(adminSpy)
+        .createSecondaryIndexes(namespace, table, tableMetadata.getSecondaryIndexNames(), true);
+  }
+
   @Test
   public void addNewColumnToTable_ShouldWorkProperly() throws ExecutionException {
     // Arrange

core/src/test/java/com/scalar/db/storage/cosmos/CosmosAdminTest.java

@@ -893,6 +893,13 @@ public void repairTable_withStoredProcedure_ShouldNotAddStoredProcedure()
     when(scripts.getStoredProcedure(CosmosAdmin.STORED_PROCEDURE_FILE_NAME))
         .thenReturn(storedProcedure);
 
+    // Existing container properties
+    CosmosContainerResponse response = mock(CosmosContainerResponse.class);
+    when(database.createContainerIfNotExists(table, "/concatenatedPartitionKey"))
+        .thenReturn(response);
+    CosmosContainerProperties properties = mock(CosmosContainerProperties.class);
+    when(response.getProperties()).thenReturn(properties);
+
     // Act Assert
     admin.repairTable(namespace, table, tableMetadata, Collections.emptyMap());
 
@@ -943,6 +950,13 @@ public void repairTable_withoutStoredProcedure_ShouldCreateStoredProcedure()
     when(cosmosException.getStatusCode()).thenReturn(404);
     when(storedProcedure.read()).thenThrow(cosmosException);
 
+    // Existing container properties
+    CosmosContainerResponse response = mock(CosmosContainerResponse.class);
+    when(database.createContainerIfNotExists(table, "/concatenatedPartitionKey"))
+        .thenReturn(response);
+    CosmosContainerProperties properties = mock(CosmosContainerProperties.class);
+    when(response.getProperties()).thenReturn(properties);
+
     // Act
     admin.repairTable(namespace, table, tableMetadata, Collections.emptyMap());
 
@@ -996,6 +1010,13 @@ public void repairTable_ShouldCreateTableIfNotExistsAndUpsertMetadata()
     when(cosmosException.getStatusCode()).thenReturn(404);
     when(storedProcedure.read()).thenThrow(cosmosException);
 
+    // Existing container properties
+    CosmosContainerResponse response = mock(CosmosContainerResponse.class);
+    when(database.createContainerIfNotExists(table, "/concatenatedPartitionKey"))
+        .thenReturn(response);
+    CosmosContainerProperties properties = mock(CosmosContainerProperties.class);
+    when(response.getProperties()).thenReturn(properties);
+
     // Act
     admin.repairTable(namespace, table, tableMetadata, Collections.emptyMap());
 
@@ -1024,6 +1045,49 @@ public void repairTable_ShouldCreateTableIfNotExistsAndUpsertMetadata()
     verify(scripts).createStoredProcedure(any());
   }
 
+  @Test
+  public void repairTable_WhenTableAlreadyExistsWithoutIndex_ShouldCreateIndex()
+      throws ExecutionException {
+    // Arrange
+    String namespace = "ns";
+    String table = "tbl";
+    TableMetadata tableMetadata =
+        TableMetadata.newBuilder()
+            .addColumn("c1", DataType.INT)
+            .addColumn("c2", DataType.TEXT)
+            .addColumn("c3", DataType.BIGINT)
+            .addPartitionKey("c1")
+            .addSecondaryIndex("c3")
+            .build();
+    when(client.getDatabase(namespace)).thenReturn(database);
+    when(database.getContainer(table)).thenReturn(container);
+    // Metadata container
+    CosmosContainer metadataContainer = mock(CosmosContainer.class);
+    CosmosDatabase metadataDatabase = mock(CosmosDatabase.class);
+    when(client.getDatabase(METADATA_DATABASE)).thenReturn(metadataDatabase);
+    when(metadataDatabase.getContainer(CosmosAdmin.TABLE_METADATA_CONTAINER))
+        .thenReturn(metadataContainer);
+    // Stored procedure exists
+    CosmosScripts scripts = mock(CosmosScripts.class);
+    when(container.getScripts()).thenReturn(scripts);
+    CosmosStoredProcedure storedProcedure = mock(CosmosStoredProcedure.class);
+    when(scripts.getStoredProcedure(CosmosAdmin.STORED_PROCEDURE_FILE_NAME))
+        .thenReturn(storedProcedure);
+    // Existing container properties
+    CosmosContainerResponse response = mock(CosmosContainerResponse.class);
+    when(database.createContainerIfNotExists(table, "/concatenatedPartitionKey"))
+        .thenReturn(response);
+    CosmosContainerProperties properties = mock(CosmosContainerProperties.class);
+    when(response.getProperties()).thenReturn(properties);
+
+    // Act
+    admin.repairTable(namespace, table, tableMetadata, Collections.emptyMap());
+
+    // Assert
+    verify(container, times(1)).replace(properties);
+    verify(properties, times(1)).setIndexingPolicy(any(IndexingPolicy.class));
+  }
+
   @Test
   public void addNewColumnToTable_ShouldWorkProperly() throws ExecutionException {
     // Arrange