Backport to branch(3) : Add support for administrative operations in Blob adapter

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitAdminIntegrationTestWithObjectStorage.java

@@ -1,7 +1,9 @@
 package com.scalar.db.storage.objectstorage;
 
+import com.scalar.db.config.DatabaseConfig;
 import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminIntegrationTestBase;
-import com.scalar.db.util.AdminTestUtils;
+import com.scalar.db.transaction.consensuscommit.ConsensusCommitConfig;
+import com.scalar.db.transaction.consensuscommit.Coordinator;
 import java.util.Properties;
 import org.junit.jupiter.api.Disabled;
 
@@ -14,8 +16,16 @@ protected Properties getProps(String testName) {
   }
 
   @Override
-  protected AdminTestUtils getAdminTestUtils(String testName) {
-    return new ObjectStorageAdminTestUtils(getProperties(testName));
+  protected String getSystemNamespaceName(Properties properties) {
+    return ObjectStorageUtils.getObjectStorageConfig(new DatabaseConfig(properties))
+        .getMetadataNamespace();
+  }
+
+  @Override
+  protected String getCoordinatorNamespaceName(String testName) {
+    return new ConsensusCommitConfig(new DatabaseConfig(getProperties(testName)))
+        .getCoordinatorNamespace()
+        .orElse(Coordinator.NAMESPACE);
   }
 
   @Override
@@ -130,9 +140,4 @@ public void renameTable_ForExistingTableWithIndexes_ShouldRenameTableAndIndexesC
   @Override
   @Disabled("Object Storage does not support renaming tables")
   public void renameTable_IfOnlyOneTableExists_ShouldRenameTableCorrectly() {}
-
-  @Override
-  @Disabled("The ScalarDB environment does not need to be upgraded with Object Storage")
-  public void
-      upgrade_WhenMetadataTableExistsButNotNamespacesTable_ShouldCreateNamespacesTableAndImportExistingNamespaces() {}
 }

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ConsensusCommitAdminRepairTableIntegrationTestWithObjectStorage.java

@@ -0,0 +1,29 @@
+package com.scalar.db.storage.objectstorage;
+
+import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminRepairTableIntegrationTestBase;
+import com.scalar.db.util.AdminTestUtils;
+import java.util.Properties;
+import org.junit.jupiter.api.Disabled;
+
+public class ConsensusCommitAdminRepairTableIntegrationTestWithObjectStorage
+    extends ConsensusCommitAdminRepairTableIntegrationTestBase {
+
+  @Override
+  protected Properties getProps(String testName) {
+    return ObjectStorageEnv.getProperties(testName);
+  }
+
+  @Override
+  protected AdminTestUtils getAdminTestUtils(String testName) {
+    return new ObjectStorageAdminTestUtils(getProperties(testName));
+  }
+
+  @Override
+  @Disabled("Object Storage recreates missing coordinator tables")
+  public void
+      repairTableAndCoordinatorTable_CoordinatorTablesDoNotExist_ShouldThrowIllegalArgumentException() {}
+
+  @Override
+  @Disabled("Object Storage recreates missing coordinator tables")
+  public void repairTable_ForNonExistingTable_ShouldThrowIllegalArgument() {}
+}

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageAdminCaseSensitivityIntegrationTest.java

@@ -1,6 +1,7 @@
 package com.scalar.db.storage.objectstorage;
 
 import com.scalar.db.api.DistributedStorageAdminCaseSensitivityIntegrationTestBase;
+import com.scalar.db.config.DatabaseConfig;
 import com.scalar.db.util.AdminTestUtils;
 import java.util.Map;
 import java.util.Properties;
@@ -14,6 +15,12 @@ protected Properties getProperties(String testName) {
     return ObjectStorageEnv.getProperties(testName);
   }
 
+  @Override
+  protected String getSystemNamespaceName(Properties properties) {
+    return ObjectStorageUtils.getObjectStorageConfig(new DatabaseConfig(properties))
+        .getMetadataNamespace();
+  }
+
   @Override
   protected Map<String, String> getCreationOptions() {
     return ObjectStorageEnv.getCreationOptions();
@@ -145,9 +152,4 @@ public void renameTable_ForExistingTableWithIndexes_ShouldRenameTableAndIndexesC
   @Override
   @Disabled("Object Storage does not support renaming tables")
   public void renameTable_IfOnlyOneTableExists_ShouldRenameTableCorrectly() {}
-
-  @Override
-  @Disabled("The ScalarDB environment does not need to be upgraded with Object Storage")
-  public void
-      upgrade_WhenMetadataTableExistsButNotNamespacesTable_ShouldCreateNamespacesTableAndImportExistingNamespaces() {}
 }

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageAdminIntegrationTest.java

@@ -1,6 +1,7 @@
 package com.scalar.db.storage.objectstorage;
 
 import com.scalar.db.api.DistributedStorageAdminIntegrationTestBase;
+import com.scalar.db.config.DatabaseConfig;
 import com.scalar.db.util.AdminTestUtils;
 import java.util.Properties;
 import org.junit.jupiter.api.Disabled;
@@ -12,6 +13,12 @@ protected Properties getProperties(String testName) {
     return ObjectStorageEnv.getProperties(testName);
   }
 
+  @Override
+  protected String getSystemNamespaceName(Properties properties) {
+    return ObjectStorageUtils.getObjectStorageConfig(new DatabaseConfig(properties))
+        .getMetadataNamespace();
+  }
+
   @Override
   protected boolean isIndexOnBooleanColumnSupported() {
     return false;
@@ -143,9 +150,4 @@ public void renameTable_ForExistingTableWithIndexes_ShouldRenameTableAndIndexesC
   @Override
   @Disabled("Object Storage does not support renaming tables")
   public void renameTable_IfOnlyOneTableExists_ShouldRenameTableCorrectly() {}
-
-  @Override
-  @Disabled("The ScalarDB environment does not need to be upgraded with Object Storage")
-  public void
-      upgrade_WhenMetadataTableExistsButNotNamespacesTable_ShouldCreateNamespacesTableAndImportExistingNamespaces() {}
 }

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageAdminRepairTableIntegrationTest.java

@@ -0,0 +1,24 @@
+package com.scalar.db.storage.objectstorage;
+
+import com.scalar.db.api.DistributedStorageAdminRepairTableIntegrationTestBase;
+import com.scalar.db.util.AdminTestUtils;
+import java.util.Properties;
+import org.junit.jupiter.api.Disabled;
+
+public class ObjectStorageAdminRepairTableIntegrationTest
+    extends DistributedStorageAdminRepairTableIntegrationTestBase {
+
+  @Override
+  protected Properties getProperties(String testName) {
+    return ObjectStorageEnv.getProperties(testName);
+  }
+
+  @Override
+  protected AdminTestUtils getAdminTestUtils(String testName) {
+    return new ObjectStorageAdminTestUtils(getProperties(testName));
+  }
+
+  @Override
+  @Disabled("Object Storage recreates missing coordinator tables")
+  public void repairTable_ForNonExistingTable_ShouldThrowIllegalArgument() {}
+}

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageAdminTestUtils.java

@@ -21,27 +21,11 @@ public ObjectStorageAdminTestUtils(Properties properties) {
     metadataNamespace = objectStorageConfig.getMetadataNamespace();
   }
 
-  @Override
-  public void dropNamespacesTable() {
-    // Object Storage does not have a concept of table
-  }
-
   @Override
   public void dropMetadataTable() {
     // Object Storage does not have a concept of table
   }
 
-  @Override
-  public void truncateNamespacesTable() throws Exception {
-    try {
-      wrapper.delete(
-          ObjectStorageUtils.getObjectKey(
-              metadataNamespace, ObjectStorageAdmin.NAMESPACE_METADATA_TABLE));
-    } catch (PreconditionFailedException e) {
-      // The namespace metadata table object does not exist, so do nothing
-    }
-  }
-
   @Override
   public void truncateMetadataTable() throws Exception {
     try {
@@ -103,11 +87,6 @@ public void deleteMetadata(String namespace, String table) throws Exception {
     }
   }
 
-  @Override
-  public void dropNamespace(String namespace) {
-    // Object Storage does not have a concept of namespace
-  }
-
   @Override
   public boolean namespaceExists(String namespace) {
     // Object Storage does not have a concept of namespace

core/src/integration-test/java/com/scalar/db/storage/objectstorage/ObjectStorageEnv.java

@@ -37,7 +37,7 @@ public static Properties getProperties(String testName) {
 
     // Add testName as a metadata namespace suffix
     properties.setProperty(
-        DatabaseConfig.SYSTEM_NAMESPACE_NAME,
+        BlobStorageConfig.TABLE_METADATA_NAMESPACE,
         DatabaseConfig.DEFAULT_SYSTEM_NAMESPACE_NAME + "_" + testName);
 
     return properties;

core/src/integration-test/java/com/scalar/db/storage/objectstorage/SingleCrudOperationTransactionAdminIntegrationTestWithObjectStorage.java

@@ -1,5 +1,6 @@
 package com.scalar.db.storage.objectstorage;
 
+import com.scalar.db.config.DatabaseConfig;
 import com.scalar.db.transaction.singlecrudoperation.SingleCrudOperationTransactionAdminIntegrationTestBase;
 import java.util.Properties;
 import org.junit.jupiter.api.Disabled;
@@ -12,6 +13,12 @@ protected Properties getProps(String testName) {
     return ObjectStorageEnv.getProperties(testName);
   }
 
+  @Override
+  protected String getSystemNamespaceName(Properties properties) {
+    return ObjectStorageUtils.getObjectStorageConfig(new DatabaseConfig(properties))
+        .getMetadataNamespace();
+  }
+
   @Override
   @Disabled("Temporarily disabled because it includes DML operations")
   public void truncateTable_ShouldTruncateProperly() {}
@@ -124,9 +131,4 @@ public void renameTable_ForExistingTableWithIndexes_ShouldRenameTableAndIndexesC
   @Override
   @Disabled("Object Storage does not support renaming tables")
   public void renameTable_IfOnlyOneTableExists_ShouldRenameTableCorrectly() {}
-
-  @Override
-  @Disabled("The ScalarDB environment does not need to be upgraded with Object Storage")
-  public void
-      upgrade_WhenMetadataTableExistsButNotNamespacesTable_ShouldCreateNamespacesTableAndImportExistingNamespaces() {}
 }

core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageAdmin.java

@@ -14,6 +14,7 @@
 import com.scalar.db.util.ScalarDbUtils;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -229,29 +230,6 @@ public boolean namespaceExists(String namespace) throws ExecutionException {
     }
   }
 
-  @Override
-  public void repairNamespace(String namespace, Map<String, String> options)
-      throws ExecutionException {
-    try {
-      // Upsert the namespace metadata
-      Map<String, String> readVersionMap = new HashMap<>();
-      Map<String, ObjectStorageNamespaceMetadata> metadataTable =
-          getNamespaceMetadataTable(readVersionMap);
-      if (metadataTable.isEmpty()) {
-        insertMetadataTable(
-            NAMESPACE_METADATA_TABLE,
-            Collections.singletonMap(namespace, new ObjectStorageNamespaceMetadata(namespace)));
-      } else {
-        metadataTable.put(namespace, new ObjectStorageNamespaceMetadata(namespace));
-        updateMetadataTable(
-            NAMESPACE_METADATA_TABLE, metadataTable, readVersionMap.get(NAMESPACE_METADATA_TABLE));
-      }
-    } catch (Exception e) {
-      throw new ExecutionException(
-          String.format("Failed to repair the namespace %s", namespace), e);
-    }
-  }
-
   @Override
   public void importTable(
       String namespace,
@@ -346,17 +324,14 @@ public void renameTable(String namespace, String oldTableName, String newTableNa
   @Override
   public Set<String> getNamespaceNames() throws ExecutionException {
     try {
-      return getNamespaceMetadataTable().keySet();
+      Set<String> namespaceNames = new HashSet<>(getNamespaceMetadataTable().keySet());
+      namespaceNames.add(metadataNamespace);
+      return namespaceNames;
     } catch (Exception e) {
       throw new ExecutionException("Failed to get the namespace names", e);
     }
   }
 
-  @Override
-  public void upgrade(Map<String, String> options) throws ExecutionException {
-    // Currently, nothing needs to be upgraded. Do nothing.
-  }
-
   private Map<String, ObjectStorageNamespaceMetadata> getNamespaceMetadataTable()
       throws ExecutionException {
     return getNamespaceMetadataTable(null);

core/src/main/java/com/scalar/db/storage/objectstorage/ObjectStorageProvider.java

@@ -15,6 +15,6 @@ default DistributedStorage createDistributedStorage(DatabaseConfig config) {
 
   @Override
   default DistributedStorageAdmin createDistributedStorageAdmin(DatabaseConfig config) {
-    return new CommonDistributedStorageAdmin(new ObjectStorageAdmin(config), config);
+    return new CommonDistributedStorageAdmin(new ObjectStorageAdmin(config), true);
   }
 }

core/src/main/java/com/scalar/db/storage/objectstorage/blobstorage/BlobStorageConfig.java

@@ -2,6 +2,7 @@
 
 import static com.scalar.db.config.ConfigUtils.getInt;
 import static com.scalar.db.config.ConfigUtils.getLong;
+import static com.scalar.db.config.ConfigUtils.getString;
 
 import com.scalar.db.common.CoreError;
 import com.scalar.db.config.DatabaseConfig;
@@ -12,6 +13,7 @@
 public class BlobStorageConfig implements ObjectStorageConfig {
   public static final String STORAGE_NAME = "blob-storage";
   public static final String PREFIX = DatabaseConfig.PREFIX + STORAGE_NAME + ".";
+  public static final String TABLE_METADATA_NAMESPACE = PREFIX + "table_metadata.namespace";
 
   public static final String PARALLEL_UPLOAD_BLOCK_SIZE_IN_BYTES =
       PREFIX + "parallel_upload_block_size_in_bytes";
@@ -58,7 +60,7 @@ public BlobStorageConfig(DatabaseConfig databaseConfig) {
     }
     username = databaseConfig.getUsername().orElse(null);
     password = databaseConfig.getPassword().orElse(null);
-    metadataNamespace = databaseConfig.getSystemNamespaceName();
+    metadataNamespace = getString(databaseConfig.getProperties(), TABLE_METADATA_NAMESPACE, null);
 
     if (databaseConfig.getScanFetchSize() != DatabaseConfig.DEFAULT_SCAN_FETCH_SIZE) {
       logger.warn(