[PLAT-12874] Import Universe to operator - individual resources

Summary:
Implemented the OperatorImportResource task, along with each
resource's specific import functionality. The resources implemented:
1. Secrets
2. Storage Config
3. Backup Schedule
4. Release
5. Universe

Provider CRD is not yet merged, and backups have already been implemented.
Backups are integrated to the import task though

Test Plan: unit tests

Reviewers: anijhawan, vkumar, anabaria, #yba-api-review

Reviewed By: anijhawan, #yba-api-review

Subscribers: yugaware

Differential Revision: https://phorge.dev.yugabyte.com/D46338

Author: shubin-yb

managed/build.sbt

@@ -242,6 +242,7 @@ libraryDependencies ++= Seq(
   "io.fabric8" % "kubernetes-client" % "6.8.0",
   "io.fabric8" % "kubernetes-client-api" % "6.8.0",
   "io.fabric8" % "kubernetes-model" % "6.8.0",
+  "io.fabric8" % "kubernetes-server-mock" % "6.8.0",
   "org.modelmapper" % "modelmapper" % "2.4.4",
   "com.datadoghq" % "datadog-api-client" % "2.25.0" classifier "shaded-jar",
   "javax.xml.bind" % "jaxb-api" % "2.3.1",
@@ -468,7 +469,7 @@ generateCrdObjects / fileInputs += baseDirectory.value.toGlob /
 generateCrdObjects := {
   if (generateCrdObjects.inputFileChanges.hasChanges) {
     ybLog("Generating crd classes...")
-    val generatedSourcesDirectory = baseDirectory.value / "target/scala-2.13/"
+    val generatedSourcesDirectory = baseDirectory.value / "target/operatorCRD/"
     val command = s"mvn generate-sources -DoutputDirectory=$generatedSourcesDirectory"
     val status = Process(command, baseDirectory.value / "src/main/java/com/yugabyte/yw/common/operator/").!
     status
@@ -477,6 +478,7 @@ generateCrdObjects := {
     0
   }
 }
+Compile / unmanagedSourceDirectories += baseDirectory.value / "target/operatorCRD/"
 
 downloadThirdPartyDeps := {
   ybLog("Downloading third-party dependencies...")

managed/src/main/java/MainModule.java

@@ -84,7 +84,9 @@
 import com.yugabyte.yw.common.operator.OperatorStatusUpdaterFactory;
 import com.yugabyte.yw.common.operator.YBInformerFactory;
 import com.yugabyte.yw.common.operator.YBReconcilerFactory;
+import com.yugabyte.yw.common.operator.utils.KubernetesClientFactory;
 import com.yugabyte.yw.common.operator.utils.OperatorUtils;
+import com.yugabyte.yw.common.operator.utils.UniverseImporter;
 import com.yugabyte.yw.common.rbac.PermissionUtil;
 import com.yugabyte.yw.common.rbac.RoleBindingUtil;
 import com.yugabyte.yw.common.rbac.RoleUtil;
@@ -94,7 +96,6 @@
 import com.yugabyte.yw.common.ybflyway.YBFlywayInit;
 import com.yugabyte.yw.controllers.MetricGrafanaController;
 import com.yugabyte.yw.controllers.PlatformHttpActionAdapter;
-import com.yugabyte.yw.controllers.handlers.OperatorResourceMigrateHandler;
 import com.yugabyte.yw.metrics.MetricQueryHelper;
 import com.yugabyte.yw.models.CertificateInfo;
 import com.yugabyte.yw.models.HealthCheck;
@@ -303,7 +304,8 @@ public void configure() {
     bind(ReleasesUtils.class).asEagerSingleton();
     bind(ReleaseContainerFactory.class).asEagerSingleton();
     bind(SoftwareUpgradeHelper.class).asEagerSingleton();
-    bind(OperatorResourceMigrateHandler.class).asEagerSingleton();
+    bind(KubernetesClientFactory.class).asEagerSingleton();
+    bind(UniverseImporter.class).asEagerSingleton();
 
     // Destroy current session on SSO logout.
     final LogoutController logoutController = new LogoutController();

managed/src/main/java/api/v2/controllers/UniverseApiControllerImp.java

@@ -18,6 +18,7 @@
 import api.v2.models.UniverseEditGFlags;
 import api.v2.models.UniverseEditKubernetesOverrides;
 import api.v2.models.UniverseEditSpec;
+import api.v2.models.UniverseOperatorImportReq;
 import api.v2.models.UniverseQueryLogsExport;
 import api.v2.models.UniverseResourceDetails;
 import api.v2.models.UniverseRestart;
@@ -220,4 +221,16 @@ public YBATask configureMetricsExport(
       Request request, UUID cUUID, UUID uniUUID, ConfigureMetricsExportSpec req) throws Exception {
     return universeUpgradeHandler.configureMetricsExport(request, cUUID, uniUUID, req);
   }
+
+  @Override
+  public YBATask operatorImportUniverse(
+      Request request, UUID cUUID, UUID uniUUID, UniverseOperatorImportReq req) throws Exception {
+    return universeHandler.operatorImportUniverse(request, cUUID, uniUUID, req);
+  }
+
+  @Override
+  public void operatorImportUniversePrecheck(
+      Request request, UUID cUUID, UUID uniUUID, UniverseOperatorImportReq req) throws Exception {
+    universeHandler.precheckOperatorImportUniverse(request, cUUID, uniUUID, req);
+  }
 }

managed/src/main/java/api/v2/handlers/UniverseManagementHandler.java

@@ -19,6 +19,7 @@
 import api.v2.models.UniverseCreateSpec;
 import api.v2.models.UniverseDeleteSpec;
 import api.v2.models.UniverseEditSpec;
+import api.v2.models.UniverseOperatorImportReq;
 import api.v2.models.UniverseSpec;
 import api.v2.models.YBATask;
 import api.v2.utils.ApiControllerUtils;
@@ -709,4 +710,76 @@ public api.v2.models.UniverseResourceDetails getUniverseResources(
     }
     return v2Response;
   }
+
+  public void precheckOperatorImportUniverse(
+      Request request, UUID cUUID, UUID uniUUID, UniverseOperatorImportReq req) {
+    Customer customer = Customer.getOrBadRequest(cUUID);
+    Universe universe = Universe.getOrBadRequest(uniUUID, customer);
+
+    // validate the universe is kubernetes
+    if (!Util.isKubernetesBasedUniverse(universe)) {
+      log.error(
+          "Universe {} is not a Kubernetes universe, cannot migrate to operator",
+          universe.getName());
+      throw new PlatformServiceException(BAD_REQUEST, "Universe is not a Kubernetes universe");
+    }
+    if (!confGetter.getGlobalConf(GlobalConfKeys.KubernetesOperatorEnabled)) {
+      log.error("Operator is not enabled, cannot migrate universe {}", universe.getName());
+      throw new PlatformServiceException(
+          BAD_REQUEST,
+          "Operator is not enabled. Please enable the runtime config"
+              + " 'yb.kubernetes.operator.enabled' and restart YBA");
+    }
+
+    // Check if the namespace is the same as the operator namespace if it is set
+    boolean migrateNamespaceSet = req.getNamespace() != null && !req.getNamespace().isEmpty();
+    String operatorNamespace = confGetter.getGlobalConf(GlobalConfKeys.KubernetesOperatorNamespace);
+    boolean operatorNamespaceSet = operatorNamespace != null && !operatorNamespace.isEmpty();
+    if (migrateNamespaceSet
+        && operatorNamespaceSet
+        && !req.getNamespace().equals(operatorNamespace)) {
+      log.error(
+          "Namespace {} is not the same as the operator namespace {}",
+          req.getNamespace(),
+          operatorNamespace);
+      throw new PlatformServiceException(
+          BAD_REQUEST,
+          "Namespace is not the same as the operator namespace. Please set the namespace to the"
+              + " same as the operator namespace");
+    }
+
+    // Read Replicas clusters are not supported
+    if (universe.getUniverseDetails().clusters.size() > 1) {
+      log.error(
+          "Universe {} has read-only clusters, cannot migrate to operator", universe.getName());
+      throw new PlatformServiceException(BAD_REQUEST, "Universe has read-only clusters");
+    }
+
+    // XCluster is not supported by operator
+    if (!XClusterConfig.getByUniverseUuid(universe.getUniverseUUID()).isEmpty()) {
+      log.error("Universe {} has xClusterInfo set, cannot migrate to operator", universe.getName());
+      throw new PlatformServiceException(
+          BAD_REQUEST, "Cannot migrate universes in an xcluster setup.");
+    }
+
+    // AZ Level overrides are not supported by operator
+    Map<String, String> azOverrides =
+        universe.getUniverseDetails().getPrimaryCluster().userIntent.azOverrides;
+    if (azOverrides != null && azOverrides.size() > 0) {
+      log.error(
+          "Universe {} has AZ level overrides set, cannot migrate to operator", universe.getName());
+      throw new PlatformServiceException(
+          BAD_REQUEST, "Cannot migrate universes with AZ level overrides.");
+    }
+  }
+
+  public YBATask operatorImportUniverse(
+      Request request, UUID cUUID, UUID uniUUID, UniverseOperatorImportReq req) {
+    Customer customer = Customer.getOrBadRequest(cUUID);
+    Universe universe = Universe.getOrBadRequest(uniUUID, customer);
+    // TODO: Implement the migration logic
+    UUID taskUuid = UUID.randomUUID();
+    YBATask ybaTask = new YBATask().taskUuid(taskUuid).resourceUuid(universe.getUniverseUUID());
+    return ybaTask;
+  }
 }

managed/src/main/java/com/yugabyte/yw/commissioner/tasks/OperatorImportUniverse.java

@@ -1,5 +1,6 @@
 package com.yugabyte.yw.commissioner.tasks;
 
+import static com.yugabyte.yw.commissioner.tasks.subtasks.OperatorImportResource.*;
 import static play.mvc.Http.Status.BAD_REQUEST;
 
 import com.google.inject.Inject;
@@ -11,8 +12,8 @@
 import com.yugabyte.yw.commissioner.tasks.subtasks.OperatorImportResource;
 import com.yugabyte.yw.common.PlatformServiceException;
 import com.yugabyte.yw.common.Util;
+import com.yugabyte.yw.common.operator.StorageConfigReconciler;
 import com.yugabyte.yw.forms.BackupRequestParams;
-import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.UserIntent;
 import com.yugabyte.yw.forms.UniverseTaskParams;
 import com.yugabyte.yw.models.Backup;
 import com.yugabyte.yw.models.Customer;
@@ -47,7 +48,8 @@ public OperatorImportUniverse(BaseTaskDependencies baseTaskDependencies) {
   }
 
   public static class Params extends UniverseTaskParams {
-    UUID universeUUID;
+    // UniverseUUID is also required, but already defined in the base class.
+    String namespace;
   }
 
   @Override
@@ -71,9 +73,10 @@ private static interface ImportResourceTaskCreator {
 
   @Override
   public void run() {
-    Universe universe =
-        lockAndFreezeUniverseForUpdate(
-            taskParams().universeUUID, taskParams().expectedUniverseVersion, null);
+    log.info(
+        "Running OperatorImportUniverse task with universe UUID: {}",
+        taskParams().getUniverseUUID());
+    Universe universe = lockAndFreezeUniverseForUpdate(taskParams().getUniverseUUID(), -1, null);
 
     for (ImportResourceTaskCreator creator : importResourceTaskCreators) {
       log.trace("creating subtaskGroup with creator {}", creator);
@@ -94,13 +97,17 @@ public void run() {
   }
 
   private void createImportSecretSubtask(
-      String secretName, String secretValue, SubTaskGroup group) {
+      String secretName, String secretKey, String secretValue, SubTaskGroup group) {
     OperatorImportResource task = createTask(OperatorImportResource.class);
     OperatorImportResource.Params params = new OperatorImportResource.Params();
     params.secretName = secretName;
+    params.secretKey = secretKey;
     params.secretValue = secretValue;
     params.resourceType = OperatorImportResource.Params.ResourceType.SECRET;
+    params.namespace = taskParams().namespace;
+
     initializeTask(group, task, params);
+    log.trace("initialized task for secret");
   }
 
   private void createImportStorageConfigSubtask(
@@ -118,24 +125,41 @@ private void createImportStorageConfigSubtask(
           String.format(
               "no storage config %s found or is not a storage config", storageConfigUUID));
     }
+
+    OperatorImportResource task = createTask(OperatorImportResource.class);
+    OperatorImportResource.Params params = new OperatorImportResource.Params();
+    params.storageConfigUUID = storageConfigUUID;
+    params.resourceType = OperatorImportResource.Params.ResourceType.STORAGE_CONFIG;
+    params.namespace = taskParams().namespace;
+
     // Create Secret task(s)
     switch (storageConfig.getName()) {
       case Util.S3:
         CustomerConfigStorageS3Data s3Data =
             (CustomerConfigStorageS3Data) storageConfig.getDataObject();
         createImportSecretSubtask(
-            "awsSecretAccessKeySecret", s3Data.awsSecretAccessKey, secretGroup);
+            "awsSecretAccessKeySecret",
+            StorageConfigReconciler.AWS_SECRET_ACCESS_KEY_SECRET_KEY,
+            s3Data.awsSecretAccessKey,
+            secretGroup);
         break;
       case Util.GCS:
         CustomerConfigStorageGCSData gcsData =
             (CustomerConfigStorageGCSData) storageConfig.getDataObject();
         createImportSecretSubtask(
-            "gcsCredentialsJsonSecret", gcsData.gcsCredentialsJson, secretGroup);
+            "gcsCredentialsJsonSecret",
+            StorageConfigReconciler.GCS_CREDENTIALS_JSON_SECRET_KEY,
+            gcsData.gcsCredentialsJson,
+            secretGroup);
         break;
       case Util.AZ:
         CustomerConfigStorageAzureData azData =
             (CustomerConfigStorageAzureData) storageConfig.getDataObject();
-        createImportSecretSubtask("azureStorageSasTokenSecret", azData.azureSasToken, secretGroup);
+        createImportSecretSubtask(
+            "azureStorageSasTokenSecret",
+            StorageConfigReconciler.AZURE_STORAGE_SAS_TOKEN_SECRET_KEY,
+            azData.azureSasToken,
+            secretGroup);
         break;
       case Util.NFS:
         // No secrets for NFS
@@ -144,11 +168,8 @@ private void createImportStorageConfigSubtask(
         throw new RuntimeException("Unknown storage config type: " + storageConfig.getName());
     }
 
-    OperatorImportResource task = createTask(OperatorImportResource.class);
-    OperatorImportResource.Params params = new OperatorImportResource.Params();
-    params.storageConfigUUID = storageConfigUUID;
-    params.resourceType = OperatorImportResource.Params.ResourceType.STORAGE_CONFIG;
     initializeTask(storageCfgGroup, task, params);
+    log.trace("initialized task for storage config");
   }
 
   private List<SubTaskGroup> createImportReleaseSubtasks(Universe universe) {
@@ -159,7 +180,9 @@ private List<SubTaskGroup> createImportReleaseSubtasks(Universe universe) {
     params.releaseVersion =
         universe.getUniverseDetails().getPrimaryCluster().userIntent.ybSoftwareVersion;
     params.resourceType = OperatorImportResource.Params.ResourceType.RELEASE;
+    params.namespace = taskParams().namespace;
     initializeTask(group, task, params);
+    log.trace("initialized task for release");
     return List.of(group);
   }
 
@@ -174,35 +197,27 @@ private List<SubTaskGroup> createImportProviderSubtasks(Universe universe) {
     params.providerUUID =
         UUID.fromString(universe.getUniverseDetails().getPrimaryCluster().userIntent.provider);
     params.resourceType = OperatorImportResource.Params.ResourceType.PROVIDER;
+    params.namespace = taskParams().namespace;
     initializeTask(group, task, params);
+    log.trace("initialized task for provider");
     return List.of(group);
   }
 
   private List<SubTaskGroup> createImportUniverseSubtasks(Universe universe) {
     List<SubTaskGroup> groups = new ArrayList<>();
-    String secretName;
-    UserIntent userIntent = universe.getUniverseDetails().getPrimaryCluster().userIntent;
-    SubTaskGroup secretGroup =
-        createSubTaskGroup(UniverseImportSecretTaskName, SubTaskGroupType.OperatorImportResource);
-    if (userIntent.isYSQLAuthEnabled()) {
-      secretName = universe.getName() + "-ysqlpassword";
-      createImportSecretSubtask(secretName, userIntent.ysqlPassword, secretGroup);
-    }
-    if (userIntent.enableYCQLAuth) {
-      secretName = universe.getName() + "-ycqlpassword";
-      createImportSecretSubtask(secretName, userIntent.ycqlPassword, secretGroup);
-    }
-    if (secretGroup.getSubTaskCount() > 0) {
-      groups.add(secretGroup);
-    }
+
+    // Create universe params first so we can reference them when creating secrets
+    OperatorImportResource.Params universeParams = new OperatorImportResource.Params();
+    universeParams.universeUUID = universe.getUniverseUUID();
+    universeParams.resourceType = OperatorImportResource.Params.ResourceType.UNIVERSE;
+    universeParams.namespace = taskParams().namespace;
+
     SubTaskGroup group =
         createSubTaskGroup("ImportUniverse", SubTaskGroupType.OperatorImportResource);
     groups.add(group);
     OperatorImportResource task = createTask(OperatorImportResource.class);
-    OperatorImportResource.Params params = new OperatorImportResource.Params();
-    params.universeUUID = universe.getUniverseUUID();
-    params.resourceType = OperatorImportResource.Params.ResourceType.UNIVERSE;
-    initializeTask(group, task, params);
+    initializeTask(group, task, universeParams);
+    log.trace("initialized task for universe");
     return groups;
   }
 
@@ -222,14 +237,20 @@ private List<SubTaskGroup> createImportBackupSchedulesSubtasks(Universe universe
           // Create tasks for migrating storage configs
           BackupRequestParams backupParams =
               Json.mapper().convertValue(schedule.getTaskParams(), BackupRequestParams.class);
-          createImportStorageConfigSubtask(backupParams.scheduleUUID, storageCfgGroup, secretGroup);
+          createImportStorageConfigSubtask(
+              backupParams.storageConfigUUID, storageCfgGroup, secretGroup);
 
           // Now migrate the actual schedule.
           OperatorImportResource task = createTask(OperatorImportResource.class);
           OperatorImportResource.Params params = new OperatorImportResource.Params();
           params.backupScheduleUUID = schedule.getScheduleUUID();
           params.resourceType = OperatorImportResource.Params.ResourceType.BACKUP_SCHEDULE;
+          params.namespace = taskParams().namespace;
+
+          // Add any secrets from storage configs to this backup schedule's secret map
+          // The storage config subtask will have populated the secret map
           initializeTask(group, task, params);
+          log.trace("initialized task for backup schedule");
         });
     return List.of(secretGroup, storageCfgGroup, group);
   }
@@ -254,7 +275,12 @@ private List<SubTaskGroup> createImportBackupsSubtasks(Universe universe) {
           OperatorImportResource.Params params = new OperatorImportResource.Params();
           params.backupUUID = backup.getBackupUUID();
           params.resourceType = OperatorImportResource.Params.ResourceType.BACKUP;
+          params.namespace = taskParams().namespace;
+          params.customerUUID = customer.getUuid();
+          // Add any secrets from storage configs to this backup's secret map
+          // The storage config subtask will have populated the secret map
           initializeTask(group, task, params);
+          log.trace("initialized task for backup");
         });
     return List.of(group);
   }