feat: Converts clusters with multiple replications_specs, region_configs and shards (#22)

* remove version from pinned_fcv in example

* failing test

* multi repls

* num_shards

* error conditions

* fillTagsLabelsOpt

* fillRegionConfigs

* feedback about literal expressions

* change preview provider v2 doc

Author: lantoli

README.md

@@ -16,11 +16,13 @@ Install the plugin by running:
 atlas plugin install github.com/mongodb-labs/atlas-cli-plugin-terraform
 ```
 
-## Convert cluster to advanced_cluster v2
+## Convert mongodbatlas_cluster to mongodbatlas_advanced_cluster (preview provider v2)
 
 ### Usage
 
-If you want to convert a Terraform configuration from `mongodbatlas_cluster` to `mongodbatlas_advanced_cluster` schema v2, use the following command:
+**Note**: In order to use the **Preview for MongoDB Atlas Provider v2** of `mongodbatlas_advanced_cluster`, you need to set the environment variable `MONGODB_ATLAS_PREVIEW_PROVIDER_V2_ADVANCED_CLUSTER` to `true`.
+
+If you want to convert a Terraform configuration from `mongodbatlas_cluster` to `mongodbatlas_advanced_cluster`, use the following command:
 ```bash
 atlas terraform clusterToAdvancedCluster --file in.tf --output out.tf
 ```
@@ -35,7 +37,8 @@ If you want to overwrite the output file if it exists, or even use the same outp
 ### Limitations
 
 - The plugin doesn't support `regions_config` without `electable_nodes` as there can be some issues with `priority` when they only have `analytics_nodes` and/or `electable_nodes`.
-- `priority` is required in `regions_config` and must be a resolved number between 7 and 1, e.g. `var.priority` is not supported. This is to allow reordering them by descending priority as this is expected in `mongodbatlas_advanced_cluster`.
+- [`priority`](https://registry.terraform.io/providers/mongodb/mongodbatlas/latest/docs/resources/cluster#priority-1) is required in `regions_config` and must be a numeric [literal expression](https://developer.hashicorp.com/nomad/docs/job-specification/hcl2/expressions#literal-expressions) between 7 and 1, e.g. `var.priority` is not supported. This is to allow reordering them by descending priority as this is expected in `mongodbatlas_advanced_cluster`.
+- [`num_shards`](https://registry.terraform.io/providers/mongodb/mongodbatlas/latest/docs/resources/cluster#num_shards-2) in `replication_specs` must be a numeric [literal expression](https://developer.hashicorp.com/nomad/docs/job-specification/hcl2/expressions#literal-expressions), e.g. `var.num_shards` is not supported. This is to allow creating a `replication_specs` element per shard in `mongodbatlas_advanced_cluster`.
 - `dynamic` blocks to generate `replication_specs`, `regions_config`, etc. are not supported.
 
 ## Contributing

internal/convert/const_names.go

@@ -44,6 +44,7 @@ const (
 	nElectableNodes             = "electable_nodes"
 	nReadOnlyNodes              = "read_only_nodes"
 	nAnalyticsNodes             = "analytics_nodes"
+	nZoneName                   = "zone_name"
 	nKey                        = "key"
 	nValue                      = "value"
 )

internal/convert/convert.go

@@ -24,6 +24,7 @@ const (
 	errRepSpecs    = "setting " + nRepSpecs
 	errConfigs     = "setting " + nConfig
 	errPriority    = "setting " + nPriority
+	errNumShards   = "setting " + nNumShards
 )
 
 type attrVals struct {
@@ -34,8 +35,8 @@ type attrVals struct {
 // ClusterToAdvancedCluster transforms all mongodbatlas_cluster definitions in a
 // Terraform configuration file into mongodbatlas_advanced_cluster schema v2 definitions.
 // All other resources and data sources are left untouched.
-// Note: hclwrite.Tokens are used instead of cty.Value so expressions like var.region can be preserved.
-// cty.Value only supports resolved values.
+// Note: hclwrite.Tokens are used instead of cty.Value so expressions with interpolations like var.region can be preserved.
+// cty.Value only supports literal expressions.
 func ClusterToAdvancedCluster(config []byte) ([]byte, error) {
 	parser, err := hcl.GetParser(config)
 	if err != nil {
@@ -55,9 +56,9 @@ func ClusterToAdvancedCluster(config []byte) ([]byte, error) {
 		resource.SetLabels(labels)
 
 		if resourceb.FirstMatchingBlock(nRepSpecs, nil) != nil {
-			err = fillReplicationSpecs(resourceb)
+			err = fillCluster(resourceb)
 		} else {
-			err = fillFreeTier(resourceb)
+			err = fillFreeTierCluster(resourceb)
 		}
 		if err != nil {
 			return nil, err
@@ -70,8 +71,8 @@ func ClusterToAdvancedCluster(config []byte) ([]byte, error) {
 	return parser.Bytes(), nil
 }
 
-// fillFreeTier is the entry point to convert clusters in free tier
-func fillFreeTier(resourceb *hclwrite.Body) error {
+// fillFreeTierCluster is the entry point to convert clusters in free tier
+func fillFreeTierCluster(resourceb *hclwrite.Body) error {
 	resourceb.SetAttributeValue(nClusterType, cty.StringVal(valClusterType))
 	config := hclwrite.NewEmptyFile()
 	configb := config.Body()
@@ -97,73 +98,128 @@ func fillFreeTier(resourceb *hclwrite.Body) error {
 	return nil
 }
 
-// fillReplicationSpecs is the entry point to convert clusters with replications_specs (all but free tier)
-func fillReplicationSpecs(resourceb *hclwrite.Body) error {
+// fillCluster is the entry point to convert clusters with replications_specs (all but free tier)
+func fillCluster(resourceb *hclwrite.Body) error {
 	root, errRoot := popRootAttrs(resourceb)
 	if errRoot != nil {
 		return errRoot
 	}
 	resourceb.RemoveAttribute(nNumShards) // num_shards in root is not relevant, only in replication_specs
 	// ok to fail as cloud_backup is optional
 	_ = hcl.MoveAttr(resourceb, resourceb, nCloudBackup, nBackupEnabled, errRepSpecs)
-
-	// at least one replication_specs exists here, if not it would be a free tier cluster
-	repSpecsSrc := resourceb.FirstMatchingBlock(nRepSpecs, nil)
-	if err := checkDynamicBlock(repSpecsSrc.Body()); err != nil {
+	if err := fillReplicationSpecs(resourceb, root); err != nil {
 		return err
 	}
-	configs, errConfigs := getRegionConfigs(repSpecsSrc, root)
-	if errConfigs != nil {
-		return errConfigs
-	}
-	repSpecs := hclwrite.NewEmptyFile()
-	repSpecs.Body().SetAttributeRaw(nConfig, configs)
-	resourceb.SetAttributeRaw(nRepSpecs, hcl.TokensArraySingle(repSpecs.Body()))
-	tags, errTags := getTagsLabelsOpt(resourceb, nTags)
-	if errTags != nil {
-		return errTags
-	}
-	if tags != nil {
-		resourceb.SetAttributeRaw(nTags, tags)
-	}
-	labels, errLabels := getTagsLabelsOpt(resourceb, nLabels)
-	if errLabels != nil {
-		return errLabels
+	if err := fillTagsLabelsOpt(resourceb, nTags); err != nil {
+		return err
 	}
-	if labels != nil {
-		resourceb.SetAttributeRaw(nLabels, labels)
+	if err := fillTagsLabelsOpt(resourceb, nLabels); err != nil {
+		return err
 	}
 	fillBlockOpt(resourceb, nTimeouts)
 	fillBlockOpt(resourceb, nAdvConf)
 	fillBlockOpt(resourceb, nBiConnector)
 	fillBlockOpt(resourceb, nPinnedFCV)
-	resourceb.RemoveBlock(repSpecsSrc)
 	return nil
 }
 
-func getRegionConfigs(repSpecsSrc *hclwrite.Block, root attrVals) (hclwrite.Tokens, error) {
+func fillReplicationSpecs(resourceb *hclwrite.Body, root attrVals) error {
+	// at least one replication_specs exists here, if not it would be a free tier cluster
+	var specbs []*hclwrite.Body
+	for {
+		var (
+			specSrc = resourceb.FirstMatchingBlock(nRepSpecs, nil)
+			spec    = hclwrite.NewEmptyFile()
+			specb   = spec.Body()
+		)
+		if specSrc == nil {
+			break
+		}
+		specbSrc := specSrc.Body()
+		if err := checkDynamicBlock(specbSrc); err != nil {
+			return err
+		}
+		// ok to fail as zone_name is optional
+		_ = hcl.MoveAttr(specbSrc, specb, nZoneName, nZoneName, errRepSpecs)
+		shards := specbSrc.GetAttribute(nNumShards)
+		if shards == nil {
+			return fmt.Errorf("%s: %s not found", errRepSpecs, nNumShards)
+		}
+		shardsVal, err := hcl.GetAttrInt(shards, errNumShards)
+		if err != nil {
+			return err
+		}
+		if err := fillRegionConfigs(specb, specbSrc, root); err != nil {
+			return err
+		}
+		for range shardsVal {
+			specbs = append(specbs, specb)
+		}
+		resourceb.RemoveBlock(specSrc)
+	}
+	resourceb.SetAttributeRaw(nRepSpecs, hcl.TokensArray(specbs))
+	return nil
+}
+
+func fillTagsLabelsOpt(resourceb *hclwrite.Body, name string) error {
+	var (
+		file  = hclwrite.NewEmptyFile()
+		fileb = file.Body()
+		found = false
+	)
+	for {
+		block := resourceb.FirstMatchingBlock(name, nil)
+		if block == nil {
+			break
+		}
+		key := block.Body().GetAttribute(nKey)
+		value := block.Body().GetAttribute(nValue)
+		if key == nil || value == nil {
+			return fmt.Errorf("%s: %s or %s not found", name, nKey, nValue)
+		}
+		setKeyValue(fileb, key, value)
+		resourceb.RemoveBlock(block)
+		found = true
+	}
+	if found {
+		resourceb.SetAttributeRaw(name, hcl.TokensObject(fileb))
+	}
+	return nil
+}
+
+func fillBlockOpt(resourceb *hclwrite.Body, name string) {
+	block := resourceb.FirstMatchingBlock(name, nil)
+	if block == nil {
+		return
+	}
+	resourceb.RemoveBlock(block)
+	resourceb.SetAttributeRaw(name, hcl.TokensObject(block.Body()))
+}
+
+func fillRegionConfigs(specb, specbSrc *hclwrite.Body, root attrVals) error {
 	var configs []*hclwrite.Body
 	for {
-		configSrc := repSpecsSrc.Body().FirstMatchingBlock(nConfigSrc, nil)
+		configSrc := specbSrc.FirstMatchingBlock(nConfigSrc, nil)
 		if configSrc == nil {
 			break
 		}
 		config, err := getRegionConfig(configSrc, root)
 		if err != nil {
-			return nil, err
+			return err
 		}
 		configs = append(configs, config.Body())
-		repSpecsSrc.Body().RemoveBlock(configSrc)
+		specbSrc.RemoveBlock(configSrc)
 	}
 	if len(configs) == 0 {
-		return nil, fmt.Errorf("%s: %s not found", errRepSpecs, nConfigSrc)
+		return fmt.Errorf("%s: %s not found", errRepSpecs, nConfigSrc)
 	}
 	sort.Slice(configs, func(i, j int) bool {
 		pi, _ := hcl.GetAttrInt(configs[i].GetAttribute(nPriority), errPriority)
 		pj, _ := hcl.GetAttrInt(configs[j].GetAttribute(nPriority), errPriority)
 		return pi > pj
 	})
-	return hcl.TokensArray(configs), nil
+	specb.SetAttributeRaw(nConfig, hcl.TokensArray(configs))
+	return nil
 }
 
 func getRegionConfig(configSrc *hclwrite.Block, root attrVals) (*hclwrite.File, error) {
@@ -245,41 +301,6 @@ func getAutoScalingOpt(opt map[string]hclwrite.Tokens) hclwrite.Tokens {
 	return hcl.TokensObject(fileb)
 }
 
-func getTagsLabelsOpt(resourceb *hclwrite.Body, name string) (hclwrite.Tokens, error) {
-	var (
-		file  = hclwrite.NewEmptyFile()
-		fileb = file.Body()
-		found = false
-	)
-	for {
-		block := resourceb.FirstMatchingBlock(name, nil)
-		if block == nil {
-			break
-		}
-		key := block.Body().GetAttribute(nKey)
-		value := block.Body().GetAttribute(nValue)
-		if key == nil || value == nil {
-			return nil, fmt.Errorf("%s: %s or %s not found", name, nKey, nValue)
-		}
-		setKeyValue(fileb, key, value)
-		resourceb.RemoveBlock(block)
-		found = true
-	}
-	if !found {
-		return nil, nil
-	}
-	return hcl.TokensObject(fileb), nil
-}
-
-func fillBlockOpt(resourceb *hclwrite.Body, name string) {
-	block := resourceb.FirstMatchingBlock(name, nil)
-	if block == nil {
-		return
-	}
-	resourceb.RemoveBlock(block)
-	resourceb.SetAttributeRaw(name, hcl.TokensObject(block.Body()))
-}
-
 func checkDynamicBlock(body *hclwrite.Body) error {
 	for _, block := range body.Blocks() {
 		if block.Type() == "dynamic" {
@@ -297,7 +318,7 @@ func setKeyValue(body *hclwrite.Body, key, value *hclwrite.Attribute) {
 		}
 	} else {
 		keyStr = strings.TrimSpace(string(key.Expr().BuildTokens(nil).Bytes()))
-		keyStr = "(" + keyStr + ")" // wrap in parentheses so unresolved expressions can be used as attribute names
+		keyStr = "(" + keyStr + ")" // wrap in parentheses so non-literal expressions can be used as attribute names
 	}
 	body.SetAttributeRaw(keyStr, value.Expr().BuildTokens(nil))
 }

internal/convert/testdata/clu2adv/adv_config_bi_connector_pinned_fcv.in.tf

@@ -16,7 +16,6 @@ resource "mongodbatlas_cluster" "this" {
   pinned_fcv {
     # comments in pinned_fcv are kept
     expiration_date = var.fcv_date
-    version         = var.fcv_date
   }
   replication_specs {
     num_shards = 1

internal/convert/testdata/clu2adv/adv_config_bi_connector_pinned_fcv.out.tf

@@ -29,7 +29,6 @@ resource "mongodbatlas_advanced_cluster" "this" {
   pinned_fcv = {
     # comments in pinned_fcv are kept
     expiration_date = var.fcv_date
-    version         = var.fcv_date
   }
 
   # Generated by atlas-cli-plugin-terraform.

internal/convert/testdata/clu2adv/errors.json

@@ -6,7 +6,9 @@
 	"regions_config_missing_electable_nodes": "setting replication_specs: attribute electable_nodes not found",
 	"regions_config_missing_priority": "setting replication_specs: priority not found",
 	"regions_config_out_of_range_priority": "setting priority: priority is 0 but must be between 1 and 7",
-	"regions_config_unresolved_priority": "setting priority: failed to evaluate number",
+	"regions_config_non_literal_priority": "setting priority: failed to evaluate number",
 	"replication_specs_unsupported_dynamic": "dynamic blocks are not supported",
-	"regions_config_unsupported_dynamic": "dynamic blocks are not supported"
+	"regions_config_unsupported_dynamic": "dynamic blocks are not supported",
+	"replication_specs_non_literal_num_shards": "setting num_shards: failed to evaluate number",
+	"replication_specs_missing_num_shards": "num_shards not found"
 }

internal/convert/testdata/clu2adv/multi_replicaction_specs.in.tf

@@ -0,0 +1,59 @@
+resource "mongodbatlas_cluster" "multirep" {
+  project_id                  = var.project_id
+  name                        = "multirep"
+  disk_size_gb                = 80
+  num_shards                  = 1
+  cloud_backup                = false
+  cluster_type                = "GEOSHARDED"
+  provider_name               = "AWS"
+  provider_instance_size_name = "M10"
+  replication_specs {
+    zone_name  = "Zone 1"
+    num_shards = 1
+    regions_config {
+      region_name     = "US_EAST_1"
+      electable_nodes = 3
+      priority        = 7
+    }
+  }
+  replication_specs {
+    zone_name  = "Zone 2"
+    num_shards = 1
+    regions_config {
+      region_name     = "US_WEST_2"
+      electable_nodes = 3
+      priority        = 7
+    }
+  }
+}
+
+resource "mongodbatlas_cluster" "geo" {
+  project_id                  = "1234"
+  name                        = "geo"
+  disk_size_gb                = 80
+  num_shards                  = 1
+  cloud_backup                = false
+  cluster_type                = "GEOSHARDED"
+  provider_name               = "AWS"
+  provider_instance_size_name = "M10"
+  replication_specs {
+    zone_name  = "Zone 1"
+    num_shards = 2
+    regions_config {
+      region_name     = "US_EAST_1"
+      electable_nodes = 3
+      priority        = 7
+      read_only_nodes = 0
+    }
+  }
+  replication_specs {
+    zone_name  = "Zone 2"
+    num_shards = 3
+    regions_config {
+      region_name     = "US_WEST_2"
+      electable_nodes = 3
+      priority        = 7
+      read_only_nodes = 0
+    }
+  }
+}