Added concurrecy control for discovery

Author: vimalmeena238

internal/resourcediscovery/export_compartment.go

@@ -45,6 +45,10 @@ type ResourceDiscoveryStage int
 const (
 	Discovery       ResourceDiscoveryStage = 1
 	GeneratingState                        = 2
+	// maximum goroutines for finding resources in a step at base level.
+	// Value of MaxParallelFindResources is decided based on perf analysis and experiments.
+	// Note: SubResources will be discovered sequentially.
+	MaxParallelFindResource = 16
 )
 
 var (
@@ -315,7 +319,6 @@ func RunExportCommand(args *tf_export.ExportCommandArgs) (err error, status Stat
 	if len(ctx.ErrorList.Errors) > 0 {
 		// If the errors were from discovery of resources return partial success status
 		error, status := getListOfNotDiscoveredResources(ctx)
-
 		return error, status
 	}
 	return nil, StatusSuccess
@@ -616,15 +619,15 @@ func generateStateParallel(ctx *tf_export.ResourceDiscoveryContext, steps []reso
 
 			// Write temp state file for each service, this step will import resources into a separate state file for each service in parallel
 			// If writing temp configuration thrown error, won't attempt to write temp state
-			// Write temp state file for each service, this step will import resources into a separate state file for each service in parallel
-			if err := step.writeTmpState(); err != nil {
-				var tfError tf_export.ResourceDiscoveryCustomError
-				tfError = tf_export.ResourceDiscoveryCustomError{
-					TypeOfError: tf_export.WriteTmpStateError,
-					Message:     errors.New(tf_export.WriteTmpStateErrorMessage),
-					Suggestion:  tf_export.WriteTmpStateErrorSuggestion,
+			if err == nil {
+				if err = step.writeTmpState(); err != nil {
+					tfError := tf_export.ResourceDiscoveryCustomError{
+						TypeOfError: tf_export.WriteTmpStateError,
+						Message:     errors.New(tf_export.WriteTmpStateErrorMessage),
+						Suggestion:  tf_export.WriteTmpStateErrorSuggestion}
+
+					errorChannel <- tfError.Error()
 				}
-				errorChannel <- tfError.Error()
 			}
 
 			utils.Debugf("writing temp config and state: Completed step %d", i)
@@ -886,7 +889,7 @@ func getDiscoverResourceWithGraphSteps(ctx *tf_export.ResourceDiscoveryContext)
 	return result, nil
 }
 
-func findResources(ctx *tf_export.ResourceDiscoveryContext, root *tf_export.OCIResource, resourceGraph tf_export.TerraformResourceGraph) (foundResources []*tf_export.OCIResource, err error) {
+func findResources(ctx *tf_export.ResourceDiscoveryContext, root *tf_export.OCIResource, resourceGraph tf_export.TerraformResourceGraph, discoveryParallelism bool) (foundResources []*tf_export.OCIResource, err error) {
 	// findResources will never return error, it will add the errors encountered to the errorList and print those after the discovery finishes
 	// If find resources needs to fail in some scenario, this func needs to be modified to return error instead of continuing discovery
 	// Errors so far are API errors or the errors when service/feature is not available
@@ -901,22 +904,31 @@ func findResources(ctx *tf_export.ResourceDiscoveryContext, root *tf_export.OCIR
 	utils.Logf("[INFO] resource discovery: visiting %s\n", root.GetTerraformReference())
 	utils.Debugf("[DEBUG] resource discovery: visiting %s\n", root.GetTerraformReference())
 
-	utils.Logf("[INFO] number of child resource types for %s: %d\n", root.getTerraformReference(), len(childResourceTypes))
+	utils.Logf("[INFO] number of child resource types for %s: %d\n", root.GetTerraformReference(), len(childResourceTypes))
 
 	findResourcesStart := time.Now()
 	var findResourcesWg sync.WaitGroup
 	findResourcesWg.Add(len(childResourceTypes))
 
-	ch := make(chan struct{}, len(childResourceTypes))
+	// setting parallelism argument false for subResources to control concurrency and thrashing
+	var ch chan struct{}
+	if discoveryParallelism == true {
+		ch = make(chan struct{}, MaxParallelFindResource)
+		// setting parallelism argument false for subResources to control concurrency and thrashing
+		discoveryParallelism = false
+	} else {
+		ch = make(chan struct{}, 1)
+	}
 
 	for i, childType := range childResourceTypes {
 
 		ch <- struct{}{}
 
-		go func(i int, childType TerraformResourceAssociation) {
+		go func(i int, childType tf_export.TerraformResourceAssociation) {
 			utils.Debugf("[DEBUG] findResources: finding resources for resource type index: %d", i)
 
 			defer func(tfMeta *tf_export.TerraformResourceAssociation, err *error) {
+				<-ch
 				if r := recover(); r != nil {
 					utils.Logf("[WARN] recovered from panic in findResourcesGeneric for resource: %s \n continuing discovery...", tfMeta.ResourceClass)
 					returnErr := fmt.Errorf("panic in findResourcesGeneric for resource %s", tfMeta.ResourceClass)
@@ -928,8 +940,8 @@ func findResources(ctx *tf_export.ResourceDiscoveryContext, root *tf_export.OCIR
 			}(&childType, &err)
 
 			findResourceFn := tf_export.FindResourcesGeneric
-			if childType.findResourcesOverrideFn != nil {
-				findResourceFn = childType.findResourcesOverrideFn
+			if childType.FindResourcesOverrideFn != nil {
+				findResourceFn = childType.FindResourcesOverrideFn
 			}
 			results, err := findResourceFn(ctx, &childType, root, &resourceGraph)
 			if err != nil {
@@ -976,7 +988,7 @@ func findResources(ctx *tf_export.ResourceDiscoveryContext, root *tf_export.OCIR
 
 				}
 
-				subResources, err := findResources(ctx, resource, resourceGraph)
+				subResources, err := findResources(ctx, resource, resourceGraph, discoveryParallelism)
 				if err != nil {
 					continue
 				}
@@ -985,14 +997,13 @@ func findResources(ctx *tf_export.ResourceDiscoveryContext, root *tf_export.OCIR
 				foundResourcesLock.Unlock()
 			}
 			utils.Debugf("[DEBUG] findResources: Completed for resource type index %d", i)
-			<-ch
 		}(i, childType)
 	}
 
 	// Wait for all steps to complete findResources
 	findResourcesWg.Wait()
 	totalFindResourcesTime := time.Since(findResourcesStart)
-	utils.Debugf("finding resources for %s took %v\n", root.getTerraformReference(), totalFindResourcesTime)
+	utils.Debugf("finding resources for %s took %v\n", root.GetTerraformReference(), totalFindResourcesTime)
 
 	return foundResources, nil
 }

internal/resourcediscovery/export_compartment_test.go

@@ -1124,7 +1124,7 @@ func TestUnitFindResources_basic(t *testing.T) {
 	ctx := &tf_export.ResourceDiscoveryContext{
 		ErrorList: tf_export.ErrorList{},
 	}
-	results, err := findResources(ctx, rootResource, compartmentTestingResourceGraph)
+	results, err := findResources(ctx, rootResource, compartmentTestingResourceGraph, true)
 	if err != nil {
 		t.Logf("got error from findResources: %v", err)
 		t.Fail()
@@ -1164,7 +1164,7 @@ func TestUnitFindResources_404Error(t *testing.T) {
 	ctx := &tf_export.ResourceDiscoveryContext{
 		ErrorList: tf_export.ErrorList{},
 	}
-	results, err := findResources(ctx, rootResource, compartmentTestingResourceGraphWith404ErrorResource)
+	results, err := findResources(ctx, rootResource, compartmentTestingResourceGraphWith404ErrorResource, true)
 	if err != nil {
 		t.Logf("got error from findResources: %v", err)
 		t.Fail()
@@ -1198,7 +1198,7 @@ func TestUnitFindResources_panic(t *testing.T) {
 	ctx := &tf_export.ResourceDiscoveryContext{
 		ErrorList: tf_export.ErrorList{},
 	}
-	results, err := findResources(ctx, rootResource, compartmentTestingResourceGraphWithPanicResource)
+	results, err := findResources(ctx, rootResource, compartmentTestingResourceGraphWithPanicResource, true)
 	if err != nil {
 		t.Logf("got error from findResources: %v", err)
 		t.Fail()
@@ -1222,7 +1222,7 @@ func TestUnitFindResources_errorList(t *testing.T) {
 	ctx := &tf_export.ResourceDiscoveryContext{
 		ErrorList: tf_export.ErrorList{},
 	}
-	_, err := findResources(ctx, rootResource, compartmentTestingResourceGraphWithFaultyChildResource)
+	_, err := findResources(ctx, rootResource, compartmentTestingResourceGraphWithFaultyChildResource, true)
 	if err != nil {
 		t.Logf("got error from findResources: %v", err)
 		t.Fail()
@@ -1272,7 +1272,7 @@ func TestUnitFindResources_restrictedOcids(t *testing.T) {
 			ErrorList:           tf_export.ErrorList{},
 		}
 
-		results, err := findResources(ctx, rootResource, compartmentTestingResourceGraph)
+		results, err := findResources(ctx, rootResource, compartmentTestingResourceGraph, true)
 		if err != nil {
 			t.Logf("got error from findResources: %v", err)
 			t.Fail()
@@ -1308,7 +1308,7 @@ func TestUnitFindResources_overrideFn(t *testing.T) {
 	ctx := &tf_export.ResourceDiscoveryContext{
 		ErrorList: tf_export.ErrorList{},
 	}
-	results, err := findResources(ctx, rootResource, compartmentTestingResourceGraph)
+	results, err := findResources(ctx, rootResource, compartmentTestingResourceGraph, true)
 	if err != nil {
 		t.Logf("got error from findResources: %v", err)
 		t.Fail()
@@ -1348,7 +1348,7 @@ func TestUnitFindResources_processResourceFn(t *testing.T) {
 		ErrorList: tf_export.ErrorList{},
 	}
 
-	results, err := findResources(ctx, rootResource, compartmentTestingResourceGraph)
+	results, err := findResources(ctx, rootResource, compartmentTestingResourceGraph, true)
 	if err != nil {
 		t.Logf("got error from findResources: %v", err)
 		t.Fail()
@@ -1437,7 +1437,7 @@ func TestUnitGetHCLString_basic(t *testing.T) {
 	ctx := &tf_export.ResourceDiscoveryContext{
 		ErrorList: tf_export.ErrorList{},
 	}
-	results, err := findResources(ctx, rootResource, compartmentTestingResourceGraph)
+	results, err := findResources(ctx, rootResource, compartmentTestingResourceGraph, true)
 	if err != nil {
 		t.Logf("got error from findResources: %v", err)
 		t.Fail()
@@ -1513,7 +1513,7 @@ func TestUnitGetHCLStringFromMap(t *testing.T) {
 	rootResource := getRootCompartmentResource()
 
 	ctx := &tf_export.ResourceDiscoveryContext{}
-	results, err := findResources(ctx, rootResource, compartmentTestingResourceGraph)
+	results, err := findResources(ctx, rootResource, compartmentTestingResourceGraph, true)
 	if err != nil {
 		t.Logf("got error from findResources: %v", err)
 		t.Fail()
@@ -1545,7 +1545,7 @@ func TestUnitGetHCLString_missingFields(t *testing.T) {
 	ctx := &tf_export.ResourceDiscoveryContext{
 		ErrorList: tf_export.ErrorList{},
 	}
-	results, err := findResources(ctx, rootResource, compartmentTestingResourceGraph)
+	results, err := findResources(ctx, rootResource, compartmentTestingResourceGraph, true)
 	if err != nil {
 		t.Logf("got error from findResources: %v", err)
 		t.Fail()
@@ -1591,7 +1591,7 @@ func TestUnitGetHCLString_interpolationMap(t *testing.T) {
 	ctx := &tf_export.ResourceDiscoveryContext{
 		ErrorList: tf_export.ErrorList{},
 	}
-	results, err := findResources(ctx, rootResource, compartmentTestingResourceGraph)
+	results, err := findResources(ctx, rootResource, compartmentTestingResourceGraph, true)
 	if err != nil {
 		t.Logf("got error from findResources: %v", err)
 		t.Fail()
@@ -1641,7 +1641,7 @@ func TestUnitGetHCLString_tfSyntaxVersion(t *testing.T) {
 	rootResource := getRootCompartmentResource()
 
 	ctx := &tf_export.ResourceDiscoveryContext{}
-	results, err := findResources(ctx, rootResource, compartmentTestingResourceGraph)
+	results, err := findResources(ctx, rootResource, compartmentTestingResourceGraph, true)
 	if err != nil {
 		t.Logf("got error from findResources: %v", err)
 		t.Fail()
@@ -2012,7 +2012,7 @@ func TestUnitGetHCLString_logging(t *testing.T) {
 	ctx := &tf_export.ResourceDiscoveryContext{
 		ErrorList: tf_export.ErrorList{},
 	}
-	_, err = findResources(ctx, rootResource, compartmentTestingResourceGraph)
+	_, err = findResources(ctx, rootResource, compartmentTestingResourceGraph, true)
 	if err != nil {
 		t.Logf("got error from findResources: %v", err)
 		t.Fail()

internal/resourcediscovery/export_resource_helpers.go

@@ -124,6 +124,7 @@ type resourceDiscoveryBaseStep struct {
 	discoveredResources         []*tf_export.OCIResource
 	omittedResources            []*tf_export.OCIResource
 	tempState                   interface{}
+	discoveryParallelism        bool
 	timeTakenForDiscovery       time.Duration
 	timeTakenForGeneratingState time.Duration
 }
@@ -408,7 +409,11 @@ func (r *resourceDiscoveryWithGraph) discover() error {
 	var err error
 	var ociResources []*tf_export.OCIResource
 
-	ociResources, err = findResources(r.ctx, r.root, r.resourceGraph)
+	// for root step, setting discovery parallelism on
+	// turn it off for further levels
+	r.discoveryParallelism = true
+
+	ociResources, err = findResources(r.ctx, r.root, r.resourceGraph, r.discoveryParallelism)
 	if err != nil {
 		return err
 	}
@@ -534,7 +539,7 @@ func (r *resourceDiscoveryWithTargetIds) discover() error {
 
 		if _, hasRelatedResources := tf_export.ExportRelatedResourcesGraph[resourceHint.ResourceClass]; hasRelatedResources && r.ctx.IsExportWithRelatedResources {
 			utils.Logf("[INFO] resource discovery: finding related resources for %s\n", resourceHint.ResourceClass)
-			ociResources, err := findResources(r.ctx, ociResource, tf_export.ExportRelatedResourcesGraph)
+			ociResources, err := findResources(r.ctx, ociResource, tf_export.ExportRelatedResourcesGraph, r.discoveryParallelism)
 			if err != nil {
 				return err
 			}

website/docs/d/core_volume.html.markdown

@@ -46,7 +46,11 @@ The following attributes are exported:
 * `display_name` - A user-friendly name. Does not have to be unique, and it's changeable. Avoid entering confidential information. 
 * `freeform_tags` - Free-form tags for this resource. Each tag is a simple key-value pair with no predefined name, type, or namespace. For more information, see [Resource Tags](https://docs.cloud.oracle.com/iaas/Content/General/Concepts/resourcetags.htm).  Example: `{"Department": "Finance"}` 
 * `id` - The OCID of the volume.
+<<<<<<< ours
+* `is_auto_tune_enabled` - Specifies whether the auto-tune performance is enabled for this volume. This field is deprecated. Use the `DetachedVolumeAutotunePolicy` instead to enable the volume for detached autotune. 
+=======
 * `is_auto_tune_enabled` - Specifies whether the auto-tune performance is enabled for this boot volume.
+>>>>>>> theirs
 * `is_hydrated` - Specifies whether the cloned volume's data has finished copying from the source volume or backup. 
 * `kms_key_id` - The OCID of the Key Management key which is the master encryption key for the volume. 
 * `size_in_gbs` - The size of the volume in GBs.

website/docs/d/datascience_model_deployment.html.markdown

@@ -54,8 +54,13 @@ The following attributes are exported:
 		* `instance_configuration` - The model deployment instance configuration
 			* `instance_shape_name` - The shape used to launch the model deployment instances.
 			* `model_deployment_instance_shape_config_details` - Details for the model-deployment instance shape configuration.
+<<<<<<< ours
+				* `memory_in_gbs` - A model-deployment instance of type VM.Standard.E3.Flex or VM.Standard.E4.Flex allows the memory to be specified with in the range of 6 to 1024 GB. VM.Standard3.Flex memory range is between 6 to 512 GB and VM.Optimized3.Flex memory range is between 6 to 256 GB. 
+				* `ocpus` - A model-deployment instance of type VM.Standard.E3.Flex or VM.Standard.E4.Flex allows the ocpu count to be specified with in the range of 1 to 64 ocpu. VM.Standard3.Flex OCPU range is between 1 to 32 ocpu and for VM.Optimized3.Flex OCPU range is 1 to 18 ocpu. 
+=======
 				* `memory_in_gbs` - A model-deployment instance of type VM.Standard.E3.Flex or VM.Standard.E4.Flex allows the memory to be specified with in the range of 6 to 1024 GB. VM.Standard3.Flex memory range is between 6 and 512 GB and VM.Optimized3.Flex memory range is between 6 and 256 GB.
 				* `ocpus` - A model-deployment instance of type VM.Standard.E3.Flex or VM.Standard.E4.Flex allows the ocpu count to be specified with in the range of 1 to 64 ocpu. VM.Standard3.Flex OCPU range is between 1 and 32 ocpu and for VM.Optimized3.Flex OCPU range is 1 to 18 ocpu.
+>>>>>>> theirs
 		* `model_id` - The OCID of the model you want to deploy.
 		* `scaling_policy` - The scaling policy to apply to each model of the deployment.
 			* `instance_count` - The number of instances for the model deployment.

website/docs/d/datascience_model_deployments.html.markdown

@@ -73,8 +73,13 @@ The following attributes are exported:
 		* `instance_configuration` - The model deployment instance configuration
 			* `instance_shape_name` - The shape used to launch the model deployment instances.
 			* `model_deployment_instance_shape_config_details` - Details for the model-deployment instance shape configuration.
+<<<<<<< ours
+				* `memory_in_gbs` - A model-deployment instance of type VM.Standard.E3.Flex or VM.Standard.E4.Flex allows the memory to be specified with in the range of 6 to 1024 GB. VM.Standard3.Flex memory range is between 6 to 512 GB and VM.Optimized3.Flex memory range is between 6 to 256 GB. 
+				* `ocpus` - A model-deployment instance of type VM.Standard.E3.Flex or VM.Standard.E4.Flex allows the ocpu count to be specified with in the range of 1 to 64 ocpu. VM.Standard3.Flex OCPU range is between 1 to 32 ocpu and for VM.Optimized3.Flex OCPU range is 1 to 18 ocpu. 
+=======
 				* `memory_in_gbs` - A model-deployment instance of type VM.Standard.E3.Flex or VM.Standard.E4.Flex allows the memory to be specified with in the range of 6 to 1024 GB. VM.Standard3.Flex memory range is between 6 and 512 GB and VM.Optimized3.Flex memory range is between 6 and 256 GB.
 				* `ocpus` - A model-deployment instance of type VM.Standard.E3.Flex or VM.Standard.E4.Flex allows the ocpu count to be specified with in the range of 1 to 64 ocpu. VM.Standard3.Flex OCPU range is between 1 and 32 ocpu and for VM.Optimized3.Flex OCPU range is 1 to 18 ocpu.
+>>>>>>> theirs
 		* `model_id` - The OCID of the model you want to deploy.
 		* `scaling_policy` - The scaling policy to apply to each model of the deployment.
 			* `instance_count` - The number of instances for the model deployment.

website/docs/d/opsi_host_insight.html.markdown

@@ -33,6 +33,11 @@ The following arguments are supported:
 The following attributes are exported:
 
 * `compartment_id` - The [OCID](https://docs.cloud.oracle.com/iaas/Content/General/Concepts/identifiers.htm) of the compartment.
+<<<<<<< ours
+* `compute_id` - The [OCID](https://docs.cloud.oracle.com/iaas/Content/General/Concepts/identifiers.htm) of the Compute Instance
+* `defined_tags` - Defined tags for this resource. Each key is predefined and scoped to a namespace. Example: `{"foo-namespace.bar-key": "value"}` 
+=======
+>>>>>>> theirs
 * `enterprise_manager_bridge_id` - OPSI Enterprise Manager Bridge OCID
 * `enterprise_manager_entity_display_name` - Enterprise Manager Entity Display Name
 * `enterprise_manager_entity_identifier` - Enterprise Manager Entity Unique Identifier

website/docs/d/opsi_host_insights.html.markdown

@@ -39,7 +39,11 @@ The following arguments are supported:
 * `compartment_id_in_subtree` - (Optional) A flag to search all resources within a given compartment and all sub-compartments. 
 * `enterprise_manager_bridge_id` - (Applicable when entity_source=EM_MANAGED_EXTERNAL_HOST) Unique Enterprise Manager bridge identifier
 * `exadata_insight_id` - (Applicable when entity_source=EM_MANAGED_EXTERNAL_HOST) [OCID](https://docs.cloud.oracle.com/iaas/Content/General/Concepts/identifiers.htm) of exadata insight resource. 
+<<<<<<< ours
+* `host_type` - (Optional) Filter by one or more host types. Possible values are CLOUD-HOST, EXTERNAL-HOST 
+=======
 * `host_type` - (Optional) Filter by one or more host types. Possible value is EXTERNAL-HOST.
+>>>>>>> theirs
 * `id` - (Optional) Optional list of host insight resource [OCIDs](https://docs.cloud.oracle.com/iaas/Content/General/Concepts/identifiers.htm). 
 * `platform_type` - (Optional) Filter by one or more platform types. Supported platformType(s) for MACS-managed external host insight: [LINUX, SOLARIS, WINDOWS]. Supported platformType(s) for MACS-managed cloud host insight: [LINUX]. Supported platformType(s) for EM-managed external host insight: [LINUX, SOLARIS, SUNOS, ZLINUX, WINDOWS].
 * `state` - (Optional) Lifecycle states