Merge pull request #1565 from snyk/fea/facade_enum

add a facade to implement enum library api

Author: eliecharra

enumeration/diagnostic.go

@@ -0,0 +1,12 @@
+package enumeration
+
+import "github.com/snyk/driftctl/enumeration/resource"
+
+type Diagnostic interface {
+	Code() string
+	Message() string
+	ResourceType() string
+	Resource() *resource.Resource
+}
+
+type Diagnostics []Diagnostic

enumeration/enum.go

@@ -0,0 +1,32 @@
+package enumeration
+
+import (
+	"time"
+
+	"github.com/snyk/driftctl/enumeration/resource"
+)
+
+type EnumerateInput struct {
+	ResourceTypes []string
+}
+
+type EnumerateOutput struct {
+	// Resources is a map of resources by type. Every listed resource type will
+	// have a key in the map. The value will be either nil or an empty slice if
+	// no resources of that type were found.
+	Resources map[string][]*resource.Resource
+
+	// Timings is map of list durations by resource type. This aids understanding
+	// which resource types took the most time to list.
+	Timings map[string]time.Duration
+
+	// Diagnostics contains messages and errors that arose during the list operation.
+	// If the diagnostic is associated with a resource type, the ResourceType()
+	// call will indicate which type. If associated with a resource, the Resource()
+	// call will indicate which resource.
+	Diagnostics Diagnostics
+}
+
+type Enumerator interface {
+	Enumerate(*EnumerateInput) (*EnumerateOutput, error)
+}

enumeration/enumerator/cloud_enumerator.go

@@ -0,0 +1,266 @@
+package enumerator
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/snyk/driftctl/enumeration"
+
+	"github.com/sirupsen/logrus"
+	"github.com/snyk/driftctl/enumeration/alerter"
+	"github.com/snyk/driftctl/enumeration/parallel"
+	"github.com/snyk/driftctl/enumeration/remote"
+	"github.com/snyk/driftctl/enumeration/remote/common"
+	"github.com/snyk/driftctl/enumeration/resource"
+	"github.com/snyk/driftctl/enumeration/terraform"
+)
+
+type CloudEnumerator struct {
+	alerter              *sliceAlerter
+	progress             enumeration.ProgressCounter
+	remoteLibrary        *common.RemoteLibrary
+	providerLibrary      *terraform.ProviderLibrary
+	enumeratorRunner     *parallel.ParallelRunner
+	detailsFetcherRunner *parallel.ParallelRunner
+	to                   string
+}
+
+type cloudEnumeratorBuilder struct {
+	cloud           string
+	providerVersion string
+	configDirectory string
+}
+
+// WithCloud Choose which cloud to use for enumeration and refresh
+// TODO could be inferred with types listed
+func (b *cloudEnumeratorBuilder) WithCloud(cloud string) *cloudEnumeratorBuilder {
+	b.cloud = cloud
+	return b
+}
+
+// WithProviderVersion optionally choose the provider version used for refresh
+func (b *cloudEnumeratorBuilder) WithProviderVersion(providerVersion string) *cloudEnumeratorBuilder {
+	b.providerVersion = providerVersion
+	return b
+}
+
+// WithConfigDirectory optionally choose the directory used to download terraform provider used for refresh
+func (b *cloudEnumeratorBuilder) WithConfigDirectory(configDir string) *cloudEnumeratorBuilder {
+	b.configDirectory = configDir
+	return b
+}
+
+func (b *cloudEnumeratorBuilder) Build() (*CloudEnumerator, error) {
+	enumerator := &CloudEnumerator{
+		enumeratorRunner:     parallel.NewParallelRunner(context.TODO(), 10),
+		detailsFetcherRunner: parallel.NewParallelRunner(context.TODO(), 10),
+		providerLibrary:      terraform.NewProviderLibrary(),
+		remoteLibrary:        common.NewRemoteLibrary(),
+		alerter:              &sliceAlerter{},
+		progress:             &dummyCounter{},
+	}
+
+	if b.configDirectory == "" {
+		tempDir, err := os.MkdirTemp("", "enumerator")
+		if err != nil {
+			return nil, err
+		}
+		b.configDirectory = tempDir
+	}
+
+	err := enumerator.init(fmt.Sprintf("%s+tf", b.cloud), b.providerVersion, b.configDirectory)
+
+	return enumerator, err
+}
+
+func NewCloudEnumerator() *cloudEnumeratorBuilder {
+	return &cloudEnumeratorBuilder{}
+}
+
+func (e *CloudEnumerator) init(to, providerVersion, configDirectory string) error {
+	e.to = to
+
+	resFactory := terraform.NewTerraformResourceFactory()
+
+	err := remote.Activate(to, providerVersion, e.alerter, e.providerLibrary, e.remoteLibrary, e.progress, resFactory, configDirectory)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (e *CloudEnumerator) Enumerate(input *enumeration.EnumerateInput) (*enumeration.EnumerateOutput, error) {
+	types := map[string]struct{}{}
+	for _, resourceType := range input.ResourceTypes {
+		types[resourceType] = struct{}{}
+	}
+	filter := typeFilter{types: types}
+
+	for _, enumerator := range e.remoteLibrary.Enumerators() {
+		if filter.IsTypeIgnored(enumerator.SupportedType()) {
+			logrus.WithFields(logrus.Fields{
+				"type": enumerator.SupportedType(),
+			}).Debug("Ignored enumeration of resources since it is ignored in filter")
+			continue
+		}
+		enumerator := enumerator
+		e.enumeratorRunner.Run(func() (interface{}, error) {
+			resources, err := enumerator.Enumerate()
+			if err != nil {
+				err := remote.HandleResourceEnumerationError(err, e.alerter)
+				if err == nil {
+					return []*resource.Resource{}, nil
+				}
+				return nil, err
+			}
+			for _, res := range resources {
+				if res == nil {
+					continue
+				}
+				logrus.WithFields(logrus.Fields{
+					"id":   res.ResourceId(),
+					"type": res.ResourceType(),
+				}).Debug("Found cloud resource")
+			}
+			return resources, nil
+		})
+	}
+
+	results, err := e.retrieveRunnerResults(e.enumeratorRunner)
+	if err != nil {
+		return nil, err
+	}
+
+	mapRes := mapByType(results)
+
+	return &enumeration.EnumerateOutput{
+		Resources:   mapRes,
+		Timings:     nil,
+		Diagnostics: nil,
+	}, nil
+}
+
+func (e *CloudEnumerator) Refresh(input *enumeration.RefreshInput) (*enumeration.RefreshOutput, error) {
+	for _, resByType := range input.Resources {
+		for _, res := range resByType {
+			res := res
+			e.detailsFetcherRunner.Run(func() (interface{}, error) {
+				fetcher := e.remoteLibrary.GetDetailsFetcher(resource.ResourceType(res.ResourceType()))
+				if fetcher == nil {
+					return []*resource.Resource{res}, nil
+				}
+
+				resourceWithDetails, err := fetcher.ReadDetails(res)
+				if err != nil {
+					if err := remote.HandleResourceDetailsFetchingError(err, e.alerter); err != nil {
+						return nil, err
+					}
+					return []*resource.Resource{}, nil
+				}
+				return []*resource.Resource{resourceWithDetails}, nil
+			})
+		}
+	}
+
+	results, err := e.retrieveRunnerResults(e.detailsFetcherRunner)
+	if err != nil {
+		return nil, err
+	}
+
+	mapRes := mapByType(results)
+
+	return &enumeration.RefreshOutput{
+		Resources:   mapRes,
+		Diagnostics: nil,
+	}, nil
+}
+
+func (e *CloudEnumerator) GetSchema() (*enumeration.GetSchemasOutput, error) {
+	panic("GetSchema is not implemented..")
+}
+
+func (e *CloudEnumerator) retrieveRunnerResults(runner *parallel.ParallelRunner) ([]*resource.Resource, error) {
+	results := make([]*resource.Resource, 0)
+loop:
+	for {
+		select {
+		case resources, ok := <-runner.Read():
+			if !ok || resources == nil {
+				break loop
+			}
+
+			for _, res := range resources.([]*resource.Resource) {
+				if res != nil {
+					results = append(results, res)
+				}
+			}
+		case <-runner.DoneChan():
+			break loop
+		}
+	}
+	return results, runner.Err()
+}
+
+func (e *CloudEnumerator) List(typ string) ([]*resource.Resource, error) {
+	enumInput := &enumeration.EnumerateInput{ResourceTypes: []string{typ}}
+	enumerate, err := e.Enumerate(enumInput)
+	if err != nil {
+		return nil, err
+	}
+
+	refreshInput := &enumeration.RefreshInput{Resources: enumerate.Resources}
+	refresh, err := e.Refresh(refreshInput)
+	if err != nil {
+		return nil, err
+	}
+	return refresh.Resources[typ], nil
+}
+
+type sliceAlerter struct {
+	lock   sync.Mutex
+	alerts alerter.Alerts
+}
+
+func (d *sliceAlerter) Alerts() alerter.Alerts {
+	return d.alerts
+}
+
+func (d *sliceAlerter) SendAlert(key string, alert alerter.Alert) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+	d.alerts[key] = append(d.alerts[key], alert)
+}
+
+type typeFilter struct {
+	types map[string]struct{}
+}
+
+func (u *typeFilter) IsTypeIgnored(ty resource.ResourceType) bool {
+	_, ok := u.types[ty.String()]
+	return !ok
+}
+
+func (u *typeFilter) IsResourceIgnored(res *resource.Resource) bool {
+	_, ok := u.types[res.Type]
+	return !ok
+}
+
+func (u *typeFilter) IsFieldIgnored(res *resource.Resource, path []string) bool {
+	return false
+}
+
+type dummyCounter struct {
+}
+
+func (d *dummyCounter) Inc() {
+}
+
+func mapByType(results []*resource.Resource) map[string][]*resource.Resource {
+	mapRes := map[string][]*resource.Resource{}
+	for _, result := range results {
+		mapRes[result.Type] = append(mapRes[result.Type], result)
+	}
+	return mapRes
+}

enumeration/refresh.go

@@ -0,0 +1,25 @@
+package enumeration
+
+import (
+	"github.com/hashicorp/terraform/terraform"
+	"github.com/snyk/driftctl/enumeration/resource"
+)
+
+type RefreshInput struct {
+	// Resources to refresh
+	Resources map[string][]*resource.Resource
+}
+
+type RefreshOutput struct {
+	Resources   map[string][]*resource.Resource
+	Diagnostics Diagnostics
+}
+
+type GetSchemasOutput struct {
+	Schema *terraform.ProviderSchema
+}
+
+type Refresher interface {
+	Refresh(input *RefreshInput) (*RefreshOutput, error)
+	GetSchema() (*GetSchemasOutput, error)
+}

enumeration/remote/aws/init.go

@@ -17,7 +17,7 @@ import (
  * Required to use Scanner
  */
 
-func Init(version string, alerter *alerter.Alerter, providerLibrary *terraform.ProviderLibrary, remoteLibrary *common.RemoteLibrary, progress enumeration.ProgressCounter, factory resource.ResourceFactory, configDir string) error {
+func Init(version string, alerter alerter.AlerterInterface, providerLibrary *terraform.ProviderLibrary, remoteLibrary *common.RemoteLibrary, progress enumeration.ProgressCounter, factory resource.ResourceFactory, configDir string) error {
 
 	provider, err := NewAWSTerraformProvider(version, progress, configDir)
 	if err != nil {

enumeration/remote/azurerm/init.go

@@ -13,7 +13,7 @@ import (
 	"github.com/snyk/driftctl/enumeration/terraform"
 )
 
-func Init(version string, alerter *alerter.Alerter, providerLibrary *terraform.ProviderLibrary, remoteLibrary *common.RemoteLibrary, progress enumeration.ProgressCounter, factory resource.ResourceFactory, configDir string) error {
+func Init(version string, alerter alerter.AlerterInterface, providerLibrary *terraform.ProviderLibrary, remoteLibrary *common.RemoteLibrary, progress enumeration.ProgressCounter, factory resource.ResourceFactory, configDir string) error {
 
 	provider, err := NewAzureTerraformProvider(version, progress, configDir)
 	if err != nil {

enumeration/remote/github/init.go

@@ -15,7 +15,7 @@ import (
  * Required to use Scanner
  */
 
-func Init(version string, alerter *alerter.Alerter, providerLibrary *terraform.ProviderLibrary, remoteLibrary *common.RemoteLibrary, progress enumeration.ProgressCounter, factory resource.ResourceFactory, configDir string) error {
+func Init(version string, alerter alerter.AlerterInterface, providerLibrary *terraform.ProviderLibrary, remoteLibrary *common.RemoteLibrary, progress enumeration.ProgressCounter, factory resource.ResourceFactory, configDir string) error {
 
 	provider, err := NewGithubTerraformProvider(version, progress, configDir)
 	if err != nil {

enumeration/remote/google/init.go

@@ -18,7 +18,7 @@ import (
 	"google.golang.org/api/cloudresourcemanager/v1"
 )
 
-func Init(version string, alerter *alerter.Alerter, providerLibrary *terraform.ProviderLibrary, remoteLibrary *common.RemoteLibrary, progress enumeration.ProgressCounter, factory resource.ResourceFactory, configDir string) error {
+func Init(version string, alerter alerter.AlerterInterface, providerLibrary *terraform.ProviderLibrary, remoteLibrary *common.RemoteLibrary, progress enumeration.ProgressCounter, factory resource.ResourceFactory, configDir string) error {
 
 	provider, err := NewGCPTerraformProvider(version, progress, configDir)
 	if err != nil {

enumeration/remote/remote.go

@@ -29,7 +29,7 @@ func IsSupported(remote string) bool {
 	return false
 }
 
-func Activate(remote, version string, alerter *alerter.Alerter, providerLibrary *terraform.ProviderLibrary, remoteLibrary *common.RemoteLibrary, progress enumeration.ProgressCounter, factory resource.ResourceFactory, configDir string) error {
+func Activate(remote, version string, alerter alerter.AlerterInterface, providerLibrary *terraform.ProviderLibrary, remoteLibrary *common.RemoteLibrary, progress enumeration.ProgressCounter, factory resource.ResourceFactory, configDir string) error {
 	switch remote {
 	case common.RemoteAWSTerraform:
 		return aws.Init(version, alerter, providerLibrary, remoteLibrary, progress, factory, configDir)