Unit Testing Framework w/ Feature Group sample (#66)

Summary:
Added unit testing framework (and code coverage), with an example of a unit test in feature group.

Code Coverage: 1.8%

Files Added:

- test/mocks/aws-sdk-go/sagemaker/SageMakerAPI.go - Contains unit testing api mocks.
- pkg/resource/feature_group/manager_test_suite_test.go - Contains resource specific helper functions for unit testing, and the provideResourceManagerWithMockSDKAPI function for unit testing.
- pkg/resource/feature_group/testdata/feature_group/v1alpha1/fg_invalid_before_create.yaml - Input sample file for unit testing.
- pkg/resource/feature_group/testdata/feature_group/v1alpha1/fg_invalid_create_attempted.yaml - Output (expected) sample file for unit testing.
- pkg/resource/feature_group/testdata/test_suite.yaml - Contains unit testing scenarios (which input/output to test along with the functions (Create/ReadOne/Update/Delete) to test).
- pkg/testutil/test_suite_config.go - Defines structures related to the unit testing test suite.
- pkg/testutil/test_suite_runner.go - Contains main workflow functions for running unit tests.
- pkg/testutil/util.go - Contains helper functions used in test_suite_runner.go.
- test/scripts/install-mockery.sh - A script that installs the mockery CLI tool that is used to build Go mocks for our interfaces to use in unit testing.

Files Edited:

- Makefile - Defined unit testing, mock, and unit testing coverage related testing calls.

Author: BriannaRoskind

ATTRIBUTION.md

@@ -32,6 +32,7 @@ License version 2.0, we include the full text of the package's License below.
 * `github.com/spf13/pflag`
 * `github.com/stretchr/testify`
 * `golang.org/x/mod`
+* `go.uber.org/zap`
 * `k8s.io/api`
 * `k8s.io/apimachinery`
 * `k8s.io/client-go`

Makefile

@@ -3,6 +3,11 @@ SHELL := /bin/bash # Use bash syntax
 # Set up variables
 GO111MODULE=on
 
+AWS_SDK_GO_VERSION="$(shell echo $(shell go list -m -f '{{.Version}}' github.com/aws/aws-sdk-go))"
+AWS_SDK_GO_VERSIONED_PATH="$(shell echo github.com/aws/aws-sdk-go@$(AWS_SDK_GO_VERSION))"
+SAGEMAKER_API_PATH="$(shell echo $(shell go env GOPATH))/pkg/mod/$(AWS_SDK_GO_VERSIONED_PATH)/service/sagemaker/sagemakeriface"
+SERVICE_CONTROLLER_SRC_PATH="$(shell pwd)"
+
 # Build ldflags
 VERSION ?= "v0.0.0"
 GITCOMMIT=$(shell git rev-parse HEAD)
@@ -11,13 +16,32 @@ GO_LDFLAGS=-ldflags "-X main.version=$(VERSION) \
 			-X main.buildHash=$(GITCOMMIT) \
 			-X main.buildDate=$(BUILDDATE)"
 
-.PHONY: all test
+.PHONY: all test clean-mocks mocks
 
 all: test
 
 test: 				## Run code tests
 	go test -v ./...
 
+test-cover: | mocks				## Run code tests with resources coverage
+	go test -coverpkg=./pkg/resource/... -covermode=count -coverprofile=coverage.out ./...
+	go tool cover -func=coverage.out
+
+clean-mocks:	## Remove mocks directory
+	rm -r mocks
+
+install-mockery:
+	@test/scripts/install-mockery.sh
+
+mocks: install-mockery ## Build mocks
+	go get -d $(AWS_SDK_GO_VERSIONED_PATH)
+	@echo "building mocks for $(SAGEMAKER_API_PATH) ... "
+	@pushd $(SAGEMAKER_API_PATH) 1>/dev/null; \
+	$(SERVICE_CONTROLLER_SRC_PATH)/bin/mockery --all --dir=. --output=$(SERVICE_CONTROLLER_SRC_PATH)/mocks/aws-sdk-go/sagemaker/ ; \
+	popd 1>/dev/null;
+	@echo "ok."
+
+
 help:           	## Show this help.
 	@grep -F -h "##" $(MAKEFILE_LIST) | grep -F -v grep | sed -e 's/\\$$//' \
 		| awk -F'[:#]' '{print $$1 = sprintf("%-30s", $$1), $$4}'

go.mod

@@ -5,8 +5,12 @@ go 1.14
 require (
 	github.com/aws-controllers-k8s/runtime v0.6.0
 	github.com/aws/aws-sdk-go v1.38.11
+	github.com/ghodss/yaml v1.0.0
 	github.com/go-logr/logr v0.1.0
+	github.com/google/go-cmp v0.3.1
 	github.com/spf13/pflag v1.0.5
+	github.com/stretchr/testify v1.5.1
+	go.uber.org/zap v1.10.0
 	k8s.io/api v0.18.2
 	k8s.io/apimachinery v0.18.6
 	k8s.io/client-go v0.18.2

go.sum

@@ -71,6 +71,7 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=

pkg/resource/feature_group/manager_test_suite_test.go

@@ -0,0 +1,114 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package feature_group
+
+import (
+	"errors"
+	"fmt"
+	mocksvcsdkapi "github.com/aws-controllers-k8s/sagemaker-controller/test/mocks/aws-sdk-go/sagemaker"
+	"github.com/aws-controllers-k8s/sagemaker-controller/pkg/testutil"
+	acktypes "github.com/aws-controllers-k8s/runtime/pkg/types"
+	svcsdk "github.com/aws/aws-sdk-go/service/sagemaker"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"path/filepath"
+	"testing"
+	ctrlrtzap "sigs.k8s.io/controller-runtime/pkg/log/zap"
+	ackmetrics "github.com/aws-controllers-k8s/runtime/pkg/metrics"
+	"go.uber.org/zap/zapcore"
+)
+
+// provideResourceManagerWithMockSDKAPI accepts MockSageMakerAPI and returns pointer to resourceManager
+// the returned resourceManager is configured to use mockapi api.
+func provideResourceManagerWithMockSDKAPI(mockSageMakerAPI *mocksvcsdkapi.SageMakerAPI) *resourceManager {
+	zapOptions := ctrlrtzap.Options{
+		Development: true,
+		Level:       zapcore.InfoLevel,
+	}
+	fakeLogger := ctrlrtzap.New(ctrlrtzap.UseFlagOptions(&zapOptions))
+	return &resourceManager{
+		rr:           nil,
+		awsAccountID: "",
+		awsRegion:    "",
+		sess:         nil,
+		sdkapi:       mockSageMakerAPI,
+		log:          fakeLogger,
+		metrics:      ackmetrics.NewMetrics("sagemaker"),
+	}
+}
+
+// TestDeclarativeTestSuite runs the test suite for feature group
+func TestDeclarativeTestSuite(t *testing.T) {
+     	defer func() {
+     	   if r := recover(); r != nil {
+     	      fmt.Println(testutil.RecoverPanicString, r)
+	      t.Fail()
+     	   }
+        }()
+	var ts = testutil.TestSuite{}
+	testutil.LoadFromFixture(filepath.Join("testdata", "test_suite.yaml"), &ts)
+	var delegate = testRunnerDelegate{t: t}
+	var runner = testutil.TestSuiteRunner{TestSuite: &ts, Delegate: &delegate}
+	runner.RunTests()
+}
+
+// testRunnerDelegate implements testutil.TestRunnerDelegate
+type testRunnerDelegate struct {
+	t *testing.T
+}
+
+func (d *testRunnerDelegate) ResourceDescriptor() acktypes.AWSResourceDescriptor {
+	return &resourceDescriptor{}
+}
+
+func (d *testRunnerDelegate) ResourceManager(mocksdkapi *mocksvcsdkapi.SageMakerAPI) acktypes.AWSResourceManager {
+	return provideResourceManagerWithMockSDKAPI(mocksdkapi)
+}
+
+func (d *testRunnerDelegate) GoTestRunner() *testing.T {
+	return d.t
+}
+
+func (d *testRunnerDelegate) EmptyServiceAPIOutput(apiName string) (interface{}, error) {
+	if apiName == "" {
+		return nil, errors.New("no API name specified")
+	}
+	//TODO: use reflection, template to auto generate this block/method.
+	switch apiName {
+	case "CreateFeatureGroupWithContext":
+		var output svcsdk.CreateFeatureGroupOutput
+	    	return &output, nil
+	case "DeleteFeatureGroupWithContext":
+		var output svcsdk.DeleteFeatureGroupOutput
+		return &output, nil
+	case "DescribeFeatureGroupWithContext":
+		var output svcsdk.DescribeFeatureGroupOutput
+		return &output, nil
+	}
+	return nil, errors.New(fmt.Sprintf("no matching API name found for: %s", apiName))
+}
+
+func (d *testRunnerDelegate) Equal(a acktypes.AWSResource, b acktypes.AWSResource) bool {
+	ac := a.(*resource)
+	bc := b.(*resource)
+	opts := []cmp.Option{cmpopts.EquateEmpty()}
+
+	if cmp.Equal(ac.ko.Status, bc.ko.Status, opts...) {
+		return true
+	} else {
+		fmt.Printf("Difference (-expected +actual):\n\n")
+		fmt.Println(cmp.Diff(ac.ko.Status, bc.ko.Status, opts...))
+		return false
+	}
+}

pkg/resource/feature_group/testdata/feature_group/v1alpha1/fg_invalid_before_create.yaml

@@ -0,0 +1,17 @@
+apiVersion: sagemaker.services.k8s.aws/v1alpha1
+kind: FeatureGroup
+metadata:
+  name: unit-testing-feature-group
+spec:
+  eventTimeFeatureName: EventTime
+  featureDefinitions:
+  - featureName: TransactionID
+    featureType: Integral
+  - featureName: EventTime
+    featureType: Fractional
+  featureGroupName: !-intentionally-invalid-name
+  offlineStoreConfig:
+    s3StorageConfig:
+      s3URI: s3://source-data-bucket-592697580195-us-west-2/sagemaker/feature-group-data
+  recordIdentifierFeatureName: TransactionID
+  roleARN: arn:aws:iam::123456789012:role/ack-sagemaker-execution-role

pkg/resource/feature_group/testdata/feature_group/v1alpha1/fg_invalid_create_attempted.yaml

@@ -0,0 +1,25 @@
+apiVersion: sagemaker.services.k8s.aws/v1alpha1
+kind: FeatureGroup
+metadata:
+  creationTimestamp: null
+  name: unit-testing-feature-group
+spec:
+  eventTimeFeatureName: EventTime
+  featureDefinitions:
+  - featureName: TransactionID
+    featureType: Integral
+  - featureName: EventTime
+    featureType: Fractional
+  featureGroupName: ""
+  offlineStoreConfig:
+    s3StorageConfig:
+      s3URI: s3://source-data-bucket-592697580195-us-west-2/sagemaker/feature-group-data
+  recordIdentifierFeatureName: TransactionID
+  roleARN: arn:aws:iam::123456789012:role/ack-sagemaker-execution-role
+status:
+  ackResourceMetadata:
+    ownerAccountID: ""
+  conditions:
+  - message: The feature group name must start with an alphanumeric character.
+    status: "True"
+    type: ACK.Terminal

pkg/resource/feature_group/testdata/test_suite.yaml

@@ -0,0 +1,17 @@
+tests:
+  - name: "Feature group create test."
+    description: "Feature group CRD tests"
+    scenarios:
+     - name: "Create=InvalidInput"
+       description: "Given one of the parameters is invalid, ko.Status shows a terminal condition"
+       given:
+         desired_state: "feature_group/v1alpha1/fg_invalid_before_create.yaml"
+         svc_api:
+           - operation: CreateFeatureGroupWithContext
+             error:
+               code: InvalidParameterValue
+               message: "The feature group name must start with an alphanumeric character."
+       invoke: Create
+       expect:
+         latest_state: "feature_group/v1alpha1/fg_invalid_create_attempted.yaml"
+         error: resource is in terminal condition

pkg/testutil/test_suite_config.go

@@ -0,0 +1,71 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package testutil
+
+// TestSuite represents instructions to run unit tests using test fixtures and mock service apis
+type TestSuite struct {
+	Tests []TestConfig `json:"tests"`
+}
+
+// TestConfig represents declarative unit test
+type TestConfig struct {
+	Name        string         `json:"name"`
+	Description string         `json:"description"`
+	Scenarios   []TestScenario `json:"scenarios"`
+}
+
+// TestScenario represents declarative test scenario details
+type TestScenario struct {
+	Name        string `json:"name"`
+	Description string `json:"description"`
+	// Fixture lets you specify test scenario given input fixtures
+	Fixture Fixture `json:"given"`
+	// UnitUnderTest lets you specify the unit to test
+	// For example resource manager API: ReadOne, Create, Update, Delete
+	UnitUnderTest string `json:"invoke"`
+	// Expect lets you specify test scenario expected outcome fixtures
+	Expect Expect `json:"expect"`
+}
+
+// Fixture represents test scenario fixture to load from file paths
+type Fixture struct {
+	// DesiredState lets you specify fixture path to load the desired state fixture
+	DesiredState string `json:"desired_state"`
+	// LatestState lets you specify fixture path to load the current state fixture
+	LatestState string `json:"latest_state"`
+	// ServiceAPIs lets you specify fixture path to mock service sdk api response
+	ServiceAPIs []ServiceAPI `json:"svc_api"`
+}
+
+// ServiceAPI represents details about the the service sdk api and fixture path to mock its response
+type ServiceAPI struct {
+	Operation       string           `json:"operation"`
+	Output          string           `json:"output_fixture"`
+	ServiceAPIError *ServiceAPIError `json:"error,omitempty"`
+}
+
+// ServiceAPIError contains the specification for the error of the mock API response
+type ServiceAPIError struct {
+	// Code here is usually the type of fault/error, not the HTTP status code
+	Code    string `json:"code"`
+	Message string `json:"message"`
+}
+
+// Expect represents test scenario expected outcome fixture to load from file path
+type Expect struct {
+	LatestState string `json:"latest_state"`
+	// Error is a string matching the message of the expected error returned from the ResourceManager operation.
+	// Possible errors can be found in runtime/pkg/errors/error.go
+	Error string `json:"error"`
+}