chore: adds cloudformation for publish (#2586)

Adds CloudFormation partial file for publish.
Adds CloudFormation for publish environment variable.
Adds publish to various service/job CloudFormation files.
Adds publish yml file to workload.

Addresses #2550

By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.

Author: xar-tol

internal/pkg/addon/storage_test.go

@@ -110,7 +110,7 @@ func TestS3_MarshalBinary(t *testing.T) {
 
 func TestRDS_MarshalBinary(t *testing.T) {
 	testCases := map[string]struct {
-		engine       string
+		engine           string
 		mockDependencies func(ctrl *gomock.Controller, r *RDS)
 
 		wantedBinary []byte

internal/pkg/aws/s3/s3.go

@@ -23,7 +23,7 @@ import (
 
 const (
 	artifactDirName = "manual"
-	notFound = "NotFound"
+	notFound        = "NotFound"
 )
 
 type s3ManagerAPI interface {
@@ -33,7 +33,7 @@ type s3ManagerAPI interface {
 type s3API interface {
 	ListObjectVersions(input *s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error)
 	DeleteObjects(input *s3.DeleteObjectsInput) (*s3.DeleteObjectsOutput, error)
-	HeadBucket (input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
+	HeadBucket(input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error)
 }
 
 // NamedBinary is a named binary to be uploaded.
@@ -110,8 +110,8 @@ func (s *S3) EmptyBucket(bucket string) error {
 	var err error
 
 	// Bucket is exists check to make sure the bucket exists before proceeding in emptying it
-	isExists, err:= s.isBucketExists(bucket)
-	if err!= nil {
+	isExists, err := s.isBucketExists(bucket)
+	if err != nil {
 		return fmt.Errorf("unable to determine the existance of bucket %s: %w", bucket, err)
 	}
 

internal/pkg/cli/task_delete_test.go

@@ -57,7 +57,7 @@ func TestDeleteTaskOpts_Validate(t *testing.T) {
 		"with all flags": {
 			inAppName: "phonetool",
 			inEnvName: "test",
-			inName: "oneoff",
+			inName:    "oneoff",
 			setupMocks: func(m validateMocks) {
 				m.store.EXPECT().GetApplication("phonetool").Return(&config.Application{Name: "phonetool"}, nil)
 				m.store.EXPECT().GetEnvironment("phonetool", "test").Return(&config.Environment{Name: "test", App: "phonetool"}, nil)
@@ -69,8 +69,8 @@ func TestDeleteTaskOpts_Validate(t *testing.T) {
 		"task does not exist": {
 			inAppName: "phonetool",
 			inEnvName: "test",
-			inName: "oneoff",
-			want: errors.New("get task: some error"),
+			inName:    "oneoff",
+			want:      errors.New("get task: some error"),
 			setupMocks: func(m validateMocks) {
 				m.store.EXPECT().GetApplication("phonetool").Return(&config.Application{Name: "phonetool"}, nil)
 				m.store.EXPECT().GetEnvironment("phonetool", "test").Return(&config.Environment{Name: "test", App: "phonetool"}, nil)
@@ -81,10 +81,10 @@ func TestDeleteTaskOpts_Validate(t *testing.T) {
 		"with default cluster flag set": {
 			inDefaultCluster: true,
 			inName:           "oneoff",
-			setupMocks:       func(m validateMocks) {
+			setupMocks: func(m validateMocks) {
 				m.cfn.EXPECT().GetTaskStack("oneoff")
 			},
-			want:             nil,
+			want: nil,
 		},
 		"with default cluster and env flag": {
 			inDefaultCluster: true,
@@ -112,7 +112,7 @@ func TestDeleteTaskOpts_Validate(t *testing.T) {
 
 			mocks := validateMocks{
 				store: mockstore,
-				cfn: mocktaskStackManager,
+				cfn:   mocktaskStackManager,
 			}
 
 			tc.setupMocks(mocks)

internal/pkg/deploy/cloudformation/stack/backend_svc.go

@@ -118,26 +118,26 @@ func (s *BackendService) Template() (string, error) {
 		return "", err
 	}
 	content, err := s.parser.ParseBackendService(template.WorkloadOpts{
-		Variables:            s.manifest.BackendServiceConfig.Variables,
-		Secrets:              s.manifest.BackendServiceConfig.Secrets,
-		NestedStack:          outputs,
-		Sidecars:             sidecars,
-		Autoscaling:          autoscaling,
-		CapacityProviders:    capacityProviders,
-		DesiredCountOnSpot:   desiredCountOnSpot,
-		ExecuteCommand:       convertExecuteCommand(&s.manifest.ExecuteCommand),
-		WorkloadType:         manifest.BackendServiceType,
-		HealthCheck:          s.manifest.BackendServiceConfig.ImageConfig.HealthCheckOpts(),
-		LogConfig:            convertLogging(s.manifest.Logging),
-		DockerLabels:         s.manifest.ImageConfig.DockerLabels,
-		DesiredCountLambda:   desiredCountLambda.String(),
-		EnvControllerLambda:  envControllerLambda.String(),
-		Storage:              storage,
-		Network:              convertNetworkConfig(s.manifest.Network),
-		EntryPoint:           entrypoint,
-		Command:              command,
-		DependsOn:            dependencies,
-		CredentialsParameter: aws.StringValue(s.manifest.ImageConfig.Credentials),
+		Variables:                s.manifest.BackendServiceConfig.Variables,
+		Secrets:                  s.manifest.BackendServiceConfig.Secrets,
+		NestedStack:              outputs,
+		Sidecars:                 sidecars,
+		Autoscaling:              autoscaling,
+		CapacityProviders:        capacityProviders,
+		DesiredCountOnSpot:       desiredCountOnSpot,
+		ExecuteCommand:           convertExecuteCommand(&s.manifest.ExecuteCommand),
+		WorkloadType:             manifest.BackendServiceType,
+		HealthCheck:              s.manifest.BackendServiceConfig.ImageConfig.HealthCheckOpts(),
+		LogConfig:                convertLogging(s.manifest.Logging),
+		DockerLabels:             s.manifest.ImageConfig.DockerLabels,
+		DesiredCountLambda:       desiredCountLambda.String(),
+		EnvControllerLambda:      envControllerLambda.String(),
+		Storage:                  storage,
+		Network:                  convertNetworkConfig(s.manifest.Network),
+		EntryPoint:               entrypoint,
+		Command:                  command,
+		DependsOn:                dependencies,
+		CredentialsParameter:     aws.StringValue(s.manifest.ImageConfig.Credentials),
 		ServiceDiscoveryEndpoint: s.rc.ServiceDiscoveryEndpoint,
 	})
 	if err != nil {

internal/pkg/deploy/cloudformation/stack/scheduled_job.go

@@ -163,21 +163,21 @@ func (j *ScheduledJob) Template() (string, error) {
 	}
 
 	content, err := j.parser.ParseScheduledJob(template.WorkloadOpts{
-		Variables:            j.manifest.Variables,
-		Secrets:              j.manifest.Secrets,
-		NestedStack:          outputs,
-		Sidecars:             sidecars,
-		ScheduleExpression:   schedule,
-		StateMachine:         stateMachine,
-		HealthCheck:          j.manifest.ImageConfig.HealthCheckOpts(),
-		LogConfig:            convertLogging(j.manifest.Logging),
-		DockerLabels:         j.manifest.ImageConfig.DockerLabels,
-		Storage:              storage,
-		Network:              convertNetworkConfig(j.manifest.Network),
-		EntryPoint:           entrypoint,
-		Command:              command,
-		DependsOn:            dependencies,
-		CredentialsParameter: aws.StringValue(j.manifest.ImageConfig.Credentials),
+		Variables:                j.manifest.Variables,
+		Secrets:                  j.manifest.Secrets,
+		NestedStack:              outputs,
+		Sidecars:                 sidecars,
+		ScheduleExpression:       schedule,
+		StateMachine:             stateMachine,
+		HealthCheck:              j.manifest.ImageConfig.HealthCheckOpts(),
+		LogConfig:                convertLogging(j.manifest.Logging),
+		DockerLabels:             j.manifest.ImageConfig.DockerLabels,
+		Storage:                  storage,
+		Network:                  convertNetworkConfig(j.manifest.Network),
+		EntryPoint:               entrypoint,
+		Command:                  command,
+		DependsOn:                dependencies,
+		CredentialsParameter:     aws.StringValue(j.manifest.ImageConfig.Credentials),
 		ServiceDiscoveryEndpoint: j.rc.ServiceDiscoveryEndpoint,
 
 		EnvControllerLambda: envControllerLambda.String(),

internal/pkg/deploy/cloudformation/stack/transformers.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/endpoints"
 	"github.com/aws/copilot-cli/internal/pkg/manifest"
 	"github.com/aws/copilot-cli/internal/pkg/template"
 )
@@ -586,14 +587,18 @@ func convertCommand(command *manifest.CommandOverride) ([]string, error) {
 	return out, nil
 }
 
-func convertPublish(p *manifest.PublishConfig) (*template.PublishOpts, error) {
+func convertPublish(p *manifest.PublishConfig, accountID, region, app, env, svc string) (*template.PublishOpts, error) {
 	if p == nil || len(p.Topics) == 0 {
 		return nil, nil
 	}
+	partition, ok := endpoints.PartitionForRegion(endpoints.DefaultPartitions(), region)
+	if !ok {
+		return nil, fmt.Errorf("find the partition for region %s", region)
+	}
 	publishers := template.PublishOpts{}
 	// convert the topics to template Topics
 	for _, topic := range p.Topics {
-		t, err := convertTopic(topic)
+		t, err := convertTopic(topic, accountID, partition.ID(), region, app, env, svc)
 		if err != nil {
 			return nil, err
 		}
@@ -604,19 +609,23 @@ func convertPublish(p *manifest.PublishConfig) (*template.PublishOpts, error) {
 	return &publishers, nil
 }
 
-func convertTopic(t manifest.Topic) (*template.Topics, error) {
-	err := validatePubSubName(t.Name)
-	if err != nil {
+func convertTopic(t manifest.Topic, accountID, partition, region, app, env, svc string) (*template.Topic, error) {
+	// topic should have a valid name and valid service worker names
+	if err := validatePubSubName(t.Name); err != nil {
 		return nil, err
 	}
-
-	workerErr := validateWorkerNames(t.AllowedWorkers)
-	if workerErr != nil {
-		return nil, workerErr
+	if err := validateWorkerNames(t.AllowedWorkers); err != nil {
+		return nil, err
 	}
 
-	return &template.Topics{
+	return &template.Topic{
 		Name:           t.Name,
 		AllowedWorkers: t.AllowedWorkers,
+		AccountID:      accountID,
+		Partition:      partition,
+		Region:         region,
+		App:            app,
+		Env:            env,
+		Svc:            svc,
 	}, nil
 }

internal/pkg/deploy/cloudformation/stack/transformers_test.go

@@ -1459,6 +1459,12 @@ func Test_convertImageDependsOn(t *testing.T) {
 }
 
 func Test_convertPublish(t *testing.T) {
+	accountId := "123456789123"
+	partition := "aws"
+	region := "us-west-2"
+	app := "testapp"
+	env := "testenv"
+	svc := "hello"
 	testCases := map[string]struct {
 		inPublish *manifest.PublishConfig
 
@@ -1486,9 +1492,15 @@ func Test_convertPublish(t *testing.T) {
 				},
 			},
 			wanted: &template.PublishOpts{
-				Topics: []*template.Topics{
+				Topics: []*template.Topic{
 					{
-						Name: aws.String("topic1"),
+						Name:      aws.String("topic1"),
+						AccountID: accountId,
+						Partition: partition,
+						Region:    region,
+						App:       app,
+						Env:       env,
+						Svc:       svc,
 					},
 				},
 			},
@@ -1503,10 +1515,16 @@ func Test_convertPublish(t *testing.T) {
 				},
 			},
 			wanted: &template.PublishOpts{
-				Topics: []*template.Topics{
+				Topics: []*template.Topic{
 					{
 						Name:           aws.String("topic1"),
 						AllowedWorkers: []string{"worker1"},
+						AccountID:      accountId,
+						Partition:      partition,
+						Region:         region,
+						App:            app,
+						Env:            env,
+						Svc:            svc,
 					},
 				},
 			},
@@ -1536,7 +1554,7 @@ func Test_convertPublish(t *testing.T) {
 	}
 	for name, tc := range testCases {
 		t.Run(name, func(t *testing.T) {
-			got, err := convertPublish(tc.inPublish)
+			got, err := convertPublish(tc.inPublish, accountId, region, app, env, svc)
 			if tc.wantedError != nil {
 				require.EqualError(t, err, tc.wantedError.Error())
 			} else {

internal/pkg/deploy/cloudformation/stack/validate.go

@@ -26,7 +26,7 @@ var (
 	errNoContainerPath          = errors.New("`path` cannot be empty")
 	errNoSourceVolume           = errors.New("`source_volume` cannot be empty")
 	errEmptyEFSConfig           = errors.New("bad EFS configuration: `efs` cannot be empty")
-	errMissingPublishTopicField = errors.New("topic `name` cannot be empty")
+	errMissingPublishTopicField = errors.New("field `publish.topics[].name` cannot be empty")
 )
 
 // Conditional errors.

internal/pkg/deploy/pipeline_test.go

@@ -152,8 +152,8 @@ func TestPipelineBuildFromManifest(t *testing.T) {
 	const defaultImage = "aws/codebuild/amazonlinux2-x86_64-standard:3.0"
 
 	testCases := map[string]struct {
-		mfBuild             *manifest.Build
-		expectedBuild       *Build
+		mfBuild       *manifest.Build
+		expectedBuild *Build
 	}{
 		"set default image if not be specified in manifest": {
 			mfBuild: nil,

internal/pkg/manifest/svc_test.go

@@ -80,7 +80,7 @@ environments:
 					LoadBalancedWebServiceConfig: LoadBalancedWebServiceConfig{
 						ImageConfig: ImageWithPortAndHealthcheck{
 							ImageWithPort: ImageWithPort{Image: Image{Build: BuildArgsOrString{},
-								Location: aws.String("foo/bar"),
+								Location:    aws.String("foo/bar"),
 								Credentials: aws.String("some arn"),
 							}, Port: aws.Uint16(80)},
 						},