Add service metadata configuration file (#126)

Description of changes:
Creates a new specification for a `metadata.yaml` file that resides in the root of the controller repository. The purpose of this file is to provide human-friendly display information about the service (long name, acronym, links) and to provide a source of truth for API versions.

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

Author: RedbackThomson

cmd/ack-generate/command/apis.go

@@ -23,6 +23,7 @@ import (
 	"github.com/spf13/cobra"
 
 	ackgenerate "github.com/aws-controllers-k8s/code-generator/pkg/generate/ack"
+	ackmetadata "github.com/aws-controllers-k8s/code-generator/pkg/metadata"
 	ackmodel "github.com/aws-controllers-k8s/code-generator/pkg/model"
 	"github.com/aws-controllers-k8s/code-generator/pkg/util"
 )
@@ -59,10 +60,10 @@ func init() {
 // saveGeneratedMetadata saves the parameters used to generate APIs and checksum
 // of the generated code.
 func saveGeneratedMetadata(cmd *cobra.Command, args []string) error {
-	err := ackgenerate.CreateGenerationMetadata(
+	err := ackmetadata.CreateGenerationMetadata(
 		optGenVersion,
 		filepath.Join(optOutputPath, "apis"),
-		ackgenerate.UpdateReasonAPIGeneration,
+		ackmetadata.UpdateReasonAPIGeneration,
 		optAWSSDKGoVersion,
 		optGeneratorConfigPath,
 	)

cmd/ack-generate/command/root.go

@@ -41,6 +41,7 @@ var (
 	optDryRun              bool
 	sdkDir                 string
 	optGeneratorConfigPath string
+	optMetadataConfigPath  string
 	optOutputPath          string
 )
 
@@ -111,6 +112,9 @@ func init() {
 	rootCmd.PersistentFlags().StringVar(
 		&optGeneratorConfigPath, "generator-config-path", "", "Path to file containing instructions for code generation to use",
 	)
+	rootCmd.PersistentFlags().StringVar(
+		&optMetadataConfigPath, "metadata-config-path", "", "Path to file containing service metadata to use",
+	)
 	rootCmd.PersistentFlags().StringVarP(
 		&optOutputPath, "output", "o", "", "Path to directory to output generated files.",
 	)

go.sum

@@ -67,8 +67,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
-github.com/aws-controllers-k8s/runtime v0.4.0 h1:qsCp1OKXoSMtjS+6zlH65PN3/UkVJNbI/ybwYjPmhrY=
-github.com/aws-controllers-k8s/runtime v0.4.0/go.mod h1:xA2F18PJerBHaqrS4de1lpP7skeSMeStkmh+3x5sWvw=
+github.com/aws-controllers-k8s/runtime v0.5.0 h1:lV2QxuEHHKS+nbJ7lYgPjwVBY6Aqw+RHJF8eDZZeswM=
+github.com/aws-controllers-k8s/runtime v0.5.0/go.mod h1:xA2F18PJerBHaqrS4de1lpP7skeSMeStkmh+3x5sWvw=
 github.com/aws/aws-sdk-go v1.37.4 h1:tWxrpMK/oRSXVnjUzhGeCWLR00fW0WF4V4sycYPPrJ8=
 github.com/aws/aws-sdk-go v1.37.4/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -475,6 +475,7 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=
@@ -584,6 +585,7 @@ golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPI
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0 h1:sfUMP1Gu8qASkorDVjnMuvgJzwFbTZSeXFiGBYAVdl4=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
@@ -622,6 +624,7 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@@ -688,6 +691,7 @@ golang.org/x/sys v0.0.0-20201112073958-5cba982894dd h1:5CtCZbICpIOFdgO940moixOPj
 golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=

pkg/model/multiversion/api_info.go renamed to pkg/metadata/api_info.go

@@ -11,17 +11,12 @@
 // express or implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package multiversion
-
-// TODO(a-hilaly) move this file outside of pkg/model/multiversion. Idealy we
-// Should be able to access APIStatus and APIInfo to prevent regenerating removed or
-// deprecated APIs.
+package metadata
 
 type APIStatus string
 
 const (
-	APIStatusUnknown    APIStatus = "unknown"
-	APIStatusAvailable            = "available"
+	APIStatusAvailable  APIStatus = "available"
 	APIStatusRemoved              = "removed"
 	APIStatusDeprecated           = "deprecated"
 )

pkg/generate/ack/output.go renamed to pkg/metadata/generation_metadata.go

@@ -11,7 +11,7 @@
 // express or implied. See the License for the specific language governing
 // permissions and limitations under the License.
 
-package ack
+package metadata
 
 import (
 	"crypto/sha1"

pkg/metadata/service_metadata.go

@@ -0,0 +1,120 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package metadata
+
+import (
+	"errors"
+	"io/ioutil"
+
+	"github.com/ghodss/yaml"
+)
+
+var (
+	ErrNoServiceMetadataFile = errors.New("expected metadata file path, none provided")
+	ErrNoAvailableVersions   = errors.New("service metadata contains no available versions")
+)
+
+// ServiceMetadata consists of information about the service and relative links as well
+// as a list of supported/deprecated versions
+type ServiceMetadata struct {
+	Service ServiceDetails `json:"service"`
+	// A list of all generated API versions of the service
+	APIVersions []ServiceVersion `json:"api_versions"`
+}
+
+// ServiceDetails contains string identifiers and relevant links for the
+// service
+type ServiceDetails struct {
+	// The full display name for the service. eg. Amazon Elastic Kubernetes
+	// Service
+	FullName string `json:"full_name"`
+	// The short name (abbreviation) for the service. eg. S3
+	ShortName string `json:"short_name"`
+	// The URL of the service's homepage
+	Link string `json:"link"`
+	// The URL of the service's main documentation/user guide
+	Documentation string `json:"documentation"`
+}
+
+// ServiceVersion describes the status of all existing version of the controller
+type ServiceVersion struct {
+	APIVersion string    `json:"api_version"`
+	Status     APIStatus `json:"status"`
+}
+
+// GetLatestAPIVersion returns the latest available API version.
+// This should always be used by the generators as the source of truth
+// for what version to build.
+func (m *ServiceMetadata) GetLatestAPIVersion() (string, error) {
+	availableVersions := m.GetAvailableAPIVersions()
+
+	if len(availableVersions) == 0 {
+		return "", ErrNoAvailableVersions
+	}
+
+	return availableVersions[len(availableVersions)-1], nil
+}
+
+// GetDeprecatedAPIVersions returns all API versions that have been marked as
+// deprecated
+func (m *ServiceMetadata) GetDeprecatedAPIVersions() []string {
+	return m.getVersionsByStatus(APIStatusDeprecated)
+}
+
+// GetRemovedAPIVersions returns all API versions that have been marked as
+// removed
+func (m *ServiceMetadata) GetRemovedAPIVersions() []string {
+	return m.getVersionsByStatus(APIStatusRemoved)
+}
+
+// GetAvailableAPIVersions returns all API versions that have been marked as
+// available
+func (m *ServiceMetadata) GetAvailableAPIVersions() []string {
+	return m.getVersionsByStatus(APIStatusAvailable)
+}
+
+// getVersionsByStatus filters all of the versions by their respective statuses
+// and returns their API versions
+func (m *ServiceMetadata) getVersionsByStatus(status APIStatus) []string {
+	if len(m.APIVersions) == 0 {
+		return []string{}
+	}
+
+	versions := []string{}
+	for _, v := range m.APIVersions {
+		if v.Status == status {
+			versions = append(versions, v.APIVersion)
+		}
+	}
+	return versions
+}
+
+// NewServiceMetadata returns a new Metadata object given a supplied
+// path to a metadata file
+func NewServiceMetadata(
+	metadataPath string,
+) (ServiceMetadata, error) {
+	if metadataPath == "" {
+		return ServiceMetadata{}, ErrNoServiceMetadataFile
+	}
+	content, err := ioutil.ReadFile(metadataPath)
+	if err != nil {
+		return ServiceMetadata{}, err
+	}
+	gc := ServiceMetadata{}
+	if err = yaml.Unmarshal(content, &gc); err != nil {
+		return ServiceMetadata{}, err
+	}
+	return gc, nil
+}

pkg/model/multiversion/manager.go

@@ -21,6 +21,7 @@ import (
 	"gopkg.in/src-d/go-git.v4"
 
 	ackgenconfig "github.com/aws-controllers-k8s/code-generator/pkg/generate/config"
+	ackmetadata "github.com/aws-controllers-k8s/code-generator/pkg/metadata"
 	ackmodel "github.com/aws-controllers-k8s/code-generator/pkg/model"
 	"github.com/aws-controllers-k8s/code-generator/pkg/util"
 )
@@ -35,31 +36,36 @@ var (
 // of each non-deprecated version with their correspending ackmodel.Model
 // and APIInfos.
 type APIVersionManager struct {
-	gitRepo *git.Repository
+	gitRepo  *git.Repository
+	metadata ackmetadata.ServiceMetadata
 
 	hubVersion    string
 	spokeVersions []string
 
-	deprecatedVersions []string
-	removedVersions    []string
-
-	apiInfos map[string]APIInfo
+	apiInfos map[string]ackmetadata.APIInfo
 	models   map[string]*ackmodel.Model
 }
 
 // NewAPIVersionManager initialises and returns a new APIVersionManager.
 func NewAPIVersionManager(
 	sdkCacheDir string,
+	metadataPath string,
 	serviceAlias string,
 	hubVersion string,
-	apisInfo map[string]APIInfo,
+	apisInfo map[string]ackmetadata.APIInfo,
 	defaultConfig ackgenconfig.Config,
 ) (*APIVersionManager, error) {
 	if len(apisInfo) == 0 {
 		return nil, fmt.Errorf("empty apisInfo")
 	}
 
-	spokeVersions := make([]string, 0, len(apisInfo)-1)
+	metadata, err := ackmetadata.NewServiceMetadata(metadataPath)
+	if err != nil {
+		return nil, err
+	}
+
+	spokeVersions := []string{}
+
 	gitRepo, err := util.LoadRepository(sdkCacheDir)
 	if err != nil {
 		return nil, fmt.Errorf("cannot read sdk git repository: %v", err)
@@ -68,11 +74,19 @@ func NewAPIVersionManager(
 	SDKAPIHelper := ackmodel.NewSDKHelper(sdkCacheDir)
 
 	// create model for each non-deprecated api version
-	models := make(map[string]*ackmodel.Model, len(apisInfo))
-	for apiVersion, apiInfo := range apisInfo {
-		// TODO(a-hilaly) filter deprecated and removed api versions
-		if apiVersion != hubVersion {
-			spokeVersions = append(spokeVersions, apiVersion)
+	models := map[string]*ackmodel.Model{}
+	for _, version := range metadata.APIVersions {
+		if version.Status == ackmetadata.APIStatusDeprecated || version.Status == ackmetadata.APIStatusRemoved {
+			continue
+		}
+
+		if version.APIVersion != hubVersion {
+			spokeVersions = append(spokeVersions, version.APIVersion)
+		}
+
+		apiInfo, ok := apisInfo[version.APIVersion]
+		if !ok {
+			return nil, fmt.Errorf("could not find API info for API version %s", version.APIVersion)
 		}
 
 		err = SDKAPIHelper.WithSDKVersion(apiInfo.AWSSDKVersion)
@@ -87,25 +101,26 @@ func NewAPIVersionManager(
 
 		i, err := ackmodel.New(
 			SDKAPI,
-			apiVersion,
+			version.APIVersion,
 			apiInfo.GeneratorConfigPath,
 			defaultConfig,
 		)
 		if err != nil {
-			return nil, fmt.Errorf("cannot create model for API version %s: %v", apiVersion, err)
+			return nil, fmt.Errorf("cannot create model for API version %s: %v", version.APIVersion, err)
 		}
-		models[apiVersion] = i
+		models[version.APIVersion] = i
 	}
 
 	sort.Strings(spokeVersions)
 	model := &APIVersionManager{
 		gitRepo:       gitRepo,
+		metadata:      metadata,
 		hubVersion:    hubVersion,
 		spokeVersions: spokeVersions,
 		apiInfos:      apisInfo,
 		models:        models,
 	}
-	// TODO(hilalymh): Audit deprecated and removed versions
+
 	return model, nil
 }
 
@@ -140,10 +155,10 @@ func (m *APIVersionManager) VerifyAPIVersions(apiVersions ...string) error {
 		if !ok {
 			return fmt.Errorf("%v: %s", ErrAPIVersionNotFound, apiVersion)
 		}
-		if apiInfo.Status == APIStatusDeprecated {
+		if apiInfo.Status == ackmetadata.APIStatusDeprecated {
 			return fmt.Errorf("%v: %s", ErrAPIVersionDeprecated, apiVersion)
 		}
-		if apiInfo.Status == APIStatusRemoved {
+		if apiInfo.Status == ackmetadata.APIStatusRemoved {
 			return fmt.Errorf("%v: %s", ErrAPIVersionRemoved, apiVersion)
 		}
 	}

pkg/testdata/models/metadata.yaml

@@ -0,0 +1,8 @@
+service:
+  full_name: Amazon Test Service
+  short_name: ATS
+  link: https://example.com/
+  documentation: https://docs.example.com/
+versions:
+- api_version: v1alpha1
+  status: available