add selector flag to the scorecard subcommand (#2096)

* add dry-run and selector flags to the scorecard subcommand

* make default selector flag value empty string

* add selector implementation and remove dryrun which will be added later

* remove dryrun from CHANGELOG since that is not in this PR

* update scorecard doc to include selector flag

* refactor to delete tests based on selector

* remove weights in test suite as part of removing a test when using selector flag, update sdk scorecard docs to include selector flag and v1alpha2 output instead of v1alpha1 output

* add validation for v1alpha2 to not allow external plugins, add test for this validation

* Update internal/scorecard/config.go

Co-Authored-By: Joe Lanford <[email protected]>

Author: Jeff McCormick

CHANGELOG.md

@@ -37,6 +37,7 @@
 - Added the Go version, OS, and architecture to the output of `operator-sdk version` ([#1863](https://github.com/operator-framework/operator-sdk/pull/1863))
 - Added support for `ppc64le-linux` for the `operator-sdk` binary and the Helm operator base image. ([#1533](https://github.com/operator-framework/operator-sdk/pull/1533))
 - Added new `--version` flag to the `operator-sdk scorecard` command to support a new output format for the scorecard. ([#1916](https://github.com/operator-framework/operator-sdk/pull/1916)
+- Added new `--selector` flag to the `operator-sdk scorecard` command to support filtering scorecard tests based on labels added to each test. ([#1916](https://github.com/operator-framework/operator-sdk/pull/1916)
 
 ### Changed
 

cmd/operator-sdk/scorecard/cmd.go

@@ -37,14 +37,16 @@ func NewCmd() *cobra.Command {
 	scorecardCmd.Flags().String(scorecard.ConfigOpt, "", fmt.Sprintf("config file (default is '<project_dir>/%s'; the config file's extension and format can be .yaml, .json, or .toml)", scorecard.DefaultConfigFile))
 	scorecardCmd.Flags().String(scplugins.KubeconfigOpt, "", "Path to kubeconfig of custom resource created in cluster")
 	scorecardCmd.Flags().StringP(scorecard.OutputFormatOpt, "o", scorecard.TextOutputFormat, fmt.Sprintf("Output format for results. Valid values: %s, %s", scorecard.TextOutputFormat, scorecard.JSONOutputFormat))
-	scorecardCmd.Flags().String(schelpers.VersionOpt, schelpers.DefaultScorecardVersion, fmt.Sprintf("scorecard version (tech preview version is '%s'", schelpers.LatestScorecardVersion))
+	scorecardCmd.Flags().String(schelpers.VersionOpt, schelpers.DefaultScorecardVersion, "scorecard version. Valid values: v1alpha1, v1alpha2")
+	scorecardCmd.Flags().StringP(scorecard.SelectorOpt, "l", "", "selector (label query) to filter tests on (only valid when version is v1alpha2)")
 
 	// TODO: make config file global and make this a top level flag
 	viper.BindPFlag(scorecard.ConfigOpt, scorecardCmd.Flags().Lookup(scorecard.ConfigOpt))
 
 	viper.BindPFlag("scorecard."+scplugins.KubeconfigOpt, scorecardCmd.Flags().Lookup(scplugins.KubeconfigOpt))
 	viper.BindPFlag("scorecard."+scorecard.OutputFormatOpt, scorecardCmd.Flags().Lookup(scorecard.OutputFormatOpt))
 	viper.BindPFlag("scorecard."+schelpers.VersionOpt, scorecardCmd.Flags().Lookup(schelpers.VersionOpt))
+	viper.BindPFlag("scorecard."+scorecard.SelectorOpt, scorecardCmd.Flags().Lookup(scorecard.SelectorOpt))
 
 	return scorecardCmd
 }

doc/sdk-cli-reference.md

@@ -420,28 +420,26 @@ Run scorecard tests on an operator
 * `-o, --output` string - Output format for results. Valid values: `human-readable` or `json` (default `human-readable`)
 * `proxy-image` string - Image name for scorecard proxy (default "quay.io/operator-framework/scorecard-proxy")
 * `proxy-pull-policy` string - Pull policy for scorecard proxy image (default "Always")
+* `selector` string - Selector (label query) to filter tests on (only valid when version is v1alpha2)
 * `-h, --help` - help for scorecard
 * `version` string - The scorecard version to run (default v1alpha1), the tech preview version is v1alpha2.
 
 ### Example
 
 ```console
-$ operator-sdk scorecard --cr-manifest deploy/crds/cache.example.com_v1alpha1_memcached_cr.yaml --csv-path deploy/olm-catalog/memcached-operator/0.0.2/memcached-operator.v0.0.2.clusterserviceversion.yaml
-Basic Operator:
-        Spec Block Exists: 1/1 points
-        Status Block Exist: 1/1 points
-        Operator actions are reflected in status: 1/1 points
-        Writing into CRs has an effect: 1/1 points
-OLM Integration:
-        Provided APIs have validation: 1/1
-        Owned CRDs have resources listed: 1/1 points
-        CRs have at least 1 example: 0/1 points
-        Spec fields with descriptors: 1/1 points
-        Status fields with descriptors: 0/1 points
-
-Total Score: 84%
-SUGGESTION: Add an alm-examples annotation to your CSV to pass the CRs have at least 1 example test
-SUGGESTION: Add a status descriptor for nodes
+$ operator-sdk scorecard --cr-manifest deploy/crds/cache.example.com_v1alpha1_memcached_cr.yaml --csv-path deploy/olm-catalog/memcached-operator/0.0.2/memcached-operator.v0.0.2.clusterserviceversion.yaml -o text
+basic:
+	Writing into CRs has an effect     : pass
+	Spec Block Exists                  : pass
+	Status Block Exists                : pass
+olm:
+	Spec fields with descriptors       : pass
+	Status fields with descriptors     : pass
+	Provided APIs have validation      : fail
+	Owned CRDs have resources listed   : pass
+	CRs have at least 1 example        : pass
+SUGGESTION: Add CRD validation for Memcached/v1alpha1
+SUGGESTION: If it would be helpful to an end-user to understand or troubleshoot your CR, consider adding resources [deployments/v1 services/v1 configmaps/v1 memcacheds/v1alpha1 replicasets/v1] to the resources section for owned CRD Memcached
 ```
 
 ## test

doc/test-framework/scorecard.md

@@ -106,7 +106,8 @@ While most configuration is done via a config file, there are a few important ar
 | `--config`  | string | Path to config file (default `<project_dir>/.osdk-scorecard`; file type and extension can be any of `.yaml`, `.json`, or `.toml`). If a config file is not provided and a config file is not found at the default location, the scorecard will exit with an error. |
 | `--output`, `-o`  | string | Output format. Valid options are: `text` and `json`. The default format is `text`, which is designed to be a simpler human readable format. The `json` format uses the JSON schema output format used for plugins defined later in this document. |
 | `--kubeconfig`, `-o`  | string |  path to kubeconfig. It sets the kubeconfig internally for internal plugins and sets the `KUBECONFIG` env var to the provided value for external plugins. If an external plugin specifically sets the `KUBECONFIG` env var, the kubeconfig from the specified env var will be used for that plugin instead. |
-| `--version`  | string |  The version of scorecard to run, v1alpha1 is the default, whereas v1alpha2 is the tech preview. |
+| `--version`  | string |  The version of scorecard to run, v1alpha2 is the default, valid values are v1alpha and v1alpha2. |
+| `--selector`, `-l`  | string |  The label selector to filter tests on, only valid in version v1alpha2. |
 
 ### Config File Options
 

hack/tests/subcommand-scorecard.sh

@@ -5,6 +5,7 @@ CONFIG_PATH=".test-osdk-scorecard.yaml"
 CONFIG_PATH_V1ALPHA1=".test-osdk-scorecard-v1alpha1.yaml"
 CONFIG_PATH_DISABLE=".osdk-scorecard-disable.yaml"
 CONFIG_PATH_INVALID=".osdk-scorecard-invalid.yaml"
+CONFIG_PATH_V1ALPHA2=".osdk-scorecard-v1alpha2.yaml"
 
 set -ex
 
@@ -14,10 +15,13 @@ set -ex
 # the test framework directory has all the manifests needed to run the cluster
 pushd test/test-framework
 
+# test to see if scorecard fails when version is v1alpha2 and when external plugins are configured
+operator-sdk scorecard --version v1alpha2 --config "$CONFIG_PATH" |& grep '^.*error validating plugin config.*$'
+
 # test to see if v1alpha2 is used from the command line
-commandoutput="$(operator-sdk scorecard --version v1alpha2 --config "$CONFIG_PATH" 2>&1)"
+commandoutput="$(operator-sdk scorecard --version v1alpha2 --config "$CONFIG_PATH_V1ALPHA2" 2>&1)"
 failCount=`echo $commandoutput | grep -o ": fail" | wc -l`
-expectedFailCount=7
+expectedFailCount=3
 if [ $failCount -ne $expectedFailCount ]
 then
 	echo "expected fail count $expectedFailCount, got $failCount"

internal/scorecard/config.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"strings"
 
+	schelpers "github.com/operator-framework/operator-sdk/internal/scorecard/helpers"
 	"gopkg.in/yaml.v2"
 )
 
@@ -49,7 +50,7 @@ func (e externalPluginEnv) String() string {
 }
 
 // validateConfig takes a viper config for a plugin and returns a nil error if valid or an error explaining why the config is invalid
-func validateConfig(config pluginConfig, idx int) error {
+func validateConfig(config pluginConfig, idx int, version string) error {
 	// find plugin config type
 	pluginType := ""
 	if config.Basic != nil {
@@ -66,6 +67,9 @@ func validateConfig(config pluginConfig, idx int) error {
 			return fmt.Errorf("plugin config can only contain one of: basic, olm, external")
 		}
 		pluginType = "external"
+		if schelpers.IsV1alpha2(version) {
+			return fmt.Errorf("revert to v1alpha1 to use external plugins: external plugins are not currently supported with v1alpha2")
+		}
 	}
 	if pluginType == "" {
 		marshalledConfig, err := yaml.Marshal(config)

internal/scorecard/helpers/test_definitions.go

@@ -20,6 +20,7 @@ import (
 	"strings"
 
 	scapiv1alpha1 "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha1"
+	"k8s.io/apimachinery/pkg/labels"
 )
 
 // Type Definitions
@@ -102,6 +103,19 @@ func (ts *TestSuite) TotalScore() (score int) {
 	return int(floatScore * (100 / addedWeights))
 }
 
+// ApplySelector apply label selectors removing tests that do not match
+func (ts *TestSuite) ApplySelector(selector labels.Selector) {
+	for i := 0; i < len(ts.Tests); i++ {
+		t := ts.Tests[i]
+		if !selector.Matches(labels.Set(t.GetLabels())) {
+			// Remove the test
+			ts.Tests = append(ts.Tests[:i], ts.Tests[i+1:]...)
+			delete(ts.Weights, t.GetName())
+			i--
+		}
+	}
+}
+
 // Run runs all Tests in a TestSuite
 func (ts *TestSuite) Run(ctx context.Context) {
 	for _, test := range ts.Tests {

internal/scorecard/plugins/basic_tests.go

@@ -100,6 +100,7 @@ func NewBasicTestSuite(conf BasicTestConfig) *schelpers.TestSuite {
 		"Basic Tests",
 		"Test suite that runs basic, functional operator tests",
 	)
+
 	ts.AddTest(NewCheckSpecTest(conf), 1.5)
 	ts.AddTest(NewCheckStatusTest(conf), 1)
 	ts.AddTest(NewWritingIntoCRsHasEffectTest(conf), 1)

internal/scorecard/plugins/config.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/pkg/errors"
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
 )
 
 const (
@@ -39,18 +40,20 @@ const (
 )
 
 type BasicAndOLMPluginConfig struct {
-	Namespace          string        `mapstructure:"namespace"`
-	Kubeconfig         string        `mapstructure:"kubeconfig"`
-	InitTimeout        int           `mapstructure:"init-timeout"`
-	OLMDeployed        bool          `mapstructure:"olm-deployed"`
-	NamespacedManifest string        `mapstructure:"namespaced-manifest"`
-	GlobalManifest     string        `mapstructure:"global-manifest"`
-	CRManifest         []string      `mapstructure:"cr-manifest"`
-	CSVManifest        string        `mapstructure:"csv-path"`
-	ProxyImage         string        `mapstructure:"proxy-image"`
-	ProxyPullPolicy    v1.PullPolicy `mapstructure:"proxy-pull-policy"`
-	CRDsDir            string        `mapstructure:"crds-dir"`
-	DeployDir          string        `mapstructure:"deploy-dir"`
+	Namespace          string          `mapstructure:"namespace"`
+	Kubeconfig         string          `mapstructure:"kubeconfig"`
+	InitTimeout        int             `mapstructure:"init-timeout"`
+	OLMDeployed        bool            `mapstructure:"olm-deployed"`
+	NamespacedManifest string          `mapstructure:"namespaced-manifest"`
+	GlobalManifest     string          `mapstructure:"global-manifest"`
+	CRManifest         []string        `mapstructure:"cr-manifest"`
+	CSVManifest        string          `mapstructure:"csv-path"`
+	ProxyImage         string          `mapstructure:"proxy-image"`
+	ProxyPullPolicy    v1.PullPolicy   `mapstructure:"proxy-pull-policy"`
+	CRDsDir            string          `mapstructure:"crds-dir"`
+	DeployDir          string          `mapstructure:"deploy-dir"`
+	Selector           labels.Selector `mapstructure:"selector"`
+	Version            string          `mapstructure:"version"`
 }
 
 func validateScorecardPluginFlags(config BasicAndOLMPluginConfig, pluginType PluginType) error {

internal/scorecard/plugins/plugin_runner.go

@@ -292,6 +292,9 @@ func RunInternalPlugin(pluginType PluginType, config BasicAndOLMPluginConfig, lo
 				ProxyPod: proxyPodGlobal,
 			}
 			basicTests := NewBasicTestSuite(conf)
+			if schelpers.IsV1alpha2(config.Version) {
+				basicTests.ApplySelector(config.Selector)
+			}
 			basicTests.Run(context.TODO())
 			logs, err := ioutil.ReadAll(logReadWriter)
 			if err != nil {
@@ -310,6 +313,9 @@ func RunInternalPlugin(pluginType PluginType, config BasicAndOLMPluginConfig, lo
 				ProxyPod: proxyPodGlobal,
 			}
 			olmTests := NewOLMTestSuite(conf)
+			if schelpers.IsV1alpha2(config.Version) {
+				olmTests.ApplySelector(config.Selector)
+			}
 			olmTests.Run(context.TODO())
 			logs, err := ioutil.ReadAll(logReadWriter)
 			if err != nil {