diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..f3eba9f073 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,36 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + # Workflow files stored in the + # default location of `.github/workflows` + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: ":seedling:" + labels: + - "ok-to-test" + + # Maintain dependencies for go + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: ":seedling:" + # Ignore K8 packages as these are done manually + ignore: + - dependency-name: "k8s.io/api" + - dependency-name: "k8s.io/apiextensions-apiserver" + - dependency-name: "k8s.io/apimachinery" + - dependency-name: "k8s.io/client-go" + - dependency-name: "k8s.io/component-base" + labels: + - "ok-to-test" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index fe4c2e7a72..f6b8370e57 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,4 +1,4 @@ - + diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 2cf235d808..4d89839ee9 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -15,9 +15,9 @@ jobs: - "" - tools/setup-envtest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: golangci-lint - uses: golangci/golangci-lint-action@v2 + uses: golangci/golangci-lint-action@v3 with: - version: v1.40.1 + version: v1.51.1 working-directory: ${{matrix.working-directory}} diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index caa342f44b..51bb083ed5 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -9,6 +9,6 @@ jobs: steps: - name: Verifier action id: verifier - uses: kubernetes-sigs/kubebuilder-release-tools@v0.1 + uses: kubernetes-sigs/kubebuilder-release-tools@v0.3.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index c2c72faf34..294685952b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,6 @@ # Tools binaries. hack/tools/bin + +junit-report.xml +/artifacts \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml index a5a5dad615..8c98246a31 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,46 +1,46 @@ linters: disable-all: true enable: - - asciicheck - - bodyclose - - deadcode - - depguard - - dogsled - - errcheck - - exportloopref - - goconst - - gocritic - - gocyclo - - godot - - gofmt - - goimports - - goprintffuncname - - gosec - - gosimple - - govet - - ifshort - - importas - - ineffassign - - misspell - - nakedret - - nilerr - - nolintlint - - prealloc - - revive - - rowserrcheck - - staticcheck - - structcheck - - stylecheck - - typecheck - - unconvert - - unparam - - varcheck - - whitespace + - asasalint + - asciicheck + - bidichk + - bodyclose + - depguard + - dogsled + - dupl + - errcheck + - errchkjson + - errorlint + - exhaustive + - exportloopref + - goconst + - gocritic + - gocyclo + - gofmt + - goimports + - goprintffuncname + - gosec + - gosimple + - govet + - importas + - ineffassign + - makezero + - misspell + - nakedret + - nilerr + - nolintlint + - prealloc + - revive + - staticcheck + - stylecheck + - tagliatelle + - typecheck + - unconvert + - unparam + - unused + - whitespace linters-settings: - ifshort: - # Maximum length of variable declaration measured in number of characters, after which linter won't suggest using short syntax. - max-decl-chars: 50 importas: no-unaliased: true alias: @@ -59,9 +59,13 @@ linters-settings: - pkg: sigs.k8s.io/controller-runtime alias: ctrl staticcheck: - go: "1.16" + go: "1.19" stylecheck: - go: "1.16" + go: "1.19" + depguard: + include-go-root: true + packages: + - io/ioutil # https://go.dev/doc/go1.16#ioutil issues: max-same-issues: 0 @@ -71,60 +75,74 @@ issues: exclude-use-default: false # List of regexps of issue texts to exclude, empty list by default. exclude: - # The following are being worked on to remove their exclusion. This list should be reduced or go away all together over time. - # If it is decided they will not be addressed they should be moved above this comment. - - Subprocess launch(ed with variable|ing should be audited) - - (G204|G104|G307) - - "ST1000: at least one file in a package should have a package comment" + # The following are being worked on to remove their exclusion. This list should be reduced or go away all together over time. + # If it is decided they will not be addressed they should be moved above this comment. + - Subprocess launch(ed with variable|ing should be audited) + - (G204|G104|G307) + - "ST1000: at least one file in a package should have a package comment" exclude-rules: - - linters: - - gosec - text: "G108: Profiling endpoint is automatically exposed on /debug/pprof" - - linters: - - revive - text: "exported: exported method .*\\.(Reconcile|SetupWithManager|SetupWebhookWithManager) should have comment or be unexported" - - linters: - - errcheck - text: Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked - # With Go 1.16, the new embed directive can be used with an un-named import, - # revive (previously, golint) only allows these to be imported in a main.go, which wouldn't work for us. - # This directive allows the embed package to be imported with an underscore everywhere. - - linters: - - revive - source: _ "embed" - # Exclude some packages or code to require comments, for example test code, or fake clients. - - linters: - - revive - text: exported (method|function|type|const) (.+) should have comment or be unexported - source: (func|type).*Fake.* - - linters: - - revive - text: exported (method|function|type|const) (.+) should have comment or be unexported - path: fake_\.go - # Disable unparam "always receives" which might not be really - # useful when building libraries. - - linters: - - unparam - text: always receives - # Dot imports for gomega or ginkgo are allowed - # within test files. - - path: _test\.go - text: should not use dot imports - - path: _test\.go - text: cyclomatic complexity - - path: _test\.go - text: "G107: Potential HTTP request made with variable url" - # Append should be able to assign to a different var/slice. - - linters: - - gocritic - text: "appendAssign: append result not assigned to the same slice" - - linters: - - gocritic - text: "singleCaseSwitch: should rewrite switch statement to if statement" + - linters: + - gosec + text: "G108: Profiling endpoint is automatically exposed on /debug/pprof" + - linters: + - revive + text: "exported: exported method .*\\.(Reconcile|SetupWithManager|SetupWebhookWithManager) should have comment or be unexported" + - linters: + - errcheck + text: Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked + - linters: + - staticcheck + text: "SA1019: .* The component config package has been deprecated and will be removed in a future release." + # With Go 1.16, the new embed directive can be used with an un-named import, + # revive (previously, golint) only allows these to be imported in a main.go, which wouldn't work for us. + # This directive allows the embed package to be imported with an underscore everywhere. + - linters: + - revive + source: _ "embed" + # Exclude some packages or code to require comments, for example test code, or fake clients. + - linters: + - revive + text: exported (method|function|type|const) (.+) should have comment or be unexported + source: (func|type).*Fake.* + - linters: + - revive + text: exported (method|function|type|const) (.+) should have comment or be unexported + path: fake_\.go + # Disable unparam "always receives" which might not be really + # useful when building libraries. + - linters: + - unparam + text: always receives + # Dot imports for gomega and ginkgo are allowed + # within test files. + - path: _test\.go + text: should not use dot imports + - path: _test\.go + text: cyclomatic complexity + - path: _test\.go + text: "G107: Potential HTTP request made with variable url" + # Append should be able to assign to a different var/slice. + - linters: + - gocritic + text: "appendAssign: append result not assigned to the same slice" + - linters: + - gocritic + text: "singleCaseSwitch: should rewrite switch statement to if statement" + # It considers all file access to a filename that comes from a variable problematic, + # which is naiv at best. + - linters: + - gosec + text: "G304: Potential file inclusion via variable" + - linters: + - revive + text: "package-comments: should have a package comment" + - linters: + - dupl + path: _test\.go run: timeout: 10m skip-files: - - "zz_generated.*\\.go$" - - ".*conversion.*\\.go$" + - "zz_generated.*\\.go$" + - ".*conversion.*\\.go$" allow-parallel-runners: true diff --git a/FAQ.md b/FAQ.md index cfc2997924..c21b29e287 100644 --- a/FAQ.md +++ b/FAQ.md @@ -30,13 +30,13 @@ on your situation. take this approach: the StatefulSet controller appends a specific number to each pod that it creates, while the Deployment controller hashes the pod template spec and appends that. - + - In the few cases when you cannot take advantage of deterministic names (e.g. when using generateName), it may be useful in to track which actions you took, and assume that they need to be repeated if they don't occur after a given time (e.g. using a requeue result). This is what the ReplicaSet controller does. - + In general, write your controller with the assumption that information will eventually be correct, but may be slightly out of date. Make sure that your reconcile function enforces the entire state of the world each @@ -48,17 +48,17 @@ generally cover most circumstances. ### Q: Where's the fake client? How do I use it? **A**: The fake client -[exists](https://godoc.org/sigs.k8s.io/controller-runtime/pkg/client/fake), +[exists](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/client/fake), but we generally recommend using -[envtest.Environment](https://godoc.org/sigs.k8s.io/controller-runtime/pkg/envtest#Environment) +[envtest.Environment](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/envtest#Environment) to test against a real API server. In our experience, tests using fake clients gradually re-implement poorly-written impressions of a real API server, which leads to hard-to-maintain, complex test code. -### Q: How should I write tests? Any suggestions for getting started? +### Q: How should I write tests? Any suggestions for getting started? - Use the aforementioned - [envtest.Environment](https://godoc.org/sigs.k8s.io/controller-runtime/pkg/envtest#Environment) + [envtest.Environment](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/envtest#Environment) to spin up a real API server instead of trying to mock one out. - Structure your tests to check that the state of the world is as you @@ -77,5 +77,5 @@ mapping between Go types and group-version-kinds in Kubernetes. In general, your application should have its own Scheme containing the types from the API groups that it needs (be they Kubernetes types or your own). See the [scheme builder -docs](https://godoc.org/sigs.k8s.io/controller-runtime/pkg/scheme) for +docs](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/scheme) for more information. diff --git a/Makefile b/Makefile index f2cd38cad8..e7e167d10b 100644 --- a/Makefile +++ b/Makefile @@ -40,6 +40,7 @@ TOOLS_BIN_DIR := $(TOOLS_DIR)/bin GOLANGCI_LINT := $(abspath $(TOOLS_BIN_DIR)/golangci-lint) GO_APIDIFF := $(TOOLS_BIN_DIR)/go-apidiff CONTROLLER_GEN := $(TOOLS_BIN_DIR)/controller-gen +ENVTEST_DIR := $(abspath tools/setup-envtest) # The help will print out all targets with their descriptions organized bellow their categories. The categories are represented by `##@` and the target descriptions by `##`. # The awk commands is responsible to read the entire set of makefiles included in this invocation, looking for lines of the file as xyz: ## something, and then pretty-format the target and help. Then, if there's a line with ##@ something, that gets pretty-printed as a category. @@ -97,6 +98,7 @@ lint-fix: $(GOLANGCI_LINT) ## Lint the codebase and run auto-fixers if supported modules: ## Runs go mod to ensure modules are up to date. go mod tidy cd $(TOOLS_DIR); go mod tidy + cd $(ENVTEST_DIR); go mod tidy .PHONY: generate generate: $(CONTROLLER_GEN) ## Runs controller-gen for internal types for config file @@ -115,7 +117,15 @@ clean-bin: ## Remove all generated binaries. rm -rf hack/tools/bin .PHONY: verify-modules -verify-modules: modules - @if !(git diff --quiet HEAD -- go.sum go.mod); then \ +verify-modules: modules ## Verify go modules are up to date + @if !(git diff --quiet HEAD -- go.sum go.mod $(TOOLS_DIR)/go.mod $(TOOLS_DIR)/go.sum $(ENVTEST_DIR)/go.mod $(ENVTEST_DIR)/go.sum); then \ + git diff; \ echo "go module files are out of date, please run 'make modules'"; exit 1; \ fi + +.PHONY: verify-generate +verify-generate: generate ## Verify generated files are up to date + @if !(git diff --quiet HEAD); then \ + git diff; \ + echo "generated files are out of date, run make generate"; exit 1; \ + fi diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index d93e1794fb..3d1d2f0cb8 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -4,28 +4,26 @@ aliases: # active folks who can be contacted to perform admin-related # tasks on the repo, or otherwise approve any PRS. controller-runtime-admins: - - droot - - mengqiy - - pwittrock + - vincepri + - joelanford # non-admin folks who have write-access and can approve any PRs in the repo controller-runtime-maintainers: - vincepri + - joelanford # non-admin folks who can approve any PRs in the repo controller-runtime-approvers: - - gerred - - shawn-hurley - - joelanford - alvaroaleman + - fillzpp + - sbueringer # folks who can review and LGTM any PRs in the repo (doesn't # include approvers & admins -- those count too via the OWNERS # file) controller-runtime-reviewers: - - alenkacz - - vincepri - - alexeldeib + - varshaprasad96 + - inteon # folks to can approve things in the directly-ported # testing_frameworks portions of the codebase @@ -37,3 +35,7 @@ aliases: # but are no longer directly involved controller-runtime-emeritus-maintainers: - directxman12 + controller-runtime-emeritus-admins: + - droot + - mengqiy + - pwittrock diff --git a/README.md b/README.md index e4cbabb00a..484881dce4 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ [![Go Report Card](https://goreportcard.com/badge/sigs.k8s.io/controller-runtime)](https://goreportcard.com/report/sigs.k8s.io/controller-runtime) +[![godoc](https://pkg.go.dev/badge/sigs.k8s.io/controller-runtime)](https://pkg.go.dev/sigs.k8s.io/controller-runtime) # Kubernetes controller-runtime Project @@ -52,7 +53,7 @@ in sig apimachinery. You can reach the maintainers of this project at: -- Slack channel: [#kubebuilder](http://slack.k8s.io/#kubebuilder) +- Slack channel: [#controller-runtime](https://kubernetes.slack.com/archives/C02MRBMN00Z) - Google Group: [kubebuilder@googlegroups.com](https://groups.google.com/forum/#!forum/kubebuilder) ## Contributing diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000000..134a73a31b --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,47 @@ +# Release Process + +The Kubernetes controller-runtime Project is released on an as-needed basis. The process is as follows: + +**Note:** Releases are done from the `release-MAJOR.MINOR` branches. For PATCH releases is not required +to create a new branch you will just need to ensure that all big fixes are cherry-picked into the respective +`release-MAJOR.MINOR` branch. To know more about versioning check https://semver.org/. + +## How to do a release + +### Create the new branch and the release tag + +1. Create a new branch `git checkout -b release-` from master +2. Push the new branch to the remote repository + +### Now, let's generate the changelog + +1. Create the changelog from the new branch `release-` (`git checkout release-`). +You will need to use the [kubebuilder-release-tools][kubebuilder-release-tools] to generate the notes. See [here][release-notes-generation] + +> **Note** +> - You will need to have checkout locally from the remote repository the previous branch +> - Also, ensure that you fetch all tags from the remote `git fetch --all --tags` + +### Draft a new release from GitHub + +1. Create a new tag with the correct version from the new `release-` branch +2. Add the changelog on it and publish. Now, the code source is released ! + +### Add a new Prow test the for the new branch release + +1. Create a new prow test under [github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-sigs/controller-runtime](https://github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-sigs/controller-runtime) +for the new `release-` branch. (i.e. for the `0.11.0` release see the PR: https://github.com/kubernetes/test-infra/pull/25205) +2. Ping the infra PR in the controller-runtime slack channel for reviews. + +### Announce the new release: + +1. Publish on the Slack channel the new release, i.e: + +```` +:announce: Controller-Runtime v0.12.0 has been released! +This release includes a Kubernetes dependency bump to v1.24. +For more info, see the release page: https://github.com/kubernetes-sigs/controller-runtime/releases. + :tada: Thanks to all our contributors! +```` + +2. An announcement email is sent to `kubebuilder@googlegroups.com` with the subject `[ANNOUNCE] Controller-Runtime $VERSION is released` diff --git a/TMP-LOGGING.md b/TMP-LOGGING.md index 9ee4b2a431..97e091fd48 100644 --- a/TMP-LOGGING.md +++ b/TMP-LOGGING.md @@ -21,7 +21,7 @@ log.Printf("starting reconciliation for pod %s/%s", podNamespace, podName) In controller-runtime, we'd instead write: ```go -logger.Info("starting reconciliation", "pod", req.NamespacedNamed) +logger.Info("starting reconciliation", "pod", req.NamespacedName) ``` or even write @@ -51,7 +51,7 @@ You can configure the logging implementation using `"sigs.k8s.io/controller-runtime/pkg/log".SetLogger`. That package also contains the convenience functions for setting up Zap. -You can get a handle to the the "root" logger using +You can get a handle to the "root" logger using `"sigs.k8s.io/controller-runtime/pkg/log".Log`, and can then call `WithName` to create individual named loggers. You can call `WithName` repeatedly to chain names together: @@ -75,7 +75,7 @@ allKubernetesObjectsEverywhere) ``` While it's possible to use higher log levels, it's recommended that you -stick with `V(1)` or V(0)` (which is equivalent to not specifying `V`), +stick with `V(1)` or `V(0)` (which is equivalent to not specifying `V`), and then filter later based on key-value pairs or messages; different numbers tend to lose meaning easily over time, and you'll be left wondering why particular logs lines are at `V(5)` instead of `V(7)`. diff --git a/alias.go b/alias.go index 29f964dcbe..237963889c 100644 --- a/alias.go +++ b/alias.go @@ -70,6 +70,10 @@ type TypeMeta = metav1.TypeMeta type ObjectMeta = metav1.ObjectMeta var ( + // RegisterFlags registers flag variables to the given FlagSet if not already registered. + // It uses the default command line FlagSet, if none is provided. Currently, it only registers the kubeconfig flag. + RegisterFlags = config.RegisterFlags + // GetConfigOrDie creates a *rest.Config for talking to a Kubernetes apiserver. // If --kubeconfig is set, will use the kubeconfig file at that location. Otherwise will assume running // in cluster and use the cluster provided kubeconfig. @@ -95,6 +99,8 @@ var ( // ConfigFile returns the cfg.File function for deferred config file loading, // this is passed into Options{}.From() to populate the Options fields for // the manager. + // + // Deprecated: This is deprecated in favor of using Options directly. ConfigFile = cfg.File // NewControllerManagedBy returns a new controller builder that will be started by the provided Manager. @@ -135,7 +141,7 @@ var ( // The logger, when used with controllers, can be expected to contain basic information about the object // that's being reconciled like: // - `reconciler group` and `reconciler kind` coming from the For(...) object passed in when building a controller. - // - `name` and `namespace` injected from the reconciliation request. + // - `name` and `namespace` from the reconciliation request. // // This is meant to be used with the context supplied in a struct that satisfies the Reconciler interface. LoggerFrom = log.FromContext diff --git a/designs/README.md b/designs/README.md index 3ed7de5ad3..bf8b5000a9 100644 --- a/designs/README.md +++ b/designs/README.md @@ -1,12 +1,12 @@ Designs ======= -These are design documents for changes to Controller Runtime They exist -to help document the design processes that go into writing Controller -Runtime, but may not be up-to-date (more below). +These are the design documents for changes to Controller Runtime. They +exist to help document the design processes that go into writing +Controller Runtime, but may not be up-to-date (more below). Not all changes to Controller Runtime need a design document -- only major -ones. use your best judgement. +ones. Use your best judgement. When submitting a design document, we encourage having written a proof-of-concept, and it's perfectly acceptable to submit the @@ -17,7 +17,7 @@ proof-of-concept process can help iron out wrinkles and can help with the ## Out-of-Date Designs **Controller Runtime documentation -[GoDoc](https://godoc.org/sigs.k8s.io/controller-runtime) should be +[GoDoc](https://pkg.go.dev/sigs.k8s.io/controller-runtime) should be considered the canonical, update-to-date reference and architectural documentation** for Controller Runtime. @@ -29,8 +29,8 @@ why things changed. For example: # Out of Date -This change is out of date. It turns out curly braces a frustrating to +This change is out of date. It turns out curly braces are frustrating to type, so we had to abandon functions entirely, and have users specify custom functionality using strings of Common LISP instead. See #000 for more information. -``` \ No newline at end of file +``` diff --git a/designs/component-config.md b/designs/component-config.md index 00d6d59391..8aebec4f96 100644 --- a/designs/component-config.md +++ b/designs/component-config.md @@ -37,7 +37,7 @@ Currently controllers that use `controller-runtime` need to configure the `ctrl. ## Motivation -This change is important because: +This change is important because: - it will help make it easier for controllers to be configured by other machine processes - it will reduce the required flags required to start a controller - allow for configuration types which aren't natively supported by flags @@ -65,7 +65,7 @@ This change is important because: ## Proposal -The `ctrl.Manager` _SHOULD_ support loading configurations from `ComponentConfig` like objects. +The `ctrl.Manager` _SHOULD_ support loading configurations from `ComponentConfig` like objects. An interface for that object with getters for the specific configuration parameters is created to bridge existing patterns. Without breaking the current `ctrl.NewManager` which uses an exported `ctrl.Options{}` the `manager.go` can expose a new func, `NewFromComponentConfig()` this would be able to loop through the getters to populate an internal `ctrl.Options{}` and pass that into `New()`. @@ -101,7 +101,7 @@ type ManagerConfiguration interface { func NewFromComponentConfig(config *rest.Config, scheme *runtime.Scheme, filename string, managerconfig ManagerConfiguration) (Manager, error) { codecs := serializer.NewCodecFactory(scheme) if err := decodeComponentConfigFileInto(codecs, filename, managerconfig); err != nil { - + } options := Options{} @@ -139,7 +139,7 @@ import ( // ControllerManagerConfiguration defines the embedded RuntimeConfiguration for controller-runtime clients. type ControllerManagerConfiguration struct { - Namespace string `json:"namespace,omitempty"` + Namespace string `json:"namespace,omitempty"` SyncPeriod *time.Duration `json:"syncPeriod,omitempty"` @@ -168,7 +168,7 @@ type ControllerManagerConfigurationHealth struct { #### Default ComponentConfig Type -To enable `controller-runtime` to have a default `ComponentConfig` struct which can be used instead of requiring each controller or extension to build it's own `ComponentConfig` type, we can create a `DefaultControllerConfiguration` type which can exist in `pkg/api/config/v1alpha1/types.go`. This will allow the controller authors to use this before needing to implement their own type with additional configs. +To enable `controller-runtime` to have a default `ComponentConfig` struct which can be used instead of requiring each controller or extension to build its own `ComponentConfig` type, we can create a `DefaultControllerConfiguration` type which can exist in `pkg/api/config/v1alpha1/types.go`. This will allow the controller authors to use this before needing to implement their own type with additional configs. ```golang // pkg/api/config/v1alpha1/types.go @@ -212,12 +212,12 @@ if err != nil { } ``` -The above example uses `configname` which is the name of the file to load the configuration from and uses `scheme` to get the specific serializer, eg `serializer.NewCodecFactory(scheme)`. This will allow the configuration to be unmarshalled into the `runtime.Object` type and passed into the +The above example uses `configname` which is the name of the file to load the configuration from and uses `scheme` to get the specific serializer, eg `serializer.NewCodecFactory(scheme)`. This will allow the configuration to be unmarshalled into the `runtime.Object` type and passed into the `ctrl.NewManagerFromComponentConfig()` as a `ManagerConfiguration` interface. #### Using Flags w/ ComponentConfig -Since this design still requires setting up the initial `ComponentConfig` type and passing in a pointer to `ctrl.NewFromComponentConfig()` if you want to allow for the use of flags, your controller can use any of the different flagging interfaces. eg [`flag`](https://golang.org/pkg/flag/), [`pflag`](https://godoc.org/github.com/spf13/pflag), [`flagnum`](https://godoc.org/github.com/luci/luci-go/common/flag/flagenum) and set values on the `ComponentConfig` type prior to passing the pointer into the `ctrl.NewFromComponentConfig()`, example below. +Since this design still requires setting up the initial `ComponentConfig` type and passing in a pointer to `ctrl.NewFromComponentConfig()` if you want to allow for the use of flags, your controller can use any of the different flagging interfaces. eg [`flag`](https://golang.org/pkg/flag/), [`pflag`](https://pkg.go.dev/github.com/spf13/pflag), [`flagnum`](https://pkg.go.dev/github.com/luci/luci-go/common/flag/flagenum) and set values on the `ComponentConfig` type prior to passing the pointer into the `ctrl.NewFromComponentConfig()`, example below. ```golang leaderElect := true @@ -247,7 +247,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" configv1alpha1 "sigs.k8s.io/controller-runtime/pkg/apis/config/v1alpha1" -) +) type ControllerNameConfigurationSpec struct { configv1alpha1.ControllerManagerConfiguration `json:",inline"` diff --git a/designs/use-selectors-at-cache.md b/designs/use-selectors-at-cache.md index c9175e9d38..1d7ec6ecfb 100644 --- a/designs/use-selectors-at-cache.md +++ b/designs/use-selectors-at-cache.md @@ -1,5 +1,5 @@ -Filter cache ListWatch using selectors -=================== +# Filter cache ListWatch using selectors + ## Motivation Controller-Runtime controllers use a cache to subscribe to events from @@ -32,6 +32,7 @@ This proposal is related to the following issue [2] Add a new selector code at `pkg/cache/internal/selector.go` with common structs and helpers + ```golang package internal @@ -53,14 +54,12 @@ type Selector struct { func (s Selector) ApplyToList(listOpts *metav1.ListOptions) { ... } - +``` Add a type alias to `pkg/cache/cache.go` to internal ```golang - type SelectorsByObject internal.SelectorsByObject - ``` Extend `cache.Options` as follows: @@ -79,7 +78,6 @@ Add new builder function that will return a cache constructor using the passed cache.Options, users can set SelectorsByObject there to filter out cache, it will convert SelectorByObject to SelectorsByGVK - ```golang func BuilderWithOptions(options cache.Options) NewCacheFunc { ... @@ -119,7 +117,6 @@ implications of using a different cache than the default one ) ``` - [1] https://github.com/nmstate/kubernetes-nmstate/pull/687 [2] https://github.com/kubernetes-sigs/controller-runtime/issues/244 [3] https://github.com/kubernetes-sigs/controller-runtime/pull/1404 diff --git a/doc.go b/doc.go index 758662085d..9b604d522b 100644 --- a/doc.go +++ b/doc.go @@ -23,13 +23,14 @@ limitations under the License. // and uncommon cases should be possible. In general, controller-runtime tries // to guide users towards Kubernetes controller best-practices. // -// Getting Started +// # Getting Started // // The main entrypoint for controller-runtime is this root package, which // contains all of the common types needed to get started building controllers: -// import ( -// ctrl "sigs.k8s.io/controller-runtime" -// ) +// +// import ( +// ctrl "sigs.k8s.io/controller-runtime" +// ) // // The examples in this package walk through a basic controller setup. The // kubebuilder book (https://book.kubebuilder.io) has some more in-depth @@ -38,7 +39,7 @@ limitations under the License. // controller-runtime favors structs with sane defaults over constructors, so // it's fairly common to see structs being used directly in controller-runtime. // -// Organization +// # Organization // // A brief-ish walkthrough of the layout of this library can be found below. Each // package contains more information about how to use it. @@ -47,18 +48,18 @@ limitations under the License. // controllers can be found at // https://github.com/kubernetes-sigs/controller-runtime/blob/master/FAQ.md. // -// Managers +// # Managers // // Every controller and webhook is ultimately run by a Manager (pkg/manager). A // manager is responsible for running controllers and webhooks, and setting up -// common dependencies (pkg/runtime/inject), like shared caches and clients, as +// common dependencies, like shared caches and clients, as // well as managing leader election (pkg/leaderelection). Managers are // generally configured to gracefully shut down controllers on pod termination // by wiring up a signal handler (pkg/manager/signals). // -// Controllers +// # Controllers // -// Controllers (pkg/controller) use events (pkg/events) to eventually trigger +// Controllers (pkg/controller) use events (pkg/event) to eventually trigger // reconcile requests. They may be constructed manually, but are often // constructed with a Builder (pkg/builder), which eases the wiring of event // sources (pkg/source), like Kubernetes API object changes, to event handlers @@ -67,7 +68,7 @@ limitations under the License. // trigger reconciles. There are pre-written utilities for the common cases, and // interfaces and helpers for advanced cases. // -// Reconcilers +// # Reconcilers // // Controller logic is implemented in terms of Reconcilers (pkg/reconcile). A // Reconciler implements a function which takes a reconcile Request containing @@ -75,7 +76,7 @@ limitations under the License. // and returns a Response or an error indicating whether to requeue for a // second round of processing. // -// Clients and Caches +// # Clients and Caches // // Reconcilers use Clients (pkg/client) to access API objects. The default // client provided by the manager reads from a local shared cache (pkg/cache) @@ -91,23 +92,23 @@ limitations under the License. // may retrieve event recorders (pkg/recorder) to emit events using the // manager. // -// Schemes +// # Schemes // // Clients, Caches, and many other things in Kubernetes use Schemes // (pkg/scheme) to associate Go types to Kubernetes API Kinds // (Group-Version-Kinds, to be specific). // -// Webhooks +// # Webhooks // // Similarly, webhooks (pkg/webhook/admission) may be implemented directly, but // are often constructed using a builder (pkg/webhook/admission/builder). They // are run via a server (pkg/webhook) which is managed by a Manager. // -// Logging and Metrics +// # Logging and Metrics // // Logging (pkg/log) in controller-runtime is done via structured logs, using a // log set of interfaces called logr -// (https://godoc.org/github.com/go-logr/logr). While controller-runtime +// (https://pkg.go.dev/github.com/go-logr/logr). While controller-runtime // provides easy setup for using Zap (https://go.uber.org/zap, pkg/log/zap), // you can provide any implementation of logr as the base logger for // controller-runtime. @@ -117,7 +118,7 @@ limitations under the License. // serve these by an HTTP endpoint, and additional metrics may be registered to // this Registry as normal. // -// Testing +// # Testing // // You can easily build integration and unit tests for your controllers and // webhooks using the test Environment (pkg/envtest). This will automatically diff --git a/example_test.go b/example_test.go index de4ae736a5..381959d5bb 100644 --- a/example_test.go +++ b/example_test.go @@ -26,6 +26,9 @@ import ( corev1 "k8s.io/api/core/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + + // since we invoke tests with -ginkgo.junit-report we need to import ginkgo. + _ "github.com/onsi/ginkgo/v2" ) // This example creates a simple application Controller that is configured for ReplicaSets and Pods. @@ -34,7 +37,6 @@ import ( // ReplicaSetReconciler. // // * Start the application. -// TODO(pwittrock): Update this example when we have better dependency injection support. func Example() { var log = ctrl.Log.WithName("builder-examples") @@ -63,16 +65,15 @@ func Example() { // This example creates a simple application Controller that is configured for ReplicaSets and Pods. // This application controller will be running leader election with the provided configuration in the manager options. // If leader election configuration is not provided, controller runs leader election with default values. -// Default values taken from: https://github.com/kubernetes/apiserver/blob/master/pkg/apis/config/v1alpha1/defaults.go -// defaultLeaseDuration = 15 * time.Second -// defaultRenewDeadline = 10 * time.Second -// defaultRetryPeriod = 2 * time.Second +// Default values taken from: https://github.com/kubernetes/component-base/blob/master/config/v1alpha1/defaults.go +// * defaultLeaseDuration = 15 * time.Second +// * defaultRenewDeadline = 10 * time.Second +// * defaultRetryPeriod = 2 * time.Second // // * Create a new application for ReplicaSets that manages Pods owned by the ReplicaSet and calls into // ReplicaSetReconciler. // // * Start the application. -// TODO(pwittrock): Update this example when we have better dependency injection support. func Example_updateLeaderElectionDurations() { var log = ctrl.Log.WithName("builder-examples") leaseDuration := 100 * time.Second @@ -135,7 +136,7 @@ func (a *ReplicaSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) // Update the ReplicaSet rs.Labels["pod-count"] = fmt.Sprintf("%v", len(pods.Items)) - err = a.Update(context.TODO(), rs) + err = a.Update(ctx, rs) if err != nil { return ctrl.Result{}, err } diff --git a/examples/builtins/main.go b/examples/builtins/main.go index ff1f0dfa3b..8ea173b248 100644 --- a/examples/builtins/main.go +++ b/examples/builtins/main.go @@ -22,6 +22,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -30,7 +31,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/manager/signals" "sigs.k8s.io/controller-runtime/pkg/source" - "sigs.k8s.io/controller-runtime/pkg/webhook" ) func init() { @@ -59,25 +59,26 @@ func main() { } // Watch ReplicaSets and enqueue ReplicaSet object key - if err := c.Watch(&source.Kind{Type: &appsv1.ReplicaSet{}}, &handler.EnqueueRequestForObject{}); err != nil { + if err := c.Watch(source.Kind(mgr.GetCache(), &appsv1.ReplicaSet{}), &handler.EnqueueRequestForObject{}); err != nil { entryLog.Error(err, "unable to watch ReplicaSets") os.Exit(1) } // Watch Pods and enqueue owning ReplicaSet key - if err := c.Watch(&source.Kind{Type: &corev1.Pod{}}, - &handler.EnqueueRequestForOwner{OwnerType: &appsv1.ReplicaSet{}, IsController: true}); err != nil { + if err := c.Watch(source.Kind(mgr.GetCache(), &corev1.Pod{}), + handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &appsv1.ReplicaSet{}, handler.OnlyControllerOwner())); err != nil { entryLog.Error(err, "unable to watch Pods") os.Exit(1) } - // Setup webhooks - entryLog.Info("setting up webhook server") - hookServer := mgr.GetWebhookServer() - - entryLog.Info("registering webhooks to the webhook server") - hookServer.Register("/mutate-v1-pod", &webhook.Admission{Handler: &podAnnotator{Client: mgr.GetClient()}}) - hookServer.Register("/validate-v1-pod", &webhook.Admission{Handler: &podValidator{Client: mgr.GetClient()}}) + if err := builder.WebhookManagedBy(mgr). + For(&corev1.Pod{}). + WithDefaulter(&podAnnotator{}). + WithValidator(&podValidator{}). + Complete(); err != nil { + entryLog.Error(err, "unable to create webhook", "webhook", "Pod") + os.Exit(1) + } entryLog.Info("starting manager") if err := mgr.Start(signals.SetupSignalHandler()); err != nil { diff --git a/examples/builtins/mutatingwebhook.go b/examples/builtins/mutatingwebhook.go index a4f4eee508..c3e3bc396a 100644 --- a/examples/builtins/mutatingwebhook.go +++ b/examples/builtins/mutatingwebhook.go @@ -18,49 +18,31 @@ package main import ( "context" - "encoding/json" - "net/http" + "fmt" corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + "k8s.io/apimachinery/pkg/runtime" + + logf "sigs.k8s.io/controller-runtime/pkg/log" ) // +kubebuilder:webhook:path=/mutate-v1-pod,mutating=true,failurePolicy=fail,groups="",resources=pods,verbs=create;update,versions=v1,name=mpod.kb.io // podAnnotator annotates Pods -type podAnnotator struct { - Client client.Client - decoder *admission.Decoder -} +type podAnnotator struct{} -// podAnnotator adds an annotation to every incoming pods. -func (a *podAnnotator) Handle(ctx context.Context, req admission.Request) admission.Response { - pod := &corev1.Pod{} - - err := a.decoder.Decode(req, pod) - if err != nil { - return admission.Errored(http.StatusBadRequest, err) +func (a *podAnnotator) Default(ctx context.Context, obj runtime.Object) error { + log := logf.FromContext(ctx) + pod, ok := obj.(*corev1.Pod) + if !ok { + return fmt.Errorf("expected a Pod but got a %T", obj) } if pod.Annotations == nil { pod.Annotations = map[string]string{} } pod.Annotations["example-mutating-admission-webhook"] = "foo" + log.Info("Annotated Pod") - marshaledPod, err := json.Marshal(pod) - if err != nil { - return admission.Errored(http.StatusInternalServerError, err) - } - - return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod) -} - -// podAnnotator implements admission.DecoderInjector. -// A decoder will be automatically injected. - -// InjectDecoder injects the decoder. -func (a *podAnnotator) InjectDecoder(d *admission.Decoder) error { - a.decoder = d return nil } diff --git a/examples/builtins/validatingwebhook.go b/examples/builtins/validatingwebhook.go index 57bc526574..e6094598bb 100644 --- a/examples/builtins/validatingwebhook.go +++ b/examples/builtins/validatingwebhook.go @@ -19,47 +19,47 @@ package main import ( "context" "fmt" - "net/http" corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + "k8s.io/apimachinery/pkg/runtime" + + logf "sigs.k8s.io/controller-runtime/pkg/log" ) // +kubebuilder:webhook:path=/validate-v1-pod,mutating=false,failurePolicy=fail,groups="",resources=pods,verbs=create;update,versions=v1,name=vpod.kb.io // podValidator validates Pods -type podValidator struct { - Client client.Client - decoder *admission.Decoder -} +type podValidator struct{} -// podValidator admits a pod if a specific annotation exists. -func (v *podValidator) Handle(ctx context.Context, req admission.Request) admission.Response { - pod := &corev1.Pod{} - - err := v.decoder.Decode(req, pod) - if err != nil { - return admission.Errored(http.StatusBadRequest, err) +// validate admits a pod if a specific annotation exists. +func (v *podValidator) validate(ctx context.Context, obj runtime.Object) error { + log := logf.FromContext(ctx) + pod, ok := obj.(*corev1.Pod) + if !ok { + return fmt.Errorf("expected a Pod but got a %T", obj) } + log.Info("Validating Pod") key := "example-mutating-admission-webhook" anno, found := pod.Annotations[key] if !found { - return admission.Denied(fmt.Sprintf("missing annotation %s", key)) + return fmt.Errorf("missing annotation %s", key) } if anno != "foo" { - return admission.Denied(fmt.Sprintf("annotation %s did not have value %q", key, "foo")) + return fmt.Errorf("annotation %s did not have value %q", key, "foo") } - return admission.Allowed("") + return nil +} + +func (v *podValidator) ValidateCreate(ctx context.Context, obj runtime.Object) error { + return v.validate(ctx, obj) } -// podValidator implements admission.DecoderInjector. -// A decoder will be automatically injected. +func (v *podValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error { + return v.validate(ctx, newObj) +} -// InjectDecoder injects the decoder. -func (v *podValidator) InjectDecoder(d *admission.Decoder) error { - v.decoder = d - return nil +func (v *podValidator) ValidateDelete(ctx context.Context, obj runtime.Object) error { + return v.validate(ctx, obj) } diff --git a/examples/configfile/custom/v1alpha1/zz_generated.deepcopy.go b/examples/configfile/custom/v1alpha1/zz_generated.deepcopy.go index eff37f0a37..b9d3b6b4b9 100644 --- a/examples/configfile/custom/v1alpha1/zz_generated.deepcopy.go +++ b/examples/configfile/custom/v1alpha1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated // Code generated by controller-gen. DO NOT EDIT. diff --git a/examples/scratch-env/main.go b/examples/scratch-env/main.go index 6be4d127c2..b8305ffed3 100644 --- a/examples/scratch-env/main.go +++ b/examples/scratch-env/main.go @@ -18,14 +18,14 @@ package main import ( goflag "flag" - "io/ioutil" "os" flag "github.com/spf13/pflag" + "go.uber.org/zap" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/log/zap" + logzap "sigs.k8s.io/controller-runtime/pkg/log/zap" ) var ( @@ -36,8 +36,9 @@ var ( // have a separate function so we can return an exit code w/o skipping defers func runMain() int { - loggerOpts := &zap.Options{ + loggerOpts := &logzap.Options{ Development: true, // a sane default + ZapOpts: []zap.Option{zap.AddCaller()}, } { var goFlagSet goflag.FlagSet @@ -45,7 +46,8 @@ func runMain() int { flag.CommandLine.AddGoFlagSet(&goFlagSet) } flag.Parse() - ctrl.SetLogger(zap.New(zap.UseFlagOptions(loggerOpts))) + ctrl.SetLogger(logzap.New(logzap.UseFlagOptions(loggerOpts))) + ctrl.Log.Info("Starting...") log := ctrl.Log.WithName("main") @@ -83,7 +85,7 @@ func runMain() int { } // TODO(directxman12): add support for writing to a new context in an existing file - kubeconfigFile, err := ioutil.TempFile("", "scratch-env-kubeconfig-") + kubeconfigFile, err := os.CreateTemp("", "scratch-env-kubeconfig-") if err != nil { log.Error(err, "unable to create kubeconfig file, continuing on without it") return 1 diff --git a/go.mod b/go.mod index 1423d76de0..5fe225193e 100644 --- a/go.mod +++ b/go.mod @@ -1,29 +1,71 @@ module sigs.k8s.io/controller-runtime -go 1.16 +go 1.19 require ( - github.com/evanphx/json-patch v4.11.0+incompatible - github.com/fsnotify/fsnotify v1.4.9 - github.com/go-logr/logr v0.4.0 - github.com/go-logr/zapr v0.4.0 - github.com/googleapis/gnostic v0.5.5 // indirect - github.com/hashicorp/golang-lru v0.5.4 // indirect - github.com/imdario/mergo v0.3.12 // indirect - github.com/onsi/ginkgo v1.16.4 - github.com/onsi/gomega v1.13.0 - github.com/prometheus/client_golang v1.11.0 - github.com/prometheus/client_model v0.2.0 - go.uber.org/goleak v1.1.10 - go.uber.org/zap v1.17.0 - golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 + github.com/evanphx/json-patch/v5 v5.6.0 + github.com/fsnotify/fsnotify v1.6.0 + github.com/go-logr/logr v1.2.3 + github.com/go-logr/zapr v1.2.3 + github.com/google/go-cmp v0.5.9 + github.com/onsi/ginkgo/v2 v2.8.1 + github.com/onsi/gomega v1.26.0 + github.com/prometheus/client_golang v1.14.0 + github.com/prometheus/client_model v0.3.0 + go.uber.org/goleak v1.2.1 + go.uber.org/zap v1.24.0 + golang.org/x/sys v0.5.0 + golang.org/x/time v0.3.0 gomodules.xyz/jsonpatch/v2 v2.2.0 + k8s.io/api v0.26.1 + k8s.io/apiextensions-apiserver v0.26.1 + k8s.io/apimachinery v0.26.1 + k8s.io/client-go v0.26.1 + k8s.io/component-base v0.26.1 + k8s.io/klog/v2 v2.90.0 + k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 + sigs.k8s.io/yaml v1.3.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/emicklei/go-restful/v3 v3.9.0 // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.20.0 // indirect + github.com/go-openapi/swag v0.19.14 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/google/gofuzz v1.1.0 // indirect + github.com/google/uuid v1.1.2 // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.6 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/common v0.37.0 // indirect + github.com/prometheus/procfs v0.8.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + golang.org/x/net v0.6.0 // indirect + golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect + golang.org/x/term v0.5.0 // indirect + golang.org/x/text v0.7.0 // indirect google.golang.org/appengine v1.6.7 // indirect - k8s.io/api v0.21.2 - k8s.io/apiextensions-apiserver v0.21.2 - k8s.io/apimachinery v0.21.2 - k8s.io/client-go v0.21.2 - k8s.io/component-base v0.21.2 - k8s.io/utils v0.0.0-20210527160623-6fdb442a123b - sigs.k8s.io/yaml v1.2.0 + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect + sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) diff --git a/go.sum b/go.sum index 12cf1b35ba..dd84fc2214 100644 --- a/go.sum +++ b/go.sum @@ -8,144 +8,117 @@ cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= -github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= +github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= -github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs= -github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= +github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM= -github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= +github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= +github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -159,84 +132,54 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= -github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= -github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -245,173 +188,106 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= +github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= +github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= -github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/onsi/ginkgo/v2 v2.8.1 h1:xFTEVwOFa1D/Ty24Ws1npBWkDYEV9BqZrsDxVrVkrrU= +github.com/onsi/ginkgo/v2 v2.8.1/go.mod h1:N1/NbDngAFcSLdyZ+/aYTYGSlq9qMCS/cNKGJjy+csc= +github.com/onsi/gomega v1.26.0 h1:03cDLK28U6hWvCAns6NeydX3zIm4SF3ci69ulidS32Q= +github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -433,7 +309,6 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -443,14 +318,9 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -460,9 +330,8 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -470,32 +339,39 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -504,15 +380,9 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -523,58 +393,59 @@ golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs= -golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -589,17 +460,22 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= @@ -612,12 +488,19 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -637,17 +520,31 @@ google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -659,26 +556,18 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -689,46 +578,38 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.21.2 h1:vz7DqmRsXTCSa6pNxXwQ1IYeAZgdIsua+DZU+o+SX3Y= -k8s.io/api v0.21.2/go.mod h1:Lv6UGJZ1rlMI1qusN8ruAp9PUBFyBwpEHAdG24vIsiU= -k8s.io/apiextensions-apiserver v0.21.2 h1:+exKMRep4pDrphEafRvpEi79wTnCFMqKf8LBtlA3yrE= -k8s.io/apiextensions-apiserver v0.21.2/go.mod h1:+Axoz5/l3AYpGLlhJDfcVQzCerVYq3K3CvDMvw6X1RA= -k8s.io/apimachinery v0.21.2 h1:vezUc/BHqWlQDnZ+XkrpXSmnANSLbpnlpwo0Lhk0gpc= -k8s.io/apimachinery v0.21.2/go.mod h1:CdTY8fU/BlvAbJ2z/8kBwimGki5Zp8/fbVuLY8gJumM= -k8s.io/apiserver v0.21.2/go.mod h1:lN4yBoGyiNT7SC1dmNk0ue6a5Wi6O3SWOIw91TsucQw= -k8s.io/client-go v0.21.2 h1:Q1j4L/iMN4pTw6Y4DWppBoUxgKO8LbffEMVEV00MUp0= -k8s.io/client-go v0.21.2/go.mod h1:HdJ9iknWpbl3vMGtib6T2PyI/VYxiZfq936WNVHBRrA= -k8s.io/code-generator v0.21.2/go.mod h1:8mXJDCB7HcRo1xiEQstcguZkbxZaqeUOrO9SsicWs3U= -k8s.io/component-base v0.21.2 h1:EsnmFFoJ86cEywC0DoIkAUiEV6fjgauNugiw1lmIjs4= -k8s.io/component-base v0.21.2/go.mod h1:9lvmIThzdlrJj5Hp8Z/TOgIkdfsNARQ1pT+3PByuiuc= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= -k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= -k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210527160623-6fdb442a123b h1:MSqsVQ3pZvPGTqCjptfimO2WjG7A9un2zcpiHkA6M/s= -k8s.io/utils v0.0.0-20210527160623-6fdb442a123b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.26.1 h1:f+SWYiPd/GsiWwVRz+NbFyCgvv75Pk9NK6dlkZgpCRQ= +k8s.io/api v0.26.1/go.mod h1:xd/GBNgR0f707+ATNyPmQ1oyKSgndzXij81FzWGsejg= +k8s.io/apiextensions-apiserver v0.26.1 h1:cB8h1SRk6e/+i3NOrQgSFij1B2S0Y0wDoNl66bn8RMI= +k8s.io/apiextensions-apiserver v0.26.1/go.mod h1:AptjOSXDGuE0JICx/Em15PaoO7buLwTs0dGleIHixSM= +k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ= +k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= +k8s.io/client-go v0.26.1 h1:87CXzYJnAMGaa/IDDfRdhTzxk/wzGZ+/HUQpqgVSZXU= +k8s.io/client-go v0.26.1/go.mod h1:IWNSglg+rQ3OcvDkhY6+QLeasV4OYHDjdqeWkDQZwGE= +k8s.io/component-base v0.26.1 h1:4ahudpeQXHZL5kko+iDHqLj/FSGAEUnSVO0EBbgDd+4= +k8s.io/component-base v0.26.1/go.mod h1:VHrLR0b58oC035w6YQiBSbtsf0ThuSwXP+p5dD/kAWU= +k8s.io/klog/v2 v2.90.0 h1:VkTxIV/FjRXn1fgNNcKGM8cfmL1Z33ZjXRTVxKCoF5M= +k8s.io/klog/v2 v2.90.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= +k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y= +k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= -sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/hack/check-everything.sh b/hack/check-everything.sh index b6d3472bcc..09f2a901d4 100755 --- a/hack/check-everything.sh +++ b/hack/check-everything.sh @@ -24,9 +24,11 @@ source ${hack_dir}/common.sh tmp_root=/tmp kb_root_dir=$tmp_root/kubebuilder -ENVTEST_K8S_VERSION=${ENVTEST_K8S_VERSION:-"1.21.2"} +# Run verification scripts. +${hack_dir}/verify.sh -# set up envtest tools if necessary +# Envtest. +ENVTEST_K8S_VERSION=${ENVTEST_K8S_VERSION:-"1.24.2"} header_text "installing envtest tools@${ENVTEST_K8S_VERSION} with setup-envtest if necessary" tmp_bin=/tmp/cr-tests-bin @@ -35,9 +37,9 @@ tmp_bin=/tmp/cr-tests-bin cd ${hack_dir}/../tools/setup-envtest GOBIN=${tmp_bin} go install . ) -source <(${tmp_bin}/setup-envtest use --use-env -p env ${ENVTEST_K8S_VERSION}) +export KUBEBUILDER_ASSETS="$(${tmp_bin}/setup-envtest use --use-env -p path "${ENVTEST_K8S_VERSION}")" -${hack_dir}/verify.sh +# Run tests. ${hack_dir}/test-all.sh header_text "confirming examples compile (via go install)" diff --git a/hack/ensure-golangci-lint.sh b/hack/ensure-golangci-lint.sh index 9e9ef03167..9210f959b0 100755 --- a/hack/ensure-golangci-lint.sh +++ b/hack/ensure-golangci-lint.sh @@ -103,6 +103,11 @@ get_binaries() { linux/mips64le) BINARIES="golangci-lint" ;; linux/ppc64le) BINARIES="golangci-lint" ;; linux/s390x) BINARIES="golangci-lint" ;; + linux/riscv64) BINARIES="golangci-lint" ;; + netbsd/386) BINARIES="golangci-lint" ;; + netbsd/amd64) BINARIES="golangci-lint" ;; + netbsd/armv6) BINARIES="golangci-lint" ;; + netbsd/armv7) BINARIES="golangci-lint" ;; windows/386) BINARIES="golangci-lint" ;; windows/amd64) BINARIES="golangci-lint" ;; windows/arm64) BINARIES="golangci-lint" ;; @@ -209,9 +214,10 @@ log_crit() { uname_os() { os=$(uname -s | tr '[:upper:]' '[:lower:]') case "$os" in - cygwin_nt*) os="windows" ;; + msys*) os="windows" ;; mingw*) os="windows" ;; - msys_nt*) os="windows" ;; + cygwin*) os="windows" ;; + win*) os="windows" ;; esac echo "$os" } @@ -244,7 +250,7 @@ uname_os_check() { solaris) return 0 ;; windows) return 0 ;; esac - log_crit "uname_os_check '$(uname -s)' got converted to '$os' which is not a GOOS value. Please file bug at https://github.com/client9/shlib" + log_crit "uname_os_check '$(uname -s)' got converted to '$os' which is not a GOOS value." return 1 } uname_arch_check() { @@ -263,9 +269,10 @@ uname_arch_check() { mips64) return 0 ;; mips64le) return 0 ;; s390x) return 0 ;; + riscv64) return 0 ;; amd64p32) return 0 ;; esac - log_crit "uname_arch_check '$(uname -m)' got converted to '$arch' which is not a GOARCH value. Please file bug report at https://github.com/client9/shlib" + log_crit "uname_arch_check '$(uname -m)' got converted to '$arch' which is not a GOARCH value." return 1 } untar() { @@ -327,11 +334,14 @@ http_copy() { github_release() { owner_repo=$1 version=$2 - test -z "$version" && version="latest" - giturl="https://github.com/${owner_repo}/releases/${version}" + if [ -z "$version" ]; then + giturl="https://api.github.com/repos/${owner_repo}/releases/latest" + else + giturl="https://api.github.com/repos/${owner_repo}/releases/tags/${version}" + fi json=$(http_copy "$giturl" "Accept:application/json") test -z "$json" && return 1 - version=$(echo "$json" | tr -s '\n' ' ' | sed 's/.*"tag_name":"//' | sed 's/".*//') + version=$(echo "$json" | tr -s '\n' ' ' | sed 's/.*"tag_name": "//' | sed 's/".*//') test -z "$version" && return 1 echo "$version" } diff --git a/hack/test-all.sh b/hack/test-all.sh index e538e78b63..34d841cfd0 100755 --- a/hack/test-all.sh +++ b/hack/test-all.sh @@ -20,20 +20,21 @@ source $(dirname ${BASH_SOURCE})/common.sh header_text "running go test" -# On MacOS there is a strange race condition -# between port allocation of envtest suites when go test -# runs all the tests in parallel without any limits (spins up around 10+ environments). -# -# To avoid flakes, set we're setting the go-test parallel flag to -# to limit the number of parallel executions. -# -# TODO(community): Investigate this behavior further. -if [[ "${OSTYPE}" == "darwin"* ]]; then - P_FLAG="-p=1" +if [[ -n ${ARTIFACTS:-} ]]; then + GINKGO_ARGS="-ginkgo.junit-report=junit-report.xml" fi -go test -race ${P_FLAG} ${MOD_OPT} ./... +result=0 +go test -v -race ${P_FLAG} ${MOD_OPT} ./... --ginkgo.fail-fast ${GINKGO_ARGS} || result=$? if [[ -n ${ARTIFACTS:-} ]]; then - if grep -Rin '' ${ARTIFACTS}/*; then exit 1; fi + mkdir -p ${ARTIFACTS} + for file in `find . -name *junit-report.xml`; do + new_file=${file#./} + new_file=${new_file%/junit-report.xml} + new_file=${new_file//"/"/"-"} + mv "$file" "$ARTIFACTS/junit_${new_file}.xml" + done fi + +exit $result diff --git a/hack/tools/go.mod b/hack/tools/go.mod index eb97ee1853..2ca09f77af 100644 --- a/hack/tools/go.mod +++ b/hack/tools/go.mod @@ -3,9 +3,6 @@ module sigs.k8s.io/controller-runtime/hack/tools go 1.16 require ( - github.com/joelanford/go-apidiff v0.1.0 - github.com/sergi/go-diff v1.1.0 // indirect - github.com/stretchr/testify v1.7.0 // indirect - golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5 // indirect - sigs.k8s.io/controller-tools v0.6.1 + github.com/joelanford/go-apidiff v0.5.0 + sigs.k8s.io/controller-tools v0.11.3 ) diff --git a/hack/tools/go.sum b/hack/tools/go.sum index 0fa3e73ac9..4b83d24705 100644 --- a/hack/tools/go.sum +++ b/hack/tools/go.sum @@ -9,42 +9,66 @@ cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6T cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= -github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= +github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 h1:YoJbenK9C67SkzkDfmQuVln04ygHj3vjZfd9FL+GmQQ= +github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= -github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= +github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= +github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220418222510-f25a4f6275ed/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= +github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -52,20 +76,37 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= +github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -73,13 +114,12 @@ github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8Nz github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -88,70 +128,103 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc= -github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= +github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= +github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-billy/v5 v5.3.1 h1:CPiOUAzKtMRvolEKw+bG1PLRpT7D3LIs3/3ey4Aiu34= +github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-git-fixtures/v4 v4.2.1 h1:n9gGL1Ct/yIw+nfsfr8s4+sbhT+Ncu2SubfXjIWgci8= +github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0= +github.com/go-git/go-git/v5 v5.4.2 h1:BXyZu9t0VkbiHtqrsvdq39UDhGJTl1h55VW6CSC4aY4= +github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/gobuffalo/flect v0.2.3 h1:f/ZukRnSNA/DUpSNDadko7Qc0PhGvsew35p/2tu+CRY= -github.com/gobuffalo/flect v0.2.3/go.mod h1:vmkQwuZYhN5Pc4ljYQZzP+1sq+NEkK+lh20jmEmX3jc= +github.com/gobuffalo/flect v0.3.0 h1:erfPWM+K1rFNIQeRPdeEXxo8yFr/PO17lhRnS8FUrtk= +github.com/gobuffalo/flect v0.3.0/go.mod h1:5pf3aGnsvqvCj50AVni7mJJF8ICxGZ8HomberC3pXLE= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -161,43 +234,65 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/cel-go v0.12.6/go.mod h1:Jk7ljRzLBhkmiAwBoUxB1sZSCVBAzkqPF25olK/iRDw= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -220,35 +315,46 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/joelanford/go-apidiff v0.1.0 h1:bt/247wfLDKFnCC5jYdapR3WY2laJMPB9apfc1U9Idw= -github.com/joelanford/go-apidiff v0.1.0/go.mod h1:wgVWgVCwYYkjcYpJtBnWYkyUYZfVovO3Y5pX49mJsqs= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/joelanford/go-apidiff v0.5.0 h1:vGTUv9jSAHNtvX+NEb0Oa5XPBceQPqY/W/L3Zf/hPWg= +github.com/joelanford/go-apidiff v0.5.0/go.mod h1:HhwH55VeVftiJaI08m+nyIySuZq1xHl5uBoGehKM/tI= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= -github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck= +github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -256,17 +362,18 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= +github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -277,40 +384,52 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= +github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= +github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= +github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= +github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= +github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= +github.com/onsi/ginkgo/v2 v2.6.1/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= -github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= +github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= +github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= +github.com/onsi/gomega v1.24.2 h1:J/tulyYK6JwBldPViHJReihxxZ+22FHs0piGjQAvoUE= +github.com/onsi/gomega v1.24.2/go.mod h1:gs3J10IS7Z7r7eXRoNJIrNqU4ToQukCJhFtKrWgHWnk= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -320,94 +439,144 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= -github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= +github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= -github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= -github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= +github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= +github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= +go.etcd.io/etcd/client/pkg/v3 v3.5.5/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= +go.etcd.io/etcd/client/v2 v2.305.5/go.mod h1:zQjKllfqfBVyVStbt4FaosoX2iYd8fV/GRy/PbowgP4= +go.etcd.io/etcd/client/v3 v3.5.5/go.mod h1:aApjR4WGlSumpnJ2kloS75h6aHUmAyaPLjHMxpc7E7c= +go.etcd.io/etcd/pkg/v3 v3.5.5/go.mod h1:6ksYFxttiUGzC2uxyqiyOEvhAiD0tuIqSZkX3TyPdaE= +go.etcd.io/etcd/raft/v3 v3.5.5/go.mod h1:76TA48q03g1y1VpTue92jZLr9lIHKUNcYdZOOGyx8rI= +go.etcd.io/etcd/server/v3 v3.5.5/go.mod h1:rZ95vDw/jrvsbj9XpTqPrTAB9/kzchVdhRirySPkUBc= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0/go.mod h1:h8TWwRAhQpOd0aM5nYsRD8+flnkj+526GEIVlarH7eY= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0/go.mod h1:9NiG9I2aHTKkcxqCILhjtyNA1QEiCjdBACv4IvrFQ+c= +go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= +go.opentelemetry.io/otel v1.8.0/go.mod h1:2pkj+iMj0o03Y+cW6/m8Y4WkRdYN3AvCXCnzRMp9yvM= +go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0/go.mod h1:78XhIg8Ht9vR4tbLNUhXsiOnE2HOuSeKAiAcoVQEpOY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.1/go.mod h1:Kv8liBeVNFkkkbilbgWRpV+wWuu+H5xdOT6HAgd30iw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0/go.mod h1:Krqnjl22jUJ0HgMzw5eveuCvFDXY4nSYb4F8t5gdrag= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.1/go.mod h1:xOvWoTOrQjxjW61xtOmD/WKGRYb/P4NzRo3bs65U6Rk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0/go.mod h1:OfUCyyIiDvNXHWpcWgbF+MWvqPZiNa3YDEnivcnYsV0= +go.opentelemetry.io/otel/metric v0.31.0/go.mod h1:ohmwj9KTSIeBnDBm/ZwH2PSZxZzoOaG2xZeekTRzL5A= +go.opentelemetry.io/otel/sdk v1.0.1/go.mod h1:HrdXne+BiwsOHYYkBE5ysIcv2bvdZstxzmCQhxTcZkI= +go.opentelemetry.io/otel/sdk v1.10.0/go.mod h1:vO06iKzD5baltJz1zarxMCNHFpUlUiOy4s65ECtn6kE= +go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= +go.opentelemetry.io/otel/trace v1.8.0/go.mod h1:0Bt3PXY8w+3pheS3hQUt+wow8b1ojPaTBoTCh2zIFI4= +go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSsOPU9brfSHJg= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -415,27 +584,26 @@ golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5 h1:FR+oGxGfbQu1d+jglI3rCkjAjUnhRSZcUxr+DqlDLNo= -golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/exp v0.0.0-20220921164117-439092de6870 h1:j8b6j9gzSigH28O5SjSpQSSh9lFd6f5D/q0aHjNTulc= +golang.org/x/exp v0.0.0-20220921164117-439092de6870/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -448,6 +616,8 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -456,9 +626,14 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -475,10 +650,9 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191003171128-d98b1b443823/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -486,25 +660,71 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -514,20 +734,17 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191003212358-c178f38b412c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -542,36 +759,81 @@ golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -582,16 +844,13 @@ golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191004183538-27eeabb02079/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -607,18 +866,40 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.3 h1:L69ShwSZEyCsLKoAxDKeMvLDZkumEe8gXUZAjab0tX8= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -629,12 +910,32 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -652,16 +953,79 @@ google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -674,13 +1038,16 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= @@ -689,12 +1056,6 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= -gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= -gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOAJK+LsJg= -gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= -gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE= -gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= @@ -702,6 +1063,7 @@ gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRN gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -709,8 +1071,10 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -719,34 +1083,38 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.21.2 h1:vz7DqmRsXTCSa6pNxXwQ1IYeAZgdIsua+DZU+o+SX3Y= -k8s.io/api v0.21.2/go.mod h1:Lv6UGJZ1rlMI1qusN8ruAp9PUBFyBwpEHAdG24vIsiU= -k8s.io/apiextensions-apiserver v0.21.2 h1:+exKMRep4pDrphEafRvpEi79wTnCFMqKf8LBtlA3yrE= -k8s.io/apiextensions-apiserver v0.21.2/go.mod h1:+Axoz5/l3AYpGLlhJDfcVQzCerVYq3K3CvDMvw6X1RA= -k8s.io/apimachinery v0.21.2 h1:vezUc/BHqWlQDnZ+XkrpXSmnANSLbpnlpwo0Lhk0gpc= -k8s.io/apimachinery v0.21.2/go.mod h1:CdTY8fU/BlvAbJ2z/8kBwimGki5Zp8/fbVuLY8gJumM= -k8s.io/apiserver v0.21.2/go.mod h1:lN4yBoGyiNT7SC1dmNk0ue6a5Wi6O3SWOIw91TsucQw= -k8s.io/client-go v0.21.2/go.mod h1:HdJ9iknWpbl3vMGtib6T2PyI/VYxiZfq936WNVHBRrA= -k8s.io/code-generator v0.21.2/go.mod h1:8mXJDCB7HcRo1xiEQstcguZkbxZaqeUOrO9SsicWs3U= -k8s.io/component-base v0.21.2/go.mod h1:9lvmIThzdlrJj5Hp8Z/TOgIkdfsNARQ1pT+3PByuiuc= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.26.1 h1:f+SWYiPd/GsiWwVRz+NbFyCgvv75Pk9NK6dlkZgpCRQ= +k8s.io/api v0.26.1/go.mod h1:xd/GBNgR0f707+ATNyPmQ1oyKSgndzXij81FzWGsejg= +k8s.io/apiextensions-apiserver v0.26.1 h1:cB8h1SRk6e/+i3NOrQgSFij1B2S0Y0wDoNl66bn8RMI= +k8s.io/apiextensions-apiserver v0.26.1/go.mod h1:AptjOSXDGuE0JICx/Em15PaoO7buLwTs0dGleIHixSM= +k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ= +k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= +k8s.io/apiserver v0.26.1/go.mod h1:wr75z634Cv+sifswE9HlAo5FQ7UoUauIICRlOE+5dCg= +k8s.io/client-go v0.26.1/go.mod h1:IWNSglg+rQ3OcvDkhY6+QLeasV4OYHDjdqeWkDQZwGE= +k8s.io/code-generator v0.26.1/go.mod h1:OMoJ5Dqx1wgaQzKgc+ZWaZPfGjdRq/Y3WubFrZmeI3I= +k8s.io/component-base v0.26.1/go.mod h1:VHrLR0b58oC035w6YQiBSbtsf0ThuSwXP+p5dD/kAWU= +k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20220902162205-c0856e24416d/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= -k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= +k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kms v0.26.1/go.mod h1:ReC1IEGuxgfN+PDCIpR6w8+XMmDE7uJhxcCwMZFdIYc= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= +k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20221107191617-1a15be271d1d h1:0Smp/HP1OH4Rvhe+4B8nWGERtlqAGSftbSbbmm45oFs= +k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/controller-tools v0.6.1 h1:nODRx2YrSNcaGd+90+CVC9SGEG6ygHlz3nSJmweR5as= -sigs.k8s.io/controller-tools v0.6.1/go.mod h1:U6O1RF5w17iX2d+teSXELpJsdexmrTb126DMeJM8r+U= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= -sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35/go.mod h1:WxjusMwXlKzfAs4p9km6XJRndVt2FROgMVCE4cdohFo= +sigs.k8s.io/controller-tools v0.11.3 h1:T1xzLkog9saiyQSLz1XOImu4OcbdXWytc5cmYsBeBiE= +sigs.k8s.io/controller-tools v0.11.3/go.mod h1:qcfX7jfcfYD/b7lAhvqAyTbt/px4GpvN88WKLFFv7p8= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/hack/tools/golangci-lint b/hack/tools/golangci-lint deleted file mode 100755 index 8a446d2e9e..0000000000 Binary files a/hack/tools/golangci-lint and /dev/null differ diff --git a/hack/verify.sh b/hack/verify.sh index e44299797c..ad48128e43 100755 --- a/hack/verify.sh +++ b/hack/verify.sh @@ -24,8 +24,22 @@ cd "${REPO_ROOT}" header_text "running generate" make generate +# Only run verify-generate in CI, otherwise running generate +# locally (which is a valid operation) causes `make test` to fail. +if [[ -n ${CI} ]]; then + header_text "verifying generate" + make verify-generate +fi + +header_text "running modules" +make modules + +# Only run verify-modules in CI, otherwise updating +# go module locally (which is a valid operation) causes `make test` to fail. +if [[ -n ${CI} ]]; then + header_text "verifying modules" + make verify-modules +fi + header_text "running golangci-lint" make lint - -header_text "verifying modules" -make modules verify-modules diff --git a/pkg/builder/builder_suite_test.go b/pkg/builder/builder_suite_test.go index 0dc260d4bf..aec31ddfcf 100644 --- a/pkg/builder/builder_suite_test.go +++ b/pkg/builder/builder_suite_test.go @@ -19,16 +19,15 @@ package builder import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" "sigs.k8s.io/controller-runtime/pkg/internal/testing/addr" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -38,14 +37,13 @@ import ( func TestBuilder(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "application Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "application Suite") } var testenv *envtest.Environment var cfg *rest.Config -var _ = BeforeSuite(func(done Done) { +var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) testenv = &envtest.Environment{} @@ -63,9 +61,7 @@ var _ = BeforeSuite(func(done Done) { webhook.DefaultPort, _, err = addr.Suggest("") Expect(err).NotTo(HaveOccurred()) - - close(done) -}, 60) +}) var _ = AfterSuite(func() { Expect(testenv.Stop()).To(Succeed()) @@ -80,27 +76,32 @@ var _ = AfterSuite(func() { func addCRDToEnvironment(env *envtest.Environment, gvks ...schema.GroupVersionKind) { for _, gvk := range gvks { plural, singular := meta.UnsafeGuessKindToResource(gvk) - crd := &apiextensionsv1beta1.CustomResourceDefinition{ + crd := &apiextensionsv1.CustomResourceDefinition{ TypeMeta: metav1.TypeMeta{ - APIVersion: "apiextensions.k8s.io/v1beta1", + APIVersion: "apiextensions.k8s.io/v1", Kind: "CustomResourceDefinition", }, ObjectMeta: metav1.ObjectMeta{ Name: plural.Resource + "." + gvk.Group, }, - Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{ - Group: gvk.Group, - Version: gvk.Version, - Names: apiextensionsv1beta1.CustomResourceDefinitionNames{ + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: gvk.Group, + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: plural.Resource, Singular: singular.Resource, Kind: gvk.Kind, }, - Versions: []apiextensionsv1beta1.CustomResourceDefinitionVersion{ + Scope: apiextensionsv1.NamespaceScoped, + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ { Name: gvk.Version, Served: true, Storage: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", + }, + }, }, }, }, diff --git a/pkg/builder/controller.go b/pkg/builder/controller.go index 2cd4ce9de1..570cfd63d0 100644 --- a/pkg/builder/controller.go +++ b/pkg/builder/controller.go @@ -17,17 +17,20 @@ limitations under the License. package builder import ( + "errors" "fmt" "strings" "github.com/go-logr/logr" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" + internalsource "sigs.k8s.io/controller-runtime/pkg/internal/source" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -94,14 +97,20 @@ func (blder *Builder) For(object client.Object, opts ...ForOption) *Builder { // OwnsInput represents the information set by Owns method. type OwnsInput struct { + matchEveryOwner bool object client.Object predicates []predicate.Predicate objectProjection objectProjection } // Owns defines types of Objects being *generated* by the ControllerManagedBy, and configures the ControllerManagedBy to respond to -// create / delete / update events by *reconciling the owner object*. This is the equivalent of calling -// Watches(&source.Kind{Type: }, &handler.EnqueueRequestForOwner{OwnerType: apiType, IsController: true}). +// create / delete / update events by *reconciling the owner object*. +// +// The default behavior reconciles only the first controller-type OwnerReference of the given type. +// Use Owns(object, builder.MatchEveryOwner) to reconcile all owners. +// +// By default, this is the equivalent of calling +// Watches(object, handler.EnqueueRequestForOwner([...], ownerType, OnlyControllerOwner())). func (blder *Builder) Owns(object client.Object, opts ...OwnsOption) *Builder { input := OwnsInput{object: object} for _, opt := range opts { @@ -120,10 +129,54 @@ type WatchesInput struct { objectProjection objectProjection } -// Watches exposes the lower-level ControllerManagedBy Watches functions through the builder. Consider using -// Owns or For instead of Watches directly. +// Watches defines the type of Object to watch, and configures the ControllerManagedBy to respond to create / delete / +// update events by *reconciling the object* with the given EventHandler. +// +// This is the equivalent of calling +// WatchesRawSource(source.Kind(scheme, object), eventhandler, opts...). +func (blder *Builder) Watches(object client.Object, eventhandler handler.EventHandler, opts ...WatchesOption) *Builder { + src := source.Kind(blder.mgr.GetCache(), object) + return blder.WatchesRawSource(src, eventhandler, opts...) +} + +// WatchesMetadata is the same as Watches, but forces the internal cache to only watch PartialObjectMetadata. +// +// This is useful when watching lots of objects, really big objects, or objects for which you only know +// the GVK, but not the structure. You'll need to pass metav1.PartialObjectMetadata to the client +// when fetching objects in your reconciler, otherwise you'll end up with a duplicate structured or unstructured cache. +// +// When watching a resource with metadata only, for example the v1.Pod, you should not Get and List using the v1.Pod type. +// Instead, you should use the special metav1.PartialObjectMetadata type. +// +// ❌ Incorrect: +// +// pod := &v1.Pod{} +// mgr.GetClient().Get(ctx, nsAndName, pod) +// +// ✅ Correct: +// +// pod := &metav1.PartialObjectMetadata{} +// pod.SetGroupVersionKind(schema.GroupVersionKind{ +// Group: "", +// Version: "v1", +// Kind: "Pod", +// }) +// mgr.GetClient().Get(ctx, nsAndName, pod) +// +// In the first case, controller-runtime will create another cache for the +// concrete type on top of the metadata cache; this increases memory +// consumption and leads to race conditions as caches are not in sync. +func (blder *Builder) WatchesMetadata(object client.Object, eventhandler handler.EventHandler, opts ...WatchesOption) *Builder { + opts = append(opts, OnlyMetadata) + return blder.Watches(object, eventhandler, opts...) +} + +// WatchesRawSource exposes the lower-level ControllerManagedBy Watches functions through the builder. // Specified predicates are registered only for given source. -func (blder *Builder) Watches(src source.Source, eventhandler handler.EventHandler, opts ...WatchesOption) *Builder { +// +// STOP! Consider using For(...), Owns(...), Watches(...), WatchesMetadata(...) instead. +// This method is only exposed for more advanced use cases, most users should use higher level functions. +func (blder *Builder) WatchesRawSource(src source.Source, eventhandler handler.EventHandler, opts ...WatchesOption) *Builder { input := WatchesInput{src: src, eventhandler: eventhandler} for _, opt := range opts { opt.ApplyToWatches(&input) @@ -148,9 +201,9 @@ func (blder *Builder) WithOptions(options controller.Options) *Builder { return blder } -// WithLogger overrides the controller options's logger used. -func (blder *Builder) WithLogger(log logr.Logger) *Builder { - blder.ctrlOptions.Log = log +// WithLogConstructor overrides the controller options's LogConstructor. +func (blder *Builder) WithLogConstructor(logConstructor func(*reconcile.Request) logr.Logger) *Builder { + blder.ctrlOptions.LogConstructor = logConstructor return blder } @@ -181,10 +234,6 @@ func (blder *Builder) Build(r reconcile.Reconciler) (controller.Controller, erro if blder.forInput.err != nil { return nil, blder.forInput.err } - // Checking the reconcile type exist or not - if blder.forInput.object == nil { - return nil, fmt.Errorf("must provide an object for reconciliation") - } // Set the ControllerManagedBy if err := blder.doController(r); err != nil { @@ -218,28 +267,38 @@ func (blder *Builder) project(obj client.Object, proj objectProjection) (client. func (blder *Builder) doWatch() error { // Reconcile type - typeForSrc, err := blder.project(blder.forInput.object, blder.forInput.objectProjection) - if err != nil { - return err - } - src := &source.Kind{Type: typeForSrc} - hdler := &handler.EnqueueRequestForObject{} - allPredicates := append(blder.globalPredicates, blder.forInput.predicates...) - if err := blder.ctrl.Watch(src, hdler, allPredicates...); err != nil { - return err + if blder.forInput.object != nil { + obj, err := blder.project(blder.forInput.object, blder.forInput.objectProjection) + if err != nil { + return err + } + src := source.Kind(blder.mgr.GetCache(), obj) + hdler := &handler.EnqueueRequestForObject{} + allPredicates := append(blder.globalPredicates, blder.forInput.predicates...) + if err := blder.ctrl.Watch(src, hdler, allPredicates...); err != nil { + return err + } } // Watches the managed types + if len(blder.ownsInput) > 0 && blder.forInput.object == nil { + return errors.New("Owns() can only be used together with For()") + } for _, own := range blder.ownsInput { - typeForSrc, err := blder.project(own.object, own.objectProjection) + obj, err := blder.project(own.object, own.objectProjection) if err != nil { return err } - src := &source.Kind{Type: typeForSrc} - hdler := &handler.EnqueueRequestForOwner{ - OwnerType: blder.forInput.object, - IsController: true, + src := source.Kind(blder.mgr.GetCache(), obj) + opts := []handler.OwnerOption{} + if !own.matchEveryOwner { + opts = append(opts, handler.OnlyControllerOwner()) } + hdler := handler.EnqueueRequestForOwner( + blder.mgr.GetScheme(), blder.mgr.GetRESTMapper(), + blder.forInput.object, + opts..., + ) allPredicates := append([]predicate.Predicate(nil), blder.globalPredicates...) allPredicates = append(allPredicates, own.predicates...) if err := blder.ctrl.Watch(src, hdler, allPredicates...); err != nil { @@ -248,12 +307,15 @@ func (blder *Builder) doWatch() error { } // Do the watch requests + if len(blder.watchesInput) == 0 && blder.forInput.object == nil { + return errors.New("there are no watches configured, controller will never get triggered. Use For(), Owns() or Watches() to set them up") + } for _, w := range blder.watchesInput { allPredicates := append([]predicate.Predicate(nil), blder.globalPredicates...) allPredicates = append(allPredicates, w.predicates...) - // If the source of this watch is of type *source.Kind, project it. - if srckind, ok := w.src.(*source.Kind); ok { + // If the source of this watch is of type Kind, project it. + if srckind, ok := w.src.(*internalsource.Kind); ok { typeForSrc, err := blder.project(srckind.Type, w.objectProjection) if err != nil { return err @@ -268,11 +330,14 @@ func (blder *Builder) doWatch() error { return nil } -func (blder *Builder) getControllerName(gvk schema.GroupVersionKind) string { +func (blder *Builder) getControllerName(gvk schema.GroupVersionKind, hasGVK bool) (string, error) { if blder.name != "" { - return blder.name + return blder.name, nil + } + if !hasGVK { + return "", errors.New("one of For() or Named() must be called") } - return strings.ToLower(gvk.Kind) + return strings.ToLower(gvk.Kind), nil } func (blder *Builder) doController(r reconcile.Reconciler) error { @@ -285,13 +350,18 @@ func (blder *Builder) doController(r reconcile.Reconciler) error { // Retrieve the GVK from the object we're reconciling // to prepopulate logger information, and to optionally generate a default name. - gvk, err := getGvk(blder.forInput.object, blder.mgr.GetScheme()) - if err != nil { - return err + var gvk schema.GroupVersionKind + hasGVK := blder.forInput.object != nil + if hasGVK { + var err error + gvk, err = getGvk(blder.forInput.object, blder.mgr.GetScheme()) + if err != nil { + return err + } } // Setup concurrency. - if ctrlOptions.MaxConcurrentReconciles == 0 { + if ctrlOptions.MaxConcurrentReconciles == 0 && hasGVK { groupKind := gvk.GroupKind().String() if concurrency, ok := globalOpts.GroupKindConcurrency[groupKind]; ok && concurrency > 0 { @@ -300,17 +370,42 @@ func (blder *Builder) doController(r reconcile.Reconciler) error { } // Setup cache sync timeout. - if ctrlOptions.CacheSyncTimeout == 0 && globalOpts.CacheSyncTimeout != nil { - ctrlOptions.CacheSyncTimeout = *globalOpts.CacheSyncTimeout + if ctrlOptions.CacheSyncTimeout == 0 && globalOpts.CacheSyncTimeout > 0 { + ctrlOptions.CacheSyncTimeout = globalOpts.CacheSyncTimeout + } + + controllerName, err := blder.getControllerName(gvk, hasGVK) + if err != nil { + return err } // Setup the logger. - if ctrlOptions.Log == nil { - ctrlOptions.Log = blder.mgr.GetLogger() + if ctrlOptions.LogConstructor == nil { + log := blder.mgr.GetLogger().WithValues( + "controller", controllerName, + ) + if hasGVK { + log = log.WithValues( + "controllerGroup", gvk.Group, + "controllerKind", gvk.Kind, + ) + } + + ctrlOptions.LogConstructor = func(req *reconcile.Request) logr.Logger { + log := log + if req != nil { + if hasGVK { + log = log.WithValues(gvk.Kind, klog.KRef(req.Namespace, req.Name)) + } + log = log.WithValues( + "namespace", req.Namespace, "name", req.Name, + ) + } + return log + } } - ctrlOptions.Log = ctrlOptions.Log.WithValues("reconciler group", gvk.Group, "reconciler kind", gvk.Kind) // Build the controller and return. - blder.ctrl, err = newController(blder.getControllerName(gvk), blder.mgr, ctrlOptions) + blder.ctrl, err = newController(controllerName, blder.mgr, ctrlOptions) return err } diff --git a/pkg/builder/controller_test.go b/pkg/builder/controller_test.go index ca486fbacd..eb5ae8197d 100644 --- a/pkg/builder/controller_test.go +++ b/pkg/builder/controller_test.go @@ -23,7 +23,7 @@ import ( "sync/atomic" "github.com/go-logr/logr" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -33,10 +33,11 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" "k8s.io/client-go/util/workqueue" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -44,7 +45,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/scheme" - "sigs.k8s.io/controller-runtime/pkg/source" ) type typedNoop struct{} @@ -57,10 +57,21 @@ type testLogger struct { logr.Logger } -func (l *testLogger) WithName(_ string) logr.Logger { +func (l *testLogger) Init(logr.RuntimeInfo) { +} + +func (l *testLogger) Enabled(int) bool { + return true +} + +func (l *testLogger) Info(level int, msg string, keysAndValues ...interface{}) { +} + +func (l *testLogger) WithValues(keysAndValues ...interface{}) logr.LogSink { return l } -func (l *testLogger) WithValues(_ ...interface{}) logr.Logger { + +func (l *testLogger) WithName(name string) logr.LogSink { return l } @@ -101,16 +112,55 @@ var _ = Describe("application", func() { Expect(instance).To(BeNil()) }) - It("should return an error if For function is not called", func() { + It("should return an error if For and Named function are not called", func() { + By("creating a controller manager") + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + instance, err := ControllerManagedBy(m). + Watches(&appsv1.ReplicaSet{}, &handler.EnqueueRequestForObject{}). + Build(noop) + Expect(err).To(MatchError(ContainSubstring("one of For() or Named() must be called"))) + Expect(instance).To(BeNil()) + }) + + It("should return an error when using Owns without For", func() { By("creating a controller manager") m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) instance, err := ControllerManagedBy(m). + Named("my_controller"). Owns(&appsv1.ReplicaSet{}). Build(noop) - Expect(err).To(MatchError(ContainSubstring("must provide an object for reconciliation"))) + Expect(err).To(MatchError(ContainSubstring("Owns() can only be used together with For()"))) Expect(instance).To(BeNil()) + + }) + + It("should return an error when there are no watches", func() { + By("creating a controller manager") + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + instance, err := ControllerManagedBy(m). + Named("my_controller"). + Build(noop) + Expect(err).To(MatchError(ContainSubstring("there are no watches configured, controller will never get triggered. Use For(), Owns() or Watches() to set them up"))) + Expect(instance).To(BeNil()) + }) + + It("should allow creating a controllerw without calling For", func() { + By("creating a controller manager") + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + instance, err := ControllerManagedBy(m). + Named("my_controller"). + Watches(&appsv1.ReplicaSet{}, &handler.EnqueueRequestForObject{}). + Build(noop) + Expect(err).NotTo(HaveOccurred()) + Expect(instance).NotTo(BeNil()) }) It("should return an error if there is no GVK for an object, and thus we can't default the controller name", func() { @@ -167,7 +217,7 @@ var _ = Describe("application", func() { instance, err := ControllerManagedBy(m). For(&appsv1.ReplicaSet{}). Owns(&appsv1.ReplicaSet{}). - WithOptions(controller.Options{MaxConcurrentReconciles: maxConcurrentReconciles}). + WithOptions(controller.Options{Controller: config.Controller{MaxConcurrentReconciles: maxConcurrentReconciles}}). Build(noop) Expect(err).NotTo(HaveOccurred()) Expect(instance).NotTo(BeNil()) @@ -185,7 +235,7 @@ var _ = Describe("application", func() { By("creating a controller manager") m, err := manager.New(cfg, manager.Options{ - Controller: v1alpha1.ControllerConfigurationSpec{ + Controller: config.Controller{ GroupKindConcurrency: map[string]int{ "ReplicaSet.apps": maxConcurrentReconciles, }, @@ -227,10 +277,10 @@ var _ = Describe("application", func() { logger := &testLogger{} newController = func(name string, mgr manager.Manager, options controller.Options) (controller.Controller, error) { - if options.Log == logger { + if options.LogConstructor(nil).GetSink() == logger { return controller.New(name, mgr, options) } - return nil, fmt.Errorf("logger expected %T but found %T", logger, options.Log) + return nil, fmt.Errorf("logger expected %T but found %T", logger, options.LogConstructor) } By("creating a controller manager") @@ -240,7 +290,9 @@ var _ = Describe("application", func() { instance, err := ControllerManagedBy(m). For(&appsv1.ReplicaSet{}). Owns(&appsv1.ReplicaSet{}). - WithLogger(logger). + WithLogConstructor(func(request *reconcile.Request) logr.Logger { + return logr.New(logger) + }). Build(noop) Expect(err).NotTo(HaveOccurred()) Expect(instance).NotTo(BeNil()) @@ -297,7 +349,7 @@ var _ = Describe("application", func() { }) Describe("Start with ControllerManagedBy", func() { - It("should Reconcile Owns objects", func(done Done) { + It("should Reconcile Owns objects", func() { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -307,29 +359,59 @@ var _ = Describe("application", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - doReconcileTest(ctx, "3", bldr, m, false) - close(done) - }, 10) + doReconcileTest(ctx, "3", m, false, bldr) + }) - It("should Reconcile Watches objects", func(done Done) { + It("should Reconcile Owns objects for every owner", func() { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) bldr := ControllerManagedBy(m). For(&appsv1.Deployment{}). + Owns(&appsv1.ReplicaSet{}, MatchEveryOwner) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + doReconcileTest(ctx, "12", m, false, bldr) + }) + + It("should Reconcile Watches objects", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + bldr := ControllerManagedBy(m). + For(&appsv1.Deployment{}). + Watches( // Equivalent of Owns + &appsv1.ReplicaSet{}, + handler.EnqueueRequestForOwner(m.GetScheme(), m.GetRESTMapper(), &appsv1.Deployment{}, handler.OnlyControllerOwner()), + ) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + doReconcileTest(ctx, "4", m, true, bldr) + }) + + It("should Reconcile without For", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + bldr := ControllerManagedBy(m). + Named("Deployment"). + Watches( // Equivalent of For + &appsv1.Deployment{}, &handler.EnqueueRequestForObject{}). Watches( // Equivalent of Owns - &source.Kind{Type: &appsv1.ReplicaSet{}}, - &handler.EnqueueRequestForOwner{OwnerType: &appsv1.Deployment{}, IsController: true}) + &appsv1.ReplicaSet{}, + handler.EnqueueRequestForOwner(m.GetScheme(), m.GetRESTMapper(), &appsv1.Deployment{}, handler.OnlyControllerOwner()), + ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - doReconcileTest(ctx, "4", bldr, m, true) - close(done) - }, 10) + doReconcileTest(ctx, "9", m, true, bldr) + }) }) Describe("Set custom predicates", func() { - It("should execute registered predicates only for assigned kind", func(done Done) { + It("should execute registered predicates only for assigned kind", func() { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -380,13 +462,11 @@ var _ = Describe("application", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - doReconcileTest(ctx, "5", bldr, m, true) + doReconcileTest(ctx, "5", m, true, bldr) Expect(deployPrctExecuted).To(BeTrue(), "Deploy predicated should be called at least once") Expect(replicaSetPrctExecuted).To(BeTrue(), "ReplicaSet predicated should be called at least once") Expect(allPrctExecuted).To(BeNumerically(">=", 2), "Global Predicated should be called at least twice") - - close(done) }) }) @@ -400,14 +480,24 @@ var _ = Describe("application", func() { Expect(err).NotTo(HaveOccurred()) }) + It("should support multiple controllers watching the same metadata kind", func() { + bldr1 := ControllerManagedBy(mgr).For(&appsv1.Deployment{}, OnlyMetadata) + bldr2 := ControllerManagedBy(mgr).For(&appsv1.Deployment{}, OnlyMetadata) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + doReconcileTest(ctx, "6", mgr, true, bldr1, bldr2) + }) + It("should support watching For, Owns, and Watch as metadata", func() { statefulSetMaps := make(chan *metav1.PartialObjectMetadata) bldr := ControllerManagedBy(mgr). For(&appsv1.Deployment{}, OnlyMetadata). Owns(&appsv1.ReplicaSet{}, OnlyMetadata). - Watches(&source.Kind{Type: &appsv1.StatefulSet{}}, - handler.EnqueueRequestsFromMapFunc(func(o client.Object) []reconcile.Request { + Watches(&appsv1.StatefulSet{}, + handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []reconcile.Request { defer GinkgoRecover() ometa := o.(*metav1.PartialObjectMetadata) @@ -425,7 +515,7 @@ var _ = Describe("application", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - doReconcileTest(ctx, "8", bldr, mgr, true) + doReconcileTest(ctx, "8", mgr, true, bldr) By("Creating a new stateful set") set := &appsv1.StatefulSet{ @@ -500,7 +590,7 @@ func (c *nonTypedOnlyCache) GetInformerForKind(ctx context.Context, gvk schema.G // TODO(directxman12): this function has too many arguments, and the whole // "nameSuffix" think is a bit of a hack It should be cleaned up significantly by someone with a bit of time. -func doReconcileTest(ctx context.Context, nameSuffix string, blder *Builder, mgr manager.Manager, complete bool) { +func doReconcileTest(ctx context.Context, nameSuffix string, mgr manager.Manager, complete bool, blders ...*Builder) { deployName := "deploy-name-" + nameSuffix rsName := "rs-name-" + nameSuffix @@ -516,22 +606,23 @@ func doReconcileTest(ctx context.Context, nameSuffix string, blder *Builder, mgr return reconcile.Result{}, nil }) - if complete { - err := blder.Complete(fn) - Expect(err).NotTo(HaveOccurred()) - } else { - var err error - var c controller.Controller - c, err = blder.Build(fn) - Expect(err).NotTo(HaveOccurred()) - Expect(c).NotTo(BeNil()) + for _, blder := range blders { + if complete { + err := blder.Complete(fn) + Expect(err).NotTo(HaveOccurred()) + } else { + var err error + var c controller.Controller + c, err = blder.Build(fn) + Expect(err).NotTo(HaveOccurred()) + Expect(c).NotTo(BeNil()) + } } By("Starting the application") go func() { defer GinkgoRecover() Expect(mgr.Start(ctx)).NotTo(HaveOccurred()) - By("Stopping the application") }() By("Creating a Deployment") @@ -558,7 +649,7 @@ func doReconcileTest(ctx context.Context, nameSuffix string, blder *Builder, mgr }, }, } - err := mgr.GetClient().Create(context.TODO(), dep) + err := mgr.GetClient().Create(ctx, dep) Expect(err).NotTo(HaveOccurred()) By("Waiting for the Deployment Reconcile") @@ -567,7 +658,6 @@ func doReconcileTest(ctx context.Context, nameSuffix string, blder *Builder, mgr By("Creating a ReplicaSet") // Expect a Reconcile when an Owned object is managedObjects. - t := true rs := &appsv1.ReplicaSet{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", @@ -578,7 +668,7 @@ func doReconcileTest(ctx context.Context, nameSuffix string, blder *Builder, mgr Name: deployName, Kind: "Deployment", APIVersion: "apps/v1", - Controller: &t, + Controller: pointer.Bool(true), UID: dep.UID, }, }, @@ -588,7 +678,7 @@ func doReconcileTest(ctx context.Context, nameSuffix string, blder *Builder, mgr Template: dep.Spec.Template, }, } - err = mgr.GetClient().Create(context.TODO(), rs) + err = mgr.GetClient().Create(ctx, rs) Expect(err).NotTo(HaveOccurred()) By("Waiting for the ReplicaSet Reconcile") diff --git a/pkg/builder/doc.go b/pkg/builder/doc.go index 09126576b2..e4df1b709f 100644 --- a/pkg/builder/doc.go +++ b/pkg/builder/doc.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package builder provides wraps other controller-runtime libraries and exposes simple +// Package builder wraps other controller-runtime libraries and exposes simple // patterns for building common Controllers. // // Projects built with the builder package can trivially be rebased on top of the underlying diff --git a/pkg/builder/example_test.go b/pkg/builder/example_test.go index 955c46b562..652c9b5833 100644 --- a/pkg/builder/example_test.go +++ b/pkg/builder/example_test.go @@ -107,7 +107,9 @@ func ExampleBuilder() { ControllerManagedBy(mgr). // Create the ControllerManagedBy For(&appsv1.ReplicaSet{}). // ReplicaSet is the Application API Owns(&corev1.Pod{}). // ReplicaSet owns Pods created by it - Complete(&ReplicaSetReconciler{}) + Complete(&ReplicaSetReconciler{ + Client: mgr.GetClient(), + }) if err != nil { log.Error(err, "could not create controller") os.Exit(1) @@ -155,8 +157,3 @@ func (a *ReplicaSetReconciler) Reconcile(ctx context.Context, req reconcile.Requ return reconcile.Result{}, nil } - -func (a *ReplicaSetReconciler) InjectClient(c client.Client) error { - a.Client = c - return nil -} diff --git a/pkg/builder/options.go b/pkg/builder/options.go index 7bb4273094..bce2065efa 100644 --- a/pkg/builder/options.go +++ b/pkg/builder/options.go @@ -101,12 +101,35 @@ func (p projectAs) ApplyToWatches(opts *WatchesInput) { var ( // OnlyMetadata tells the controller to *only* cache metadata, and to watch - // the the API server in metadata-only form. This is useful when watching + // the API server in metadata-only form. This is useful when watching // lots of objects, really big objects, or objects for which you only know - // the the GVK, but not the structure. You'll need to pass + // the GVK, but not the structure. You'll need to pass // metav1.PartialObjectMetadata to the client when fetching objects in your // reconciler, otherwise you'll end up with a duplicate structured or // unstructured cache. + // + // When watching a resource with OnlyMetadata, for example the v1.Pod, you + // should not Get and List using the v1.Pod type. Instead, you should use + // the special metav1.PartialObjectMetadata type. + // + // ❌ Incorrect: + // + // pod := &v1.Pod{} + // mgr.GetClient().Get(ctx, nsAndName, pod) + // + // ✅ Correct: + // + // pod := &metav1.PartialObjectMetadata{} + // pod.SetGroupVersionKind(schema.GroupVersionKind{ + // Group: "", + // Version: "v1", + // Kind: "Pod", + // }) + // mgr.GetClient().Get(ctx, nsAndName, pod) + // + // In the first case, controller-runtime will create another cache for the + // concrete type on top of the metadata cache; this increases memory + // consumption and leads to race conditions as caches are not in sync. OnlyMetadata = projectAs(projectAsMetadata) _ ForOption = OnlyMetadata @@ -115,3 +138,19 @@ var ( ) // }}} + +// MatchEveryOwner determines whether the watch should be filtered based on +// controller ownership. As in, when the OwnerReference.Controller field is set. +// +// If passed as an option, +// the handler receives notification for every owner of the object with the given type. +// If unset (default), the handler receives notification only for the first +// OwnerReference with `Controller: true`. +var MatchEveryOwner = &matchEveryOwner{} + +type matchEveryOwner struct{} + +// ApplyToOwns applies this configuration to the given OwnsInput options. +func (o matchEveryOwner) ApplyToOwns(opts *OwnsInput) { + opts.matchEveryOwner = true +} diff --git a/pkg/builder/webhook.go b/pkg/builder/webhook.go index d24877d303..4cb971cea4 100644 --- a/pkg/builder/webhook.go +++ b/pkg/builder/webhook.go @@ -17,13 +17,17 @@ limitations under the License. package builder import ( + "errors" "net/http" "net/url" "strings" + "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -32,10 +36,14 @@ import ( // WebhookBuilder builds a Webhook. type WebhookBuilder struct { - apiType runtime.Object - gvk schema.GroupVersionKind - mgr manager.Manager - config *rest.Config + apiType runtime.Object + withDefaulter admission.CustomDefaulter + withValidator admission.CustomValidator + gvk schema.GroupVersionKind + mgr manager.Manager + config *rest.Config + recoverPanic bool + logConstructor func(base logr.Logger, req *admission.Request) logr.Logger } // WebhookManagedBy allows inform its manager.Manager. @@ -53,11 +61,38 @@ func (blder *WebhookBuilder) For(apiType runtime.Object) *WebhookBuilder { return blder } +// WithDefaulter takes a admission.WithDefaulter interface, a MutatingWebhook will be wired for this type. +func (blder *WebhookBuilder) WithDefaulter(defaulter admission.CustomDefaulter) *WebhookBuilder { + blder.withDefaulter = defaulter + return blder +} + +// WithValidator takes a admission.WithValidator interface, a ValidatingWebhook will be wired for this type. +func (blder *WebhookBuilder) WithValidator(validator admission.CustomValidator) *WebhookBuilder { + blder.withValidator = validator + return blder +} + +// WithLogConstructor overrides the webhook's LogConstructor. +func (blder *WebhookBuilder) WithLogConstructor(logConstructor func(base logr.Logger, req *admission.Request) logr.Logger) *WebhookBuilder { + blder.logConstructor = logConstructor + return blder +} + +// RecoverPanic indicates whether the panic caused by webhook should be recovered. +func (blder *WebhookBuilder) RecoverPanic() *WebhookBuilder { + blder.recoverPanic = true + return blder +} + // Complete builds the webhook. func (blder *WebhookBuilder) Complete() error { // Set the Config blder.loadRestConfig() + // Configure the default LogConstructor + blder.setLogConstructor() + // Set the Webhook if needed return blder.registerWebhooks() } @@ -68,10 +103,33 @@ func (blder *WebhookBuilder) loadRestConfig() { } } +func (blder *WebhookBuilder) setLogConstructor() { + if blder.logConstructor == nil { + blder.logConstructor = func(base logr.Logger, req *admission.Request) logr.Logger { + log := base.WithValues( + "webhookGroup", blder.gvk.Group, + "webhookKind", blder.gvk.Kind, + ) + if req != nil { + return log.WithValues( + blder.gvk.Kind, klog.KRef(req.Namespace, req.Name), + "namespace", req.Namespace, "name", req.Name, + "resource", req.Resource, "user", req.UserInfo.Username, + ) + } + return log + } + } +} + func (blder *WebhookBuilder) registerWebhooks() error { + typ, err := blder.getType() + if err != nil { + return err + } + // Create webhook(s) for each type - var err error - blder.gvk, err = apiutil.GVKForObject(blder.apiType, blder.mgr.GetScheme()) + blder.gvk, err = apiutil.GVKForObject(typ, blder.mgr.GetScheme()) if err != nil { return err } @@ -88,13 +146,9 @@ func (blder *WebhookBuilder) registerWebhooks() error { // registerDefaultingWebhook registers a defaulting webhook if th. func (blder *WebhookBuilder) registerDefaultingWebhook() { - defaulter, isDefaulter := blder.apiType.(admission.Defaulter) - if !isDefaulter { - log.Info("skip registering a mutating webhook, admission.Defaulter interface is not implemented", "GVK", blder.gvk) - return - } - mwh := admission.DefaultingWebhookFor(defaulter) + mwh := blder.getDefaultingWebhook() if mwh != nil { + mwh.LogConstructor = blder.logConstructor path := generateMutatePath(blder.gvk) // Checking if the path is already registered. @@ -108,14 +162,23 @@ func (blder *WebhookBuilder) registerDefaultingWebhook() { } } -func (blder *WebhookBuilder) registerValidatingWebhook() { - validator, isValidator := blder.apiType.(admission.Validator) - if !isValidator { - log.Info("skip registering a validating webhook, admission.Validator interface is not implemented", "GVK", blder.gvk) - return +func (blder *WebhookBuilder) getDefaultingWebhook() *admission.Webhook { + if defaulter := blder.withDefaulter; defaulter != nil { + return admission.WithCustomDefaulter(blder.mgr.GetScheme(), blder.apiType, defaulter).WithRecoverPanic(blder.recoverPanic) } - vwh := admission.ValidatingWebhookFor(validator) + if defaulter, ok := blder.apiType.(admission.Defaulter); ok { + return admission.DefaultingWebhookFor(blder.mgr.GetScheme(), defaulter).WithRecoverPanic(blder.recoverPanic) + } + log.Info( + "skip registering a mutating webhook, object does not implement admission.Defaulter or WithDefaulter wasn't called", + "GVK", blder.gvk) + return nil +} + +func (blder *WebhookBuilder) registerValidatingWebhook() { + vwh := blder.getValidatingWebhook() if vwh != nil { + vwh.LogConstructor = blder.logConstructor path := generateValidatePath(blder.gvk) // Checking if the path is already registered. @@ -129,22 +192,42 @@ func (blder *WebhookBuilder) registerValidatingWebhook() { } } +func (blder *WebhookBuilder) getValidatingWebhook() *admission.Webhook { + if validator := blder.withValidator; validator != nil { + return admission.WithCustomValidator(blder.mgr.GetScheme(), blder.apiType, validator).WithRecoverPanic(blder.recoverPanic) + } + if validator, ok := blder.apiType.(admission.Validator); ok { + return admission.ValidatingWebhookFor(blder.mgr.GetScheme(), validator).WithRecoverPanic(blder.recoverPanic) + } + log.Info( + "skip registering a validating webhook, object does not implement admission.Validator or WithValidator wasn't called", + "GVK", blder.gvk) + return nil +} + func (blder *WebhookBuilder) registerConversionWebhook() error { ok, err := conversion.IsConvertible(blder.mgr.GetScheme(), blder.apiType) if err != nil { - log.Error(err, "conversion check failed", "object", blder.apiType) + log.Error(err, "conversion check failed", "GVK", blder.gvk) return err } if ok { if !blder.isAlreadyHandled("/convert") { - blder.mgr.GetWebhookServer().Register("/convert", &conversion.Webhook{}) + blder.mgr.GetWebhookServer().Register("/convert", conversion.NewWebhookHandler(blder.mgr.GetScheme())) } - log.Info("conversion webhook enabled", "object", blder.apiType) + log.Info("Conversion webhook enabled", "GVK", blder.gvk) } return nil } +func (blder *WebhookBuilder) getType() (runtime.Object, error) { + if blder.apiType != nil { + return blder.apiType, nil + } + return nil, errors.New("For() must be called with a valid object") +} + func (blder *WebhookBuilder) isAlreadyHandled(path string) bool { if blder.mgr.GetWebhookServer().WebhookMux == nil { return false diff --git a/pkg/builder/webhook_test.go b/pkg/builder/webhook_test.go index 53fc95468d..2ee1e7bfb4 100644 --- a/pkg/builder/webhook_test.go +++ b/pkg/builder/webhook_test.go @@ -20,18 +20,23 @@ import ( "context" "errors" "fmt" + "io" "net/http" "net/http/httptest" "os" "strings" - . "github.com/onsi/ginkgo" + "github.com/go-logr/logr" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/onsi/gomega/gbytes" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/controller" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/scheme" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -49,11 +54,17 @@ var _ = Describe("webhook", func() { }) func runTests(admissionReviewVersion string) { - var stop chan struct{} + var ( + stop chan struct{} + logBuffer *gbytes.Buffer + testingLogger logr.Logger + ) BeforeEach(func() { stop = make(chan struct{}) newController = controller.New + logBuffer = gbytes.NewBuffer() + testingLogger = zap.New(zap.JSONEncoder(), zap.WriteTo(io.MultiWriter(logBuffer, GinkgoWriter))) }) AfterEach(func() { @@ -104,8 +115,6 @@ func runTests(admissionReviewVersion string) { ctx, cancel := context.WithCancel(context.Background()) cancel() - // TODO: we may want to improve it to make it be able to inject dependencies, - // but not always try to load certs and return not found error. err = svr.Start(ctx) if err != nil && !os.IsNotExist(err) { ExpectWithOffset(1, err).NotTo(HaveOccurred()) @@ -134,6 +143,148 @@ func runTests(admissionReviewVersion string) { ExpectWithOffset(1, w.Code).To(Equal(http.StatusNotFound)) }) + It("should scaffold a defaulting webhook which recovers from panics", func() { + By("creating a controller manager") + m, err := manager.New(cfg, manager.Options{}) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("registering the type in the Scheme") + builder := scheme.Builder{GroupVersion: testDefaulterGVK.GroupVersion()} + builder.Register(&TestDefaulter{}, &TestDefaulterList{}) + err = builder.AddToScheme(m.GetScheme()) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + err = WebhookManagedBy(m). + For(&TestDefaulter{Panic: true}). + RecoverPanic(). + Complete() + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + svr := m.GetWebhookServer() + ExpectWithOffset(1, svr).NotTo(BeNil()) + + reader := strings.NewReader(`{ + "kind":"AdmissionReview", + "apiVersion":"admission.k8s.io/` + admissionReviewVersion + `", + "request":{ + "uid":"07e52e8d-4513-11e9-a716-42010a800270", + "kind":{ + "group":"", + "version":"v1", + "kind":"TestDefaulter" + }, + "resource":{ + "group":"", + "version":"v1", + "resource":"testdefaulter" + }, + "namespace":"default", + "operation":"CREATE", + "object":{ + "replica":1, + "panic":true + }, + "oldObject":null + } +}`) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err = svr.Start(ctx) + if err != nil && !os.IsNotExist(err) { + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + } + + By("sending a request to a mutating webhook path") + path := generateMutatePath(testDefaulterGVK) + req := httptest.NewRequest("POST", "http://svc-name.svc-ns.svc"+path, reader) + req.Header.Add("Content-Type", "application/json") + w := httptest.NewRecorder() + svr.WebhookMux.ServeHTTP(w, req) + ExpectWithOffset(1, w.Code).To(Equal(http.StatusOK)) + By("sanity checking the response contains reasonable fields") + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"allowed":false`)) + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"code":500`)) + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"message":"panic: fake panic test [recovered]`)) + }) + + It("should scaffold a defaulting webhook with a custom defaulter", func() { + By("creating a controller manager") + m, err := manager.New(cfg, manager.Options{}) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("registering the type in the Scheme") + builder := scheme.Builder{GroupVersion: testDefaulterGVK.GroupVersion()} + builder.Register(&TestDefaulter{}, &TestDefaulterList{}) + err = builder.AddToScheme(m.GetScheme()) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + err = WebhookManagedBy(m). + WithDefaulter(&TestCustomDefaulter{}). + For(&TestDefaulter{}). + WithLogConstructor(func(base logr.Logger, req *admission.Request) logr.Logger { + return admission.DefaultLogConstructor(testingLogger, req) + }). + Complete() + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + svr := m.GetWebhookServer() + ExpectWithOffset(1, svr).NotTo(BeNil()) + + reader := strings.NewReader(`{ + "kind":"AdmissionReview", + "apiVersion":"admission.k8s.io/` + admissionReviewVersion + `", + "request":{ + "uid":"07e52e8d-4513-11e9-a716-42010a800270", + "kind":{ + "group":"foo.test.org", + "version":"v1", + "kind":"TestDefaulter" + }, + "resource":{ + "group":"foo.test.org", + "version":"v1", + "resource":"testdefaulter" + }, + "namespace":"default", + "name":"foo", + "operation":"CREATE", + "object":{ + "replica":1 + }, + "oldObject":null + } +}`) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err = svr.Start(ctx) + if err != nil && !os.IsNotExist(err) { + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + } + + By("sending a request to a mutating webhook path") + path := generateMutatePath(testDefaulterGVK) + req := httptest.NewRequest("POST", "http://svc-name.svc-ns.svc"+path, reader) + req.Header.Add("Content-Type", "application/json") + w := httptest.NewRecorder() + svr.WebhookMux.ServeHTTP(w, req) + ExpectWithOffset(1, w.Code).To(Equal(http.StatusOK)) + By("sanity checking the response contains reasonable fields") + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"allowed":true`)) + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"patch":`)) + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"code":200`)) + EventuallyWithOffset(1, logBuffer).Should(gbytes.Say(`"msg":"Defaulting object","object":{"name":"foo","namespace":"default"},"namespace":"default","name":"foo","resource":{"group":"foo.test.org","version":"v1","resource":"testdefaulter"},"user":"","requestID":"07e52e8d-4513-11e9-a716-42010a800270"`)) + + By("sending a request to a validating webhook path that doesn't exist") + path = generateValidatePath(testDefaulterGVK) + _, err = reader.Seek(0, 0) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + req = httptest.NewRequest("POST", "http://svc-name.svc-ns.svc"+path, reader) + req.Header.Add("Content-Type", "application/json") + w = httptest.NewRecorder() + svr.WebhookMux.ServeHTTP(w, req) + ExpectWithOffset(1, w.Code).To(Equal(http.StatusNotFound)) + }) + It("should scaffold a validating webhook if the type implements the Validator interface", func() { By("creating a controller manager") m, err := manager.New(cfg, manager.Options{}) @@ -180,8 +331,6 @@ func runTests(admissionReviewVersion string) { ctx, cancel := context.WithCancel(context.Background()) cancel() - // TODO: we may want to improve it to make it be able to inject dependencies, - // but not always try to load certs and return not found error. err = svr.Start(ctx) if err != nil && !os.IsNotExist(err) { ExpectWithOffset(1, err).NotTo(HaveOccurred()) @@ -209,6 +358,150 @@ func runTests(admissionReviewVersion string) { ExpectWithOffset(1, w.Body).To(ContainSubstring(`"code":403`)) }) + It("should scaffold a validating webhook which recovers from panics", func() { + By("creating a controller manager") + m, err := manager.New(cfg, manager.Options{}) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("registering the type in the Scheme") + builder := scheme.Builder{GroupVersion: testValidatorGVK.GroupVersion()} + builder.Register(&TestValidator{}, &TestValidatorList{}) + err = builder.AddToScheme(m.GetScheme()) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + err = WebhookManagedBy(m). + For(&TestValidator{Panic: true}). + RecoverPanic(). + Complete() + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + svr := m.GetWebhookServer() + ExpectWithOffset(1, svr).NotTo(BeNil()) + + reader := strings.NewReader(`{ + "kind":"AdmissionReview", + "apiVersion":"admission.k8s.io/` + admissionReviewVersion + `", + "request":{ + "uid":"07e52e8d-4513-11e9-a716-42010a800270", + "kind":{ + "group":"", + "version":"v1", + "kind":"TestValidator" + }, + "resource":{ + "group":"", + "version":"v1", + "resource":"testvalidator" + }, + "namespace":"default", + "operation":"CREATE", + "object":{ + "replica":2, + "panic":true + } + } +}`) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err = svr.Start(ctx) + if err != nil && !os.IsNotExist(err) { + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + } + + By("sending a request to a validating webhook path") + path := generateValidatePath(testValidatorGVK) + _, err = reader.Seek(0, 0) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + req := httptest.NewRequest("POST", "http://svc-name.svc-ns.svc"+path, reader) + req.Header.Add("Content-Type", "application/json") + w := httptest.NewRecorder() + svr.WebhookMux.ServeHTTP(w, req) + ExpectWithOffset(1, w.Code).To(Equal(http.StatusOK)) + By("sanity checking the response contains reasonable field") + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"allowed":false`)) + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"code":500`)) + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"message":"panic: fake panic test [recovered]`)) + }) + + It("should scaffold a validating webhook with a custom validator", func() { + By("creating a controller manager") + m, err := manager.New(cfg, manager.Options{}) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + By("registering the type in the Scheme") + builder := scheme.Builder{GroupVersion: testValidatorGVK.GroupVersion()} + builder.Register(&TestValidator{}, &TestValidatorList{}) + err = builder.AddToScheme(m.GetScheme()) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + err = WebhookManagedBy(m). + WithValidator(&TestCustomValidator{}). + For(&TestValidator{}). + WithLogConstructor(func(base logr.Logger, req *admission.Request) logr.Logger { + return admission.DefaultLogConstructor(testingLogger, req) + }). + Complete() + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + svr := m.GetWebhookServer() + ExpectWithOffset(1, svr).NotTo(BeNil()) + + reader := strings.NewReader(`{ + "kind":"AdmissionReview", + "apiVersion":"admission.k8s.io/` + admissionReviewVersion + `", + "request":{ + "uid":"07e52e8d-4513-11e9-a716-42010a800270", + "kind":{ + "group":"foo.test.org", + "version":"v1", + "kind":"TestValidator" + }, + "resource":{ + "group":"foo.test.org", + "version":"v1", + "resource":"testvalidator" + }, + "namespace":"default", + "name":"foo", + "operation":"UPDATE", + "object":{ + "replica":1 + }, + "oldObject":{ + "replica":2 + } + } +}`) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err = svr.Start(ctx) + if err != nil && !os.IsNotExist(err) { + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + } + + By("sending a request to a mutating webhook path that doesn't exist") + path := generateMutatePath(testValidatorGVK) + req := httptest.NewRequest("POST", "http://svc-name.svc-ns.svc"+path, reader) + req.Header.Add("Content-Type", "application/json") + w := httptest.NewRecorder() + svr.WebhookMux.ServeHTTP(w, req) + ExpectWithOffset(1, w.Code).To(Equal(http.StatusNotFound)) + + By("sending a request to a validating webhook path") + path = generateValidatePath(testValidatorGVK) + _, err = reader.Seek(0, 0) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + req = httptest.NewRequest("POST", "http://svc-name.svc-ns.svc"+path, reader) + req.Header.Add("Content-Type", "application/json") + w = httptest.NewRecorder() + svr.WebhookMux.ServeHTTP(w, req) + ExpectWithOffset(1, w.Code).To(Equal(http.StatusOK)) + By("sanity checking the response contains reasonable field") + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"allowed":false`)) + ExpectWithOffset(1, w.Body).To(ContainSubstring(`"code":403`)) + EventuallyWithOffset(1, logBuffer).Should(gbytes.Say(`"msg":"Validating object","object":{"name":"foo","namespace":"default"},"namespace":"default","name":"foo","resource":{"group":"foo.test.org","version":"v1","resource":"testvalidator"},"user":"","requestID":"07e52e8d-4513-11e9-a716-42010a800270"`)) + }) + It("should scaffold defaulting and validating webhooks if the type implements both Defaulter and Validator interfaces", func() { By("creating a controller manager") m, err := manager.New(cfg, manager.Options{}) @@ -253,8 +546,6 @@ func runTests(admissionReviewVersion string) { ctx, cancel := context.WithCancel(context.Background()) cancel() - // TODO: we may want to improve it to make it be able to inject dependencies, - // but not always try to load certs and return not found error. err = svr.Start(ctx) if err != nil && !os.IsNotExist(err) { ExpectWithOffset(1, err).NotTo(HaveOccurred()) @@ -331,8 +622,6 @@ func runTests(admissionReviewVersion string) { }`) cancel() - // TODO: we may want to improve it to make it be able to inject dependencies, - // but not always try to load certs and return not found error. err = svr.Start(ctx) if err != nil && !os.IsNotExist(err) { ExpectWithOffset(1, err).NotTo(HaveOccurred()) @@ -388,11 +677,14 @@ func runTests(admissionReviewVersion string) { // TestDefaulter. var _ runtime.Object = &TestDefaulter{} +const testDefaulterKind = "TestDefaulter" + type TestDefaulter struct { - Replica int `json:"replica,omitempty"` + Replica int `json:"replica,omitempty"` + Panic bool `json:"panic,omitempty"` } -var testDefaulterGVK = schema.GroupVersionKind{Group: "foo.test.org", Version: "v1", Kind: "TestDefaulter"} +var testDefaulterGVK = schema.GroupVersionKind{Group: "foo.test.org", Version: "v1", Kind: testDefaulterKind} func (d *TestDefaulter) GetObjectKind() schema.ObjectKind { return d } func (d *TestDefaulter) DeepCopyObject() runtime.Object { @@ -415,6 +707,9 @@ func (*TestDefaulterList) GetObjectKind() schema.ObjectKind { return nil } func (*TestDefaulterList) DeepCopyObject() runtime.Object { return nil } func (d *TestDefaulter) Default() { + if d.Panic { + panic("fake panic test") + } if d.Replica < 2 { d.Replica = 2 } @@ -423,11 +718,14 @@ func (d *TestDefaulter) Default() { // TestValidator. var _ runtime.Object = &TestValidator{} +const testValidatorKind = "TestValidator" + type TestValidator struct { - Replica int `json:"replica,omitempty"` + Replica int `json:"replica,omitempty"` + Panic bool `json:"panic,omitempty"` } -var testValidatorGVK = schema.GroupVersionKind{Group: "foo.test.org", Version: "v1", Kind: "TestValidator"} +var testValidatorGVK = schema.GroupVersionKind{Group: "foo.test.org", Version: "v1", Kind: testValidatorKind} func (v *TestValidator) GetObjectKind() schema.ObjectKind { return v } func (v *TestValidator) DeepCopyObject() runtime.Object { @@ -452,6 +750,9 @@ func (*TestValidatorList) DeepCopyObject() runtime.Object { return nil } var _ admission.Validator = &TestValidator{} func (v *TestValidator) ValidateCreate() error { + if v.Panic { + panic("fake panic test") + } if v.Replica < 0 { return errors.New("number of replica should be greater than or equal to 0") } @@ -459,6 +760,9 @@ func (v *TestValidator) ValidateCreate() error { } func (v *TestValidator) ValidateUpdate(old runtime.Object) error { + if v.Panic { + panic("fake panic test") + } if v.Replica < 0 { return errors.New("number of replica should be greater than or equal to 0") } @@ -471,6 +775,9 @@ func (v *TestValidator) ValidateUpdate(old runtime.Object) error { } func (v *TestValidator) ValidateDelete() error { + if v.Panic { + panic("fake panic test") + } if v.Replica > 0 { return errors.New("number of replica should be less than or equal to 0 to delete") } @@ -537,3 +844,87 @@ func (dv *TestDefaultValidator) ValidateDelete() error { } return nil } + +// TestCustomDefaulter. + +type TestCustomDefaulter struct{} + +func (*TestCustomDefaulter) Default(ctx context.Context, obj runtime.Object) error { + logf.FromContext(ctx).Info("Defaulting object") + req, err := admission.RequestFromContext(ctx) + if err != nil { + return fmt.Errorf("expected admission.Request in ctx: %w", err) + } + if req.Kind.Kind != testDefaulterKind { + return fmt.Errorf("expected Kind TestDefaulter got %q", req.Kind.Kind) + } + + d := obj.(*TestDefaulter) //nolint:ifshort + if d.Replica < 2 { + d.Replica = 2 + } + return nil +} + +var _ admission.CustomDefaulter = &TestCustomDefaulter{} + +// TestCustomValidator. + +type TestCustomValidator struct{} + +func (*TestCustomValidator) ValidateCreate(ctx context.Context, obj runtime.Object) error { + logf.FromContext(ctx).Info("Validating object") + req, err := admission.RequestFromContext(ctx) + if err != nil { + return fmt.Errorf("expected admission.Request in ctx: %w", err) + } + if req.Kind.Kind != testValidatorKind { + return fmt.Errorf("expected Kind TestValidator got %q", req.Kind.Kind) + } + + v := obj.(*TestValidator) //nolint:ifshort + if v.Replica < 0 { + return errors.New("number of replica should be greater than or equal to 0") + } + return nil +} + +func (*TestCustomValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error { + logf.FromContext(ctx).Info("Validating object") + req, err := admission.RequestFromContext(ctx) + if err != nil { + return fmt.Errorf("expected admission.Request in ctx: %w", err) + } + if req.Kind.Kind != testValidatorKind { + return fmt.Errorf("expected Kind TestValidator got %q", req.Kind.Kind) + } + + v := newObj.(*TestValidator) + old := oldObj.(*TestValidator) + if v.Replica < 0 { + return errors.New("number of replica should be greater than or equal to 0") + } + if v.Replica < old.Replica { + return fmt.Errorf("new replica %v should not be fewer than old replica %v", v.Replica, old.Replica) + } + return nil +} + +func (*TestCustomValidator) ValidateDelete(ctx context.Context, obj runtime.Object) error { + logf.FromContext(ctx).Info("Validating object") + req, err := admission.RequestFromContext(ctx) + if err != nil { + return fmt.Errorf("expected admission.Request in ctx: %w", err) + } + if req.Kind.Kind != testValidatorKind { + return fmt.Errorf("expected Kind TestValidator got %q", req.Kind.Kind) + } + + v := obj.(*TestValidator) //nolint:ifshort + if v.Replica > 0 { + return errors.New("number of replica should be less than or equal to 0 to delete") + } + return nil +} + +var _ admission.CustomValidator = &TestCustomValidator{} diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 6862fd62bd..f74e8e1077 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -19,21 +19,30 @@ package cache import ( "context" "fmt" + "net/http" + "reflect" "time" "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" toolscache "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/cache/internal" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" logf "sigs.k8s.io/controller-runtime/pkg/internal/log" ) -var log = logf.RuntimeLog.WithName("object-cache") +var ( + log = logf.RuntimeLog.WithName("object-cache") + defaultResyncTime = 10 * time.Hour +) // Cache knows how to load Kubernetes objects, fetch informers to request // to receive events for Kubernetes objects (at a low-level), @@ -74,11 +83,19 @@ type Informer interface { // AddEventHandler adds an event handler to the shared informer using the shared informer's resync // period. Events to a single handler are delivered sequentially, but there is no coordination // between different handlers. - AddEventHandler(handler toolscache.ResourceEventHandler) + // It returns a registration handle for the handler that can be used to remove + // the handler again. + AddEventHandler(handler toolscache.ResourceEventHandler) (toolscache.ResourceEventHandlerRegistration, error) // AddEventHandlerWithResyncPeriod adds an event handler to the shared informer using the // specified resync period. Events to a single handler are delivered sequentially, but there is // no coordination between different handlers. - AddEventHandlerWithResyncPeriod(handler toolscache.ResourceEventHandler, resyncPeriod time.Duration) + // It returns a registration handle for the handler that can be used to remove + // the handler again and an error if the handler cannot be added. + AddEventHandlerWithResyncPeriod(handler toolscache.ResourceEventHandler, resyncPeriod time.Duration) (toolscache.ResourceEventHandlerRegistration, error) + // RemoveEventHandler removes a formerly added event handler given by + // its registration handle. + // This function is guaranteed to be idempotent, and thread-safe. + RemoveEventHandler(handle toolscache.ResourceEventHandlerRegistration) error // AddIndexers adds more indexers to this store. If you call this after you already have data // in the store, the results are undefined. AddIndexers(indexers toolscache.Indexers) error @@ -86,36 +103,73 @@ type Informer interface { HasSynced() bool } -// SelectorsByObject associate a client.Object's GVK to a field/label selector. -type SelectorsByObject map[client.Object]internal.Selector - // Options are the optional arguments for creating a new InformersMap object. type Options struct { + // HTTPClient is the http client to use for the REST client + HTTPClient *http.Client + // Scheme is the scheme to use for mapping objects to GroupVersionKinds Scheme *runtime.Scheme // Mapper is the RESTMapper to use for mapping GroupVersionKinds to Resources Mapper meta.RESTMapper - // Resync is the base frequency the informers are resynced. + // ResyncEvery is the base frequency the informers are resynced. // Defaults to defaultResyncTime. - // A 10 percent jitter will be added to the Resync period between informers + // A 10 percent jitter will be added to the ResyncEvery period between informers // So that all informers will not send list requests simultaneously. - Resync *time.Duration + ResyncEvery *time.Duration - // Namespace restricts the cache's ListWatch to the desired namespace + // Namespaces restricts the cache's ListWatch to the desired namespaces // Default watches all namespaces - Namespace string + Namespaces []string + + // DefaultLabelSelector will be used as a label selectors for all object types + // unless they have a more specific selector set in ByObject. + DefaultLabelSelector labels.Selector + + // DefaultFieldSelector will be used as a field selectors for all object types + // unless they have a more specific selector set in ByObject. + DefaultFieldSelector fields.Selector - // SelectorsByObject restricts the cache's ListWatch to the desired - // fields per GVK at the specified object, the map's value must implement - // Selector [1] using for example a Set [2] - // [1] https://pkg.go.dev/k8s.io/apimachinery/pkg/fields#Selector - // [2] https://pkg.go.dev/k8s.io/apimachinery/pkg/fields#Set - SelectorsByObject SelectorsByObject + // DefaultTransform will be used as transform for all object types + // unless they have a more specific transform set in ByObject. + DefaultTransform toolscache.TransformFunc + + // ByObject restricts the cache's ListWatch to the desired fields per GVK at the specified object. + ByObject map[client.Object]ByObject + + // UnsafeDisableDeepCopy indicates not to deep copy objects during get or + // list objects for EVERY object. + // Be very careful with this, when enabled you must DeepCopy any object before mutating it, + // otherwise you will mutate the object in the cache. + // + // This is a global setting for all objects, and can be overridden by the ByObject setting. + UnsafeDisableDeepCopy *bool } -var defaultResyncTime = 10 * time.Hour +// ByObject offers more fine-grained control over the cache's ListWatch by object. +type ByObject struct { + // Label represents a label selector for the object. + Label labels.Selector + + // Field represents a field selector for the object. + Field fields.Selector + + // Transform is a map from objects to transformer functions which + // get applied when objects of the transformation are about to be committed + // to cache. + // + // This function is called both for new objects to enter the cache, + // and for updated objects. + Transform toolscache.TransformFunc + + // UnsafeDisableDeepCopy indicates not to deep copy objects during get or + // list objects per GVK at the specified object. + // Be very careful with this, when enabled you must DeepCopy any object before mutating it, + // otherwise you will mutate the object in the cache. + UnsafeDisableDeepCopy *bool +} // New initializes and returns a new Cache. func New(config *rest.Config, opts Options) (Cache, error) { @@ -123,39 +177,229 @@ func New(config *rest.Config, opts Options) (Cache, error) { if err != nil { return nil, err } - selectorsByGVK, err := convertToSelectorsByGVK(opts.SelectorsByObject, opts.Scheme) + if len(opts.Namespaces) == 0 { + opts.Namespaces = []string{metav1.NamespaceAll} + } + if len(opts.Namespaces) > 1 { + return newMultiNamespaceCache(config, opts) + } + byGVK, err := convertToInformerOptsByGVK(opts.ByObject, opts.Scheme) if err != nil { return nil, err } - im := internal.NewInformersMap(config, opts.Scheme, opts.Mapper, *opts.Resync, opts.Namespace, selectorsByGVK) - return &informerCache{InformersMap: im}, nil + // Set the default selector and transform. + byGVK[schema.GroupVersionKind{}] = internal.InformersOptsByGVK{ + Selector: internal.Selector{ + Label: opts.DefaultLabelSelector, + Field: opts.DefaultFieldSelector, + }, + Transform: opts.DefaultTransform, + UnsafeDisableDeepCopy: opts.UnsafeDisableDeepCopy, + } + + return &informerCache{ + scheme: opts.Scheme, + Informers: internal.NewInformers(config, &internal.InformersOpts{ + HTTPClient: opts.HTTPClient, + Scheme: opts.Scheme, + Mapper: opts.Mapper, + ResyncPeriod: *opts.ResyncEvery, + Namespace: opts.Namespaces[0], + ByGVK: byGVK, + }), + }, nil } -// BuilderWithOptions returns a Cache constructor that will build the a cache +// BuilderWithOptions returns a Cache constructor that will build a cache // honoring the options argument, this is useful to specify options like -// SelectorsByObject -// WARNING: if SelectorsByObject is specified. filtered out resources are not -// returned. +// ByObjects, DefaultSelector, DefaultTransform, etc. +// WARNING: If ByObject selectors are specified, filtered out resources are not +// returned. +// WARNING: If ByObject UnsafeDisableDeepCopy is enabled, you must DeepCopy any object +// returned from cache get/list before mutating it. func BuilderWithOptions(options Options) NewCacheFunc { - return func(config *rest.Config, opts Options) (Cache, error) { - if opts.Scheme == nil { - opts.Scheme = options.Scheme + return func(config *rest.Config, inherited Options) (Cache, error) { + var err error + inherited, err = defaultOpts(config, inherited) + if err != nil { + return nil, err } - if opts.Mapper == nil { - opts.Mapper = options.Mapper + options, err = defaultOpts(config, options) + if err != nil { + return nil, err } - if opts.Resync == nil { - opts.Resync = options.Resync + combined, err := options.inheritFrom(inherited) + if err != nil { + return nil, err } - if opts.Namespace == "" { - opts.Namespace = options.Namespace + return New(config, *combined) + } +} + +func (options Options) inheritFrom(inherited Options) (*Options, error) { + var ( + combined Options + err error + ) + combined.Scheme = combineScheme(inherited.Scheme, options.Scheme) + combined.Mapper = selectMapper(inherited.Mapper, options.Mapper) + combined.ResyncEvery = selectResync(inherited.ResyncEvery, options.ResyncEvery) + combined.Namespaces = selectNamespaces(inherited.Namespaces, options.Namespaces) + combined.DefaultLabelSelector = combineSelector( + internal.Selector{Label: inherited.DefaultLabelSelector}, + internal.Selector{Label: options.DefaultLabelSelector}, + ).Label + combined.DefaultFieldSelector = combineSelector( + internal.Selector{Field: inherited.DefaultFieldSelector}, + internal.Selector{Field: options.DefaultFieldSelector}, + ).Field + combined.DefaultTransform = combineTransform(inherited.DefaultTransform, options.DefaultTransform) + combined.ByObject, err = combineByObject(inherited, options, combined.Scheme) + if options.UnsafeDisableDeepCopy != nil { + combined.UnsafeDisableDeepCopy = options.UnsafeDisableDeepCopy + } else { + combined.UnsafeDisableDeepCopy = inherited.UnsafeDisableDeepCopy + } + if err != nil { + return nil, err + } + return &combined, nil +} + +func combineScheme(schemes ...*runtime.Scheme) *runtime.Scheme { + var out *runtime.Scheme + for _, sch := range schemes { + if sch == nil { + continue + } + for gvk, t := range sch.AllKnownTypes() { + if out == nil { + out = runtime.NewScheme() + } + out.AddKnownTypeWithName(gvk, reflect.New(t).Interface().(runtime.Object)) } - opts.SelectorsByObject = options.SelectorsByObject - return New(config, opts) + } + return out +} + +func selectMapper(def, override meta.RESTMapper) meta.RESTMapper { + if override != nil { + return override + } + return def +} + +func selectResync(def, override *time.Duration) *time.Duration { + if override != nil { + return override + } + return def +} + +func selectNamespaces(def, override []string) []string { + if len(override) > 0 { + return override + } + return def +} + +func combineByObject(inherited, options Options, scheme *runtime.Scheme) (map[client.Object]ByObject, error) { + optionsByGVK, err := convertToInformerOptsByGVK(options.ByObject, scheme) + if err != nil { + return nil, err + } + inheritedByGVK, err := convertToInformerOptsByGVK(inherited.ByObject, scheme) + if err != nil { + return nil, err + } + for gvk, inheritedByGVK := range inheritedByGVK { + unsafeDisableDeepCopy := options.UnsafeDisableDeepCopy + if current, ok := optionsByGVK[gvk]; ok { + unsafeDisableDeepCopy = current.UnsafeDisableDeepCopy + } + optionsByGVK[gvk] = internal.InformersOptsByGVK{ + Selector: combineSelector(inheritedByGVK.Selector, optionsByGVK[gvk].Selector), + Transform: combineTransform(inheritedByGVK.Transform, optionsByGVK[gvk].Transform), + UnsafeDisableDeepCopy: unsafeDisableDeepCopy, + } + } + return convertToByObject(optionsByGVK, scheme) +} + +func combineSelector(selectors ...internal.Selector) internal.Selector { + ls := make([]labels.Selector, 0, len(selectors)) + fs := make([]fields.Selector, 0, len(selectors)) + for _, s := range selectors { + ls = append(ls, s.Label) + fs = append(fs, s.Field) + } + return internal.Selector{ + Label: combineLabelSelectors(ls...), + Field: combineFieldSelectors(fs...), + } +} + +func combineLabelSelectors(ls ...labels.Selector) labels.Selector { + var combined labels.Selector + for _, l := range ls { + if l == nil { + continue + } + if combined == nil { + combined = labels.NewSelector() + } + reqs, _ := l.Requirements() + combined = combined.Add(reqs...) + } + return combined +} + +func combineFieldSelectors(fs ...fields.Selector) fields.Selector { + nonNil := fs[:0] + for _, f := range fs { + if f == nil { + continue + } + nonNil = append(nonNil, f) + } + if len(nonNil) == 0 { + return nil + } + if len(nonNil) == 1 { + return nonNil[0] + } + return fields.AndSelectors(nonNil...) +} + +func combineTransform(inherited, current toolscache.TransformFunc) toolscache.TransformFunc { + if inherited == nil { + return current + } + if current == nil { + return inherited + } + return func(in interface{}) (interface{}, error) { + mid, err := inherited(in) + if err != nil { + return nil, err + } + return current(mid) } } func defaultOpts(config *rest.Config, opts Options) (Options, error) { + logger := log.WithName("setup") + + // Use the rest HTTP client for the provided config if unset + if opts.HTTPClient == nil { + var err error + opts.HTTPClient, err = rest.HTTPClientFor(config) + if err != nil { + logger.Error(err, "Failed to get HTTP client") + return opts, fmt.Errorf("could not create HTTP client from config: %w", err) + } + } + // Use the default Kubernetes Scheme if unset if opts.Scheme == nil { opts.Scheme = scheme.Scheme @@ -164,28 +408,62 @@ func defaultOpts(config *rest.Config, opts Options) (Options, error) { // Construct a new Mapper if unset if opts.Mapper == nil { var err error - opts.Mapper, err = apiutil.NewDiscoveryRESTMapper(config) + opts.Mapper, err = apiutil.NewDiscoveryRESTMapper(config, opts.HTTPClient) if err != nil { - log.WithName("setup").Error(err, "Failed to get API Group-Resources") - return opts, fmt.Errorf("could not create RESTMapper from config") + logger.Error(err, "Failed to get API Group-Resources") + return opts, fmt.Errorf("could not create RESTMapper from config: %w", err) } } // Default the resync period to 10 hours if unset - if opts.Resync == nil { - opts.Resync = &defaultResyncTime + if opts.ResyncEvery == nil { + opts.ResyncEvery = &defaultResyncTime } return opts, nil } -func convertToSelectorsByGVK(selectorsByObject SelectorsByObject, scheme *runtime.Scheme) (internal.SelectorsByGVK, error) { - selectorsByGVK := internal.SelectorsByGVK{} - for object, selector := range selectorsByObject { +func convertToInformerOptsByGVK(in map[client.Object]ByObject, scheme *runtime.Scheme) (map[schema.GroupVersionKind]internal.InformersOptsByGVK, error) { + out := map[schema.GroupVersionKind]internal.InformersOptsByGVK{} + for object, byObject := range in { gvk, err := apiutil.GVKForObject(object, scheme) if err != nil { return nil, err } - selectorsByGVK[gvk] = selector + if _, ok := out[gvk]; ok { + return nil, fmt.Errorf("duplicate cache options for GVK %v, cache.Options.ByObject has multiple types with the same GroupVersionKind", gvk) + } + out[gvk] = internal.InformersOptsByGVK{ + Selector: internal.Selector{ + Field: byObject.Field, + Label: byObject.Label, + }, + Transform: byObject.Transform, + UnsafeDisableDeepCopy: byObject.UnsafeDisableDeepCopy, + } + } + return out, nil +} + +func convertToByObject(in map[schema.GroupVersionKind]internal.InformersOptsByGVK, scheme *runtime.Scheme) (map[client.Object]ByObject, error) { + out := map[client.Object]ByObject{} + for gvk, opts := range in { + if gvk == (schema.GroupVersionKind{}) { + continue + } + obj, err := scheme.New(gvk) + if err != nil { + return nil, err + } + cObj, ok := obj.(client.Object) + if !ok { + return nil, fmt.Errorf("object %T for GVK %q does not implement client.Object", obj, gvk) + } + out[cObj] = ByObject{ + Field: opts.Selector.Field, + Label: opts.Selector.Label, + Transform: opts.Transform, + UnsafeDisableDeepCopy: opts.UnsafeDisableDeepCopy, + } } - return selectorsByGVK, nil + return out, nil } diff --git a/pkg/cache/cache_suite_test.go b/pkg/cache/cache_suite_test.go index 900e87e56e..a9a5152ce8 100644 --- a/pkg/cache/cache_suite_test.go +++ b/pkg/cache/cache_suite_test.go @@ -19,27 +19,25 @@ package cache_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Cache Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Cache Suite") } var testenv *envtest.Environment var cfg *rest.Config var clientset *kubernetes.Clientset -var _ = BeforeSuite(func(done Done) { +var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) testenv = &envtest.Environment{} @@ -50,9 +48,7 @@ var _ = BeforeSuite(func(done Done) { clientset, err = kubernetes.NewForConfig(cfg) Expect(err).NotTo(HaveOccurred()) - - close(done) -}, 60) +}) var _ = AfterSuite(func() { Expect(testenv.Stop()).To(Succeed()) diff --git a/pkg/cache/cache_test.go b/pkg/cache/cache_test.go index 49e8e97cfe..36213c6ffd 100644 --- a/pkg/cache/cache_test.go +++ b/pkg/cache/cache_test.go @@ -19,21 +19,26 @@ package cache_test import ( "context" "fmt" + "reflect" + "sort" + "strconv" + "strings" - . "github.com/onsi/ginkgo" - . "github.com/onsi/ginkgo/extensions/table" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" kscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" kcache "k8s.io/client-go/tools/cache" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" @@ -71,6 +76,33 @@ func createPodWithLabels(name, namespace string, restartPolicy corev1.RestartPol return pod } +func createSvc(name, namespace string, cl client.Client) client.Object { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{Port: 1}}, + }, + } + err := cl.Create(context.Background(), svc) + Expect(err).NotTo(HaveOccurred()) + return svc +} + +func createSA(name, namespace string, cl client.Client) client.Object { + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + err := cl.Create(context.Background(), sa) + Expect(err).NotTo(HaveOccurred()) + return sa +} + func createPod(name, namespace string, restartPolicy corev1.RestartPolicy) client.Object { return createPodWithLabels(name, namespace, restartPolicy, nil) } @@ -83,13 +115,306 @@ func deletePod(pod client.Object) { } var _ = Describe("Informer Cache", func() { - CacheTest(cache.New) + CacheTest(cache.New, cache.Options{}) }) var _ = Describe("Multi-Namespace Informer Cache", func() { - CacheTest(cache.MultiNamespacedCacheBuilder([]string{testNamespaceOne, testNamespaceTwo, "default"})) + CacheTest(cache.MultiNamespacedCacheBuilder([]string{testNamespaceOne, testNamespaceTwo, "default"}), cache.Options{}) +}) + +var _ = Describe("Informer Cache without global DeepCopy", func() { + CacheTest(cache.New, cache.Options{ + UnsafeDisableDeepCopy: pointer.Bool(true), + }) +}) + +var _ = Describe("Cache with transformers", func() { + var ( + informerCache cache.Cache + informerCacheCtx context.Context + informerCacheCancel context.CancelFunc + knownPod1 client.Object + knownPod2 client.Object + knownPod3 client.Object + knownPod4 client.Object + knownPod5 client.Object + knownPod6 client.Object + ) + + getTransformValue := func(obj client.Object) string { + accessor, err := meta.Accessor(obj) + if err == nil { + annotations := accessor.GetAnnotations() + if val, exists := annotations["transformed"]; exists { + return val + } + } + return "" + } + + BeforeEach(func() { + informerCacheCtx, informerCacheCancel = context.WithCancel(context.Background()) + Expect(cfg).NotTo(BeNil()) + + By("creating three pods") + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + err = ensureNode(testNodeOne, cl) + Expect(err).NotTo(HaveOccurred()) + err = ensureNamespace(testNamespaceOne, cl) + Expect(err).NotTo(HaveOccurred()) + err = ensureNamespace(testNamespaceTwo, cl) + Expect(err).NotTo(HaveOccurred()) + err = ensureNamespace(testNamespaceThree, cl) + Expect(err).NotTo(HaveOccurred()) + // Includes restart policy since these objects are indexed on this field. + knownPod1 = createPod("test-pod-1", testNamespaceOne, corev1.RestartPolicyNever) + knownPod2 = createPod("test-pod-2", testNamespaceTwo, corev1.RestartPolicyAlways) + knownPod3 = createPodWithLabels("test-pod-3", testNamespaceTwo, corev1.RestartPolicyOnFailure, map[string]string{"common-label": "common"}) + knownPod4 = createPodWithLabels("test-pod-4", testNamespaceThree, corev1.RestartPolicyNever, map[string]string{"common-label": "common"}) + knownPod5 = createPod("test-pod-5", testNamespaceOne, corev1.RestartPolicyNever) + knownPod6 = createPod("test-pod-6", testNamespaceTwo, corev1.RestartPolicyAlways) + + podGVK := schema.GroupVersionKind{ + Kind: "Pod", + Version: "v1", + } + + knownPod1.GetObjectKind().SetGroupVersionKind(podGVK) + knownPod2.GetObjectKind().SetGroupVersionKind(podGVK) + knownPod3.GetObjectKind().SetGroupVersionKind(podGVK) + knownPod4.GetObjectKind().SetGroupVersionKind(podGVK) + knownPod5.GetObjectKind().SetGroupVersionKind(podGVK) + knownPod6.GetObjectKind().SetGroupVersionKind(podGVK) + + By("creating the informer cache") + informerCache, err = cache.New(cfg, cache.Options{ + DefaultTransform: func(i interface{}) (interface{}, error) { + obj := i.(runtime.Object) + Expect(obj).NotTo(BeNil()) + + accessor, err := meta.Accessor(obj) + Expect(err).To(BeNil()) + annotations := accessor.GetAnnotations() + + if _, exists := annotations["transformed"]; exists { + // Avoid performing transformation multiple times. + return i, nil + } + + if annotations == nil { + annotations = make(map[string]string) + } + annotations["transformed"] = "default" + accessor.SetAnnotations(annotations) + return i, nil + }, + ByObject: map[client.Object]cache.ByObject{ + &corev1.Pod{}: { + Transform: func(i interface{}) (interface{}, error) { + obj := i.(runtime.Object) + Expect(obj).NotTo(BeNil()) + accessor, err := meta.Accessor(obj) + Expect(err).To(BeNil()) + + annotations := accessor.GetAnnotations() + if _, exists := annotations["transformed"]; exists { + // Avoid performing transformation multiple times. + return i, nil + } + + if annotations == nil { + annotations = make(map[string]string) + } + annotations["transformed"] = "explicit" + accessor.SetAnnotations(annotations) + return i, nil + }, + }, + }, + }) + Expect(err).NotTo(HaveOccurred()) + By("running the cache and waiting for it to sync") + // pass as an arg so that we don't race between close and re-assign + go func(ctx context.Context) { + defer GinkgoRecover() + Expect(informerCache.Start(ctx)).To(Succeed()) + }(informerCacheCtx) + Expect(informerCache.WaitForCacheSync(informerCacheCtx)).To(BeTrue()) + }) + + AfterEach(func() { + By("cleaning up created pods") + deletePod(knownPod1) + deletePod(knownPod2) + deletePod(knownPod3) + deletePod(knownPod4) + deletePod(knownPod5) + deletePod(knownPod6) + + informerCacheCancel() + }) + + Context("with structured objects", func() { + It("should apply transformers to explicitly specified GVKS", func() { + By("listing pods") + out := corev1.PodList{} + Expect(informerCache.List(context.Background(), &out)).To(Succeed()) + + By("verifying that the returned pods were transformed") + for i := 0; i < len(out.Items); i++ { + Expect(getTransformValue(&out.Items[i])).To(BeIdenticalTo("explicit")) + } + }) + + It("should apply default transformer to objects when none is specified", func() { + By("getting the Kubernetes service") + svc := &corev1.Service{} + svcKey := client.ObjectKey{Namespace: "default", Name: "kubernetes"} + Expect(informerCache.Get(context.Background(), svcKey, svc)).To(Succeed()) + + By("verifying that the returned service was transformed") + Expect(getTransformValue(svc)).To(BeIdenticalTo("default")) + }) + }) + + Context("with unstructured objects", func() { + It("should apply transformers to explicitly specified GVKS", func() { + By("listing pods") + out := unstructured.UnstructuredList{} + out.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "PodList", + }) + Expect(informerCache.List(context.Background(), &out)).To(Succeed()) + + By("verifying that the returned pods were transformed") + for i := 0; i < len(out.Items); i++ { + Expect(getTransformValue(&out.Items[i])).To(BeIdenticalTo("explicit")) + } + }) + + It("should apply default transformer to objects when none is specified", func() { + By("getting the Kubernetes service") + svc := &unstructured.Unstructured{} + svc.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Service", + }) + svcKey := client.ObjectKey{Namespace: "default", Name: "kubernetes"} + Expect(informerCache.Get(context.Background(), svcKey, svc)).To(Succeed()) + + By("verifying that the returned service was transformed") + Expect(getTransformValue(svc)).To(BeIdenticalTo("default")) + }) + }) + + Context("with metadata-only objects", func() { + It("should apply transformers to explicitly specified GVKS", func() { + By("listing pods") + out := metav1.PartialObjectMetadataList{} + out.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "PodList", + }) + Expect(informerCache.List(context.Background(), &out)).To(Succeed()) + + By("verifying that the returned pods were transformed") + for i := 0; i < len(out.Items); i++ { + Expect(getTransformValue(&out.Items[i])).To(BeIdenticalTo("explicit")) + } + }) + It("should apply default transformer to objects when none is specified", func() { + By("getting the Kubernetes service") + svc := &metav1.PartialObjectMetadata{} + svc.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Service", + }) + svcKey := client.ObjectKey{Namespace: "default", Name: "kubernetes"} + Expect(informerCache.Get(context.Background(), svcKey, svc)).To(Succeed()) + + By("verifying that the returned service was transformed") + Expect(getTransformValue(svc)).To(BeIdenticalTo("default")) + }) + }) }) -func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (cache.Cache, error)) { +var _ = Describe("Cache with selectors", func() { + defer GinkgoRecover() + var ( + informerCache cache.Cache + informerCacheCtx context.Context + informerCacheCancel context.CancelFunc + ) + + BeforeEach(func() { + informerCacheCtx, informerCacheCancel = context.WithCancel(context.Background()) + Expect(cfg).NotTo(BeNil()) + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + err = ensureNamespace(testNamespaceOne, cl) + Expect(err).NotTo(HaveOccurred()) + err = ensureNamespace(testNamespaceTwo, cl) + Expect(err).NotTo(HaveOccurred()) + for idx, namespace := range []string{testNamespaceOne, testNamespaceTwo} { + _ = createSA("test-sa-"+strconv.Itoa(idx), namespace, cl) + _ = createSvc("test-svc-"+strconv.Itoa(idx), namespace, cl) + } + + opts := cache.Options{ + DefaultFieldSelector: fields.OneTermEqualSelector("metadata.namespace", testNamespaceTwo), + ByObject: map[client.Object]cache.ByObject{ + &corev1.ServiceAccount{}: {Field: fields.OneTermEqualSelector("metadata.namespace", testNamespaceOne)}, + }, + } + + By("creating the informer cache") + informerCache, err = cache.New(cfg, opts) + Expect(err).NotTo(HaveOccurred()) + By("running the cache and waiting for it to sync") + // pass as an arg so that we don't race between close and re-assign + go func(ctx context.Context) { + defer GinkgoRecover() + Expect(informerCache.Start(ctx)).To(Succeed()) + }(informerCacheCtx) + Expect(informerCache.WaitForCacheSync(informerCacheCtx)).To(BeTrue()) + }) + + AfterEach(func() { + ctx := context.Background() + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + for idx, namespace := range []string{testNamespaceOne, testNamespaceTwo} { + err = cl.Delete(ctx, &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: "test-sa-" + strconv.Itoa(idx)}}) + Expect(err).NotTo(HaveOccurred()) + err = cl.Delete(ctx, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: "test-svc-" + strconv.Itoa(idx)}}) + Expect(err).NotTo(HaveOccurred()) + } + informerCacheCancel() + }) + + It("Should list serviceaccounts and find exactly one in namespace "+testNamespaceOne, func() { + var sas corev1.ServiceAccountList + err := informerCache.List(informerCacheCtx, &sas) + Expect(err).NotTo(HaveOccurred()) + Expect(len(sas.Items)).To(Equal(1)) + Expect(sas.Items[0].Namespace).To(Equal(testNamespaceOne)) + }) + + It("Should list services and find exactly one in namespace "+testNamespaceTwo, func() { + var svcs corev1.ServiceList + err := informerCache.List(informerCacheCtx, &svcs) + Expect(err).NotTo(HaveOccurred()) + Expect(len(svcs.Items)).To(Equal(1)) + Expect(svcs.Items[0].Namespace).To(Equal(testNamespaceTwo)) + }) +}) + +func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (cache.Cache, error), opts cache.Options) { Describe("Cache test", func() { var ( informerCache cache.Cache @@ -139,7 +464,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca knownPod6.GetObjectKind().SetGroupVersionKind(podGVK) By("creating the informer cache") - informerCache, err = createCacheFunc(cfg, cache.Options{}) + informerCache, err = createCacheFunc(cfg, opts) Expect(err).NotTo(HaveOccurred()) By("running the cache and waiting for it to sync") // pass as an arg so that we don't race between close and re-assign @@ -228,18 +553,20 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca } }) - It("should be able to list objects with GVK populated", func() { - By("listing pods") - out := &corev1.PodList{} - Expect(informerCache.List(context.Background(), out)).To(Succeed()) - - By("verifying that the returned pods have GVK populated") - Expect(out.Items).NotTo(BeEmpty()) - Expect(out.Items).Should(SatisfyAny(HaveLen(5), HaveLen(6))) - for _, p := range out.Items { - Expect(p.GroupVersionKind()).To(Equal(corev1.SchemeGroupVersion.WithKind("Pod"))) - } - }) + if !isPodDisableDeepCopy(opts) { + It("should be able to list objects with GVK populated", func() { + By("listing pods") + out := &corev1.PodList{} + Expect(informerCache.List(context.Background(), out)).To(Succeed()) + + By("verifying that the returned pods have GVK populated") + Expect(out.Items).NotTo(BeEmpty()) + Expect(out.Items).Should(SatisfyAny(HaveLen(5), HaveLen(6))) + for _, p := range out.Items { + Expect(p.GroupVersionKind()).To(Equal(corev1.SchemeGroupVersion.WithKind("Pod"))) + } + }) + } It("should be able to list objects by namespace", func() { By("listing pods in test-namespace-1") @@ -255,21 +582,53 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca } }) - It("should deep copy the object unless told otherwise", func() { - By("retrieving a specific pod from the cache") - out := &corev1.Pod{} - podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} - Expect(informerCache.Get(context.Background(), podKey, out)).To(Succeed()) + if !isPodDisableDeepCopy(opts) { + It("should deep copy the object unless told otherwise", func() { + By("retrieving a specific pod from the cache") + out := &corev1.Pod{} + podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} + Expect(informerCache.Get(context.Background(), podKey, out)).To(Succeed()) - By("verifying the retrieved pod is equal to a known pod") - Expect(out).To(Equal(knownPod2)) + By("verifying the retrieved pod is equal to a known pod") + Expect(out).To(Equal(knownPod2)) - By("altering a field in the retrieved pod") - *out.Spec.ActiveDeadlineSeconds = 4 + By("altering a field in the retrieved pod") + *out.Spec.ActiveDeadlineSeconds = 4 - By("verifying the pods are no longer equal") - Expect(out).NotTo(Equal(knownPod2)) - }) + By("verifying the pods are no longer equal") + Expect(out).NotTo(Equal(knownPod2)) + }) + } else { + It("should not deep copy the object if UnsafeDisableDeepCopy is enabled", func() { + By("getting a specific pod from the cache twice") + podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} + out1 := &corev1.Pod{} + Expect(informerCache.Get(context.Background(), podKey, out1)).To(Succeed()) + out2 := &corev1.Pod{} + Expect(informerCache.Get(context.Background(), podKey, out2)).To(Succeed()) + + By("verifying the pointer fields in pod have the same addresses") + Expect(out1).To(Equal(out2)) + Expect(reflect.ValueOf(out1.Labels).Pointer()).To(BeIdenticalTo(reflect.ValueOf(out2.Labels).Pointer())) + + By("listing pods from the cache twice") + outList1 := &corev1.PodList{} + Expect(informerCache.List(context.Background(), outList1, client.InNamespace(testNamespaceOne))).To(Succeed()) + outList2 := &corev1.PodList{} + Expect(informerCache.List(context.Background(), outList2, client.InNamespace(testNamespaceOne))).To(Succeed()) + + By("verifying the pointer fields in pod have the same addresses") + Expect(len(outList1.Items)).To(Equal(len(outList2.Items))) + sort.SliceStable(outList1.Items, func(i, j int) bool { return outList1.Items[i].Name <= outList1.Items[j].Name }) + sort.SliceStable(outList2.Items, func(i, j int) bool { return outList2.Items[i].Name <= outList2.Items[j].Name }) + for i := range outList1.Items { + a := &outList1.Items[i] + b := &outList2.Items[i] + Expect(a).To(Equal(b)) + Expect(reflect.ValueOf(a.Labels).Pointer()).To(BeIdenticalTo(reflect.ValueOf(b.Labels).Pointer())) + } + }) + } It("should return an error if the object is not found", func() { By("getting a service that does not exists") @@ -312,6 +671,15 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(informerCache.List(context.Background(), listObj, opts)).To(Succeed()) Expect(listObj.Items).Should(HaveLen(3)) }) + + It("should return a limited result set matching the correct label", func() { + listObj := &corev1.PodList{} + labelOpt := client.MatchingLabels(map[string]string{"common-label": "common"}) + limitOpt := client.Limit(1) + By("verifying that only Limit (1) number of objects are retrieved from the cache") + Expect(informerCache.List(context.Background(), listObj, labelOpt, limitOpt)).To(Succeed()) + Expect(listObj.Items).Should(HaveLen(1)) + }) }) Context("with unstructured objects", func() { @@ -422,7 +790,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca It("should be able to restrict cache to a namespace", func() { By("creating a namespaced cache") - namespacedCache, err := cache.New(cfg, cache.Options{Namespace: testNamespaceOne}) + namespacedCache, err := cache.New(cfg, cache.Options{Namespaces: []string{testNamespaceOne}}) Expect(err).NotTo(HaveOccurred()) By("running the cache and waiting for it to sync") @@ -476,30 +844,66 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(namespacedCache.Get(context.Background(), key2, node)).To(Succeed()) }) - It("should deep copy the object unless told otherwise", func() { - By("retrieving a specific pod from the cache") - out := &unstructured.Unstructured{} - out.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "Pod", + if !isPodDisableDeepCopy(opts) { + It("should deep copy the object unless told otherwise", func() { + By("retrieving a specific pod from the cache") + out := &unstructured.Unstructured{} + out.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Pod", + }) + uKnownPod2 := &unstructured.Unstructured{} + Expect(kscheme.Scheme.Convert(knownPod2, uKnownPod2, nil)).To(Succeed()) + + podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} + Expect(informerCache.Get(context.Background(), podKey, out)).To(Succeed()) + + By("verifying the retrieved pod is equal to a known pod") + Expect(out).To(Equal(uKnownPod2)) + + By("altering a field in the retrieved pod") + m, _ := out.Object["spec"].(map[string]interface{}) + m["activeDeadlineSeconds"] = 4 + + By("verifying the pods are no longer equal") + Expect(out).NotTo(Equal(knownPod2)) }) - uKnownPod2 := &unstructured.Unstructured{} - Expect(kscheme.Scheme.Convert(knownPod2, uKnownPod2, nil)).To(Succeed()) - - podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} - Expect(informerCache.Get(context.Background(), podKey, out)).To(Succeed()) - - By("verifying the retrieved pod is equal to a known pod") - Expect(out).To(Equal(uKnownPod2)) - - By("altering a field in the retrieved pod") - m, _ := out.Object["spec"].(map[string]interface{}) - m["activeDeadlineSeconds"] = 4 - - By("verifying the pods are no longer equal") - Expect(out).NotTo(Equal(knownPod2)) - }) + } else { + It("should not deep copy the object if UnsafeDisableDeepCopy is enabled", func() { + By("getting a specific pod from the cache twice") + podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} + out1 := &unstructured.Unstructured{} + out1.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}) + Expect(informerCache.Get(context.Background(), podKey, out1)).To(Succeed()) + out2 := &unstructured.Unstructured{} + out2.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}) + Expect(informerCache.Get(context.Background(), podKey, out2)).To(Succeed()) + + By("verifying the pointer fields in pod have the same addresses") + Expect(out1).To(Equal(out2)) + Expect(reflect.ValueOf(out1.Object).Pointer()).To(BeIdenticalTo(reflect.ValueOf(out2.Object).Pointer())) + + By("listing pods from the cache twice") + outList1 := &unstructured.UnstructuredList{} + outList1.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "PodList"}) + Expect(informerCache.List(context.Background(), outList1, client.InNamespace(testNamespaceOne))).To(Succeed()) + outList2 := &unstructured.UnstructuredList{} + outList2.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "PodList"}) + Expect(informerCache.List(context.Background(), outList2, client.InNamespace(testNamespaceOne))).To(Succeed()) + + By("verifying the pointer fields in pod have the same addresses") + Expect(len(outList1.Items)).To(Equal(len(outList2.Items))) + sort.SliceStable(outList1.Items, func(i, j int) bool { return outList1.Items[i].GetName() <= outList1.Items[j].GetName() }) + sort.SliceStable(outList2.Items, func(i, j int) bool { return outList2.Items[i].GetName() <= outList2.Items[j].GetName() }) + for i := range outList1.Items { + a := &outList1.Items[i] + b := &outList2.Items[i] + Expect(a).To(Equal(b)) + Expect(reflect.ValueOf(a.Object).Pointer()).To(BeIdenticalTo(reflect.ValueOf(b.Object).Pointer())) + } + }) + } It("should return an error if the object is not found", func() { By("getting a service that does not exists") @@ -667,7 +1071,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca It("should be able to restrict cache to a namespace", func() { By("creating a namespaced cache") - namespacedCache, err := cache.New(cfg, cache.Options{Namespace: testNamespaceOne}) + namespacedCache, err := cache.New(cfg, cache.Options{Namespaces: []string{testNamespaceOne}}) Expect(err).NotTo(HaveOccurred()) By("running the cache and waiting for it to sync") @@ -721,34 +1125,71 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(namespacedCache.Get(context.Background(), key2, node)).To(Succeed()) }) - It("should deep copy the object unless told otherwise", func() { - By("retrieving a specific pod from the cache") - out := &metav1.PartialObjectMetadata{} - out.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "Pod", + if !isPodDisableDeepCopy(opts) { + It("should deep copy the object unless told otherwise", func() { + By("retrieving a specific pod from the cache") + out := &metav1.PartialObjectMetadata{} + out.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Pod", + }) + uKnownPod2 := &metav1.PartialObjectMetadata{} + knownPod2.(*corev1.Pod).ObjectMeta.DeepCopyInto(&uKnownPod2.ObjectMeta) + uKnownPod2.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Pod", + }) + + podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} + Expect(informerCache.Get(context.Background(), podKey, out)).To(Succeed()) + + By("verifying the retrieved pod is equal to a known pod") + Expect(out).To(Equal(uKnownPod2)) + + By("altering a field in the retrieved pod") + out.Labels["foo"] = "bar" + + By("verifying the pods are no longer equal") + Expect(out).NotTo(Equal(knownPod2)) }) - uKnownPod2 := &metav1.PartialObjectMetadata{} - knownPod2.(*corev1.Pod).ObjectMeta.DeepCopyInto(&uKnownPod2.ObjectMeta) - uKnownPod2.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "Pod", + } else { + It("should not deep copy the object if UnsafeDisableDeepCopy is enabled", func() { + By("getting a specific pod from the cache twice") + podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} + out1 := &metav1.PartialObjectMetadata{} + out1.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}) + Expect(informerCache.Get(context.Background(), podKey, out1)).To(Succeed()) + out2 := &metav1.PartialObjectMetadata{} + out2.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}) + Expect(informerCache.Get(context.Background(), podKey, out2)).To(Succeed()) + + By("verifying the pods have the same pointer addresses") + By("verifying the pointer fields in pod have the same addresses") + Expect(out1).To(Equal(out2)) + Expect(reflect.ValueOf(out1.Labels).Pointer()).To(BeIdenticalTo(reflect.ValueOf(out2.Labels).Pointer())) + + By("listing pods from the cache twice") + outList1 := &metav1.PartialObjectMetadataList{} + outList1.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "PodList"}) + Expect(informerCache.List(context.Background(), outList1, client.InNamespace(testNamespaceOne))).To(Succeed()) + outList2 := &metav1.PartialObjectMetadataList{} + outList2.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "PodList"}) + Expect(informerCache.List(context.Background(), outList2, client.InNamespace(testNamespaceOne))).To(Succeed()) + + By("verifying the pointer fields in pod have the same addresses") + Expect(len(outList1.Items)).To(Equal(len(outList2.Items))) + sort.SliceStable(outList1.Items, func(i, j int) bool { return outList1.Items[i].Name <= outList1.Items[j].Name }) + sort.SliceStable(outList2.Items, func(i, j int) bool { return outList2.Items[i].Name <= outList2.Items[j].Name }) + for i := range outList1.Items { + a := &outList1.Items[i] + b := &outList2.Items[i] + Expect(a).To(Equal(b)) + Expect(reflect.ValueOf(a.Labels).Pointer()).To(BeIdenticalTo(reflect.ValueOf(b.Labels).Pointer())) + } }) - - podKey := client.ObjectKey{Name: "test-pod-2", Namespace: testNamespaceTwo} - Expect(informerCache.Get(context.Background(), podKey, out)).To(Succeed()) - - By("verifying the retrieved pod is equal to a known pod") - Expect(out).To(Equal(uKnownPod2)) - - By("altering a field in the retrieved pod") - out.Labels["foo"] = "bar" - - By("verifying the pods are no longer equal") - Expect(out).NotTo(Equal(knownPod2)) - }) + } It("should return an error if the object is not found", func() { By("getting a service that does not exists") @@ -784,7 +1225,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca By("creating the cache") builder := cache.BuilderWithOptions( cache.Options{ - SelectorsByObject: cache.SelectorsByObject{ + ByObject: map[client.Object]cache.ByObject{ &corev1.Pod{}: { Label: labels.Set(tc.labelSelectors).AsSelector(), Field: fields.Set(tc.fieldSelectors).AsSelector(), @@ -885,7 +1326,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }) Describe("as an Informer", func() { Context("with structured objects", func() { - It("should be able to get informer for the object", func(done Done) { + It("should be able to get informer for the object", func() { By("getting a shared index informer for a pod") pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -911,7 +1352,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca addFunc := func(obj interface{}) { out <- obj } - sii.AddEventHandler(kcache.ResourceEventHandlerFuncs{AddFunc: addFunc}) + _, _ = sii.AddEventHandler(kcache.ResourceEventHandlerFuncs{AddFunc: addFunc}) By("adding an object") cl, err := client.New(cfg, client.Options{}) @@ -921,9 +1362,8 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca By("verifying the object is received on the channel") Eventually(out).Should(Receive(Equal(pod))) - close(done) }) - It("should be able to get an informer by group/version/kind", func(done Done) { + It("should be able to get an informer by group/version/kind", func() { By("getting an shared index informer for gvk = core/v1/pod") gvk := schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} sii, err := informerCache.GetInformerForKind(context.TODO(), gvk) @@ -936,7 +1376,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca addFunc := func(obj interface{}) { out <- obj } - sii.AddEventHandler(kcache.ResourceEventHandlerFuncs{AddFunc: addFunc}) + _, _ = sii.AddEventHandler(kcache.ResourceEventHandlerFuncs{AddFunc: addFunc}) By("adding an object") cl, err := client.New(cfg, client.Options{}) @@ -960,7 +1400,6 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca By("verifying the object is received on the channel") Eventually(out).Should(Receive(Equal(pod))) - close(done) }) It("should be able to index an object field then retrieve objects by that field", func() { By("creating the cache") @@ -1028,9 +1467,43 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(sii).To(BeNil()) Expect(apierrors.IsTimeout(err)).To(BeTrue()) }) + + It("should be able not to change indexer values after indexing cluster-scope objects", func() { + By("creating the cache") + informer, err := cache.New(cfg, cache.Options{}) + Expect(err).NotTo(HaveOccurred()) + + By("indexing the Namespace objects with fixed values before starting") + pod := &corev1.Namespace{} + indexerValues := []string{"a", "b", "c"} + fieldName := "fixedValues" + indexFunc := func(obj client.Object) []string { + return indexerValues + } + Expect(informer.IndexField(context.TODO(), pod, fieldName, indexFunc)).To(Succeed()) + + By("running the cache and waiting for it to sync") + go func() { + defer GinkgoRecover() + Expect(informer.Start(informerCacheCtx)).To(Succeed()) + }() + Expect(informer.WaitForCacheSync(informerCacheCtx)).NotTo(BeFalse()) + + By("listing Namespaces with fixed indexer") + listObj := &corev1.NamespaceList{} + Expect(informer.List(context.Background(), listObj, + client.MatchingFields{fieldName: "a"})).To(Succeed()) + Expect(listObj.Items).NotTo(BeZero()) + + By("verifying the indexing does not change fixed returned values") + Expect(indexerValues).Should(HaveLen(3)) + Expect(indexerValues[0]).To(Equal("a")) + Expect(indexerValues[1]).To(Equal("b")) + Expect(indexerValues[2]).To(Equal("c")) + }) }) Context("with unstructured objects", func() { - It("should be able to get informer for the object", func(done Done) { + It("should be able to get informer for the object", func() { By("getting a shared index informer for a pod") pod := &unstructured.Unstructured{ @@ -1062,7 +1535,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca addFunc := func(obj interface{}) { out <- obj } - sii.AddEventHandler(kcache.ResourceEventHandlerFuncs{AddFunc: addFunc}) + _, _ = sii.AddEventHandler(kcache.ResourceEventHandlerFuncs{AddFunc: addFunc}) By("adding an object") cl, err := client.New(cfg, client.Options{}) @@ -1072,8 +1545,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca By("verifying the object is received on the channel") Eventually(out).Should(Receive(Equal(pod))) - close(done) - }, 3) + }) It("should be able to index an object field then retrieve objects by that field", func() { By("creating the cache") @@ -1123,7 +1595,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Expect(listObj.Items).Should(HaveLen(1)) actual := listObj.Items[0] Expect(actual.GetName()).To(Equal("test-pod-3")) - }, 3) + }) It("should allow for get informer to be cancelled", func() { By("cancelling the context") @@ -1145,7 +1617,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }) }) Context("with metadata-only objects", func() { - It("should be able to get informer for the object", func(done Done) { + It("should be able to get informer for the object", func() { By("getting a shared index informer for a pod") pod := &corev1.Pod{ @@ -1181,7 +1653,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca addFunc := func(obj interface{}) { out <- obj } - sii.AddEventHandler(kcache.ResourceEventHandlerFuncs{AddFunc: addFunc}) + _, _ = sii.AddEventHandler(kcache.ResourceEventHandlerFuncs{AddFunc: addFunc}) By("adding an object") cl, err := client.New(cfg, client.Options{}) @@ -1193,8 +1665,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca By("verifying the object's metadata is received on the channel") Eventually(out).Should(Receive(Equal(podMeta))) - close(done) - }, 3) + }) It("should be able to index an object field then retrieve objects by that field", func() { By("creating the cache") @@ -1248,7 +1719,7 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca Version: "v1", Kind: "Pod", })) - }, 3) + }) It("should allow for get informer to be cancelled", func() { By("creating a context and cancelling it") @@ -1271,6 +1742,29 @@ func CacheTest(createCacheFunc func(config *rest.Config, opts cache.Options) (ca }) }) }) + Describe("use UnsafeDisableDeepCopy list options", func() { + It("should be able to change object in informer cache", func() { + By("listing pods") + out := corev1.PodList{} + Expect(informerCache.List(context.Background(), &out, client.UnsafeDisableDeepCopy)).To(Succeed()) + for _, item := range out.Items { + if strings.Compare(item.Name, "test-pod-3") == 0 { // test-pod-3 has labels + item.Labels["UnsafeDisableDeepCopy"] = "true" + break + } + } + + By("verifying that the returned pods were changed") + out2 := corev1.PodList{} + Expect(informerCache.List(context.Background(), &out, client.UnsafeDisableDeepCopy)).To(Succeed()) + for _, item := range out2.Items { + if strings.Compare(item.Name, "test-pod-3") == 0 { + Expect(item.Labels["UnsafeDisableDeepCopy"]).To(Equal("true")) + break + } + } + }) + }) }) } @@ -1314,3 +1808,13 @@ func isKubeService(svc metav1.Object) bool { // grumble grumble linters grumble grumble return svc.GetNamespace() == "default" && svc.GetName() == "kubernetes" } + +func isPodDisableDeepCopy(opts cache.Options) bool { + if opts.ByObject[&corev1.Pod{}].UnsafeDisableDeepCopy != nil { + return *opts.ByObject[&corev1.Pod{}].UnsafeDisableDeepCopy + } + if opts.UnsafeDisableDeepCopy != nil { + return *opts.UnsafeDisableDeepCopy + } + return false +} diff --git a/pkg/cache/cache_unit_test.go b/pkg/cache/cache_unit_test.go new file mode 100644 index 0000000000..1006c72812 --- /dev/null +++ b/pkg/cache/cache_unit_test.go @@ -0,0 +1,546 @@ +package cache + +import ( + "reflect" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/pointer" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var _ = Describe("cache.inheritFrom", func() { + defer GinkgoRecover() + + var ( + inherited Options + specified Options + gv schema.GroupVersion + coreScheme *runtime.Scheme + ) + + BeforeEach(func() { + inherited = Options{} + specified = Options{} + gv = schema.GroupVersion{ + Group: "example.com", + Version: "v1alpha1", + } + coreScheme = runtime.NewScheme() + Expect(scheme.AddToScheme(coreScheme)).To(Succeed()) + }) + + Context("Scheme", func() { + It("is nil when specified and inherited are unset", func() { + Expect(checkError(specified.inheritFrom(inherited)).Scheme).To(BeNil()) + }) + It("is specified when only specified is set", func() { + specified.Scheme = runtime.NewScheme() + specified.Scheme.AddKnownTypes(gv, &unstructured.Unstructured{}) + Expect(specified.Scheme.KnownTypes(gv)).To(HaveLen(1)) + + Expect(checkError(specified.inheritFrom(inherited)).Scheme.KnownTypes(gv)).To(HaveLen(1)) + }) + It("is inherited when only inherited is set", func() { + inherited.Scheme = runtime.NewScheme() + inherited.Scheme.AddKnownTypes(gv, &unstructured.Unstructured{}) + Expect(inherited.Scheme.KnownTypes(gv)).To(HaveLen(1)) + + combined := checkError(specified.inheritFrom(inherited)) + Expect(combined.Scheme).NotTo(BeNil()) + Expect(combined.Scheme.KnownTypes(gv)).To(HaveLen(1)) + }) + It("is combined when both inherited and specified are set", func() { + specified.Scheme = runtime.NewScheme() + specified.Scheme.AddKnownTypes(gv, &unstructured.Unstructured{}) + Expect(specified.Scheme.AllKnownTypes()).To(HaveLen(1)) + + inherited.Scheme = runtime.NewScheme() + inherited.Scheme.AddKnownTypes(schema.GroupVersion{Group: "example.com", Version: "v1"}, &unstructured.Unstructured{}) + Expect(inherited.Scheme.AllKnownTypes()).To(HaveLen(1)) + + Expect(checkError(specified.inheritFrom(inherited)).Scheme.AllKnownTypes()).To(HaveLen(2)) + }) + }) + Context("Mapper", func() { + It("is nil when specified and inherited are unset", func() { + Expect(checkError(specified.inheritFrom(inherited)).Mapper).To(BeNil()) + }) + It("is unchanged when only specified is set", func() { + specified.Mapper = meta.NewDefaultRESTMapper(nil) + Expect(checkError(specified.inheritFrom(inherited)).Mapper).To(Equal(specified.Mapper)) + }) + It("is inherited when only inherited is set", func() { + inherited.Mapper = meta.NewDefaultRESTMapper(nil) + Expect(checkError(specified.inheritFrom(inherited)).Mapper).To(Equal(inherited.Mapper)) + }) + It("is unchanged when both inherited and specified are set", func() { + specified.Mapper = meta.NewDefaultRESTMapper(nil) + inherited.Mapper = meta.NewDefaultRESTMapper([]schema.GroupVersion{gv}) + Expect(checkError(specified.inheritFrom(inherited)).Mapper).To(Equal(specified.Mapper)) + }) + }) + Context("Resync", func() { + It("is nil when specified and inherited are unset", func() { + Expect(checkError(specified.inheritFrom(inherited)).ResyncEvery).To(BeNil()) + }) + It("is unchanged when only specified is set", func() { + specified.ResyncEvery = pointer.Duration(time.Second) + Expect(checkError(specified.inheritFrom(inherited)).ResyncEvery).To(Equal(specified.ResyncEvery)) + }) + It("is inherited when only inherited is set", func() { + inherited.ResyncEvery = pointer.Duration(time.Second) + Expect(checkError(specified.inheritFrom(inherited)).ResyncEvery).To(Equal(inherited.ResyncEvery)) + }) + It("is unchanged when both inherited and specified are set", func() { + specified.ResyncEvery = pointer.Duration(time.Second) + inherited.ResyncEvery = pointer.Duration(time.Minute) + Expect(checkError(specified.inheritFrom(inherited)).ResyncEvery).To(Equal(specified.ResyncEvery)) + }) + }) + Context("Namespace", func() { + It("has zero length when Namespaces specified and inherited are unset", func() { + Expect(checkError(specified.inheritFrom(inherited)).Namespaces).To(HaveLen(0)) + }) + It("is unchanged when only specified is set", func() { + specified.Namespaces = []string{"specified"} + Expect(checkError(specified.inheritFrom(inherited)).Namespaces).To(Equal(specified.Namespaces)) + }) + It("is inherited when only inherited is set", func() { + inherited.Namespaces = []string{"inherited"} + Expect(checkError(specified.inheritFrom(inherited)).Namespaces).To(Equal(inherited.Namespaces)) + }) + It("in unchanged when both inherited and specified are set", func() { + specified.Namespaces = []string{"specified"} + inherited.Namespaces = []string{"inherited"} + Expect(checkError(specified.inheritFrom(inherited)).Namespaces).To(Equal(specified.Namespaces)) + }) + }) + Context("SelectorsByObject", func() { + It("is unchanged when specified and inherited are unset", func() { + Expect(checkError(specified.inheritFrom(inherited)).ByObject).To(HaveLen(0)) + }) + It("is unchanged when only specified is set", func() { + specified.Scheme = coreScheme + specified.ByObject = map[client.Object]ByObject{ + &corev1.Pod{}: {}, + } + Expect(checkError(specified.inheritFrom(inherited)).ByObject).To(HaveLen(1)) + }) + It("is inherited when only inherited is set", func() { + inherited.Scheme = coreScheme + inherited.ByObject = map[client.Object]ByObject{ + &corev1.ConfigMap{}: {}, + } + Expect(checkError(specified.inheritFrom(inherited)).ByObject).To(HaveLen(1)) + }) + It("is combined when both inherited and specified are set", func() { + specified.Scheme = coreScheme + inherited.Scheme = coreScheme + specified.ByObject = map[client.Object]ByObject{ + &corev1.Pod{}: {}, + } + inherited.ByObject = map[client.Object]ByObject{ + &corev1.ConfigMap{}: {}, + } + Expect(checkError(specified.inheritFrom(inherited)).ByObject).To(HaveLen(2)) + }) + It("combines selectors if specified and inherited specify selectors for the same object", func() { + specified.Scheme = coreScheme + inherited.Scheme = coreScheme + specified.ByObject = map[client.Object]ByObject{ + &corev1.Pod{}: { + Label: labels.Set{"specified": "true"}.AsSelector(), + Field: fields.Set{"metadata.name": "specified"}.AsSelector(), + }, + } + inherited.ByObject = map[client.Object]ByObject{ + &corev1.Pod{}: { + Label: labels.Set{"inherited": "true"}.AsSelector(), + Field: fields.Set{"metadata.namespace": "inherited"}.AsSelector(), + }, + } + combined := checkError(specified.inheritFrom(inherited)).ByObject + Expect(combined).To(HaveLen(1)) + var ( + obj client.Object + byObject ByObject + ) + for obj, byObject = range combined { + } + Expect(obj).To(BeAssignableToTypeOf(&corev1.Pod{})) + + Expect(byObject.Label.Matches(labels.Set{"specified": "true"})).To(BeFalse()) + Expect(byObject.Label.Matches(labels.Set{"inherited": "true"})).To(BeFalse()) + Expect(byObject.Label.Matches(labels.Set{"specified": "true", "inherited": "true"})).To(BeTrue()) + + Expect(byObject.Field.Matches(fields.Set{"metadata.name": "specified", "metadata.namespace": "other"})).To(BeFalse()) + Expect(byObject.Field.Matches(fields.Set{"metadata.name": "other", "metadata.namespace": "inherited"})).To(BeFalse()) + Expect(byObject.Field.Matches(fields.Set{"metadata.name": "specified", "metadata.namespace": "inherited"})).To(BeTrue()) + }) + It("uses inherited scheme for inherited selectors", func() { + inherited.Scheme = coreScheme + inherited.ByObject = map[client.Object]ByObject{ + &corev1.ConfigMap{}: {}, + } + Expect(checkError(specified.inheritFrom(inherited)).ByObject).To(HaveLen(1)) + }) + It("uses inherited scheme for specified selectors", func() { + inherited.Scheme = coreScheme + specified.ByObject = map[client.Object]ByObject{ + &corev1.ConfigMap{}: {}, + } + Expect(checkError(specified.inheritFrom(inherited)).ByObject).To(HaveLen(1)) + }) + It("uses specified scheme for specified selectors", func() { + specified.Scheme = coreScheme + specified.ByObject = map[client.Object]ByObject{ + &corev1.ConfigMap{}: {}, + } + Expect(checkError(specified.inheritFrom(inherited)).ByObject).To(HaveLen(1)) + }) + }) + Context("DefaultSelector", func() { + It("is unchanged when specified and inherited are unset", func() { + Expect(specified.DefaultLabelSelector).To(BeNil()) + Expect(inherited.DefaultLabelSelector).To(BeNil()) + Expect(specified.DefaultFieldSelector).To(BeNil()) + Expect(inherited.DefaultFieldSelector).To(BeNil()) + Expect(checkError(specified.inheritFrom(inherited)).DefaultLabelSelector).To(BeNil()) + Expect(checkError(specified.inheritFrom(inherited)).DefaultFieldSelector).To(BeNil()) + }) + It("is unchanged when only specified is set", func() { + specified.DefaultLabelSelector = labels.Set{"specified": "true"}.AsSelector() + specified.DefaultFieldSelector = fields.Set{"specified": "true"}.AsSelector() + Expect(checkError(specified.inheritFrom(inherited)).DefaultLabelSelector).To(Equal(specified.DefaultLabelSelector)) + Expect(checkError(specified.inheritFrom(inherited)).DefaultFieldSelector).To(Equal(specified.DefaultFieldSelector)) + }) + It("is inherited when only inherited is set", func() { + inherited.DefaultLabelSelector = labels.Set{"inherited": "true"}.AsSelector() + inherited.DefaultFieldSelector = fields.Set{"inherited": "true"}.AsSelector() + Expect(checkError(specified.inheritFrom(inherited)).DefaultLabelSelector).To(Equal(inherited.DefaultLabelSelector)) + Expect(checkError(specified.inheritFrom(inherited)).DefaultFieldSelector).To(Equal(inherited.DefaultFieldSelector)) + }) + It("is combined when both inherited and specified are set", func() { + specified.DefaultLabelSelector = labels.Set{"specified": "true"}.AsSelector() + specified.DefaultFieldSelector = fields.Set{"metadata.name": "specified"}.AsSelector() + + inherited.DefaultLabelSelector = labels.Set{"inherited": "true"}.AsSelector() + inherited.DefaultFieldSelector = fields.Set{"metadata.namespace": "inherited"}.AsSelector() + { + combined := checkError(specified.inheritFrom(inherited)).DefaultLabelSelector + Expect(combined).NotTo(BeNil()) + Expect(combined.Matches(labels.Set{"specified": "true"})).To(BeFalse()) + Expect(combined.Matches(labels.Set{"inherited": "true"})).To(BeFalse()) + Expect(combined.Matches(labels.Set{"specified": "true", "inherited": "true"})).To(BeTrue()) + } + + { + combined := checkError(specified.inheritFrom(inherited)).DefaultFieldSelector + Expect(combined.Matches(fields.Set{"metadata.name": "specified", "metadata.namespace": "other"})).To(BeFalse()) + Expect(combined.Matches(fields.Set{"metadata.name": "other", "metadata.namespace": "inherited"})).To(BeFalse()) + Expect(combined.Matches(fields.Set{"metadata.name": "specified", "metadata.namespace": "inherited"})).To(BeTrue()) + } + + }) + }) + Context("UnsafeDisableDeepCopyByObject", func() { + It("is unchanged when specified and inherited are unset", func() { + Expect(checkError(specified.inheritFrom(inherited)).ByObject).To(HaveLen(0)) + }) + It("is unchanged when only specified is set", func() { + specified.Scheme = coreScheme + specified.UnsafeDisableDeepCopy = pointer.Bool(true) + Expect(*(checkError(specified.inheritFrom(inherited)).UnsafeDisableDeepCopy)).To(BeTrue()) + }) + It("is inherited when only inherited is set", func() { + inherited.Scheme = coreScheme + inherited.UnsafeDisableDeepCopy = pointer.Bool(true) + Expect(*(checkError(specified.inheritFrom(inherited)).UnsafeDisableDeepCopy)).To(BeTrue()) + }) + It("is combined when both inherited and specified are set for different keys", func() { + specified.Scheme = coreScheme + inherited.Scheme = coreScheme + specified.ByObject = map[client.Object]ByObject{ + &corev1.Pod{}: { + UnsafeDisableDeepCopy: pointer.Bool(true), + }, + } + inherited.ByObject = map[client.Object]ByObject{ + &corev1.ConfigMap{}: { + UnsafeDisableDeepCopy: pointer.Bool(true), + }, + } + Expect(checkError(specified.inheritFrom(inherited)).ByObject).To(HaveLen(2)) + }) + It("is true when inherited=false and specified=true for the same key", func() { + specified.Scheme = coreScheme + inherited.Scheme = coreScheme + specified.ByObject = map[client.Object]ByObject{ + &corev1.Pod{}: { + UnsafeDisableDeepCopy: pointer.Bool(true), + }, + } + inherited.ByObject = map[client.Object]ByObject{ + &corev1.Pod{}: { + UnsafeDisableDeepCopy: pointer.Bool(false), + }, + } + combined := checkError(specified.inheritFrom(inherited)).ByObject + Expect(combined).To(HaveLen(1)) + + var ( + obj client.Object + byObject ByObject + ) + for obj, byObject = range combined { + } + Expect(obj).To(BeAssignableToTypeOf(&corev1.Pod{})) + Expect(byObject.UnsafeDisableDeepCopy).ToNot(BeNil()) + Expect(*byObject.UnsafeDisableDeepCopy).To(BeTrue()) + }) + It("is false when inherited=true and specified=false for the same key", func() { + specified.Scheme = coreScheme + inherited.Scheme = coreScheme + specified.ByObject = map[client.Object]ByObject{ + &corev1.Pod{}: { + UnsafeDisableDeepCopy: pointer.Bool(false), + }, + } + inherited.ByObject = map[client.Object]ByObject{ + &corev1.Pod{}: { + UnsafeDisableDeepCopy: pointer.Bool(true), + }, + } + combined := checkError(specified.inheritFrom(inherited)).ByObject + Expect(combined).To(HaveLen(1)) + + var ( + obj client.Object + byObject ByObject + ) + for obj, byObject = range combined { + } + Expect(obj).To(BeAssignableToTypeOf(&corev1.Pod{})) + Expect(byObject.UnsafeDisableDeepCopy).ToNot(BeNil()) + Expect(*byObject.UnsafeDisableDeepCopy).To(BeFalse()) + }) + }) + Context("TransformByObject", func() { + type transformed struct { + podSpecified bool + podInherited bool + configmapSpecified bool + configmapInherited bool + } + var tx transformed + BeforeEach(func() { + tx = transformed{} + }) + It("is unchanged when specified and inherited are unset", func() { + Expect(checkError(specified.inheritFrom(inherited)).ByObject).To(HaveLen(0)) + }) + It("is unchanged when only specified is set", func() { + specified.Scheme = coreScheme + specified.ByObject = map[client.Object]ByObject{ + &corev1.Pod{}: { + Transform: func(i interface{}) (interface{}, error) { + ti := i.(transformed) + ti.podSpecified = true + return ti, nil + }, + }, + } + combined := checkError(specified.inheritFrom(inherited)).ByObject + Expect(combined).To(HaveLen(1)) + for obj, byObject := range combined { + Expect(obj).To(BeAssignableToTypeOf(&corev1.Pod{})) + out, _ := byObject.Transform(tx) + Expect(out).To(And( + BeAssignableToTypeOf(tx), + WithTransform(func(i transformed) bool { return i.podSpecified }, BeTrue()), + WithTransform(func(i transformed) bool { return i.podInherited }, BeFalse()), + )) + } + }) + It("is inherited when only inherited is set", func() { + inherited.Scheme = coreScheme + inherited.ByObject = map[client.Object]ByObject{ + &corev1.Pod{}: { + Transform: func(i interface{}) (interface{}, error) { + ti := i.(transformed) + ti.podInherited = true + return ti, nil + }, + }, + } + combined := checkError(specified.inheritFrom(inherited)).ByObject + Expect(combined).To(HaveLen(1)) + for obj, byObject := range combined { + Expect(obj).To(BeAssignableToTypeOf(&corev1.Pod{})) + out, _ := byObject.Transform(tx) + Expect(out).To(And( + BeAssignableToTypeOf(tx), + WithTransform(func(i transformed) bool { return i.podSpecified }, BeFalse()), + WithTransform(func(i transformed) bool { return i.podInherited }, BeTrue()), + )) + } + }) + It("is combined when both inherited and specified are set for different keys", func() { + specified.Scheme = coreScheme + inherited.Scheme = coreScheme + specified.ByObject = map[client.Object]ByObject{ + &corev1.Pod{}: { + Transform: func(i interface{}) (interface{}, error) { + ti := i.(transformed) + ti.podSpecified = true + return ti, nil + }, + }, + } + inherited.ByObject = map[client.Object]ByObject{ + &corev1.ConfigMap{}: { + Transform: func(i interface{}) (interface{}, error) { + ti := i.(transformed) + ti.configmapInherited = true + return ti, nil + }, + }, + } + combined := checkError(specified.inheritFrom(inherited)).ByObject + Expect(combined).To(HaveLen(2)) + for obj, byObject := range combined { + out, _ := byObject.Transform(tx) + if reflect.TypeOf(obj) == reflect.TypeOf(&corev1.Pod{}) { + Expect(out).To(And( + BeAssignableToTypeOf(tx), + WithTransform(func(i transformed) bool { return i.podSpecified }, BeTrue()), + WithTransform(func(i transformed) bool { return i.podInherited }, BeFalse()), + WithTransform(func(i transformed) bool { return i.configmapSpecified }, BeFalse()), + WithTransform(func(i transformed) bool { return i.configmapInherited }, BeFalse()), + )) + } + if reflect.TypeOf(obj) == reflect.TypeOf(&corev1.ConfigMap{}) { + Expect(out).To(And( + BeAssignableToTypeOf(tx), + WithTransform(func(i transformed) bool { return i.podSpecified }, BeFalse()), + WithTransform(func(i transformed) bool { return i.podInherited }, BeFalse()), + WithTransform(func(i transformed) bool { return i.configmapSpecified }, BeFalse()), + WithTransform(func(i transformed) bool { return i.configmapInherited }, BeTrue()), + )) + } + } + }) + It("is combined into a single transform function when both inherited and specified are set for the same key", func() { + specified.Scheme = coreScheme + inherited.Scheme = coreScheme + specified.ByObject = map[client.Object]ByObject{ + &corev1.Pod{}: { + Transform: func(i interface{}) (interface{}, error) { + ti := i.(transformed) + ti.podSpecified = true + return ti, nil + }, + }, + } + inherited.ByObject = map[client.Object]ByObject{ + &corev1.Pod{}: { + Transform: func(i interface{}) (interface{}, error) { + ti := i.(transformed) + ti.podInherited = true + return ti, nil + }, + }, + } + combined := checkError(specified.inheritFrom(inherited)).ByObject + Expect(combined).To(HaveLen(1)) + for obj, byObject := range combined { + Expect(obj).To(BeAssignableToTypeOf(&corev1.Pod{})) + out, _ := byObject.Transform(tx) + Expect(out).To(And( + BeAssignableToTypeOf(tx), + WithTransform(func(i transformed) bool { return i.podSpecified }, BeTrue()), + WithTransform(func(i transformed) bool { return i.podInherited }, BeTrue()), + WithTransform(func(i transformed) bool { return i.configmapSpecified }, BeFalse()), + WithTransform(func(i transformed) bool { return i.configmapInherited }, BeFalse()), + )) + } + }) + }) + Context("DefaultTransform", func() { + type transformed struct { + specified bool + inherited bool + } + var tx transformed + BeforeEach(func() { + tx = transformed{} + }) + It("is unchanged when specified and inherited are unset", func() { + Expect(checkError(specified.inheritFrom(inherited)).DefaultTransform).To(BeNil()) + }) + It("is unchanged when only specified is set", func() { + specified.DefaultTransform = func(i interface{}) (interface{}, error) { + ti := i.(transformed) + ti.specified = true + return ti, nil + } + combined := checkError(specified.inheritFrom(inherited)).DefaultTransform + out, _ := combined(tx) + Expect(out).To(And( + BeAssignableToTypeOf(tx), + WithTransform(func(i transformed) bool { return i.specified }, BeTrue()), + WithTransform(func(i transformed) bool { return i.inherited }, BeFalse()), + )) + }) + It("is inherited when only inherited is set", func() { + inherited.DefaultTransform = func(i interface{}) (interface{}, error) { + ti := i.(transformed) + ti.inherited = true + return ti, nil + } + combined := checkError(specified.inheritFrom(inherited)).DefaultTransform + out, _ := combined(tx) + Expect(out).To(And( + BeAssignableToTypeOf(tx), + WithTransform(func(i transformed) bool { return i.specified }, BeFalse()), + WithTransform(func(i transformed) bool { return i.inherited }, BeTrue()), + )) + }) + It("is combined when the transform function is defined in both inherited and specified", func() { + specified.DefaultTransform = func(i interface{}) (interface{}, error) { + ti := i.(transformed) + ti.specified = true + return ti, nil + } + inherited.DefaultTransform = func(i interface{}) (interface{}, error) { + ti := i.(transformed) + ti.inherited = true + return ti, nil + } + combined := checkError(specified.inheritFrom(inherited)).DefaultTransform + Expect(combined).NotTo(BeNil()) + out, _ := combined(tx) + Expect(out).To(And( + BeAssignableToTypeOf(tx), + WithTransform(func(i transformed) bool { return i.specified }, BeTrue()), + WithTransform(func(i transformed) bool { return i.inherited }, BeTrue()), + )) + }) + }) +}) + +func checkError[T any](v T, err error) T { + Expect(err).To(BeNil()) + return v +} diff --git a/pkg/cache/informer_cache.go b/pkg/cache/informer_cache.go index 90647c8e33..771244d52a 100644 --- a/pkg/cache/informer_cache.go +++ b/pkg/cache/informer_cache.go @@ -19,10 +19,10 @@ package cache import ( "context" "fmt" - "reflect" "strings" apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -45,19 +45,21 @@ func (*ErrCacheNotStarted) Error() string { return "the cache is not started, can not read objects" } -// informerCache is a Kubernetes Object cache populated from InformersMap. informerCache wraps an InformersMap. +// informerCache is a Kubernetes Object cache populated from internal.Informers. +// informerCache wraps internal.Informers. type informerCache struct { - *internal.InformersMap + scheme *runtime.Scheme + *internal.Informers } // Get implements Reader. -func (ip *informerCache) Get(ctx context.Context, key client.ObjectKey, out client.Object) error { - gvk, err := apiutil.GVKForObject(out, ip.Scheme) +func (ic *informerCache) Get(ctx context.Context, key client.ObjectKey, out client.Object, opts ...client.GetOption) error { + gvk, err := apiutil.GVKForObject(out, ic.scheme) if err != nil { return err } - started, cache, err := ip.InformersMap.Get(ctx, gvk, out) + started, cache, err := ic.Informers.Get(ctx, gvk, out) if err != nil { return err } @@ -69,13 +71,13 @@ func (ip *informerCache) Get(ctx context.Context, key client.ObjectKey, out clie } // List implements Reader. -func (ip *informerCache) List(ctx context.Context, out client.ObjectList, opts ...client.ListOption) error { - gvk, cacheTypeObj, err := ip.objectTypeForListObject(out) +func (ic *informerCache) List(ctx context.Context, out client.ObjectList, opts ...client.ListOption) error { + gvk, cacheTypeObj, err := ic.objectTypeForListObject(out) if err != nil { return err } - started, cache, err := ip.InformersMap.Get(ctx, *gvk, cacheTypeObj) + started, cache, err := ic.Informers.Get(ctx, *gvk, cacheTypeObj) if err != nil { return err } @@ -90,54 +92,46 @@ func (ip *informerCache) List(ctx context.Context, out client.ObjectList, opts . // objectTypeForListObject tries to find the runtime.Object and associated GVK // for a single object corresponding to the passed-in list type. We need them // because they are used as cache map key. -func (ip *informerCache) objectTypeForListObject(list client.ObjectList) (*schema.GroupVersionKind, runtime.Object, error) { - gvk, err := apiutil.GVKForObject(list, ip.Scheme) +func (ic *informerCache) objectTypeForListObject(list client.ObjectList) (*schema.GroupVersionKind, runtime.Object, error) { + gvk, err := apiutil.GVKForObject(list, ic.scheme) if err != nil { return nil, nil, err } - if !strings.HasSuffix(gvk.Kind, "List") { - return nil, nil, fmt.Errorf("non-list type %T (kind %q) passed as output", list, gvk) - } - // we need the non-list GVK, so chop off the "List" from the end of the kind - gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] - _, isUnstructured := list.(*unstructured.UnstructuredList) - var cacheTypeObj runtime.Object - if isUnstructured { + // We need the non-list GVK, so chop off the "List" from the end of the kind. + gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") + + // Handle unstructured.UnstructuredList. + if _, isUnstructured := list.(runtime.Unstructured); isUnstructured { u := &unstructured.Unstructured{} u.SetGroupVersionKind(gvk) - cacheTypeObj = u - } else { - itemsPtr, err := apimeta.GetItemsPtr(list) - if err != nil { - return nil, nil, err - } - // http://knowyourmeme.com/memes/this-is-fine - elemType := reflect.Indirect(reflect.ValueOf(itemsPtr)).Type().Elem() - if elemType.Kind() != reflect.Ptr { - elemType = reflect.PtrTo(elemType) - } - - cacheTypeValue := reflect.Zero(elemType) - var ok bool - cacheTypeObj, ok = cacheTypeValue.Interface().(runtime.Object) - if !ok { - return nil, nil, fmt.Errorf("cannot get cache for %T, its element %T is not a runtime.Object", list, cacheTypeValue.Interface()) - } + return &gvk, u, nil + } + // Handle metav1.PartialObjectMetadataList. + if _, isPartialObjectMetadata := list.(*metav1.PartialObjectMetadataList); isPartialObjectMetadata { + pom := &metav1.PartialObjectMetadata{} + pom.SetGroupVersionKind(gvk) + return &gvk, pom, nil } + // Any other list type should have a corresponding non-list type registered + // in the scheme. Use that to create a new instance of the non-list type. + cacheTypeObj, err := ic.scheme.New(gvk) + if err != nil { + return nil, nil, err + } return &gvk, cacheTypeObj, nil } // GetInformerForKind returns the informer for the GroupVersionKind. -func (ip *informerCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (Informer, error) { +func (ic *informerCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (Informer, error) { // Map the gvk to an object - obj, err := ip.Scheme.New(gvk) + obj, err := ic.scheme.New(gvk) if err != nil { return nil, err } - _, i, err := ip.InformersMap.Get(ctx, gvk, obj) + _, i, err := ic.Informers.Get(ctx, gvk, obj) if err != nil { return nil, err } @@ -145,13 +139,13 @@ func (ip *informerCache) GetInformerForKind(ctx context.Context, gvk schema.Grou } // GetInformer returns the informer for the obj. -func (ip *informerCache) GetInformer(ctx context.Context, obj client.Object) (Informer, error) { - gvk, err := apiutil.GVKForObject(obj, ip.Scheme) +func (ic *informerCache) GetInformer(ctx context.Context, obj client.Object) (Informer, error) { + gvk, err := apiutil.GVKForObject(obj, ic.scheme) if err != nil { return nil, err } - _, i, err := ip.InformersMap.Get(ctx, gvk, obj) + _, i, err := ic.Informers.Get(ctx, gvk, obj) if err != nil { return nil, err } @@ -160,7 +154,7 @@ func (ip *informerCache) GetInformer(ctx context.Context, obj client.Object) (In // NeedLeaderElection implements the LeaderElectionRunnable interface // to indicate that this can be started without requiring the leader lock. -func (ip *informerCache) NeedLeaderElection() bool { +func (ic *informerCache) NeedLeaderElection() bool { return false } @@ -169,8 +163,8 @@ func (ip *informerCache) NeedLeaderElection() bool { // to List. For one-to-one compatibility with "normal" field selectors, only return one value. // The values may be anything. They will automatically be prefixed with the namespace of the // given object, if present. The objects passed are guaranteed to be objects of the correct type. -func (ip *informerCache) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { - informer, err := ip.GetInformer(ctx, obj) +func (ic *informerCache) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { + informer, err := ic.GetInformer(ctx, obj) if err != nil { return err } @@ -193,8 +187,8 @@ func indexByField(indexer Informer, field string, extractor client.IndexerFunc) rawVals := extractor(obj) var vals []string if ns == "" { - // if we're not doubling the keys for the namespaced case, just re-use what was returned to us - vals = rawVals + // if we're not doubling the keys for the namespaced case, just create a new slice with same length + vals = make([]string, len(rawVals)) } else { // if we need to add non-namespaced versions too, double the length vals = make([]string, len(rawVals)*2) diff --git a/pkg/cache/informer_cache_test.go b/pkg/cache/informer_cache_test.go index 6a19eaa366..267fa139c8 100644 --- a/pkg/cache/informer_cache_test.go +++ b/pkg/cache/informer_cache_test.go @@ -17,7 +17,7 @@ limitations under the License. package cache_test import ( - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/rest" @@ -31,7 +31,9 @@ var _ = Describe("informerCache", func() { It("should not require LeaderElection", func() { cfg := &rest.Config{} - mapper, err := apiutil.NewDynamicRESTMapper(cfg, apiutil.WithLazyDiscovery) + httpClient, err := rest.HTTPClientFor(cfg) + Expect(err).ToNot(HaveOccurred()) + mapper, err := apiutil.NewDynamicRESTMapper(cfg, httpClient, apiutil.WithLazyDiscovery) Expect(err).ToNot(HaveOccurred()) c, err := cache.New(cfg, cache.Options{Mapper: mapper}) diff --git a/pkg/cache/informer_cache_unit_test.go b/pkg/cache/informer_cache_unit_test.go index 6f66e4bd89..130059bc40 100644 --- a/pkg/cache/informer_cache_unit_test.go +++ b/pkg/cache/informer_cache_unit_test.go @@ -17,10 +17,11 @@ limitations under the License. package cache import ( - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -38,7 +39,8 @@ const ( var _ = Describe("ip.objectTypeForListObject", func() { ip := &informerCache{ - InformersMap: &internal.InformersMap{Scheme: scheme.Scheme}, + scheme: scheme.Scheme, + Informers: &internal.Informers{}, } It("should find the object type for unstructured lists", func() { @@ -54,7 +56,21 @@ var _ = Describe("ip.objectTypeForListObject", func() { referenceUnstructured := &unstructured.Unstructured{} referenceUnstructured.SetGroupVersionKind(*gvk) Expect(obj).To(Equal(referenceUnstructured)) + }) + + It("should find the object type for partial object metadata lists", func() { + partialList := &metav1.PartialObjectMetadataList{} + partialList.APIVersion = ("v1") + partialList.Kind = "PodList" + gvk, obj, err := ip.objectTypeForListObject(partialList) + Expect(err).ToNot(HaveOccurred()) + Expect(gvk.Group).To(Equal("")) + Expect(gvk.Version).To(Equal("v1")) + Expect(gvk.Kind).To(Equal("Pod")) + referencePartial := &metav1.PartialObjectMetadata{} + referencePartial.SetGroupVersionKind(*gvk) + Expect(obj).To(Equal(referencePartial)) }) It("should find the object type of a list with a slice of literals items field", func() { @@ -63,21 +79,20 @@ var _ = Describe("ip.objectTypeForListObject", func() { Expect(gvk.Group).To(Equal("")) Expect(gvk.Version).To(Equal("v1")) Expect(gvk.Kind).To(Equal("Pod")) - var referencePod *corev1.Pod + referencePod := &corev1.Pod{} Expect(obj).To(Equal(referencePod)) - }) It("should find the object type of a list with a slice of pointers items field", func() { By("registering the type", func() { - ip.Scheme = runtime.NewScheme() + ip.scheme = runtime.NewScheme() err := (&crscheme.Builder{ GroupVersion: schema.GroupVersion{Group: itemPointerSliceTypeGroupName, Version: itemPointerSliceTypeVersion}, }). Register( &controllertest.UnconventionalListType{}, &controllertest.UnconventionalListTypeList{}, - ).AddToScheme(ip.Scheme) + ).AddToScheme(ip.scheme) Expect(err).To(BeNil()) }) @@ -87,7 +102,7 @@ var _ = Describe("ip.objectTypeForListObject", func() { Expect(gvk.Group).To(Equal(itemPointerSliceTypeGroupName)) Expect(gvk.Version).To(Equal(itemPointerSliceTypeVersion)) Expect(gvk.Kind).To(Equal("UnconventionalListType")) - var referenceObject *controllertest.UnconventionalListType + referenceObject := &controllertest.UnconventionalListType{} Expect(obj).To(Equal(referenceObject)) }) }) diff --git a/pkg/cache/informertest/fake_cache.go b/pkg/cache/informertest/fake_cache.go index e115d380f7..da3bf8e0d4 100644 --- a/pkg/cache/informertest/fake_cache.go +++ b/pkg/cache/informertest/fake_cache.go @@ -131,7 +131,7 @@ func (c *FakeInformers) IndexField(ctx context.Context, obj client.Object, field } // Get implements Cache. -func (c *FakeInformers) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error { +func (c *FakeInformers) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { return nil } diff --git a/pkg/cache/internal/cache_reader.go b/pkg/cache/internal/cache_reader.go index 5a495693ed..f78b083382 100644 --- a/pkg/cache/internal/cache_reader.go +++ b/pkg/cache/internal/cache_reader.go @@ -23,12 +23,11 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/selection" "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/internal/field/selector" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -46,10 +45,15 @@ type CacheReader struct { // scopeName is the scope of the resource (namespaced or cluster-scoped). scopeName apimeta.RESTScopeName + + // disableDeepCopy indicates not to deep copy objects during get or list objects. + // Be very careful with this, when enabled you must DeepCopy any object before mutating it, + // otherwise you will mutate the object in the cache. + disableDeepCopy bool } // Get checks the indexer for the object and writes a copy of it if found. -func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out client.Object) error { +func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out client.Object, opts ...client.GetOption) error { if c.scopeName == apimeta.RESTScopeNameRoot { key.Namespace = "" } @@ -76,9 +80,13 @@ func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out client.Ob return fmt.Errorf("cache contained %T, which is not an Object", obj) } - // deep copy to avoid mutating cache - // TODO(directxman12): revisit the decision to always deepcopy - obj = obj.(runtime.Object).DeepCopyObject() + if c.disableDeepCopy { + // skip deep copy which might be unsafe + // you must DeepCopy any object before mutating it outside + } else { + // deep copy to avoid mutating cache + obj = obj.(runtime.Object).DeepCopyObject() + } // Copy the value of the item in the cache to the returned value // TODO(directxman12): this is a terrible hack, pls fix (we should have deepcopyinto) @@ -88,7 +96,9 @@ func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out client.Ob return fmt.Errorf("cache had type %s, but %s was asked for", objVal.Type(), outVal.Type()) } reflect.Indirect(outVal).Set(reflect.Indirect(objVal)) - out.GetObjectKind().SetGroupVersionKind(c.groupVersionKind) + if !c.disableDeepCopy { + out.GetObjectKind().SetGroupVersionKind(c.groupVersionKind) + } return nil } @@ -105,7 +115,7 @@ func (c *CacheReader) List(_ context.Context, out client.ObjectList, opts ...cli case listOpts.FieldSelector != nil: // TODO(directxman12): support more complicated field selectors by // combining multiple indices, GetIndexers, etc - field, val, requiresExact := requiresExactMatch(listOpts.FieldSelector) + field, val, requiresExact := selector.RequiresExactMatch(listOpts.FieldSelector) if !requiresExact { return fmt.Errorf("non-exact field matches are not supported by the cache") } @@ -129,10 +139,10 @@ func (c *CacheReader) List(_ context.Context, out client.ObjectList, opts ...cli limitSet := listOpts.Limit > 0 runtimeObjs := make([]runtime.Object, 0, len(objs)) - for i, item := range objs { + for _, item := range objs { // if the Limit option is set and the number of items // listed exceeds this limit, then stop reading. - if limitSet && int64(i) >= listOpts.Limit { + if limitSet && int64(len(runtimeObjs)) >= listOpts.Limit { break } obj, isObj := item.(runtime.Object) @@ -150,8 +160,15 @@ func (c *CacheReader) List(_ context.Context, out client.ObjectList, opts ...cli } } - outObj := obj.DeepCopyObject() - outObj.GetObjectKind().SetGroupVersionKind(c.groupVersionKind) + var outObj runtime.Object + if c.disableDeepCopy || (listOpts.UnsafeDisableDeepCopy != nil && *listOpts.UnsafeDisableDeepCopy) { + // skip deep copy which might be unsafe + // you must DeepCopy any object before mutating it outside + outObj = obj + } else { + outObj = obj.DeepCopyObject() + outObj.GetObjectKind().SetGroupVersionKind(c.groupVersionKind) + } runtimeObjs = append(runtimeObjs, outObj) } return apimeta.SetList(out, runtimeObjs) @@ -168,19 +185,6 @@ func objectKeyToStoreKey(k client.ObjectKey) string { return k.Namespace + "/" + k.Name } -// requiresExactMatch checks if the given field selector is of the form `k=v` or `k==v`. -func requiresExactMatch(sel fields.Selector) (field, val string, required bool) { - reqs := sel.Requirements() - if len(reqs) != 1 { - return "", "", false - } - req := reqs[0] - if req.Operator != selection.Equals && req.Operator != selection.DoubleEquals { - return "", "", false - } - return req.Field, req.Value, true -} - // FieldIndexName constructs the name of the index over the given field, // for use with an indexer. func FieldIndexName(field string) string { diff --git a/pkg/cache/internal/deleg_map.go b/pkg/cache/internal/deleg_map.go deleted file mode 100644 index 841f1657eb..0000000000 --- a/pkg/cache/internal/deleg_map.go +++ /dev/null @@ -1,124 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package internal - -import ( - "context" - "time" - - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/cache" -) - -// InformersMap create and caches Informers for (runtime.Object, schema.GroupVersionKind) pairs. -// It uses a standard parameter codec constructed based on the given generated Scheme. -type InformersMap struct { - // we abstract over the details of structured/unstructured/metadata with the specificInformerMaps - // TODO(directxman12): genericize this over different projections now that we have 3 different maps - - structured *specificInformersMap - unstructured *specificInformersMap - metadata *specificInformersMap - - // Scheme maps runtime.Objects to GroupVersionKinds - Scheme *runtime.Scheme -} - -// NewInformersMap creates a new InformersMap that can create informers for -// both structured and unstructured objects. -func NewInformersMap(config *rest.Config, - scheme *runtime.Scheme, - mapper meta.RESTMapper, - resync time.Duration, - namespace string, - selectors SelectorsByGVK, -) *InformersMap { - return &InformersMap{ - structured: newStructuredInformersMap(config, scheme, mapper, resync, namespace, selectors), - unstructured: newUnstructuredInformersMap(config, scheme, mapper, resync, namespace, selectors), - metadata: newMetadataInformersMap(config, scheme, mapper, resync, namespace, selectors), - - Scheme: scheme, - } -} - -// Start calls Run on each of the informers and sets started to true. Blocks on the context. -func (m *InformersMap) Start(ctx context.Context) error { - go m.structured.Start(ctx) - go m.unstructured.Start(ctx) - go m.metadata.Start(ctx) - <-ctx.Done() - return nil -} - -// WaitForCacheSync waits until all the caches have been started and synced. -func (m *InformersMap) WaitForCacheSync(ctx context.Context) bool { - syncedFuncs := append([]cache.InformerSynced(nil), m.structured.HasSyncedFuncs()...) - syncedFuncs = append(syncedFuncs, m.unstructured.HasSyncedFuncs()...) - syncedFuncs = append(syncedFuncs, m.metadata.HasSyncedFuncs()...) - - if !m.structured.waitForStarted(ctx) { - return false - } - if !m.unstructured.waitForStarted(ctx) { - return false - } - if !m.metadata.waitForStarted(ctx) { - return false - } - return cache.WaitForCacheSync(ctx.Done(), syncedFuncs...) -} - -// Get will create a new Informer and add it to the map of InformersMap if none exists. Returns -// the Informer from the map. -func (m *InformersMap) Get(ctx context.Context, gvk schema.GroupVersionKind, obj runtime.Object) (bool, *MapEntry, error) { - switch obj.(type) { - case *unstructured.Unstructured: - return m.unstructured.Get(ctx, gvk, obj) - case *unstructured.UnstructuredList: - return m.unstructured.Get(ctx, gvk, obj) - case *metav1.PartialObjectMetadata: - return m.metadata.Get(ctx, gvk, obj) - case *metav1.PartialObjectMetadataList: - return m.metadata.Get(ctx, gvk, obj) - default: - return m.structured.Get(ctx, gvk, obj) - } -} - -// newStructuredInformersMap creates a new InformersMap for structured objects. -func newStructuredInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration, - namespace string, selectors SelectorsByGVK) *specificInformersMap { - return newSpecificInformersMap(config, scheme, mapper, resync, namespace, selectors, createStructuredListWatch) -} - -// newUnstructuredInformersMap creates a new InformersMap for unstructured objects. -func newUnstructuredInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration, - namespace string, selectors SelectorsByGVK) *specificInformersMap { - return newSpecificInformersMap(config, scheme, mapper, resync, namespace, selectors, createUnstructuredListWatch) -} - -// newMetadataInformersMap creates a new InformersMap for metadata-only objects. -func newMetadataInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration, - namespace string, selectors SelectorsByGVK) *specificInformersMap { - return newSpecificInformersMap(config, scheme, mapper, resync, namespace, selectors, createMetadataListWatch) -} diff --git a/pkg/cache/internal/informers.go b/pkg/cache/internal/informers.go new file mode 100644 index 0000000000..02d770e660 --- /dev/null +++ b/pkg/cache/internal/informers.go @@ -0,0 +1,564 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "context" + "fmt" + "math/rand" + "net/http" + "sync" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/metadata" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" +) + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +// InformersOpts configures an InformerMap. +type InformersOpts struct { + HTTPClient *http.Client + Scheme *runtime.Scheme + Mapper meta.RESTMapper + ResyncPeriod time.Duration + Namespace string + ByGVK map[schema.GroupVersionKind]InformersOptsByGVK +} + +// InformersOptsByGVK configured additional by group version kind (or object) +// in an InformerMap. +type InformersOptsByGVK struct { + Selector Selector + Transform cache.TransformFunc + UnsafeDisableDeepCopy *bool +} + +// NewInformers creates a new InformersMap that can create informers under the hood. +func NewInformers(config *rest.Config, options *InformersOpts) *Informers { + return &Informers{ + config: config, + httpClient: options.HTTPClient, + scheme: options.Scheme, + mapper: options.Mapper, + tracker: tracker{ + Structured: make(map[schema.GroupVersionKind]*Cache), + Unstructured: make(map[schema.GroupVersionKind]*Cache), + Metadata: make(map[schema.GroupVersionKind]*Cache), + }, + codecs: serializer.NewCodecFactory(options.Scheme), + paramCodec: runtime.NewParameterCodec(options.Scheme), + resync: options.ResyncPeriod, + startWait: make(chan struct{}), + namespace: options.Namespace, + byGVK: options.ByGVK, + } +} + +// Cache contains the cached data for an Cache. +type Cache struct { + // Informer is the cached informer + Informer cache.SharedIndexInformer + + // CacheReader wraps Informer and implements the CacheReader interface for a single type + Reader CacheReader +} + +type tracker struct { + Structured map[schema.GroupVersionKind]*Cache + Unstructured map[schema.GroupVersionKind]*Cache + Metadata map[schema.GroupVersionKind]*Cache +} + +// Informers create and caches Informers for (runtime.Object, schema.GroupVersionKind) pairs. +// It uses a standard parameter codec constructed based on the given generated Scheme. +type Informers struct { + // httpClient is used to create a new REST client + httpClient *http.Client + + // scheme maps runtime.Objects to GroupVersionKinds + scheme *runtime.Scheme + + // config is used to talk to the apiserver + config *rest.Config + + // mapper maps GroupVersionKinds to Resources + mapper meta.RESTMapper + + // tracker tracks informers keyed by their type and groupVersionKind + tracker tracker + + // codecs is used to create a new REST client + codecs serializer.CodecFactory + + // paramCodec is used by list and watch + paramCodec runtime.ParameterCodec + + // resync is the base frequency the informers are resynced + // a 10 percent jitter will be added to the resync period between informers + // so that all informers will not send list requests simultaneously. + resync time.Duration + + // mu guards access to the map + mu sync.RWMutex + + // started is true if the informers have been started + started bool + + // startWait is a channel that is closed after the + // informer has been started. + startWait chan struct{} + + // waitGroup is the wait group that is used to wait for all informers to stop + waitGroup sync.WaitGroup + + // stopped is true if the informers have been stopped + stopped bool + + // ctx is the context to stop informers + ctx context.Context + + // namespace is the namespace that all ListWatches are restricted to + // default or empty string means all namespaces + namespace string + + byGVK map[schema.GroupVersionKind]InformersOptsByGVK +} + +func (ip *Informers) getSelector(gvk schema.GroupVersionKind) Selector { + if ip.byGVK == nil { + return Selector{} + } + if res, ok := ip.byGVK[gvk]; ok { + return res.Selector + } + if res, ok := ip.byGVK[schema.GroupVersionKind{}]; ok { + return res.Selector + } + return Selector{} +} + +func (ip *Informers) getTransform(gvk schema.GroupVersionKind) cache.TransformFunc { + if ip.byGVK == nil { + return nil + } + if res, ok := ip.byGVK[gvk]; ok { + return res.Transform + } + if res, ok := ip.byGVK[schema.GroupVersionKind{}]; ok { + return res.Transform + } + return nil +} + +func (ip *Informers) getDisableDeepCopy(gvk schema.GroupVersionKind) bool { + if ip.byGVK == nil { + return false + } + if res, ok := ip.byGVK[gvk]; ok && res.UnsafeDisableDeepCopy != nil { + return *res.UnsafeDisableDeepCopy + } + if res, ok := ip.byGVK[schema.GroupVersionKind{}]; ok && res.UnsafeDisableDeepCopy != nil { + return *res.UnsafeDisableDeepCopy + } + return false +} + +// Start calls Run on each of the informers and sets started to true. Blocks on the context. +// It doesn't return start because it can't return an error, and it's not a runnable directly. +func (ip *Informers) Start(ctx context.Context) error { + func() { + ip.mu.Lock() + defer ip.mu.Unlock() + + // Set the context so it can be passed to informers that are added later + ip.ctx = ctx + + // Start each informer + for _, i := range ip.tracker.Structured { + ip.startInformerLocked(i.Informer) + } + for _, i := range ip.tracker.Unstructured { + ip.startInformerLocked(i.Informer) + } + for _, i := range ip.tracker.Metadata { + ip.startInformerLocked(i.Informer) + } + + // Set started to true so we immediately start any informers added later. + ip.started = true + close(ip.startWait) + }() + <-ctx.Done() // Block until the context is done + ip.mu.Lock() + ip.stopped = true // Set stopped to true so we don't start any new informers + ip.mu.Unlock() + ip.waitGroup.Wait() // Block until all informers have stopped + return nil +} + +func (ip *Informers) startInformerLocked(informer cache.SharedIndexInformer) { + // Don't start the informer in case we are already waiting for the items in + // the waitGroup to finish, since waitGroups don't support waiting and adding + // at the same time. + if ip.stopped { + return + } + + ip.waitGroup.Add(1) + go func() { + defer ip.waitGroup.Done() + informer.Run(ip.ctx.Done()) + }() +} + +func (ip *Informers) waitForStarted(ctx context.Context) bool { + select { + case <-ip.startWait: + return true + case <-ctx.Done(): + return false + } +} + +// getHasSyncedFuncs returns all the HasSynced functions for the informers in this map. +func (ip *Informers) getHasSyncedFuncs() []cache.InformerSynced { + ip.mu.RLock() + defer ip.mu.RUnlock() + + res := make([]cache.InformerSynced, 0, + len(ip.tracker.Structured)+len(ip.tracker.Unstructured)+len(ip.tracker.Metadata), + ) + for _, i := range ip.tracker.Structured { + res = append(res, i.Informer.HasSynced) + } + for _, i := range ip.tracker.Unstructured { + res = append(res, i.Informer.HasSynced) + } + for _, i := range ip.tracker.Metadata { + res = append(res, i.Informer.HasSynced) + } + return res +} + +// WaitForCacheSync waits until all the caches have been started and synced. +func (ip *Informers) WaitForCacheSync(ctx context.Context) bool { + if !ip.waitForStarted(ctx) { + return false + } + return cache.WaitForCacheSync(ctx.Done(), ip.getHasSyncedFuncs()...) +} + +func (ip *Informers) get(gvk schema.GroupVersionKind, obj runtime.Object) (res *Cache, started bool, ok bool) { + ip.mu.RLock() + defer ip.mu.RUnlock() + i, ok := ip.informersByType(obj)[gvk] + return i, ip.started, ok +} + +// Get will create a new Informer and add it to the map of specificInformersMap if none exists. Returns +// the Informer from the map. +func (ip *Informers) Get(ctx context.Context, gvk schema.GroupVersionKind, obj runtime.Object) (bool, *Cache, error) { + // Return the informer if it is found + i, started, ok := ip.get(gvk, obj) + if !ok { + var err error + if i, started, err = ip.addInformerToMap(gvk, obj); err != nil { + return started, nil, err + } + } + + if started && !i.Informer.HasSynced() { + // Wait for it to sync before returning the Informer so that folks don't read from a stale cache. + if !cache.WaitForCacheSync(ctx.Done(), i.Informer.HasSynced) { + return started, nil, apierrors.NewTimeoutError(fmt.Sprintf("failed waiting for %T Informer to sync", obj), 0) + } + } + + return started, i, nil +} + +func (ip *Informers) informersByType(obj runtime.Object) map[schema.GroupVersionKind]*Cache { + switch obj.(type) { + case runtime.Unstructured: + return ip.tracker.Unstructured + case *metav1.PartialObjectMetadata, *metav1.PartialObjectMetadataList: + return ip.tracker.Metadata + default: + return ip.tracker.Structured + } +} + +func (ip *Informers) addInformerToMap(gvk schema.GroupVersionKind, obj runtime.Object) (*Cache, bool, error) { + ip.mu.Lock() + defer ip.mu.Unlock() + + // Check the cache to see if we already have an Informer. If we do, return the Informer. + // This is for the case where 2 routines tried to get the informer when it wasn't in the map + // so neither returned early, but the first one created it. + if i, ok := ip.informersByType(obj)[gvk]; ok { + return i, ip.started, nil + } + + // Create a NewSharedIndexInformer and add it to the map. + listWatcher, err := ip.makeListWatcher(gvk, obj) + if err != nil { + return nil, false, err + } + sharedIndexInformer := cache.NewSharedIndexInformer(&cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + ip.getSelector(gvk).ApplyToList(&opts) + return listWatcher.ListFunc(opts) + }, + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + ip.getSelector(gvk).ApplyToList(&opts) + opts.Watch = true // Watch needs to be set to true separately + return listWatcher.WatchFunc(opts) + }, + }, obj, calculateResyncPeriod(ip.resync), cache.Indexers{ + cache.NamespaceIndex: cache.MetaNamespaceIndexFunc, + }) + + // Check to see if there is a transformer for this gvk + if err := sharedIndexInformer.SetTransform(ip.getTransform(gvk)); err != nil { + return nil, false, err + } + + mapping, err := ip.mapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + return nil, false, err + } + + // Create the new entry and set it in the map. + i := &Cache{ + Informer: sharedIndexInformer, + Reader: CacheReader{ + indexer: sharedIndexInformer.GetIndexer(), + groupVersionKind: gvk, + scopeName: mapping.Scope.Name(), + disableDeepCopy: ip.getDisableDeepCopy(gvk), + }, + } + ip.informersByType(obj)[gvk] = i + + // Start the informer in case the InformersMap has started, otherwise it will be + // started when the InformersMap starts. + if ip.started { + ip.startInformerLocked(i.Informer) + } + return i, ip.started, nil +} + +func (ip *Informers) makeListWatcher(gvk schema.GroupVersionKind, obj runtime.Object) (*cache.ListWatch, error) { + // Kubernetes APIs work against Resources, not GroupVersionKinds. Map the + // groupVersionKind to the Resource API we will use. + mapping, err := ip.mapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + return nil, err + } + + // Figure out if the GVK we're dealing with is global, or namespace scoped. + var namespace string + if mapping.Scope.Name() == meta.RESTScopeNameNamespace { + namespace = restrictNamespaceBySelector(ip.namespace, ip.getSelector(gvk)) + } + + switch obj.(type) { + // + // Unstructured + // + case runtime.Unstructured: + // If the rest configuration has a negotiated serializer passed in, + // we should remove it and use the one that the dynamic client sets for us. + cfg := rest.CopyConfig(ip.config) + cfg.NegotiatedSerializer = nil + dynamicClient, err := dynamic.NewForConfigAndClient(cfg, ip.httpClient) + if err != nil { + return nil, err + } + resources := dynamicClient.Resource(mapping.Resource) + return &cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + if namespace != "" { + return resources.Namespace(namespace).List(ip.ctx, opts) + } + return resources.List(ip.ctx, opts) + }, + // Setup the watch function + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + if namespace != "" { + return resources.Namespace(namespace).Watch(ip.ctx, opts) + } + return resources.Watch(ip.ctx, opts) + }, + }, nil + // + // Metadata + // + case *metav1.PartialObjectMetadata, *metav1.PartialObjectMetadataList: + // Always clear the negotiated serializer and use the one + // set from the metadata client. + cfg := rest.CopyConfig(ip.config) + cfg.NegotiatedSerializer = nil + + // Grab the metadata metadataClient. + metadataClient, err := metadata.NewForConfigAndClient(cfg, ip.httpClient) + if err != nil { + return nil, err + } + resources := metadataClient.Resource(mapping.Resource) + + return &cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + var ( + list *metav1.PartialObjectMetadataList + err error + ) + if namespace != "" { + list, err = resources.Namespace(namespace).List(ip.ctx, opts) + } else { + list, err = resources.List(ip.ctx, opts) + } + if list != nil { + for i := range list.Items { + list.Items[i].SetGroupVersionKind(gvk) + } + } + return list, err + }, + // Setup the watch function + WatchFunc: func(opts metav1.ListOptions) (watcher watch.Interface, err error) { + if namespace != "" { + watcher, err = resources.Namespace(namespace).Watch(ip.ctx, opts) + } else { + watcher, err = resources.Watch(ip.ctx, opts) + } + if err != nil { + return nil, err + } + return newGVKFixupWatcher(gvk, watcher), nil + }, + }, nil + // + // Structured. + // + default: + client, err := apiutil.RESTClientForGVK(gvk, false, ip.config, ip.codecs, ip.httpClient) + if err != nil { + return nil, err + } + listGVK := gvk.GroupVersion().WithKind(gvk.Kind + "List") + listObj, err := ip.scheme.New(listGVK) + if err != nil { + return nil, err + } + return &cache.ListWatch{ + ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { + // Build the request. + req := client.Get().Resource(mapping.Resource.Resource).VersionedParams(&opts, ip.paramCodec) + if namespace != "" { + req.Namespace(namespace) + } + + // Create the resulting object, and execute the request. + res := listObj.DeepCopyObject() + if err := req.Do(ip.ctx).Into(res); err != nil { + return nil, err + } + return res, nil + }, + // Setup the watch function + WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { + // Build the request. + req := client.Get().Resource(mapping.Resource.Resource).VersionedParams(&opts, ip.paramCodec) + if namespace != "" { + req.Namespace(namespace) + } + // Call the watch. + return req.Watch(ip.ctx) + }, + }, nil + } +} + +// newGVKFixupWatcher adds a wrapper that preserves the GVK information when +// events come in. +// +// This works around a bug where GVK information is not passed into mapping +// functions when using the OnlyMetadata option in the builder. +// This issue is most likely caused by kubernetes/kubernetes#80609. +// See kubernetes-sigs/controller-runtime#1484. +// +// This was originally implemented as a cache.ResourceEventHandler wrapper but +// that contained a data race which was resolved by setting the GVK in a watch +// wrapper, before the objects are written to the cache. +// See kubernetes-sigs/controller-runtime#1650. +// +// The original watch wrapper was found to be incompatible with +// k8s.io/client-go/tools/cache.Reflector so it has been re-implemented as a +// watch.Filter which is compatible. +// See kubernetes-sigs/controller-runtime#1789. +func newGVKFixupWatcher(gvk schema.GroupVersionKind, watcher watch.Interface) watch.Interface { + return watch.Filter( + watcher, + func(in watch.Event) (watch.Event, bool) { + in.Object.GetObjectKind().SetGroupVersionKind(gvk) + return in, true + }, + ) +} + +// calculateResyncPeriod returns a duration based on the desired input +// this is so that multiple controllers don't get into lock-step and all +// hammer the apiserver with list requests simultaneously. +func calculateResyncPeriod(resync time.Duration) time.Duration { + // the factor will fall into [0.9, 1.1) + factor := rand.Float64()/5.0 + 0.9 //nolint:gosec + return time.Duration(float64(resync.Nanoseconds()) * factor) +} + +// restrictNamespaceBySelector returns either a global restriction for all ListWatches +// if not default/empty, or the namespace that a ListWatch for the specific resource +// is restricted to, based on a specified field selector for metadata.namespace field. +func restrictNamespaceBySelector(namespaceOpt string, s Selector) string { + if namespaceOpt != "" { + // namespace is already restricted + return namespaceOpt + } + fieldSelector := s.Field + if fieldSelector == nil || fieldSelector.Empty() { + return "" + } + // check whether a selector includes the namespace field + value, found := fieldSelector.RequiresExactMatch("metadata.namespace") + if found { + return value + } + return "" +} diff --git a/pkg/cache/internal/informers_map.go b/pkg/cache/internal/informers_map.go deleted file mode 100644 index bef54d302e..0000000000 --- a/pkg/cache/internal/informers_map.go +++ /dev/null @@ -1,388 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package internal - -import ( - "context" - "fmt" - "math/rand" - "sync" - "time" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/apimachinery/pkg/watch" - "k8s.io/client-go/dynamic" - "k8s.io/client-go/metadata" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/cache" - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" -) - -func init() { - rand.Seed(time.Now().UnixNano()) -} - -// clientListWatcherFunc knows how to create a ListWatcher. -type createListWatcherFunc func(gvk schema.GroupVersionKind, ip *specificInformersMap) (*cache.ListWatch, error) - -// newSpecificInformersMap returns a new specificInformersMap (like -// the generical InformersMap, except that it doesn't implement WaitForCacheSync). -func newSpecificInformersMap(config *rest.Config, - scheme *runtime.Scheme, - mapper meta.RESTMapper, - resync time.Duration, - namespace string, - selectors SelectorsByGVK, - createListWatcher createListWatcherFunc) *specificInformersMap { - ip := &specificInformersMap{ - config: config, - Scheme: scheme, - mapper: mapper, - informersByGVK: make(map[schema.GroupVersionKind]*MapEntry), - codecs: serializer.NewCodecFactory(scheme), - paramCodec: runtime.NewParameterCodec(scheme), - resync: resync, - startWait: make(chan struct{}), - createListWatcher: createListWatcher, - namespace: namespace, - selectors: selectors, - } - return ip -} - -// MapEntry contains the cached data for an Informer. -type MapEntry struct { - // Informer is the cached informer - Informer cache.SharedIndexInformer - - // CacheReader wraps Informer and implements the CacheReader interface for a single type - Reader CacheReader -} - -// specificInformersMap create and caches Informers for (runtime.Object, schema.GroupVersionKind) pairs. -// It uses a standard parameter codec constructed based on the given generated Scheme. -type specificInformersMap struct { - // Scheme maps runtime.Objects to GroupVersionKinds - Scheme *runtime.Scheme - - // config is used to talk to the apiserver - config *rest.Config - - // mapper maps GroupVersionKinds to Resources - mapper meta.RESTMapper - - // informersByGVK is the cache of informers keyed by groupVersionKind - informersByGVK map[schema.GroupVersionKind]*MapEntry - - // codecs is used to create a new REST client - codecs serializer.CodecFactory - - // paramCodec is used by list and watch - paramCodec runtime.ParameterCodec - - // stop is the stop channel to stop informers - stop <-chan struct{} - - // resync is the base frequency the informers are resynced - // a 10 percent jitter will be added to the resync period between informers - // so that all informers will not send list requests simultaneously. - resync time.Duration - - // mu guards access to the map - mu sync.RWMutex - - // start is true if the informers have been started - started bool - - // startWait is a channel that is closed after the - // informer has been started. - startWait chan struct{} - - // createClient knows how to create a client and a list object, - // and allows for abstracting over the particulars of structured vs - // unstructured objects. - createListWatcher createListWatcherFunc - - // namespace is the namespace that all ListWatches are restricted to - // default or empty string means all namespaces - namespace string - - // selectors are the label or field selectors that will be added to the - // ListWatch ListOptions. - selectors SelectorsByGVK -} - -// Start calls Run on each of the informers and sets started to true. Blocks on the context. -// It doesn't return start because it can't return an error, and it's not a runnable directly. -func (ip *specificInformersMap) Start(ctx context.Context) { - func() { - ip.mu.Lock() - defer ip.mu.Unlock() - - // Set the stop channel so it can be passed to informers that are added later - ip.stop = ctx.Done() - - // Start each informer - for _, informer := range ip.informersByGVK { - go informer.Informer.Run(ctx.Done()) - } - - // Set started to true so we immediately start any informers added later. - ip.started = true - close(ip.startWait) - }() - <-ctx.Done() -} - -func (ip *specificInformersMap) waitForStarted(ctx context.Context) bool { - select { - case <-ip.startWait: - return true - case <-ctx.Done(): - return false - } -} - -// HasSyncedFuncs returns all the HasSynced functions for the informers in this map. -func (ip *specificInformersMap) HasSyncedFuncs() []cache.InformerSynced { - ip.mu.RLock() - defer ip.mu.RUnlock() - syncedFuncs := make([]cache.InformerSynced, 0, len(ip.informersByGVK)) - for _, informer := range ip.informersByGVK { - syncedFuncs = append(syncedFuncs, informer.Informer.HasSynced) - } - return syncedFuncs -} - -// Get will create a new Informer and add it to the map of specificInformersMap if none exists. Returns -// the Informer from the map. -func (ip *specificInformersMap) Get(ctx context.Context, gvk schema.GroupVersionKind, obj runtime.Object) (bool, *MapEntry, error) { - // Return the informer if it is found - i, started, ok := func() (*MapEntry, bool, bool) { - ip.mu.RLock() - defer ip.mu.RUnlock() - i, ok := ip.informersByGVK[gvk] - return i, ip.started, ok - }() - - if !ok { - var err error - if i, started, err = ip.addInformerToMap(gvk, obj); err != nil { - return started, nil, err - } - } - - if started && !i.Informer.HasSynced() { - // Wait for it to sync before returning the Informer so that folks don't read from a stale cache. - if !cache.WaitForCacheSync(ctx.Done(), i.Informer.HasSynced) { - return started, nil, apierrors.NewTimeoutError(fmt.Sprintf("failed waiting for %T Informer to sync", obj), 0) - } - } - - return started, i, nil -} - -func (ip *specificInformersMap) addInformerToMap(gvk schema.GroupVersionKind, obj runtime.Object) (*MapEntry, bool, error) { - ip.mu.Lock() - defer ip.mu.Unlock() - - // Check the cache to see if we already have an Informer. If we do, return the Informer. - // This is for the case where 2 routines tried to get the informer when it wasn't in the map - // so neither returned early, but the first one created it. - if i, ok := ip.informersByGVK[gvk]; ok { - return i, ip.started, nil - } - - // Create a NewSharedIndexInformer and add it to the map. - var lw *cache.ListWatch - lw, err := ip.createListWatcher(gvk, ip) - if err != nil { - return nil, false, err - } - ni := cache.NewSharedIndexInformer(lw, obj, resyncPeriod(ip.resync)(), cache.Indexers{ - cache.NamespaceIndex: cache.MetaNamespaceIndexFunc, - }) - rm, err := ip.mapper.RESTMapping(gvk.GroupKind(), gvk.Version) - if err != nil { - return nil, false, err - } - - switch obj.(type) { - case *metav1.PartialObjectMetadata, *metav1.PartialObjectMetadataList: - ni = metadataSharedIndexInformerPreserveGVK(gvk, ni) - default: - } - - i := &MapEntry{ - Informer: ni, - Reader: CacheReader{indexer: ni.GetIndexer(), groupVersionKind: gvk, scopeName: rm.Scope.Name()}, - } - ip.informersByGVK[gvk] = i - - // Start the Informer if need by - // TODO(seans): write thorough tests and document what happens here - can you add indexers? - // can you add eventhandlers? - if ip.started { - go i.Informer.Run(ip.stop) - } - return i, ip.started, nil -} - -// newListWatch returns a new ListWatch object that can be used to create a SharedIndexInformer. -func createStructuredListWatch(gvk schema.GroupVersionKind, ip *specificInformersMap) (*cache.ListWatch, error) { - // Kubernetes APIs work against Resources, not GroupVersionKinds. Map the - // groupVersionKind to the Resource API we will use. - mapping, err := ip.mapper.RESTMapping(gvk.GroupKind(), gvk.Version) - if err != nil { - return nil, err - } - - client, err := apiutil.RESTClientForGVK(gvk, false, ip.config, ip.codecs) - if err != nil { - return nil, err - } - listGVK := gvk.GroupVersion().WithKind(gvk.Kind + "List") - listObj, err := ip.Scheme.New(listGVK) - if err != nil { - return nil, err - } - - // TODO: the functions that make use of this ListWatch should be adapted to - // pass in their own contexts instead of relying on this fixed one here. - ctx := context.TODO() - // Create a new ListWatch for the obj - return &cache.ListWatch{ - ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - ip.selectors[gvk].ApplyToList(&opts) - res := listObj.DeepCopyObject() - isNamespaceScoped := ip.namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot - err := client.Get().NamespaceIfScoped(ip.namespace, isNamespaceScoped).Resource(mapping.Resource.Resource).VersionedParams(&opts, ip.paramCodec).Do(ctx).Into(res) - return res, err - }, - // Setup the watch function - WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - ip.selectors[gvk].ApplyToList(&opts) - // Watch needs to be set to true separately - opts.Watch = true - isNamespaceScoped := ip.namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot - return client.Get().NamespaceIfScoped(ip.namespace, isNamespaceScoped).Resource(mapping.Resource.Resource).VersionedParams(&opts, ip.paramCodec).Watch(ctx) - }, - }, nil -} - -func createUnstructuredListWatch(gvk schema.GroupVersionKind, ip *specificInformersMap) (*cache.ListWatch, error) { - // Kubernetes APIs work against Resources, not GroupVersionKinds. Map the - // groupVersionKind to the Resource API we will use. - mapping, err := ip.mapper.RESTMapping(gvk.GroupKind(), gvk.Version) - if err != nil { - return nil, err - } - - // If the rest configuration has a negotiated serializer passed in, - // we should remove it and use the one that the dynamic client sets for us. - cfg := rest.CopyConfig(ip.config) - cfg.NegotiatedSerializer = nil - dynamicClient, err := dynamic.NewForConfig(cfg) - if err != nil { - return nil, err - } - - // TODO: the functions that make use of this ListWatch should be adapted to - // pass in their own contexts instead of relying on this fixed one here. - ctx := context.TODO() - // Create a new ListWatch for the obj - return &cache.ListWatch{ - ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - ip.selectors[gvk].ApplyToList(&opts) - if ip.namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot { - return dynamicClient.Resource(mapping.Resource).Namespace(ip.namespace).List(ctx, opts) - } - return dynamicClient.Resource(mapping.Resource).List(ctx, opts) - }, - // Setup the watch function - WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - ip.selectors[gvk].ApplyToList(&opts) - // Watch needs to be set to true separately - opts.Watch = true - if ip.namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot { - return dynamicClient.Resource(mapping.Resource).Namespace(ip.namespace).Watch(ctx, opts) - } - return dynamicClient.Resource(mapping.Resource).Watch(ctx, opts) - }, - }, nil -} - -func createMetadataListWatch(gvk schema.GroupVersionKind, ip *specificInformersMap) (*cache.ListWatch, error) { - // Kubernetes APIs work against Resources, not GroupVersionKinds. Map the - // groupVersionKind to the Resource API we will use. - mapping, err := ip.mapper.RESTMapping(gvk.GroupKind(), gvk.Version) - if err != nil { - return nil, err - } - - // Always clear the negotiated serializer and use the one - // set from the metadata client. - cfg := rest.CopyConfig(ip.config) - cfg.NegotiatedSerializer = nil - - // grab the metadata client - client, err := metadata.NewForConfig(cfg) - if err != nil { - return nil, err - } - - // TODO: the functions that make use of this ListWatch should be adapted to - // pass in their own contexts instead of relying on this fixed one here. - ctx := context.TODO() - - // create the relevant listwatch - return &cache.ListWatch{ - ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { - ip.selectors[gvk].ApplyToList(&opts) - if ip.namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot { - return client.Resource(mapping.Resource).Namespace(ip.namespace).List(ctx, opts) - } - return client.Resource(mapping.Resource).List(ctx, opts) - }, - // Setup the watch function - WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { - ip.selectors[gvk].ApplyToList(&opts) - // Watch needs to be set to true separately - opts.Watch = true - if ip.namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot { - return client.Resource(mapping.Resource).Namespace(ip.namespace).Watch(ctx, opts) - } - return client.Resource(mapping.Resource).Watch(ctx, opts) - }, - }, nil -} - -// resyncPeriod returns a function which generates a duration each time it is -// invoked; this is so that multiple controllers don't get into lock-step and all -// hammer the apiserver with list requests simultaneously. -func resyncPeriod(resync time.Duration) func() time.Duration { - return func() time.Duration { - // the factor will fall into [0.9, 1.1) - factor := rand.Float64()/5.0 + 0.9 //nolint:gosec - return time.Duration(float64(resync.Nanoseconds()) * factor) - } -} diff --git a/pkg/cache/internal/informers_test.go b/pkg/cache/internal/informers_test.go new file mode 100644 index 0000000000..854a39c1f1 --- /dev/null +++ b/pkg/cache/internal/informers_test.go @@ -0,0 +1,94 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/watch" +) + +// Test that gvkFixupWatcher behaves like watch.FakeWatcher +// and that it overrides the GVK. +// These tests are adapted from the watch.FakeWatcher tests in: +// https://github.com/kubernetes/kubernetes/blob/adbda068c1808fcc8a64a94269e0766b5c46ec41/staging/src/k8s.io/apimachinery/pkg/watch/watch_test.go#L33-L78 +var _ = Describe("gvkFixupWatcher", func() { + It("behaves like watch.FakeWatcher", func() { + newTestType := func(name string) runtime.Object { + return &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + } + + f := watch.NewFake() + // This is the GVK which we expect the wrapper to set on all the events + expectedGVK := schema.GroupVersionKind{ + Group: "testgroup", + Version: "v1test2", + Kind: "TestKind", + } + gvkfw := newGVKFixupWatcher(expectedGVK, f) + + table := []struct { + t watch.EventType + s runtime.Object + }{ + {watch.Added, newTestType("foo")}, + {watch.Modified, newTestType("qux")}, + {watch.Modified, newTestType("bar")}, + {watch.Deleted, newTestType("bar")}, + {watch.Error, newTestType("error: blah")}, + } + + consumer := func(w watch.Interface) { + for _, expect := range table { + By(fmt.Sprintf("Fixing up watch.EventType: %v and passing it on", expect.t)) + got, ok := <-w.ResultChan() + Expect(ok).To(BeTrue(), "closed early") + Expect(expect.t).To(Equal(got.Type), "unexpected Event.Type or out-of-order Event") + Expect(got.Object).To(BeAssignableToTypeOf(&metav1.PartialObjectMetadata{}), "unexpected Event.Object type") + a := got.Object.(*metav1.PartialObjectMetadata) + Expect(got.Object.GetObjectKind().GroupVersionKind()).To(Equal(expectedGVK), "GVK was not fixed up") + expected := expect.s.DeepCopyObject() + expected.GetObjectKind().SetGroupVersionKind(schema.GroupVersionKind{}) + actual := a.DeepCopyObject() + actual.GetObjectKind().SetGroupVersionKind(schema.GroupVersionKind{}) + Expect(actual).To(Equal(expected), "unexpected change to the Object") + } + Eventually(w.ResultChan()).Should(BeClosed()) + } + + sender := func() { + f.Add(newTestType("foo")) + f.Action(watch.Modified, newTestType("qux")) + f.Modify(newTestType("bar")) + f.Delete(newTestType("bar")) + f.Error(newTestType("error: blah")) + f.Stop() + } + + go sender() + consumer(gvkfw) + }) +}) diff --git a/pkg/runtime/inject/inject_suite_test.go b/pkg/cache/internal/internal_suite_test.go similarity index 68% rename from pkg/runtime/inject/inject_suite_test.go rename to pkg/cache/internal/internal_suite_test.go index 98cf79ab3b..25ec0f1dbc 100644 --- a/pkg/runtime/inject/inject_suite_test.go +++ b/pkg/cache/internal/internal_suite_test.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2022 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,18 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. */ -package inject +package internal import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Runtime Injection Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Cache Internal Suite") } diff --git a/pkg/cache/internal/metadata_infomer_wrapper.go b/pkg/cache/internal/metadata_infomer_wrapper.go deleted file mode 100644 index c0fa24a5c1..0000000000 --- a/pkg/cache/internal/metadata_infomer_wrapper.go +++ /dev/null @@ -1,71 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package internal - -import ( - "time" - - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/tools/cache" -) - -func metadataSharedIndexInformerPreserveGVK(gvk schema.GroupVersionKind, si cache.SharedIndexInformer) cache.SharedIndexInformer { - return &sharedInformerWrapper{ - gvk: gvk, - SharedIndexInformer: si, - } -} - -type sharedInformerWrapper struct { - gvk schema.GroupVersionKind - cache.SharedIndexInformer -} - -func (s *sharedInformerWrapper) AddEventHandler(handler cache.ResourceEventHandler) { - s.SharedIndexInformer.AddEventHandler(&handlerPreserveGVK{s.gvk, handler}) -} - -func (s *sharedInformerWrapper) AddEventHandlerWithResyncPeriod(handler cache.ResourceEventHandler, resyncPeriod time.Duration) { - s.SharedIndexInformer.AddEventHandlerWithResyncPeriod(&handlerPreserveGVK{s.gvk, handler}, resyncPeriod) -} - -type handlerPreserveGVK struct { - gvk schema.GroupVersionKind - cache.ResourceEventHandler -} - -func (h *handlerPreserveGVK) resetGroupVersionKind(obj interface{}) { - if v, ok := obj.(schema.ObjectKind); ok { - v.SetGroupVersionKind(h.gvk) - } -} - -func (h *handlerPreserveGVK) OnAdd(obj interface{}) { - h.resetGroupVersionKind(obj) - h.ResourceEventHandler.OnAdd(obj) -} - -func (h *handlerPreserveGVK) OnUpdate(oldObj, newObj interface{}) { - h.resetGroupVersionKind(oldObj) - h.resetGroupVersionKind(newObj) - h.ResourceEventHandler.OnUpdate(oldObj, newObj) -} - -func (h *handlerPreserveGVK) OnDelete(obj interface{}) { - h.resetGroupVersionKind(obj) - h.ResourceEventHandler.OnDelete(obj) -} diff --git a/pkg/cache/internal/selector.go b/pkg/cache/internal/selector.go index cd9c580008..c674379b99 100644 --- a/pkg/cache/internal/selector.go +++ b/pkg/cache/internal/selector.go @@ -20,12 +20,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime/schema" ) -// SelectorsByGVK associate a GroupVersionKind to a field/label selector. -type SelectorsByGVK map[schema.GroupVersionKind]Selector - // Selector specify the label/field selector to fill in ListOptions. type Selector struct { Label labels.Selector diff --git a/pkg/cache/internal/transformers.go b/pkg/cache/internal/transformers.go new file mode 100644 index 0000000000..0725f550c5 --- /dev/null +++ b/pkg/cache/internal/transformers.go @@ -0,0 +1,55 @@ +package internal + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/tools/cache" + + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" +) + +// TransformFuncByGVK provides access to the correct transform function for +// any given GVK. +type TransformFuncByGVK interface { + Set(runtime.Object, *runtime.Scheme, cache.TransformFunc) error + Get(schema.GroupVersionKind) cache.TransformFunc + SetDefault(transformer cache.TransformFunc) +} + +type transformFuncByGVK struct { + defaultTransform cache.TransformFunc + transformers map[schema.GroupVersionKind]cache.TransformFunc +} + +// TransformFuncByGVKFromMap creates a TransformFuncByGVK from a map that +// maps GVKs to TransformFuncs. +func TransformFuncByGVKFromMap(in map[schema.GroupVersionKind]cache.TransformFunc) TransformFuncByGVK { + byGVK := &transformFuncByGVK{} + if defaultFunc, hasDefault := in[schema.GroupVersionKind{}]; hasDefault { + byGVK.defaultTransform = defaultFunc + } + delete(in, schema.GroupVersionKind{}) + byGVK.transformers = in + return byGVK +} + +func (t *transformFuncByGVK) SetDefault(transformer cache.TransformFunc) { + t.defaultTransform = transformer +} + +func (t *transformFuncByGVK) Set(obj runtime.Object, scheme *runtime.Scheme, transformer cache.TransformFunc) error { + gvk, err := apiutil.GVKForObject(obj, scheme) + if err != nil { + return err + } + + t.transformers[gvk] = transformer + return nil +} + +func (t transformFuncByGVK) Get(gvk schema.GroupVersionKind) cache.TransformFunc { + if val, ok := t.transformers[gvk]; ok { + return val + } + return t.defaultTransform +} diff --git a/pkg/cache/multi_namespace_cache.go b/pkg/cache/multi_namespace_cache.go index dc29651b01..ee33fb0dd5 100644 --- a/pkg/cache/multi_namespace_cache.go +++ b/pkg/cache/multi_namespace_cache.go @@ -28,7 +28,7 @@ import ( "k8s.io/client-go/rest" toolscache "k8s.io/client-go/tools/cache" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/internal/objectutil" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" ) // NewCacheFunc - Function for creating a new cache from the options and a rest config. @@ -43,31 +43,43 @@ const globalCache = "_cluster-scope" // a global cache for cluster scoped resource. Note that this is not intended // to be used for excluding namespaces, this is better done via a Predicate. Also note that // you may face performance issues when using this with a high number of namespaces. +// +// Deprecated: Use cache.Options.View.Namespaces instead. func MultiNamespacedCacheBuilder(namespaces []string) NewCacheFunc { return func(config *rest.Config, opts Options) (Cache, error) { - opts, err := defaultOpts(config, opts) - if err != nil { - return nil, err - } + opts.Namespaces = namespaces + return newMultiNamespaceCache(config, opts) + } +} - caches := map[string]Cache{} +func newMultiNamespaceCache(config *rest.Config, opts Options) (Cache, error) { + if len(opts.Namespaces) < 2 { + return nil, fmt.Errorf("must specify more than one namespace to use multi-namespace cache") + } + opts, err := defaultOpts(config, opts) + if err != nil { + return nil, err + } - // create a cache for cluster scoped resources - gCache, err := New(config, opts) + // Create every namespace cache. + caches := map[string]Cache{} + for _, ns := range opts.Namespaces { + opts.Namespaces = []string{ns} + c, err := New(config, opts) if err != nil { - return nil, fmt.Errorf("error creating global cache %v", err) + return nil, err } + caches[ns] = c + } - for _, ns := range namespaces { - opts.Namespace = ns - c, err := New(config, opts) - if err != nil { - return nil, err - } - caches[ns] = c - } - return &multiNamespaceCache{namespaceToCache: caches, Scheme: opts.Scheme, RESTMapper: opts.Mapper, clusterCache: gCache}, nil + // Create a cache for cluster scoped resources. + opts.Namespaces = []string{} + gCache, err := New(config, opts) + if err != nil { + return nil, fmt.Errorf("error creating global cache: %w", err) } + + return &multiNamespaceCache{namespaceToCache: caches, Scheme: opts.Scheme, RESTMapper: opts.Mapper, clusterCache: gCache}, nil } // multiNamespaceCache knows how to handle multiple namespaced caches @@ -89,7 +101,7 @@ func (c *multiNamespaceCache) GetInformer(ctx context.Context, obj client.Object // If the object is clusterscoped, get the informer from clusterCache, // if not use the namespaced caches. - isNamespaced, err := objectutil.IsAPINamespaced(obj, c.Scheme, c.RESTMapper) + isNamespaced, err := apiutil.IsObjectNamespaced(obj, c.Scheme, c.RESTMapper) if err != nil { return nil, err } @@ -119,7 +131,7 @@ func (c *multiNamespaceCache) GetInformerForKind(ctx context.Context, gvk schema // If the object is clusterscoped, get the informer from clusterCache, // if not use the namespaced caches. - isNamespaced, err := objectutil.IsAPINamespacedWithGVK(gvk, c.Scheme, c.RESTMapper) + isNamespaced, err := apiutil.IsGVKNamespaced(gvk, c.RESTMapper) if err != nil { return nil, err } @@ -183,7 +195,7 @@ func (c *multiNamespaceCache) WaitForCacheSync(ctx context.Context) bool { } func (c *multiNamespaceCache) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { - isNamespaced, err := objectutil.IsAPINamespaced(obj, c.Scheme, c.RESTMapper) + isNamespaced, err := apiutil.IsObjectNamespaced(obj, c.Scheme, c.RESTMapper) if err != nil { return nil //nolint:nilerr } @@ -200,8 +212,8 @@ func (c *multiNamespaceCache) IndexField(ctx context.Context, obj client.Object, return nil } -func (c *multiNamespaceCache) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error { - isNamespaced, err := objectutil.IsAPINamespaced(obj, c.Scheme, c.RESTMapper) +func (c *multiNamespaceCache) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + isNamespaced, err := apiutil.IsObjectNamespaced(obj, c.Scheme, c.RESTMapper) if err != nil { return err } @@ -223,7 +235,7 @@ func (c *multiNamespaceCache) List(ctx context.Context, list client.ObjectList, listOpts := client.ListOptions{} listOpts.ApplyOptions(opts) - isNamespaced, err := objectutil.IsAPINamespaced(list, c.Scheme, c.RESTMapper) + isNamespaced, err := apiutil.IsObjectNamespaced(list, c.Scheme, c.RESTMapper) if err != nil { return err } @@ -296,17 +308,47 @@ type multiNamespaceInformer struct { var _ Informer = &multiNamespaceInformer{} // AddEventHandler adds the handler to each namespaced informer. -func (i *multiNamespaceInformer) AddEventHandler(handler toolscache.ResourceEventHandler) { - for _, informer := range i.namespaceToInformer { - informer.AddEventHandler(handler) +func (i *multiNamespaceInformer) AddEventHandler(handler toolscache.ResourceEventHandler) (toolscache.ResourceEventHandlerRegistration, error) { + handles := make(map[string]toolscache.ResourceEventHandlerRegistration, len(i.namespaceToInformer)) + for ns, informer := range i.namespaceToInformer { + registration, err := informer.AddEventHandler(handler) + if err != nil { + return nil, err + } + handles[ns] = registration } + return handles, nil } // AddEventHandlerWithResyncPeriod adds the handler with a resync period to each namespaced informer. -func (i *multiNamespaceInformer) AddEventHandlerWithResyncPeriod(handler toolscache.ResourceEventHandler, resyncPeriod time.Duration) { - for _, informer := range i.namespaceToInformer { - informer.AddEventHandlerWithResyncPeriod(handler, resyncPeriod) +func (i *multiNamespaceInformer) AddEventHandlerWithResyncPeriod(handler toolscache.ResourceEventHandler, resyncPeriod time.Duration) (toolscache.ResourceEventHandlerRegistration, error) { + handles := make(map[string]toolscache.ResourceEventHandlerRegistration, len(i.namespaceToInformer)) + for ns, informer := range i.namespaceToInformer { + registration, err := informer.AddEventHandlerWithResyncPeriod(handler, resyncPeriod) + if err != nil { + return nil, err + } + handles[ns] = registration } + return handles, nil +} + +// RemoveEventHandler removes a formerly added event handler given by its registration handle. +func (i *multiNamespaceInformer) RemoveEventHandler(h toolscache.ResourceEventHandlerRegistration) error { + handles, ok := h.(map[string]toolscache.ResourceEventHandlerRegistration) + if !ok { + return fmt.Errorf("it is not the registration returned by multiNamespaceInformer") + } + for ns, informer := range i.namespaceToInformer { + registration, ok := handles[ns] + if !ok { + continue + } + if err := informer.RemoveEventHandler(registration); err != nil { + return err + } + } + return nil } // AddIndexers adds the indexer for each namespaced informer. diff --git a/pkg/certwatcher/certwatcher.go b/pkg/certwatcher/certwatcher.go index e8e0e17a2b..f7e6033b17 100644 --- a/pkg/certwatcher/certwatcher.go +++ b/pkg/certwatcher/certwatcher.go @@ -19,9 +19,15 @@ package certwatcher import ( "context" "crypto/tls" + "fmt" "sync" + "time" "github.com/fsnotify/fsnotify" + kerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics" logf "sigs.k8s.io/controller-runtime/pkg/internal/log" ) @@ -71,11 +77,24 @@ func (cw *CertWatcher) GetCertificate(_ *tls.ClientHelloInfo) (*tls.Certificate, // Start starts the watch on the certificate and key files. func (cw *CertWatcher) Start(ctx context.Context) error { - files := []string{cw.certPath, cw.keyPath} - - for _, f := range files { - if err := cw.watcher.Add(f); err != nil { - return err + files := sets.New(cw.certPath, cw.keyPath) + + { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + var watchErr error + if err := wait.PollImmediateUntilWithContext(ctx, 1*time.Second, func(ctx context.Context) (done bool, err error) { + for _, f := range files.UnsortedList() { + if err := cw.watcher.Add(f); err != nil { + watchErr = err + return false, nil //nolint:nilerr // We want to keep trying. + } + // We've added the watch, remove it from the set. + files.Delete(f) + } + return true, nil + }); err != nil { + return fmt.Errorf("failed to add watches: %w", kerrors.NewAggregate([]error{err, watchErr})) } } @@ -116,8 +135,10 @@ func (cw *CertWatcher) Watch() { // and updates the current certificate on the watcher. If a callback is set, it // is invoked with the new certificate. func (cw *CertWatcher) ReadCertificate() error { + metrics.ReadCertificateTotal.Inc() cert, err := tls.LoadX509KeyPair(cw.certPath, cw.keyPath) if err != nil { + metrics.ReadCertificateErrors.Inc() return err } @@ -151,13 +172,13 @@ func (cw *CertWatcher) handleEvent(event fsnotify.Event) { } func isWrite(event fsnotify.Event) bool { - return event.Op&fsnotify.Write == fsnotify.Write + return event.Op.Has(fsnotify.Write) } func isCreate(event fsnotify.Event) bool { - return event.Op&fsnotify.Create == fsnotify.Create + return event.Op.Has(fsnotify.Create) } func isRemove(event fsnotify.Event) bool { - return event.Op&fsnotify.Remove == fsnotify.Remove + return event.Op.Has(fsnotify.Remove) } diff --git a/pkg/certwatcher/certwatcher_suite_test.go b/pkg/certwatcher/certwatcher_suite_test.go index dfbd40a524..a44a968c89 100644 --- a/pkg/certwatcher/certwatcher_suite_test.go +++ b/pkg/certwatcher/certwatcher_suite_test.go @@ -20,9 +20,8 @@ import ( "os" "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) @@ -34,19 +33,15 @@ var ( func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "CertWatcher Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "CertWatcher Suite") } -var _ = BeforeSuite(func(done Done) { +var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) +}) - close(done) -}, 60) - -var _ = AfterSuite(func(done Done) { +var _ = AfterSuite(func() { for _, file := range []string{certPath, keyPath} { _ = os.Remove(file) } - close(done) -}, 60) +}) diff --git a/pkg/certwatcher/certwatcher_test.go b/pkg/certwatcher/certwatcher_test.go index 287645df92..c7349ea80d 100644 --- a/pkg/certwatcher/certwatcher_test.go +++ b/pkg/certwatcher/certwatcher_test.go @@ -23,14 +23,17 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "fmt" "math/big" "net" "os" "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus/testutil" "sigs.k8s.io/controller-runtime/pkg/certwatcher" + "sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics" ) var _ = Describe("CertWatcher", func() { @@ -109,6 +112,64 @@ var _ = Describe("CertWatcher", func() { ctxCancel() Eventually(doneCh, "4s").Should(BeClosed()) }) + + Context("prometheus metric read_certificate_total", func() { + var readCertificateTotalBefore float64 + var readCertificateErrorsBefore float64 + + BeforeEach(func() { + readCertificateTotalBefore = testutil.ToFloat64(metrics.ReadCertificateTotal) + readCertificateErrorsBefore = testutil.ToFloat64(metrics.ReadCertificateErrors) + }) + + It("should get updated on successful certificate read", func() { + doneCh := startWatcher() + + Eventually(func() error { + readCertificateTotalAfter := testutil.ToFloat64(metrics.ReadCertificateTotal) + if readCertificateTotalAfter != readCertificateTotalBefore+1.0 { + return fmt.Errorf("metric read certificate total expected: %v and got: %v", readCertificateTotalBefore+1.0, readCertificateTotalAfter) + } + return nil + }, "4s").Should(Succeed()) + + ctxCancel() + Eventually(doneCh, "4s").Should(BeClosed()) + }) + + It("should get updated on read certificate errors", func() { + doneCh := startWatcher() + + Eventually(func() error { + readCertificateTotalAfter := testutil.ToFloat64(metrics.ReadCertificateTotal) + if readCertificateTotalAfter != readCertificateTotalBefore+1.0 { + return fmt.Errorf("metric read certificate total expected: %v and got: %v", readCertificateTotalBefore+1.0, readCertificateTotalAfter) + } + readCertificateTotalBefore = readCertificateTotalAfter + return nil + }, "4s").Should(Succeed()) + + Expect(os.Remove(keyPath)).To(BeNil()) + + Eventually(func() error { + readCertificateTotalAfter := testutil.ToFloat64(metrics.ReadCertificateTotal) + if readCertificateTotalAfter != readCertificateTotalBefore+1.0 { + return fmt.Errorf("metric read certificate total expected: %v and got: %v", readCertificateTotalBefore+1.0, readCertificateTotalAfter) + } + return nil + }, "4s").Should(Succeed()) + Eventually(func() error { + readCertificateErrorsAfter := testutil.ToFloat64(metrics.ReadCertificateErrors) + if readCertificateErrorsAfter != readCertificateErrorsBefore+1.0 { + return fmt.Errorf("metric read certificate errors expected: %v and got: %v", readCertificateErrorsBefore+1.0, readCertificateErrorsAfter) + } + return nil + }, "4s").Should(Succeed()) + + ctxCancel() + Eventually(doneCh, "4s").Should(BeClosed()) + }) + }) }) }) @@ -167,7 +228,7 @@ func writeCerts(certPath, keyPath, ip string) error { return err } - keyOut, err := os.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) //nolint:gosec + keyOut, err := os.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { return err } diff --git a/pkg/certwatcher/example_test.go b/pkg/certwatcher/example_test.go index cea5280125..e322aeebfc 100644 --- a/pkg/certwatcher/example_test.go +++ b/pkg/certwatcher/example_test.go @@ -20,6 +20,7 @@ import ( "context" "crypto/tls" "net/http" + "time" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/certwatcher" @@ -32,7 +33,7 @@ func Example() { // Setup Context ctx := ctrl.SetupSignalHandler() - // Initialize a new cert watcher with cert/key pari + // Initialize a new cert watcher with cert/key pair watcher, err := certwatcher.New("ssl/tls.crt", "ssl/tls.key") if err != nil { panic(err) @@ -56,13 +57,16 @@ func Example() { // Initialize your tls server srv := &http.Server{ - Handler: &sampleServer{}, + Handler: &sampleServer{}, + ReadHeaderTimeout: 5 * time.Second, } // Start goroutine for handling server shutdown. go func() { <-ctx.Done() - if err := srv.Shutdown(context.Background()); err != nil { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := srv.Shutdown(ctx); err != nil { panic(err) } }() diff --git a/pkg/certwatcher/metrics/metrics.go b/pkg/certwatcher/metrics/metrics.go new file mode 100644 index 0000000000..05869eff03 --- /dev/null +++ b/pkg/certwatcher/metrics/metrics.go @@ -0,0 +1,45 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +var ( + // ReadCertificateTotal is a prometheus counter metrics which holds the total + // number of certificate reads. + ReadCertificateTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "certwatcher_read_certificate_total", + Help: "Total number of certificate reads", + }) + + // ReadCertificateErrors is a prometheus counter metrics which holds the total + // number of errors from certificate read. + ReadCertificateErrors = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "certwatcher_read_certificate_errors_total", + Help: "Total number of certificate read errors", + }) +) + +func init() { + metrics.Registry.MustRegister( + ReadCertificateTotal, + ReadCertificateErrors, + ) +} diff --git a/pkg/client/apiutil/apimachinery.go b/pkg/client/apiutil/apimachinery.go index 2611a20c64..6a1bfb546e 100644 --- a/pkg/client/apiutil/apimachinery.go +++ b/pkg/client/apiutil/apimachinery.go @@ -20,7 +20,10 @@ limitations under the License. package apiutil import ( + "errors" "fmt" + "net/http" + "reflect" "sync" "k8s.io/apimachinery/pkg/api/meta" @@ -29,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/client-go/discovery" + "k8s.io/client-go/dynamic" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/restmapper" @@ -58,9 +62,13 @@ func AddToProtobufScheme(addToScheme func(*runtime.Scheme) error) error { // NewDiscoveryRESTMapper constructs a new RESTMapper based on discovery // information fetched by a new client with the given config. -func NewDiscoveryRESTMapper(c *rest.Config) (meta.RESTMapper, error) { +func NewDiscoveryRESTMapper(c *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) { + if httpClient == nil { + return nil, fmt.Errorf("httpClient must not be nil, consider using rest.HTTPClientFor(c) to create a client") + } + // Get a mapper - dc, err := discovery.NewDiscoveryClientForConfig(c) + dc, err := discovery.NewDiscoveryClientForConfigAndClient(c, httpClient) if err != nil { return nil, err } @@ -71,6 +79,36 @@ func NewDiscoveryRESTMapper(c *rest.Config) (meta.RESTMapper, error) { return restmapper.NewDiscoveryRESTMapper(gr), nil } +// IsObjectNamespaced returns true if the object is namespace scoped. +// For unstructured objects the gvk is found from the object itself. +func IsObjectNamespaced(obj runtime.Object, scheme *runtime.Scheme, restmapper meta.RESTMapper) (bool, error) { + gvk, err := GVKForObject(obj, scheme) + if err != nil { + return false, err + } + + return IsGVKNamespaced(gvk, restmapper) +} + +// IsGVKNamespaced returns true if the object having the provided +// GVK is namespace scoped. +func IsGVKNamespaced(gvk schema.GroupVersionKind, restmapper meta.RESTMapper) (bool, error) { + restmapping, err := restmapper.RESTMapping(schema.GroupKind{Group: gvk.Group, Kind: gvk.Kind}) + if err != nil { + return false, fmt.Errorf("failed to get restmapping: %w", err) + } + + scope := restmapping.Scope.Name() + if scope == "" { + return false, errors.New("scope cannot be identified, empty scope returned") + } + + if scope != meta.RESTScopeNameRoot { + return true, nil + } + return false, nil +} + // GVKForObject finds the GroupVersionKind associated with the given object, if there is only a single such GVK. func GVKForObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupVersionKind, error) { // TODO(directxman12): do we want to generalize this to arbitrary container types? @@ -80,7 +118,7 @@ func GVKForObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupVersi // (unstructured, partial, etc) // check for PartialObjectMetadata, which is analogous to unstructured, but isn't handled by ObjectKinds - _, isPartial := obj.(*metav1.PartialObjectMetadata) //nolint:ifshort + _, isPartial := obj.(*metav1.PartialObjectMetadata) _, isPartialList := obj.(*metav1.PartialObjectMetadataList) if isPartial || isPartialList { // we require that the GVK be populated in order to recognize the object @@ -94,6 +132,7 @@ func GVKForObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupVersi return gvk, nil } + // Use the given scheme to retrieve all the GVKs for the object. gvks, isUnversioned, err := scheme.ObjectKinds(obj) if err != nil { return schema.GroupVersionKind{}, err @@ -102,36 +141,49 @@ func GVKForObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupVersi return schema.GroupVersionKind{}, fmt.Errorf("cannot create group-version-kind for unversioned type %T", obj) } - if len(gvks) < 1 { - return schema.GroupVersionKind{}, fmt.Errorf("no group-version-kinds associated with type %T", obj) - } - if len(gvks) > 1 { - // this should only trigger for things like metav1.XYZ -- - // normal versioned types should be fine + switch { + case len(gvks) < 1: + // If the object has no GVK, the object might not have been registered with the scheme. + // or it's not a valid object. + return schema.GroupVersionKind{}, fmt.Errorf("no GroupVersionKind associated with Go type %T, was the type registered with the Scheme?", obj) + case len(gvks) > 1: + err := fmt.Errorf("multiple GroupVersionKinds associated with Go type %T within the Scheme, this can happen when a type is registered for multiple GVKs at the same time", obj) + + // We've found multiple GVKs for the object. + currentGVK := obj.GetObjectKind().GroupVersionKind() + if !currentGVK.Empty() { + // If the base object has a GVK, check if it's in the list of GVKs before using it. + for _, gvk := range gvks { + if gvk == currentGVK { + return gvk, nil + } + } + + return schema.GroupVersionKind{}, fmt.Errorf( + "%w: the object's supplied GroupVersionKind %q was not found in the Scheme's list; refusing to guess at one: %q", err, currentGVK, gvks) + } + + // This should only trigger for things like metav1.XYZ -- + // normal versioned types should be fine. + // + // See https://github.com/kubernetes-sigs/controller-runtime/issues/362 + // for more information. return schema.GroupVersionKind{}, fmt.Errorf( - "multiple group-version-kinds associated with type %T, refusing to guess at one", obj) + "%w: callers can either fix their type registration to only register it once, or specify the GroupVersionKind to use for object passed in; refusing to guess at one: %q", err, gvks) + default: + // In any other case, we've found a single GVK for the object. + return gvks[0], nil } - return gvks[0], nil } // RESTClientForGVK constructs a new rest.Interface capable of accessing the resource associated // with the given GroupVersionKind. The REST client will be configured to use the negotiated serializer from // baseConfig, if set, otherwise a default serializer will be set. -func RESTClientForGVK(gvk schema.GroupVersionKind, isUnstructured bool, baseConfig *rest.Config, codecs serializer.CodecFactory) (rest.Interface, error) { - return rest.RESTClientFor(createRestConfig(gvk, isUnstructured, baseConfig, codecs)) -} - -// serializerWithDecodedGVK is a CodecFactory that overrides the DecoderToVersion of a WithoutConversionCodecFactory -// in order to avoid clearing the GVK from the decoded object. -// -// See https://github.com/kubernetes/kubernetes/issues/80609. -type serializerWithDecodedGVK struct { - serializer.WithoutConversionCodecFactory -} - -// DecoderToVersion returns an decoder that does not do conversion. -func (f serializerWithDecodedGVK) DecoderToVersion(serializer runtime.Decoder, _ runtime.GroupVersioner) runtime.Decoder { - return serializer +func RESTClientForGVK(gvk schema.GroupVersionKind, isUnstructured bool, baseConfig *rest.Config, codecs serializer.CodecFactory, httpClient *http.Client) (rest.Interface, error) { + if httpClient == nil { + return nil, fmt.Errorf("httpClient must not be nil, consider using rest.HTTPClientFor(c) to create a client") + } + return rest.RESTClientForConfigAndClient(createRestConfig(gvk, isUnstructured, baseConfig, codecs), httpClient) } // createRestConfig copies the base config and updates needed fields for a new rest config. @@ -157,15 +209,38 @@ func createRestConfig(gvk schema.GroupVersionKind, isUnstructured bool, baseConf protobufSchemeLock.RUnlock() } - if cfg.NegotiatedSerializer == nil { - if isUnstructured { - // If the object is unstructured, we need to preserve the GVK information. - // Use our own custom serializer. - cfg.NegotiatedSerializer = serializerWithDecodedGVK{serializer.WithoutConversionCodecFactory{CodecFactory: codecs}} - } else { - cfg.NegotiatedSerializer = serializer.WithoutConversionCodecFactory{CodecFactory: codecs} - } + if isUnstructured { + // If the object is unstructured, we use the client-go dynamic serializer. + cfg = dynamic.ConfigFor(cfg) + } else { + cfg.NegotiatedSerializer = serializerWithTargetZeroingDecode{NegotiatedSerializer: serializer.WithoutConversionCodecFactory{CodecFactory: codecs}} } return cfg } + +type serializerWithTargetZeroingDecode struct { + runtime.NegotiatedSerializer +} + +func (s serializerWithTargetZeroingDecode) DecoderToVersion(serializer runtime.Decoder, r runtime.GroupVersioner) runtime.Decoder { + return targetZeroingDecoder{upstream: s.NegotiatedSerializer.DecoderToVersion(serializer, r)} +} + +type targetZeroingDecoder struct { + upstream runtime.Decoder +} + +func (t targetZeroingDecoder) Decode(data []byte, defaults *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error) { + zero(into) + return t.upstream.Decode(data, defaults, into) +} + +// zero zeros the value of a pointer. +func zero(x interface{}) { + if x == nil { + return + } + res := reflect.ValueOf(x).Elem() + res.Set(reflect.Zero(res.Type())) +} diff --git a/pkg/client/apiutil/apiutil_suite_test.go b/pkg/client/apiutil/apiutil_suite_test.go index 4a4d7905a5..7fe960b917 100644 --- a/pkg/client/apiutil/apiutil_suite_test.go +++ b/pkg/client/apiutil/apiutil_suite_test.go @@ -14,15 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package apiutil_test +package apiutil import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -30,17 +29,14 @@ import ( func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "API Utilities Test Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "API Utilities Test Suite") } var cfg *rest.Config -var _ = BeforeSuite(func(done Done) { +var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) // for things that technically need a rest.Config for defaulting, but don't actually use them cfg = &rest.Config{} - - close(done) -}, 60) +}) diff --git a/pkg/client/apiutil/dynamicrestmapper.go b/pkg/client/apiutil/dynamicrestmapper.go index 56a00371ff..d263c518cb 100644 --- a/pkg/client/apiutil/dynamicrestmapper.go +++ b/pkg/client/apiutil/dynamicrestmapper.go @@ -17,8 +17,10 @@ limitations under the License. package apiutil import ( - "errors" + "fmt" + "net/http" "sync" + "sync/atomic" "golang.org/x/time/rate" "k8s.io/apimachinery/pkg/api/meta" @@ -38,7 +40,10 @@ type dynamicRESTMapper struct { lazy bool // Used for lazy init. - initOnce sync.Once + inited uint32 + initMtx sync.Mutex + + useLazyRestmapper bool } // DynamicRESTMapperOption is a functional option on the dynamicRESTMapper. @@ -59,6 +64,12 @@ var WithLazyDiscovery DynamicRESTMapperOption = func(drm *dynamicRESTMapper) err return nil } +// WithExperimentalLazyMapper enables experimental more advanced Lazy Restmapping mechanism. +var WithExperimentalLazyMapper DynamicRESTMapperOption = func(drm *dynamicRESTMapper) error { + drm.useLazyRestmapper = true + return nil +} + // WithCustomMapper supports setting a custom RESTMapper refresher instead of // the default method, which uses a discovery client. // @@ -74,8 +85,12 @@ func WithCustomMapper(newMapper func() (meta.RESTMapper, error)) DynamicRESTMapp // NewDynamicRESTMapper returns a dynamic RESTMapper for cfg. The dynamic // RESTMapper dynamically discovers resource types at runtime. opts // configure the RESTMapper. -func NewDynamicRESTMapper(cfg *rest.Config, opts ...DynamicRESTMapperOption) (meta.RESTMapper, error) { - client, err := discovery.NewDiscoveryClientForConfig(cfg) +func NewDynamicRESTMapper(cfg *rest.Config, httpClient *http.Client, opts ...DynamicRESTMapperOption) (meta.RESTMapper, error) { + if httpClient == nil { + return nil, fmt.Errorf("httpClient must not be nil, consider using rest.HTTPClientFor(c) to create a client") + } + + client, err := discovery.NewDiscoveryClientForConfigAndClient(cfg, httpClient) if err != nil { return nil, err } @@ -94,6 +109,9 @@ func NewDynamicRESTMapper(cfg *rest.Config, opts ...DynamicRESTMapperOption) (me return nil, err } } + if drm.useLazyRestmapper { + return newLazyRESTMapperWithClient(client) + } if !drm.lazy { if err := drm.setStaticMapper(); err != nil { return nil, err @@ -125,18 +143,25 @@ func (drm *dynamicRESTMapper) setStaticMapper() error { // init initializes drm only once if drm is lazy. func (drm *dynamicRESTMapper) init() (err error) { - drm.initOnce.Do(func() { - if drm.lazy { - err = drm.setStaticMapper() + // skip init if drm is not lazy or has initialized + if !drm.lazy || atomic.LoadUint32(&drm.inited) != 0 { + return nil + } + + drm.initMtx.Lock() + defer drm.initMtx.Unlock() + if drm.inited == 0 { + if err = drm.setStaticMapper(); err == nil { + atomic.StoreUint32(&drm.inited, 1) } - }) + } return err } // checkAndReload attempts to call the given callback, which is assumed to be dependent // on the data in the restmapper. // -// If the callback returns an error that matches the given error, it will attempt to reload +// If the callback returns an error matching meta.IsNoMatchErr, it will attempt to reload // the RESTMapper's data and re-call the callback once that's occurred. // If the callback returns any other error, the function will return immediately regardless. // @@ -145,7 +170,7 @@ func (drm *dynamicRESTMapper) init() (err error) { // the callback. // It's thread-safe, and worries about thread-safety for the callback (so the callback does // not need to attempt to lock the restmapper). -func (drm *dynamicRESTMapper) checkAndReload(needsReloadErr error, checkNeedsReload func() error) error { +func (drm *dynamicRESTMapper) checkAndReload(checkNeedsReload func() error) error { // first, check the common path -- data is fresh enough // (use an IIFE for the lock's defer) err := func() error { @@ -155,10 +180,7 @@ func (drm *dynamicRESTMapper) checkAndReload(needsReloadErr error, checkNeedsRel return checkNeedsReload() }() - // NB(directxman12): `Is` and `As` have a confusing relationship -- - // `Is` is like `== or does this implement .Is`, whereas `As` says - // `can I type-assert into` - needsReload := errors.As(err, &needsReloadErr) + needsReload := meta.IsNoMatchError(err) if !needsReload { return err } @@ -169,7 +191,7 @@ func (drm *dynamicRESTMapper) checkAndReload(needsReloadErr error, checkNeedsRel // ... and double-check that we didn't reload in the meantime err = checkNeedsReload() - needsReload = errors.As(err, &needsReloadErr) + needsReload = meta.IsNoMatchError(err) if !needsReload { return err } @@ -197,7 +219,7 @@ func (drm *dynamicRESTMapper) KindFor(resource schema.GroupVersionResource) (sch return schema.GroupVersionKind{}, err } var gvk schema.GroupVersionKind - err := drm.checkAndReload(&meta.NoResourceMatchError{}, func() error { + err := drm.checkAndReload(func() error { var err error gvk, err = drm.staticMapper.KindFor(resource) return err @@ -210,7 +232,7 @@ func (drm *dynamicRESTMapper) KindsFor(resource schema.GroupVersionResource) ([] return nil, err } var gvks []schema.GroupVersionKind - err := drm.checkAndReload(&meta.NoResourceMatchError{}, func() error { + err := drm.checkAndReload(func() error { var err error gvks, err = drm.staticMapper.KindsFor(resource) return err @@ -224,7 +246,7 @@ func (drm *dynamicRESTMapper) ResourceFor(input schema.GroupVersionResource) (sc } var gvr schema.GroupVersionResource - err := drm.checkAndReload(&meta.NoResourceMatchError{}, func() error { + err := drm.checkAndReload(func() error { var err error gvr, err = drm.staticMapper.ResourceFor(input) return err @@ -237,7 +259,7 @@ func (drm *dynamicRESTMapper) ResourcesFor(input schema.GroupVersionResource) ([ return nil, err } var gvrs []schema.GroupVersionResource - err := drm.checkAndReload(&meta.NoResourceMatchError{}, func() error { + err := drm.checkAndReload(func() error { var err error gvrs, err = drm.staticMapper.ResourcesFor(input) return err @@ -250,7 +272,7 @@ func (drm *dynamicRESTMapper) RESTMapping(gk schema.GroupKind, versions ...strin return nil, err } var mapping *meta.RESTMapping - err := drm.checkAndReload(&meta.NoKindMatchError{}, func() error { + err := drm.checkAndReload(func() error { var err error mapping, err = drm.staticMapper.RESTMapping(gk, versions...) return err @@ -263,7 +285,7 @@ func (drm *dynamicRESTMapper) RESTMappings(gk schema.GroupKind, versions ...stri return nil, err } var mappings []*meta.RESTMapping - err := drm.checkAndReload(&meta.NoKindMatchError{}, func() error { + err := drm.checkAndReload(func() error { var err error mappings, err = drm.staticMapper.RESTMappings(gk, versions...) return err @@ -276,7 +298,7 @@ func (drm *dynamicRESTMapper) ResourceSingularizer(resource string) (string, err return "", err } var singular string - err := drm.checkAndReload(&meta.NoResourceMatchError{}, func() error { + err := drm.checkAndReload(func() error { var err error singular, err = drm.staticMapper.ResourceSingularizer(resource) return err diff --git a/pkg/client/apiutil/dynamicrestmapper_test.go b/pkg/client/apiutil/dynamicrestmapper_test.go index 1e294b793d..43b8bfff35 100644 --- a/pkg/client/apiutil/dynamicrestmapper_test.go +++ b/pkg/client/apiutil/dynamicrestmapper_test.go @@ -14,20 +14,20 @@ See the License for the specific language governing permissions and limitations under the License. */ -package apiutil_test +package apiutil import ( + "fmt" "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/onsi/gomega/format" "github.com/onsi/gomega/types" "golang.org/x/time/rate" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime/schema" - - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "k8s.io/client-go/rest" ) var ( @@ -51,8 +51,10 @@ var _ = Describe("Dynamic REST Mapper", func() { baseMapper.Add(targetGVK, meta.RESTScopeNamespace) } + httpClient, err := rest.HTTPClientFor(cfg) + Expect(err).ToNot(HaveOccurred()) lim = rate.NewLimiter(rate.Limit(5), 5) - mapper, err = apiutil.NewDynamicRESTMapper(cfg, apiutil.WithLimiter(lim), apiutil.WithCustomMapper(func() (meta.RESTMapper, error) { + mapper, err = NewDynamicRESTMapper(cfg, httpClient, WithLimiter(lim), WithCustomMapper(func() (meta.RESTMapper, error) { baseMapper := meta.NewDefaultRESTMapper(nil) addToMapper(baseMapper) @@ -146,11 +148,30 @@ var _ = Describe("Dynamic REST Mapper", func() { By("ensuring that it was only refreshed once") Expect(count).To(Equal(1)) }) - } - PIt("should lazily initialize if the lazy option is used", func() { - - }) + It("should lazily initialize if the lazy option is used", func() { + var err error + var failedOnce bool + mockErr := fmt.Errorf("mock failed once") + httpClient, err := rest.HTTPClientFor(cfg) + Expect(err).ToNot(HaveOccurred()) + mapper, err = NewDynamicRESTMapper(cfg, httpClient, WithLazyDiscovery, WithCustomMapper(func() (meta.RESTMapper, error) { + // Make newMapper fail once + if !failedOnce { + failedOnce = true + return nil, mockErr + } + baseMapper := meta.NewDefaultRESTMapper(nil) + addToMapper(baseMapper) + return baseMapper, nil + })) + Expect(err).NotTo(HaveOccurred()) + Expect(mapper.(*dynamicRESTMapper).staticMapper).To(BeNil()) + + Expect(callWithTarget()).To(MatchError(mockErr)) + Expect(callWithTarget()).To(Succeed()) + }) + } Describe("KindFor", func() { mapperTest(func() error { diff --git a/pkg/client/apiutil/lazyrestmapper.go b/pkg/client/apiutil/lazyrestmapper.go new file mode 100644 index 0000000000..70c6a11dbc --- /dev/null +++ b/pkg/client/apiutil/lazyrestmapper.go @@ -0,0 +1,248 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiutil + +import ( + "fmt" + "sync" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" + "k8s.io/client-go/restmapper" +) + +// lazyRESTMapper is a RESTMapper that will lazily query the provided +// client for discovery information to do REST mappings. +type lazyRESTMapper struct { + mapper meta.RESTMapper + client *discovery.DiscoveryClient + knownGroups map[string]*restmapper.APIGroupResources + apiGroups *metav1.APIGroupList + + // mutex to provide thread-safe mapper reloading. + mu sync.Mutex +} + +// newLazyRESTMapperWithClient initializes a LazyRESTMapper with a custom discovery client. +func newLazyRESTMapperWithClient(discoveryClient *discovery.DiscoveryClient) (meta.RESTMapper, error) { + return &lazyRESTMapper{ + mapper: restmapper.NewDiscoveryRESTMapper([]*restmapper.APIGroupResources{}), + client: discoveryClient, + knownGroups: map[string]*restmapper.APIGroupResources{}, + }, nil +} + +// KindFor implements Mapper.KindFor. +func (m *lazyRESTMapper) KindFor(resource schema.GroupVersionResource) (schema.GroupVersionKind, error) { + res, err := m.mapper.KindFor(resource) + if meta.IsNoMatchError(err) { + if err = m.addKnownGroupAndReload(resource.Group, resource.Version); err != nil { + return res, err + } + + res, err = m.mapper.KindFor(resource) + } + + return res, err +} + +// KindsFor implements Mapper.KindsFor. +func (m *lazyRESTMapper) KindsFor(resource schema.GroupVersionResource) ([]schema.GroupVersionKind, error) { + res, err := m.mapper.KindsFor(resource) + if meta.IsNoMatchError(err) { + if err = m.addKnownGroupAndReload(resource.Group, resource.Version); err != nil { + return res, err + } + + res, err = m.mapper.KindsFor(resource) + } + + return res, err +} + +// ResourceFor implements Mapper.ResourceFor. +func (m *lazyRESTMapper) ResourceFor(input schema.GroupVersionResource) (schema.GroupVersionResource, error) { + res, err := m.mapper.ResourceFor(input) + if meta.IsNoMatchError(err) { + if err = m.addKnownGroupAndReload(input.Group, input.Version); err != nil { + return res, err + } + + res, err = m.mapper.ResourceFor(input) + } + + return res, err +} + +// ResourcesFor implements Mapper.ResourcesFor. +func (m *lazyRESTMapper) ResourcesFor(input schema.GroupVersionResource) ([]schema.GroupVersionResource, error) { + res, err := m.mapper.ResourcesFor(input) + if meta.IsNoMatchError(err) { + if err = m.addKnownGroupAndReload(input.Group, input.Version); err != nil { + return res, err + } + + res, err = m.mapper.ResourcesFor(input) + } + + return res, err +} + +// RESTMapping implements Mapper.RESTMapping. +func (m *lazyRESTMapper) RESTMapping(gk schema.GroupKind, versions ...string) (*meta.RESTMapping, error) { + res, err := m.mapper.RESTMapping(gk, versions...) + if meta.IsNoMatchError(err) { + if err = m.addKnownGroupAndReload(gk.Group, versions...); err != nil { + return res, err + } + + res, err = m.mapper.RESTMapping(gk, versions...) + } + + return res, err +} + +// RESTMappings implements Mapper.RESTMappings. +func (m *lazyRESTMapper) RESTMappings(gk schema.GroupKind, versions ...string) ([]*meta.RESTMapping, error) { + res, err := m.mapper.RESTMappings(gk, versions...) + if meta.IsNoMatchError(err) { + if err = m.addKnownGroupAndReload(gk.Group, versions...); err != nil { + return res, err + } + + res, err = m.mapper.RESTMappings(gk, versions...) + } + + return res, err +} + +// ResourceSingularizer implements Mapper.ResourceSingularizer. +func (m *lazyRESTMapper) ResourceSingularizer(resource string) (string, error) { + return m.mapper.ResourceSingularizer(resource) +} + +// addKnownGroupAndReload reloads the mapper with updated information about missing API group. +// versions can be specified for partial updates, for instance for v1beta1 version only. +func (m *lazyRESTMapper) addKnownGroupAndReload(groupName string, versions ...string) error { + m.mu.Lock() + defer m.mu.Unlock() + + // If no specific versions are set by user, we will scan all available ones for the API group. + // This operation requires 2 requests: /api and /apis, but only once. For all subsequent calls + // this data will be taken from cache. + if len(versions) == 0 { + apiGroup, err := m.findAPIGroupByName(groupName) + if err != nil { + return err + } + for _, version := range apiGroup.Versions { + versions = append(versions, version.Version) + } + } + + // Create or fetch group resources from cache. + groupResources := &restmapper.APIGroupResources{ + Group: metav1.APIGroup{Name: groupName}, + VersionedResources: make(map[string][]metav1.APIResource), + } + if _, ok := m.knownGroups[groupName]; ok { + groupResources = m.knownGroups[groupName] + } + + // Update information for group resources about versioned resources. + // The number of API calls is equal to the number of versions: /apis//. + groupVersionResources, err := m.fetchGroupVersionResources(groupName, versions...) + if err != nil { + return fmt.Errorf("failed to get API group resources: %w", err) + } + for version, resources := range groupVersionResources { + groupResources.VersionedResources[version.Version] = resources.APIResources + } + + // Update information for group resources about the API group by adding new versions. + for _, version := range versions { + groupResources.Group.Versions = append(groupResources.Group.Versions, metav1.GroupVersionForDiscovery{ + GroupVersion: metav1.GroupVersion{Group: groupName, Version: version}.String(), + Version: version, + }) + } + + // Update data in the cache. + m.knownGroups[groupName] = groupResources + + // Finally, update the group with received information and regenerate the mapper. + updatedGroupResources := make([]*restmapper.APIGroupResources, 0, len(m.knownGroups)) + for _, agr := range m.knownGroups { + updatedGroupResources = append(updatedGroupResources, agr) + } + + m.mapper = restmapper.NewDiscoveryRESTMapper(updatedGroupResources) + + return nil +} + +// findAPIGroupByName returns API group by its name. +func (m *lazyRESTMapper) findAPIGroupByName(groupName string) (metav1.APIGroup, error) { + // Ensure that required info about existing API groups is received and stored in the mapper. + // It will make 2 API calls to /api and /apis, but only once. + if m.apiGroups == nil { + apiGroups, err := m.client.ServerGroups() + if err != nil { + return metav1.APIGroup{}, fmt.Errorf("failed to get server groups: %w", err) + } + if len(apiGroups.Groups) == 0 { + return metav1.APIGroup{}, fmt.Errorf("received an empty API groups list") + } + + m.apiGroups = apiGroups + } + + for i := range m.apiGroups.Groups { + if groupName == (&m.apiGroups.Groups[i]).Name { + return m.apiGroups.Groups[i], nil + } + } + + return metav1.APIGroup{}, fmt.Errorf("failed to find API group %s", groupName) +} + +// fetchGroupVersionResources fetches the resources for the specified group and its versions. +func (m *lazyRESTMapper) fetchGroupVersionResources(groupName string, versions ...string) (map[schema.GroupVersion]*metav1.APIResourceList, error) { + groupVersionResources := make(map[schema.GroupVersion]*metav1.APIResourceList) + failedGroups := make(map[schema.GroupVersion]error) + + for _, version := range versions { + groupVersion := schema.GroupVersion{Group: groupName, Version: version} + + apiResourceList, err := m.client.ServerResourcesForGroupVersion(groupVersion.String()) + if err != nil { + failedGroups[groupVersion] = err + } + if apiResourceList != nil { + // even in case of error, some fallback might have been returned. + groupVersionResources[groupVersion] = apiResourceList + } + } + + if len(failedGroups) > 0 { + return nil, &discovery.ErrGroupDiscoveryFailed{Groups: failedGroups} + } + + return groupVersionResources, nil +} diff --git a/pkg/client/apiutil/lazyrestmapper_test.go b/pkg/client/apiutil/lazyrestmapper_test.go new file mode 100644 index 0000000000..a0027e77ff --- /dev/null +++ b/pkg/client/apiutil/lazyrestmapper_test.go @@ -0,0 +1,404 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiutil_test + +import ( + "net/http" + "testing" + + _ "github.com/onsi/ginkgo/v2" + gmg "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +// countingRoundTripper is used to count HTTP requests. +type countingRoundTripper struct { + roundTripper http.RoundTripper + requestCount int +} + +func newCountingRoundTripper(rt http.RoundTripper) *countingRoundTripper { + return &countingRoundTripper{roundTripper: rt} +} + +// RoundTrip implements http.RoundTripper.RoundTrip that additionally counts requests. +func (crt *countingRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { + crt.requestCount++ + + return crt.roundTripper.RoundTrip(r) +} + +// GetRequestCount returns how many requests have been made. +func (crt *countingRoundTripper) GetRequestCount() int { + return crt.requestCount +} + +// Reset sets the counter to 0. +func (crt *countingRoundTripper) Reset() { + crt.requestCount = 0 +} + +func setupEnvtest(t *testing.T) (*rest.Config, func(t *testing.T)) { + t.Log("Setup envtest") + + g := gmg.NewWithT(t) + testEnv := &envtest.Environment{ + CRDDirectoryPaths: []string{"testdata"}, + } + + cfg, err := testEnv.Start() + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(cfg).NotTo(gmg.BeNil()) + + teardownFunc := func(t *testing.T) { + t.Log("Stop envtest") + g.Expect(testEnv.Stop()).To(gmg.Succeed()) + } + + return cfg, teardownFunc +} + +func TestLazyRestMapperProvider(t *testing.T) { + restCfg, tearDownFn := setupEnvtest(t) + defer tearDownFn(t) + + t.Run("LazyRESTMapper should fetch data based on the request", func(t *testing.T) { + g := gmg.NewWithT(t) + + // To initialize mapper does 2 requests: + // GET https://host/api + // GET https://host/apis + // Then, for each new group it performs just one request to the API server: + // GET https://host/apis// + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient, apiutil.WithExperimentalLazyMapper) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + // There are no requests before any call + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "deployment"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("deployment")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) + + mappings, err := lazyRestMapper.RESTMappings(schema.GroupKind{Group: "", Kind: "pod"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(len(mappings)).To(gmg.Equal(1)) + g.Expect(mappings[0].GroupVersionKind.Kind).To(gmg.Equal("pod")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) + + kind, err := lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "networking.k8s.io", Version: "v1", Resource: "ingresses"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(kind.Kind).To(gmg.Equal("Ingress")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(5)) + + kinds, err := lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "authentication.k8s.io", Version: "v1", Resource: "tokenreviews"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(len(kinds)).To(gmg.Equal(1)) + g.Expect(kinds[0].Kind).To(gmg.Equal("TokenReview")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(6)) + + resource, err := lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "scheduling.k8s.io", Version: "v1", Resource: "priorityclasses"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(resource.Resource).To(gmg.Equal("priorityclasses")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(7)) + + resources, err := lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "policy", Version: "v1", Resource: "poddisruptionbudgets"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(len(resources)).To(gmg.Equal(1)) + g.Expect(resources[0].Resource).To(gmg.Equal("poddisruptionbudgets")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(8)) + }) + + t.Run("LazyRESTMapper should cache fetched data and doesn't perform any additional requests", func(t *testing.T) { + g := gmg.NewWithT(t) + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient, apiutil.WithExperimentalLazyMapper) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "deployment"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("deployment")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) + + // Data taken from cache - there are no more additional requests. + + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "deployment"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("deployment")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) + + kind, err := lazyRestMapper.KindFor((schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployment"})) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(kind.Kind).To(gmg.Equal("Deployment")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) + + resource, err := lazyRestMapper.ResourceFor((schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployment"})) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(resource.Resource).To(gmg.Equal("deployments")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) + }) + + t.Run("LazyRESTMapper should work correctly with empty versions list", func(t *testing.T) { + g := gmg.NewWithT(t) + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient, apiutil.WithExperimentalLazyMapper) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + // crew.example.com has 2 versions: v1 and v2 + + // If no versions were provided by user, we fetch all of them. + // Here we expect 4 calls. + // To initialize: + // #1: GET https://host/api + // #2: GET https://host/apis + // Then, for each version it performs one request to the API server: + // #3: GET https://host/apis/crew.example.com/v1 + // #4: GET https://host/apis/crew.example.com/v2 + mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) + + // All subsequent calls won't send requests to the server. + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) + }) + + t.Run("LazyRESTMapper should work correctly with multiple API group versions", func(t *testing.T) { + g := gmg.NewWithT(t) + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient, apiutil.WithExperimentalLazyMapper) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + // We explicitly ask for 2 versions: v1 and v2. + // For each version it performs one request to the API server: + // #1: GET https://host/apis/crew.example.com/v1 + // #2: GET https://host/apis/crew.example.com/v2 + mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v1", "v2") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) + + // All subsequent calls won't send requests to the server as everything is stored in the cache. + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v1") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) + + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}) + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) + }) + + t.Run("LazyRESTMapper should work correctly with different API group versions", func(t *testing.T) { + g := gmg.NewWithT(t) + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient, apiutil.WithExperimentalLazyMapper) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + g.Expect(crt.GetRequestCount()).To(gmg.Equal(0)) + + // Now we want resources for crew.example.com/v1 version only. + // Here we expect 1 call: + // #1: GET https://host/apis/crew.example.com/v1 + mapping, err := lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v1") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(1)) + + // Get additional resources from v2. + // It sends another request: + // #2: GET https://host/apis/crew.example.com/v2 + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v2") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) + + // No subsequent calls require additional API requests. + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v1") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) + + mapping, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "crew.example.com", Kind: "driver"}, "v1", "v2") + g.Expect(err).NotTo(gmg.HaveOccurred()) + g.Expect(mapping.GroupVersionKind.Kind).To(gmg.Equal("driver")) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) + }) + + t.Run("LazyRESTMapper should return an error if the group doesn't exist", func(t *testing.T) { + g := gmg.NewWithT(t) + + // After initialization for each invalid group the mapper performs just 1 request to the API server. + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient, apiutil.WithExperimentalLazyMapper) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "INVALID1"}, "v1") + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(1)) + + _, err = lazyRestMapper.RESTMappings(schema.GroupKind{Group: "INVALID2"}, "v1") + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) + + _, err = lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "INVALID3", Version: "v1"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) + + _, err = lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "INVALID4", Version: "v1"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) + + _, err = lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "INVALID5", Version: "v1"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(5)) + + _, err = lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "INVALID6", Version: "v1"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(6)) + }) + + t.Run("LazyRESTMapper should return an error if a resource doesn't exist", func(t *testing.T) { + g := gmg.NewWithT(t) + + // After initialization for each invalid resource the mapper performs just 1 request to the API server. + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient, apiutil.WithExperimentalLazyMapper) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "INVALID"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) + + _, err = lazyRestMapper.RESTMappings(schema.GroupKind{Group: "", Kind: "INVALID"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) + + _, err = lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "networking.k8s.io", Version: "v1", Resource: "INVALID"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(5)) + + _, err = lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "authentication.k8s.io", Version: "v1", Resource: "INVALID"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(6)) + + _, err = lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "scheduling.k8s.io", Version: "v1", Resource: "INVALID"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(7)) + + _, err = lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "policy", Version: "v1", Resource: "INVALID"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(8)) + }) + + t.Run("LazyRESTMapper should return an error if the version doesn't exist", func(t *testing.T) { + g := gmg.NewWithT(t) + + // After initialization, for each invalid resource mapper performs 1 requests to the API server. + + httpClient, err := rest.HTTPClientFor(restCfg) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + crt := newCountingRoundTripper(httpClient.Transport) + httpClient.Transport = crt + + lazyRestMapper, err := apiutil.NewDynamicRESTMapper(restCfg, httpClient, apiutil.WithExperimentalLazyMapper) + g.Expect(err).NotTo(gmg.HaveOccurred()) + + _, err = lazyRestMapper.RESTMapping(schema.GroupKind{Group: "apps", Kind: "deployment"}, "INVALID") + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(1)) + + _, err = lazyRestMapper.RESTMappings(schema.GroupKind{Group: "", Kind: "pod"}, "INVALID") + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(2)) + + _, err = lazyRestMapper.KindFor(schema.GroupVersionResource{Group: "networking.k8s.io", Version: "INVALID", Resource: "ingresses"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(3)) + + _, err = lazyRestMapper.KindsFor(schema.GroupVersionResource{Group: "authentication.k8s.io", Version: "INVALID", Resource: "tokenreviews"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(4)) + + _, err = lazyRestMapper.ResourceFor(schema.GroupVersionResource{Group: "scheduling.k8s.io", Version: "INVALID", Resource: "priorityclasses"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(5)) + + _, err = lazyRestMapper.ResourcesFor(schema.GroupVersionResource{Group: "policy", Version: "INVALID", Resource: "poddisruptionbudgets"}) + g.Expect(err).To(gmg.HaveOccurred()) + g.Expect(crt.GetRequestCount()).To(gmg.Equal(6)) + }) +} diff --git a/pkg/client/apiutil/testdata/crd.yaml b/pkg/client/apiutil/testdata/crd.yaml new file mode 100644 index 0000000000..5bb2d73f69 --- /dev/null +++ b/pkg/client/apiutil/testdata/crd.yaml @@ -0,0 +1,62 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + name: drivers.crew.example.com +spec: + group: crew.example.com + names: + kind: Driver + plural: drivers + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + description: Driver is the Schema for the drivers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + type: object + status: + type: object + type: object + - name: v2 + served: true + storage: false + schema: + openAPIV3Schema: + description: Driver is the Schema for the drivers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + type: object + status: + type: object + type: object +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/pkg/client/client.go b/pkg/client/client.go index bbe36c4673..49a398b3cc 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -18,12 +18,13 @@ package client import ( "context" + "errors" "fmt" + "net/http" "strings" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -35,6 +36,28 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" ) +// Options are creation options for a Client. +type Options struct { + // HTTPClient is the HTTP client to use for requests. + HTTPClient *http.Client + + // Scheme, if provided, will be used to map go structs to GroupVersionKinds + Scheme *runtime.Scheme + + // Mapper, if provided, will be used to map GroupVersionKinds to Resources + Mapper meta.RESTMapper + + // Cache, if provided, is used to read objects from the cache. + Cache *CacheOptions + + // WarningHandler is used to configure the warning handler responsible for + // surfacing and handling warnings messages sent by the API server. + WarningHandler WarningHandlerOptions + + // DryRun instructs the client to only perform dry run requests. + DryRun *bool +} + // WarningHandlerOptions are options for configuring a // warning handler for the client which is responsible // for surfacing API Server warnings. @@ -45,23 +68,25 @@ type WarningHandlerOptions struct { // AllowDuplicateLogs does not deduplicate the to-be // logged surfaced warnings messages. See // log.WarningHandlerOptions for considerations - // regarding deuplication + // regarding deduplication AllowDuplicateLogs bool } -// Options are creation options for a Client. -type Options struct { - // Scheme, if provided, will be used to map go structs to GroupVersionKinds - Scheme *runtime.Scheme - - // Mapper, if provided, will be used to map GroupVersionKinds to Resources - Mapper meta.RESTMapper - - // Opts is used to configure the warning handler responsible for - // surfacing and handling warnings messages sent by the API server. - Opts WarningHandlerOptions +// CacheOptions are options for creating a cache-backed client. +type CacheOptions struct { + // Reader is a cache-backed reader that will be used to read objects from the cache. + // +required + Reader Reader + // DisableFor is a list of objects that should not be read from the cache. + DisableFor []Object + // Unstructured is a flag that indicates whether the cache-backed client should + // read unstructured objects or lists from the cache. + Unstructured bool } +// NewClientFunc allows a user to define how to create a client. +type NewClientFunc func(config *rest.Config, options Options) (Client, error) + // New returns a new Client using the provided config and Options. // The returned client reads *and* writes directly from the server // (it doesn't use object caches). It understands how to work with @@ -72,8 +97,12 @@ type Options struct { // corresponding group, version, and kind for the given type. In the // case of unstructured types, the group, version, and kind will be extracted // from the corresponding fields on the object. -func New(config *rest.Config, options Options) (Client, error) { - return newClient(config, options) +func New(config *rest.Config, options Options) (c Client, err error) { + c, err = newClient(config, options) + if err == nil && options.DryRun != nil && *options.DryRun { + c = NewDryRunClient(c) + } + return c, err } func newClient(config *rest.Config, options Options) (*client, error) { @@ -81,23 +110,31 @@ func newClient(config *rest.Config, options Options) (*client, error) { return nil, fmt.Errorf("must provide non-nil rest.Config to client.New") } - if !options.Opts.SuppressWarnings { + if !options.WarningHandler.SuppressWarnings { // surface warnings logger := log.Log.WithName("KubeAPIWarningLogger") // Set a WarningHandler, the default WarningHandler // is log.KubeAPIWarningLogger with deduplication enabled. // See log.KubeAPIWarningLoggerOptions for considerations // regarding deduplication. - rest.SetDefaultWarningHandler( - log.NewKubeAPIWarningLogger( - logger, - log.KubeAPIWarningLoggerOptions{ - Deduplicate: !options.Opts.AllowDuplicateLogs, - }, - ), + config = rest.CopyConfig(config) + config.WarningHandler = log.NewKubeAPIWarningLogger( + logger, + log.KubeAPIWarningLoggerOptions{ + Deduplicate: !options.WarningHandler.AllowDuplicateLogs, + }, ) } + // Use the rest HTTP client for the provided config if unset + if options.HTTPClient == nil { + var err error + options.HTTPClient, err = rest.HTTPClientFor(config) + if err != nil { + return nil, err + } + } + // Init a scheme if none provided if options.Scheme == nil { options.Scheme = scheme.Scheme @@ -106,34 +143,35 @@ func newClient(config *rest.Config, options Options) (*client, error) { // Init a Mapper if none provided if options.Mapper == nil { var err error - options.Mapper, err = apiutil.NewDynamicRESTMapper(config) + options.Mapper, err = apiutil.NewDynamicRESTMapper(config, options.HTTPClient) if err != nil { return nil, err } } - clientcache := &clientCache{ - config: config, - scheme: options.Scheme, - mapper: options.Mapper, - codecs: serializer.NewCodecFactory(options.Scheme), + resources := &clientRestResources{ + httpClient: options.HTTPClient, + config: config, + scheme: options.Scheme, + mapper: options.Mapper, + codecs: serializer.NewCodecFactory(options.Scheme), structuredResourceByType: make(map[schema.GroupVersionKind]*resourceMeta), unstructuredResourceByType: make(map[schema.GroupVersionKind]*resourceMeta), } - rawMetaClient, err := metadata.NewForConfig(config) + rawMetaClient, err := metadata.NewForConfigAndClient(config, options.HTTPClient) if err != nil { return nil, fmt.Errorf("unable to construct metadata-only client for use as part of client: %w", err) } c := &client{ typedClient: typedClient{ - cache: clientcache, + resources: resources, paramCodec: runtime.NewParameterCodec(options.Scheme), }, unstructuredClient: unstructuredClient{ - cache: clientcache, + resources: resources, paramCodec: noConversionParamCodec{}, }, metadataClient: metadataClient{ @@ -143,20 +181,65 @@ func newClient(config *rest.Config, options Options) (*client, error) { scheme: options.Scheme, mapper: options.Mapper, } + if options.Cache == nil || options.Cache.Reader == nil { + return c, nil + } + + // We want a cache if we're here. + // Set the cache. + c.cache = options.Cache.Reader + // Load uncached GVKs. + c.cacheUnstructured = options.Cache.Unstructured + uncachedGVKs := map[schema.GroupVersionKind]struct{}{} + for _, obj := range options.Cache.DisableFor { + gvk, err := c.GroupVersionKindFor(obj) + if err != nil { + return nil, err + } + uncachedGVKs[gvk] = struct{}{} + } return c, nil } var _ Client = &client{} -// client is a client.Client that reads and writes directly from/to an API server. It lazily initializes -// new clients at the time they are used, and caches the client. +// client is a client.Client that reads and writes directly from/to an API server. +// It lazily initializes new clients at the time they are used. type client struct { typedClient typedClient unstructuredClient unstructuredClient metadataClient metadataClient scheme *runtime.Scheme mapper meta.RESTMapper + + cache Reader + uncachedGVKs map[schema.GroupVersionKind]struct{} + cacheUnstructured bool +} + +func (c *client) shouldBypassCache(obj runtime.Object) (bool, error) { + if c.cache == nil { + return true, nil + } + + gvk, err := c.GroupVersionKindFor(obj) + if err != nil { + return false, err + } + // TODO: this is producing unsafe guesses that don't actually work, + // but it matches ~99% of the cases out there. + if meta.IsListType(obj) { + gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") + } + if _, isUncached := c.uncachedGVKs[gvk]; isUncached { + return true, nil + } + if !c.cacheUnstructured { + _, isUnstructured := obj.(runtime.Unstructured) + return isUnstructured, nil + } + return false, nil } // resetGroupVersionKind is a helper function to restore and preserve GroupVersionKind on an object. @@ -168,6 +251,16 @@ func (c *client) resetGroupVersionKind(obj runtime.Object, gvk schema.GroupVersi } } +// GroupVersionKindFor returns the GroupVersionKind for the given object. +func (c *client) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + return apiutil.GVKForObject(obj, c.scheme) +} + +// IsObjectNamespaced returns true if the GroupVersionKind of the object is namespaced. +func (c *client) IsObjectNamespaced(obj runtime.Object) (bool, error) { + return apiutil.IsObjectNamespaced(obj, c.scheme, c.mapper) +} + // Scheme returns the scheme this client is using. func (c *client) Scheme() *runtime.Scheme { return c.scheme @@ -181,7 +274,7 @@ func (c *client) RESTMapper() meta.RESTMapper { // Create implements client.Client. func (c *client) Create(ctx context.Context, obj Object, opts ...CreateOption) error { switch obj.(type) { - case *unstructured.Unstructured: + case runtime.Unstructured: return c.unstructuredClient.Create(ctx, obj, opts...) case *metav1.PartialObjectMetadata: return fmt.Errorf("cannot create using only metadata") @@ -194,7 +287,7 @@ func (c *client) Create(ctx context.Context, obj Object, opts ...CreateOption) e func (c *client) Update(ctx context.Context, obj Object, opts ...UpdateOption) error { defer c.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind()) switch obj.(type) { - case *unstructured.Unstructured: + case runtime.Unstructured: return c.unstructuredClient.Update(ctx, obj, opts...) case *metav1.PartialObjectMetadata: return fmt.Errorf("cannot update using only metadata -- did you mean to patch?") @@ -206,7 +299,7 @@ func (c *client) Update(ctx context.Context, obj Object, opts ...UpdateOption) e // Delete implements client.Client. func (c *client) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error { switch obj.(type) { - case *unstructured.Unstructured: + case runtime.Unstructured: return c.unstructuredClient.Delete(ctx, obj, opts...) case *metav1.PartialObjectMetadata: return c.metadataClient.Delete(ctx, obj, opts...) @@ -218,7 +311,7 @@ func (c *client) Delete(ctx context.Context, obj Object, opts ...DeleteOption) e // DeleteAllOf implements client.Client. func (c *client) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error { switch obj.(type) { - case *unstructured.Unstructured: + case runtime.Unstructured: return c.unstructuredClient.DeleteAllOf(ctx, obj, opts...) case *metav1.PartialObjectMetadata: return c.metadataClient.DeleteAllOf(ctx, obj, opts...) @@ -231,7 +324,7 @@ func (c *client) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllO func (c *client) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { defer c.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind()) switch obj.(type) { - case *unstructured.Unstructured: + case runtime.Unstructured: return c.unstructuredClient.Patch(ctx, obj, patch, opts...) case *metav1.PartialObjectMetadata: return c.metadataClient.Patch(ctx, obj, patch, opts...) @@ -241,23 +334,35 @@ func (c *client) Patch(ctx context.Context, obj Object, patch Patch, opts ...Pat } // Get implements client.Client. -func (c *client) Get(ctx context.Context, key ObjectKey, obj Object) error { +func (c *client) Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error { + if isUncached, err := c.shouldBypassCache(obj); err != nil { + return err + } else if !isUncached { + return c.cache.Get(ctx, key, obj, opts...) + } + switch obj.(type) { - case *unstructured.Unstructured: - return c.unstructuredClient.Get(ctx, key, obj) + case runtime.Unstructured: + return c.unstructuredClient.Get(ctx, key, obj, opts...) case *metav1.PartialObjectMetadata: // Metadata only object should always preserve the GVK coming in from the caller. defer c.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind()) - return c.metadataClient.Get(ctx, key, obj) + return c.metadataClient.Get(ctx, key, obj, opts...) default: - return c.typedClient.Get(ctx, key, obj) + return c.typedClient.Get(ctx, key, obj, opts...) } } // List implements client.Client. func (c *client) List(ctx context.Context, obj ObjectList, opts ...ListOption) error { + if isUncached, err := c.shouldBypassCache(obj); err != nil { + return err + } else if !isUncached { + return c.cache.List(ctx, obj, opts...) + } + switch x := obj.(type) { - case *unstructured.UnstructuredList: + case runtime.Unstructured: return c.unstructuredClient.List(ctx, obj, opts...) case *metav1.PartialObjectMetadataList: // Metadata only object should always preserve the GVK. @@ -289,40 +394,194 @@ func (c *client) List(ctx context.Context, obj ObjectList, opts ...ListOption) e } // Status implements client.StatusClient. -func (c *client) Status() StatusWriter { - return &statusWriter{client: c} +func (c *client) Status() SubResourceWriter { + return c.SubResource("status") +} + +func (c *client) SubResource(subResource string) SubResourceClient { + return &subResourceClient{client: c, subResource: subResource} +} + +// subResourceClient is client.SubResourceWriter that writes to subresources. +type subResourceClient struct { + client *client + subResource string } -// statusWriter is client.StatusWriter that writes status subresource. -type statusWriter struct { - client *client +// ensure subResourceClient implements client.SubResourceClient. +var _ SubResourceClient = &subResourceClient{} + +// SubResourceGetOptions holds all the possible configuration +// for a subresource Get request. +type SubResourceGetOptions struct { + Raw *metav1.GetOptions +} + +// ApplyToSubResourceGet updates the configuaration to the given get options. +func (getOpt *SubResourceGetOptions) ApplyToSubResourceGet(o *SubResourceGetOptions) { + if getOpt.Raw != nil { + o.Raw = getOpt.Raw + } +} + +// ApplyOptions applues the given options. +func (getOpt *SubResourceGetOptions) ApplyOptions(opts []SubResourceGetOption) *SubResourceGetOptions { + for _, o := range opts { + o.ApplyToSubResourceGet(getOpt) + } + + return getOpt +} + +// AsGetOptions returns the configured options as *metav1.GetOptions. +func (getOpt *SubResourceGetOptions) AsGetOptions() *metav1.GetOptions { + if getOpt.Raw == nil { + return &metav1.GetOptions{} + } + return getOpt.Raw +} + +// SubResourceUpdateOptions holds all the possible configuration +// for a subresource update request. +type SubResourceUpdateOptions struct { + UpdateOptions + SubResourceBody Object +} + +// ApplyToSubResourceUpdate updates the configuration on the given create options +func (uo *SubResourceUpdateOptions) ApplyToSubResourceUpdate(o *SubResourceUpdateOptions) { + uo.UpdateOptions.ApplyToUpdate(&o.UpdateOptions) + if uo.SubResourceBody != nil { + o.SubResourceBody = uo.SubResourceBody + } } -// ensure statusWriter implements client.StatusWriter. -var _ StatusWriter = &statusWriter{} +// ApplyOptions applies the given options. +func (uo *SubResourceUpdateOptions) ApplyOptions(opts []SubResourceUpdateOption) *SubResourceUpdateOptions { + for _, o := range opts { + o.ApplyToSubResourceUpdate(uo) + } + + return uo +} + +// SubResourceUpdateAndPatchOption is an option that can be used for either +// a subresource update or patch request. +type SubResourceUpdateAndPatchOption interface { + SubResourceUpdateOption + SubResourcePatchOption +} + +// WithSubResourceBody returns an option that uses the given body +// for a subresource Update or Patch operation. +func WithSubResourceBody(body Object) SubResourceUpdateAndPatchOption { + return &withSubresourceBody{body: body} +} + +type withSubresourceBody struct { + body Object +} + +func (wsr *withSubresourceBody) ApplyToSubResourceUpdate(o *SubResourceUpdateOptions) { + o.SubResourceBody = wsr.body +} + +func (wsr *withSubresourceBody) ApplyToSubResourcePatch(o *SubResourcePatchOptions) { + o.SubResourceBody = wsr.body +} + +// SubResourceCreateOptions are all the possible configurations for a subresource +// create request. +type SubResourceCreateOptions struct { + CreateOptions +} -// Update implements client.StatusWriter. -func (sw *statusWriter) Update(ctx context.Context, obj Object, opts ...UpdateOption) error { - defer sw.client.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind()) +// ApplyOptions applies the given options. +func (co *SubResourceCreateOptions) ApplyOptions(opts []SubResourceCreateOption) *SubResourceCreateOptions { + for _, o := range opts { + o.ApplyToSubResourceCreate(co) + } + + return co +} + +// ApplyToSubresourceCreate applies the the configuration on the given create options. +func (co *SubResourceCreateOptions) ApplyToSubresourceCreate(o *SubResourceCreateOptions) { + co.CreateOptions.ApplyToCreate(&co.CreateOptions) +} + +// SubResourcePatchOptions holds all possible configurations for a subresource patch +// request. +type SubResourcePatchOptions struct { + PatchOptions + SubResourceBody Object +} + +// ApplyOptions applies the given options. +func (po *SubResourcePatchOptions) ApplyOptions(opts []SubResourcePatchOption) *SubResourcePatchOptions { + for _, o := range opts { + o.ApplyToSubResourcePatch(po) + } + + return po +} + +// ApplyToSubResourcePatch applies the configuration on the given patch options. +func (po *SubResourcePatchOptions) ApplyToSubResourcePatch(o *SubResourcePatchOptions) { + po.PatchOptions.ApplyToPatch(&o.PatchOptions) + if po.SubResourceBody != nil { + o.SubResourceBody = po.SubResourceBody + } +} + +func (sc *subResourceClient) Get(ctx context.Context, obj Object, subResource Object, opts ...SubResourceGetOption) error { switch obj.(type) { - case *unstructured.Unstructured: - return sw.client.unstructuredClient.UpdateStatus(ctx, obj, opts...) + case runtime.Unstructured: + return sc.client.unstructuredClient.GetSubResource(ctx, obj, subResource, sc.subResource, opts...) + case *metav1.PartialObjectMetadata: + return errors.New("can not get subresource using only metadata") + default: + return sc.client.typedClient.GetSubResource(ctx, obj, subResource, sc.subResource, opts...) + } +} + +// Create implements client.SubResourceClient +func (sc *subResourceClient) Create(ctx context.Context, obj Object, subResource Object, opts ...SubResourceCreateOption) error { + defer sc.client.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind()) + defer sc.client.resetGroupVersionKind(subResource, subResource.GetObjectKind().GroupVersionKind()) + + switch obj.(type) { + case runtime.Unstructured: + return sc.client.unstructuredClient.CreateSubResource(ctx, obj, subResource, sc.subResource, opts...) case *metav1.PartialObjectMetadata: return fmt.Errorf("cannot update status using only metadata -- did you mean to patch?") default: - return sw.client.typedClient.UpdateStatus(ctx, obj, opts...) + return sc.client.typedClient.CreateSubResource(ctx, obj, subResource, sc.subResource, opts...) } } -// Patch implements client.Client. -func (sw *statusWriter) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { - defer sw.client.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind()) +// Update implements client.SubResourceClient +func (sc *subResourceClient) Update(ctx context.Context, obj Object, opts ...SubResourceUpdateOption) error { + defer sc.client.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind()) + switch obj.(type) { + case runtime.Unstructured: + return sc.client.unstructuredClient.UpdateSubResource(ctx, obj, sc.subResource, opts...) + case *metav1.PartialObjectMetadata: + return fmt.Errorf("cannot update status using only metadata -- did you mean to patch?") + default: + return sc.client.typedClient.UpdateSubResource(ctx, obj, sc.subResource, opts...) + } +} + +// Patch implements client.SubResourceWriter. +func (sc *subResourceClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...SubResourcePatchOption) error { + defer sc.client.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind()) switch obj.(type) { - case *unstructured.Unstructured: - return sw.client.unstructuredClient.PatchStatus(ctx, obj, patch, opts...) + case runtime.Unstructured: + return sc.client.unstructuredClient.PatchSubResource(ctx, obj, sc.subResource, patch, opts...) case *metav1.PartialObjectMetadata: - return sw.client.metadataClient.PatchStatus(ctx, obj, patch, opts...) + return sc.client.metadataClient.PatchSubResource(ctx, obj, sc.subResource, patch, opts...) default: - return sw.client.typedClient.PatchStatus(ctx, obj, patch, opts...) + return sc.client.typedClient.PatchSubResource(ctx, obj, sc.subResource, patch, opts...) } } diff --git a/pkg/client/client_cache.go b/pkg/client/client_rest_resources.go similarity index 82% rename from pkg/client/client_cache.go rename to pkg/client/client_rest_resources.go index 857a0b38a7..2d07879520 100644 --- a/pkg/client/client_cache.go +++ b/pkg/client/client_rest_resources.go @@ -17,12 +17,12 @@ limitations under the License. package client import ( + "net/http" "strings" "sync" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -30,8 +30,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/apiutil" ) -// clientCache creates and caches rest clients and metadata for Kubernetes types. -type clientCache struct { +// clientRestResources creates and stores rest clients and metadata for Kubernetes types. +type clientRestResources struct { + // httpClient is the http client to use for requests + httpClient *http.Client + // config is the rest.Config to talk to an apiserver config *rest.Config @@ -44,22 +47,22 @@ type clientCache struct { // codecs are used to create a REST client for a gvk codecs serializer.CodecFactory - // structuredResourceByType caches structured type metadata + // structuredResourceByType stores structured type metadata structuredResourceByType map[schema.GroupVersionKind]*resourceMeta - // unstructuredResourceByType caches unstructured type metadata + // unstructuredResourceByType stores unstructured type metadata unstructuredResourceByType map[schema.GroupVersionKind]*resourceMeta mu sync.RWMutex } // newResource maps obj to a Kubernetes Resource and constructs a client for that Resource. // If the object is a list, the resource represents the item's type instead. -func (c *clientCache) newResource(gvk schema.GroupVersionKind, isList, isUnstructured bool) (*resourceMeta, error) { +func (c *clientRestResources) newResource(gvk schema.GroupVersionKind, isList, isUnstructured bool) (*resourceMeta, error) { if strings.HasSuffix(gvk.Kind, "List") && isList { // if this was a list, treat it as a request for the item's resource gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] } - client, err := apiutil.RESTClientForGVK(gvk, isUnstructured, c.config, c.codecs) + client, err := apiutil.RESTClientForGVK(gvk, isUnstructured, c.config, c.codecs, c.httpClient) if err != nil { return nil, err } @@ -72,15 +75,13 @@ func (c *clientCache) newResource(gvk schema.GroupVersionKind, isList, isUnstruc // getResource returns the resource meta information for the given type of object. // If the object is a list, the resource represents the item's type instead. -func (c *clientCache) getResource(obj runtime.Object) (*resourceMeta, error) { +func (c *clientRestResources) getResource(obj runtime.Object) (*resourceMeta, error) { gvk, err := apiutil.GVKForObject(obj, c.scheme) if err != nil { return nil, err } - _, isUnstructured := obj.(*unstructured.Unstructured) - _, isUnstructuredList := obj.(*unstructured.UnstructuredList) - isUnstructured = isUnstructured || isUnstructuredList + _, isUnstructured := obj.(runtime.Unstructured) // It's better to do creation work twice than to not let multiple // people make requests at once @@ -108,7 +109,7 @@ func (c *clientCache) getResource(obj runtime.Object) (*resourceMeta, error) { } // getObjMeta returns objMeta containing both type and object metadata and state. -func (c *clientCache) getObjMeta(obj runtime.Object) (*objMeta, error) { +func (c *clientRestResources) getObjMeta(obj runtime.Object) (*objMeta, error) { r, err := c.getResource(obj) if err != nil { return nil, err @@ -120,7 +121,7 @@ func (c *clientCache) getObjMeta(obj runtime.Object) (*objMeta, error) { return &objMeta{resourceMeta: r, Object: m}, err } -// resourceMeta caches state for a Kubernetes type. +// resourceMeta stores state for a Kubernetes type. type resourceMeta struct { // client is the rest client used to talk to the apiserver rest.Interface diff --git a/pkg/client/client_suite_test.go b/pkg/client/client_suite_test.go index e0e02575b2..f3942502d3 100644 --- a/pkg/client/client_suite_test.go +++ b/pkg/client/client_suite_test.go @@ -19,31 +19,31 @@ package client_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/examples/crd/pkg" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) -func TestSource(t *testing.T) { +func TestClient(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Client Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Client Suite") } var testenv *envtest.Environment var cfg *rest.Config var clientset *kubernetes.Clientset -var _ = BeforeSuite(func(done Done) { +var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) - testenv = &envtest.Environment{} + testenv = &envtest.Environment{CRDDirectoryPaths: []string{"./testdata"}} var err error cfg, err = testenv.Start() @@ -52,8 +52,8 @@ var _ = BeforeSuite(func(done Done) { clientset, err = kubernetes.NewForConfig(cfg) Expect(err).NotTo(HaveOccurred()) - close(done) -}, 60) + Expect(pkg.AddToScheme(scheme.Scheme)).NotTo(HaveOccurred()) +}) var _ = AfterSuite(func() { Expect(testenv.Stop()).To(Succeed()) diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index ebd6e47744..e2f53008e9 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -18,27 +18,34 @@ package client_test import ( "context" + "encoding/json" "fmt" + "reflect" "sync/atomic" "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + authenticationv1 "k8s.io/api/authentication/v1" + autoscalingv1 "k8s.io/api/autoscaling/v1" + certificatesv1 "k8s.io/api/certificates/v1" corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" kscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/examples/crd/pkg" "sigs.k8s.io/controller-runtime/pkg/client" ) -const serverSideTimeoutSeconds = 10 - func deleteDeployment(ctx context.Context, dep *appsv1.Deployment, ns string) { _, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) if err == nil { @@ -131,12 +138,14 @@ var _ = Describe("Client", func() { var dep *appsv1.Deployment var pod *corev1.Pod var node *corev1.Node + var serviceAccount *corev1.ServiceAccount + var csr *certificatesv1.CertificateSigningRequest var count uint64 = 0 var replicaCount int32 = 2 var ns = "default" ctx := context.TODO() - BeforeEach(func(done Done) { + BeforeEach(func() { atomic.AddUint64(&count, 1) dep = &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("deployment-name-%v", count), Namespace: ns, Labels: map[string]string{"app": fmt.Sprintf("bar-%v", count)}}, @@ -165,13 +174,35 @@ var _ = Describe("Client", func() { ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("node-name-%v", count)}, Spec: corev1.NodeSpec{}, } + serviceAccount = &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("sa-%v", count), Namespace: ns}} + csr = &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("csr-%v", count)}, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "org.io/my-signer", + Request: []byte(`-----BEGIN CERTIFICATE REQUEST----- +MIIChzCCAW8CAQAwQjELMAkGA1UEBhMCWFgxFTATBgNVBAcMDERlZmF1bHQgQ2l0 +eTEcMBoGA1UECgwTRGVmYXVsdCBDb21wYW55IEx0ZDCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBANe06dLX/bDNm6mVEnKdJexcJM6WKMFSt5o6BEdD1+Ki +WyUcvfNgIBbwAZjkF9U1r7+KuDcc6XYFnb6ky1wPo4C+XwcIIx7Nnbf8IdWJukPb +2BCsqO4NCsG6kKFavmH9J3q//nwKUvlQE+AJ2MPuOAZTwZ4KskghiGuS8hyk6/PZ +XH9QhV7Jma43bDzQozd2C7OujRBhLsuP94KSu839RRFWd9ms3XHgTxLxb7nxwZDx +9l7/ZVAObJoQYlHENqs12NCVP4gpJfbcY8/rd+IG4ftcZEmpeO4kKO+d2TpRKQqw +bjCMoAdD5Y43iLTtyql4qRnbMe3nxYG2+1inEryuV/cCAwEAAaAAMA0GCSqGSIb3 +DQEBCwUAA4IBAQDH5hDByRN7wERQtC/o6uc8Y+yhjq9YcBJjjbnD6Vwru5pOdWtx +qfKkkXI5KNOdEhWzLnJyOcWHjj8UoHqI3AjxGC7dTM95eGjxQGUpsUOX8JSd4MiZ +cct4g4BKBj02AGqZLiEgN+PLCYAmEaYU7oZc4OAh6WzMrljNRsj66awMQpw8O1eY +YuBa8vwz8ko8vn/pn7IrFu8cZ+EA3rluJ+budX/QrEGi1hijg27q7/Qr0wNI9f1v +086mLKdqaBTkblXWEvF3WP4CcLNyrSNi4eu+G0fcAgGp1F/Nqh0MuWKSOLprv5Om +U5wwSivyi7vmegHKmblOzNVKA5qPO8zWzqBC +-----END CERTIFICATE REQUEST-----`), + Usages: []certificatesv1.KeyUsage{certificatesv1.UsageClientAuth}, + }, + } scheme = kscheme.Scheme - - close(done) - }, serverSideTimeoutSeconds) + }) var delOptions *metav1.DeleteOptions - AfterEach(func(done Done) { + AfterEach(func() { // Cleanup var zero int64 = 0 policy := metav1.DeletePropagationForeground @@ -185,63 +216,72 @@ var _ = Describe("Client", func() { err = clientset.CoreV1().Nodes().Delete(ctx, node.Name, *delOptions) Expect(err).NotTo(HaveOccurred()) } - close(done) - }, serverSideTimeoutSeconds) + err = clientset.CoreV1().ServiceAccounts(ns).Delete(ctx, serviceAccount.Name, *delOptions) + Expect(client.IgnoreNotFound(err)).NotTo(HaveOccurred()) + + err = clientset.CertificatesV1().CertificateSigningRequests().Delete(ctx, csr.Name, *delOptions) + Expect(client.IgnoreNotFound(err)).NotTo(HaveOccurred()) + }) - // TODO(seans): Cast "cl" as "client" struct from "Client" interface. Then validate the - // instance values for the "client" struct. Describe("New", func() { - It("should return a new Client", func(done Done) { + It("should return a new Client", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) - - close(done) }) - It("should fail if the config is nil", func(done Done) { + It("should fail if the config is nil", func() { cl, err := client.New(nil, client.Options{}) Expect(err).To(HaveOccurred()) Expect(cl).To(BeNil()) - - close(done) }) - // TODO(seans): cast as client struct and inspect Scheme - It("should use the provided Scheme if provided", func(done Done) { + It("should use the provided Scheme if provided", func() { cl, err := client.New(cfg, client.Options{Scheme: scheme}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) - - close(done) + Expect(cl.Scheme()).ToNot(BeNil()) + Expect(cl.Scheme()).To(Equal(scheme)) }) - // TODO(seans): cast as client struct and inspect Scheme - It("should default the Scheme if not provided", func(done Done) { + It("should default the Scheme if not provided", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) - - close(done) + Expect(cl.Scheme()).ToNot(BeNil()) + Expect(cl.Scheme()).To(Equal(kscheme.Scheme)) }) - PIt("should use the provided Mapper if provided", func() { - + It("should use the provided Mapper if provided", func() { + mapper := meta.NewDefaultRESTMapper([]schema.GroupVersion{}) + cl, err := client.New(cfg, client.Options{Mapper: mapper}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + Expect(cl.RESTMapper()).ToNot(BeNil()) + Expect(cl.RESTMapper()).To(Equal(mapper)) }) - // TODO(seans): cast as client struct and inspect Mapper - It("should create a Mapper if not provided", func(done Done) { + It("should create a Mapper if not provided", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) + Expect(cl.RESTMapper()).ToNot(BeNil()) + }) - close(done) + It("should use the provided reader cache if provided, on get and list", func() { + cache := &fakeReader{} + cl, err := client.New(cfg, client.Options{Cache: &client.CacheOptions{Reader: cache}}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + Expect(cl.Get(ctx, client.ObjectKey{Name: "test"}, &appsv1.Deployment{})).To(Succeed()) + Expect(cl.List(ctx, &appsv1.DeploymentList{})).To(Succeed()) + Expect(cache.Called).To(Equal(2)) }) }) Describe("Create", func() { Context("with structured objects", func() { - It("should create a new object from a go struct", func(done Done) { + It("should create a new object from a go struct", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -256,11 +296,9 @@ var _ = Describe("Client", func() { By("writing the result back to the go struct") Expect(dep).To(Equal(actual)) - - close(done) }) - It("should create a new object non-namespace object from a go struct", func(done Done) { + It("should create a new object non-namespace object from a go struct", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -275,11 +313,9 @@ var _ = Describe("Client", func() { By("writing the result back to the go struct") Expect(node).To(Equal(actual)) - - close(done) }) - It("should fail if the object already exists", func(done Done) { + It("should fail if the object already exists", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -298,11 +334,9 @@ var _ = Describe("Client", func() { err = cl.Create(context.TODO(), old) Expect(err).To(HaveOccurred()) Expect(apierrors.IsAlreadyExists(err)).To(BeTrue()) - - close(done) }) - It("should fail if the object does not pass server-side validation", func(done Done) { + It("should fail if the object does not pass server-side validation", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -312,9 +346,7 @@ var _ = Describe("Client", func() { Expect(err).To(HaveOccurred()) // TODO(seans): Add test to validate the returned error. Problems currently with // different returned error locally versus travis. - - close(done) - }, serverSideTimeoutSeconds) + }) It("should fail if the object cannot be mapped to a GVK", func() { By("creating client with empty Scheme") @@ -335,7 +367,22 @@ var _ = Describe("Client", func() { }) Context("with the DryRun option", func() { - It("should not create a new object", func(done Done) { + It("should not create a new object, global option", func() { + cl, err := client.New(cfg, client.Options{DryRun: pointer.Bool(true)}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("creating the object (with DryRun)") + err = cl.Create(context.TODO(), dep) + Expect(err).NotTo(HaveOccurred()) + + actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).To(HaveOccurred()) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + Expect(actual).To(Equal(&appsv1.Deployment{})) + }) + + It("should not create a new object, inline option", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -348,14 +395,12 @@ var _ = Describe("Client", func() { Expect(err).To(HaveOccurred()) Expect(apierrors.IsNotFound(err)).To(BeTrue()) Expect(actual).To(Equal(&appsv1.Deployment{})) - - close(done) }) }) }) Context("with unstructured objects", func() { - It("should create a new object from a go struct", func(done Done) { + It("should create a new object from a go struct", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -376,10 +421,9 @@ var _ = Describe("Client", func() { actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) - close(done) }) - It("should create a new non-namespace object ", func(done Done) { + It("should create a new non-namespace object ", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -406,11 +450,9 @@ var _ = Describe("Client", func() { By("writing the result back to the go struct") Expect(u).To(Equal(au)) - - close(done) }) - It("should fail if the object already exists", func(done Done) { + It("should fail if the object already exists", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -437,11 +479,9 @@ var _ = Describe("Client", func() { err = cl.Create(context.TODO(), u) Expect(err).To(HaveOccurred()) Expect(apierrors.IsAlreadyExists(err)).To(BeTrue()) - - close(done) }) - It("should fail if the object does not pass server-side validation", func(done Done) { + It("should fail if the object does not pass server-side validation", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -458,9 +498,7 @@ var _ = Describe("Client", func() { Expect(err).To(HaveOccurred()) // TODO(seans): Add test to validate the returned error. Problems currently with // different returned error locally versus travis. - - close(done) - }, serverSideTimeoutSeconds) + }) }) @@ -475,7 +513,7 @@ var _ = Describe("Client", func() { }) Context("with the DryRun option", func() { - It("should not create a new object from a go struct", func(done Done) { + It("should not create a new object from a go struct", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -497,15 +535,13 @@ var _ = Describe("Client", func() { Expect(err).To(HaveOccurred()) Expect(apierrors.IsNotFound(err)).To(BeTrue()) Expect(actual).To(Equal(&appsv1.Deployment{})) - - close(done) }) }) }) Describe("Update", func() { Context("with structured objects", func() { - It("should update an existing object from a go struct", func(done Done) { + It("should update an existing object from a go struct", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -524,11 +560,9 @@ var _ = Describe("Client", func() { Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) Expect(actual.Annotations["foo"]).To(Equal("bar")) - - close(done) }) - It("should update and preserve type information", func(done Done) { + It("should update and preserve type information", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -544,11 +578,9 @@ var _ = Describe("Client", func() { By("validating updated Deployment has type information") Expect(dep.GroupVersionKind()).To(Equal(depGvk)) - - close(done) }) - It("should update an existing object non-namespace object from a go struct", func(done Done) { + It("should update an existing object non-namespace object from a go struct", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -566,11 +598,9 @@ var _ = Describe("Client", func() { Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) Expect(actual.Annotations["foo"]).To(Equal("bar")) - - close(done) }) - It("should fail if the object does not exist", func(done Done) { + It("should fail if the object does not exist", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -578,8 +608,6 @@ var _ = Describe("Client", func() { By("updating non-existent object") err = cl.Update(context.TODO(), dep) Expect(err).To(HaveOccurred()) - - close(done) }) PIt("should fail if the object does not pass server-side validation", func() { @@ -590,7 +618,7 @@ var _ = Describe("Client", func() { }) - It("should fail if the object cannot be mapped to a GVK", func(done Done) { + It("should fail if the object cannot be mapped to a GVK", func() { By("creating client with empty Scheme") emptyScheme := runtime.NewScheme() cl, err := client.New(cfg, client.Options{Scheme: emptyScheme}) @@ -606,8 +634,6 @@ var _ = Describe("Client", func() { err = cl.Update(context.TODO(), dep) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("no kind is registered for the type")) - - close(done) }) PIt("should fail if the GVK cannot be mapped to a Resource", func() { @@ -615,7 +641,7 @@ var _ = Describe("Client", func() { }) }) Context("with unstructured objects", func() { - It("should update an existing object from a go struct", func(done Done) { + It("should update an existing object from a go struct", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -641,11 +667,9 @@ var _ = Describe("Client", func() { Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) Expect(actual.Annotations["foo"]).To(Equal("bar")) - - close(done) }) - It("should update and preserve type information", func(done Done) { + It("should update and preserve type information", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -664,11 +688,9 @@ var _ = Describe("Client", func() { By("validating updated Deployment has type information") Expect(u.GroupVersionKind()).To(Equal(depGvk)) - - close(done) }) - It("should update an existing object non-namespace object from a go struct", func(done Done) { + It("should update an existing object non-namespace object from a go struct", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -693,10 +715,8 @@ var _ = Describe("Client", func() { Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) Expect(actual.Annotations["foo"]).To(Equal("bar")) - - close(done) }) - It("should fail if the object does not exist", func(done Done) { + It("should fail if the object does not exist", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -707,8 +727,6 @@ var _ = Describe("Client", func() { u.SetGroupVersionKind(depGvk) err = cl.Update(context.TODO(), dep) Expect(err).To(HaveOccurred()) - - close(done) }) }) Context("with metadata objects", func() { @@ -725,7 +743,7 @@ var _ = Describe("Client", func() { Describe("Patch", func() { Context("Metadata Client", func() { - It("should merge patch with options", func(done Done) { + It("should merge patch with options", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -751,14 +769,437 @@ var _ = Describe("Client", func() { By("validating patch options were applied") Expect(testOption.applied).To(Equal(true)) - close(done) }) }) }) + Describe("SubResourceClient", func() { + Context("with structured objects", func() { + It("should be able to read the Scale subresource", func() { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating a deployment") + dep, err := clientset.AppsV1().Deployments(dep.Namespace).Create(ctx, dep, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("reading the scale subresource") + scale := &autoscalingv1.Scale{} + err = cl.SubResource("scale").Get(ctx, dep, scale) + Expect(err).NotTo(HaveOccurred()) + Expect(scale.Spec.Replicas).To(Equal(*dep.Spec.Replicas)) + }) + It("should be able to create ServiceAccount tokens", func() { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating the serviceAccount") + _, err = clientset.CoreV1().ServiceAccounts(serviceAccount.Namespace).Create(ctx, serviceAccount, metav1.CreateOptions{}) + Expect((err)).NotTo(HaveOccurred()) + + token := &authenticationv1.TokenRequest{} + err = cl.SubResource("token").Create(ctx, serviceAccount, token) + Expect(err).NotTo(HaveOccurred()) + + Expect(token.Status.Token).NotTo(Equal("")) + }) + + It("should be able to create Pod evictions", func() { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + // Make the pod valid + pod.Spec.Containers = []corev1.Container{{Name: "foo", Image: "busybox"}} + + By("Creating the pod") + pod, err = clientset.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Creating the eviction") + eviction := &policyv1.Eviction{ + DeleteOptions: &metav1.DeleteOptions{GracePeriodSeconds: ptr(int64(0))}, + } + err = cl.SubResource("eviction").Create(ctx, pod, eviction) + Expect((err)).NotTo(HaveOccurred()) + + By("Asserting the pod is gone") + _, err = clientset.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("should be able to create Pod bindings", func() { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + // Make the pod valid + pod.Spec.Containers = []corev1.Container{{Name: "foo", Image: "busybox"}} + + By("Creating the pod") + pod, err = clientset.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Creating the binding") + binding := &corev1.Binding{ + Target: corev1.ObjectReference{Name: node.Name}, + } + err = cl.SubResource("binding").Create(ctx, pod, binding) + Expect((err)).NotTo(HaveOccurred()) + + By("Asserting the pod is bound") + pod, err = clientset.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(pod.Spec.NodeName).To(Equal(node.Name)) + }) + + It("should be able to approve CSRs", func() { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating the CSR") + csr, err := clientset.CertificatesV1().CertificateSigningRequests().Create(ctx, csr, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Approving the CSR") + csr.Status.Conditions = append(csr.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateApproved, + Status: corev1.ConditionTrue, + }) + err = cl.SubResource("approval").Update(ctx, csr) + Expect(err).NotTo(HaveOccurred()) + + By("Asserting the CSR is approved") + csr, err = clientset.CertificatesV1().CertificateSigningRequests().Get(ctx, csr.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(csr.Status.Conditions[0].Type).To(Equal(certificatesv1.CertificateApproved)) + Expect(csr.Status.Conditions[0].Status).To(Equal(corev1.ConditionTrue)) + }) + + It("should be able to approve CSRs using Patch", func() { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating the CSR") + csr, err := clientset.CertificatesV1().CertificateSigningRequests().Create(ctx, csr, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Approving the CSR") + patch := client.MergeFrom(csr.DeepCopy()) + csr.Status.Conditions = append(csr.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateApproved, + Status: corev1.ConditionTrue, + }) + err = cl.SubResource("approval").Patch(ctx, csr, patch) + Expect(err).NotTo(HaveOccurred()) + + By("Asserting the CSR is approved") + csr, err = clientset.CertificatesV1().CertificateSigningRequests().Get(ctx, csr.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(csr.Status.Conditions[0].Type).To(Equal(certificatesv1.CertificateApproved)) + Expect(csr.Status.Conditions[0].Status).To(Equal(corev1.ConditionTrue)) + }) + + It("should be able to update the scale subresource", func() { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating a deployment") + dep, err := clientset.AppsV1().Deployments(dep.Namespace).Create(ctx, dep, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Updating the scale subresurce") + replicaCount := *dep.Spec.Replicas + scale := &autoscalingv1.Scale{Spec: autoscalingv1.ScaleSpec{Replicas: replicaCount}} + err = cl.SubResource("scale").Update(ctx, dep, client.WithSubResourceBody(scale)) + Expect(err).NotTo(HaveOccurred()) + + By("Asserting replicas got updated") + dep, err = clientset.AppsV1().Deployments(dep.Namespace).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(*dep.Spec.Replicas).To(Equal(replicaCount)) + }) + + It("should be able to patch the scale subresource", func() { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating a deployment") + dep, err := clientset.AppsV1().Deployments(dep.Namespace).Create(ctx, dep, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Updating the scale subresurce") + replicaCount := *dep.Spec.Replicas + patch := client.MergeFrom(&autoscalingv1.Scale{}) + scale := &autoscalingv1.Scale{Spec: autoscalingv1.ScaleSpec{Replicas: replicaCount}} + err = cl.SubResource("scale").Patch(ctx, dep, patch, client.WithSubResourceBody(scale)) + Expect(err).NotTo(HaveOccurred()) + + By("Asserting replicas got updated") + dep, err = clientset.AppsV1().Deployments(dep.Namespace).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(*dep.Spec.Replicas).To(Equal(replicaCount)) + }) + }) + + Context("with unstructured objects", func() { + It("should be able to read the Scale subresource", func() { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating a deployment") + dep, err := clientset.AppsV1().Deployments(dep.Namespace).Create(ctx, dep, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + dep.APIVersion = appsv1.SchemeGroupVersion.String() + dep.Kind = reflect.TypeOf(dep).Elem().Name() + depUnstructured, err := toUnstructured(dep) + Expect(err).NotTo(HaveOccurred()) + + By("reading the scale subresource") + scale := &unstructured.Unstructured{} + scale.SetAPIVersion("autoscaling/v1") + scale.SetKind("Scale") + err = cl.SubResource("scale").Get(ctx, depUnstructured, scale) + Expect(err).NotTo(HaveOccurred()) + + val, found, err := unstructured.NestedInt64(scale.UnstructuredContent(), "spec", "replicas") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(int32(val)).To(Equal(*dep.Spec.Replicas)) + }) + It("should be able to create ServiceAccount tokens", func() { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating the serviceAccount") + _, err = clientset.CoreV1().ServiceAccounts(serviceAccount.Namespace).Create(ctx, serviceAccount, metav1.CreateOptions{}) + Expect((err)).NotTo(HaveOccurred()) + + serviceAccount.APIVersion = "v1" + serviceAccount.Kind = "ServiceAccount" + serviceAccountUnstructured, err := toUnstructured(serviceAccount) + Expect(err).NotTo(HaveOccurred()) + + token := &unstructured.Unstructured{} + token.SetAPIVersion("authentication.k8s.io/v1") + token.SetKind("TokenRequest") + err = cl.SubResource("token").Create(ctx, serviceAccountUnstructured, token) + Expect(err).NotTo(HaveOccurred()) + Expect(token.GetAPIVersion()).To(Equal("authentication.k8s.io/v1")) + Expect(token.GetKind()).To(Equal("TokenRequest")) + + val, found, err := unstructured.NestedString(token.UnstructuredContent(), "status", "token") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(val).NotTo(Equal("")) + }) + + It("should be able to create Pod evictions", func() { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + // Make the pod valid + pod.Spec.Containers = []corev1.Container{{Name: "foo", Image: "busybox"}} + + By("Creating the pod") + pod, err = clientset.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + pod.APIVersion = "v1" + pod.Kind = "Pod" + podUnstructured, err := toUnstructured(pod) + Expect(err).NotTo(HaveOccurred()) + + By("Creating the eviction") + eviction := &unstructured.Unstructured{} + eviction.SetAPIVersion("policy/v1") + eviction.SetKind("Eviction") + err = unstructured.SetNestedField(eviction.UnstructuredContent(), int64(0), "deleteOptions", "gracePeriodSeconds") + Expect(err).NotTo(HaveOccurred()) + err = cl.SubResource("eviction").Create(ctx, podUnstructured, eviction) + Expect(err).NotTo(HaveOccurred()) + Expect(eviction.GetAPIVersion()).To(Equal("policy/v1")) + Expect(eviction.GetKind()).To(Equal("Eviction")) + + By("Asserting the pod is gone") + _, err = clientset.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{}) + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + + It("should be able to create Pod bindings", func() { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + // Make the pod valid + pod.Spec.Containers = []corev1.Container{{Name: "foo", Image: "busybox"}} + + By("Creating the pod") + pod, err = clientset.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + pod.APIVersion = "v1" + pod.Kind = "Pod" + podUnstructured, err := toUnstructured(pod) + Expect(err).NotTo(HaveOccurred()) + + By("Creating the binding") + binding := &unstructured.Unstructured{} + binding.SetAPIVersion("v1") + binding.SetKind("Binding") + err = unstructured.SetNestedField(binding.UnstructuredContent(), node.Name, "target", "name") + Expect(err).NotTo(HaveOccurred()) + + err = cl.SubResource("binding").Create(ctx, podUnstructured, binding) + Expect((err)).NotTo(HaveOccurred()) + Expect(binding.GetAPIVersion()).To(Equal("v1")) + Expect(binding.GetKind()).To(Equal("Binding")) + + By("Asserting the pod is bound") + pod, err = clientset.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(pod.Spec.NodeName).To(Equal(node.Name)) + }) + + It("should be able to approve CSRs", func() { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating the CSR") + csr, err := clientset.CertificatesV1().CertificateSigningRequests().Create(ctx, csr, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Approving the CSR") + csr.Status.Conditions = append(csr.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateApproved, + Status: corev1.ConditionTrue, + }) + csr.APIVersion = "certificates.k8s.io/v1" + csr.Kind = "CertificateSigningRequest" + csrUnstructured, err := toUnstructured(csr) + Expect(err).NotTo(HaveOccurred()) + + err = cl.SubResource("approval").Update(ctx, csrUnstructured) + Expect(err).NotTo(HaveOccurred()) + Expect(csrUnstructured.GetAPIVersion()).To(Equal("certificates.k8s.io/v1")) + Expect(csrUnstructured.GetKind()).To(Equal("CertificateSigningRequest")) + + By("Asserting the CSR is approved") + csr, err = clientset.CertificatesV1().CertificateSigningRequests().Get(ctx, csr.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(csr.Status.Conditions[0].Type).To(Equal(certificatesv1.CertificateApproved)) + Expect(csr.Status.Conditions[0].Status).To(Equal(corev1.ConditionTrue)) + }) + + It("should be able to approve CSRs using Patch", func() { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating the CSR") + csr, err := clientset.CertificatesV1().CertificateSigningRequests().Create(ctx, csr, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By("Approving the CSR") + patch := client.MergeFrom(csr.DeepCopy()) + csr.Status.Conditions = append(csr.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateApproved, + Status: corev1.ConditionTrue, + }) + csr.APIVersion = "certificates.k8s.io/v1" + csr.Kind = "CertificateSigningRequest" + csrUnstructured, err := toUnstructured(csr) + Expect(err).NotTo(HaveOccurred()) + + err = cl.SubResource("approval").Patch(ctx, csrUnstructured, patch) + Expect(err).NotTo(HaveOccurred()) + Expect(csrUnstructured.GetAPIVersion()).To(Equal("certificates.k8s.io/v1")) + Expect(csrUnstructured.GetKind()).To(Equal("CertificateSigningRequest")) + + By("Asserting the CSR is approved") + csr, err = clientset.CertificatesV1().CertificateSigningRequests().Get(ctx, csr.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(csr.Status.Conditions[0].Type).To(Equal(certificatesv1.CertificateApproved)) + Expect(csr.Status.Conditions[0].Status).To(Equal(corev1.ConditionTrue)) + }) + + It("should be able to update the scale subresource", func() { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating a deployment") + dep, err := clientset.AppsV1().Deployments(dep.Namespace).Create(ctx, dep, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + dep.APIVersion = "apps/v1" + dep.Kind = "Deployment" + depUnstructured, err := toUnstructured(dep) + Expect(err).NotTo(HaveOccurred()) + + By("Updating the scale subresurce") + replicaCount := *dep.Spec.Replicas + scale := &unstructured.Unstructured{} + scale.SetAPIVersion("autoscaling/v1") + scale.SetKind("Scale") + Expect(unstructured.SetNestedField(scale.Object, int64(replicaCount), "spec", "replicas")).NotTo(HaveOccurred()) + err = cl.SubResource("scale").Update(ctx, depUnstructured, client.WithSubResourceBody(scale)) + Expect(err).NotTo(HaveOccurred()) + Expect(scale.GetAPIVersion()).To(Equal("autoscaling/v1")) + Expect(scale.GetKind()).To(Equal("Scale")) + + By("Asserting replicas got updated") + dep, err = clientset.AppsV1().Deployments(dep.Namespace).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(*dep.Spec.Replicas).To(Equal(replicaCount)) + }) + + It("should be able to patch the scale subresource", func() { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("Creating a deployment") + dep, err := clientset.AppsV1().Deployments(dep.Namespace).Create(ctx, dep, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + dep.APIVersion = "apps/v1" + dep.Kind = "Deployment" + depUnstructured, err := toUnstructured(dep) + Expect(err).NotTo(HaveOccurred()) + + By("Updating the scale subresurce") + replicaCount := *dep.Spec.Replicas + scale := &unstructured.Unstructured{} + scale.SetAPIVersion("autoscaling/v1") + scale.SetKind("Scale") + patch := client.MergeFrom(scale.DeepCopy()) + Expect(unstructured.SetNestedField(scale.Object, int64(replicaCount), "spec", "replicas")).NotTo(HaveOccurred()) + err = cl.SubResource("scale").Patch(ctx, depUnstructured, patch, client.WithSubResourceBody(scale)) + Expect(err).NotTo(HaveOccurred()) + Expect(scale.GetAPIVersion()).To(Equal("autoscaling/v1")) + Expect(scale.GetKind()).To(Equal("Scale")) + + By("Asserting replicas got updated") + dep, err = clientset.AppsV1().Deployments(dep.Namespace).Get(ctx, dep.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(*dep.Spec.Replicas).To(Equal(replicaCount)) + }) + }) + + }) + Describe("StatusClient", func() { Context("with structured objects", func() { - It("should update status of an existing object", func(done Done) { + It("should update status of an existing object", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -777,11 +1218,9 @@ var _ = Describe("Client", func() { Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) Expect(actual.Status.Replicas).To(BeEquivalentTo(1)) - - close(done) }) - It("should update status and preserve type information", func(done Done) { + It("should update status and preserve type information", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -798,11 +1237,9 @@ var _ = Describe("Client", func() { By("validating updated Deployment has type information") Expect(dep.GroupVersionKind()).To(Equal(depGvk)) - - close(done) }) - It("should patch status and preserve type information", func(done Done) { + It("should patch status and preserve type information", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -820,11 +1257,9 @@ var _ = Describe("Client", func() { By("validating updated Deployment has type information") Expect(dep.GroupVersionKind()).To(Equal(depGvk)) - - close(done) }) - It("should not update spec of an existing object", func(done Done) { + It("should not update spec of an existing object", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -846,11 +1281,9 @@ var _ = Describe("Client", func() { Expect(actual).NotTo(BeNil()) Expect(actual.Status.Replicas).To(BeEquivalentTo(1)) Expect(*actual.Spec.Replicas).To(BeEquivalentTo(replicaCount)) - - close(done) }) - It("should update an existing object non-namespace object", func(done Done) { + It("should update an existing object non-namespace object", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -868,11 +1301,9 @@ var _ = Describe("Client", func() { Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) Expect(actual.Status.Phase).To(Equal(corev1.NodeRunning)) - - close(done) }) - It("should fail if the object does not exist", func(done Done) { + It("should fail if the object does not exist", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -880,11 +1311,9 @@ var _ = Describe("Client", func() { By("updating status of a non-existent object") err = cl.Status().Update(context.TODO(), dep) Expect(err).To(HaveOccurred()) - - close(done) }) - It("should fail if the object cannot be mapped to a GVK", func(done Done) { + It("should fail if the object cannot be mapped to a GVK", func() { By("creating client with empty Scheme") emptyScheme := runtime.NewScheme() cl, err := client.New(cfg, client.Options{Scheme: emptyScheme}) @@ -900,8 +1329,6 @@ var _ = Describe("Client", func() { err = cl.Status().Update(context.TODO(), dep) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("no kind is registered for the type")) - - close(done) }) PIt("should fail if the GVK cannot be mapped to a Resource", func() { @@ -914,7 +1341,7 @@ var _ = Describe("Client", func() { }) Context("with unstructured objects", func() { - It("should update status of an existing object", func(done Done) { + It("should update status of an existing object", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -935,11 +1362,9 @@ var _ = Describe("Client", func() { Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) Expect(actual.Status.Replicas).To(BeEquivalentTo(1)) - - close(done) }) - It("should update status and preserve type information", func(done Done) { + It("should update status and preserve type information", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -957,11 +1382,9 @@ var _ = Describe("Client", func() { By("validating updated Deployment has type information") Expect(u.GroupVersionKind()).To(Equal(depGvk)) - - close(done) }) - It("should patch status and preserve type information", func(done Done) { + It("should patch status and preserve type information", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -986,11 +1409,9 @@ var _ = Describe("Client", func() { Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) Expect(actual.Status.Replicas).To(BeEquivalentTo(1)) - - close(done) }) - It("should not update spec of an existing object", func(done Done) { + It("should not update spec of an existing object", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1014,11 +1435,9 @@ var _ = Describe("Client", func() { Expect(actual).NotTo(BeNil()) Expect(actual.Status.Replicas).To(BeEquivalentTo(1)) Expect(*actual.Spec.Replicas).To(BeEquivalentTo(replicaCount)) - - close(done) }) - It("should update an existing object non-namespace object", func(done Done) { + It("should update an existing object non-namespace object", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1038,11 +1457,9 @@ var _ = Describe("Client", func() { Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) Expect(actual.Status.Phase).To(Equal(corev1.NodeRunning)) - - close(done) }) - It("should fail if the object does not exist", func(done Done) { + It("should fail if the object does not exist", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1052,8 +1469,6 @@ var _ = Describe("Client", func() { Expect(scheme.Convert(dep, u, nil)).To(Succeed()) err = cl.Status().Update(context.TODO(), u) Expect(err).To(HaveOccurred()) - - close(done) }) PIt("should fail if the GVK cannot be mapped to a Resource", func() { @@ -1075,7 +1490,7 @@ var _ = Describe("Client", func() { Expect(cl.Status().Update(context.TODO(), obj)).NotTo(Succeed()) }) - It("should patch status and preserve type information", func(done Done) { + It("should patch status and preserve type information", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1099,15 +1514,13 @@ var _ = Describe("Client", func() { Expect(err).NotTo(HaveOccurred()) Expect(actual).NotTo(BeNil()) Expect(actual.Annotations).To(HaveKeyWithValue("some-new-annotation", "some-new-value")) - - close(done) }) }) }) Describe("Delete", func() { Context("with structured objects", func() { - It("should delete an existing object from a go struct", func(done Done) { + It("should delete an existing object from a go struct", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1124,11 +1537,9 @@ var _ = Describe("Client", func() { By("validating the Deployment no longer exists") _, err = clientset.AppsV1().Deployments(ns).Get(ctx, depName, metav1.GetOptions{}) Expect(err).To(HaveOccurred()) - - close(done) }) - It("should delete an existing object non-namespace object from a go struct", func(done Done) { + It("should delete an existing object non-namespace object from a go struct", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1145,11 +1556,9 @@ var _ = Describe("Client", func() { By("validating the Node no longer exists") _, err = clientset.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) Expect(err).To(HaveOccurred()) - - close(done) }) - It("should fail if the object does not exist", func(done Done) { + It("should fail if the object does not exist", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1157,15 +1566,13 @@ var _ = Describe("Client", func() { By("Deleting node before it is ever created") err = cl.Delete(context.TODO(), node) Expect(err).To(HaveOccurred()) - - close(done) }) PIt("should fail if the object doesn't have meta", func() { }) - It("should fail if the object cannot be mapped to a GVK", func(done Done) { + It("should fail if the object cannot be mapped to a GVK", func() { By("creating client with empty Scheme") emptyScheme := runtime.NewScheme() cl, err := client.New(cfg, client.Options{Scheme: emptyScheme}) @@ -1180,15 +1587,13 @@ var _ = Describe("Client", func() { err = cl.Delete(context.TODO(), dep) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("no kind is registered for the type")) - - close(done) }) PIt("should fail if the GVK cannot be mapped to a Resource", func() { }) - It("should delete a collection of objects", func(done Done) { + It("should delete a collection of objects", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1196,7 +1601,7 @@ var _ = Describe("Client", func() { By("initially creating two Deployments") dep2 := dep.DeepCopy() - dep2.Name = dep2.Name + "-2" + dep2.Name += "-2" dep, err = clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1215,12 +1620,10 @@ var _ = Describe("Client", func() { Expect(err).To(HaveOccurred()) _, err = clientset.AppsV1().Deployments(ns).Get(ctx, dep2Name, metav1.GetOptions{}) Expect(err).To(HaveOccurred()) - - close(done) }) }) Context("with unstructured objects", func() { - It("should delete an existing object from a go struct", func(done Done) { + It("should delete an existing object from a go struct", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1244,11 +1647,9 @@ var _ = Describe("Client", func() { By("validating the Deployment no longer exists") _, err = clientset.AppsV1().Deployments(ns).Get(ctx, depName, metav1.GetOptions{}) Expect(err).To(HaveOccurred()) - - close(done) }) - It("should delete an existing object non-namespace object from a go struct", func(done Done) { + It("should delete an existing object non-namespace object from a go struct", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1272,11 +1673,9 @@ var _ = Describe("Client", func() { By("validating the Node no longer exists") _, err = clientset.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) Expect(err).To(HaveOccurred()) - - close(done) }) - It("should fail if the object does not exist", func(done Done) { + It("should fail if the object does not exist", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1291,11 +1690,9 @@ var _ = Describe("Client", func() { }) err = cl.Delete(context.TODO(), node) Expect(err).To(HaveOccurred()) - - close(done) }) - It("should delete a collection of object", func(done Done) { + It("should delete a collection of object", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1303,7 +1700,7 @@ var _ = Describe("Client", func() { By("initially creating two Deployments") dep2 := dep.DeepCopy() - dep2.Name = dep2.Name + "-2" + dep2.Name += "-2" dep, err = clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1329,12 +1726,10 @@ var _ = Describe("Client", func() { Expect(err).To(HaveOccurred()) _, err = clientset.AppsV1().Deployments(ns).Get(ctx, dep2Name, metav1.GetOptions{}) Expect(err).To(HaveOccurred()) - - close(done) }) }) Context("with metadata objects", func() { - It("should delete an existing object from a go struct", func(done Done) { + It("should delete an existing object from a go struct", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1351,11 +1746,9 @@ var _ = Describe("Client", func() { By("validating the Deployment no longer exists") _, err = clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) Expect(err).To(HaveOccurred()) - - close(done) }) - It("should delete an existing object non-namespace object from a go struct", func(done Done) { + It("should delete an existing object non-namespace object from a go struct", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1372,11 +1765,9 @@ var _ = Describe("Client", func() { By("validating the Node no longer exists") _, err = clientset.CoreV1().Nodes().Get(ctx, node.Name, metav1.GetOptions{}) Expect(err).To(HaveOccurred()) - - close(done) }) - It("should fail if the object does not exist", func(done Done) { + It("should fail if the object does not exist", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1385,11 +1776,9 @@ var _ = Describe("Client", func() { metaObj := metaOnlyFromObj(node, scheme) err = cl.Delete(context.TODO(), metaObj) Expect(err).To(HaveOccurred()) - - close(done) }) - It("should delete a collection of object", func(done Done) { + It("should delete a collection of object", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1397,7 +1786,7 @@ var _ = Describe("Client", func() { By("initially creating two Deployments") dep2 := dep.DeepCopy() - dep2.Name = dep2.Name + "-2" + dep2.Name += "-2" dep, err = clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1417,15 +1806,13 @@ var _ = Describe("Client", func() { Expect(err).To(HaveOccurred()) _, err = clientset.AppsV1().Deployments(ns).Get(ctx, dep2Name, metav1.GetOptions{}) Expect(err).To(HaveOccurred()) - - close(done) }) }) }) Describe("Get", func() { Context("with structured objects", func() { - It("should fetch an existing object for a go struct", func(done Done) { + It("should fetch an existing object for a go struct", func() { By("first creating the Deployment") dep, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1443,11 +1830,9 @@ var _ = Describe("Client", func() { By("validating the fetched deployment equals the created one") Expect(dep).To(Equal(&actual)) - - close(done) }) - It("should fetch an existing non-namespace object for a go struct", func(done Done) { + It("should fetch an existing non-namespace object for a go struct", func() { By("first creating the object") node, err := clientset.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1464,11 +1849,9 @@ var _ = Describe("Client", func() { Expect(actual).NotTo(BeNil()) Expect(node).To(Equal(&actual)) - - close(done) }) - It("should fail if the object does not exist", func(done Done) { + It("should fail if the object does not exist", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1478,8 +1861,6 @@ var _ = Describe("Client", func() { var actual appsv1.Deployment err = cl.Get(context.TODO(), key, &actual) Expect(err).To(HaveOccurred()) - - close(done) }) PIt("should fail if the object doesn't have meta", func() { @@ -1508,10 +1889,37 @@ var _ = Describe("Client", func() { PIt("should fail if the GVK cannot be mapped to a Resource", func() { }) + + // Test this with an integrated type and a CRD to make sure it covers both proto + // and json deserialization. + for idx, object := range []client.Object{&corev1.ConfigMap{}, &pkg.ChaosPod{}} { + idx, object := idx, object + It(fmt.Sprintf("should not retain any data in the obj variable that is not on the server for %T", object), func() { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + object.SetName(fmt.Sprintf("retain-test-%d", idx)) + object.SetNamespace(ns) + + By("First creating the object") + toCreate := object.DeepCopyObject().(client.Object) + Expect(cl.Create(ctx, toCreate)).NotTo(HaveOccurred()) + + By("Fetching it into a variable that has finalizers set") + toGetInto := object.DeepCopyObject().(client.Object) + toGetInto.SetFinalizers([]string{"some-finalizer"}) + Expect(cl.Get(ctx, client.ObjectKeyFromObject(object), toGetInto)).NotTo(HaveOccurred()) + + By("Ensuring the created and the received object are equal") + Expect(toCreate).Should(Equal(toGetInto)) + }) + } + }) Context("with unstructured objects", func() { - It("should fetch an existing object", func(done Done) { + It("should fetch an existing object", func() { By("first creating the Deployment") dep, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1538,11 +1946,9 @@ var _ = Describe("Client", func() { By("validating the fetched Deployment equals the created one") Expect(u).To(Equal(&actual)) - - close(done) }) - It("should fetch an existing non-namespace object", func(done Done) { + It("should fetch an existing non-namespace object", func() { By("first creating the Node") node, err := clientset.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1569,11 +1975,9 @@ var _ = Describe("Client", func() { By("validating the fetched Node equals the created one") Expect(u).To(Equal(&actual)) - - close(done) }) - It("should fail if the object does not exist", func(done Done) { + It("should fail if the object does not exist", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1583,12 +1987,34 @@ var _ = Describe("Client", func() { u := &unstructured.Unstructured{} err = cl.Get(context.TODO(), key, u) Expect(err).To(HaveOccurred()) + }) + + It("should not retain any data in the obj variable that is not on the server", func() { + object := &unstructured.Unstructured{} + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + object.SetName("retain-unstructured") + object.SetNamespace(ns) + object.SetAPIVersion("chaosapps.metamagical.io/v1") + object.SetKind("ChaosPod") - close(done) + By("First creating the object") + toCreate := object.DeepCopyObject().(client.Object) + Expect(cl.Create(ctx, toCreate)).NotTo(HaveOccurred()) + + By("Fetching it into a variable that has finalizers set") + toGetInto := object.DeepCopyObject().(client.Object) + toGetInto.SetFinalizers([]string{"some-finalizer"}) + Expect(cl.Get(ctx, client.ObjectKeyFromObject(object), toGetInto)).NotTo(HaveOccurred()) + + By("Ensuring the created and the received object are equal") + Expect(toCreate).Should(Equal(toGetInto)) }) }) Context("with metadata objects", func() { - It("should fetch an existing object for a go struct", func(done Done) { + It("should fetch an existing object for a go struct", func() { By("first creating the Deployment") dep, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1615,11 +2041,9 @@ var _ = Describe("Client", func() { By("validating the fetched deployment equals the created one") Expect(metaOnlyFromObj(dep, scheme)).To(Equal(&actual)) - - close(done) }) - It("should fetch an existing non-namespace object for a go struct", func(done Done) { + It("should fetch an existing non-namespace object for a go struct", func() { By("first creating the object") node, err := clientset.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1640,11 +2064,9 @@ var _ = Describe("Client", func() { Expect(actual).NotTo(BeNil()) Expect(metaOnlyFromObj(node, scheme)).To(Equal(&actual)) - - close(done) }) - It("should fail if the object does not exist", func(done Done) { + It("should fail if the object does not exist", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -1659,8 +2081,6 @@ var _ = Describe("Client", func() { }) err = cl.Get(context.TODO(), key, &actual) Expect(err).To(HaveOccurred()) - - close(done) }) PIt("should fail if the object doesn't have meta", func() { @@ -1670,12 +2090,33 @@ var _ = Describe("Client", func() { PIt("should fail if the GVK cannot be mapped to a Resource", func() { }) + + It("should not retain any data in the obj variable that is not on the server", func() { + cl, err := client.New(cfg, client.Options{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cl).NotTo(BeNil()) + + By("First creating the object") + toCreate := &pkg.ChaosPod{ObjectMeta: metav1.ObjectMeta{Name: "retain-metadata", Namespace: ns}} + Expect(cl.Create(ctx, toCreate)).NotTo(HaveOccurred()) + + By("Fetching it into a variable that has finalizers set") + toGetInto := &metav1.PartialObjectMetadata{ + TypeMeta: metav1.TypeMeta{APIVersion: "chaosapps.metamagical.io/v1", Kind: "ChaosPod"}, + ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "retain-metadata"}, + } + toGetInto.SetFinalizers([]string{"some-finalizer"}) + Expect(cl.Get(ctx, client.ObjectKeyFromObject(toGetInto), toGetInto)).NotTo(HaveOccurred()) + + By("Ensuring the created and the received objects metadata are equal") + Expect(toCreate.ObjectMeta).Should(Equal(toGetInto.ObjectMeta)) + }) }) }) Describe("List", func() { Context("with structured objects", func() { - It("should fetch collection of objects", func(done Done) { + It("should fetch collection of objects", func() { By("creating an initial object") dep, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1696,11 +2137,9 @@ var _ = Describe("Client", func() { } } Expect(hasDep).To(BeTrue()) + }) - close(done) - }, serverSideTimeoutSeconds) - - It("should fetch unstructured collection of objects", func(done Done) { + It("should fetch unstructured collection of objects", func() { By("create an initial object") _, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1732,10 +2171,9 @@ var _ = Describe("Client", func() { } } Expect(hasDep).To(BeTrue()) - close(done) - }, serverSideTimeoutSeconds) + }) - It("should fetch unstructured collection of objects, even if scheme is empty", func(done Done) { + It("should fetch unstructured collection of objects, even if scheme is empty", func() { By("create an initial object") _, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -1762,10 +2200,9 @@ var _ = Describe("Client", func() { } } Expect(hasDep).To(BeTrue()) - close(done) - }, serverSideTimeoutSeconds) + }) - It("should return an empty list if there are no matching objects", func(done Done) { + It("should return an empty list if there are no matching objects", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -1775,12 +2212,10 @@ var _ = Describe("Client", func() { By("validating no Deployments are returned") Expect(deps.Items).To(BeEmpty()) - - close(done) - }, serverSideTimeoutSeconds) + }) // TODO(seans): get label selector test working - It("should filter results by label selector", func(done Done) { + It("should filter results by label selector", func() { By("creating a Deployment with the app=frontend label") depFrontend := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -1838,11 +2273,9 @@ var _ = Describe("Client", func() { deleteDeployment(ctx, depFrontend, ns) deleteDeployment(ctx, depBackend, ns) + }) - close(done) - }, serverSideTimeoutSeconds) - - It("should filter results by namespace selector", func(done Done) { + It("should filter results by namespace selector", func() { By("creating a Deployment in test-namespace-1") tns1 := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-namespace-1"}} _, err := clientset.CoreV1().Namespaces().Create(ctx, tns1, metav1.CreateOptions{}) @@ -1899,11 +2332,9 @@ var _ = Describe("Client", func() { deleteDeployment(ctx, depBackend, "test-namespace-2") deleteNamespace(ctx, tns1) deleteNamespace(ctx, tns2) + }) - close(done) - }, serverSideTimeoutSeconds) - - It("should filter results by field selector", func(done Done) { + It("should filter results by field selector", func() { By("creating a Deployment with name deployment-frontend") depFrontend := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Name: "deployment-frontend", Namespace: ns}, @@ -1953,11 +2384,9 @@ var _ = Describe("Client", func() { deleteDeployment(ctx, depFrontend, ns) deleteDeployment(ctx, depBackend, ns) + }) - close(done) - }, serverSideTimeoutSeconds) - - It("should filter results by namespace selector and label selector", func(done Done) { + It("should filter results by namespace selector and label selector", func() { By("creating a Deployment in test-namespace-3 with the app=frontend label") tns3 := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-namespace-3"}} _, err := clientset.CoreV1().Namespaces().Create(ctx, tns3, metav1.CreateOptions{}) @@ -2048,9 +2477,7 @@ var _ = Describe("Client", func() { deleteDeployment(ctx, depFrontend4, "test-namespace-4") deleteNamespace(ctx, tns3) deleteNamespace(ctx, tns4) - - close(done) - }, serverSideTimeoutSeconds) + }) It("should filter results using limit and continue options", func() { @@ -2133,7 +2560,7 @@ var _ = Describe("Client", func() { Expect(deps.Continue).To(BeEmpty()) Expect(deps.Items[0].Name).To(Equal(dep3.Name)) Expect(deps.Items[1].Name).To(Equal(dep4.Name)) - }, serverSideTimeoutSeconds) + }) PIt("should fail if the object doesn't have meta", func() { @@ -2149,7 +2576,7 @@ var _ = Describe("Client", func() { }) Context("with unstructured objects", func() { - It("should fetch collection of objects", func(done Done) { + It("should fetch collection of objects", func() { By("create an initial object") _, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -2176,10 +2603,9 @@ var _ = Describe("Client", func() { } } Expect(hasDep).To(BeTrue()) - close(done) - }, serverSideTimeoutSeconds) + }) - It("should return an empty list if there are no matching objects", func(done Done) { + It("should return an empty list if there are no matching objects", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -2194,11 +2620,9 @@ var _ = Describe("Client", func() { By("validating no Deployments are returned") Expect(deps.Items).To(BeEmpty()) + }) - close(done) - }, serverSideTimeoutSeconds) - - It("should filter results by namespace selector", func(done Done) { + It("should filter results by namespace selector", func() { By("creating a Deployment in test-namespace-5") tns1 := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-namespace-5"}} _, err := clientset.CoreV1().Namespaces().Create(ctx, tns1, metav1.CreateOptions{}) @@ -2260,11 +2684,9 @@ var _ = Describe("Client", func() { deleteDeployment(ctx, depBackend, "test-namespace-6") deleteNamespace(ctx, tns1) deleteNamespace(ctx, tns2) + }) - close(done) - }, serverSideTimeoutSeconds) - - It("should filter results by field selector", func(done Done) { + It("should filter results by field selector", func() { By("creating a Deployment with name deployment-frontend") depFrontend := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Name: "deployment-frontend", Namespace: ns}, @@ -2319,11 +2741,9 @@ var _ = Describe("Client", func() { deleteDeployment(ctx, depFrontend, ns) deleteDeployment(ctx, depBackend, ns) + }) - close(done) - }, serverSideTimeoutSeconds) - - It("should filter results by namespace selector and label selector", func(done Done) { + It("should filter results by namespace selector and label selector", func() { By("creating a Deployment in test-namespace-7 with the app=frontend label") tns3 := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-namespace-7"}} _, err := clientset.CoreV1().Namespaces().Create(ctx, tns3, metav1.CreateOptions{}) @@ -2417,9 +2837,7 @@ var _ = Describe("Client", func() { deleteDeployment(ctx, depFrontend4, "test-namespace-8") deleteNamespace(ctx, tns3) deleteNamespace(ctx, tns4) - - close(done) - }, serverSideTimeoutSeconds) + }) PIt("should fail if the object doesn't have meta", func() { @@ -2430,7 +2848,7 @@ var _ = Describe("Client", func() { }) }) Context("with metadata objects", func() { - It("should fetch collection of objects", func(done Done) { + It("should fetch collection of objects", func() { By("creating an initial object") dep, err := clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -2467,11 +2885,9 @@ var _ = Describe("Client", func() { } } Expect(hasDep).To(BeTrue()) + }) - close(done) - }, serverSideTimeoutSeconds) - - It("should return an empty list if there are no matching objects", func(done Done) { + It("should return an empty list if there are no matching objects", func() { cl, err := client.New(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -2486,12 +2902,10 @@ var _ = Describe("Client", func() { By("validating no Deployments are returned") Expect(metaList.Items).To(BeEmpty()) - - close(done) - }, serverSideTimeoutSeconds) + }) // TODO(seans): get label selector test working - It("should filter results by label selector", func(done Done) { + It("should filter results by label selector", func() { By("creating a Deployment with the app=frontend label") depFrontend := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -2554,11 +2968,9 @@ var _ = Describe("Client", func() { deleteDeployment(ctx, depFrontend, ns) deleteDeployment(ctx, depBackend, ns) + }) - close(done) - }, serverSideTimeoutSeconds) - - It("should filter results by namespace selector", func(done Done) { + It("should filter results by namespace selector", func() { By("creating a Deployment in test-namespace-1") tns1 := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-namespace-1"}} _, err := clientset.CoreV1().Namespaces().Create(ctx, tns1, metav1.CreateOptions{}) @@ -2620,11 +3032,9 @@ var _ = Describe("Client", func() { deleteDeployment(ctx, depBackend, "test-namespace-2") deleteNamespace(ctx, tns1) deleteNamespace(ctx, tns2) + }) - close(done) - }, serverSideTimeoutSeconds) - - It("should filter results by field selector", func(done Done) { + It("should filter results by field selector", func() { By("creating a Deployment with name deployment-frontend") depFrontend := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Name: "deployment-frontend", Namespace: ns}, @@ -2679,11 +3089,9 @@ var _ = Describe("Client", func() { deleteDeployment(ctx, depFrontend, ns) deleteDeployment(ctx, depBackend, ns) + }) - close(done) - }, serverSideTimeoutSeconds) - - It("should filter results by namespace selector and label selector", func(done Done) { + It("should filter results by namespace selector and label selector", func() { By("creating a Deployment in test-namespace-3 with the app=frontend label") tns3 := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-namespace-3"}} _, err := clientset.CoreV1().Namespaces().Create(ctx, tns3, metav1.CreateOptions{}) @@ -2779,9 +3187,7 @@ var _ = Describe("Client", func() { deleteDeployment(ctx, depFrontend4, "test-namespace-4") deleteNamespace(ctx, tns3) deleteNamespace(ctx, tns4) - - close(done) - }, serverSideTimeoutSeconds) + }) It("should filter results using limit and continue options", func() { @@ -2879,7 +3285,7 @@ var _ = Describe("Client", func() { Expect(metaList.Continue).To(BeEmpty()) Expect(metaList.Items[0].Name).To(Equal(dep3.Name)) Expect(metaList.Items[1].Name).To(Equal(dep4.Name)) - }, serverSideTimeoutSeconds) + }) PIt("should fail if the object doesn't have meta", func() { @@ -2998,6 +3404,24 @@ var _ = Describe("Client", func() { }) }) + Describe("GetOptions", func() { + It("should be convertable to metav1.GetOptions", func() { + o := (&client.GetOptions{}).ApplyOptions([]client.GetOption{ + &client.GetOptions{Raw: &metav1.GetOptions{ResourceVersion: "RV0"}}, + }) + mo := o.AsGetOptions() + Expect(mo).NotTo(BeNil()) + Expect(mo.ResourceVersion).To(Equal("RV0")) + }) + + It("should produce empty metav1.GetOptions if nil", func() { + var o *client.GetOptions + Expect(o.AsGetOptions()).To(Equal(&metav1.GetOptions{})) + o = &client.GetOptions{} + Expect(o.AsGetOptions()).To(Equal(&metav1.GetOptions{})) + }) + }) + Describe("ListOptions", func() { It("should be convertable to metav1.ListOptions", func() { lo := (&client.ListOptions{}).ApplyOptions([]client.ListOption{ @@ -3149,20 +3573,19 @@ var _ = Describe("Client", func() { }) }) -var _ = Describe("DelegatingClient", func() { +var _ = Describe("ClientWithCache", func() { Describe("Get", func() { It("should call cache reader when structured object", func() { cachedReader := &fakeReader{} - cl, err := client.New(cfg, client.Options{}) - Expect(err).NotTo(HaveOccurred()) - dReader, err := client.NewDelegatingClient(client.NewDelegatingClientInput{ - CacheReader: cachedReader, - Client: cl, + cl, err := client.New(cfg, client.Options{ + Cache: &client.CacheOptions{ + Reader: cachedReader, + }, }) Expect(err).NotTo(HaveOccurred()) var actual appsv1.Deployment key := client.ObjectKey{Namespace: "ns", Name: "name"} - Expect(dReader.Get(context.TODO(), key, &actual)).To(Succeed()) + Expect(cl.Get(context.TODO(), key, &actual)).To(Succeed()) Expect(1).To(Equal(cachedReader.Called)) }) @@ -3198,11 +3621,10 @@ var _ = Describe("DelegatingClient", func() { }) It("should call client reader when not cached", func() { cachedReader := &fakeReader{} - cl, err := client.New(cfg, client.Options{}) - Expect(err).NotTo(HaveOccurred()) - dReader, err := client.NewDelegatingClient(client.NewDelegatingClientInput{ - CacheReader: cachedReader, - Client: cl, + cl, err := client.New(cfg, client.Options{ + Cache: &client.CacheOptions{ + Reader: cachedReader, + }, }) Expect(err).NotTo(HaveOccurred()) @@ -3214,17 +3636,16 @@ var _ = Describe("DelegatingClient", func() { }) actual.SetName(dep.Name) key := client.ObjectKey{Namespace: dep.Namespace, Name: dep.Name} - Expect(dReader.Get(context.TODO(), key, actual)).To(Succeed()) + Expect(cl.Get(context.TODO(), key, actual)).To(Succeed()) Expect(0).To(Equal(cachedReader.Called)) }) It("should call cache reader when cached", func() { cachedReader := &fakeReader{} - cl, err := client.New(cfg, client.Options{}) - Expect(err).NotTo(HaveOccurred()) - dReader, err := client.NewDelegatingClient(client.NewDelegatingClientInput{ - CacheReader: cachedReader, - Client: cl, - CacheUnstructured: true, + cl, err := client.New(cfg, client.Options{ + Cache: &client.CacheOptions{ + Reader: cachedReader, + Unstructured: true, + }, }) Expect(err).NotTo(HaveOccurred()) @@ -3236,7 +3657,7 @@ var _ = Describe("DelegatingClient", func() { }) actual.SetName(dep.Name) key := client.ObjectKey{Namespace: dep.Namespace, Name: dep.Name} - Expect(dReader.Get(context.TODO(), key, actual)).To(Succeed()) + Expect(cl.Get(context.TODO(), key, actual)).To(Succeed()) Expect(1).To(Equal(cachedReader.Called)) }) }) @@ -3244,26 +3665,24 @@ var _ = Describe("DelegatingClient", func() { Describe("List", func() { It("should call cache reader when structured object", func() { cachedReader := &fakeReader{} - cl, err := client.New(cfg, client.Options{}) - Expect(err).NotTo(HaveOccurred()) - dReader, err := client.NewDelegatingClient(client.NewDelegatingClientInput{ - CacheReader: cachedReader, - Client: cl, + cl, err := client.New(cfg, client.Options{ + Cache: &client.CacheOptions{ + Reader: cachedReader, + }, }) Expect(err).NotTo(HaveOccurred()) var actual appsv1.DeploymentList - Expect(dReader.List(context.Background(), &actual)).To(Succeed()) + Expect(cl.List(context.Background(), &actual)).To(Succeed()) Expect(1).To(Equal(cachedReader.Called)) }) When("listing unstructured objects", func() { It("should call client reader when not cached", func() { cachedReader := &fakeReader{} - cl, err := client.New(cfg, client.Options{}) - Expect(err).NotTo(HaveOccurred()) - dReader, err := client.NewDelegatingClient(client.NewDelegatingClientInput{ - CacheReader: cachedReader, - Client: cl, + cl, err := client.New(cfg, client.Options{ + Cache: &client.CacheOptions{ + Reader: cachedReader, + }, }) Expect(err).NotTo(HaveOccurred()) @@ -3273,17 +3692,16 @@ var _ = Describe("DelegatingClient", func() { Kind: "DeploymentList", Version: "v1", }) - Expect(dReader.List(context.Background(), actual)).To(Succeed()) + Expect(cl.List(context.Background(), actual)).To(Succeed()) Expect(0).To(Equal(cachedReader.Called)) }) It("should call cache reader when cached", func() { cachedReader := &fakeReader{} - cl, err := client.New(cfg, client.Options{}) - Expect(err).NotTo(HaveOccurred()) - dReader, err := client.NewDelegatingClient(client.NewDelegatingClientInput{ - CacheReader: cachedReader, - Client: cl, - CacheUnstructured: true, + cl, err := client.New(cfg, client.Options{ + Cache: &client.CacheOptions{ + Reader: cachedReader, + Unstructured: true, + }, }) Expect(err).NotTo(HaveOccurred()) @@ -3293,7 +3711,7 @@ var _ = Describe("DelegatingClient", func() { Kind: "DeploymentList", Version: "v1", }) - Expect(dReader.List(context.Background(), actual)).To(Succeed()) + Expect(cl.List(context.Background(), actual)).To(Succeed()) Expect(1).To(Equal(cachedReader.Called)) }) }) @@ -3470,11 +3888,37 @@ var _ = Describe("IgnoreNotFound", func() { }) }) +var _ = Describe("IgnoreAlreadyExists", func() { + It("should return nil on a 'AlreadyExists' error", func() { + By("creating a AlreadyExists error") + err := apierrors.NewAlreadyExists(schema.GroupResource{}, "") + + By("returning no error") + Expect(client.IgnoreAlreadyExists(err)).To(Succeed()) + }) + + It("should return the error on a status other than already exists", func() { + By("creating a BadRequest error") + err := apierrors.NewBadRequest("") + + By("returning an error") + Expect(client.IgnoreAlreadyExists(err)).To(HaveOccurred()) + }) + + It("should return the error on a non-status error", func() { + By("creating an fmt error") + err := fmt.Errorf("arbitrary error") + + By("returning an error") + Expect(client.IgnoreAlreadyExists(err)).To(HaveOccurred()) + }) +}) + type fakeReader struct { Called int } -func (f *fakeReader) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error { +func (f *fakeReader) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { f.Called++ return nil } @@ -3483,3 +3927,16 @@ func (f *fakeReader) List(ctx context.Context, list client.ObjectList, opts ...c f.Called++ return nil } + +func ptr[T any](to T) *T { + return &to +} + +func toUnstructured(o client.Object) (*unstructured.Unstructured, error) { + serialized, err := json.Marshal(o) + if err != nil { + return nil, err + } + u := &unstructured.Unstructured{} + return u, json.Unmarshal(serialized, u) +} diff --git a/pkg/client/config/config.go b/pkg/client/config/config.go index 235a7e450b..5f0a6d4b1d 100644 --- a/pkg/client/config/config.go +++ b/pkg/client/config/config.go @@ -21,7 +21,7 @@ import ( "fmt" "os" "os/user" - "path" + "path/filepath" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -29,15 +29,32 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/internal/log" ) +// KubeconfigFlagName is the name of the kubeconfig flag +const KubeconfigFlagName = "kubeconfig" + var ( kubeconfig string log = logf.RuntimeLog.WithName("client").WithName("config") ) +// init registers the "kubeconfig" flag to the default command line FlagSet. +// TODO: This should be removed, as it potentially leads to redefined flag errors for users, if they already +// have registered the "kubeconfig" flag to the command line FlagSet in other parts of their code. func init() { - // TODO: Fix this to allow double vendoring this library but still register flags on behalf of users - flag.StringVar(&kubeconfig, "kubeconfig", "", - "Paths to a kubeconfig. Only required if out-of-cluster.") + RegisterFlags(flag.CommandLine) +} + +// RegisterFlags registers flag variables to the given FlagSet if not already registered. +// It uses the default command line FlagSet, if none is provided. Currently, it only registers the kubeconfig flag. +func RegisterFlags(fs *flag.FlagSet) { + if fs == nil { + fs = flag.CommandLine + } + if f := fs.Lookup(KubeconfigFlagName); f != nil { + kubeconfig = f.Value.String() + } else { + fs.StringVar(&kubeconfig, KubeconfigFlagName, "", "Paths to a kubeconfig. Only required if out-of-cluster.") + } } // GetConfig creates a *rest.Config for talking to a Kubernetes API server. @@ -47,7 +64,7 @@ func init() { // It also applies saner defaults for QPS and burst based on the Kubernetes // controller manager defaults (20 QPS, 30 burst) // -// Config precedence +// Config precedence: // // * --kubeconfig flag pointing at a file // @@ -67,7 +84,7 @@ func GetConfig() (*rest.Config, error) { // It also applies saner defaults for QPS and burst based on the Kubernetes // controller manager defaults (20 QPS, 30 burst) // -// Config precedence +// Config precedence: // // * --kubeconfig flag pointing at a file // @@ -81,12 +98,12 @@ func GetConfigWithContext(context string) (*rest.Config, error) { if err != nil { return nil, err } - if cfg.QPS == 0.0 { cfg.QPS = 20.0 - cfg.Burst = 30.0 } - + if cfg.Burst == 0 { + cfg.Burst = 30 + } return cfg, nil } @@ -96,7 +113,7 @@ func GetConfigWithContext(context string) (*rest.Config, error) { var loadInClusterConfig = rest.InClusterConfig // loadConfig loads a REST Config as per the rules specified in GetConfig. -func loadConfig(context string) (*rest.Config, error) { +func loadConfig(context string) (config *rest.Config, configErr error) { // If a flag is specified with the config location, use that if len(kubeconfig) > 0 { return loadConfigWithContext("", &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}, context) @@ -106,9 +123,16 @@ func loadConfig(context string) (*rest.Config, error) { // try the in-cluster config. kubeconfigPath := os.Getenv(clientcmd.RecommendedConfigPathEnvVar) if len(kubeconfigPath) == 0 { - if c, err := loadInClusterConfig(); err == nil { + c, err := loadInClusterConfig() + if err == nil { return c, nil } + + defer func() { + if configErr != nil { + log.Error(err, "unable to load in-cluster config") + } + }() } // If the recommended kubeconfig env variable is set, or there @@ -123,9 +147,9 @@ func loadConfig(context string) (*rest.Config, error) { if _, ok := os.LookupEnv("HOME"); !ok { u, err := user.Current() if err != nil { - return nil, fmt.Errorf("could not get current user: %v", err) + return nil, fmt.Errorf("could not get current user: %w", err) } - loadingRules.Precedence = append(loadingRules.Precedence, path.Join(u.HomeDir, clientcmd.RecommendedHomeDir, clientcmd.RecommendedFileName)) + loadingRules.Precedence = append(loadingRules.Precedence, filepath.Join(u.HomeDir, clientcmd.RecommendedHomeDir, clientcmd.RecommendedFileName)) } return loadConfigWithContext("", loadingRules, context) diff --git a/pkg/client/config/config_suite_test.go b/pkg/client/config/config_suite_test.go index b69c2e1e30..626613cef4 100644 --- a/pkg/client/config/config_suite_test.go +++ b/pkg/client/config/config_suite_test.go @@ -19,22 +19,18 @@ package config import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestConfig(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Client Config Test Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Client Config Test Suite") } -var _ = BeforeSuite(func(done Done) { +var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) - - close(done) -}, 60) +}) diff --git a/pkg/client/config/config_test.go b/pkg/client/config/config_test.go index ed9761e50f..058ff33c1f 100644 --- a/pkg/client/config/config_test.go +++ b/pkg/client/config/config_test.go @@ -17,12 +17,11 @@ limitations under the License. package config import ( - "io/ioutil" "os" "path/filepath" "strings" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -45,7 +44,7 @@ var _ = Describe("Config", func() { BeforeEach(func() { // create temporary directory for test case var err error - dir, err = ioutil.TempDir("", "cr-test") + dir, err = os.MkdirTemp("", "cr-test") Expect(err).NotTo(HaveOccurred()) // override $HOME/.kube/config @@ -192,7 +191,7 @@ func setConfigs(tc testCase, dir string) { func createFiles(files map[string]string, dir string) error { for path, data := range files { - if err := ioutil.WriteFile(filepath.Join(dir, path), []byte(data), 0644); err != nil { //nolint:gosec + if err := os.WriteFile(filepath.Join(dir, path), []byte(data), 0644); err != nil { //nolint:gosec return err } } diff --git a/pkg/client/doc.go b/pkg/client/doc.go index 2965e5fa94..b2e2024942 100644 --- a/pkg/client/doc.go +++ b/pkg/client/doc.go @@ -17,7 +17,7 @@ limitations under the License. // Package client contains functionality for interacting with Kubernetes API // servers. // -// Clients +// # Clients // // Clients are split into two interfaces -- Readers and Writers. Readers // get and list, while writers create, update, and delete. @@ -25,18 +25,18 @@ limitations under the License. // The New function can be used to create a new client that talks directly // to the API server. // -// A common pattern in Kubernetes to read from a cache and write to the API -// server. This pattern is covered by the DelegatingClient type, which can -// be used to have a client whose Reader is different from the Writer. +// It is a common pattern in Kubernetes to read from a cache and write to the API +// server. This pattern is covered by the creating the Client with a Cache. // -// Options +// # Options // // Many client operations in Kubernetes support options. These options are // represented as variadic arguments at the end of a given method call. // For instance, to use a label selector on list, you can call -// err := someReader.List(context.Background(), &podList, client.MatchingLabels{"somelabel": "someval"}) // -// Indexing +// err := someReader.List(context.Background(), &podList, client.MatchingLabels{"somelabel": "someval"}) +// +// # Indexing // // Indexes may be added to caches using a FieldIndexer. This allows you to easily // and efficiently look up objects with certain properties. You can then make diff --git a/pkg/client/dryrun.go b/pkg/client/dryrun.go index ea25ea2530..bbcdd38321 100644 --- a/pkg/client/dryrun.go +++ b/pkg/client/dryrun.go @@ -21,6 +21,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" ) // NewDryRunClient wraps an existing client and enforces DryRun mode @@ -46,6 +47,16 @@ func (c *dryRunClient) RESTMapper() meta.RESTMapper { return c.client.RESTMapper() } +// GroupVersionKindFor returns the GroupVersionKind for the given object. +func (c *dryRunClient) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + return c.client.GroupVersionKindFor(obj) +} + +// IsObjectNamespaced returns true if the GroupVersionKind of the object is namespaced. +func (c *dryRunClient) IsObjectNamespaced(obj runtime.Object) (bool, error) { + return c.client.IsObjectNamespaced(obj) +} + // Create implements client.Client. func (c *dryRunClient) Create(ctx context.Context, obj Object, opts ...CreateOption) error { return c.client.Create(ctx, obj, append(opts, DryRunAll)...) @@ -72,8 +83,8 @@ func (c *dryRunClient) Patch(ctx context.Context, obj Object, patch Patch, opts } // Get implements client.Client. -func (c *dryRunClient) Get(ctx context.Context, key ObjectKey, obj Object) error { - return c.client.Get(ctx, key, obj) +func (c *dryRunClient) Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error { + return c.client.Get(ctx, key, obj, opts...) } // List implements client.Client. @@ -82,25 +93,38 @@ func (c *dryRunClient) List(ctx context.Context, obj ObjectList, opts ...ListOpt } // Status implements client.StatusClient. -func (c *dryRunClient) Status() StatusWriter { - return &dryRunStatusWriter{client: c.client.Status()} +func (c *dryRunClient) Status() SubResourceWriter { + return c.SubResource("status") +} + +// SubResource implements client.SubResourceClient. +func (c *dryRunClient) SubResource(subResource string) SubResourceClient { + return &dryRunSubResourceClient{client: c.client.SubResource(subResource)} } -// ensure dryRunStatusWriter implements client.StatusWriter. -var _ StatusWriter = &dryRunStatusWriter{} +// ensure dryRunSubResourceWriter implements client.SubResourceWriter. +var _ SubResourceWriter = &dryRunSubResourceClient{} -// dryRunStatusWriter is client.StatusWriter that writes status subresource with dryRun mode +// dryRunSubResourceClient is client.SubResourceWriter that writes status subresource with dryRun mode // enforced. -type dryRunStatusWriter struct { - client StatusWriter +type dryRunSubResourceClient struct { + client SubResourceClient +} + +func (sw *dryRunSubResourceClient) Get(ctx context.Context, obj, subResource Object, opts ...SubResourceGetOption) error { + return sw.client.Get(ctx, obj, subResource, opts...) +} + +func (sw *dryRunSubResourceClient) Create(ctx context.Context, obj, subResource Object, opts ...SubResourceCreateOption) error { + return sw.client.Create(ctx, obj, subResource, append(opts, DryRunAll)...) } -// Update implements client.StatusWriter. -func (sw *dryRunStatusWriter) Update(ctx context.Context, obj Object, opts ...UpdateOption) error { +// Update implements client.SubResourceWriter. +func (sw *dryRunSubResourceClient) Update(ctx context.Context, obj Object, opts ...SubResourceUpdateOption) error { return sw.client.Update(ctx, obj, append(opts, DryRunAll)...) } -// Patch implements client.StatusWriter. -func (sw *dryRunStatusWriter) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { +// Patch implements client.SubResourceWriter. +func (sw *dryRunSubResourceClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...SubResourcePatchOption) error { return sw.client.Patch(ctx, obj, patch, append(opts, DryRunAll)...) } diff --git a/pkg/client/dryrun_test.go b/pkg/client/dryrun_test.go index 0a46e5617d..72907fefab 100644 --- a/pkg/client/dryrun_test.go +++ b/pkg/client/dryrun_test.go @@ -21,13 +21,14 @@ import ( "fmt" "sync/atomic" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -40,10 +41,10 @@ var _ = Describe("DryRunClient", func() { ctx := context.Background() getClient := func() client.Client { - nonDryRunClient, err := client.New(cfg, client.Options{}) + cl, err := client.New(cfg, client.Options{DryRun: pointer.Bool(true)}) Expect(err).NotTo(HaveOccurred()) - Expect(nonDryRunClient).NotTo(BeNil()) - return client.NewDryRunClient(nonDryRunClient) + Expect(cl).NotTo(BeNil()) + return cl } BeforeEach(func() { @@ -226,7 +227,7 @@ var _ = Describe("DryRunClient", func() { It("should not change objects via update status with opts", func() { changedDep := dep.DeepCopy() changedDep.Status.Replicas = 99 - opts := &client.UpdateOptions{DryRun: []string{"Bye", "Pippa"}} + opts := &client.SubResourceUpdateOptions{UpdateOptions: client.UpdateOptions{DryRun: []string{"Bye", "Pippa"}}} Expect(getClient().Status().Update(ctx, changedDep, opts)).NotTo(HaveOccurred()) @@ -252,7 +253,7 @@ var _ = Describe("DryRunClient", func() { changedDep := dep.DeepCopy() changedDep.Status.Replicas = 99 - opts := &client.PatchOptions{DryRun: []string{"Bye", "Pippa"}} + opts := &client.SubResourcePatchOptions{PatchOptions: client.PatchOptions{DryRun: []string{"Bye", "Pippa"}}} Expect(getClient().Status().Patch(ctx, changedDep, client.MergeFrom(dep), opts)).ToNot(HaveOccurred()) diff --git a/pkg/client/example_test.go b/pkg/client/example_test.go index 1be098bbbb..c69caabcd2 100644 --- a/pkg/client/example_test.go +++ b/pkg/client/example_test.go @@ -53,7 +53,7 @@ func ExampleNew() { } } -// This example shows how to use the client with typed and unstructured objects to retrieve a objects. +// This example shows how to use the client with typed and unstructured objects to retrieve an object. func ExampleClient_get() { // Using a typed object. pod := &corev1.Pod{} diff --git a/pkg/client/fake/client.go b/pkg/client/fake/client.go index ded8a60d33..b6a1f07886 100644 --- a/pkg/client/fake/client.go +++ b/pkg/client/fake/client.go @@ -21,6 +21,7 @@ import ( "encoding/json" "errors" "fmt" + "reflect" "strconv" "strings" "sync" @@ -29,6 +30,8 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" utilrand "k8s.io/apimachinery/pkg/util/rand" @@ -36,6 +39,7 @@ import ( "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/testing" + "sigs.k8s.io/controller-runtime/pkg/internal/field/selector" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" @@ -48,8 +52,14 @@ type versionedTracker struct { } type fakeClient struct { - tracker versionedTracker - scheme *runtime.Scheme + tracker versionedTracker + scheme *runtime.Scheme + restMapper meta.RESTMapper + + // indexes maps each GroupVersionKind (GVK) to the indexes registered for that GVK. + // The inner map maps from index name to IndexerFunc. + indexes map[schema.GroupVersionKind]map[string]client.IndexerFunc + schemeWriteLock sync.Mutex } @@ -86,9 +96,15 @@ func NewClientBuilder() *ClientBuilder { // ClientBuilder builds a fake client. type ClientBuilder struct { scheme *runtime.Scheme + restMapper meta.RESTMapper initObject []client.Object initLists []client.ObjectList initRuntimeObjects []runtime.Object + objectTracker testing.ObjectTracker + + // indexes maps each GroupVersionKind (GVK) to the indexes registered for that GVK. + // The inner map maps from index name to IndexerFunc. + indexes map[schema.GroupVersionKind]map[string]client.IndexerFunc } // WithScheme sets this builder's internal scheme. @@ -98,6 +114,15 @@ func (f *ClientBuilder) WithScheme(scheme *runtime.Scheme) *ClientBuilder { return f } +// WithRESTMapper sets this builder's restMapper. +// The restMapper is directly set as mapper in the Client. This can be used for example +// with a meta.DefaultRESTMapper to provide a static rest mapping. +// If not set, defaults to an empty meta.DefaultRESTMapper. +func (f *ClientBuilder) WithRESTMapper(restMapper meta.RESTMapper) *ClientBuilder { + f.restMapper = restMapper + return f +} + // WithObjects can be optionally used to initialize this fake client with client.Object(s). func (f *ClientBuilder) WithObjects(initObjs ...client.Object) *ClientBuilder { f.initObject = append(f.initObject, initObjs...) @@ -116,13 +141,67 @@ func (f *ClientBuilder) WithRuntimeObjects(initRuntimeObjs ...runtime.Object) *C return f } +// WithObjectTracker can be optionally used to initialize this fake client with testing.ObjectTracker. +func (f *ClientBuilder) WithObjectTracker(ot testing.ObjectTracker) *ClientBuilder { + f.objectTracker = ot + return f +} + +// WithIndex can be optionally used to register an index with name `field` and indexer `extractValue` +// for API objects of the same GroupVersionKind (GVK) as `obj` in the fake client. +// It can be invoked multiple times, both with objects of the same GVK or different ones. +// Invoking WithIndex twice with the same `field` and GVK (via `obj`) arguments will panic. +// WithIndex retrieves the GVK of `obj` using the scheme registered via WithScheme if +// WithScheme was previously invoked, the default scheme otherwise. +func (f *ClientBuilder) WithIndex(obj runtime.Object, field string, extractValue client.IndexerFunc) *ClientBuilder { + objScheme := f.scheme + if objScheme == nil { + objScheme = scheme.Scheme + } + + gvk, err := apiutil.GVKForObject(obj, objScheme) + if err != nil { + panic(err) + } + + // If this is the first index being registered, we initialize the map storing all the indexes. + if f.indexes == nil { + f.indexes = make(map[schema.GroupVersionKind]map[string]client.IndexerFunc) + } + + // If this is the first index being registered for the GroupVersionKind of `obj`, we initialize + // the map storing the indexes for that GroupVersionKind. + if f.indexes[gvk] == nil { + f.indexes[gvk] = make(map[string]client.IndexerFunc) + } + + if _, fieldAlreadyIndexed := f.indexes[gvk][field]; fieldAlreadyIndexed { + panic(fmt.Errorf("indexer conflict: field %s for GroupVersionKind %v is already indexed", + field, gvk)) + } + + f.indexes[gvk][field] = extractValue + + return f +} + // Build builds and returns a new fake client. func (f *ClientBuilder) Build() client.WithWatch { if f.scheme == nil { f.scheme = scheme.Scheme } + if f.restMapper == nil { + f.restMapper = meta.NewDefaultRESTMapper([]schema.GroupVersion{}) + } + + var tracker versionedTracker + + if f.objectTracker == nil { + tracker = versionedTracker{ObjectTracker: testing.NewObjectTracker(f.scheme, scheme.Codecs.UniversalDecoder()), scheme: f.scheme} + } else { + tracker = versionedTracker{ObjectTracker: f.objectTracker, scheme: f.scheme} + } - tracker := versionedTracker{ObjectTracker: testing.NewObjectTracker(f.scheme, scheme.Codecs.UniversalDecoder()), scheme: f.scheme} for _, obj := range f.initObject { if err := tracker.Add(obj); err != nil { panic(fmt.Errorf("failed to add object %v to fake client: %w", obj, err)) @@ -139,8 +218,10 @@ func (f *ClientBuilder) Build() client.WithWatch { } } return &fakeClient{ - tracker: tracker, - scheme: f.scheme, + tracker: tracker, + scheme: f.scheme, + restMapper: f.restMapper, + indexes: f.indexes, } } @@ -169,6 +250,11 @@ func (t versionedTracker) Add(obj runtime.Object) error { // be recognized accessor.SetResourceVersion(trackerAddResourceVersion) } + + obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj) + if err != nil { + return err + } if err := t.ObjectTracker.Add(obj); err != nil { return err } @@ -180,7 +266,7 @@ func (t versionedTracker) Add(obj runtime.Object) error { func (t versionedTracker) Create(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error { accessor, err := meta.Accessor(obj) if err != nil { - return fmt.Errorf("failed to get accessor for object: %v", err) + return fmt.Errorf("failed to get accessor for object: %w", err) } if accessor.GetName() == "" { return apierrors.NewInvalid( @@ -192,17 +278,51 @@ func (t versionedTracker) Create(gvr schema.GroupVersionResource, obj runtime.Ob return apierrors.NewBadRequest("resourceVersion can not be set for Create requests") } accessor.SetResourceVersion("1") + obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj) + if err != nil { + return err + } if err := t.ObjectTracker.Create(gvr, obj, ns); err != nil { accessor.SetResourceVersion("") return err } + return nil } +// convertFromUnstructuredIfNecessary will convert runtime.Unstructured for a GVK that is recognized +// by the schema into the whatever the schema produces with New() for said GVK. +// This is required because the tracker unconditionally saves on manipulations, but its List() implementation +// tries to assign whatever it finds into a ListType it gets from schema.New() - Thus we have to ensure +// we save as the very same type, otherwise subsequent List requests will fail. +func convertFromUnstructuredIfNecessary(s *runtime.Scheme, o runtime.Object) (runtime.Object, error) { + gvk := o.GetObjectKind().GroupVersionKind() + + u, isUnstructured := o.(runtime.Unstructured) + if !isUnstructured || !s.Recognizes(gvk) { + return o, nil + } + + typed, err := s.New(gvk) + if err != nil { + return nil, fmt.Errorf("scheme recognizes %s but failed to produce an object for it: %w", gvk, err) + } + + unstructuredSerialized, err := json.Marshal(u) + if err != nil { + return nil, fmt.Errorf("failed to serialize %T: %w", unstructuredSerialized, err) + } + if err := json.Unmarshal(unstructuredSerialized, typed); err != nil { + return nil, fmt.Errorf("failed to unmarshal the content of %T into %T: %w", u, typed, err) + } + + return typed, nil +} + func (t versionedTracker) Update(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error { accessor, err := meta.Accessor(obj) if err != nil { - return fmt.Errorf("failed to get accessor for object: %v", err) + return fmt.Errorf("failed to get accessor for object: %w", err) } if accessor.GetName() == "" { @@ -248,17 +368,21 @@ func (t versionedTracker) Update(gvr schema.GroupVersionResource, obj runtime.Ob } intResourceVersion, err := strconv.ParseUint(oldAccessor.GetResourceVersion(), 10, 64) if err != nil { - return fmt.Errorf("can not convert resourceVersion %q to int: %v", oldAccessor.GetResourceVersion(), err) + return fmt.Errorf("can not convert resourceVersion %q to int: %w", oldAccessor.GetResourceVersion(), err) } intResourceVersion++ accessor.SetResourceVersion(strconv.FormatUint(intResourceVersion, 10)) if !accessor.GetDeletionTimestamp().IsZero() && len(accessor.GetFinalizers()) == 0 { return t.ObjectTracker.Delete(gvr, accessor.GetNamespace(), accessor.GetName()) } + obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj) + if err != nil { + return err + } return t.ObjectTracker.Update(gvr, obj, ns) } -func (c *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error { +func (c *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { gvr, err := getGVRFromObject(obj, c.scheme) if err != nil { return err @@ -284,6 +408,7 @@ func (c *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.O return err } decoder := scheme.Codecs.UniversalDecoder() + zero(obj) _, _, err = decoder.Decode(j, nil, obj) return err } @@ -294,9 +419,7 @@ func (c *fakeClient) Watch(ctx context.Context, list client.ObjectList, opts ... return nil, err } - if strings.HasSuffix(gvk.Kind, "List") { - gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] - } + gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") listOpts := client.ListOptions{} listOpts.ApplyOptions(opts) @@ -313,12 +436,10 @@ func (c *fakeClient) List(ctx context.Context, obj client.ObjectList, opts ...cl originalKind := gvk.Kind - if strings.HasSuffix(gvk.Kind, "List") { - gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] - } + gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") - if _, isUnstructuredList := obj.(*unstructured.UnstructuredList); isUnstructuredList && !c.scheme.Recognizes(gvk) { - // We need tor register the ListKind with UnstructuredList: + if _, isUnstructuredList := obj.(runtime.Unstructured); isUnstructuredList && !c.scheme.Recognizes(gvk) { + // We need to register the ListKind with UnstructuredList: // https://github.com/kubernetes/kubernetes/blob/7b2776b89fb1be28d4e9203bdeec079be903c103/staging/src/k8s.io/client-go/dynamic/fake/simple.go#L44-L51 c.schemeWriteLock.Lock() c.scheme.AddKnownTypeWithName(gvk.GroupVersion().WithKind(gvk.Kind+"List"), &unstructured.UnstructuredList{}) @@ -346,26 +467,94 @@ func (c *fakeClient) List(ctx context.Context, obj client.ObjectList, opts ...cl return err } decoder := scheme.Codecs.UniversalDecoder() + zero(obj) _, _, err = decoder.Decode(j, nil, obj) if err != nil { return err } - if listOpts.LabelSelector != nil { - objs, err := meta.ExtractList(obj) + if listOpts.LabelSelector == nil && listOpts.FieldSelector == nil { + return nil + } + + // If we're here, either a label or field selector are specified (or both), so before we return + // the list we must filter it. If both selectors are set, they are ANDed. + objs, err := meta.ExtractList(obj) + if err != nil { + return err + } + + filteredList, err := c.filterList(objs, gvk, listOpts.LabelSelector, listOpts.FieldSelector) + if err != nil { + return err + } + + return meta.SetList(obj, filteredList) +} + +func (c *fakeClient) filterList(list []runtime.Object, gvk schema.GroupVersionKind, ls labels.Selector, fs fields.Selector) ([]runtime.Object, error) { + // Filter the objects with the label selector + filteredList := list + if ls != nil { + objsFilteredByLabel, err := objectutil.FilterWithLabels(list, ls) if err != nil { - return err + return nil, err } - filteredObjs, err := objectutil.FilterWithLabels(objs, listOpts.LabelSelector) + filteredList = objsFilteredByLabel + } + + // Filter the result of the previous pass with the field selector + if fs != nil { + objsFilteredByField, err := c.filterWithFields(filteredList, gvk, fs) if err != nil { - return err + return nil, err } - err = meta.SetList(obj, filteredObjs) - if err != nil { - return err + filteredList = objsFilteredByField + } + + return filteredList, nil +} + +func (c *fakeClient) filterWithFields(list []runtime.Object, gvk schema.GroupVersionKind, fs fields.Selector) ([]runtime.Object, error) { + // We only allow filtering on the basis of a single field to ensure consistency with the + // behavior of the cache reader (which we're faking here). + fieldKey, fieldVal, requiresExact := selector.RequiresExactMatch(fs) + if !requiresExact { + return nil, fmt.Errorf("field selector %s is not in one of the two supported forms \"key==val\" or \"key=val\"", + fs) + } + + // Field selection is mimicked via indexes, so there's no sane answer this function can give + // if there are no indexes registered for the GroupVersionKind of the objects in the list. + indexes := c.indexes[gvk] + if len(indexes) == 0 || indexes[fieldKey] == nil { + return nil, fmt.Errorf("List on GroupVersionKind %v specifies selector on field %s, but no "+ + "index with name %s has been registered for GroupVersionKind %v", gvk, fieldKey, fieldKey, gvk) + } + + indexExtractor := indexes[fieldKey] + filteredList := make([]runtime.Object, 0, len(list)) + for _, obj := range list { + if c.objMatchesFieldSelector(obj, indexExtractor, fieldVal) { + filteredList = append(filteredList, obj) } } - return nil + return filteredList, nil +} + +func (c *fakeClient) objMatchesFieldSelector(o runtime.Object, extractIndex client.IndexerFunc, val string) bool { + obj, isClientObject := o.(client.Object) + if !isClientObject { + panic(fmt.Errorf("expected object %v to be of type client.Object, but it's not", o)) + } + + for _, extractedVal := range extractIndex(obj) { + if extractedVal == val { + return true + } + } + + return false } func (c *fakeClient) Scheme() *runtime.Scheme { @@ -373,8 +562,17 @@ func (c *fakeClient) Scheme() *runtime.Scheme { } func (c *fakeClient) RESTMapper() meta.RESTMapper { - // TODO: Implement a fake RESTMapper. - return nil + return c.restMapper +} + +// GroupVersionKindFor returns the GroupVersionKind for the given object. +func (c *fakeClient) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + return apiutil.GVKForObject(obj, c.scheme) +} + +// IsObjectNamespaced returns true if the GroupVersionKind of the object is namespaced. +func (c *fakeClient) IsObjectNamespaced(obj runtime.Object) (bool, error) { + return apiutil.IsObjectNamespaced(obj, c.scheme, c.restMapper) } func (c *fakeClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { @@ -419,6 +617,34 @@ func (c *fakeClient) Delete(ctx context.Context, obj client.Object, opts ...clie delOptions := client.DeleteOptions{} delOptions.ApplyOptions(opts) + for _, dryRunOpt := range delOptions.DryRun { + if dryRunOpt == metav1.DryRunAll { + return nil + } + } + + // Check the ResourceVersion if that Precondition was specified. + if delOptions.Preconditions != nil && delOptions.Preconditions.ResourceVersion != nil { + name := accessor.GetName() + dbObj, err := c.tracker.Get(gvr, accessor.GetNamespace(), name) + if err != nil { + return err + } + oldAccessor, err := meta.Accessor(dbObj) + if err != nil { + return err + } + actualRV := oldAccessor.GetResourceVersion() + expectRV := *delOptions.Preconditions.ResourceVersion + if actualRV != expectRV { + msg := fmt.Sprintf( + "the ResourceVersion in the precondition (%s) does not match the ResourceVersion in record (%s). "+ + "The object might have been modified", + expectRV, actualRV) + return apierrors.NewConflict(gvr.GroupResource(), name, errors.New(msg)) + } + } + return c.deleteObject(gvr, accessor) } @@ -431,6 +657,12 @@ func (c *fakeClient) DeleteAllOf(ctx context.Context, obj client.Object, opts .. dcOptions := client.DeleteAllOfOptions{} dcOptions.ApplyOptions(opts) + for _, dryRunOpt := range dcOptions.DryRun { + if dryRunOpt == metav1.DryRunAll { + return nil + } + } + gvr, _ := meta.UnsafeGuessKindToResource(gvk) o, err := c.tracker.List(gvr, gvk, dcOptions.Namespace) if err != nil { @@ -527,12 +759,17 @@ func (c *fakeClient) Patch(ctx context.Context, obj client.Object, patch client. return err } decoder := scheme.Codecs.UniversalDecoder() + zero(obj) _, _, err = decoder.Decode(j, nil, obj) return err } -func (c *fakeClient) Status() client.StatusWriter { - return &fakeStatusWriter{client: c} +func (c *fakeClient) Status() client.SubResourceWriter { + return c.SubResource("status") +} + +func (c *fakeClient) SubResource(subResource string) client.SubResourceClient { + return &fakeSubResourceClient{client: c} } func (c *fakeClient) deleteObject(gvr schema.GroupVersionResource, accessor metav1.Object) error { @@ -561,20 +798,44 @@ func getGVRFromObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupV return gvr, nil } -type fakeStatusWriter struct { +type fakeSubResourceClient struct { client *fakeClient } -func (sw *fakeStatusWriter) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { - // TODO(droot): This results in full update of the obj (spec + status). Need - // a way to update status field only. - return sw.client.Update(ctx, obj, opts...) +func (sw *fakeSubResourceClient) Get(ctx context.Context, obj, subResource client.Object, opts ...client.SubResourceGetOption) error { + panic("fakeSubResourceClient does not support get") +} + +func (sw *fakeSubResourceClient) Create(ctx context.Context, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error { + panic("fakeSubResourceWriter does not support create") +} + +func (sw *fakeSubResourceClient) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + // TODO(droot): This results in full update of the obj (spec + subresources). Need + // a way to update subresource only. + updateOptions := client.SubResourceUpdateOptions{} + updateOptions.ApplyOptions(opts) + + body := obj + if updateOptions.SubResourceBody != nil { + body = updateOptions.SubResourceBody + } + return sw.client.Update(ctx, body, &updateOptions.UpdateOptions) } -func (sw *fakeStatusWriter) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { - // TODO(droot): This results in full update of the obj (spec + status). Need - // a way to update status field only. - return sw.client.Patch(ctx, obj, patch, opts...) +func (sw *fakeSubResourceClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + // TODO(droot): This results in full update of the obj (spec + subresources). Need + // a way to update subresource only. + + patchOptions := client.SubResourcePatchOptions{} + patchOptions.ApplyOptions(opts) + + body := obj + if patchOptions.SubResourceBody != nil { + body = patchOptions.SubResourceBody + } + + return sw.client.Patch(ctx, body, patch, &patchOptions.PatchOptions) } func allowsUnconditionalUpdate(gvk schema.GroupVersionKind) bool { @@ -673,3 +934,12 @@ func allowsCreateOnUpdate(gvk schema.GroupVersionKind) bool { return false } + +// zero zeros the value of a pointer. +func zero(x interface{}) { + if x == nil { + return + } + res := reflect.ValueOf(x).Elem() + res.Set(reflect.Zero(res.Type())) +} diff --git a/pkg/client/fake/client_suite_test.go b/pkg/client/fake/client_suite_test.go index b697144d8b..66590f0b58 100644 --- a/pkg/client/fake/client_suite_test.go +++ b/pkg/client/fake/client_suite_test.go @@ -19,9 +19,8 @@ package fake import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -29,11 +28,9 @@ import ( func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Fake client Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Fake client Suite") } -var _ = BeforeSuite(func(done Done) { +var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) - close(done) -}, 60) +}) diff --git a/pkg/client/fake/client_test.go b/pkg/client/fake/client_test.go index e59e68d3ad..570cd744ad 100644 --- a/pkg/client/fake/client_test.go +++ b/pkg/client/fake/client_test.go @@ -20,11 +20,15 @@ import ( "context" "encoding/json" "fmt" + "strconv" "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes/fake" appsv1 "k8s.io/api/apps/v1" coordinationv1 "k8s.io/api/coordination/v1" @@ -44,6 +48,7 @@ var _ = Describe("Fake client", func() { var cl client.WithWatch BeforeEach(func() { + replicas := int32(1) dep = &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ APIVersion: "apps/v1", @@ -54,6 +59,12 @@ var _ = Describe("Fake client", func() { Namespace: "ns1", ResourceVersion: trackerAddResourceVersion, }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RecreateDeploymentStrategyType, + }, + }, } dep2 = &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ @@ -68,6 +79,9 @@ var _ = Describe("Fake client", func() { }, ResourceVersion: trackerAddResourceVersion, }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + }, } cm = &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ @@ -85,7 +99,7 @@ var _ = Describe("Fake client", func() { } }) - AssertClientBehavior := func() { + AssertClientWithoutIndexBehavior := func() { It("should be able to Get", func() { By("Getting a deployment") namespacedName := types.NamespacedName{ @@ -140,6 +154,51 @@ var _ = Describe("Fake client", func() { Expect(list.Items).To(HaveLen(2)) }) + It("should be able to retrieve registered objects that got manipulated as unstructured", func() { + list := func() { + By("Listing all endpoints in a namespace") + list := &unstructured.UnstructuredList{} + list.SetAPIVersion("v1") + list.SetKind("EndpointsList") + err := cl.List(context.Background(), list, client.InNamespace("ns1")) + Expect(err).To(BeNil()) + Expect(list.Items).To(HaveLen(1)) + } + + unstructuredEndpoint := func() *unstructured.Unstructured { + item := &unstructured.Unstructured{} + item.SetAPIVersion("v1") + item.SetKind("Endpoints") + item.SetName("test-endpoint") + item.SetNamespace("ns1") + return item + } + + By("Adding the object during client initialization") + cl = NewFakeClient(unstructuredEndpoint()) + list() + Expect(cl.Delete(context.Background(), unstructuredEndpoint())).To(BeNil()) + + By("Creating an object") + item := unstructuredEndpoint() + err := cl.Create(context.Background(), item) + Expect(err).To(BeNil()) + list() + + By("Updating the object") + item.SetAnnotations(map[string]string{"foo": "bar"}) + err = cl.Update(context.Background(), item) + Expect(err).To(BeNil()) + list() + + By("Patching the object") + old := item.DeepCopy() + item.SetAnnotations(map[string]string{"bar": "baz"}) + err = cl.Patch(context.Background(), item, client.MergeFrom(old)) + Expect(err).To(BeNil()) + list() + }) + It("should be able to Create an unregistered type using unstructured", func() { item := &unstructured.Unstructured{} item.SetAPIVersion("custom/v1") @@ -568,7 +627,33 @@ var _ = Describe("Fake client", func() { Expect(obj.ObjectMeta.ResourceVersion).To(Equal(trackerAddResourceVersion)) }) - It("should be able to Delete", func() { + It("should reject Delete with a mismatched ResourceVersion", func() { + bogusRV := "bogus" + By("Deleting with a mismatched ResourceVersion Precondition") + err := cl.Delete(context.Background(), dep, client.Preconditions{ResourceVersion: &bogusRV}) + Expect(apierrors.IsConflict(err)).To(BeTrue()) + + list := &appsv1.DeploymentList{} + err = cl.List(context.Background(), list, client.InNamespace("ns1")) + Expect(err).To(BeNil()) + Expect(list.Items).To(HaveLen(2)) + Expect(list.Items).To(ConsistOf(*dep, *dep2)) + }) + + It("should successfully Delete with a matching ResourceVersion", func() { + goodRV := trackerAddResourceVersion + By("Deleting with a matching ResourceVersion Precondition") + err := cl.Delete(context.Background(), dep, client.Preconditions{ResourceVersion: &goodRV}) + Expect(err).To(BeNil()) + + list := &appsv1.DeploymentList{} + err = cl.List(context.Background(), list, client.InNamespace("ns1")) + Expect(err).To(BeNil()) + Expect(list.Items).To(HaveLen(1)) + Expect(list.Items).To(ConsistOf(*dep2)) + }) + + It("should be able to Delete with no ResourceVersion Precondition", func() { By("Deleting a deployment") err := cl.Delete(context.Background(), dep) Expect(err).To(BeNil()) @@ -581,6 +666,21 @@ var _ = Describe("Fake client", func() { Expect(list.Items).To(ConsistOf(*dep2)) }) + It("should be able to Delete with no opts even if object's ResourceVersion doesn't match server", func() { + By("Deleting a deployment") + depCopy := dep.DeepCopy() + depCopy.ResourceVersion = "bogus" + err := cl.Delete(context.Background(), depCopy) + Expect(err).To(BeNil()) + + By("Listing all deployments in the namespace") + list := &appsv1.DeploymentList{} + err = cl.List(context.Background(), list, client.InNamespace("ns1")) + Expect(err).To(BeNil()) + Expect(list.Items).To(HaveLen(1)) + Expect(list.Items).To(ConsistOf(*dep2)) + }) + It("should handle finalizers on Update", func() { namespacedName := types.NamespacedName{ Name: "test-cm", @@ -744,6 +844,27 @@ var _ = Describe("Fake client", func() { Expect(obj).To(Equal(cm)) Expect(obj.ObjectMeta.ResourceVersion).To(Equal(trackerAddResourceVersion)) }) + + It("Should not Delete the object", func() { + By("Deleting a configmap with DryRun with Delete()") + err := cl.Delete(context.Background(), cm, client.DryRunAll) + Expect(err).To(BeNil()) + + By("Deleting a configmap with DryRun with DeleteAllOf()") + err = cl.DeleteAllOf(context.Background(), cm, client.DryRunAll) + Expect(err).To(BeNil()) + + By("Getting the configmap") + namespacedName := types.NamespacedName{ + Name: "test-cm", + Namespace: "ns2", + } + obj := &corev1.ConfigMap{} + err = cl.Get(context.Background(), namespacedName, obj) + Expect(err).To(BeNil()) + Expect(obj).To(Equal(cm)) + Expect(obj.ObjectMeta.ResourceVersion).To(Equal(trackerAddResourceVersion)) + }) }) It("should be able to Patch", func() { @@ -810,20 +931,60 @@ var _ = Describe("Fake client", func() { err = cl.Get(context.Background(), namespacedName, obj) Expect(apierrors.IsNotFound(err)).To(BeTrue()) }) + + It("should remove finalizers of the object on Patch", func() { + namespacedName := types.NamespacedName{ + Name: "test-cm", + Namespace: "patch-finalizers-in-obj", + } + By("Creating a new object") + obj := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespacedName.Name, + Namespace: namespacedName.Namespace, + Finalizers: []string{"finalizers.sigs.k8s.io/test"}, + }, + Data: map[string]string{ + "test-key": "new-value", + }, + } + err := cl.Create(context.Background(), obj) + Expect(err).To(BeNil()) + + By("Removing the finalizer") + mergePatch, err := json.Marshal(map[string]interface{}{ + "metadata": map[string]interface{}{ + "$deleteFromPrimitiveList/finalizers": []string{ + "finalizers.sigs.k8s.io/test", + }, + }, + }) + Expect(err).To(BeNil()) + err = cl.Patch(context.Background(), obj, client.RawPatch(types.StrategicMergePatchType, mergePatch)) + Expect(err).To(BeNil()) + + By("Check the finalizer has been removed in the object") + Expect(len(obj.Finalizers)).To(Equal(0)) + + By("Check the finalizer has been removed in client") + newObj := &corev1.ConfigMap{} + err = cl.Get(context.Background(), namespacedName, newObj) + Expect(err).To(BeNil()) + Expect(len(newObj.Finalizers)).To(Equal(0)) + }) } Context("with default scheme.Scheme", func() { - BeforeEach(func(done Done) { + BeforeEach(func() { cl = NewClientBuilder(). WithObjects(dep, dep2, cm). Build() - close(done) }) - AssertClientBehavior() + AssertClientWithoutIndexBehavior() }) Context("with given scheme", func() { - BeforeEach(func(done Done) { + BeforeEach(func() { scheme := runtime.NewScheme() Expect(corev1.AddToScheme(scheme)).To(Succeed()) Expect(appsv1.AddToScheme(scheme)).To(Succeed()) @@ -833,9 +994,163 @@ var _ = Describe("Fake client", func() { WithObjects(cm). WithLists(&appsv1.DeploymentList{Items: []appsv1.Deployment{*dep, *dep2}}). Build() - close(done) }) - AssertClientBehavior() + AssertClientWithoutIndexBehavior() + }) + + Context("with Indexes", func() { + depReplicasIndexer := func(obj client.Object) []string { + dep, ok := obj.(*appsv1.Deployment) + if !ok { + panic(fmt.Errorf("indexer function for type %T's spec.replicas field received"+ + " object of type %T, this should never happen", appsv1.Deployment{}, obj)) + } + indexVal := "" + if dep.Spec.Replicas != nil { + indexVal = strconv.Itoa(int(*dep.Spec.Replicas)) + } + return []string{indexVal} + } + + depStrategyTypeIndexer := func(obj client.Object) []string { + dep, ok := obj.(*appsv1.Deployment) + if !ok { + panic(fmt.Errorf("indexer function for type %T's spec.strategy.type field received"+ + " object of type %T, this should never happen", appsv1.Deployment{}, obj)) + } + return []string{string(dep.Spec.Strategy.Type)} + } + + var cb *ClientBuilder + BeforeEach(func() { + cb = NewClientBuilder(). + WithObjects(dep, dep2, cm). + WithIndex(&appsv1.Deployment{}, "spec.replicas", depReplicasIndexer) + }) + + Context("client has just one Index", func() { + BeforeEach(func() { cl = cb.Build() }) + + Context("behavior that doesn't use an Index", func() { + AssertClientWithoutIndexBehavior() + }) + + Context("filtered List using field selector", func() { + It("errors when there's no Index for the GroupVersionResource", func() { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("key", "val"), + } + err := cl.List(context.Background(), &corev1.ConfigMapList{}, listOpts) + Expect(err).NotTo(BeNil()) + }) + + It("errors when there's no Index matching the field name", func() { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("spec.paused", "false"), + } + err := cl.List(context.Background(), &appsv1.DeploymentList{}, listOpts) + Expect(err).NotTo(BeNil()) + }) + + It("errors when field selector uses two requirements", func() { + listOpts := &client.ListOptions{ + FieldSelector: fields.AndSelectors( + fields.OneTermEqualSelector("spec.replicas", "1"), + fields.OneTermEqualSelector("spec.strategy.type", string(appsv1.RecreateDeploymentStrategyType)), + )} + err := cl.List(context.Background(), &appsv1.DeploymentList{}, listOpts) + Expect(err).NotTo(BeNil()) + }) + + It("returns two deployments that match the only field selector requirement", func() { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("spec.replicas", "1"), + } + list := &appsv1.DeploymentList{} + Expect(cl.List(context.Background(), list, listOpts)).To(Succeed()) + Expect(list.Items).To(ConsistOf(*dep, *dep2)) + }) + + It("returns no object because no object matches the only field selector requirement", func() { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("spec.replicas", "2"), + } + list := &appsv1.DeploymentList{} + Expect(cl.List(context.Background(), list, listOpts)).To(Succeed()) + Expect(list.Items).To(BeEmpty()) + }) + + It("returns deployment that matches both the field and label selectors", func() { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("spec.replicas", "1"), + LabelSelector: labels.SelectorFromSet(dep2.Labels), + } + list := &appsv1.DeploymentList{} + Expect(cl.List(context.Background(), list, listOpts)).To(Succeed()) + Expect(list.Items).To(ConsistOf(*dep2)) + }) + + It("returns no object even if field selector matches because label selector doesn't", func() { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("spec.replicas", "1"), + LabelSelector: labels.Nothing(), + } + list := &appsv1.DeploymentList{} + Expect(cl.List(context.Background(), list, listOpts)).To(Succeed()) + Expect(list.Items).To(BeEmpty()) + }) + + It("returns no object even if label selector matches because field selector doesn't", func() { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("spec.replicas", "2"), + LabelSelector: labels.Everything(), + } + list := &appsv1.DeploymentList{} + Expect(cl.List(context.Background(), list, listOpts)).To(Succeed()) + Expect(list.Items).To(BeEmpty()) + }) + }) + }) + + Context("client has two Indexes", func() { + BeforeEach(func() { + cl = cb.WithIndex(&appsv1.Deployment{}, "spec.strategy.type", depStrategyTypeIndexer).Build() + }) + + Context("behavior that doesn't use an Index", func() { + AssertClientWithoutIndexBehavior() + }) + + Context("filtered List using field selector", func() { + It("uses the second index to retrieve the indexed objects when there are matches", func() { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("spec.strategy.type", string(appsv1.RecreateDeploymentStrategyType)), + } + list := &appsv1.DeploymentList{} + Expect(cl.List(context.Background(), list, listOpts)).To(Succeed()) + Expect(list.Items).To(ConsistOf(*dep)) + }) + + It("uses the second index to retrieve the indexed objects when there are no matches", func() { + listOpts := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("spec.strategy.type", string(appsv1.RollingUpdateDeploymentStrategyType)), + } + list := &appsv1.DeploymentList{} + Expect(cl.List(context.Background(), list, listOpts)).To(Succeed()) + Expect(list.Items).To(BeEmpty()) + }) + + It("errors when field selector uses two requirements", func() { + listOpts := &client.ListOptions{ + FieldSelector: fields.AndSelectors( + fields.OneTermEqualSelector("spec.replicas", "1"), + fields.OneTermEqualSelector("spec.strategy.type", string(appsv1.RecreateDeploymentStrategyType)), + )} + err := cl.List(context.Background(), &appsv1.DeploymentList{}, listOpts) + Expect(err).NotTo(BeNil()) + }) + }) + }) }) It("should set the ResourceVersion to 999 when adding an object to the tracker", func() { @@ -856,4 +1171,68 @@ var _ = Describe("Fake client", func() { } Expect(retrieved).To(Equal(reference)) }) + + It("should be able to build with given tracker and get resource", func() { + clientSet := fake.NewSimpleClientset(dep) + cl := NewClientBuilder().WithRuntimeObjects(dep2).WithObjectTracker(clientSet.Tracker()).Build() + + By("Getting a deployment") + namespacedName := types.NamespacedName{ + Name: "test-deployment", + Namespace: "ns1", + } + obj := &appsv1.Deployment{} + err := cl.Get(context.Background(), namespacedName, obj) + Expect(err).To(BeNil()) + Expect(obj).To(Equal(dep)) + + By("Getting a deployment from clientSet") + csDep2, err := clientSet.AppsV1().Deployments("ns1").Get(context.Background(), "test-deployment-2", metav1.GetOptions{}) + Expect(err).To(BeNil()) + Expect(csDep2).To(Equal(dep2)) + + By("Getting a new deployment") + namespacedName3 := types.NamespacedName{ + Name: "test-deployment-3", + Namespace: "ns1", + } + + dep3 := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "Deployment", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment-3", + Namespace: "ns1", + Labels: map[string]string{ + "test-label": "label-value", + }, + ResourceVersion: trackerAddResourceVersion, + }, + } + + _, err = clientSet.AppsV1().Deployments("ns1").Create(context.Background(), dep3, metav1.CreateOptions{}) + Expect(err).To(BeNil()) + + obj = &appsv1.Deployment{} + err = cl.Get(context.Background(), namespacedName3, obj) + Expect(err).To(BeNil()) + Expect(obj).To(Equal(dep3)) + }) +}) + +var _ = Describe("Fake client builder", func() { + It("panics when an index with the same name and GroupVersionKind is registered twice", func() { + // We need any realistic GroupVersionKind, the choice of apps/v1 Deployment is arbitrary. + cb := NewClientBuilder().WithIndex(&appsv1.Deployment{}, + "test-name", + func(client.Object) []string { return nil }) + + Expect(func() { + cb.WithIndex(&appsv1.Deployment{}, + "test-name", + func(client.Object) []string { return []string{"foo"} }) + }).To(Panic()) + }) }) diff --git a/pkg/client/fake/doc.go b/pkg/client/fake/doc.go index 7d680690dc..d0614666e3 100644 --- a/pkg/client/fake/doc.go +++ b/pkg/client/fake/doc.go @@ -28,12 +28,11 @@ When in doubt, it's almost always better not to use this package and instead use envtest.Environment with a real client and API server. WARNING: ⚠️ Current Limitations / Known Issues with the fake Client ⚠️ -- This client does not have a way to inject specific errors to test handled vs. unhandled errors. -- There is some support for sub resources which can cause issues with tests if you're trying to update - e.g. metadata and status in the same reconcile. -- No OpeanAPI validation is performed when creating or updating objects. -- ObjectMeta's `Generation` and `ResourceVersion` don't behave properly, Patch or Update -operations that rely on these fields will fail, or give false positives. - + - This client does not have a way to inject specific errors to test handled vs. unhandled errors. + - There is some support for sub resources which can cause issues with tests if you're trying to update + e.g. metadata and status in the same reconcile. + - No OpenAPI validation is performed when creating or updating objects. + - ObjectMeta's `Generation` and `ResourceVersion` don't behave properly, Patch or Update + operations that rely on these fields will fail, or give false positives. */ package fake diff --git a/pkg/client/interfaces.go b/pkg/client/interfaces.go index 58c2ece15b..0ddda3163d 100644 --- a/pkg/client/interfaces.go +++ b/pkg/client/interfaces.go @@ -20,6 +20,7 @@ import ( "context" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" @@ -50,7 +51,7 @@ type Reader interface { // Get retrieves an obj for the given object key from the Kubernetes Cluster. // obj must be a struct pointer so that obj can be updated with the response // returned by the Server. - Get(ctx context.Context, key ObjectKey, obj Object) error + Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error // List retrieves list of objects for a given namespace and list options. On a // successful call, Items field in the list will be populated with the @@ -60,7 +61,8 @@ type Reader interface { // Writer knows how to create, delete, and update Kubernetes objects. type Writer interface { - // Create saves the object obj in the Kubernetes cluster. + // Create saves the object obj in the Kubernetes cluster. obj must be a + // struct pointer so that obj can be updated with the content returned by the Server. Create(ctx context.Context, obj Object, opts ...CreateOption) error // Delete deletes the given obj from Kubernetes cluster. @@ -81,20 +83,80 @@ type Writer interface { // StatusClient knows how to create a client which can update status subresource // for kubernetes objects. type StatusClient interface { - Status() StatusWriter + Status() SubResourceWriter } -// StatusWriter knows how to update status subresource of a Kubernetes object. -type StatusWriter interface { +// SubResourceClientConstructor knows how to create a client which can update subresource +// for kubernetes objects. +type SubResourceClientConstructor interface { + // SubResourceClientConstructor returns a subresource client for the named subResource. Known + // upstream subResources usages are: + // - ServiceAccount token creation: + // sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}} + // token := &authenticationv1.TokenRequest{} + // c.SubResourceClient("token").Create(ctx, sa, token) + // + // - Pod eviction creation: + // pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}} + // c.SubResourceClient("eviction").Create(ctx, pod, &policyv1.Eviction{}) + // + // - Pod binding creation: + // pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}} + // binding := &corev1.Binding{Target: corev1.ObjectReference{Name: "my-node"}} + // c.SubResourceClient("binding").Create(ctx, pod, binding) + // + // - CertificateSigningRequest approval: + // csr := &certificatesv1.CertificateSigningRequest{ + // ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}, + // Status: certificatesv1.CertificateSigningRequestStatus{ + // Conditions: []certificatesv1.[]CertificateSigningRequestCondition{{ + // Type: certificatesv1.CertificateApproved, + // Status: corev1.ConditionTrue, + // }}, + // }, + // } + // c.SubResourceClient("approval").Update(ctx, csr) + // + // - Scale retrieval: + // dep := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}} + // scale := &autoscalingv1.Scale{} + // c.SubResourceClient("scale").Get(ctx, dep, scale) + // + // - Scale update: + // dep := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Namespace: "foo", Name: "bar"}} + // scale := &autoscalingv1.Scale{Spec: autoscalingv1.ScaleSpec{Replicas: 2}} + // c.SubResourceClient("scale").Update(ctx, dep, client.WithSubResourceBody(scale)) + SubResource(subResource string) SubResourceClient +} + +// StatusWriter is kept for backward compatibility. +type StatusWriter = SubResourceWriter + +// SubResourceReader knows how to read SubResources +type SubResourceReader interface { + Get(ctx context.Context, obj Object, subResource Object, opts ...SubResourceGetOption) error +} + +// SubResourceWriter knows how to update subresource of a Kubernetes object. +type SubResourceWriter interface { + // Create saves the subResource object in the Kubernetes cluster. obj must be a + // struct pointer so that obj can be updated with the content returned by the Server. + Create(ctx context.Context, obj Object, subResource Object, opts ...SubResourceCreateOption) error // Update updates the fields corresponding to the status subresource for the // given obj. obj must be a struct pointer so that obj can be updated // with the content returned by the Server. - Update(ctx context.Context, obj Object, opts ...UpdateOption) error + Update(ctx context.Context, obj Object, opts ...SubResourceUpdateOption) error // Patch patches the given object's subresource. obj must be a struct // pointer so that obj can be updated with the content returned by the // Server. - Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error + Patch(ctx context.Context, obj Object, patch Patch, opts ...SubResourcePatchOption) error +} + +// SubResourceClient knows how to perform CRU operations on Kubernetes objects. +type SubResourceClient interface { + SubResourceReader + SubResourceWriter } // Client knows how to perform CRUD operations on Kubernetes objects. @@ -102,11 +164,16 @@ type Client interface { Reader Writer StatusClient + SubResourceClientConstructor // Scheme returns the scheme this client is using. Scheme() *runtime.Scheme // RESTMapper returns the rest this client is using. RESTMapper() meta.RESTMapper + // GroupVersionKindFor returns the GroupVersionKind for the given object. + GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) + // IsObjectNamespaced returns true if the GroupVersionKind of the object is namespaced. + IsObjectNamespaced(obj runtime.Object) (bool, error) } // WithWatch supports Watch on top of the CRUD operations supported by @@ -143,3 +210,13 @@ func IgnoreNotFound(err error) error { } return err } + +// IgnoreAlreadyExists returns nil on AlreadyExists errors. +// All other values that are not AlreadyExists errors or nil are returned unmodified. +func IgnoreAlreadyExists(err error) error { + if apierrors.IsAlreadyExists(err) { + return nil + } + + return err +} diff --git a/pkg/client/metadata_client.go b/pkg/client/metadata_client.go index 59747463a4..d0c6b8e13a 100644 --- a/pkg/client/metadata_client.go +++ b/pkg/client/metadata_client.go @@ -116,7 +116,7 @@ func (mc *metadataClient) Patch(ctx context.Context, obj Object, patch Patch, op } // Get implements client.Client. -func (mc *metadataClient) Get(ctx context.Context, key ObjectKey, obj Object) error { +func (mc *metadataClient) Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error { metadata, ok := obj.(*metav1.PartialObjectMetadata) if !ok { return fmt.Errorf("metadata client did not understand object: %T", obj) @@ -124,12 +124,15 @@ func (mc *metadataClient) Get(ctx context.Context, key ObjectKey, obj Object) er gvk := metadata.GroupVersionKind() + getOpts := GetOptions{} + getOpts.ApplyOptions(opts) + resInt, err := mc.getResourceInterface(gvk, key.Namespace) if err != nil { return err } - res, err := resInt.Get(ctx, key.Name, metav1.GetOptions{}) + res, err := resInt.Get(ctx, key.Name, *getOpts.AsGetOptions()) if err != nil { return err } @@ -146,9 +149,7 @@ func (mc *metadataClient) List(ctx context.Context, obj ObjectList, opts ...List } gvk := metadata.GroupVersionKind() - if strings.HasSuffix(gvk.Kind, "List") { - gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] - } + gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") listOpts := ListOptions{} listOpts.ApplyOptions(opts) @@ -167,7 +168,7 @@ func (mc *metadataClient) List(ctx context.Context, obj ObjectList, opts ...List return nil } -func (mc *metadataClient) PatchStatus(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { +func (mc *metadataClient) PatchSubResource(ctx context.Context, obj Object, subResource string, patch Patch, opts ...SubResourcePatchOption) error { metadata, ok := obj.(*metav1.PartialObjectMetadata) if !ok { return fmt.Errorf("metadata client did not understand object: %T", obj) @@ -179,16 +180,24 @@ func (mc *metadataClient) PatchStatus(ctx context.Context, obj Object, patch Pat return err } - data, err := patch.Data(obj) + patchOpts := &SubResourcePatchOptions{} + patchOpts.ApplyOptions(opts) + + body := obj + if patchOpts.SubResourceBody != nil { + body = patchOpts.SubResourceBody + } + + data, err := patch.Data(body) if err != nil { return err } - patchOpts := &PatchOptions{} - res, err := resInt.Patch(ctx, metadata.Name, patch.Type(), data, *patchOpts.AsPatchOptions(), "status") + res, err := resInt.Patch(ctx, metadata.Name, patch.Type(), data, *patchOpts.AsPatchOptions(), subResource) if err != nil { return err } + *metadata = *res metadata.SetGroupVersionKind(gvk) // restore the GVK, which isn't set on metadata return nil diff --git a/pkg/client/namespaced_client.go b/pkg/client/namespaced_client.go index d73cc5135a..222dc79579 100644 --- a/pkg/client/namespaced_client.go +++ b/pkg/client/namespaced_client.go @@ -18,14 +18,11 @@ package client import ( "context" - "errors" "fmt" "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" ) // NewNamespacedClient wraps an existing client enforcing the namespace value. @@ -55,51 +52,21 @@ func (n *namespacedClient) RESTMapper() meta.RESTMapper { return n.client.RESTMapper() } -// isNamespaced returns true if the object is namespace scoped. -// For unstructured objects the gvk is found from the object itself. -// TODO: this is repetitive code. Remove this and use ojectutil.IsNamespaced. -func isNamespaced(c Client, obj runtime.Object) (bool, error) { - var gvk schema.GroupVersionKind - var err error - - _, isUnstructured := obj.(*unstructured.Unstructured) - _, isUnstructuredList := obj.(*unstructured.UnstructuredList) - - isUnstructured = isUnstructured || isUnstructuredList - if isUnstructured { - gvk = obj.GetObjectKind().GroupVersionKind() - } else { - gvk, err = apiutil.GVKForObject(obj, c.Scheme()) - if err != nil { - return false, err - } - } - - gk := schema.GroupKind{ - Group: gvk.Group, - Kind: gvk.Kind, - } - restmapping, err := c.RESTMapper().RESTMapping(gk) - if err != nil { - return false, fmt.Errorf("failed to get restmapping: %w", err) - } - scope := restmapping.Scope.Name() - - if scope == "" { - return false, errors.New("scope cannot be identified, empty scope returned") - } +// GroupVersionKindFor returns the GroupVersionKind for the given object. +func (n *namespacedClient) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + return n.client.GroupVersionKindFor(obj) +} - if scope != meta.RESTScopeNameRoot { - return true, nil - } - return false, nil +// IsObjectNamespaced returns true if the GroupVersionKind of the object is namespaced. +func (n *namespacedClient) IsObjectNamespaced(obj runtime.Object) (bool, error) { + return n.client.IsObjectNamespaced(obj) } -// Create implements clinet.Client. +// Create implements client.Client. func (n *namespacedClient) Create(ctx context.Context, obj Object, opts ...CreateOption) error { - isNamespaceScoped, err := isNamespaced(n.client, obj) + isNamespaceScoped, err := n.IsObjectNamespaced(obj) if err != nil { - return fmt.Errorf("error finding the scope of the object: %v", err) + return fmt.Errorf("error finding the scope of the object: %w", err) } objectNamespace := obj.GetNamespace() @@ -115,9 +82,9 @@ func (n *namespacedClient) Create(ctx context.Context, obj Object, opts ...Creat // Update implements client.Client. func (n *namespacedClient) Update(ctx context.Context, obj Object, opts ...UpdateOption) error { - isNamespaceScoped, err := isNamespaced(n.client, obj) + isNamespaceScoped, err := n.IsObjectNamespaced(obj) if err != nil { - return fmt.Errorf("error finding the scope of the object: %v", err) + return fmt.Errorf("error finding the scope of the object: %w", err) } objectNamespace := obj.GetNamespace() @@ -133,9 +100,9 @@ func (n *namespacedClient) Update(ctx context.Context, obj Object, opts ...Updat // Delete implements client.Client. func (n *namespacedClient) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error { - isNamespaceScoped, err := isNamespaced(n.client, obj) + isNamespaceScoped, err := n.IsObjectNamespaced(obj) if err != nil { - return fmt.Errorf("error finding the scope of the object: %v", err) + return fmt.Errorf("error finding the scope of the object: %w", err) } objectNamespace := obj.GetNamespace() @@ -151,9 +118,9 @@ func (n *namespacedClient) Delete(ctx context.Context, obj Object, opts ...Delet // DeleteAllOf implements client.Client. func (n *namespacedClient) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error { - isNamespaceScoped, err := isNamespaced(n.client, obj) + isNamespaceScoped, err := n.IsObjectNamespaced(obj) if err != nil { - return fmt.Errorf("error finding the scope of the object: %v", err) + return fmt.Errorf("error finding the scope of the object: %w", err) } if isNamespaceScoped { @@ -164,9 +131,9 @@ func (n *namespacedClient) DeleteAllOf(ctx context.Context, obj Object, opts ... // Patch implements client.Client. func (n *namespacedClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { - isNamespaceScoped, err := isNamespaced(n.client, obj) + isNamespaceScoped, err := n.IsObjectNamespaced(obj) if err != nil { - return fmt.Errorf("error finding the scope of the object: %v", err) + return fmt.Errorf("error finding the scope of the object: %w", err) } objectNamespace := obj.GetNamespace() @@ -181,18 +148,18 @@ func (n *namespacedClient) Patch(ctx context.Context, obj Object, patch Patch, o } // Get implements client.Client. -func (n *namespacedClient) Get(ctx context.Context, key ObjectKey, obj Object) error { - isNamespaceScoped, err := isNamespaced(n.client, obj) +func (n *namespacedClient) Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error { + isNamespaceScoped, err := n.IsObjectNamespaced(obj) if err != nil { - return fmt.Errorf("error finding the scope of the object: %v", err) + return fmt.Errorf("error finding the scope of the object: %w", err) } if isNamespaceScoped { if key.Namespace != "" && key.Namespace != n.namespace { - return fmt.Errorf("namespace %s provided for the object %s does not match the namesapce %s on the client", key.Namespace, obj.GetName(), n.namespace) + return fmt.Errorf("namespace %s provided for the object %s does not match the namespace %s on the client", key.Namespace, obj.GetName(), n.namespace) } key.Namespace = n.namespace } - return n.client.Get(ctx, key, obj) + return n.client.Get(ctx, key, obj, opts...) } // List implements client.Client. @@ -204,24 +171,65 @@ func (n *namespacedClient) List(ctx context.Context, obj ObjectList, opts ...Lis } // Status implements client.StatusClient. -func (n *namespacedClient) Status() StatusWriter { - return &namespacedClientStatusWriter{StatusClient: n.client.Status(), namespace: n.namespace, namespacedclient: n} +func (n *namespacedClient) Status() SubResourceWriter { + return n.SubResource("status") +} + +// SubResource implements client.SubResourceClient. +func (n *namespacedClient) SubResource(subResource string) SubResourceClient { + return &namespacedClientSubResourceClient{client: n.client.SubResource(subResource), namespace: n.namespace, namespacedclient: n} } -// ensure namespacedClientStatusWriter implements client.StatusWriter. -var _ StatusWriter = &namespacedClientStatusWriter{} +// ensure namespacedClientSubResourceClient implements client.SubResourceClient. +var _ SubResourceClient = &namespacedClientSubResourceClient{} -type namespacedClientStatusWriter struct { - StatusClient StatusWriter +type namespacedClientSubResourceClient struct { + client SubResourceClient namespace string namespacedclient Client } -// Update implements client.StatusWriter. -func (nsw *namespacedClientStatusWriter) Update(ctx context.Context, obj Object, opts ...UpdateOption) error { - isNamespaceScoped, err := isNamespaced(nsw.namespacedclient, obj) +func (nsw *namespacedClientSubResourceClient) Get(ctx context.Context, obj, subResource Object, opts ...SubResourceGetOption) error { + isNamespaceScoped, err := nsw.namespacedclient.IsObjectNamespaced(obj) + if err != nil { + return fmt.Errorf("error finding the scope of the object: %w", err) + } + + objectNamespace := obj.GetNamespace() + if objectNamespace != nsw.namespace && objectNamespace != "" { + return fmt.Errorf("namespace %s of the object %s does not match the namespace %s on the client", objectNamespace, obj.GetName(), nsw.namespace) + } + + if isNamespaceScoped && objectNamespace == "" { + obj.SetNamespace(nsw.namespace) + } + + return nsw.client.Get(ctx, obj, subResource, opts...) +} + +func (nsw *namespacedClientSubResourceClient) Create(ctx context.Context, obj, subResource Object, opts ...SubResourceCreateOption) error { + isNamespaceScoped, err := nsw.namespacedclient.IsObjectNamespaced(obj) + if err != nil { + return fmt.Errorf("error finding the scope of the object: %w", err) + } + + objectNamespace := obj.GetNamespace() + if objectNamespace != nsw.namespace && objectNamespace != "" { + return fmt.Errorf("namespace %s of the object %s does not match the namespace %s on the client", objectNamespace, obj.GetName(), nsw.namespace) + } + + if isNamespaceScoped && objectNamespace == "" { + obj.SetNamespace(nsw.namespace) + } + + return nsw.client.Create(ctx, obj, subResource, opts...) +} + +// Update implements client.SubResourceWriter. +func (nsw *namespacedClientSubResourceClient) Update(ctx context.Context, obj Object, opts ...SubResourceUpdateOption) error { + isNamespaceScoped, err := nsw.namespacedclient.IsObjectNamespaced(obj) if err != nil { - return fmt.Errorf("error finding the scope of the object: %v", err) + return fmt.Errorf("error finding the scope of the object: %w", err) } objectNamespace := obj.GetNamespace() @@ -232,14 +240,14 @@ func (nsw *namespacedClientStatusWriter) Update(ctx context.Context, obj Object, if isNamespaceScoped && objectNamespace == "" { obj.SetNamespace(nsw.namespace) } - return nsw.StatusClient.Update(ctx, obj, opts...) + return nsw.client.Update(ctx, obj, opts...) } -// Patch implements client.StatusWriter. -func (nsw *namespacedClientStatusWriter) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { - isNamespaceScoped, err := isNamespaced(nsw.namespacedclient, obj) +// Patch implements client.SubResourceWriter. +func (nsw *namespacedClientSubResourceClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...SubResourcePatchOption) error { + isNamespaceScoped, err := nsw.namespacedclient.IsObjectNamespaced(obj) if err != nil { - return fmt.Errorf("error finding the scope of the object: %v", err) + return fmt.Errorf("error finding the scope of the object: %w", err) } objectNamespace := obj.GetNamespace() @@ -250,5 +258,5 @@ func (nsw *namespacedClientStatusWriter) Patch(ctx context.Context, obj Object, if isNamespaceScoped && objectNamespace == "" { obj.SetNamespace(nsw.namespace) } - return nsw.StatusClient.Patch(ctx, obj, patch, opts...) + return nsw.client.Patch(ctx, obj, patch, opts...) } diff --git a/pkg/client/namespaced_client_test.go b/pkg/client/namespaced_client_test.go index 5b8f3388c8..1692b7fcd9 100644 --- a/pkg/client/namespaced_client_test.go +++ b/pkg/client/namespaced_client_test.go @@ -22,7 +22,7 @@ import ( "fmt" "sync/atomic" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" rbacv1 "k8s.io/api/rbac/v1" @@ -480,7 +480,7 @@ var _ = Describe("NamespacedClient", func() { }) }) - Describe("StatusWriter", func() { + Describe("SubResourceWriter", func() { var err error BeforeEach(func() { dep, err = clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) @@ -495,7 +495,7 @@ var _ = Describe("NamespacedClient", func() { changedDep := dep.DeepCopy() changedDep.Status.Replicas = 99 - Expect(getClient().Status().Update(ctx, changedDep)).NotTo(HaveOccurred()) + Expect(getClient().SubResource("status").Update(ctx, changedDep)).NotTo(HaveOccurred()) actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -509,14 +509,14 @@ var _ = Describe("NamespacedClient", func() { changedDep.SetNamespace("test") changedDep.Status.Replicas = 99 - Expect(getClient().Status().Update(ctx, changedDep)).To(HaveOccurred()) + Expect(getClient().SubResource("status").Update(ctx, changedDep)).To(HaveOccurred()) }) It("should change objects via status patch", func() { changedDep := dep.DeepCopy() changedDep.Status.Replicas = 99 - Expect(getClient().Status().Patch(ctx, changedDep, client.MergeFrom(dep))).NotTo(HaveOccurred()) + Expect(getClient().SubResource("status").Patch(ctx, changedDep, client.MergeFrom(dep))).NotTo(HaveOccurred()) actual, err := clientset.AppsV1().Deployments(ns).Get(ctx, dep.Name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -530,7 +530,7 @@ var _ = Describe("NamespacedClient", func() { changedDep.Status.Replicas = 99 changedDep.SetNamespace("test") - Expect(getClient().Status().Patch(ctx, changedDep, client.MergeFrom(dep))).To(HaveOccurred()) + Expect(getClient().SubResource("status").Patch(ctx, changedDep, client.MergeFrom(dep))).To(HaveOccurred()) }) }) diff --git a/pkg/client/options.go b/pkg/client/options.go index aa2299eac0..7f6f5b83ff 100644 --- a/pkg/client/options.go +++ b/pkg/client/options.go @@ -37,6 +37,12 @@ type DeleteOption interface { ApplyToDelete(*DeleteOptions) } +// GetOption is some configuration that modifies options for a get request. +type GetOption interface { + // ApplyToGet applies this configuration to the given get options. + ApplyToGet(*GetOptions) +} + // ListOption is some configuration that modifies options for a list request. type ListOption interface { // ApplyToList applies this configuration to the given list options. @@ -61,6 +67,29 @@ type DeleteAllOfOption interface { ApplyToDeleteAllOf(*DeleteAllOfOptions) } +// SubResourceGetOption modifies options for a SubResource Get request. +type SubResourceGetOption interface { + ApplyToSubResourceGet(*SubResourceGetOptions) +} + +// SubResourceUpdateOption is some configuration that modifies options for a update request. +type SubResourceUpdateOption interface { + // ApplyToSubResourceUpdate applies this configuration to the given update options. + ApplyToSubResourceUpdate(*SubResourceUpdateOptions) +} + +// SubResourceCreateOption is some configuration that modifies options for a create request. +type SubResourceCreateOption interface { + // ApplyToSubResourceCreate applies this configuration to the given create options. + ApplyToSubResourceCreate(*SubResourceCreateOptions) +} + +// SubResourcePatchOption configures a subresource patch request. +type SubResourcePatchOption interface { + // ApplyToSubResourcePatch applies the configuration on the given patch options. + ApplyToSubResourcePatch(*SubResourcePatchOptions) +} + // }}} // {{{ Multi-Type Options @@ -90,10 +119,23 @@ func (dryRunAll) ApplyToPatch(opts *PatchOptions) { func (dryRunAll) ApplyToDelete(opts *DeleteOptions) { opts.DryRun = []string{metav1.DryRunAll} } + func (dryRunAll) ApplyToDeleteAllOf(opts *DeleteAllOfOptions) { opts.DryRun = []string{metav1.DryRunAll} } +func (dryRunAll) ApplyToSubResourceCreate(opts *SubResourceCreateOptions) { + opts.DryRun = []string{metav1.DryRunAll} +} + +func (dryRunAll) ApplyToSubResourceUpdate(opts *SubResourceUpdateOptions) { + opts.DryRun = []string{metav1.DryRunAll} +} + +func (dryRunAll) ApplyToSubResourcePatch(opts *SubResourcePatchOptions) { + opts.DryRun = []string{metav1.DryRunAll} +} + // FieldOwner set the field manager name for the given server-side apply patch. type FieldOwner string @@ -112,6 +154,21 @@ func (f FieldOwner) ApplyToUpdate(opts *UpdateOptions) { opts.FieldManager = string(f) } +// ApplyToSubResourcePatch applies this configuration to the given patch options. +func (f FieldOwner) ApplyToSubResourcePatch(opts *SubResourcePatchOptions) { + opts.FieldManager = string(f) +} + +// ApplyToSubResourceCreate applies this configuration to the given create options. +func (f FieldOwner) ApplyToSubResourceCreate(opts *SubResourceCreateOptions) { + opts.FieldManager = string(f) +} + +// ApplyToSubResourceUpdate applies this configuration to the given update options. +func (f FieldOwner) ApplyToSubResourceUpdate(opts *SubResourceUpdateOptions) { + opts.FieldManager = string(f) +} + // }}} // {{{ Create Options @@ -311,6 +368,45 @@ func (p PropagationPolicy) ApplyToDeleteAllOf(opts *DeleteAllOfOptions) { // }}} +// {{{ Get Options + +// GetOptions contains options for get operation. +// Now it only has a Raw field, with support for specific resourceVersion. +type GetOptions struct { + // Raw represents raw GetOptions, as passed to the API server. Note + // that these may not be respected by all implementations of interface. + Raw *metav1.GetOptions +} + +var _ GetOption = &GetOptions{} + +// ApplyToGet implements GetOption for GetOptions. +func (o *GetOptions) ApplyToGet(lo *GetOptions) { + if o.Raw != nil { + lo.Raw = o.Raw + } +} + +// AsGetOptions returns these options as a flattened metav1.GetOptions. +// This may mutate the Raw field. +func (o *GetOptions) AsGetOptions() *metav1.GetOptions { + if o == nil || o.Raw == nil { + return &metav1.GetOptions{} + } + return o.Raw +} + +// ApplyOptions applies the given get options on these options, +// and then returns itself (for convenient chaining). +func (o *GetOptions) ApplyOptions(opts []GetOption) *GetOptions { + for _, opt := range opts { + opt.ApplyToGet(o) + } + return o +} + +// }}} + // {{{ List Options // ListOptions contains options for limiting or filtering results. @@ -318,7 +414,7 @@ func (p PropagationPolicy) ApplyToDeleteAllOf(opts *DeleteAllOfOptions) { // pre-parsed selectors (since generally, selectors will be executed // against the cache). type ListOptions struct { - // LabelSelector filters results by label. Use SetLabelSelector to + // LabelSelector filters results by label. Use labels.Parse() to // set from raw string form. LabelSelector labels.Selector // FieldSelector filters results by a particular field. In order @@ -341,6 +437,12 @@ type ListOptions struct { // it has expired. This field is not supported if watch is true in the Raw ListOptions. Continue string + // UnsafeDisableDeepCopy indicates not to deep copy objects during list objects. + // Be very careful with this, when enabled you must DeepCopy any object before mutating it, + // otherwise you will mutate the object in the cache. + // +optional + UnsafeDisableDeepCopy *bool + // Raw represents raw ListOptions, as passed to the API server. Note // that these may not be respected by all implementations of interface, // and the LabelSelector, FieldSelector, Limit and Continue fields are ignored. @@ -369,6 +471,9 @@ func (o *ListOptions) ApplyToList(lo *ListOptions) { if o.Continue != "" { lo.Continue = o.Continue } + if o.UnsafeDisableDeepCopy != nil { + lo.UnsafeDisableDeepCopy = o.UnsafeDisableDeepCopy + } } // AsListOptions returns these options as a flattened metav1.ListOptions. @@ -511,6 +616,25 @@ func (l Limit) ApplyToList(opts *ListOptions) { opts.Limit = int64(l) } +// UnsafeDisableDeepCopyOption indicates not to deep copy objects during list objects. +// Be very careful with this, when enabled you must DeepCopy any object before mutating it, +// otherwise you will mutate the object in the cache. +type UnsafeDisableDeepCopyOption bool + +// ApplyToList applies this configuration to the given an List options. +func (d UnsafeDisableDeepCopyOption) ApplyToList(opts *ListOptions) { + definitelyTrue := true + definitelyFalse := false + if d { + opts.UnsafeDisableDeepCopy = &definitelyTrue + } else { + opts.UnsafeDisableDeepCopy = &definitelyFalse + } +} + +// UnsafeDisableDeepCopy indicates not to deep copy objects during list objects. +const UnsafeDisableDeepCopy = UnsafeDisableDeepCopyOption(true) + // Continue sets a continuation token to retrieve chunks of results when using limit. // Continue does not implement DeleteAllOfOption interface because the server // does not support setting it for deletecollection operations. diff --git a/pkg/client/options_test.go b/pkg/client/options_test.go index 36447a3f3c..8885ca3544 100644 --- a/pkg/client/options_test.go +++ b/pkg/client/options_test.go @@ -17,7 +17,7 @@ limitations under the License. package client_test import ( - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -74,36 +74,45 @@ var _ = Describe("ListOptions", func() { }) }) +var _ = Describe("GetOptions", func() { + It("Should set Raw", func() { + o := &client.GetOptions{Raw: &metav1.GetOptions{ResourceVersion: "RV0"}} + newGetOpts := &client.GetOptions{} + o.ApplyToGet(newGetOpts) + Expect(newGetOpts).To(Equal(o)) + }) +}) + var _ = Describe("CreateOptions", func() { It("Should set DryRun", func() { o := &client.CreateOptions{DryRun: []string{"Hello", "Theodore"}} - newCreatOpts := &client.CreateOptions{} - o.ApplyToCreate(newCreatOpts) - Expect(newCreatOpts).To(Equal(o)) + newCreateOpts := &client.CreateOptions{} + o.ApplyToCreate(newCreateOpts) + Expect(newCreateOpts).To(Equal(o)) }) It("Should set FieldManager", func() { o := &client.CreateOptions{FieldManager: "FieldManager"} - newCreatOpts := &client.CreateOptions{} - o.ApplyToCreate(newCreatOpts) - Expect(newCreatOpts).To(Equal(o)) + newCreateOpts := &client.CreateOptions{} + o.ApplyToCreate(newCreateOpts) + Expect(newCreateOpts).To(Equal(o)) }) It("Should set Raw", func() { o := &client.CreateOptions{Raw: &metav1.CreateOptions{DryRun: []string{"Bye", "Theodore"}}} - newCreatOpts := &client.CreateOptions{} - o.ApplyToCreate(newCreatOpts) - Expect(newCreatOpts).To(Equal(o)) + newCreateOpts := &client.CreateOptions{} + o.ApplyToCreate(newCreateOpts) + Expect(newCreateOpts).To(Equal(o)) }) It("Should not set anything", func() { o := &client.CreateOptions{} - newCreatOpts := &client.CreateOptions{} - o.ApplyToCreate(newCreatOpts) - Expect(newCreatOpts).To(Equal(o)) + newCreateOpts := &client.CreateOptions{} + o.ApplyToCreate(newCreateOpts) + Expect(newCreateOpts).To(Equal(o)) }) }) var _ = Describe("DeleteOptions", func() { It("Should set GracePeriodSeconds", func() { - o := &client.DeleteOptions{GracePeriodSeconds: utilpointer.Int64Ptr(42)} + o := &client.DeleteOptions{GracePeriodSeconds: utilpointer.Int64(42)} newDeleteOpts := &client.DeleteOptions{} o.ApplyToDelete(newDeleteOpts) Expect(newDeleteOpts).To(Equal(o)) @@ -176,7 +185,7 @@ var _ = Describe("PatchOptions", func() { Expect(newPatchOpts).To(Equal(o)) }) It("Should set Force", func() { - o := &client.PatchOptions{Force: utilpointer.BoolPtr(true)} + o := &client.PatchOptions{Force: utilpointer.Bool(true)} newPatchOpts := &client.PatchOptions{} o.ApplyToPatch(newPatchOpts) Expect(newPatchOpts).To(Equal(o)) @@ -209,7 +218,7 @@ var _ = Describe("DeleteAllOfOptions", func() { Expect(newDeleteAllOfOpts).To(Equal(o)) }) It("Should set DeleleteOptions", func() { - o := &client.DeleteAllOfOptions{DeleteOptions: client.DeleteOptions{GracePeriodSeconds: utilpointer.Int64Ptr(44)}} + o := &client.DeleteAllOfOptions{DeleteOptions: client.DeleteOptions{GracePeriodSeconds: utilpointer.Int64(44)}} newDeleteAllOfOpts := &client.DeleteAllOfOptions{} o.ApplyToDeleteAllOf(newDeleteAllOfOpts) Expect(newDeleteAllOfOpts).To(Equal(o)) @@ -229,3 +238,42 @@ var _ = Describe("MatchingLabels", func() { Expect(err.Error()).To(Equal(expectedErrMsg)) }) }) + +var _ = Describe("FieldOwner", func() { + It("Should apply to PatchOptions", func() { + o := &client.PatchOptions{FieldManager: "bar"} + t := client.FieldOwner("foo") + t.ApplyToPatch(o) + Expect(o.FieldManager).To(Equal("foo")) + }) + It("Should apply to CreateOptions", func() { + o := &client.CreateOptions{FieldManager: "bar"} + t := client.FieldOwner("foo") + t.ApplyToCreate(o) + Expect(o.FieldManager).To(Equal("foo")) + }) + It("Should apply to UpdateOptions", func() { + o := &client.UpdateOptions{FieldManager: "bar"} + t := client.FieldOwner("foo") + t.ApplyToUpdate(o) + Expect(o.FieldManager).To(Equal("foo")) + }) + It("Should apply to SubResourcePatchOptions", func() { + o := &client.SubResourcePatchOptions{PatchOptions: client.PatchOptions{FieldManager: "bar"}} + t := client.FieldOwner("foo") + t.ApplyToSubResourcePatch(o) + Expect(o.FieldManager).To(Equal("foo")) + }) + It("Should apply to SubResourceCreateOptions", func() { + o := &client.SubResourceCreateOptions{CreateOptions: client.CreateOptions{FieldManager: "bar"}} + t := client.FieldOwner("foo") + t.ApplyToSubResourceCreate(o) + Expect(o.FieldManager).To(Equal("foo")) + }) + It("Should apply to SubResourceUpdateOptions", func() { + o := &client.SubResourceUpdateOptions{UpdateOptions: client.UpdateOptions{FieldManager: "bar"}} + t := client.FieldOwner("foo") + t.ApplyToSubResourceUpdate(o) + Expect(o.FieldManager).To(Equal("foo")) + }) +}) diff --git a/pkg/client/patch.go b/pkg/client/patch.go index 10984c5342..11d6083885 100644 --- a/pkg/client/patch.go +++ b/pkg/client/patch.go @@ -19,7 +19,7 @@ package client import ( "fmt" - jsonpatch "github.com/evanphx/json-patch" + jsonpatch "github.com/evanphx/json-patch/v5" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/json" "k8s.io/apimachinery/pkg/util/strategicpatch" diff --git a/pkg/client/patch_test.go b/pkg/client/patch_test.go index 7f86d2357a..2910ef56bf 100644 --- a/pkg/client/patch_test.go +++ b/pkg/client/patch_test.go @@ -45,12 +45,12 @@ func BenchmarkMergeFrom(b *testing.B) { }, }, ReadinessProbe: &corev1.Probe{ - Handler: corev1.Handler{ + ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{}, }, }, Lifecycle: &corev1.Lifecycle{ - PreStop: &corev1.Handler{ + PreStop: &corev1.LifecycleHandler{ HTTPGet: &corev1.HTTPGetAction{}, }, }, diff --git a/pkg/client/split.go b/pkg/client/split.go deleted file mode 100644 index bf4b861f39..0000000000 --- a/pkg/client/split.go +++ /dev/null @@ -1,141 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package client - -import ( - "context" - "strings" - - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" -) - -// NewDelegatingClientInput encapsulates the input parameters to create a new delegating client. -type NewDelegatingClientInput struct { - CacheReader Reader - Client Client - UncachedObjects []Object - CacheUnstructured bool -} - -// NewDelegatingClient creates a new delegating client. -// -// A delegating client forms a Client by composing separate reader, writer and -// statusclient interfaces. This way, you can have an Client that reads from a -// cache and writes to the API server. -func NewDelegatingClient(in NewDelegatingClientInput) (Client, error) { - uncachedGVKs := map[schema.GroupVersionKind]struct{}{} - for _, obj := range in.UncachedObjects { - gvk, err := apiutil.GVKForObject(obj, in.Client.Scheme()) - if err != nil { - return nil, err - } - uncachedGVKs[gvk] = struct{}{} - } - - return &delegatingClient{ - scheme: in.Client.Scheme(), - mapper: in.Client.RESTMapper(), - Reader: &delegatingReader{ - CacheReader: in.CacheReader, - ClientReader: in.Client, - scheme: in.Client.Scheme(), - uncachedGVKs: uncachedGVKs, - cacheUnstructured: in.CacheUnstructured, - }, - Writer: in.Client, - StatusClient: in.Client, - }, nil -} - -type delegatingClient struct { - Reader - Writer - StatusClient - - scheme *runtime.Scheme - mapper meta.RESTMapper -} - -// Scheme returns the scheme this client is using. -func (d *delegatingClient) Scheme() *runtime.Scheme { - return d.scheme -} - -// RESTMapper returns the rest mapper this client is using. -func (d *delegatingClient) RESTMapper() meta.RESTMapper { - return d.mapper -} - -// delegatingReader forms a Reader that will cause Get and List requests for -// unstructured types to use the ClientReader while requests for any other type -// of object with use the CacheReader. This avoids accidentally caching the -// entire cluster in the common case of loading arbitrary unstructured objects -// (e.g. from OwnerReferences). -type delegatingReader struct { - CacheReader Reader - ClientReader Reader - - uncachedGVKs map[schema.GroupVersionKind]struct{} - scheme *runtime.Scheme - cacheUnstructured bool -} - -func (d *delegatingReader) shouldBypassCache(obj runtime.Object) (bool, error) { - gvk, err := apiutil.GVKForObject(obj, d.scheme) - if err != nil { - return false, err - } - // TODO: this is producing unsafe guesses that don't actually work, - // but it matches ~99% of the cases out there. - if meta.IsListType(obj) { - gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") - } - if _, isUncached := d.uncachedGVKs[gvk]; isUncached { - return true, nil - } - if !d.cacheUnstructured { - _, isUnstructured := obj.(*unstructured.Unstructured) - _, isUnstructuredList := obj.(*unstructured.UnstructuredList) - return isUnstructured || isUnstructuredList, nil - } - return false, nil -} - -// Get retrieves an obj for a given object key from the Kubernetes Cluster. -func (d *delegatingReader) Get(ctx context.Context, key ObjectKey, obj Object) error { - if isUncached, err := d.shouldBypassCache(obj); err != nil { - return err - } else if isUncached { - return d.ClientReader.Get(ctx, key, obj) - } - return d.CacheReader.Get(ctx, key, obj) -} - -// List retrieves list of objects for a given namespace and list options. -func (d *delegatingReader) List(ctx context.Context, list ObjectList, opts ...ListOption) error { - if isUncached, err := d.shouldBypassCache(list); err != nil { - return err - } else if isUncached { - return d.ClientReader.List(ctx, list, opts...) - } - return d.CacheReader.List(ctx, list, opts...) -} diff --git a/pkg/client/testdata/examplecrd.yaml b/pkg/client/testdata/examplecrd.yaml new file mode 100644 index 0000000000..5409ee9789 --- /dev/null +++ b/pkg/client/testdata/examplecrd.yaml @@ -0,0 +1,17 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: chaospods.chaosapps.metamagical.io +spec: + group: chaosapps.metamagical.io + names: + kind: ChaosPod + plural: chaospods + scope: Namespaced + versions: + - name: "v1" + storage: true + served: true + schema: + openAPIV3Schema: + type: object diff --git a/pkg/client/typed_client.go b/pkg/client/typed_client.go index dde7b21f25..92afd9a9c2 100644 --- a/pkg/client/typed_client.go +++ b/pkg/client/typed_client.go @@ -24,24 +24,22 @@ import ( var _ Reader = &typedClient{} var _ Writer = &typedClient{} -var _ StatusWriter = &typedClient{} -// client is a client.Client that reads and writes directly from/to an API server. It lazily initializes -// new clients at the time they are used, and caches the client. type typedClient struct { - cache *clientCache + resources *clientRestResources paramCodec runtime.ParameterCodec } // Create implements client.Client. func (c *typedClient) Create(ctx context.Context, obj Object, opts ...CreateOption) error { - o, err := c.cache.getObjMeta(obj) + o, err := c.resources.getObjMeta(obj) if err != nil { return err } createOpts := &CreateOptions{} createOpts.ApplyOptions(opts) + return o.Post(). NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). Resource(o.resource()). @@ -53,13 +51,14 @@ func (c *typedClient) Create(ctx context.Context, obj Object, opts ...CreateOpti // Update implements client.Client. func (c *typedClient) Update(ctx context.Context, obj Object, opts ...UpdateOption) error { - o, err := c.cache.getObjMeta(obj) + o, err := c.resources.getObjMeta(obj) if err != nil { return err } updateOpts := &UpdateOptions{} updateOpts.ApplyOptions(opts) + return o.Put(). NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). Resource(o.resource()). @@ -72,7 +71,7 @@ func (c *typedClient) Update(ctx context.Context, obj Object, opts ...UpdateOpti // Delete implements client.Client. func (c *typedClient) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error { - o, err := c.cache.getObjMeta(obj) + o, err := c.resources.getObjMeta(obj) if err != nil { return err } @@ -91,7 +90,7 @@ func (c *typedClient) Delete(ctx context.Context, obj Object, opts ...DeleteOpti // DeleteAllOf implements client.Client. func (c *typedClient) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error { - o, err := c.cache.getObjMeta(obj) + o, err := c.resources.getObjMeta(obj) if err != nil { return err } @@ -110,7 +109,7 @@ func (c *typedClient) DeleteAllOf(ctx context.Context, obj Object, opts ...Delet // Patch implements client.Client. func (c *typedClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { - o, err := c.cache.getObjMeta(obj) + o, err := c.resources.getObjMeta(obj) if err != nil { return err } @@ -121,36 +120,43 @@ func (c *typedClient) Patch(ctx context.Context, obj Object, patch Patch, opts . } patchOpts := &PatchOptions{} + patchOpts.ApplyOptions(opts) + return o.Patch(patch.Type()). NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). Resource(o.resource()). Name(o.GetName()). - VersionedParams(patchOpts.ApplyOptions(opts).AsPatchOptions(), c.paramCodec). + VersionedParams(patchOpts.AsPatchOptions(), c.paramCodec). Body(data). Do(ctx). Into(obj) } // Get implements client.Client. -func (c *typedClient) Get(ctx context.Context, key ObjectKey, obj Object) error { - r, err := c.cache.getResource(obj) +func (c *typedClient) Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error { + r, err := c.resources.getResource(obj) if err != nil { return err } + getOpts := GetOptions{} + getOpts.ApplyOptions(opts) return r.Get(). NamespaceIfScoped(key.Namespace, r.isNamespaced()). Resource(r.resource()). + VersionedParams(getOpts.AsGetOptions(), c.paramCodec). Name(key.Name).Do(ctx).Into(obj) } // List implements client.Client. func (c *typedClient) List(ctx context.Context, obj ObjectList, opts ...ListOption) error { - r, err := c.cache.getResource(obj) + r, err := c.resources.getResource(obj) if err != nil { return err } + listOpts := ListOptions{} listOpts.ApplyOptions(opts) + return r.Get(). NamespaceIfScoped(listOpts.Namespace, r.isNamespaced()). Resource(r.resource()). @@ -159,9 +165,56 @@ func (c *typedClient) List(ctx context.Context, obj ObjectList, opts ...ListOpti Into(obj) } -// UpdateStatus used by StatusWriter to write status. -func (c *typedClient) UpdateStatus(ctx context.Context, obj Object, opts ...UpdateOption) error { - o, err := c.cache.getObjMeta(obj) +func (c *typedClient) GetSubResource(ctx context.Context, obj, subResourceObj Object, subResource string, opts ...SubResourceGetOption) error { + o, err := c.resources.getObjMeta(obj) + if err != nil { + return err + } + + if subResourceObj.GetName() == "" { + subResourceObj.SetName(obj.GetName()) + } + + getOpts := &SubResourceGetOptions{} + getOpts.ApplyOptions(opts) + + return o.Get(). + NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + Resource(o.resource()). + Name(o.GetName()). + SubResource(subResource). + VersionedParams(getOpts.AsGetOptions(), c.paramCodec). + Do(ctx). + Into(subResourceObj) +} + +func (c *typedClient) CreateSubResource(ctx context.Context, obj Object, subResourceObj Object, subResource string, opts ...SubResourceCreateOption) error { + o, err := c.resources.getObjMeta(obj) + if err != nil { + return err + } + + if subResourceObj.GetName() == "" { + subResourceObj.SetName(obj.GetName()) + } + + createOpts := &SubResourceCreateOptions{} + createOpts.ApplyOptions(opts) + + return o.Post(). + NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + Resource(o.resource()). + Name(o.GetName()). + SubResource(subResource). + Body(subResourceObj). + VersionedParams(createOpts.AsCreateOptions(), c.paramCodec). + Do(ctx). + Into(subResourceObj) +} + +// UpdateSubResource used by SubResourceWriter to write status. +func (c *typedClient) UpdateSubResource(ctx context.Context, obj Object, subResource string, opts ...SubResourceUpdateOption) error { + o, err := c.resources.getObjMeta(obj) if err != nil { return err } @@ -169,37 +222,58 @@ func (c *typedClient) UpdateStatus(ctx context.Context, obj Object, opts ...Upda // wrapped to improve the UX ? // It will be nice to receive an error saying the object doesn't implement // status subresource and check CRD definition + updateOpts := &SubResourceUpdateOptions{} + updateOpts.ApplyOptions(opts) + + body := obj + if updateOpts.SubResourceBody != nil { + body = updateOpts.SubResourceBody + } + if body.GetName() == "" { + body.SetName(obj.GetName()) + } + if body.GetNamespace() == "" { + body.SetNamespace(obj.GetNamespace()) + } + return o.Put(). NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). Resource(o.resource()). Name(o.GetName()). - SubResource("status"). - Body(obj). - VersionedParams((&UpdateOptions{}).ApplyOptions(opts).AsUpdateOptions(), c.paramCodec). + SubResource(subResource). + Body(body). + VersionedParams(updateOpts.AsUpdateOptions(), c.paramCodec). Do(ctx). - Into(obj) + Into(body) } -// PatchStatus used by StatusWriter to write status. -func (c *typedClient) PatchStatus(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { - o, err := c.cache.getObjMeta(obj) +// PatchSubResource used by SubResourceWriter to write subresource. +func (c *typedClient) PatchSubResource(ctx context.Context, obj Object, subResource string, patch Patch, opts ...SubResourcePatchOption) error { + o, err := c.resources.getObjMeta(obj) if err != nil { return err } - data, err := patch.Data(obj) + patchOpts := &SubResourcePatchOptions{} + patchOpts.ApplyOptions(opts) + + body := obj + if patchOpts.SubResourceBody != nil { + body = patchOpts.SubResourceBody + } + + data, err := patch.Data(body) if err != nil { return err } - patchOpts := &PatchOptions{} return o.Patch(patch.Type()). NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). Resource(o.resource()). Name(o.GetName()). - SubResource("status"). + SubResource(subResource). Body(data). - VersionedParams(patchOpts.ApplyOptions(opts).AsPatchOptions(), c.paramCodec). + VersionedParams(patchOpts.AsPatchOptions(), c.paramCodec). Do(ctx). - Into(obj) + Into(body) } diff --git a/pkg/client/unstructured_client.go b/pkg/client/unstructured_client.go index dcf15be275..b8d4146c9f 100644 --- a/pkg/client/unstructured_client.go +++ b/pkg/client/unstructured_client.go @@ -21,37 +21,34 @@ import ( "fmt" "strings" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" ) var _ Reader = &unstructuredClient{} var _ Writer = &unstructuredClient{} -var _ StatusWriter = &unstructuredClient{} -// client is a client.Client that reads and writes directly from/to an API server. It lazily initializes -// new clients at the time they are used, and caches the client. type unstructuredClient struct { - cache *clientCache + resources *clientRestResources paramCodec runtime.ParameterCodec } // Create implements client.Client. func (uc *unstructuredClient) Create(ctx context.Context, obj Object, opts ...CreateOption) error { - u, ok := obj.(*unstructured.Unstructured) + u, ok := obj.(runtime.Unstructured) if !ok { return fmt.Errorf("unstructured client did not understand object: %T", obj) } - gvk := u.GroupVersionKind() + gvk := u.GetObjectKind().GroupVersionKind() - o, err := uc.cache.getObjMeta(obj) + o, err := uc.resources.getObjMeta(obj) if err != nil { return err } createOpts := &CreateOptions{} createOpts.ApplyOptions(opts) + result := o.Post(). NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). Resource(o.resource()). @@ -60,26 +57,27 @@ func (uc *unstructuredClient) Create(ctx context.Context, obj Object, opts ...Cr Do(ctx). Into(obj) - u.SetGroupVersionKind(gvk) + u.GetObjectKind().SetGroupVersionKind(gvk) return result } // Update implements client.Client. func (uc *unstructuredClient) Update(ctx context.Context, obj Object, opts ...UpdateOption) error { - u, ok := obj.(*unstructured.Unstructured) + u, ok := obj.(runtime.Unstructured) if !ok { return fmt.Errorf("unstructured client did not understand object: %T", obj) } - gvk := u.GroupVersionKind() + gvk := u.GetObjectKind().GroupVersionKind() - o, err := uc.cache.getObjMeta(obj) + o, err := uc.resources.getObjMeta(obj) if err != nil { return err } updateOpts := UpdateOptions{} updateOpts.ApplyOptions(opts) + result := o.Put(). NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). Resource(o.resource()). @@ -89,24 +87,24 @@ func (uc *unstructuredClient) Update(ctx context.Context, obj Object, opts ...Up Do(ctx). Into(obj) - u.SetGroupVersionKind(gvk) + u.GetObjectKind().SetGroupVersionKind(gvk) return result } // Delete implements client.Client. func (uc *unstructuredClient) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error { - _, ok := obj.(*unstructured.Unstructured) - if !ok { + if _, ok := obj.(runtime.Unstructured); !ok { return fmt.Errorf("unstructured client did not understand object: %T", obj) } - o, err := uc.cache.getObjMeta(obj) + o, err := uc.resources.getObjMeta(obj) if err != nil { return err } deleteOpts := DeleteOptions{} deleteOpts.ApplyOptions(opts) + return o.Delete(). NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). Resource(o.resource()). @@ -118,18 +116,18 @@ func (uc *unstructuredClient) Delete(ctx context.Context, obj Object, opts ...De // DeleteAllOf implements client.Client. func (uc *unstructuredClient) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error { - _, ok := obj.(*unstructured.Unstructured) - if !ok { + if _, ok := obj.(runtime.Unstructured); !ok { return fmt.Errorf("unstructured client did not understand object: %T", obj) } - o, err := uc.cache.getObjMeta(obj) + o, err := uc.resources.getObjMeta(obj) if err != nil { return err } deleteAllOfOpts := DeleteAllOfOptions{} deleteAllOfOpts.ApplyOptions(opts) + return o.Delete(). NamespaceIfScoped(deleteAllOfOpts.ListOptions.Namespace, o.isNamespaced()). Resource(o.resource()). @@ -141,12 +139,11 @@ func (uc *unstructuredClient) DeleteAllOf(ctx context.Context, obj Object, opts // Patch implements client.Client. func (uc *unstructuredClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { - _, ok := obj.(*unstructured.Unstructured) - if !ok { + if _, ok := obj.(runtime.Unstructured); !ok { return fmt.Errorf("unstructured client did not understand object: %T", obj) } - o, err := uc.cache.getObjMeta(obj) + o, err := uc.resources.getObjMeta(obj) if err != nil { return err } @@ -157,26 +154,31 @@ func (uc *unstructuredClient) Patch(ctx context.Context, obj Object, patch Patch } patchOpts := &PatchOptions{} + patchOpts.ApplyOptions(opts) + return o.Patch(patch.Type()). NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). Resource(o.resource()). Name(o.GetName()). - VersionedParams(patchOpts.ApplyOptions(opts).AsPatchOptions(), uc.paramCodec). + VersionedParams(patchOpts.AsPatchOptions(), uc.paramCodec). Body(data). Do(ctx). Into(obj) } // Get implements client.Client. -func (uc *unstructuredClient) Get(ctx context.Context, key ObjectKey, obj Object) error { - u, ok := obj.(*unstructured.Unstructured) +func (uc *unstructuredClient) Get(ctx context.Context, key ObjectKey, obj Object, opts ...GetOption) error { + u, ok := obj.(runtime.Unstructured) if !ok { return fmt.Errorf("unstructured client did not understand object: %T", obj) } - gvk := u.GroupVersionKind() + gvk := u.GetObjectKind().GroupVersionKind() + + getOpts := GetOptions{} + getOpts.ApplyOptions(opts) - r, err := uc.cache.getResource(obj) + r, err := uc.resources.getResource(obj) if err != nil { return err } @@ -184,35 +186,34 @@ func (uc *unstructuredClient) Get(ctx context.Context, key ObjectKey, obj Object result := r.Get(). NamespaceIfScoped(key.Namespace, r.isNamespaced()). Resource(r.resource()). + VersionedParams(getOpts.AsGetOptions(), uc.paramCodec). Name(key.Name). Do(ctx). Into(obj) - u.SetGroupVersionKind(gvk) + u.GetObjectKind().SetGroupVersionKind(gvk) return result } // List implements client.Client. func (uc *unstructuredClient) List(ctx context.Context, obj ObjectList, opts ...ListOption) error { - u, ok := obj.(*unstructured.UnstructuredList) + u, ok := obj.(runtime.Unstructured) if !ok { return fmt.Errorf("unstructured client did not understand object: %T", obj) } - gvk := u.GroupVersionKind() - if strings.HasSuffix(gvk.Kind, "List") { - gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] - } + gvk := u.GetObjectKind().GroupVersionKind() + gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") - listOpts := ListOptions{} - listOpts.ApplyOptions(opts) - - r, err := uc.cache.getResource(obj) + r, err := uc.resources.getResource(obj) if err != nil { return err } + listOpts := ListOptions{} + listOpts.ApplyOptions(opts) + return r.Get(). NamespaceIfScoped(listOpts.Namespace, r.isNamespaced()). Resource(r.resource()). @@ -221,57 +222,140 @@ func (uc *unstructuredClient) List(ctx context.Context, obj ObjectList, opts ... Into(obj) } -func (uc *unstructuredClient) UpdateStatus(ctx context.Context, obj Object, opts ...UpdateOption) error { - _, ok := obj.(*unstructured.Unstructured) - if !ok { +func (uc *unstructuredClient) GetSubResource(ctx context.Context, obj, subResourceObj Object, subResource string, opts ...SubResourceGetOption) error { + if _, ok := obj.(runtime.Unstructured); !ok { + return fmt.Errorf("unstructured client did not understand object: %T", subResource) + } + + if _, ok := subResourceObj.(runtime.Unstructured); !ok { return fmt.Errorf("unstructured client did not understand object: %T", obj) } - o, err := uc.cache.getObjMeta(obj) + if subResourceObj.GetName() == "" { + subResourceObj.SetName(obj.GetName()) + } + + o, err := uc.resources.getObjMeta(obj) if err != nil { return err } + getOpts := &SubResourceGetOptions{} + getOpts.ApplyOptions(opts) + + return o.Get(). + NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + Resource(o.resource()). + Name(o.GetName()). + SubResource(subResource). + VersionedParams(getOpts.AsGetOptions(), uc.paramCodec). + Do(ctx). + Into(subResourceObj) +} + +func (uc *unstructuredClient) CreateSubResource(ctx context.Context, obj, subResourceObj Object, subResource string, opts ...SubResourceCreateOption) error { + if _, ok := obj.(runtime.Unstructured); !ok { + return fmt.Errorf("unstructured client did not understand object: %T", subResourceObj) + } + + if _, ok := subResourceObj.(runtime.Unstructured); !ok { + return fmt.Errorf("unstructured client did not understand object: %T", obj) + } + + if subResourceObj.GetName() == "" { + subResourceObj.SetName(obj.GetName()) + } + + o, err := uc.resources.getObjMeta(obj) + if err != nil { + return err + } + + createOpts := &SubResourceCreateOptions{} + createOpts.ApplyOptions(opts) + + return o.Post(). + NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). + Resource(o.resource()). + Name(o.GetName()). + SubResource(subResource). + Body(subResourceObj). + VersionedParams(createOpts.AsCreateOptions(), uc.paramCodec). + Do(ctx). + Into(subResourceObj) +} + +func (uc *unstructuredClient) UpdateSubResource(ctx context.Context, obj Object, subResource string, opts ...SubResourceUpdateOption) error { + if _, ok := obj.(runtime.Unstructured); !ok { + return fmt.Errorf("unstructured client did not understand object: %T", obj) + } + + o, err := uc.resources.getObjMeta(obj) + if err != nil { + return err + } + + updateOpts := SubResourceUpdateOptions{} + updateOpts.ApplyOptions(opts) + + body := obj + if updateOpts.SubResourceBody != nil { + body = updateOpts.SubResourceBody + } + if body.GetName() == "" { + body.SetName(obj.GetName()) + } + if body.GetNamespace() == "" { + body.SetNamespace(obj.GetNamespace()) + } + return o.Put(). NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). Resource(o.resource()). Name(o.GetName()). - SubResource("status"). - Body(obj). - VersionedParams((&UpdateOptions{}).ApplyOptions(opts).AsUpdateOptions(), uc.paramCodec). + SubResource(subResource). + Body(body). + VersionedParams(updateOpts.AsUpdateOptions(), uc.paramCodec). Do(ctx). - Into(obj) + Into(body) } -func (uc *unstructuredClient) PatchStatus(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error { - u, ok := obj.(*unstructured.Unstructured) +func (uc *unstructuredClient) PatchSubResource(ctx context.Context, obj Object, subResource string, patch Patch, opts ...SubResourcePatchOption) error { + u, ok := obj.(runtime.Unstructured) if !ok { return fmt.Errorf("unstructured client did not understand object: %T", obj) } - gvk := u.GroupVersionKind() + gvk := u.GetObjectKind().GroupVersionKind() - o, err := uc.cache.getObjMeta(obj) + o, err := uc.resources.getObjMeta(obj) if err != nil { return err } - data, err := patch.Data(obj) + patchOpts := &SubResourcePatchOptions{} + patchOpts.ApplyOptions(opts) + + body := obj + if patchOpts.SubResourceBody != nil { + body = patchOpts.SubResourceBody + } + + data, err := patch.Data(body) if err != nil { return err } - patchOpts := &PatchOptions{} result := o.Patch(patch.Type()). NamespaceIfScoped(o.GetNamespace(), o.isNamespaced()). Resource(o.resource()). Name(o.GetName()). - SubResource("status"). + SubResource(subResource). Body(data). - VersionedParams(patchOpts.ApplyOptions(opts).AsPatchOptions(), uc.paramCodec). + VersionedParams(patchOpts.AsPatchOptions(), uc.paramCodec). Do(ctx). - Into(u) + Into(body) - u.SetGroupVersionKind(gvk) + u.GetObjectKind().SetGroupVersionKind(gvk) return result } diff --git a/pkg/client/watch.go b/pkg/client/watch.go index 765ca5daa6..181b22a673 100644 --- a/pkg/client/watch.go +++ b/pkg/client/watch.go @@ -21,9 +21,8 @@ import ( "strings" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" - "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" ) @@ -33,21 +32,16 @@ func NewWithWatch(config *rest.Config, options Options) (WithWatch, error) { if err != nil { return nil, err } - dynamicClient, err := dynamic.NewForConfig(config) - if err != nil { - return nil, err - } - return &watchingClient{client: client, dynamic: dynamicClient}, nil + return &watchingClient{client: client}, nil } type watchingClient struct { *client - dynamic dynamic.Interface } func (w *watchingClient) Watch(ctx context.Context, list ObjectList, opts ...ListOption) (watch.Interface, error) { switch l := list.(type) { - case *unstructured.UnstructuredList: + case runtime.Unstructured: return w.unstructuredWatch(ctx, l, opts...) case *metav1.PartialObjectMetadataList: return w.metadataWatch(ctx, l, opts...) @@ -69,9 +63,7 @@ func (w *watchingClient) listOpts(opts ...ListOption) ListOptions { func (w *watchingClient) metadataWatch(ctx context.Context, obj *metav1.PartialObjectMetadataList, opts ...ListOption) (watch.Interface, error) { gvk := obj.GroupVersionKind() - if strings.HasSuffix(gvk.Kind, "List") { - gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] - } + gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") listOpts := w.listOpts(opts...) @@ -83,27 +75,23 @@ func (w *watchingClient) metadataWatch(ctx context.Context, obj *metav1.PartialO return resInt.Watch(ctx, *listOpts.AsListOptions()) } -func (w *watchingClient) unstructuredWatch(ctx context.Context, obj *unstructured.UnstructuredList, opts ...ListOption) (watch.Interface, error) { - gvk := obj.GroupVersionKind() - if strings.HasSuffix(gvk.Kind, "List") { - gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] - } - - r, err := w.client.unstructuredClient.cache.getResource(obj) +func (w *watchingClient) unstructuredWatch(ctx context.Context, obj runtime.Unstructured, opts ...ListOption) (watch.Interface, error) { + r, err := w.client.unstructuredClient.resources.getResource(obj) if err != nil { return nil, err } listOpts := w.listOpts(opts...) - if listOpts.Namespace != "" && r.isNamespaced() { - return w.dynamic.Resource(r.mapping.Resource).Namespace(listOpts.Namespace).Watch(ctx, *listOpts.AsListOptions()) - } - return w.dynamic.Resource(r.mapping.Resource).Watch(ctx, *listOpts.AsListOptions()) + return r.Get(). + NamespaceIfScoped(listOpts.Namespace, r.isNamespaced()). + Resource(r.resource()). + VersionedParams(listOpts.AsListOptions(), w.client.unstructuredClient.paramCodec). + Watch(ctx) } func (w *watchingClient) typedWatch(ctx context.Context, obj ObjectList, opts ...ListOption) (watch.Interface, error) { - r, err := w.client.typedClient.cache.getResource(obj) + r, err := w.client.typedClient.resources.getResource(obj) if err != nil { return nil, err } diff --git a/pkg/client/watch_test.go b/pkg/client/watch_test.go index 7d770cb09c..26d90f6550 100644 --- a/pkg/client/watch_test.go +++ b/pkg/client/watch_test.go @@ -21,7 +21,7 @@ import ( "fmt" "sync/atomic" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -40,7 +40,7 @@ var _ = Describe("ClientWithWatch", func() { var ns = "kube-public" ctx := context.TODO() - BeforeEach(func(done Done) { + BeforeEach(func() { atomic.AddUint64(&count, 1) dep = &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("watch-deployment-name-%v", count), Namespace: ns, Labels: map[string]string{"app": fmt.Sprintf("bar-%v", count)}}, @@ -59,24 +59,20 @@ var _ = Describe("ClientWithWatch", func() { var err error dep, err = clientset.AppsV1().Deployments(ns).Create(ctx, dep, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - close(done) - }, serverSideTimeoutSeconds) + }) - AfterEach(func(done Done) { + AfterEach(func() { deleteDeployment(ctx, dep, ns) - close(done) - }, serverSideTimeoutSeconds) + }) Describe("NewWithWatch", func() { - It("should return a new Client", func(done Done) { + It("should return a new Client", func() { cl, err := client.NewWithWatch(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) - - close(done) }) - watchSuite := func(through client.ObjectList, expectedType client.Object) { + watchSuite := func(through client.ObjectList, expectedType client.Object, checkGvk bool) { cl, err := client.NewWithWatch(cfg, client.Options{}) Expect(err).NotTo(HaveOccurred()) Expect(cl).NotTo(BeNil()) @@ -103,29 +99,35 @@ var _ = Describe("ClientWithWatch", func() { Expect(metaObject.GetName()).To(Equal(dep.Name)) Expect(metaObject.GetUID()).To(Equal(dep.UID)) + if checkGvk { + runtimeObject := event.Object + gvk := runtimeObject.GetObjectKind().GroupVersionKind() + Expect(gvk).To(Equal(schema.GroupVersionKind{ + Group: "apps", + Kind: "Deployment", + Version: "v1", + })) + } } - It("should receive a create event when watching the typed object", func(done Done) { - watchSuite(&appsv1.DeploymentList{}, &appsv1.Deployment{}) - close(done) - }, 15) + It("should receive a create event when watching the typed object", func() { + watchSuite(&appsv1.DeploymentList{}, &appsv1.Deployment{}, false) + }) - It("should receive a create event when watching the unstructured object", func(done Done) { + It("should receive a create event when watching the unstructured object", func() { u := &unstructured.UnstructuredList{} u.SetGroupVersionKind(schema.GroupVersionKind{ Group: "apps", Kind: "Deployment", Version: "v1", }) - watchSuite(u, &unstructured.Unstructured{}) - close(done) - }, 15) + watchSuite(u, &unstructured.Unstructured{}, true) + }) - It("should receive a create event when watching the metadata object", func(done Done) { + It("should receive a create event when watching the metadata object", func() { m := &metav1.PartialObjectMetadataList{TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}} - watchSuite(m, &metav1.PartialObjectMetadata{}) - close(done) - }, 15) + watchSuite(m, &metav1.PartialObjectMetadata{}, false) + }) }) }) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index dfd0fa9dd8..6bb23a1b75 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -19,6 +19,7 @@ package cluster import ( "context" "errors" + "net/http" "time" "github.com/go-logr/logr" @@ -27,6 +28,7 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" logf "sigs.k8s.io/controller-runtime/pkg/internal/log" @@ -37,14 +39,15 @@ import ( // Cluster provides various methods to interact with a cluster. type Cluster interface { - // SetFields will set any dependencies on an object for which the object has implemented the inject - // interface - e.g. inject.Client. - // Deprecated: use the equivalent Options field to set a field. This method will be removed in v0.10. - SetFields(interface{}) error + // GetHTTPClient returns an HTTP client that can be used to talk to the apiserver + GetHTTPClient() *http.Client // GetConfig returns an initialized Config GetConfig() *rest.Config + // GetCache returns a cache.Cache + GetCache() cache.Cache + // GetScheme returns an initialized Scheme GetScheme() *runtime.Scheme @@ -57,9 +60,6 @@ type Cluster interface { // GetFieldIndexer returns a client.FieldIndexer configured with the client GetFieldIndexer() client.FieldIndexer - // GetCache returns a cache.Cache - GetCache() cache.Cache - // GetEventRecorderFor returns a new EventRecorder for the provided name GetEventRecorderFor(name string) record.EventRecorder @@ -83,7 +83,9 @@ type Options struct { Scheme *runtime.Scheme // MapperProvider provides the rest mapper used to map go types to Kubernetes APIs - MapperProvider func(c *rest.Config) (meta.RESTMapper, error) + // + // Deprecated: Set Cache.Mapper and Client.Mapper directly instead. + MapperProvider func(c *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) // Logger is the logger that should be used by this Cluster. // If none is set, it defaults to log.Log global logger. @@ -103,23 +105,54 @@ type Options struct { // Note: If a namespace is specified, controllers can still Watch for a // cluster-scoped resource (e.g Node). For namespaced resources the cache // will only hold objects from the desired namespace. + // + // Deprecated: Use Cache.Namespaces instead. Namespace string + // HTTPClient is the http client that will be used to create the default + // Cache and Client. If not set the rest.HTTPClientFor function will be used + // to create the http client. + HTTPClient *http.Client + + // Cache is the cache.Options that will be used to create the default Cache. + // By default, the cache will watch and list requested objects in all namespaces. + Cache cache.Options + // NewCache is the function that will create the cache to be used // by the manager. If not set this will use the default new cache function. + // + // When using a custom NewCache, the Cache options will be passed to the + // NewCache function. + // + // NOTE: LOW LEVEL PRIMITIVE! + // Only use a custom NewCache if you know what you are doing. NewCache cache.NewCacheFunc + // Client is the client.Options that will be used to create the default Client. + // By default, the client will use the cache for reads and direct calls for writes. + Client client.Options + // NewClient is the func that creates the client to be used by the manager. - // If not set this will create the default DelegatingClient that will - // use the cache for reads and the client for writes. - NewClient NewClientFunc + // If not set this will create a Client backed by a Cache for read operations + // and a direct Client for write operations. + // + // When using a custom NewClient, the Client options will be passed to the + // NewClient function. + // + // NOTE: LOW LEVEL PRIMITIVE! + // Only use a custom NewClient if you know what you are doing. + NewClient client.NewClientFunc // ClientDisableCacheFor tells the client that, if any cache is used, to bypass it // for the given objects. + // + // Deprecated: Use Client.Cache.DisableFor instead. ClientDisableCacheFor []client.Object // DryRunClient specifies whether the client should be configured to enforce // dryRun mode. + // + // Deprecated: Use Client.DryRun instead. DryRunClient bool // EventBroadcaster records Events emitted by the manager and sends them to the Kubernetes API @@ -136,7 +169,7 @@ type Options struct { makeBroadcaster intrec.EventBroadcasterProducer // Dependency injection for testing - newRecorderProvider func(config *rest.Config, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster intrec.EventBroadcasterProducer) (*intrec.Provider, error) + newRecorderProvider func(config *rest.Config, httpClient *http.Client, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster intrec.EventBroadcasterProducer) (*intrec.Provider, error) } // Option can be used to manipulate Options. @@ -152,52 +185,105 @@ func New(config *rest.Config, opts ...Option) (Cluster, error) { for _, opt := range opts { opt(&options) } - options = setOptionsDefaults(options) + options, err := setOptionsDefaults(options, config) + if err != nil { + options.Logger.Error(err, "Failed to set defaults") + return nil, err + } // Create the mapper provider - mapper, err := options.MapperProvider(config) + mapper, err := options.MapperProvider(config, options.HTTPClient) if err != nil { options.Logger.Error(err, "Failed to get API Group-Resources") return nil, err } // Create the cache for the cached read client and registering informers - cache, err := options.NewCache(config, cache.Options{Scheme: options.Scheme, Mapper: mapper, Resync: options.SyncPeriod, Namespace: options.Namespace}) + cacheOpts := options.Cache + { + if cacheOpts.Scheme == nil { + cacheOpts.Scheme = options.Scheme + } + if cacheOpts.Mapper == nil { + cacheOpts.Mapper = mapper + } + if cacheOpts.HTTPClient == nil { + cacheOpts.HTTPClient = options.HTTPClient + } + if cacheOpts.ResyncEvery == nil { + cacheOpts.ResyncEvery = options.SyncPeriod + } + if len(cacheOpts.Namespaces) == 0 && options.Namespace != "" { + cacheOpts.Namespaces = []string{options.Namespace} + } + } + cache, err := options.NewCache(config, cacheOpts) if err != nil { return nil, err } - clientOptions := client.Options{Scheme: options.Scheme, Mapper: mapper} + // Create the client, and default its options. + clientOpts := options.Client + { + if clientOpts.Scheme == nil { + clientOpts.Scheme = options.Scheme + } + if clientOpts.Mapper == nil { + clientOpts.Mapper = mapper + } + if clientOpts.HTTPClient == nil { + clientOpts.HTTPClient = options.HTTPClient + } + if clientOpts.Cache == nil { + clientOpts.Cache = &client.CacheOptions{ + Unstructured: false, + } + } + if clientOpts.Cache.Reader == nil { + clientOpts.Cache.Reader = cache + } + + // For backward compatibility, the ClientDisableCacheFor option should + // be appended to the DisableFor option in the client. + clientOpts.Cache.DisableFor = append(clientOpts.Cache.DisableFor, options.ClientDisableCacheFor...) - apiReader, err := client.New(config, clientOptions) + if clientOpts.DryRun == nil && options.DryRunClient { + // For backward compatibility, the DryRunClient (if set) option should override + // the DryRun option in the client (if unset). + clientOpts.DryRun = pointer.Bool(true) + } + } + clientWriter, err := options.NewClient(config, clientOpts) if err != nil { return nil, err } - writeObj, err := options.NewClient(cache, config, clientOptions, options.ClientDisableCacheFor...) + // Create the API Reader, a client with no cache. + clientReader, err := client.New(config, client.Options{ + HTTPClient: options.HTTPClient, + Scheme: options.Scheme, + Mapper: mapper, + }) if err != nil { return nil, err } - if options.DryRunClient { - writeObj = client.NewDryRunClient(writeObj) - } - // Create the recorder provider to inject event recorders for the components. // TODO(directxman12): the log for the event provider should have a context (name, tags, etc) specific // to the particular controller that it's being injected into, rather than a generic one like is here. - recorderProvider, err := options.newRecorderProvider(config, options.Scheme, options.Logger.WithName("events"), options.makeBroadcaster) + recorderProvider, err := options.newRecorderProvider(config, options.HTTPClient, options.Scheme, options.Logger.WithName("events"), options.makeBroadcaster) if err != nil { return nil, err } return &cluster{ config: config, + httpClient: options.HTTPClient, scheme: options.Scheme, cache: cache, fieldIndexes: cache, - client: writeObj, - apiReader: apiReader, + client: clientWriter, + apiReader: clientReader, recorderProvider: recorderProvider, mapper: mapper, logger: options.Logger, @@ -205,21 +291,29 @@ func New(config *rest.Config, opts ...Option) (Cluster, error) { } // setOptionsDefaults set default values for Options fields. -func setOptionsDefaults(options Options) Options { +func setOptionsDefaults(options Options, config *rest.Config) (Options, error) { + if options.HTTPClient == nil { + var err error + options.HTTPClient, err = rest.HTTPClientFor(config) + if err != nil { + return options, err + } + } + // Use the Kubernetes client-go scheme if none is specified if options.Scheme == nil { options.Scheme = scheme.Scheme } if options.MapperProvider == nil { - options.MapperProvider = func(c *rest.Config) (meta.RESTMapper, error) { - return apiutil.NewDynamicRESTMapper(c) + options.MapperProvider = func(c *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) { + return apiutil.NewDynamicRESTMapper(c, httpClient) } } // Allow users to define how to create a new client if options.NewClient == nil { - options.NewClient = DefaultNewClient + options.NewClient = client.New } // Allow newCache to be mocked @@ -245,26 +339,9 @@ func setOptionsDefaults(options Options) Options { } } - if options.Logger == nil { + if options.Logger.GetSink() == nil { options.Logger = logf.RuntimeLog.WithName("cluster") } - return options -} - -// NewClientFunc allows a user to define how to create a client. -type NewClientFunc func(cache cache.Cache, config *rest.Config, options client.Options, uncachedObjects ...client.Object) (client.Client, error) - -// DefaultNewClient creates the default caching client. -func DefaultNewClient(cache cache.Cache, config *rest.Config, options client.Options, uncachedObjects ...client.Object) (client.Client, error) { - c, err := client.New(config, options) - if err != nil { - return nil, err - } - - return client.NewDelegatingClient(client.NewDelegatingClientInput{ - CacheReader: cache, - Client: c, - UncachedObjects: uncachedObjects, - }) + return options, nil } diff --git a/pkg/cluster/cluster_suite_test.go b/pkg/cluster/cluster_suite_test.go index 8e9b7ef2ac..dc1f9ac778 100644 --- a/pkg/cluster/cluster_suite_test.go +++ b/pkg/cluster/cluster_suite_test.go @@ -20,20 +20,18 @@ import ( "net/http" "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Cluster Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Cluster Suite") } var testenv *envtest.Environment @@ -43,7 +41,7 @@ var clientset *kubernetes.Clientset // clientTransport is used to force-close keep-alives in tests that check for leaks. var clientTransport *http.Transport -var _ = BeforeSuite(func(done Done) { +var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) testenv = &envtest.Environment{} @@ -63,9 +61,7 @@ var _ = BeforeSuite(func(done Done) { clientset, err = kubernetes.NewForConfig(cfg) Expect(err).NotTo(HaveOccurred()) - - close(done) -}, 60) +}) var _ = AfterSuite(func() { Expect(testenv.Stop()).To(Succeed()) diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index d0b03785c3..dc52b2d9b3 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -20,20 +20,18 @@ import ( "context" "errors" "fmt" + "net/http" "github.com/go-logr/logr" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "go.uber.org/goleak" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/cache/informertest" "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" ) var _ = Describe("cluster.Cluster", func() { @@ -48,27 +46,25 @@ var _ = Describe("cluster.Cluster", func() { It("should return an error if it can't create a RestMapper", func() { expected := fmt.Errorf("expected error: RestMapper") c, err := New(cfg, func(o *Options) { - o.MapperProvider = func(c *rest.Config) (meta.RESTMapper, error) { return nil, expected } + o.MapperProvider = func(c *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) { return nil, expected } }) Expect(c).To(BeNil()) Expect(err).To(Equal(expected)) }) - It("should return an error it can't create a client.Client", func(done Done) { + It("should return an error it can't create a client.Client", func() { c, err := New(cfg, func(o *Options) { - o.NewClient = func(cache cache.Cache, config *rest.Config, options client.Options, uncachedObjects ...client.Object) (client.Client, error) { + o.NewClient = func(config *rest.Config, options client.Options) (client.Client, error) { return nil, errors.New("expected error") } }) Expect(c).To(BeNil()) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("expected error")) - - close(done) }) - It("should return an error it can't create a cache.Cache", func(done Done) { + It("should return an error it can't create a cache.Cache", func() { c, err := New(cfg, func(o *Options) { o.NewCache = func(config *rest.Config, opts cache.Options) (cache.Cache, error) { return nil, fmt.Errorf("expected error") @@ -77,121 +73,39 @@ var _ = Describe("cluster.Cluster", func() { Expect(c).To(BeNil()) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("expected error")) - - close(done) }) - It("should create a client defined in by the new client function", func(done Done) { + It("should create a client defined in by the new client function", func() { c, err := New(cfg, func(o *Options) { - o.NewClient = func(cache cache.Cache, config *rest.Config, options client.Options, uncachedObjects ...client.Object) (client.Client, error) { + o.NewClient = func(config *rest.Config, options client.Options) (client.Client, error) { return nil, nil } }) Expect(c).ToNot(BeNil()) Expect(err).ToNot(HaveOccurred()) Expect(c.GetClient()).To(BeNil()) - - close(done) }) - It("should return an error it can't create a recorder.Provider", func(done Done) { + It("should return an error it can't create a recorder.Provider", func() { c, err := New(cfg, func(o *Options) { - o.newRecorderProvider = func(_ *rest.Config, _ *runtime.Scheme, _ logr.Logger, _ intrec.EventBroadcasterProducer) (*intrec.Provider, error) { + o.newRecorderProvider = func(_ *rest.Config, _ *http.Client, _ *runtime.Scheme, _ logr.Logger, _ intrec.EventBroadcasterProducer) (*intrec.Provider, error) { return nil, fmt.Errorf("expected error") } }) Expect(c).To(BeNil()) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("expected error")) - - close(done) }) }) Describe("Start", func() { - It("should stop when context is cancelled", func(done Done) { + It("should stop when context is cancelled", func() { c, err := New(cfg) Expect(err).NotTo(HaveOccurred()) ctx, cancel := context.WithCancel(context.Background()) cancel() Expect(c.Start(ctx)).NotTo(HaveOccurred()) - - close(done) - }) - }) - - Describe("SetFields", func() { - It("should inject field values", func(done Done) { - c, err := New(cfg, func(o *Options) { - o.NewCache = func(_ *rest.Config, _ cache.Options) (cache.Cache, error) { - return &informertest.FakeInformers{}, nil - } - }) - Expect(err).NotTo(HaveOccurred()) - - By("Injecting the dependencies") - err = c.SetFields(&injectable{ - scheme: func(scheme *runtime.Scheme) error { - defer GinkgoRecover() - Expect(scheme).To(Equal(c.GetScheme())) - return nil - }, - config: func(config *rest.Config) error { - defer GinkgoRecover() - Expect(config).To(Equal(c.GetConfig())) - return nil - }, - client: func(client client.Client) error { - defer GinkgoRecover() - Expect(client).To(Equal(c.GetClient())) - return nil - }, - cache: func(cache cache.Cache) error { - defer GinkgoRecover() - Expect(cache).To(Equal(c.GetCache())) - return nil - }, - log: func(logger logr.Logger) error { - defer GinkgoRecover() - Expect(logger).To(Equal(logf.RuntimeLog.WithName("cluster"))) - return nil - }, - }) - Expect(err).NotTo(HaveOccurred()) - - By("Returning an error if dependency injection fails") - - expected := fmt.Errorf("expected error") - err = c.SetFields(&injectable{ - client: func(client client.Client) error { - return expected - }, - }) - Expect(err).To(Equal(expected)) - - err = c.SetFields(&injectable{ - scheme: func(scheme *runtime.Scheme) error { - return expected - }, - }) - Expect(err).To(Equal(expected)) - - err = c.SetFields(&injectable{ - config: func(config *rest.Config) error { - return expected - }, - }) - Expect(err).To(Equal(expected)) - - err = c.SetFields(&injectable{ - cache: func(c cache.Cache) error { - return expected - }, - }) - Expect(err).To(Equal(expected)) - - close(done) }) }) @@ -254,56 +168,3 @@ var _ = Describe("cluster.Cluster", func() { Expect(c.GetAPIReader()).NotTo(BeNil()) }) }) - -var _ inject.Cache = &injectable{} -var _ inject.Client = &injectable{} -var _ inject.Scheme = &injectable{} -var _ inject.Config = &injectable{} -var _ inject.Logger = &injectable{} - -type injectable struct { - scheme func(scheme *runtime.Scheme) error - client func(client.Client) error - config func(config *rest.Config) error - cache func(cache.Cache) error - log func(logger logr.Logger) error -} - -func (i *injectable) InjectCache(c cache.Cache) error { - if i.cache == nil { - return nil - } - return i.cache(c) -} - -func (i *injectable) InjectConfig(config *rest.Config) error { - if i.config == nil { - return nil - } - return i.config(config) -} - -func (i *injectable) InjectClient(c client.Client) error { - if i.client == nil { - return nil - } - return i.client(c) -} - -func (i *injectable) InjectScheme(scheme *runtime.Scheme) error { - if i.scheme == nil { - return nil - } - return i.scheme(scheme) -} - -func (i *injectable) InjectLogger(log logr.Logger) error { - if i.log == nil { - return nil - } - return i.log(log) -} - -func (i *injectable) Start(<-chan struct{}) error { - return nil -} diff --git a/pkg/cluster/internal.go b/pkg/cluster/internal.go index 125e1d144e..2742764231 100644 --- a/pkg/cluster/internal.go +++ b/pkg/cluster/internal.go @@ -18,6 +18,7 @@ package cluster import ( "context" + "net/http" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/api/meta" @@ -28,22 +29,16 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" ) type cluster struct { // config is the rest.config used to talk to the apiserver. Required. config *rest.Config - // scheme is the scheme injected into Controllers, EventHandlers, Sources and Predicates. Defaults - // to scheme.scheme. - scheme *runtime.Scheme - - cache cache.Cache - - // TODO(directxman12): Provide an escape hatch to get individual indexers - // client is the client injected into Controllers (and EventHandlers, Sources and Predicates). - client client.Client + httpClient *http.Client + scheme *runtime.Scheme + cache cache.Cache + client client.Client // apiReader is the reader that will make requests to the api server and not the cache. apiReader client.Reader @@ -64,32 +59,14 @@ type cluster struct { logger logr.Logger } -func (c *cluster) SetFields(i interface{}) error { - if _, err := inject.ConfigInto(c.config, i); err != nil { - return err - } - if _, err := inject.ClientInto(c.client, i); err != nil { - return err - } - if _, err := inject.APIReaderInto(c.apiReader, i); err != nil { - return err - } - if _, err := inject.SchemeInto(c.scheme, i); err != nil { - return err - } - if _, err := inject.CacheInto(c.cache, i); err != nil { - return err - } - if _, err := inject.MapperInto(c.mapper, i); err != nil { - return err - } - return nil -} - func (c *cluster) GetConfig() *rest.Config { return c.config } +func (c *cluster) GetHTTPClient() *http.Client { + return c.httpClient +} + func (c *cluster) GetClient() client.Client { return c.client } diff --git a/pkg/config/config.go b/pkg/config/config.go index f23b02df00..9c7b875a86 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -18,7 +18,7 @@ package config import ( "fmt" - ioutil "io/ioutil" + "os" "sync" "k8s.io/apimachinery/pkg/runtime" @@ -29,6 +29,8 @@ import ( // ControllerManagerConfiguration defines the functions necessary to parse a config file // and to configure the Options struct for the ctrl.Manager. +// +// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. type ControllerManagerConfiguration interface { runtime.Object @@ -38,6 +40,8 @@ type ControllerManagerConfiguration interface { // DeferredFileLoader is used to configure the decoder for loading controller // runtime component config types. +// +// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. type DeferredFileLoader struct { ControllerManagerConfiguration path string @@ -50,8 +54,10 @@ type DeferredFileLoader struct { // this will also configure the defaults for the loader if nothing is // // Defaults: -// Path: "./config.yaml" -// Kind: GenericControllerManagerConfiguration +// * Path: "./config.yaml" +// * Kind: GenericControllerManagerConfiguration +// +// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. func File() *DeferredFileLoader { scheme := runtime.NewScheme() utilruntime.Must(v1alpha1.AddToScheme(scheme)) @@ -83,12 +89,6 @@ func (d *DeferredFileLoader) OfKind(obj ControllerManagerConfiguration) *Deferre return d } -// InjectScheme will configure the scheme to be used for decoding the file. -func (d *DeferredFileLoader) InjectScheme(scheme *runtime.Scheme) error { - d.scheme = scheme - return nil -} - // loadFile is used from the mutex.Once to load the file. func (d *DeferredFileLoader) loadFile() { if d.scheme == nil { @@ -96,7 +96,7 @@ func (d *DeferredFileLoader) loadFile() { return } - content, err := ioutil.ReadFile(d.path) + content, err := os.ReadFile(d.path) if err != nil { d.err = fmt.Errorf("could not read file at %s", d.path) return diff --git a/pkg/config/config_suite_test.go b/pkg/config/config_suite_test.go index 9a494dafbc..8df933ba9d 100644 --- a/pkg/config/config_suite_test.go +++ b/pkg/config/config_suite_test.go @@ -1,12 +1,9 @@ /* Copyright 2018 The Kubernetes Authors. - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,14 +16,11 @@ package config_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) func TestScheme(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Config Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Config Suite") } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index f2b5461b55..329c0296c5 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -1,12 +1,9 @@ /* Copyright 2020 The Kubernetes Authors. - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,7 +14,7 @@ limitations under the License. package config_test import ( - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" diff --git a/pkg/config/controller.go b/pkg/config/controller.go new file mode 100644 index 0000000000..b37dffaeea --- /dev/null +++ b/pkg/config/controller.go @@ -0,0 +1,49 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import "time" + +// Controller contains configuration options for a controller. +type Controller struct { + // GroupKindConcurrency is a map from a Kind to the number of concurrent reconciliation + // allowed for that controller. + // + // When a controller is registered within this manager using the builder utilities, + // users have to specify the type the controller reconciles in the For(...) call. + // If the object's kind passed matches one of the keys in this map, the concurrency + // for that controller is set to the number specified. + // + // The key is expected to be consistent in form with GroupKind.String(), + // e.g. ReplicaSet in apps group (regardless of version) would be `ReplicaSet.apps`. + GroupKindConcurrency map[string]int + + // MaxConcurrentReconciles is the maximum number of concurrent Reconciles which can be run. Defaults to 1. + MaxConcurrentReconciles int + + // CacheSyncTimeout refers to the time limit set to wait for syncing caches. + // Defaults to 2 minutes if not set. + CacheSyncTimeout time.Duration + + // RecoverPanic indicates whether the panic caused by reconcile should be recovered. + // Defaults to the Controller.RecoverPanic setting from the Manager if unset. + RecoverPanic *bool + + // NeedLeaderElection indicates whether the controller needs to use leader election. + // Defaults to true, which means the controller will use leader election. + NeedLeaderElection *bool +} diff --git a/pkg/config/doc.go b/pkg/config/doc.go index ebd8243f32..47a5a2f1d7 100644 --- a/pkg/config/doc.go +++ b/pkg/config/doc.go @@ -14,12 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package config contains functionality for interacting with ComponentConfig -// files -// -// DeferredFileLoader -// -// This uses a deferred file decoding allowing you to chain your configuration -// setup. You can pass this into manager.Options#File and it will load your -// config. +// Package config contains functionality for interacting with +// configuration for controller-runtime components. package config diff --git a/pkg/config/example_test.go b/pkg/config/example_test.go index fb1cd58b5f..3d80d68eff 100644 --- a/pkg/config/example_test.go +++ b/pkg/config/example_test.go @@ -1,12 +1,9 @@ /* Copyright 2020 The Kubernetes Authors. - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -44,41 +41,10 @@ func ExampleFile() { } // This example will load the file from a custom path. -func ExampleDeferredFileLoader_atPath() { +func ExampleFile_atPath() { loader := config.File().AtPath("/var/run/controller-runtime/config.yaml") if _, err := loader.Complete(); err != nil { fmt.Println("failed to load config") os.Exit(1) } } - -// This example sets up loader with a custom scheme. -func ExampleDeferredFileLoader_injectScheme() { - loader := config.File() - err := loader.InjectScheme(scheme) - if err != nil { - fmt.Println("failed to inject scheme") - os.Exit(1) - } - - _, err = loader.Complete() - if err != nil { - fmt.Println("failed to load config") - os.Exit(1) - } -} - -// This example sets up the loader with a custom scheme and custom type. -func ExampleDeferredFileLoader_ofKind() { - loader := config.File().OfKind(&v1alpha1.CustomControllerManagerConfiguration{}) - err := loader.InjectScheme(scheme) - if err != nil { - fmt.Println("failed to inject scheme") - os.Exit(1) - } - _, err = loader.Complete() - if err != nil { - fmt.Println("failed to load config") - os.Exit(1) - } -} diff --git a/pkg/config/v1alpha1/doc.go b/pkg/config/v1alpha1/doc.go index 1e3adbafb8..8fdf14d39a 100644 --- a/pkg/config/v1alpha1/doc.go +++ b/pkg/config/v1alpha1/doc.go @@ -17,4 +17,6 @@ limitations under the License. // Package v1alpha1 provides the ControllerManagerConfiguration used for // configuring ctrl.Manager // +kubebuilder:object:generate=true +// +// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. package v1alpha1 diff --git a/pkg/config/v1alpha1/register.go b/pkg/config/v1alpha1/register.go index 9efdbc0668..ca854bcf30 100644 --- a/pkg/config/v1alpha1/register.go +++ b/pkg/config/v1alpha1/register.go @@ -23,12 +23,18 @@ import ( var ( // GroupVersion is group version used to register these objects. + // + // Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. GroupVersion = schema.GroupVersion{Group: "controller-runtime.sigs.k8s.io", Version: "v1alpha1"} // SchemeBuilder is used to add go types to the GroupVersionKind scheme. + // + // Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} // AddToScheme adds the types in this group-version to the given scheme. + // + // Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. AddToScheme = SchemeBuilder.AddToScheme ) diff --git a/pkg/config/v1alpha1/types.go b/pkg/config/v1alpha1/types.go index e67b62e514..52c8ab300f 100644 --- a/pkg/config/v1alpha1/types.go +++ b/pkg/config/v1alpha1/types.go @@ -25,6 +25,8 @@ import ( ) // ControllerManagerConfigurationSpec defines the desired state of GenericControllerManagerConfiguration. +// +// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. type ControllerManagerConfigurationSpec struct { // SyncPeriod determines the minimum frequency at which watched resources are // reconciled. A lower period will correct entropy more quickly, but reduce @@ -60,7 +62,7 @@ type ControllerManagerConfigurationSpec struct { // +optional Controller *ControllerConfigurationSpec `json:"controller,omitempty"` - // Metrics contains thw controller metrics configuration + // Metrics contains the controller metrics configuration // +optional Metrics ControllerMetrics `json:"metrics,omitempty"` @@ -75,6 +77,11 @@ type ControllerManagerConfigurationSpec struct { // ControllerConfigurationSpec defines the global configuration for // controllers registered with the manager. +// +// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. +// +// Deprecated: Controller global configuration can now be set at the manager level, +// using the manager.Options.Controller field. type ControllerConfigurationSpec struct { // GroupKindConcurrency is a map from a Kind to the number of concurrent reconciliation // allowed for that controller. @@ -94,9 +101,15 @@ type ControllerConfigurationSpec struct { // Defaults to 2 minutes if not set. // +optional CacheSyncTimeout *time.Duration `json:"cacheSyncTimeout,omitempty"` + + // RecoverPanic indicates if panics should be recovered. + // +optional + RecoverPanic *bool `json:"recoverPanic,omitempty"` } // ControllerMetrics defines the metrics configs. +// +// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. type ControllerMetrics struct { // BindAddress is the TCP address that the controller should bind to // for serving prometheus metrics. @@ -106,9 +119,12 @@ type ControllerMetrics struct { } // ControllerHealth defines the health configs. +// +// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. type ControllerHealth struct { // HealthProbeBindAddress is the TCP address that the controller should bind to // for serving health probes + // It can be set to "0" or "" to disable serving the health probe. // +optional HealthProbeBindAddress string `json:"healthProbeBindAddress,omitempty"` @@ -122,6 +138,8 @@ type ControllerHealth struct { } // ControllerWebhook defines the webhook server for the controller. +// +// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. type ControllerWebhook struct { // Port is the port that the webhook server serves at. // It is used to set webhook.Server.Port. @@ -144,6 +162,8 @@ type ControllerWebhook struct { // +kubebuilder:object:root=true // ControllerManagerConfiguration is the Schema for the GenericControllerManagerConfigurations API. +// +// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. type ControllerManagerConfiguration struct { metav1.TypeMeta `json:",inline"` @@ -152,6 +172,8 @@ type ControllerManagerConfiguration struct { } // Complete returns the configuration for controller-runtime. +// +// Deprecated: The component config package has been deprecated and will be removed in a future release. Users should migrate to their own config implementation, please share feedback in https://github.com/kubernetes-sigs/controller-runtime/issues/895. func (c *ControllerManagerConfigurationSpec) Complete() (ControllerManagerConfigurationSpec, error) { return *c, nil } diff --git a/pkg/config/v1alpha1/zz_generated.deepcopy.go b/pkg/config/v1alpha1/zz_generated.deepcopy.go index 752fa9754c..cdc7c334be 100644 --- a/pkg/config/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/config/v1alpha1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated // Code generated by controller-gen. DO NOT EDIT. @@ -26,6 +27,11 @@ func (in *ControllerConfigurationSpec) DeepCopyInto(out *ControllerConfiguration *out = new(timex.Duration) **out = **in } + if in.RecoverPanic != nil { + in, out := &in.RecoverPanic, &out.RecoverPanic + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerConfigurationSpec. diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index c9e07562a3..f2652d10a4 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -23,6 +23,9 @@ import ( "github.com/go-logr/logr" "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + + "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/internal/controller" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -34,8 +37,7 @@ import ( // Options are the arguments for creating a new Controller. type Options struct { - // MaxConcurrentReconciles is the maximum number of concurrent Reconciles which can be run. Defaults to 1. - MaxConcurrentReconciles int + config.Controller // Reconciler reconciles an object Reconciler reconcile.Reconciler @@ -45,13 +47,9 @@ type Options struct { // The overall is a token bucket and the per-item is exponential. RateLimiter ratelimiter.RateLimiter - // Log is the logger used for this controller and passed to each reconciliation - // request via the context field. - Log logr.Logger - - // CacheSyncTimeout refers to the time limit set to wait for syncing caches. - // Defaults to 2 minutes if not set. - CacheSyncTimeout time.Duration + // LogConstructor is used to construct a logger used for this controller and passed + // to each reconciliation via the context field. + LogConstructor func(request *reconcile.Request) logr.Logger } // Controller implements a Kubernetes API. A Controller manages a work queue fed reconcile.Requests @@ -101,8 +99,20 @@ func NewUnmanaged(name string, mgr manager.Manager, options Options) (Controller return nil, fmt.Errorf("must specify Name for Controller") } - if options.Log == nil { - options.Log = mgr.GetLogger() + if options.LogConstructor == nil { + log := mgr.GetLogger().WithValues( + "controller", name, + ) + options.LogConstructor = func(req *reconcile.Request) logr.Logger { + log := log + if req != nil { + log = log.WithValues( + "object", klog.KRef(req.Namespace, req.Name), + "namespace", req.Namespace, "name", req.Name, + ) + } + return log + } } if options.MaxConcurrentReconciles <= 0 { @@ -117,9 +127,8 @@ func NewUnmanaged(name string, mgr manager.Manager, options Options) (Controller options.RateLimiter = workqueue.DefaultControllerRateLimiter() } - // Inject dependencies into Reconciler - if err := mgr.SetFields(options.Reconciler); err != nil { - return nil, err + if options.RecoverPanic == nil { + options.RecoverPanic = mgr.GetControllerOptions().RecoverPanic } // Create controller with dependencies set @@ -130,8 +139,12 @@ func NewUnmanaged(name string, mgr manager.Manager, options Options) (Controller }, MaxConcurrentReconciles: options.MaxConcurrentReconciles, CacheSyncTimeout: options.CacheSyncTimeout, - SetFields: mgr.SetFields, Name: name, - Log: options.Log.WithName("controller").WithName(name), + LogConstructor: options.LogConstructor, + RecoverPanic: options.RecoverPanic, + LeaderElected: options.NeedLeaderElection, }, nil } + +// ReconcileIDFromContext gets the reconcileID from the current context. +var ReconcileIDFromContext = controller.ReconcileIDFromContext diff --git a/pkg/controller/controller_integration_test.go b/pkg/controller/controller_integration_test.go index 762b3d9fbb..48facf1e94 100644 --- a/pkg/controller/controller_integration_test.go +++ b/pkg/controller/controller_integration_test.go @@ -31,7 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "sigs.k8s.io/controller-runtime/pkg/manager" ) @@ -48,7 +48,7 @@ var _ = Describe("controller", func() { Describe("controller", func() { // TODO(directxman12): write a whole suite of controller-client interaction tests - It("should reconcile", func(done Done) { + It("should reconcile", func() { By("Creating the Manager") cm, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -64,12 +64,13 @@ var _ = Describe("controller", func() { Expect(err).NotTo(HaveOccurred()) By("Watching Resources") - err = instance.Watch(&source.Kind{Type: &appsv1.ReplicaSet{}}, &handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.Deployment{}, - }) + err = instance.Watch( + source.Kind(cm.GetCache(), &appsv1.ReplicaSet{}), + handler.EnqueueRequestForOwner(cm.GetScheme(), cm.GetRESTMapper(), &appsv1.Deployment{}), + ) Expect(err).NotTo(HaveOccurred()) - err = instance.Watch(&source.Kind{Type: &appsv1.Deployment{}}, &handler.EnqueueRequestForObject{}) + err = instance.Watch(source.Kind(cm.GetCache(), &appsv1.Deployment{}), &handler.EnqueueRequestForObject{}) Expect(err).NotTo(HaveOccurred()) err = cm.GetClient().Get(ctx, types.NamespacedName{Name: "foo"}, &corev1.Namespace{}) @@ -169,9 +170,7 @@ var _ = Describe("controller", func() { err = cm.GetClient(). List(context.Background(), &controllertest.UnconventionalListTypeList{}) Expect(err).NotTo(HaveOccurred()) - - close(done) - }, 5) + }) }) }) diff --git a/pkg/controller/controller_suite_test.go b/pkg/controller/controller_suite_test.go index 363c1076b6..af818d12cd 100644 --- a/pkg/controller/controller_suite_test.go +++ b/pkg/controller/controller_suite_test.go @@ -20,7 +20,7 @@ import ( "net/http" "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes" @@ -29,7 +29,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllertest" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics" @@ -38,8 +37,7 @@ import ( func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Controller Integration Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Controller Integration Suite") } var testenv *envtest.Environment @@ -49,7 +47,7 @@ var clientset *kubernetes.Clientset // clientTransport is used to force-close keep-alives in tests that check for leaks. var clientTransport *http.Transport -var _ = BeforeSuite(func(done Done) { +var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) err := (&crscheme.Builder{ @@ -82,9 +80,7 @@ var _ = BeforeSuite(func(done Done) { // Prevent the metrics listener being created metrics.DefaultBindAddress = "0" - - close(done) -}, 60) +}) var _ = AfterSuite(func() { Expect(testenv.Stop()).To(Succeed()) diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index 4b0a2ee914..5f20f87f1c 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -18,21 +18,21 @@ package controller_test import ( "context" - "fmt" "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "go.uber.org/goleak" corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" - "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" + internalcontroller "sigs.k8s.io/controller-runtime/pkg/internal/controller" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/source" ) @@ -42,40 +42,24 @@ var _ = Describe("controller.Controller", func() { }) Describe("New", func() { - It("should return an error if Name is not Specified", func(done Done) { + It("should return an error if Name is not Specified", func() { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) c, err := controller.New("", m, controller.Options{Reconciler: rec}) Expect(c).To(BeNil()) Expect(err.Error()).To(ContainSubstring("must specify Name for Controller")) - - close(done) }) - It("should return an error if Reconciler is not Specified", func(done Done) { + It("should return an error if Reconciler is not Specified", func() { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) c, err := controller.New("foo", m, controller.Options{}) Expect(c).To(BeNil()) Expect(err.Error()).To(ContainSubstring("must specify Reconciler")) - - close(done) - }) - - It("NewController should return an error if injecting Reconciler fails", func(done Done) { - m, err := manager.New(cfg, manager.Options{}) - Expect(err).NotTo(HaveOccurred()) - - c, err := controller.New("foo", m, controller.Options{Reconciler: &failRec{}}) - Expect(c).To(BeNil()) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("expected error")) - - close(done) }) - It("should not return an error if two controllers are registered with different names", func(done Done) { + It("should not return an error if two controllers are registered with different names", func() { m, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -86,13 +70,12 @@ var _ = Describe("controller.Controller", func() { c2, err := controller.New("c2", m, controller.Options{Reconciler: rec}) Expect(err).NotTo(HaveOccurred()) Expect(c2).ToNot(BeNil()) - - close(done) }) It("should not leak goroutines when stopped", func() { currentGRs := goleak.IgnoreCurrent() + ctx, cancel := context.WithCancel(context.Background()) watchChan := make(chan event.GenericEvent, 1) watch := &source.Channel{Source: watchChan} watchChan <- event.GenericEvent{Object: &corev1.Pod{}} @@ -119,7 +102,6 @@ var _ = Describe("controller.Controller", func() { Expect(c.Watch(watch, &handler.EnqueueRequestForObject{})).To(Succeed()) Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.Background()) go func() { defer GinkgoRecover() Expect(m.Start(ctx)).To(Succeed()) @@ -150,18 +132,86 @@ var _ = Describe("controller.Controller", func() { clientTransport.CloseIdleConnections() Eventually(func() error { return goleak.Find(currentGRs) }).Should(Succeed()) }) - }) -}) -var _ reconcile.Reconciler = &failRec{} -var _ inject.Client = &failRec{} + It("should default RecoverPanic from the manager", func() { + m, err := manager.New(cfg, manager.Options{Controller: config.Controller{RecoverPanic: pointer.Bool(true)}}) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller", m, controller.Options{ + Reconciler: reconcile.Func(nil), + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller) + Expect(ok).To(BeTrue()) + + Expect(ctrl.RecoverPanic).NotTo(BeNil()) + Expect(*ctrl.RecoverPanic).To(BeTrue()) + }) + + It("should not override RecoverPanic on the controller", func() { + m, err := manager.New(cfg, manager.Options{Controller: config.Controller{RecoverPanic: pointer.Bool(true)}}) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller", m, controller.Options{ + Controller: config.Controller{ + RecoverPanic: pointer.Bool(false), + }, + Reconciler: reconcile.Func(nil), + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller) + Expect(ok).To(BeTrue()) -type failRec struct{} + Expect(ctrl.RecoverPanic).NotTo(BeNil()) + Expect(*ctrl.RecoverPanic).To(BeFalse()) + }) + + It("should default NeedLeaderElection on the controller to true", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller", m, controller.Options{ + Reconciler: rec, + }) + Expect(err).NotTo(HaveOccurred()) -func (*failRec) Reconcile(context.Context, reconcile.Request) (reconcile.Result, error) { - return reconcile.Result{}, nil -} + ctrl, ok := c.(*internalcontroller.Controller) + Expect(ok).To(BeTrue()) -func (*failRec) InjectClient(client.Client) error { - return fmt.Errorf("expected error") -} + Expect(ctrl.NeedLeaderElection()).To(BeTrue()) + }) + + It("should allow for setting leaderElected to false", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller", m, controller.Options{ + Controller: config.Controller{ + NeedLeaderElection: pointer.Bool(false), + }, + Reconciler: rec, + }) + Expect(err).NotTo(HaveOccurred()) + + ctrl, ok := c.(*internalcontroller.Controller) + Expect(ok).To(BeTrue()) + + Expect(ctrl.NeedLeaderElection()).To(BeFalse()) + }) + + It("should implement manager.LeaderElectionRunnable", func() { + m, err := manager.New(cfg, manager.Options{}) + Expect(err).NotTo(HaveOccurred()) + + c, err := controller.New("new-controller", m, controller.Options{ + Reconciler: rec, + }) + Expect(err).NotTo(HaveOccurred()) + + _, ok := c.(manager.LeaderElectionRunnable) + Expect(ok).To(BeTrue()) + }) + }) +}) diff --git a/pkg/controller/controllertest/util.go b/pkg/controller/controllertest/util.go index df6ae9a223..17641e9c02 100644 --- a/pkg/controller/controllertest/util.go +++ b/pkg/controller/controllertest/util.go @@ -56,9 +56,10 @@ func (f *FakeInformer) HasSynced() bool { return f.Synced } -// AddEventHandler implements the Informer interface. Adds an EventHandler to the fake Informers. -func (f *FakeInformer) AddEventHandler(handler cache.ResourceEventHandler) { +// AddEventHandler implements the Informer interface. Adds an EventHandler to the fake Informers. TODO(community): Implement Registration. +func (f *FakeInformer) AddEventHandler(handler cache.ResourceEventHandler) (cache.ResourceEventHandlerRegistration, error) { f.handlers = append(f.handlers, handler) + return nil, nil } // Run implements the Informer interface. Increments f.RunCount. @@ -88,8 +89,13 @@ func (f *FakeInformer) Delete(obj metav1.Object) { } // AddEventHandlerWithResyncPeriod does nothing. TODO(community): Implement this. -func (f *FakeInformer) AddEventHandlerWithResyncPeriod(handler cache.ResourceEventHandler, resyncPeriod time.Duration) { +func (f *FakeInformer) AddEventHandlerWithResyncPeriod(handler cache.ResourceEventHandler, resyncPeriod time.Duration) (cache.ResourceEventHandlerRegistration, error) { + return nil, nil +} +// RemoveEventHandler does nothing. TODO(community): Implement this. +func (f *FakeInformer) RemoveEventHandler(handle cache.ResourceEventHandlerRegistration) error { + return nil } // GetStore does nothing. TODO(community): Implement this. @@ -111,3 +117,13 @@ func (f *FakeInformer) LastSyncResourceVersion() string { func (f *FakeInformer) SetWatchErrorHandler(cache.WatchErrorHandler) error { return nil } + +// SetTransform does nothing. TODO(community): Implement this. +func (f *FakeInformer) SetTransform(t cache.TransformFunc) error { + return nil +} + +// IsStopped does nothing. TODO(community): Implement this. +func (f *FakeInformer) IsStopped() bool { + return false +} diff --git a/pkg/controller/controllerutil/controllerutil.go b/pkg/controller/controllerutil/controllerutil.go index 7863c3deec..344abcd288 100644 --- a/pkg/controller/controllerutil/controllerutil.go +++ b/pkg/controller/controllerutil/controllerutil.go @@ -76,8 +76,8 @@ func SetControllerReference(owner, controlled metav1.Object, scheme *runtime.Sch Kind: gvk.Kind, Name: owner.GetName(), UID: owner.GetUID(), - BlockOwnerDeletion: pointer.BoolPtr(true), - Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.Bool(true), + Controller: pointer.Bool(true), } // Return early with an error if the object is already controlled. @@ -207,7 +207,7 @@ func CreateOrUpdate(ctx context.Context, c client.Client, obj client.Object, f M return OperationResultCreated, nil } - existing := obj.DeepCopyObject() //nolint:ifshort + existing := obj.DeepCopyObject() if err := mutate(f, key, obj); err != nil { return OperationResultNone, err } @@ -345,30 +345,35 @@ func mutate(f MutateFn, key client.ObjectKey, obj client.Object) error { return nil } -// MutateFn is a function which mutates the existing object into it's desired state. +// MutateFn is a function which mutates the existing object into its desired state. type MutateFn func() error // AddFinalizer accepts an Object and adds the provided finalizer if not present. -func AddFinalizer(o client.Object, finalizer string) { +// It returns an indication of whether it updated the object's list of finalizers. +func AddFinalizer(o client.Object, finalizer string) (finalizersUpdated bool) { f := o.GetFinalizers() for _, e := range f { if e == finalizer { - return + return false } } o.SetFinalizers(append(f, finalizer)) + return true } // RemoveFinalizer accepts an Object and removes the provided finalizer if present. -func RemoveFinalizer(o client.Object, finalizer string) { +// It returns an indication of whether it updated the object's list of finalizers. +func RemoveFinalizer(o client.Object, finalizer string) (finalizersUpdated bool) { f := o.GetFinalizers() for i := 0; i < len(f); i++ { if f[i] == finalizer { f = append(f[:i], f[i+1:]...) i-- + finalizersUpdated = true } } o.SetFinalizers(f) + return } // ContainsFinalizer checks an Object that the provided finalizer is present. diff --git a/pkg/controller/controllerutil/controllerutil_suite_test.go b/pkg/controller/controllerutil/controllerutil_suite_test.go index 0cfd6d1e02..a4ac5cc746 100644 --- a/pkg/controller/controllerutil/controllerutil_suite_test.go +++ b/pkg/controller/controllerutil/controllerutil_suite_test.go @@ -19,31 +19,29 @@ package controllerutil_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) func TestControllerutil(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Controllerutil Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Controllerutil Suite") } -var t *envtest.Environment +var testenv *envtest.Environment var cfg *rest.Config var c client.Client var _ = BeforeSuite(func() { var err error - t = &envtest.Environment{} + testenv = &envtest.Environment{} - cfg, err = t.Start() + cfg, err = testenv.Start() Expect(err).NotTo(HaveOccurred()) c, err = client.New(cfg, client.Options{}) @@ -51,5 +49,5 @@ var _ = BeforeSuite(func() { }) var _ = AfterSuite(func() { - Expect(t.Stop()).To(Succeed()) + Expect(testenv.Stop()).To(Succeed()) }) diff --git a/pkg/controller/controllerutil/controllerutil_test.go b/pkg/controller/controllerutil/controllerutil_test.go index d47c691465..d176c4fb6a 100644 --- a/pkg/controller/controllerutil/controllerutil_test.go +++ b/pkg/controller/controllerutil/controllerutil_test.go @@ -21,7 +21,7 @@ import ( "fmt" "math/rand" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -101,7 +101,6 @@ var _ = Describe("Controllerutil", func() { APIVersion: "extensions/v1beta1", UID: "foo-uid-2", })) - }) }) @@ -596,7 +595,7 @@ var _ = Describe("Controllerutil", func() { assertLocalDeployWasUpdated(nil) op, err = controllerutil.CreateOrPatch(context.TODO(), c, deploy, func() error { - deploy.Spec.Replicas = pointer.Int32Ptr(5) + deploy.Spec.Replicas = pointer.Int32(5) deploy.Status.Conditions = []appsv1.DeploymentCondition{{ Type: appsv1.DeploymentProgressing, Status: corev1.ConditionTrue, @@ -700,6 +699,62 @@ var _ = Describe("Controllerutil", func() { }) }) + Describe("AddFinalizer, which returns an indication of whether it modified the object's list of finalizers,", func() { + deploy = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Finalizers: []string{}, + }, + } + + When("the object's list of finalizers has no instances of the input finalizer", func() { + It("should return true", func() { + Expect(controllerutil.AddFinalizer(deploy, testFinalizer)).To(BeTrue()) + }) + It("should add the input finalizer to the object's list of finalizers", func() { + Expect(deploy.ObjectMeta.GetFinalizers()).To(Equal([]string{testFinalizer})) + }) + }) + + When("the object's list of finalizers has an instance of the input finalizer", func() { + It("should return false", func() { + Expect(controllerutil.AddFinalizer(deploy, testFinalizer)).To(BeFalse()) + }) + It("should not modify the object's list of finalizers", func() { + Expect(deploy.ObjectMeta.GetFinalizers()).To(Equal([]string{testFinalizer})) + }) + }) + }) + + Describe("RemoveFinalizer, which returns an indication of whether it modified the object's list of finalizers,", func() { + When("the object's list of finalizers has no instances of the input finalizer", func() { + It("should return false", func() { + Expect(controllerutil.RemoveFinalizer(deploy, testFinalizer1)).To(BeFalse()) + }) + It("should not modify the object's list of finalizers", func() { + Expect(deploy.ObjectMeta.GetFinalizers()).To(Equal([]string{testFinalizer})) + }) + }) + + When("the object's list of finalizers has one instance of the input finalizer", func() { + It("should return true", func() { + Expect(controllerutil.RemoveFinalizer(deploy, testFinalizer)).To(BeTrue()) + }) + It("should remove the instance of the input finalizer from the object's list of finalizers", func() { + Expect(deploy.ObjectMeta.GetFinalizers()).To(Equal([]string{})) + }) + }) + + When("the object's list of finalizers has multiple instances of the input finalizer", func() { + It("should return true", func() { + deploy.SetFinalizers(append(deploy.Finalizers, testFinalizer, testFinalizer)) + Expect(controllerutil.RemoveFinalizer(deploy, testFinalizer)).To(BeTrue()) + }) + It("should remove each instance of the input finalizer from the object's list of finalizers", func() { + Expect(deploy.ObjectMeta.GetFinalizers()).To(Equal([]string{})) + }) + }) + }) + Describe("ContainsFinalizer", func() { It("should check that finalizer is present", func() { controllerutil.AddFinalizer(deploy, testFinalizer) @@ -714,10 +769,15 @@ var _ = Describe("Controllerutil", func() { }) }) -const testFinalizer = "foo.bar.baz" +const ( + testFinalizer = "foo.bar.baz" + testFinalizer1 = testFinalizer + "1" +) -var _ runtime.Object = &errRuntimeObj{} -var _ metav1.Object = &errMetaObj{} +var ( + _ runtime.Object = &errRuntimeObj{} + _ metav1.Object = &errMetaObj{} +) type errRuntimeObj struct { runtime.TypeMeta @@ -775,6 +835,6 @@ type errorReader struct { client.Client } -func (e errorReader) Get(ctx context.Context, key client.ObjectKey, into client.Object) error { +func (e errorReader) Get(ctx context.Context, key client.ObjectKey, into client.Object, opts ...client.GetOption) error { return fmt.Errorf("unexpected error") } diff --git a/pkg/controller/doc.go b/pkg/controller/doc.go index 667b14fdd7..228335e929 100644 --- a/pkg/controller/doc.go +++ b/pkg/controller/doc.go @@ -17,7 +17,7 @@ limitations under the License. /* Package controller provides types and functions for building Controllers. Controllers implement Kubernetes APIs. -Creation +# Creation To create a new Controller, first create a manager.Manager and pass it to the controller.New function. The Controller MUST be started by calling Manager.Start. diff --git a/pkg/controller/example_test.go b/pkg/controller/example_test.go index 3d8e399703..d4fa1aef0b 100644 --- a/pkg/controller/example_test.go +++ b/pkg/controller/example_test.go @@ -71,7 +71,7 @@ func ExampleController() { } // Watch for Pod create / update / delete events and call Reconcile - err = c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForObject{}) + err = c.Watch(source.Kind(mgr.GetCache(), &corev1.Pod{}), &handler.EnqueueRequestForObject{}) if err != nil { log.Error(err, "unable to watch pods") os.Exit(1) @@ -108,7 +108,7 @@ func ExampleController_unstructured() { Version: "v1", }) // Watch for Pod create / update / delete events and call Reconcile - err = c.Watch(&source.Kind{Type: u}, &handler.EnqueueRequestForObject{}) + err = c.Watch(source.Kind(mgr.GetCache(), u), &handler.EnqueueRequestForObject{}) if err != nil { log.Error(err, "unable to watch pods") os.Exit(1) @@ -139,7 +139,7 @@ func ExampleNewUnmanaged() { os.Exit(1) } - if err := c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForObject{}); err != nil { + if err := c.Watch(source.Kind(mgr.GetCache(), &corev1.Pod{}), &handler.EnqueueRequestForObject{}); err != nil { log.Error(err, "unable to watch pods") os.Exit(1) } diff --git a/pkg/controller/testdata/crds/unconventionallisttype.yaml b/pkg/controller/testdata/crds/unconventionallisttype.yaml index 80b0f6b3a6..3069c473e5 100644 --- a/pkg/controller/testdata/crds/unconventionallisttype.yaml +++ b/pkg/controller/testdata/crds/unconventionallisttype.yaml @@ -1,4 +1,4 @@ -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: unconventionallisttypes.chaosapps.metamagical.io @@ -8,4 +8,10 @@ spec: kind: UnconventionalListType plural: unconventionallisttypes scope: Namespaced - version: "v1" + versions: + - name: "v1" + storage: true + served: true + schema: + openAPIV3Schema: + type: object diff --git a/pkg/doc.go b/pkg/doc.go index 65e2b71169..89b380c108 100644 --- a/pkg/doc.go +++ b/pkg/doc.go @@ -18,21 +18,21 @@ limitations under the License. Package pkg provides libraries for building Controllers. Controllers implement Kubernetes APIs and are foundational to building Operators, Workload APIs, Configuration APIs, Autoscalers, and more. -Client +# Client Client provides a Read + Write client for reading and writing Kubernetes objects. -Cache +# Cache Cache provides a Read client for reading objects from a local cache. A cache may register handlers to respond to events that update the cache. -Manager +# Manager Manager is required for creating a Controller and provides the Controller shared dependencies such as clients, caches, schemes, etc. Controllers should be Started through the Manager by calling Manager.Start. -Controller +# Controller Controller implements a Kubernetes API by responding to events (object Create, Update, Delete) and ensuring that the state specified in the Spec of the object matches the state of the system. This is called a reconcile. @@ -49,7 +49,7 @@ system must be read for each reconcile. * Controllers require Watches to be configured to enqueue reconcile.Requests in response to events. -Webhook +# Webhook Admission Webhooks are a mechanism for extending kubernetes APIs. Webhooks can be configured with target event type (object Create, Update, Delete), the API server will send AdmissionRequests to them @@ -62,7 +62,7 @@ Validating webhook is used to validate if an object meets certain requirements. * Admission Webhooks require Handler(s) to be provided to process the received AdmissionReview requests. -Reconciler +# Reconciler Reconciler is a function provided to a Controller that may be called at anytime with the Name and Namespace of an object. When called, the Reconciler will ensure that the state of the system matches what is specified in the object at the @@ -84,7 +84,7 @@ a mapping (e.g. owner references) that maps the object that triggers the reconci - e.g. it doesn't matter whether a ReplicaSet was created or updated, Reconciler will always compare the number of Pods in the system against what is specified in the object at the time it is called. -Source +# Source resource.Source is an argument to Controller.Watch that provides a stream of events. Events typically come from watching Kubernetes APIs (e.g. Pod Create, Update, Delete). @@ -97,7 +97,7 @@ through the Watch API. * Users SHOULD only use the provided Source implementations instead of implementing their own for nearly all cases. -EventHandler +# EventHandler handler.EventHandler is an argument to Controller.Watch that enqueues reconcile.Requests in response to events. @@ -117,7 +117,7 @@ type - e.g. map a Node event to objects that respond to cluster resize events. * Users SHOULD only use the provided EventHandler implementations instead of implementing their own for almost all cases. -Predicate +# Predicate predicate.Predicate is an optional argument to Controller.Watch that filters events. This allows common filters to be reused and composed. @@ -129,7 +129,7 @@ reused and composed. * Users SHOULD use the provided Predicate implementations, but MAY implement additional Predicates e.g. generation changed, label selectors changed etc. -PodController Diagram +# PodController Diagram Source provides event: @@ -143,14 +143,14 @@ Reconciler is called with the Request: * Reconciler(reconcile.Request{types.NamespaceName{Name: "foo", Namespace: "bar"}}) -Usage +# Usage The following example shows creating a new Controller program which Reconciles ReplicaSet objects in response to Pod or ReplicaSet events. The Reconciler function simply adds a label to the ReplicaSet. See the examples/builtins/main.go for a usage example. -Controller Example +Controller Example: 1. Watch ReplicaSet and Pods Sources @@ -167,7 +167,7 @@ Owning ReplicaSet Namespace and Name. 2.3 Reconciler triggered by deletion of Pods from some other actor -> Read ReplicaSet and Pods, create replacement Pods. -Watching and EventHandling +# Watching and EventHandling Controllers may Watch multiple Kinds of objects (e.g. Pods, ReplicaSets and Deployments), but they reconcile only a single Type. When one Type of object must be updated in response to changes in another Type of object, @@ -185,7 +185,7 @@ Note: reconcile.Requests are deduplicated when they are enqueued. Many Pod Even may trigger only 1 reconcile invocation as each Event results in the Handler trying to enqueue the same reconcile.Request for the ReplicaSet. -Controller Writing Tips +# Controller Writing Tips Reconciler Runtime Complexity: diff --git a/pkg/envtest/crd.go b/pkg/envtest/crd.go index 2f2160d683..dc38b793b4 100644 --- a/pkg/envtest/crd.go +++ b/pkg/envtest/crd.go @@ -20,19 +20,16 @@ import ( "bufio" "bytes" "context" - "encoding/base64" + "errors" "fmt" "io" - "io/ioutil" "os" "path/filepath" "time" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" @@ -42,9 +39,10 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/util/retry" "k8s.io/utils/pointer" + "sigs.k8s.io/yaml" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/conversion" - "sigs.k8s.io/yaml" ) // CRDInstallOptions are the options for installing CRDs. @@ -62,7 +60,7 @@ type CRDInstallOptions struct { Paths []string // CRDs is a list of CRDs to install - CRDs []client.Object + CRDs []*apiextensionsv1.CustomResourceDefinition // ErrorIfPathMissing will cause an error if a Path does not exist ErrorIfPathMissing bool @@ -86,11 +84,13 @@ type CRDInstallOptions struct { WebhookOptions WebhookInstallOptions } -const defaultPollInterval = 100 * time.Millisecond -const defaultMaxWait = 10 * time.Second +const ( + defaultPollInterval = 100 * time.Millisecond + defaultMaxWait = 10 * time.Second +) // InstallCRDs installs a collection of CRDs into a cluster by reading the crd yaml files from a directory. -func InstallCRDs(config *rest.Config, options CRDInstallOptions) ([]client.Object, error) { +func InstallCRDs(config *rest.Config, options CRDInstallOptions) ([]*apiextensionsv1.CustomResourceDefinition, error) { defaultCRDOptions(&options) // Read the CRD yamls into options.CRDs @@ -142,49 +142,14 @@ func defaultCRDOptions(o *CRDInstallOptions) { } // WaitForCRDs waits for the CRDs to appear in discovery. -func WaitForCRDs(config *rest.Config, crds []client.Object, options CRDInstallOptions) error { +func WaitForCRDs(config *rest.Config, crds []*apiextensionsv1.CustomResourceDefinition, options CRDInstallOptions) error { // Add each CRD to a map of GroupVersion to Resource - waitingFor := map[schema.GroupVersion]*sets.String{} - for _, crd := range runtimeCRDListToUnstructured(crds) { + waitingFor := map[schema.GroupVersion]*sets.Set[string]{} + for _, crd := range crds { gvs := []schema.GroupVersion{} - crdGroup, _, err := unstructured.NestedString(crd.Object, "spec", "group") - if err != nil { - return err - } - crdPlural, _, err := unstructured.NestedString(crd.Object, "spec", "names", "plural") - if err != nil { - return err - } - crdVersion, _, err := unstructured.NestedString(crd.Object, "spec", "version") - if err != nil { - return err - } - versions, found, err := unstructured.NestedSlice(crd.Object, "spec", "versions") - if err != nil { - return err - } - - // gvs should be added here only if single version is found. If multiple version is found we will add those version - // based on the version is served or not. - if crdVersion != "" && !found { - gvs = append(gvs, schema.GroupVersion{Group: crdGroup, Version: crdVersion}) - } - - for _, version := range versions { - versionMap, ok := version.(map[string]interface{}) - if !ok { - continue - } - served, _, err := unstructured.NestedBool(versionMap, "served") - if err != nil { - return err - } - if served { - versionName, _, err := unstructured.NestedString(versionMap, "name") - if err != nil { - return err - } - gvs = append(gvs, schema.GroupVersion{Group: crdGroup, Version: versionName}) + for _, version := range crd.Spec.Versions { + if version.Served { + gvs = append(gvs, schema.GroupVersion{Group: crd.Spec.Group, Version: version.Name}) } } @@ -192,10 +157,10 @@ func WaitForCRDs(config *rest.Config, crds []client.Object, options CRDInstallOp log.V(1).Info("adding API in waitlist", "GV", gv) if _, found := waitingFor[gv]; !found { // Initialize the set - waitingFor[gv] = &sets.String{} + waitingFor[gv] = &sets.Set[string]{} } // Add the Resource - waitingFor[gv].Insert(crdPlural) + waitingFor[gv].Insert(crd.Spec.Names.Plural) } } @@ -210,7 +175,7 @@ type poller struct { config *rest.Config // waitingFor is the map of resources keyed by group version that have not yet been found in discovery - waitingFor map[schema.GroupVersion]*sets.String + waitingFor map[schema.GroupVersion]*sets.Set[string] } // poll checks if all the resources have been found in discovery, and returns false if not. @@ -263,7 +228,8 @@ func UninstallCRDs(config *rest.Config, options CRDInstallOptions) error { } // Uninstall each CRD - for _, crd := range runtimeCRDListToUnstructured(options.CRDs) { + for _, crd := range options.CRDs { + crd := crd log.V(1).Info("uninstalling CRD", "crd", crd.GetName()) if err := cs.Delete(context.TODO(), crd); err != nil { // If CRD is not found, we can consider success @@ -277,14 +243,15 @@ func UninstallCRDs(config *rest.Config, options CRDInstallOptions) error { } // CreateCRDs creates the CRDs. -func CreateCRDs(config *rest.Config, crds []client.Object) error { +func CreateCRDs(config *rest.Config, crds []*apiextensionsv1.CustomResourceDefinition) error { cs, err := client.New(config, client.Options{}) if err != nil { return fmt.Errorf("unable to create client: %w", err) } // Create each CRD - for _, crd := range runtimeCRDListToUnstructured(crds) { + for _, crd := range crds { + crd := crd log.V(1).Info("installing CRD", "crd", crd.GetName()) existingCrd := crd.DeepCopy() err := cs.Get(context.TODO(), client.ObjectKey{Name: crd.GetName()}, existingCrd) @@ -312,22 +279,21 @@ func CreateCRDs(config *rest.Config, crds []client.Object) error { } // renderCRDs iterate through options.Paths and extract all CRD files. -func renderCRDs(options *CRDInstallOptions) ([]client.Object, error) { - var ( - err error - info os.FileInfo - files []os.FileInfo - ) - +func renderCRDs(options *CRDInstallOptions) ([]*apiextensionsv1.CustomResourceDefinition, error) { type GVKN struct { GVK schema.GroupVersionKind Name string } - crds := map[GVKN]*unstructured.Unstructured{} + crds := map[GVKN]*apiextensionsv1.CustomResourceDefinition{} for _, path := range options.Paths { - var filePath = path + var ( + err error + info os.FileInfo + files []string + filePath = path + ) // Return the error if ErrorIfPathMissing exists if info, err = os.Stat(path); os.IsNotExist(err) { @@ -338,9 +304,15 @@ func renderCRDs(options *CRDInstallOptions) ([]client.Object, error) { } if !info.IsDir() { - filePath, files = filepath.Dir(path), []os.FileInfo{info} - } else if files, err = ioutil.ReadDir(path); err != nil { - return nil, err + filePath, files = filepath.Dir(path), []string{info.Name()} + } else { + entries, err := os.ReadDir(path) + if err != nil { + return nil, err + } + for _, e := range entries { + files = append(files, e.Name()) + } } log.V(1).Info("reading CRDs from path", "path", path) @@ -361,7 +333,7 @@ func renderCRDs(options *CRDInstallOptions) ([]client.Object, error) { } // Converting map to a list to return - res := []client.Object{} + res := []*apiextensionsv1.CustomResourceDefinition{} for _, obj := range crds { res = append(res, obj) } @@ -370,12 +342,7 @@ func renderCRDs(options *CRDInstallOptions) ([]client.Object, error) { // modifyConversionWebhooks takes all the registered CustomResourceDefinitions and applies modifications // to conditionally enable webhooks if the type is registered within the scheme. -// -// The complexity of this function is high mostly due to all the edge cases that we need to handle: -// CRDv1beta1, CRDv1, and their unstructured counterpart. -// -// We should be able to simplify this code once we drop support for v1beta1 and standardize around the typed CRDv1 object. -func modifyConversionWebhooks(crds []client.Object, scheme *runtime.Scheme, webhookOptions WebhookInstallOptions) error { //nolint:gocyclo +func modifyConversionWebhooks(crds []*apiextensionsv1.CustomResourceDefinition, scheme *runtime.Scheme, webhookOptions WebhookInstallOptions) error { if len(webhookOptions.LocalServingCAData) == 0 { return nil } @@ -397,212 +364,76 @@ func modifyConversionWebhooks(crds []client.Object, scheme *runtime.Scheme, webh if err != nil { return err } - url := pointer.StringPtr(fmt.Sprintf("https://%s/convert", hostPort)) + url := pointer.String(fmt.Sprintf("https://%s/convert", hostPort)) - for _, crd := range crds { - switch c := crd.(type) { - case *apiextensionsv1beta1.CustomResourceDefinition: - // Continue if we're preserving unknown fields. - // - // preserveUnknownFields defaults to true if `nil` in v1beta1. - if c.Spec.PreserveUnknownFields == nil || *c.Spec.PreserveUnknownFields { - continue - } - // Continue if the GroupKind isn't registered as being convertible. - if _, ok := convertibles[schema.GroupKind{ - Group: c.Spec.Group, - Kind: c.Spec.Names.Kind, - }]; !ok { - continue - } - c.Spec.Conversion.Strategy = apiextensionsv1beta1.WebhookConverter - c.Spec.Conversion.WebhookClientConfig.Service = nil - c.Spec.Conversion.WebhookClientConfig = &apiextensionsv1beta1.WebhookClientConfig{ - Service: nil, - URL: url, - CABundle: webhookOptions.LocalServingCAData, - } - case *apiextensionsv1.CustomResourceDefinition: - // Continue if we're preserving unknown fields. - if c.Spec.PreserveUnknownFields { - continue - } - // Continue if the GroupKind isn't registered as being convertible. - if _, ok := convertibles[schema.GroupKind{ - Group: c.Spec.Group, - Kind: c.Spec.Names.Kind, - }]; !ok { - continue - } - c.Spec.Conversion.Strategy = apiextensionsv1.WebhookConverter - c.Spec.Conversion.Webhook.ClientConfig.Service = nil - c.Spec.Conversion.Webhook.ClientConfig = &apiextensionsv1.WebhookClientConfig{ - Service: nil, - URL: url, - CABundle: webhookOptions.LocalServingCAData, - } - case *unstructured.Unstructured: - webhookClientConfig := map[string]interface{}{ - "url": *url, - "caBundle": base64.StdEncoding.EncodeToString(webhookOptions.LocalServingCAData), - } - - switch c.GroupVersionKind().Version { - case "v1beta1": - // Continue if we're preserving unknown fields. - // - // preserveUnknownFields defaults to true if `nil` in v1beta1. - if preserve, found, err := unstructured.NestedBool(c.Object, "spec", "preserveUnknownFields"); preserve || !found { - continue - } else if err != nil { - return err - } - - // Continue if the GroupKind isn't registered as being convertible. - group, found, err := unstructured.NestedString(c.Object, "spec", "group") - if !found { - continue - } else if err != nil { - return err - } - kind, found, err := unstructured.NestedString(c.Object, "spec", "names", "kind") - if !found { - continue - } else if err != nil { - return err - } - if _, ok := convertibles[schema.GroupKind{ - Group: group, - Kind: kind, - }]; !ok { - continue - } - - // Set the strategy. - if err := unstructured.SetNestedField( - c.Object, - string(apiextensionsv1beta1.WebhookConverter), - "spec", "conversion", "strategy"); err != nil { - return err - } - // Set the conversion review versions. - if err := unstructured.SetNestedStringSlice( - c.Object, - []string{"v1beta1"}, - "spec", "conversion", "webhook", "clientConfig"); err != nil { - return err - } - // Set the client configuration. - if err := unstructured.SetNestedMap( - c.Object, - webhookClientConfig, - "spec", "conversion", "webhookClientConfig"); err != nil { - return err - } - case "v1": - if preserve, _, err := unstructured.NestedBool(c.Object, "spec", "preserveUnknownFields"); preserve { - continue - } else if err != nil { - return err - } - - // Continue if the GroupKind isn't registered as being convertible. - group, found, err := unstructured.NestedString(c.Object, "spec", "group") - if !found { - continue - } else if err != nil { - return err - } - kind, found, err := unstructured.NestedString(c.Object, "spec", "names", "kind") - if !found { - continue - } else if err != nil { - return err - } - if _, ok := convertibles[schema.GroupKind{ - Group: group, - Kind: kind, - }]; !ok { - continue - } - - // Set the strategy. - if err := unstructured.SetNestedField( - c.Object, - string(apiextensionsv1.WebhookConverter), - "spec", "conversion", "strategy"); err != nil { - return err - } - // Set the conversion review versions. - if err := unstructured.SetNestedStringSlice( - c.Object, - []string{"v1", "v1beta1"}, - "spec", "conversion", "webhook", "conversionReviewVersions"); err != nil { - return err - } - // Set the client configuration. - if err := unstructured.SetNestedMap( - c.Object, - webhookClientConfig, - "spec", "conversion", "webhook", "clientConfig"); err != nil { - return err - } + for i := range crds { + // Continue if we're preserving unknown fields. + if crds[i].Spec.PreserveUnknownFields { + continue + } + // Continue if the GroupKind isn't registered as being convertible. + if _, ok := convertibles[schema.GroupKind{ + Group: crds[i].Spec.Group, + Kind: crds[i].Spec.Names.Kind, + }]; !ok { + continue + } + if crds[i].Spec.Conversion == nil { + crds[i].Spec.Conversion = &apiextensionsv1.CustomResourceConversion{ + Webhook: &apiextensionsv1.WebhookConversion{}, } } + crds[i].Spec.Conversion.Strategy = apiextensionsv1.WebhookConverter + crds[i].Spec.Conversion.Webhook.ConversionReviewVersions = []string{"v1", "v1beta1"} + crds[i].Spec.Conversion.Webhook.ClientConfig = &apiextensionsv1.WebhookClientConfig{ + Service: nil, + URL: url, + CABundle: webhookOptions.LocalServingCAData, + } } return nil } // readCRDs reads the CRDs from files and Unmarshals them into structs. -func readCRDs(basePath string, files []os.FileInfo) ([]*unstructured.Unstructured, error) { - var crds []*unstructured.Unstructured +func readCRDs(basePath string, files []string) ([]*apiextensionsv1.CustomResourceDefinition, error) { + var crds []*apiextensionsv1.CustomResourceDefinition // White list the file extensions that may contain CRDs crdExts := sets.NewString(".json", ".yaml", ".yml") for _, file := range files { // Only parse allowlisted file types - if !crdExts.Has(filepath.Ext(file.Name())) { + if !crdExts.Has(filepath.Ext(file)) { continue } // Unmarshal CRDs from file into structs - docs, err := readDocuments(filepath.Join(basePath, file.Name())) + docs, err := readDocuments(filepath.Join(basePath, file)) if err != nil { return nil, err } for _, doc := range docs { - crd := &unstructured.Unstructured{} + crd := &apiextensionsv1.CustomResourceDefinition{} if err = yaml.Unmarshal(doc, crd); err != nil { return nil, err } - // Check that it is actually a CRD - crdKind, _, err := unstructured.NestedString(crd.Object, "spec", "names", "kind") - if err != nil { - return nil, err - } - crdGroup, _, err := unstructured.NestedString(crd.Object, "spec", "group") - if err != nil { - return nil, err - } - - if crd.GetKind() != "CustomResourceDefinition" || crdKind == "" || crdGroup == "" { + if crd.Kind != "CustomResourceDefinition" || crd.Spec.Names.Kind == "" || crd.Spec.Group == "" { continue } crds = append(crds, crd) } - log.V(1).Info("read CRDs from file", "file", file.Name()) + log.V(1).Info("read CRDs from file", "file", file) } return crds, nil } // readDocuments reads documents from file. func readDocuments(fp string) ([][]byte, error) { - b, err := ioutil.ReadFile(fp) //nolint:gosec + b, err := os.ReadFile(fp) if err != nil { return nil, err } @@ -613,7 +444,7 @@ func readDocuments(fp string) ([][]byte, error) { // Read document doc, err := reader.Read() if err != nil { - if err == io.EOF { + if errors.Is(err, io.EOF) { break } diff --git a/pkg/envtest/crd_test.go b/pkg/envtest/crd_test.go new file mode 100644 index 0000000000..92dc48e963 --- /dev/null +++ b/pkg/envtest/crd_test.go @@ -0,0 +1,51 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package envtest + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/util/sets" +) + +var _ = Describe("Test", func() { + Describe("readCRDFiles", func() { + It("should not mix up files from different directories", func() { + opt := CRDInstallOptions{ + Paths: []string{ + "testdata/crds", + "testdata/crdv1_original", + }, + } + err := readCRDFiles(&opt) + Expect(err).NotTo(HaveOccurred()) + + expectedCRDs := sets.NewString( + "frigates.ship.example.com", + "configs.foo.example.com", + "drivers.crew.example.com", + ) + + foundCRDs := sets.NewString() + for _, crd := range opt.CRDs { + foundCRDs.Insert(crd.Name) + } + + Expect(expectedCRDs).To(Equal(foundCRDs)) + }) + }) +}) diff --git a/pkg/envtest/envtest_suite_test.go b/pkg/envtest/envtest_suite_test.go index d778c88077..f7788bf090 100644 --- a/pkg/envtest/envtest_suite_test.go +++ b/pkg/envtest/envtest_suite_test.go @@ -19,91 +19,83 @@ package envtest import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" admissionv1 "k8s.io/api/admissionregistration/v1" - admissionv1beta1 "k8s.io/api/admissionregistration/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" + logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Envtest Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Envtest Suite") } var env *Environment -var _ = BeforeSuite(func(done Done) { +var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) env = &Environment{} // we're initializing webhook here and not in webhook.go to also test the envtest install code via WebhookOptions initializeWebhookInEnvironment() _, err := env.Start() Expect(err).NotTo(HaveOccurred()) - - close(done) -}, StartTimeout) +}) func initializeWebhookInEnvironment() { - namespacedScopeV1Beta1 := admissionv1beta1.NamespacedScope namespacedScopeV1 := admissionv1.NamespacedScope - failedTypeV1Beta1 := admissionv1beta1.Fail failedTypeV1 := admissionv1.Fail - equivalentTypeV1Beta1 := admissionv1beta1.Equivalent equivalentTypeV1 := admissionv1.Equivalent - noSideEffectsV1Beta1 := admissionv1beta1.SideEffectClassNone noSideEffectsV1 := admissionv1.SideEffectClassNone webhookPathV1 := "/failing" env.WebhookInstallOptions = WebhookInstallOptions{ - ValidatingWebhooks: []client.Object{ - &admissionv1beta1.ValidatingWebhookConfiguration{ + ValidatingWebhooks: []*admissionv1.ValidatingWebhookConfiguration{ + { ObjectMeta: metav1.ObjectMeta{ Name: "deployment-validation-webhook-config", }, TypeMeta: metav1.TypeMeta{ Kind: "ValidatingWebhookConfiguration", - APIVersion: "admissionregistration.k8s.io/v1beta1", + APIVersion: "admissionregistration.k8s.io/v1", }, - Webhooks: []admissionv1beta1.ValidatingWebhook{ + Webhooks: []admissionv1.ValidatingWebhook{ { Name: "deployment-validation.kubebuilder.io", - Rules: []admissionv1beta1.RuleWithOperations{ + Rules: []admissionv1.RuleWithOperations{ { - Operations: []admissionv1beta1.OperationType{"CREATE", "UPDATE"}, - Rule: admissionv1beta1.Rule{ + Operations: []admissionv1.OperationType{"CREATE", "UPDATE"}, + Rule: admissionv1.Rule{ APIGroups: []string{"apps"}, APIVersions: []string{"v1"}, Resources: []string{"deployments"}, - Scope: &namespacedScopeV1Beta1, + Scope: &namespacedScopeV1, }, }, }, - FailurePolicy: &failedTypeV1Beta1, - MatchPolicy: &equivalentTypeV1Beta1, - SideEffects: &noSideEffectsV1Beta1, - ClientConfig: admissionv1beta1.WebhookClientConfig{ - Service: &admissionv1beta1.ServiceReference{ + FailurePolicy: &failedTypeV1, + MatchPolicy: &equivalentTypeV1, + SideEffects: &noSideEffectsV1, + ClientConfig: admissionv1.WebhookClientConfig{ + Service: &admissionv1.ServiceReference{ Name: "deployment-validation-service", Namespace: "default", Path: &webhookPathV1, }, }, + AdmissionReviewVersions: []string{"v1"}, }, }, }, - &admissionv1.ValidatingWebhookConfiguration{ + { ObjectMeta: metav1.ObjectMeta{ Name: "deployment-validation-webhook-config", }, TypeMeta: metav1.TypeMeta{ Kind: "ValidatingWebhookConfiguration", - APIVersion: "admissionregistration.k8s.io/v1beta1", + APIVersion: "admissionregistration.k8s.io/v1", }, Webhooks: []admissionv1.ValidatingWebhook{ { @@ -129,6 +121,7 @@ func initializeWebhookInEnvironment() { Path: &webhookPathV1, }, }, + AdmissionReviewVersions: []string{"v1"}, }, }, }, @@ -136,8 +129,6 @@ func initializeWebhookInEnvironment() { } } -var _ = AfterSuite(func(done Done) { +var _ = AfterSuite(func() { Expect(env.Stop()).NotTo(HaveOccurred()) - - close(done) -}, StopTimeout) +}) diff --git a/pkg/envtest/envtest_test.go b/pkg/envtest/envtest_test.go index fb5fb87a7f..21464e10be 100644 --- a/pkg/envtest/envtest_test.go +++ b/pkg/envtest/envtest_test.go @@ -21,20 +21,19 @@ import ( "path/filepath" "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" ) var _ = Describe("Test", func() { - var crds []client.Object + var crds []*apiextensionsv1.CustomResourceDefinition var err error var s *runtime.Scheme var c client.Client @@ -45,28 +44,25 @@ var _ = Describe("Test", func() { var teardownTimeoutSeconds float64 = 10 // Initialize the client - BeforeEach(func(done Done) { - crds = []client.Object{} - s = runtime.NewScheme() - err = v1beta1.AddToScheme(s) - Expect(err).NotTo(HaveOccurred()) + BeforeEach(func() { + crds = []*apiextensionsv1.CustomResourceDefinition{} + s = scheme.Scheme err = apiextensionsv1.AddToScheme(s) Expect(err).NotTo(HaveOccurred()) c, err = client.New(env.Config, client.Options{Scheme: s}) Expect(err).NotTo(HaveOccurred()) - - close(done) }) // Cleanup CRDs - AfterEach(func(done Done) { - for _, crd := range runtimeCRDListToUnstructured(crds) { + AfterEach(func() { + for _, crd := range crds { + crd := crd // Delete only if CRD exists. crdObjectKey := client.ObjectKey{ Name: crd.GetName(), } - var placeholder v1beta1.CustomResourceDefinition + var placeholder apiextensionsv1.CustomResourceDefinition if err = c.Get(context.TODO(), crdObjectKey, &placeholder); err != nil && apierrors.IsNotFound(err) { // CRD doesn't need to be deleted. @@ -79,7 +75,6 @@ var _ = Describe("Test", func() { return apierrors.IsNotFound(err) }, 5*time.Second).Should(BeTrue()) } - close(done) }, teardownTimeoutSeconds) Describe("InstallCRDs", func() { @@ -91,19 +86,19 @@ var _ = Describe("Test", func() { // Expect to find the CRDs - crdv1 := &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "frigates.ship.example.com"}, crdv1) + crd := &apiextensionsv1.CustomResourceDefinition{} + err = c.Get(context.TODO(), types.NamespacedName{Name: "frigates.ship.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) - Expect(crdv1.Spec.Names.Kind).To(Equal("Frigate")) + Expect(crd.Spec.Names.Kind).To(Equal("Frigate")) - err = WaitForCRDs(env.Config, []client.Object{ - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ + err = WaitForCRDs(env.Config, []*apiextensionsv1.CustomResourceDefinition{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ Group: "ship.example.com", - Names: v1beta1.CustomResourceDefinitionNames{ + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "frigates", }, - Versions: []v1beta1.CustomResourceDefinitionVersion{ + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ { Name: "v1", Storage: true, @@ -121,7 +116,7 @@ var _ = Describe("Test", func() { ) Expect(err).NotTo(HaveOccurred()) }) - It("should install the CRDs into the cluster using directory", func(done Done) { + It("should install the CRDs into the cluster using directory", func() { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ Paths: []string{validDirectory}, }) @@ -129,33 +124,33 @@ var _ = Describe("Test", func() { // Expect to find the CRDs - crdv1 := &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "foos.bar.example.com"}, crdv1) + crd := &apiextensionsv1.CustomResourceDefinition{} + err = c.Get(context.TODO(), types.NamespacedName{Name: "foos.bar.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) - Expect(crdv1.Spec.Names.Kind).To(Equal("Foo")) + Expect(crd.Spec.Names.Kind).To(Equal("Foo")) - crd := &v1beta1.CustomResourceDefinition{} + crd = &apiextensionsv1.CustomResourceDefinition{} err = c.Get(context.TODO(), types.NamespacedName{Name: "bazs.qux.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Baz")) - crd = &v1beta1.CustomResourceDefinition{} + crd = &apiextensionsv1.CustomResourceDefinition{} err = c.Get(context.TODO(), types.NamespacedName{Name: "captains.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Captain")) - crd = &v1beta1.CustomResourceDefinition{} + crd = &apiextensionsv1.CustomResourceDefinition{} err = c.Get(context.TODO(), types.NamespacedName{Name: "firstmates.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("FirstMate")) - crd = &v1beta1.CustomResourceDefinition{} + crd = &apiextensionsv1.CustomResourceDefinition{} err = c.Get(context.TODO(), types.NamespacedName{Name: "drivers.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Driver")) - err = WaitForCRDs(env.Config, []client.Object{ - &apiextensionsv1.CustomResourceDefinition{ + err = WaitForCRDs(env.Config, []*apiextensionsv1.CustomResourceDefinition{ + { Spec: apiextensionsv1.CustomResourceDefinitionSpec{ Group: "bar.example.com", Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ @@ -174,37 +169,64 @@ var _ = Describe("Test", func() { Plural: "foos", }}, }, - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ - Group: "qux.example.com", - Version: "v1beta1", - Names: v1beta1.CustomResourceDefinitionNames{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "qux.example.com", + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1beta1", + Storage: true, + Served: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{}, + }, + }, + }, + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "bazs", }}, }, - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ - Group: "crew.example.com", - Version: "v1beta1", - Names: v1beta1.CustomResourceDefinitionNames{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "crew.example.com", + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1beta1", + Storage: true, + Served: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{}, + }, + }, + }, + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "captains", }}, }, - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ - Group: "crew.example.com", - Version: "v1beta1", - Names: v1beta1.CustomResourceDefinitionNames{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "crew.example.com", + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1beta1", + Storage: true, + Served: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{}, + }, + }, + }, + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "firstmates", }}, }, - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ Group: "crew.example.com", - Names: v1beta1.CustomResourceDefinitionNames{ + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "drivers", }, - Versions: []v1beta1.CustomResourceDefinitionVersion{ + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ { Name: "v1", Storage: true, @@ -221,27 +243,34 @@ var _ = Describe("Test", func() { CRDInstallOptions{MaxTime: 50 * time.Millisecond, PollInterval: 15 * time.Millisecond}, ) Expect(err).NotTo(HaveOccurred()) + }) - close(done) - }, 5) - - It("should install the CRDs into the cluster using file", func(done Done) { + It("should install the CRDs into the cluster using file", func() { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ Paths: []string{filepath.Join(".", "testdata", "crds", "examplecrd3.yaml")}, }) Expect(err).NotTo(HaveOccurred()) - crd := &v1beta1.CustomResourceDefinition{} + crd := &apiextensionsv1.CustomResourceDefinition{} err = c.Get(context.TODO(), types.NamespacedName{Name: "configs.foo.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Config")) - err = WaitForCRDs(env.Config, []client.Object{ - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ - Group: "foo.example.com", - Version: "v1beta1", - Names: v1beta1.CustomResourceDefinitionNames{ + err = WaitForCRDs(env.Config, []*apiextensionsv1.CustomResourceDefinition{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "foo.example.com", + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1beta1", + Storage: true, + Served: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{}, + }, + }, + }, + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "configs", }}, }, @@ -249,11 +278,9 @@ var _ = Describe("Test", func() { CRDInstallOptions{MaxTime: 50 * time.Millisecond, PollInterval: 15 * time.Millisecond}, ) Expect(err).NotTo(HaveOccurred()) + }) - close(done) - }, 10) - - It("should be able to install CRDs using multiple files", func(done Done) { + It("should be able to install CRDs using multiple files", func() { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ Paths: []string{ filepath.Join(".", "testdata", "examplecrd.yaml"), @@ -262,11 +289,9 @@ var _ = Describe("Test", func() { }) Expect(err).NotTo(HaveOccurred()) Expect(crds).To(HaveLen(2)) + }) - close(done) - }, 10) - - It("should filter out already existent CRD", func(done Done) { + It("should filter out already existent CRD", func() { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ Paths: []string{ filepath.Join(".", "testdata"), @@ -280,8 +305,8 @@ var _ = Describe("Test", func() { Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Foo")) - err = WaitForCRDs(env.Config, []client.Object{ - &apiextensionsv1.CustomResourceDefinition{ + err = WaitForCRDs(env.Config, []*apiextensionsv1.CustomResourceDefinition{ + { Spec: apiextensionsv1.CustomResourceDefinitionSpec{ Group: "bar.example.com", Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ @@ -304,43 +329,44 @@ var _ = Describe("Test", func() { CRDInstallOptions{MaxTime: 50 * time.Millisecond, PollInterval: 15 * time.Millisecond}, ) Expect(err).NotTo(HaveOccurred()) + }) - close(done) - }, 10) - - It("should not return an not error if the directory doesn't exist", func(done Done) { + It("should not return an not error if the directory doesn't exist", func() { crds, err = InstallCRDs(env.Config, CRDInstallOptions{Paths: []string{invalidDirectory}}) Expect(err).NotTo(HaveOccurred()) + }) - close(done) - }, 5) - - It("should return an error if the directory doesn't exist", func(done Done) { + It("should return an error if the directory doesn't exist", func() { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ Paths: []string{invalidDirectory}, ErrorIfPathMissing: true, }) Expect(err).To(HaveOccurred()) + }) - close(done) - }, 5) - - It("should return an error if the file doesn't exist", func(done Done) { + It("should return an error if the file doesn't exist", func() { crds, err = InstallCRDs(env.Config, CRDInstallOptions{Paths: []string{ filepath.Join(".", "testdata", "fake.yaml")}, ErrorIfPathMissing: true, }) Expect(err).To(HaveOccurred()) + }) - close(done) - }, 5) - - It("should return an error if the resource group version isn't found", func(done Done) { + It("should return an error if the resource group version isn't found", func() { // Wait for a CRD where the Group and Version don't exist err := WaitForCRDs(env.Config, - []client.Object{ - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ - Version: "v1", - Names: v1beta1.CustomResourceDefinitionNames{ + []*apiextensionsv1.CustomResourceDefinition{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1", + Storage: true, + Served: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{}, + }, + }, + }, + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "notfound", }}, }, @@ -348,42 +374,56 @@ var _ = Describe("Test", func() { CRDInstallOptions{MaxTime: 50 * time.Millisecond, PollInterval: 15 * time.Millisecond}, ) Expect(err).To(HaveOccurred()) + }) - close(done) - }, 5) - - It("should return an error if the resource isn't found in the group version", func(done Done) { + It("should return an error if the resource isn't found in the group version", func() { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ Paths: []string{"."}, }) Expect(err).NotTo(HaveOccurred()) // Wait for a CRD that doesn't exist, but the Group and Version do - err = WaitForCRDs(env.Config, []client.Object{ - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ - Group: "qux.example.com", - Version: "v1beta1", - Names: v1beta1.CustomResourceDefinitionNames{ + err = WaitForCRDs(env.Config, []*apiextensionsv1.CustomResourceDefinition{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "qux.example.com", + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1beta1", + Storage: true, + Served: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{}, + }, + }, + }, + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "bazs", }}, }, - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ - Group: "bar.example.com", - Version: "v1beta1", - Names: v1beta1.CustomResourceDefinitionNames{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "bar.example.com", + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1beta1", + Storage: true, + Served: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{}, + }, + }, + }, + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "fake", }}, }}, CRDInstallOptions{MaxTime: 50 * time.Millisecond, PollInterval: 15 * time.Millisecond}, ) Expect(err).To(HaveOccurred()) + }) - close(done) - }, 5) - - It("should reinstall the CRDs if already present in the cluster", func(done Done) { + It("should reinstall the CRDs if already present in the cluster", func() { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ Paths: []string{filepath.Join(".", "testdata")}, @@ -392,33 +432,33 @@ var _ = Describe("Test", func() { // Expect to find the CRDs - crd := &v1beta1.CustomResourceDefinition{} + crd := &apiextensionsv1.CustomResourceDefinition{} err = c.Get(context.TODO(), types.NamespacedName{Name: "foos.bar.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Foo")) - crd = &v1beta1.CustomResourceDefinition{} + crd = &apiextensionsv1.CustomResourceDefinition{} err = c.Get(context.TODO(), types.NamespacedName{Name: "bazs.qux.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Baz")) - crd = &v1beta1.CustomResourceDefinition{} + crd = &apiextensionsv1.CustomResourceDefinition{} err = c.Get(context.TODO(), types.NamespacedName{Name: "captains.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Captain")) - crd = &v1beta1.CustomResourceDefinition{} + crd = &apiextensionsv1.CustomResourceDefinition{} err = c.Get(context.TODO(), types.NamespacedName{Name: "firstmates.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("FirstMate")) - crd = &v1beta1.CustomResourceDefinition{} + crd = &apiextensionsv1.CustomResourceDefinition{} err = c.Get(context.TODO(), types.NamespacedName{Name: "drivers.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Driver")) - err = WaitForCRDs(env.Config, []client.Object{ - &apiextensionsv1.CustomResourceDefinition{ + err = WaitForCRDs(env.Config, []*apiextensionsv1.CustomResourceDefinition{ + { Spec: apiextensionsv1.CustomResourceDefinitionSpec{ Group: "bar.example.com", Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ @@ -437,37 +477,64 @@ var _ = Describe("Test", func() { Plural: "foos", }}, }, - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ - Group: "qux.example.com", - Version: "v1beta1", - Names: v1beta1.CustomResourceDefinitionNames{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "qux.example.com", + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1beta1", + Storage: true, + Served: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{}, + }, + }, + }, + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "bazs", }}, }, - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ - Group: "crew.example.com", - Version: "v1beta1", - Names: v1beta1.CustomResourceDefinitionNames{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "crew.example.com", + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1beta1", + Storage: true, + Served: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{}, + }, + }, + }, + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "captains", }}, }, - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ - Group: "crew.example.com", - Version: "v1beta1", - Names: v1beta1.CustomResourceDefinitionNames{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "crew.example.com", + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1beta1", + Storage: true, + Served: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{}, + }, + }, + }, + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "firstmates", }}, }, - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ Group: "crew.example.com", - Names: v1beta1.CustomResourceDefinitionNames{ + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "drivers", }, - Versions: []v1beta1.CustomResourceDefinitionVersion{ + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ { Name: "v1", Storage: true, @@ -494,33 +561,33 @@ var _ = Describe("Test", func() { // Expect to find the CRDs - crd = &v1beta1.CustomResourceDefinition{} + crd = &apiextensionsv1.CustomResourceDefinition{} err = c.Get(context.TODO(), types.NamespacedName{Name: "foos.bar.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Foo")) - crd = &v1beta1.CustomResourceDefinition{} + crd = &apiextensionsv1.CustomResourceDefinition{} err = c.Get(context.TODO(), types.NamespacedName{Name: "bazs.qux.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Baz")) - crd = &v1beta1.CustomResourceDefinition{} + crd = &apiextensionsv1.CustomResourceDefinition{} err = c.Get(context.TODO(), types.NamespacedName{Name: "captains.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Captain")) - crd = &v1beta1.CustomResourceDefinition{} + crd = &apiextensionsv1.CustomResourceDefinition{} err = c.Get(context.TODO(), types.NamespacedName{Name: "firstmates.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("FirstMate")) - crd = &v1beta1.CustomResourceDefinition{} + crd = &apiextensionsv1.CustomResourceDefinition{} err = c.Get(context.TODO(), types.NamespacedName{Name: "drivers.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Driver")) - err = WaitForCRDs(env.Config, []client.Object{ - &apiextensionsv1.CustomResourceDefinition{ + err = WaitForCRDs(env.Config, []*apiextensionsv1.CustomResourceDefinition{ + { Spec: apiextensionsv1.CustomResourceDefinitionSpec{ Group: "bar.example.com", Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ @@ -539,37 +606,64 @@ var _ = Describe("Test", func() { Plural: "foos", }}, }, - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ - Group: "qux.example.com", - Version: "v1beta1", - Names: v1beta1.CustomResourceDefinitionNames{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "qux.example.com", + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1beta1", + Storage: true, + Served: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{}, + }, + }, + }, + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "bazs", }}, }, - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ - Group: "crew.example.com", - Version: "v1beta1", - Names: v1beta1.CustomResourceDefinitionNames{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "crew.example.com", + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1beta1", + Storage: true, + Served: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{}, + }, + }, + }, + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "captains", }}, }, - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ - Group: "crew.example.com", - Version: "v1beta1", - Names: v1beta1.CustomResourceDefinitionNames{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "crew.example.com", + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1beta1", + Storage: true, + Served: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{}, + }, + }, + }, + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "firstmates", }}, }, - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ Group: "crew.example.com", - Names: v1beta1.CustomResourceDefinitionNames{ + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "drivers", }, - Versions: []v1beta1.CustomResourceDefinitionVersion{ + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ { Name: "v1", Storage: true, @@ -586,12 +680,10 @@ var _ = Describe("Test", func() { CRDInstallOptions{MaxTime: 50 * time.Millisecond, PollInterval: 15 * time.Millisecond}, ) Expect(err).NotTo(HaveOccurred()) - - close(done) - }, 5) + }) }) - It("should update CRDs if already present in the cluster", func(done Done) { + It("should update CRDs if already present in the cluster", func() { // Install only the CRDv1 multi-version example crds, err = InstallCRDs(env.Config, CRDInstallOptions{ @@ -601,7 +693,7 @@ var _ = Describe("Test", func() { // Expect to find the CRDs - crd := &v1beta1.CustomResourceDefinition{} + crd := &apiextensionsv1.CustomResourceDefinition{} err = c.Get(context.TODO(), types.NamespacedName{Name: "drivers.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Driver")) @@ -610,14 +702,14 @@ var _ = Describe("Test", func() { // Store resource version for comparison later on firstRV := crd.ResourceVersion - err = WaitForCRDs(env.Config, []client.Object{ - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ + err = WaitForCRDs(env.Config, []*apiextensionsv1.CustomResourceDefinition{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ Group: "crew.example.com", - Names: v1beta1.CustomResourceDefinitionNames{ + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "drivers", }, - Versions: []v1beta1.CustomResourceDefinitionVersion{ + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ { Name: "v1", Storage: true, @@ -643,21 +735,21 @@ var _ = Describe("Test", func() { // Expect to find updated CRD - crd = &v1beta1.CustomResourceDefinition{} + crd = &apiextensionsv1.CustomResourceDefinition{} err = c.Get(context.TODO(), types.NamespacedName{Name: "drivers.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Driver")) Expect(len(crd.Spec.Versions)).To(BeEquivalentTo(3)) Expect(crd.ResourceVersion).NotTo(BeEquivalentTo(firstRV)) - err = WaitForCRDs(env.Config, []client.Object{ - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ + err = WaitForCRDs(env.Config, []*apiextensionsv1.CustomResourceDefinition{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ Group: "crew.example.com", - Names: v1beta1.CustomResourceDefinitionNames{ + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "drivers", }, - Versions: []v1beta1.CustomResourceDefinitionVersion{ + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ { Name: "v1", Storage: true, @@ -679,12 +771,10 @@ var _ = Describe("Test", func() { CRDInstallOptions{MaxTime: 50 * time.Millisecond, PollInterval: 15 * time.Millisecond}, ) Expect(err).NotTo(HaveOccurred()) - - close(done) - }, 5) + }) Describe("UninstallCRDs", func() { - It("should uninstall the CRDs from the cluster", func(done Done) { + It("should uninstall the CRDs from the cluster", func() { crds, err = InstallCRDs(env.Config, CRDInstallOptions{ Paths: []string{validDirectory}, @@ -693,33 +783,33 @@ var _ = Describe("Test", func() { // Expect to find the CRDs - crdv1 := &apiextensionsv1.CustomResourceDefinition{} - err = c.Get(context.TODO(), types.NamespacedName{Name: "foos.bar.example.com"}, crdv1) + crd := &apiextensionsv1.CustomResourceDefinition{} + err = c.Get(context.TODO(), types.NamespacedName{Name: "foos.bar.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) - Expect(crdv1.Spec.Names.Kind).To(Equal("Foo")) + Expect(crd.Spec.Names.Kind).To(Equal("Foo")) - crd := &v1beta1.CustomResourceDefinition{} + crd = &apiextensionsv1.CustomResourceDefinition{} err = c.Get(context.TODO(), types.NamespacedName{Name: "bazs.qux.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Baz")) - crd = &v1beta1.CustomResourceDefinition{} + crd = &apiextensionsv1.CustomResourceDefinition{} err = c.Get(context.TODO(), types.NamespacedName{Name: "captains.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Captain")) - crd = &v1beta1.CustomResourceDefinition{} + crd = &apiextensionsv1.CustomResourceDefinition{} err = c.Get(context.TODO(), types.NamespacedName{Name: "firstmates.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("FirstMate")) - crd = &v1beta1.CustomResourceDefinition{} + crd = &apiextensionsv1.CustomResourceDefinition{} err = c.Get(context.TODO(), types.NamespacedName{Name: "drivers.crew.example.com"}, crd) Expect(err).NotTo(HaveOccurred()) Expect(crd.Spec.Names.Kind).To(Equal("Driver")) - err = WaitForCRDs(env.Config, []client.Object{ - &apiextensionsv1.CustomResourceDefinition{ + err = WaitForCRDs(env.Config, []*apiextensionsv1.CustomResourceDefinition{ + { Spec: apiextensionsv1.CustomResourceDefinitionSpec{ Group: "bar.example.com", Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ @@ -738,37 +828,64 @@ var _ = Describe("Test", func() { Plural: "foos", }}, }, - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ - Group: "qux.example.com", - Version: "v1beta1", - Names: v1beta1.CustomResourceDefinitionNames{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "qux.example.com", + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1beta1", + Storage: true, + Served: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{}, + }, + }, + }, + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "bazs", }}, }, - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ - Group: "crew.example.com", - Version: "v1beta1", - Names: v1beta1.CustomResourceDefinitionNames{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "crew.example.com", + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1beta1", + Storage: true, + Served: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{}, + }, + }, + }, + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "captains", }}, }, - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ - Group: "crew.example.com", - Version: "v1beta1", - Names: v1beta1.CustomResourceDefinitionNames{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "crew.example.com", + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ + { + Name: "v1beta1", + Storage: true, + Served: true, + Schema: &apiextensionsv1.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{}, + }, + }, + }, + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "firstmates", }}, }, - &v1beta1.CustomResourceDefinition{ - Spec: v1beta1.CustomResourceDefinitionSpec{ + { + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ Group: "crew.example.com", - Names: v1beta1.CustomResourceDefinitionNames{ + Names: apiextensionsv1.CustomResourceDefinitionNames{ Plural: "drivers", }, - Versions: []v1beta1.CustomResourceDefinitionVersion{ + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{ { Name: "v1", Storage: true, @@ -793,31 +910,17 @@ var _ = Describe("Test", func() { // Expect to NOT find the CRDs - v1crds := []string{ + crds := []string{ "foos.bar.example.com", - } - v1placeholder := &apiextensionsv1.CustomResourceDefinition{} - Eventually(func() bool { - for _, crd := range v1crds { - err = c.Get(context.TODO(), types.NamespacedName{Name: crd}, v1placeholder) - notFound := err != nil && apierrors.IsNotFound(err) - if !notFound { - return false - } - } - return true - }, 20).Should(BeTrue()) - - v1beta1crds := []string{ "bazs.qux.example.com", "captains.crew.example.com", "firstmates.crew.example.com", "drivers.crew.example.com", } - v1beta1placeholder := &v1beta1.CustomResourceDefinition{} + placeholder := &apiextensionsv1.CustomResourceDefinition{} Eventually(func() bool { - for _, crd := range v1beta1crds { - err = c.Get(context.TODO(), types.NamespacedName{Name: crd}, v1beta1placeholder) + for _, crd := range crds { + err = c.Get(context.TODO(), types.NamespacedName{Name: crd}, placeholder) notFound := err != nil && apierrors.IsNotFound(err) if !notFound { return false @@ -825,31 +928,27 @@ var _ = Describe("Test", func() { } return true }, 20).Should(BeTrue()) - - close(done) - }, 30) + }) }) Describe("Start", func() { - It("should raise an error on invalid dir when flag is enabled", func(done Done) { + It("should raise an error on invalid dir when flag is enabled", func() { env := &Environment{ErrorIfCRDPathMissing: true, CRDDirectoryPaths: []string{invalidDirectory}} _, err := env.Start() Expect(err).To(HaveOccurred()) Expect(env.Stop()).To(Succeed()) - close(done) - }, 30) + }) - It("should not raise an error on invalid dir when flag is disabled", func(done Done) { + It("should not raise an error on invalid dir when flag is disabled", func() { env := &Environment{ErrorIfCRDPathMissing: false, CRDDirectoryPaths: []string{invalidDirectory}} _, err := env.Start() Expect(err).NotTo(HaveOccurred()) Expect(env.Stop()).To(Succeed()) - close(done) - }, 30) + }) }) Describe("Stop", func() { - It("should cleanup webhook /tmp folder with no error when using existing cluster", func(done Done) { + It("should cleanup webhook /tmp folder with no error when using existing cluster", func() { env := &Environment{} _, err := env.Start() Expect(err).NotTo(HaveOccurred()) @@ -857,7 +956,6 @@ var _ = Describe("Test", func() { // check if the /tmp/envtest-serving-certs-* dir doesnt exists any more Expect(env.WebhookInstallOptions.LocalServingCertDir).ShouldNot(BeADirectory()) - close(done) - }, 30) + }) }) }) diff --git a/pkg/envtest/ginkgo_test.go b/pkg/envtest/ginkgo_test.go deleted file mode 100644 index fba031c954..0000000000 --- a/pkg/envtest/ginkgo_test.go +++ /dev/null @@ -1,27 +0,0 @@ -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package envtest - -import ( - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" -) - -// NewlineReporter is Reporter that Prints a newline after the default Reporter output so that the results -// are correctly parsed by test automation. -// See issue https://github.com/jstemmer/go-junit-report/issues/31 -// It's re-exported here to avoid compatibility breakage/mass rewrites. -type NewlineReporter = printer.NewlineReporter diff --git a/pkg/envtest/helper.go b/pkg/envtest/helper.go index 79373dd407..d3b52017d2 100644 --- a/pkg/envtest/helper.go +++ b/pkg/envtest/helper.go @@ -18,20 +18,16 @@ package envtest import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client" + "k8s.io/client-go/kubernetes/scheme" ) var ( - crdScheme = runtime.NewScheme() + crdScheme = scheme.Scheme ) // init is required to correctly initialize the crdScheme package variable. func init() { _ = apiextensionsv1.AddToScheme(crdScheme) - _ = apiextensionsv1beta1.AddToScheme(crdScheme) } // mergePaths merges two string slices containing paths. @@ -55,32 +51,19 @@ func mergePaths(s1, s2 []string) []string { // mergeCRDs merges two CRD slices using their names. // This function makes no guarantees about order of the merged slice. -func mergeCRDs(s1, s2 []client.Object) []client.Object { - m := make(map[string]*unstructured.Unstructured) - for _, obj := range runtimeCRDListToUnstructured(s1) { +func mergeCRDs(s1, s2 []*apiextensionsv1.CustomResourceDefinition) []*apiextensionsv1.CustomResourceDefinition { + m := make(map[string]*apiextensionsv1.CustomResourceDefinition) + for _, obj := range s1 { m[obj.GetName()] = obj } - for _, obj := range runtimeCRDListToUnstructured(s2) { + for _, obj := range s2 { m[obj.GetName()] = obj } - merged := make([]client.Object, len(m)) + merged := make([]*apiextensionsv1.CustomResourceDefinition, len(m)) i := 0 for _, obj := range m { - merged[i] = obj + merged[i] = obj.DeepCopy() i++ } return merged } - -func runtimeCRDListToUnstructured(l []client.Object) []*unstructured.Unstructured { - res := []*unstructured.Unstructured{} - for _, obj := range l { - u := &unstructured.Unstructured{} - if err := crdScheme.Convert(obj, u, nil); err != nil { - log.Error(err, "error converting to unstructured object", "object-kind", obj.GetObjectKind()) - continue - } - res = append(res, u) - } - return res -} diff --git a/pkg/envtest/komega/OWNERS b/pkg/envtest/komega/OWNERS new file mode 100644 index 0000000000..ba347dae2b --- /dev/null +++ b/pkg/envtest/komega/OWNERS @@ -0,0 +1,14 @@ +approvers: + - controller-runtime-admins + - controller-runtime-maintainers + - controller-runtime-approvers + - schrej + - JoelSpeed + - sbueringer +reviewers: + - controller-runtime-admins + - controller-runtime-reviewers + - controller-runtime-approvers + - schrej + - JoelSpeed + - sbueringer diff --git a/pkg/envtest/komega/default.go b/pkg/envtest/komega/default.go new file mode 100644 index 0000000000..48fb927a20 --- /dev/null +++ b/pkg/envtest/komega/default.go @@ -0,0 +1,104 @@ +package komega + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// defaultK is the Komega used by the package global functions. +var defaultK = &komega{ctx: context.Background()} + +// SetClient sets the client used by the package global functions. +func SetClient(c client.Client) { + defaultK.client = c +} + +// SetContext sets the context used by the package global functions. +func SetContext(c context.Context) { + defaultK.ctx = c +} + +func checkDefaultClient() { + if defaultK.client == nil { + panic("Default Komega's client is not set. Use SetClient to set it.") + } +} + +// Get returns a function that fetches a resource and returns the occurring error. +// It can be used with gomega.Eventually() like this +// +// deployment := appsv1.Deployment{ ... } +// gomega.Eventually(komega.Get(&deployment)).To(gomega.Succeed()) +// +// By calling the returned function directly it can also be used with gomega.Expect(komega.Get(...)()).To(...) +func Get(obj client.Object) func() error { + checkDefaultClient() + return defaultK.Get(obj) +} + +// List returns a function that lists resources and returns the occurring error. +// It can be used with gomega.Eventually() like this +// +// deployments := v1.DeploymentList{ ... } +// gomega.Eventually(k.List(&deployments)).To(gomega.Succeed()) +// +// By calling the returned function directly it can also be used as gomega.Expect(k.List(...)()).To(...) +func List(list client.ObjectList, opts ...client.ListOption) func() error { + checkDefaultClient() + return defaultK.List(list, opts...) +} + +// Update returns a function that fetches a resource, applies the provided update function and then updates the resource. +// It can be used with gomega.Eventually() like this: +// +// deployment := appsv1.Deployment{ ... } +// gomega.Eventually(k.Update(&deployment, func (o client.Object) { +// deployment.Spec.Replicas = 3 +// return &deployment +// })).To(gomega.Succeed()) +// +// By calling the returned function directly it can also be used as gomega.Expect(k.Update(...)()).To(...) +func Update(obj client.Object, f func(), opts ...client.UpdateOption) func() error { + checkDefaultClient() + return defaultK.Update(obj, f, opts...) +} + +// UpdateStatus returns a function that fetches a resource, applies the provided update function and then updates the resource's status. +// It can be used with gomega.Eventually() like this: +// +// deployment := appsv1.Deployment{ ... } +// gomega.Eventually(k.UpdateStatus(&deployment, func (o client.Object) { +// deployment.Status.AvailableReplicas = 1 +// return &deployment +// })).To(gomega.Succeed()) +// +// By calling the returned function directly it can also be used as gomega.Expect(k.UpdateStatus(...)()).To(...) +func UpdateStatus(obj client.Object, f func(), opts ...client.SubResourceUpdateOption) func() error { + checkDefaultClient() + return defaultK.UpdateStatus(obj, f, opts...) +} + +// Object returns a function that fetches a resource and returns the object. +// It can be used with gomega.Eventually() like this: +// +// deployment := appsv1.Deployment{ ... } +// gomega.Eventually(k.Object(&deployment)).To(HaveField("Spec.Replicas", gomega.Equal(pointer.Int32(3)))) +// +// By calling the returned function directly it can also be used as gomega.Expect(k.Object(...)()).To(...) +func Object(obj client.Object) func() (client.Object, error) { + checkDefaultClient() + return defaultK.Object(obj) +} + +// ObjectList returns a function that fetches a resource and returns the object. +// It can be used with gomega.Eventually() like this: +// +// deployments := appsv1.DeploymentList{ ... } +// gomega.Eventually(k.ObjectList(&deployments)).To(HaveField("Items", HaveLen(1))) +// +// By calling the returned function directly it can also be used as gomega.Expect(k.ObjectList(...)()).To(...) +func ObjectList(list client.ObjectList, opts ...client.ListOption) func() (client.ObjectList, error) { + checkDefaultClient() + return defaultK.ObjectList(list, opts...) +} diff --git a/pkg/envtest/komega/default_test.go b/pkg/envtest/komega/default_test.go new file mode 100644 index 0000000000..238a4abd9e --- /dev/null +++ b/pkg/envtest/komega/default_test.go @@ -0,0 +1,116 @@ +package komega + +import ( + "testing" + + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" +) + +func TestDefaultGet(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + SetClient(fc) + + fetched := appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "test", + }, + } + g.Eventually(Get(&fetched)).Should(Succeed()) + + g.Expect(*fetched.Spec.Replicas).To(BeEquivalentTo(5)) +} + +func TestDefaultList(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + SetClient(fc) + + list := appsv1.DeploymentList{} + g.Eventually(List(&list)).Should(Succeed()) + + g.Expect(list.Items).To(HaveLen(1)) + depl := exampleDeployment() + g.Expect(list.Items[0]).To(And( + HaveField("ObjectMeta.Name", Equal(depl.ObjectMeta.Name)), + HaveField("ObjectMeta.Namespace", Equal(depl.ObjectMeta.Namespace)), + )) +} + +func TestDefaultUpdate(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + SetClient(fc) + + updateDeployment := appsv1.Deployment{ + ObjectMeta: exampleDeployment().ObjectMeta, + } + g.Eventually(Update(&updateDeployment, func() { + updateDeployment.Annotations = map[string]string{"updated": "true"} + })).Should(Succeed()) + + fetched := appsv1.Deployment{ + ObjectMeta: exampleDeployment().ObjectMeta, + } + g.Expect(Object(&fetched)()).To(HaveField("ObjectMeta.Annotations", HaveKeyWithValue("updated", "true"))) +} + +func TestDefaultUpdateStatus(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + SetClient(fc) + + updateDeployment := appsv1.Deployment{ + ObjectMeta: exampleDeployment().ObjectMeta, + } + g.Eventually(UpdateStatus(&updateDeployment, func() { + updateDeployment.Status.AvailableReplicas = 1 + })).Should(Succeed()) + + fetched := appsv1.Deployment{ + ObjectMeta: exampleDeployment().ObjectMeta, + } + g.Expect(Object(&fetched)()).To(HaveField("Status.AvailableReplicas", BeEquivalentTo(1))) +} + +func TestDefaultObject(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + SetClient(fc) + + fetched := appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "test", + }, + } + g.Eventually(Object(&fetched)).Should(And( + Not(BeNil()), + HaveField("Spec.Replicas", Equal(pointer.Int32(5))), + )) +} + +func TestDefaultObjectList(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + SetClient(fc) + + list := appsv1.DeploymentList{} + g.Eventually(ObjectList(&list)).Should(And( + Not(BeNil()), + HaveField("Items", And( + HaveLen(1), + ContainElement(HaveField("Spec.Replicas", Equal(pointer.Int32(5)))), + )), + )) +} diff --git a/pkg/envtest/komega/equalobject.go b/pkg/envtest/komega/equalobject.go new file mode 100644 index 0000000000..a931c2718a --- /dev/null +++ b/pkg/envtest/komega/equalobject.go @@ -0,0 +1,297 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package komega + +import ( + "fmt" + "reflect" + "strings" + + "github.com/google/go-cmp/cmp" + "github.com/onsi/gomega/format" + "github.com/onsi/gomega/types" + "k8s.io/apimachinery/pkg/runtime" +) + +// These package variables hold pre-created commonly used options that can be used to reduce the manual work involved in +// identifying the paths that need to be compared for testing equality between objects. +var ( + // IgnoreAutogeneratedMetadata contains the paths for all the metadata fields that are commonly set by the + // client and APIServer. This is used as a MatchOption for situations when only user-provided metadata is relevant. + IgnoreAutogeneratedMetadata = IgnorePaths{ + "metadata.uid", + "metadata.generation", + "metadata.creationTimestamp", + "metadata.resourceVersion", + "metadata.managedFields", + "metadata.deletionGracePeriodSeconds", + "metadata.deletionTimestamp", + "metadata.selfLink", + "metadata.generateName", + } +) + +type diffPath struct { + types []string + json []string +} + +// equalObjectMatcher is a Gomega matcher used to establish equality between two Kubernetes runtime.Objects. +type equalObjectMatcher struct { + // original holds the object that will be used to Match. + original runtime.Object + + // diffPaths contains the paths that differ between two objects. + diffPaths []diffPath + + // options holds the options that identify what should and should not be matched. + options *EqualObjectOptions +} + +// EqualObject returns a Matcher for the passed Kubernetes runtime.Object with the passed Options. This function can be +// used as a Gomega Matcher in Gomega Assertions. +func EqualObject(original runtime.Object, opts ...EqualObjectOption) types.GomegaMatcher { + matchOptions := &EqualObjectOptions{} + matchOptions = matchOptions.ApplyOptions(opts) + + return &equalObjectMatcher{ + options: matchOptions, + original: original, + } +} + +// Match compares the current object to the passed object and returns true if the objects are the same according to +// the Matcher and MatchOptions. +func (m *equalObjectMatcher) Match(actual interface{}) (success bool, err error) { + // Nil checks required first here for: + // 1) Nil equality which returns true + // 2) One object nil which returns an error + actualIsNil := reflect.ValueOf(actual).IsNil() + originalIsNil := reflect.ValueOf(m.original).IsNil() + + if actualIsNil && originalIsNil { + return true, nil + } + if actualIsNil || originalIsNil { + return false, fmt.Errorf("can not compare an object with a nil. original %v , actual %v", m.original, actual) + } + + m.diffPaths = m.calculateDiff(actual) + return len(m.diffPaths) == 0, nil +} + +// FailureMessage returns a message comparing the full objects after an unexpected failure to match has occurred. +func (m *equalObjectMatcher) FailureMessage(actual interface{}) (message string) { + return fmt.Sprintf("the following fields were expected to match but did not:\n%v\n%s", m.diffPaths, + format.Message(actual, "expected to match", m.original)) +} + +// NegatedFailureMessage returns a string stating that all fields matched, even though that was not expected. +func (m *equalObjectMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return "it was expected that some fields do not match, but all of them did" +} + +func (d diffPath) String() string { + return fmt.Sprintf("(%s/%s)", strings.Join(d.types, "."), strings.Join(d.json, ".")) +} + +// diffReporter is a custom recorder for cmp.Diff which records all paths that are +// different between two objects. +type diffReporter struct { + stack []cmp.PathStep + + diffPaths []diffPath +} + +func (r *diffReporter) PushStep(s cmp.PathStep) { + r.stack = append(r.stack, s) +} + +func (r *diffReporter) Report(res cmp.Result) { + if !res.Equal() { + r.diffPaths = append(r.diffPaths, r.currentPath()) + } +} + +// currentPath converts the current stack into string representations that match +// the IgnorePaths and MatchPaths syntax. +func (r *diffReporter) currentPath() diffPath { + p := diffPath{types: []string{""}, json: []string{""}} + for si, s := range r.stack[1:] { + switch s := s.(type) { + case cmp.StructField: + p.types = append(p.types, s.String()[1:]) + // fetch the type information from the parent struct. + // Note: si has an offset of 1 compared to r.stack as we loop over r.stack[1:], so we don't need -1 + field := r.stack[si].Type().Field(s.Index()) + p.json = append(p.json, strings.Split(field.Tag.Get("json"), ",")[0]) + case cmp.SliceIndex: + key := fmt.Sprintf("[%d]", s.Key()) + p.types[len(p.types)-1] += key + p.json[len(p.json)-1] += key + case cmp.MapIndex: + key := fmt.Sprintf("%v", s.Key()) + if strings.ContainsAny(key, ".[]/\\") { + key = fmt.Sprintf("[%s]", key) + p.types[len(p.types)-1] += key + p.json[len(p.json)-1] += key + } else { + p.types = append(p.types, key) + p.json = append(p.json, key) + } + } + } + // Empty strings were added as the first element. If they're still empty, remove them again. + if len(p.json) > 0 && len(p.json[0]) == 0 { + p.json = p.json[1:] + p.types = p.types[1:] + } + return p +} + +func (r *diffReporter) PopStep() { + r.stack = r.stack[:len(r.stack)-1] +} + +// calculateDiff calculates the difference between two objects and returns the +// paths of the fields that do not match. +func (m *equalObjectMatcher) calculateDiff(actual interface{}) []diffPath { + var original interface{} = m.original + // Remove the wrapping Object from unstructured.Unstructured to make comparison behave similar to + // regular objects. + if u, isUnstructured := actual.(runtime.Unstructured); isUnstructured { + actual = u.UnstructuredContent() + } + if u, ok := m.original.(runtime.Unstructured); ok { + original = u.UnstructuredContent() + } + r := diffReporter{} + cmp.Diff(original, actual, cmp.Reporter(&r)) + return filterDiffPaths(*m.options, r.diffPaths) +} + +// filterDiffPaths filters the diff paths using the paths in EqualObjectOptions. +func filterDiffPaths(opts EqualObjectOptions, paths []diffPath) []diffPath { + result := []diffPath{} + + for _, p := range paths { + if len(opts.matchPaths) > 0 && !hasAnyPathPrefix(p, opts.matchPaths) { + continue + } + if hasAnyPathPrefix(p, opts.ignorePaths) { + continue + } + + result = append(result, p) + } + + return result +} + +// hasPathPrefix compares the segments of a path. +func hasPathPrefix(path []string, prefix []string) bool { + for i, p := range prefix { + if i >= len(path) { + return false + } + // return false if a segment doesn't match + if path[i] != p && (i < len(prefix)-1 || !segmentHasPrefix(path[i], p)) { + return false + } + } + return true +} + +func segmentHasPrefix(s, prefix string) bool { + return len(s) >= len(prefix) && s[0:len(prefix)] == prefix && + // if it is a prefix match, make sure the next character is a [ for array/map access + (len(s) == len(prefix) || s[len(prefix)] == '[') +} + +// hasAnyPathPrefix returns true if path matches any of the path prefixes. +// It respects the name boundaries within paths, so 'ObjectMeta.Name' does not +// match 'ObjectMeta.Namespace' for example. +func hasAnyPathPrefix(path diffPath, prefixes [][]string) bool { + for _, prefix := range prefixes { + if hasPathPrefix(path.types, prefix) || hasPathPrefix(path.json, prefix) { + return true + } + } + return false +} + +// EqualObjectOption describes an Option that can be applied to a Matcher. +type EqualObjectOption interface { + // ApplyToEqualObjectMatcher applies this configuration to the given MatchOption. + ApplyToEqualObjectMatcher(options *EqualObjectOptions) +} + +// EqualObjectOptions holds the available types of EqualObjectOptions that can be applied to a Matcher. +type EqualObjectOptions struct { + ignorePaths [][]string + matchPaths [][]string +} + +// ApplyOptions adds the passed MatchOptions to the MatchOptions struct. +func (o *EqualObjectOptions) ApplyOptions(opts []EqualObjectOption) *EqualObjectOptions { + for _, opt := range opts { + opt.ApplyToEqualObjectMatcher(o) + } + return o +} + +// IgnorePaths instructs the Matcher to ignore given paths when computing a diff. +// Paths are written in a syntax similar to Go with a few special cases. Both types and +// json/yaml field names are supported. +// +// Regular Paths: +// * "ObjectMeta.Name" +// * "metadata.name" +// Arrays: +// * "metadata.ownerReferences[0].name" +// Maps, if they do not contain any of .[]/\: +// * "metadata.labels.something" +// Maps, if they contain any of .[]/\: +// * "metadata.labels[kubernetes.io/something]" +type IgnorePaths []string + +// ApplyToEqualObjectMatcher applies this configuration to the given MatchOptions. +func (i IgnorePaths) ApplyToEqualObjectMatcher(opts *EqualObjectOptions) { + for _, p := range i { + opts.ignorePaths = append(opts.ignorePaths, strings.Split(p, ".")) + } +} + +// MatchPaths instructs the Matcher to restrict its diff to the given paths. If empty the Matcher will look at all paths. +// Paths are written in a syntax similar to Go with a few special cases. Both types and +// json/yaml field names are supported. +// +// Regular Paths: +// * "ObjectMeta.Name" +// * "metadata.name" +// Arrays: +// * "metadata.ownerReferences[0].name" +// Maps, if they do not contain any of .[]/\: +// * "metadata.labels.something" +// Maps, if they contain any of .[]/\: +// * "metadata.labels[kubernetes.io/something]" +type MatchPaths []string + +// ApplyToEqualObjectMatcher applies this configuration to the given MatchOptions. +func (i MatchPaths) ApplyToEqualObjectMatcher(opts *EqualObjectOptions) { + for _, p := range i { + opts.matchPaths = append(opts.ignorePaths, strings.Split(p, ".")) + } +} diff --git a/pkg/envtest/komega/equalobject_test.go b/pkg/envtest/komega/equalobject_test.go new file mode 100644 index 0000000000..9fe10d1779 --- /dev/null +++ b/pkg/envtest/komega/equalobject_test.go @@ -0,0 +1,662 @@ +package komega + +import ( + "testing" + + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestEqualObjectMatcher(t *testing.T) { + cases := []struct { + name string + original client.Object + modified client.Object + options []EqualObjectOption + want bool + }{ + { + name: "succeed with equal objects", + original: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + }, + modified: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + }, + want: true, + }, + { + name: "fail with non equal objects", + original: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + }, + modified: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "somethingelse", + }, + }, + want: false, + }, + { + name: "succeeds if ignored fields do not match", + original: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Labels: map[string]string{"somelabel": "somevalue"}, + OwnerReferences: []metav1.OwnerReference{{ + Name: "controller", + }}, + }, + }, + modified: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "somethingelse", + Labels: map[string]string{"somelabel": "anothervalue"}, + OwnerReferences: []metav1.OwnerReference{{ + Name: "another", + }}, + }, + }, + want: true, + options: []EqualObjectOption{ + IgnorePaths{ + "ObjectMeta.Name", + "ObjectMeta.CreationTimestamp", + "ObjectMeta.Labels.somelabel", + "ObjectMeta.OwnerReferences[0].Name", + "Spec.Template.ObjectMeta", + }, + }, + }, + { + name: "succeeds if ignored fields in json notation do not match", + original: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Labels: map[string]string{"somelabel": "somevalue"}, + OwnerReferences: []metav1.OwnerReference{{ + Name: "controller", + }}, + }, + }, + modified: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "somethingelse", + Labels: map[string]string{"somelabel": "anothervalue"}, + OwnerReferences: []metav1.OwnerReference{{ + Name: "another", + }}, + }, + }, + want: true, + options: []EqualObjectOption{ + IgnorePaths{ + "metadata.name", + "metadata.creationTimestamp", + "metadata.labels.somelabel", + "metadata.ownerReferences[0].name", + "spec.template.metadata", + }, + }, + }, + { + name: "succeeds if all allowed fields match, and some others do not", + original: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "default", + }, + }, + modified: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "special", + }, + }, + want: true, + options: []EqualObjectOption{ + MatchPaths{ + "ObjectMeta.Name", + }, + }, + }, + { + name: "works with unstructured.Unstructured", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "something", + "namespace": "test", + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "somethingelse", + "namespace": "test", + }, + }, + }, + want: true, + options: []EqualObjectOption{ + IgnorePaths{ + "metadata.name", + }, + }, + }, + + // Test when objects are equal. + { + name: "Equal field (spec) both in original and in modified", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + want: true, + }, + + { + name: "Equal nested field both in original and in modified", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + }, + want: true, + }, + + // Test when there is a difference between the objects. + { + name: "Unequal field both in original and in modified", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "foo": "bar-changed", + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + want: false, + }, + { + name: "Unequal nested field both in original and modified", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A-Changed", + }, + }, + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + }, + want: false, + }, + + { + name: "Value of type map with different values", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "map": map[string]string{ + "A": "A-changed", + "B": "B", + // C missing + }, + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "map": map[string]string{ + "A": "A", + // B missing + "C": "C", + }, + }, + }, + }, + want: false, + }, + + { + name: "Value of type Array or Slice with same length but different values", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "slice": []string{ + "D", + "C", + "B", + }, + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "slice": []string{ + "A", + "B", + "C", + }, + }, + }, + }, + want: false, + }, + + // This tests specific behaviour in how Kubernetes marshals the zero value of metav1.Time{}. + { + name: "Creation timestamp set to empty value on both original and modified", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + "metadata": map[string]interface{}{ + "selfLink": "foo", + "creationTimestamp": metav1.Time{}, + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + "metadata": map[string]interface{}{ + "selfLink": "foo", + "creationTimestamp": metav1.Time{}, + }, + }, + }, + want: true, + }, + + // Cases to test diff when fields exist only in modified object. + { + name: "Field only in modified", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{}, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + want: false, + }, + { + name: "Nested field only in modified", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{}, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + }, + want: false, + }, + { + name: "Creation timestamp exists on modified but not on original", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + "metadata": map[string]interface{}{ + "selfLink": "foo", + "creationTimestamp": "2021-11-03T11:05:17Z", + }, + }, + }, + want: false, + }, + + // Test when fields exists only in the original object. + { + name: "Field only in original", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{}, + }, + want: false, + }, + { + name: "Nested field only in original", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{}, + }, + want: false, + }, + { + name: "Creation timestamp exists on original but not on modified", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + "metadata": map[string]interface{}{ + "selfLink": "foo", + "creationTimestamp": "2021-11-03T11:05:17Z", + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + + want: false, + }, + + // Test metadata fields computed by the system or in status are compared. + { + name: "Unequal Metadata fields computed by the system or in status", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{}, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "selfLink": "foo", + "uid": "foo", + "resourceVersion": "foo", + "generation": "foo", + "managedFields": "foo", + }, + "status": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + want: false, + }, + { + name: "Unequal labels and annotations", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{}, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": map[string]interface{}{ + "foo": "bar", + }, + "annotations": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + }, + want: false, + }, + + // Ignore fields MatchOption + { + name: "Unequal metadata fields ignored by IgnorePaths MatchOption", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "test", + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "test", + "selfLink": "foo", + "uid": "foo", + "resourceVersion": "foo", + "generation": "foo", + "managedFields": "foo", + }, + }, + }, + options: []EqualObjectOption{IgnoreAutogeneratedMetadata}, + want: true, + }, + { + name: "Unequal labels and annotations ignored by IgnorePaths MatchOption", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "test", + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": "test", + "labels": map[string]interface{}{ + "foo": "bar", + }, + "annotations": map[string]interface{}{ + "foo": "bar", + }, + }, + }, + }, + options: []EqualObjectOption{IgnorePaths{"metadata.labels", "metadata.annotations"}}, + want: true, + }, + { + name: "Ignore fields are not compared", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{}, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "controlPlaneEndpoint": map[string]interface{}{ + "host": "", + "port": 0, + }, + }, + }, + }, + options: []EqualObjectOption{IgnorePaths{"spec.controlPlaneEndpoint"}}, + want: true, + }, + { + name: "Not-ignored fields are still compared", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "annotations": map[string]interface{}{}, + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "metadata": map[string]interface{}{ + "annotations": map[string]interface{}{ + "ignored": "somevalue", + "superflous": "shouldcausefailure", + }, + }, + }, + }, + options: []EqualObjectOption{IgnorePaths{"metadata.annotations.ignored"}}, + want: false, + }, + + // MatchPaths MatchOption + { + name: "Unequal metadata fields not compared by setting MatchPaths MatchOption", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + }, + "metadata": map[string]interface{}{ + "selfLink": "foo", + "uid": "foo", + }, + }, + }, + options: []EqualObjectOption{MatchPaths{"spec"}}, + want: true, + }, + + // More tests + { + name: "No changes", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + "B": "B", + "C": "C", // C only in original + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + "B": "B", + }, + }, + }, + want: false, + }, + { + name: "Many changes", + original: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + // B missing + "C": "C", // C only in original + }, + }, + }, + modified: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "spec": map[string]interface{}{ + "A": "A", + "B": "B", + }, + }, + }, + want: false, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + g := NewWithT(t) + m := EqualObject(c.original, c.options...) + success, _ := m.Match(c.modified) + if !success { + t.Log(m.FailureMessage(c.modified)) + } + g.Expect(success).To(Equal(c.want)) + }) + } +} diff --git a/pkg/envtest/komega/interfaces.go b/pkg/envtest/komega/interfaces.go new file mode 100644 index 0000000000..6f7e5db35b --- /dev/null +++ b/pkg/envtest/komega/interfaces.go @@ -0,0 +1,78 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package komega + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Komega is a collection of utilites for writing tests involving a mocked +// Kubernetes API. +type Komega interface { + // Get returns a function that fetches a resource and returns the occurring error. + // It can be used with gomega.Eventually() like this + // deployment := appsv1.Deployment{ ... } + // gomega.Eventually(k.Get(&deployment)).To(gomega.Succeed()) + // By calling the returned function directly it can also be used with gomega.Expect(k.Get(...)()).To(...) + Get(client.Object) func() error + + // List returns a function that lists resources and returns the occurring error. + // It can be used with gomega.Eventually() like this + // deployments := v1.DeploymentList{ ... } + // gomega.Eventually(k.List(&deployments)).To(gomega.Succeed()) + // By calling the returned function directly it can also be used as gomega.Expect(k.List(...)()).To(...) + List(client.ObjectList, ...client.ListOption) func() error + + // Update returns a function that fetches a resource, applies the provided update function and then updates the resource. + // It can be used with gomega.Eventually() like this: + // deployment := appsv1.Deployment{ ... } + // gomega.Eventually(k.Update(&deployment, func (o client.Object) { + // deployment.Spec.Replicas = 3 + // return &deployment + // })).To(gomega.Succeed()) + // By calling the returned function directly it can also be used as gomega.Expect(k.Update(...)()).To(...) + Update(client.Object, func(), ...client.UpdateOption) func() error + + // UpdateStatus returns a function that fetches a resource, applies the provided update function and then updates the resource's status. + // It can be used with gomega.Eventually() like this: + // deployment := appsv1.Deployment{ ... } + // gomega.Eventually(k.Update(&deployment, func (o client.Object) { + // deployment.Status.AvailableReplicas = 1 + // return &deployment + // })).To(gomega.Succeed()) + // By calling the returned function directly it can also be used as gomega.Expect(k.UpdateStatus(...)()).To(...) + UpdateStatus(client.Object, func(), ...client.SubResourceUpdateOption) func() error + + // Object returns a function that fetches a resource and returns the object. + // It can be used with gomega.Eventually() like this: + // deployment := appsv1.Deployment{ ... } + // gomega.Eventually(k.Object(&deployment)).To(HaveField("Spec.Replicas", gomega.Equal(pointer.Int32(3)))) + // By calling the returned function directly it can also be used as gomega.Expect(k.Object(...)()).To(...) + Object(client.Object) func() (client.Object, error) + + // ObjectList returns a function that fetches a resource and returns the object. + // It can be used with gomega.Eventually() like this: + // deployments := appsv1.DeploymentList{ ... } + // gomega.Eventually(k.ObjectList(&deployments)).To(HaveField("Items", HaveLen(1))) + // By calling the returned function directly it can also be used as gomega.Expect(k.ObjectList(...)()).To(...) + ObjectList(client.ObjectList, ...client.ListOption) func() (client.ObjectList, error) + + // WithContext returns a copy that uses the given context. + WithContext(context.Context) Komega +} diff --git a/pkg/envtest/komega/komega.go b/pkg/envtest/komega/komega.go new file mode 100644 index 0000000000..e19d9b5f0b --- /dev/null +++ b/pkg/envtest/komega/komega.go @@ -0,0 +1,117 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package komega + +import ( + "context" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// komega is a collection of utilites for writing tests involving a mocked +// Kubernetes API. +type komega struct { + ctx context.Context + client client.Client +} + +var _ Komega = &komega{} + +// New creates a new Komega instance with the given client. +func New(c client.Client) Komega { + return &komega{ + client: c, + ctx: context.Background(), + } +} + +// WithContext returns a copy that uses the given context. +func (k komega) WithContext(ctx context.Context) Komega { + k.ctx = ctx + return &k +} + +// Get returns a function that fetches a resource and returns the occurring error. +func (k *komega) Get(obj client.Object) func() error { + key := types.NamespacedName{ + Name: obj.GetName(), + Namespace: obj.GetNamespace(), + } + return func() error { + return k.client.Get(k.ctx, key, obj) + } +} + +// List returns a function that lists resources and returns the occurring error. +func (k *komega) List(obj client.ObjectList, opts ...client.ListOption) func() error { + return func() error { + return k.client.List(k.ctx, obj, opts...) + } +} + +// Update returns a function that fetches a resource, applies the provided update function and then updates the resource. +func (k *komega) Update(obj client.Object, updateFunc func(), opts ...client.UpdateOption) func() error { + key := types.NamespacedName{ + Name: obj.GetName(), + Namespace: obj.GetNamespace(), + } + return func() error { + err := k.client.Get(k.ctx, key, obj) + if err != nil { + return err + } + updateFunc() + return k.client.Update(k.ctx, obj, opts...) + } +} + +// UpdateStatus returns a function that fetches a resource, applies the provided update function and then updates the resource's status. +func (k *komega) UpdateStatus(obj client.Object, updateFunc func(), opts ...client.SubResourceUpdateOption) func() error { + key := types.NamespacedName{ + Name: obj.GetName(), + Namespace: obj.GetNamespace(), + } + return func() error { + err := k.client.Get(k.ctx, key, obj) + if err != nil { + return err + } + updateFunc() + return k.client.Status().Update(k.ctx, obj, opts...) + } +} + +// Object returns a function that fetches a resource and returns the object. +func (k *komega) Object(obj client.Object) func() (client.Object, error) { + key := types.NamespacedName{ + Name: obj.GetName(), + Namespace: obj.GetNamespace(), + } + return func() (client.Object, error) { + err := k.client.Get(k.ctx, key, obj) + return obj, err + } +} + +// ObjectList returns a function that fetches a resource and returns the object. +func (k *komega) ObjectList(obj client.ObjectList, opts ...client.ListOption) func() (client.ObjectList, error) { + return func() (client.ObjectList, error) { + err := k.client.List(k.ctx, obj, opts...) + return obj, err + } +} diff --git a/pkg/envtest/komega/komega_test.go b/pkg/envtest/komega/komega_test.go new file mode 100644 index 0000000000..275610c8bb --- /dev/null +++ b/pkg/envtest/komega/komega_test.go @@ -0,0 +1,138 @@ +package komega + +import ( + "testing" + + _ "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func exampleDeployment() *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "test", + }, + Spec: appsv1.DeploymentSpec{ + Replicas: pointer.Int32(5), + }, + } +} + +func createFakeClient() client.Client { + return fakeclient.NewClientBuilder(). + WithObjects(exampleDeployment()). + Build() +} + +func TestGet(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + k := New(fc) + + fetched := appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "test", + }, + } + g.Eventually(k.Get(&fetched)).Should(Succeed()) + + g.Expect(*fetched.Spec.Replicas).To(BeEquivalentTo(5)) +} + +func TestList(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + k := New(fc) + + list := appsv1.DeploymentList{} + g.Eventually(k.List(&list)).Should(Succeed()) + + g.Expect(list.Items).To(HaveLen(1)) + depl := exampleDeployment() + g.Expect(list.Items[0]).To(And( + HaveField("ObjectMeta.Name", Equal(depl.ObjectMeta.Name)), + HaveField("ObjectMeta.Namespace", Equal(depl.ObjectMeta.Namespace)), + )) +} + +func TestUpdate(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + k := New(fc) + + updateDeployment := appsv1.Deployment{ + ObjectMeta: exampleDeployment().ObjectMeta, + } + g.Eventually(k.Update(&updateDeployment, func() { + updateDeployment.Annotations = map[string]string{"updated": "true"} + })).Should(Succeed()) + + fetched := appsv1.Deployment{ + ObjectMeta: exampleDeployment().ObjectMeta, + } + g.Expect(k.Object(&fetched)()).To(HaveField("ObjectMeta.Annotations", HaveKeyWithValue("updated", "true"))) +} + +func TestUpdateStatus(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + k := New(fc) + + updateDeployment := appsv1.Deployment{ + ObjectMeta: exampleDeployment().ObjectMeta, + } + g.Eventually(k.UpdateStatus(&updateDeployment, func() { + updateDeployment.Status.AvailableReplicas = 1 + })).Should(Succeed()) + + fetched := appsv1.Deployment{ + ObjectMeta: exampleDeployment().ObjectMeta, + } + g.Expect(k.Object(&fetched)()).To(HaveField("Status.AvailableReplicas", BeEquivalentTo(1))) +} + +func TestObject(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + k := New(fc) + + fetched := appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "test", + }, + } + g.Eventually(k.Object(&fetched)).Should(And( + Not(BeNil()), + HaveField("Spec.Replicas", Equal(pointer.Int32(5))), + )) +} + +func TestObjectList(t *testing.T) { + g := NewWithT(t) + + fc := createFakeClient() + k := New(fc) + + list := appsv1.DeploymentList{} + g.Eventually(k.ObjectList(&list)).Should(And( + Not(BeNil()), + HaveField("Items", And( + HaveLen(1), + ContainElement(HaveField("Spec.Replicas", Equal(pointer.Int32(5)))), + )), + )) +} diff --git a/pkg/envtest/printer/ginkgo.go b/pkg/envtest/printer/ginkgo.go deleted file mode 100644 index d835dc7721..0000000000 --- a/pkg/envtest/printer/ginkgo.go +++ /dev/null @@ -1,53 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package printer contains setup for a friendlier Ginkgo printer that's easier -// to parse by test automation. -package printer - -import ( - "fmt" - - "github.com/onsi/ginkgo" - "github.com/onsi/ginkgo/config" - "github.com/onsi/ginkgo/types" -) - -var _ ginkgo.Reporter = NewlineReporter{} - -// NewlineReporter is Reporter that Prints a newline after the default Reporter output so that the results -// are correctly parsed by test automation. -// See issue https://github.com/jstemmer/go-junit-report/issues/31 -type NewlineReporter struct{} - -// SpecSuiteWillBegin implements ginkgo.Reporter. -func (NewlineReporter) SpecSuiteWillBegin(config config.GinkgoConfigType, summary *types.SuiteSummary) { -} - -// BeforeSuiteDidRun implements ginkgo.Reporter. -func (NewlineReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) {} - -// AfterSuiteDidRun implements ginkgo.Reporter. -func (NewlineReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) {} - -// SpecWillRun implements ginkgo.Reporter. -func (NewlineReporter) SpecWillRun(specSummary *types.SpecSummary) {} - -// SpecDidComplete implements ginkgo.Reporter. -func (NewlineReporter) SpecDidComplete(specSummary *types.SpecSummary) {} - -// SpecSuiteDidEnd Prints a newline between "35 Passed | 0 Failed | 0 Pending | 0 Skipped" and "--- PASS:". -func (NewlineReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) { fmt.Printf("\n") } diff --git a/pkg/envtest/printer/prow.go b/pkg/envtest/printer/prow.go deleted file mode 100644 index 2f4009aa03..0000000000 --- a/pkg/envtest/printer/prow.go +++ /dev/null @@ -1,109 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package printer - -import ( - "fmt" - "os" - "path/filepath" - "sync" - - "github.com/onsi/ginkgo" - "github.com/onsi/ginkgo/config" - "github.com/onsi/ginkgo/reporters" - "github.com/onsi/ginkgo/types" - - "k8s.io/apimachinery/pkg/util/sets" -) - -var ( - allRegisteredSuites = sets.String{} - allRegisteredSuitesLock = &sync.Mutex{} -) - -type prowReporter struct { - junitReporter *reporters.JUnitReporter -} - -// NewProwReporter returns a prowReporter that will write out junit if running in Prow and do -// nothing otherwise. -// WARNING: It seems this does not always properly fail the test runs when there are failures, -// see https://github.com/onsi/ginkgo/issues/706 -// When using this you must make sure to grep for failures in your junit xmls and fail the run -// if there are any. -func NewProwReporter(suiteName string) ginkgo.Reporter { - allRegisteredSuitesLock.Lock() - if allRegisteredSuites.Has(suiteName) { - panic(fmt.Sprintf("Suite named %q registered more than once", suiteName)) - } - allRegisteredSuites.Insert(suiteName) - allRegisteredSuitesLock.Unlock() - - if os.Getenv("CI") == "" { - return &prowReporter{} - } - artifactsDir := os.Getenv("ARTIFACTS") - if artifactsDir == "" { - return &prowReporter{} - } - - path := filepath.Join(artifactsDir, fmt.Sprintf("junit_%s_%d.xml", suiteName, config.GinkgoConfig.ParallelNode)) - return &prowReporter{ - junitReporter: reporters.NewJUnitReporter(path), - } -} - -func (pr *prowReporter) SpecSuiteWillBegin(config config.GinkgoConfigType, summary *types.SuiteSummary) { - if pr.junitReporter != nil { - pr.junitReporter.SpecSuiteWillBegin(config, summary) - } -} - -// BeforeSuiteDidRun implements ginkgo.Reporter. -func (pr *prowReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) { - if pr.junitReporter != nil { - pr.junitReporter.BeforeSuiteDidRun(setupSummary) - } -} - -// AfterSuiteDidRun implements ginkgo.Reporter. -func (pr *prowReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) { - if pr.junitReporter != nil { - pr.junitReporter.AfterSuiteDidRun(setupSummary) - } -} - -// SpecWillRun implements ginkgo.Reporter. -func (pr *prowReporter) SpecWillRun(specSummary *types.SpecSummary) { - if pr.junitReporter != nil { - pr.junitReporter.SpecWillRun(specSummary) - } -} - -// SpecDidComplete implements ginkgo.Reporter. -func (pr *prowReporter) SpecDidComplete(specSummary *types.SpecSummary) { - if pr.junitReporter != nil { - pr.junitReporter.SpecDidComplete(specSummary) - } -} - -// SpecSuiteDidEnd Prints a newline between "35 Passed | 0 Failed | 0 Pending | 0 Skipped" and "--- PASS:". -func (pr *prowReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) { - if pr.junitReporter != nil { - pr.junitReporter.SpecSuiteDidEnd(summary) - } -} diff --git a/pkg/envtest/server.go b/pkg/envtest/server.go index 967e3060ed..4ee440df9c 100644 --- a/pkg/envtest/server.go +++ b/pkg/envtest/server.go @@ -22,30 +22,29 @@ import ( "strings" "time" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" + logf "sigs.k8s.io/controller-runtime/pkg/internal/log" "sigs.k8s.io/controller-runtime/pkg/internal/testing/controlplane" "sigs.k8s.io/controller-runtime/pkg/internal/testing/process" - - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" ) var log = logf.RuntimeLog.WithName("test-env") /* It's possible to override some defaults, by setting the following environment variables: - USE_EXISTING_CLUSTER (boolean): if set to true, envtest will use an existing cluster - TEST_ASSET_KUBE_APISERVER (string): path to the api-server binary to use - TEST_ASSET_ETCD (string): path to the etcd binary to use - TEST_ASSET_KUBECTL (string): path to the kubectl binary to use - KUBEBUILDER_ASSETS (string): directory containing the binaries to use (api-server, etcd and kubectl). Defaults to /usr/local/kubebuilder/bin. - KUBEBUILDER_CONTROLPLANE_START_TIMEOUT (string supported by time.ParseDuration): timeout for test control plane to start. Defaults to 20s. - KUBEBUILDER_CONTROLPLANE_STOP_TIMEOUT (string supported by time.ParseDuration): timeout for test control plane to start. Defaults to 20s. - KUBEBUILDER_ATTACH_CONTROL_PLANE_OUTPUT (boolean): if set to true, the control plane's stdout and stderr are attached to os.Stdout and os.Stderr - +* USE_EXISTING_CLUSTER (boolean): if set to true, envtest will use an existing cluster +* TEST_ASSET_KUBE_APISERVER (string): path to the api-server binary to use +* TEST_ASSET_ETCD (string): path to the etcd binary to use +* TEST_ASSET_KUBECTL (string): path to the kubectl binary to use +* KUBEBUILDER_ASSETS (string): directory containing the binaries to use (api-server, etcd and kubectl). Defaults to /usr/local/kubebuilder/bin. +* KUBEBUILDER_CONTROLPLANE_START_TIMEOUT (string supported by time.ParseDuration): timeout for test control plane to start. Defaults to 20s. +* KUBEBUILDER_CONTROLPLANE_STOP_TIMEOUT (string supported by time.ParseDuration): timeout for test control plane to start. Defaults to 20s. +* KUBEBUILDER_ATTACH_CONTROL_PLANE_OUTPUT (boolean): if set to true, the control plane's stdout and stderr are attached to os.Stdout and os.Stderr */ const ( envUseExistingCluster = "USE_EXISTING_CLUSTER" @@ -136,7 +135,7 @@ type Environment struct { // CRDs is a list of CRDs to install. // If both this field and CRDs field in CRDInstallOptions are specified, the // values are merged. - CRDs []client.Object + CRDs []*apiextensionsv1.CustomResourceDefinition // CRDDirectoryPaths is a list of paths containing CRD yaml or json configs. // If both this field and Paths field in CRDInstallOptions are specified, the @@ -231,17 +230,19 @@ func (te *Environment) Start() (*rest.Config, error) { if os.Getenv(envAttachOutput) == "true" { te.AttachControlPlaneOutput = true } - if apiServer.Out == nil && te.AttachControlPlaneOutput { - apiServer.Out = os.Stdout - } - if apiServer.Err == nil && te.AttachControlPlaneOutput { - apiServer.Err = os.Stderr - } - if te.ControlPlane.Etcd.Out == nil && te.AttachControlPlaneOutput { - te.ControlPlane.Etcd.Out = os.Stdout - } - if te.ControlPlane.Etcd.Err == nil && te.AttachControlPlaneOutput { - te.ControlPlane.Etcd.Err = os.Stderr + if te.AttachControlPlaneOutput { + if apiServer.Out == nil { + apiServer.Out = os.Stdout + } + if apiServer.Err == nil { + apiServer.Err = os.Stderr + } + if te.ControlPlane.Etcd.Out == nil { + te.ControlPlane.Etcd.Out = os.Stdout + } + if te.ControlPlane.Etcd.Err == nil { + te.ControlPlane.Etcd.Err = os.Stderr + } } apiServer.Path = process.BinPathFinder("kube-apiserver", te.BinaryAssetsDirectory) diff --git a/pkg/envtest/testdata/crds/examplecrd3.yaml b/pkg/envtest/testdata/crds/examplecrd3.yaml index 1b6b8e7f77..479a6e5645 100644 --- a/pkg/envtest/testdata/crds/examplecrd3.yaml +++ b/pkg/envtest/testdata/crds/examplecrd3.yaml @@ -1,4 +1,4 @@ -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: configs.foo.example.com @@ -8,4 +8,10 @@ spec: kind: Config plural: configs scope: Namespaced - version: "v1beta1" \ No newline at end of file + versions: + - name: "v1beta1" + storage: true + served: true + schema: + openAPIV3Schema: + type: object diff --git a/pkg/envtest/testdata/crds/examplecrd_unserved.yaml b/pkg/envtest/testdata/crds/examplecrd_unserved.yaml index cf759009a0..09fac4f080 100644 --- a/pkg/envtest/testdata/crds/examplecrd_unserved.yaml +++ b/pkg/envtest/testdata/crds/examplecrd_unserved.yaml @@ -1,6 +1,6 @@ --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: @@ -17,42 +17,69 @@ spec: scope: Namespaced subresources: status: {} - validation: - openAPIV3Schema: - description: Frigate is the Schema for the frigates API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: FrigateSpec defines the desired state of Frigate - properties: - foo: - description: Foo is an example field of Frigate. Edit Frigate_types.go - to remove/update - type: string - type: object - status: - description: FrigateStatus defines the observed state of Frigate - type: object - type: object - version: v1 versions: - name: v1 served: false storage: true + schema: + openAPIV3Schema: + description: Frigate is the Schema for the frigates API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: FrigateSpec defines the desired state of Frigate + properties: + foo: + description: Foo is an example field of Frigate. Edit Frigate_types.go + to remove/update + type: string + type: object + status: + description: FrigateStatus defines the observed state of Frigate + type: object + type: object - name: v1beta1 served: false storage: false + schema: + openAPIV3Schema: + description: Frigate is the Schema for the frigates API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: FrigateSpec defines the desired state of Frigate + properties: + foo: + description: Foo is an example field of Frigate. Edit Frigate_types.go + to remove/update + type: string + type: object + status: + description: FrigateStatus defines the observed state of Frigate + type: object + type: object status: acceptedNames: kind: "" diff --git a/pkg/envtest/testdata/crdv1_original/example_multiversion_crd1.yaml b/pkg/envtest/testdata/crdv1_original/example_multiversion_crd1.yaml index 1999d1e02e..5dead8186a 100644 --- a/pkg/envtest/testdata/crdv1_original/example_multiversion_crd1.yaml +++ b/pkg/envtest/testdata/crdv1_original/example_multiversion_crd1.yaml @@ -1,7 +1,6 @@ -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - creationTimestamp: null name: drivers.crew.example.com spec: group: crew.example.com @@ -9,351 +8,51 @@ spec: kind: Driver plural: drivers scope: "" - validation: - openAPIV3Schema: - description: Driver is the Schema for the drivers API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - metadata: - properties: - annotations: - additionalProperties: - type: string - description: 'Annotations is an unstructured key value map stored with - a resource that may be set by external tools to store and retrieve - arbitrary metadata. They are not queryable and should be preserved - when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations' - type: object - clusterName: - description: The name of the cluster which the object belongs to. This - is used to distinguish resources with same name and namespace in different - clusters. This field is not set anywhere right now and apiserver is - going to ignore it if set in create or update request. - type: string - creationTimestamp: - description: "CreationTimestamp is a timestamp representing the server - time when this object was created. It is not guaranteed to be set - in happens-before order across separate operations. Clients may not - set this value. It is represented in RFC3339 form and is in UTC. \n - Populated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata" - format: date-time - type: string - deletionGracePeriodSeconds: - description: Number of seconds allowed for this object to gracefully - terminate before it will be removed from the system. Only set when - deletionTimestamp is also set. May only be shortened. Read-only. - format: int64 - type: integer - deletionTimestamp: - description: "DeletionTimestamp is RFC 3339 date and time at which this - resource will be deleted. This field is set by the server when a graceful - deletion is requested by the user, and is not directly settable by - a client. The resource is expected to be deleted (no longer visible - from resource lists, and not reachable by name) after the time in - this field, once the finalizers list is empty. As long as the finalizers - list contains items, deletion is blocked. Once the deletionTimestamp - is set, this value may not be unset or be set further into the future, - although it may be shortened or the resource may be deleted prior - to this time. For example, a user may request that a pod is deleted - in 30 seconds. The Kubelet will react by sending a graceful termination - signal to the containers in the pod. After that 30 seconds, the Kubelet - will send a hard termination signal (SIGKILL) to the container and - after cleanup, remove the pod from the API. In the presence of network - partitions, this object may still exist after this timestamp, until - an administrator or automated process can determine the resource is - fully terminated. If not set, graceful deletion of the object has - not been requested. \n Populated by the system when a graceful deletion - is requested. Read-only. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata" - format: date-time - type: string - finalizers: - description: Must be empty before the object is deleted from the registry. - Each entry is an identifier for the responsible component that will - remove the entry from the list. If the deletionTimestamp of the object - is non-nil, entries in this list can only be removed. - items: - type: string - type: array - generateName: - description: "GenerateName is an optional prefix, used by the server, - to generate a unique name ONLY IF the Name field has not been provided. - If this field is used, the name returned to the client will be different - than the name passed. This value will also be combined with a unique - suffix. The provided value has the same validation rules as the Name - field, and may be truncated by the length of the suffix required to - make the value unique on the server. \n If this field is specified - and the generated name exists, the server will NOT return a 409 - - instead, it will either return 201 Created or 500 with Reason ServerTimeout - indicating a unique name could not be found in the time allotted, - and the client should retry (optionally after the time indicated in - the Retry-After header). \n Applied only if Name is not specified. - More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#idempotency" - type: string - generation: - description: A sequence number representing a specific generation of - the desired state. Populated by the system. Read-only. - format: int64 - type: integer - initializers: - description: "An initializer is a controller which enforces some system - invariant at object creation time. This field is a list of initializers - that have not yet acted on this object. If nil or empty, this object - has been completely initialized. Otherwise, the object is considered - uninitialized and is hidden (in list/watch and get calls) from clients - that haven't explicitly asked to observe uninitialized objects. \n - When an object is created, the system will populate this list with - the current set of initializers. Only privileged users may set or - modify this list. Once it is empty, it may not be modified further - by any user." - properties: - pending: - description: Pending is a list of initializers that must execute - in order before this object is visible. When the last pending - initializer is removed, and no failing result is set, the initializers - struct will be set to nil and the object is considered as initialized - and visible to all clients. - items: - properties: - name: - description: name of the process that is responsible for initializing - this object. - type: string - required: - - name - type: object - type: array - result: - description: If result is set with the Failure field, the object - will be persisted to storage and then deleted, ensuring that other - clients can observe the deletion. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this - representation of an object. Servers should convert recognized - schemas to the latest internal value, and may reject unrecognized - values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' - type: string - code: - description: Suggested HTTP return code for this status, 0 if - not set. - format: int32 - type: integer - details: - description: Extended data associated with the reason. Each - reason may define its own extended details. This field is - optional and the data returned is not guaranteed to conform - to any schema except that defined by the reason type. - properties: - causes: - description: The Causes array includes more details associated - with the StatusReason failure. Not all StatusReasons may - provide detailed causes. - items: - properties: - field: - description: "The field of the resource that has caused - this error, as named by its JSON serialization. - May include dot and postfix notation for nested - attributes. Arrays are zero-indexed. Fields may - appear more than once in an array of causes due - to fields having multiple errors. Optional. \n Examples: - \ \"name\" - the field \"name\" on the current - resource \"items[0].name\" - the field \"name\" - on the first array entry in \"items\"" - type: string - message: - description: A human-readable description of the cause - of the error. This field may be presented as-is - to a reader. - type: string - reason: - description: A machine-readable description of the - cause of the error. If this value is empty there - is no information available. - type: string - type: object - type: array - group: - description: The group attribute of the resource associated - with the status StatusReason. - type: string - kind: - description: 'The kind attribute of the resource associated - with the status StatusReason. On some operations may differ - from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - name: - description: The name attribute of the resource associated - with the status StatusReason (when there is a single name - which can be described). - type: string - retryAfterSeconds: - description: If specified, the time in seconds before the - operation should be retried. Some errors may indicate - the client must take an alternate action - for those errors - this field may indicate how long to wait before taking - the alternate action. - format: int32 - type: integer - uid: - description: 'UID of the resource. (when there is a single - resource which can be described). More info: http://kubernetes.io/docs/user-guide/identifiers#uids' - type: string - type: object - kind: - description: 'Kind is a string value representing the REST resource - this object represents. Servers may infer this from the endpoint - the client submits requests to. Cannot be updated. In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - message: - description: A human-readable description of the status of this - operation. - type: string - metadata: - description: 'Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - properties: - continue: - description: continue may be set if the user set a limit - on the number of items returned, and indicates that the - server has more data available. The value is opaque and - may be used to issue another request to the endpoint that - served this list to retrieve the next set of available - objects. Continuing a consistent list may not be possible - if the server configuration has changed or more than a - few minutes have passed. The resourceVersion field returned - when using this continue value will be identical to the - value in the first response, unless you have received - this token from an error message. - type: string - resourceVersion: - description: 'String that identifies the server''s internal - version of this object that can be used by clients to - determine when objects have changed. Value must be treated - as opaque by clients and passed unmodified back to the - server. Populated by the system. Read-only. More info: - https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency' - type: string - selfLink: - description: selfLink is a URL representing this object. - Populated by the system. Read-only. - type: string - type: object - reason: - description: A machine-readable description of why this operation - is in the "Failure" status. If this value is empty there is - no information available. A Reason clarifies an HTTP status - code but does not override it. - type: string - status: - description: 'Status of the operation. One of: "Success" or - "Failure". More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status' - type: string - type: object - required: - - pending - type: object - labels: - additionalProperties: - type: string - description: 'Map of string keys and values that can be used to organize - and categorize (scope and select) objects. May match selectors of - replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels' - type: object - name: - description: 'Name must be unique within a namespace. Is required when - creating resources, although some resources may allow a client to - request the generation of an appropriate name automatically. Name - is primarily intended for creation idempotence and configuration definition. - Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names' - type: string - namespace: - description: "Namespace defines the space within each name must be unique. - An empty namespace is equivalent to the \"default\" namespace, but - \"default\" is the canonical representation. Not all objects are required - to be scoped to a namespace - the value of this field for those objects - will be empty. \n Must be a DNS_LABEL. Cannot be updated. More info: - http://kubernetes.io/docs/user-guide/namespaces" - type: string - ownerReferences: - description: List of objects depended by this object. If ALL objects - in the list have been deleted, this object will be garbage collected. - If this object is managed by a controller, then an entry in this list - will point to this controller, with the controller field set to true. - There cannot be more than one managing controller. - items: - properties: - apiVersion: - description: API version of the referent. - type: string - blockOwnerDeletion: - description: If true, AND if the owner has the "foregroundDeletion" - finalizer, then the owner cannot be deleted from the key-value - store until this reference is removed. Defaults to false. To - set this field, a user needs "delete" permission of the owner, - otherwise 422 (Unprocessable Entity) will be returned. - type: boolean - controller: - description: If true, this reference points to the managing controller. - type: boolean - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - name: - description: 'Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names' - type: string - uid: - description: 'UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids' - type: string - required: - - apiVersion - - kind - - name - - uid - type: object - type: array - resourceVersion: - description: "An opaque value that represents the internal version of - this object that can be used by clients to determine when objects - have changed. May be used for optimistic concurrency, change detection, - and the watch operation on a resource or set of resources. Clients - must treat these values as opaque and passed unmodified back to the - server. They may only be valid for a particular resource or set of - resources. \n Populated by the system. Read-only. Value must be treated - as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency" - type: string - selfLink: - description: SelfLink is a URL representing this object. Populated by - the system. Read-only. - type: string - uid: - description: "UID is the unique in time and space value for this object. - It is typically generated by the server on successful creation of - a resource and is not allowed to change on PUT operations. \n Populated - by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids" - type: string - type: object - spec: - type: object - status: - type: object - type: object versions: - name: v1 served: true storage: true + schema: + openAPIV3Schema: + description: Driver is the Schema for the drivers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + type: object + status: + type: object + type: object - name: v2 served: true storage: false + schema: + openAPIV3Schema: + description: Driver is the Schema for the drivers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + type: object + status: + type: object + type: object status: acceptedNames: kind: "" diff --git a/pkg/envtest/testdata/crdv1_updated/example_multiversion_crd1_one_more_version.yaml b/pkg/envtest/testdata/crdv1_updated/example_multiversion_crd1_one_more_version.yaml index e388510f13..9eb0ec91a2 100644 --- a/pkg/envtest/testdata/crdv1_updated/example_multiversion_crd1_one_more_version.yaml +++ b/pkg/envtest/testdata/crdv1_updated/example_multiversion_crd1_one_more_version.yaml @@ -1,362 +1,80 @@ -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - creationTimestamp: null name: drivers.crew.example.com spec: group: crew.example.com names: kind: Driver plural: drivers - scope: "" - validation: - openAPIV3Schema: - description: Driver is the Schema for the drivers API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - metadata: - properties: - annotations: - additionalProperties: - type: string - description: 'Annotations is an unstructured key value map stored with - a resource that may be set by external tools to store and retrieve - arbitrary metadata. They are not queryable and should be preserved - when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations' - type: object - clusterName: - description: The name of the cluster which the object belongs to. This - is used to distinguish resources with same name and namespace in different - clusters. This field is not set anywhere right now and apiserver is - going to ignore it if set in create or update request. - type: string - creationTimestamp: - description: "CreationTimestamp is a timestamp representing the server - time when this object was created. It is not guaranteed to be set - in happens-before order across separate operations. Clients may not - set this value. It is represented in RFC3339 form and is in UTC. \n - Populated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata" - format: date-time - type: string - deletionGracePeriodSeconds: - description: Number of seconds allowed for this object to gracefully - terminate before it will be removed from the system. Only set when - deletionTimestamp is also set. May only be shortened. Read-only. - format: int64 - type: integer - deletionTimestamp: - description: "DeletionTimestamp is RFC 3339 date and time at which this - resource will be deleted. This field is set by the server when a graceful - deletion is requested by the user, and is not directly settable by - a client. The resource is expected to be deleted (no longer visible - from resource lists, and not reachable by name) after the time in - this field, once the finalizers list is empty. As long as the finalizers - list contains items, deletion is blocked. Once the deletionTimestamp - is set, this value may not be unset or be set further into the future, - although it may be shortened or the resource may be deleted prior - to this time. For example, a user may request that a pod is deleted - in 30 seconds. The Kubelet will react by sending a graceful termination - signal to the containers in the pod. After that 30 seconds, the Kubelet - will send a hard termination signal (SIGKILL) to the container and - after cleanup, remove the pod from the API. In the presence of network - partitions, this object may still exist after this timestamp, until - an administrator or automated process can determine the resource is - fully terminated. If not set, graceful deletion of the object has - not been requested. \n Populated by the system when a graceful deletion - is requested. Read-only. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata" - format: date-time - type: string - finalizers: - description: Must be empty before the object is deleted from the registry. - Each entry is an identifier for the responsible component that will - remove the entry from the list. If the deletionTimestamp of the object - is non-nil, entries in this list can only be removed. - items: - type: string - type: array - generateName: - description: "GenerateName is an optional prefix, used by the server, - to generate a unique name ONLY IF the Name field has not been provided. - If this field is used, the name returned to the client will be different - than the name passed. This value will also be combined with a unique - suffix. The provided value has the same validation rules as the Name - field, and may be truncated by the length of the suffix required to - make the value unique on the server. \n If this field is specified - and the generated name exists, the server will NOT return a 409 - - instead, it will either return 201 Created or 500 with Reason ServerTimeout - indicating a unique name could not be found in the time allotted, - and the client should retry (optionally after the time indicated in - the Retry-After header). \n Applied only if Name is not specified. - More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#idempotency" - type: string - generation: - description: A sequence number representing a specific generation of - the desired state. Populated by the system. Read-only. - format: int64 - type: integer - initializers: - description: "An initializer is a controller which enforces some system - invariant at object creation time. This field is a list of initializers - that have not yet acted on this object. If nil or empty, this object - has been completely initialized. Otherwise, the object is considered - uninitialized and is hidden (in list/watch and get calls) from clients - that haven't explicitly asked to observe uninitialized objects. \n - When an object is created, the system will populate this list with - the current set of initializers. Only privileged users may set or - modify this list. Once it is empty, it may not be modified further - by any user." - properties: - pending: - description: Pending is a list of initializers that must execute - in order before this object is visible. When the last pending - initializer is removed, and no failing result is set, the initializers - struct will be set to nil and the object is considered as initialized - and visible to all clients. - items: - properties: - name: - description: name of the process that is responsible for initializing - this object. - type: string - required: - - name - type: object - type: array - result: - description: If result is set with the Failure field, the object - will be persisted to storage and then deleted, ensuring that other - clients can observe the deletion. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this - representation of an object. Servers should convert recognized - schemas to the latest internal value, and may reject unrecognized - values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' - type: string - code: - description: Suggested HTTP return code for this status, 0 if - not set. - format: int32 - type: integer - details: - description: Extended data associated with the reason. Each - reason may define its own extended details. This field is - optional and the data returned is not guaranteed to conform - to any schema except that defined by the reason type. - properties: - causes: - description: The Causes array includes more details associated - with the StatusReason failure. Not all StatusReasons may - provide detailed causes. - items: - properties: - field: - description: "The field of the resource that has caused - this error, as named by its JSON serialization. - May include dot and postfix notation for nested - attributes. Arrays are zero-indexed. Fields may - appear more than once in an array of causes due - to fields having multiple errors. Optional. \n Examples: - \ \"name\" - the field \"name\" on the current - resource \"items[0].name\" - the field \"name\" - on the first array entry in \"items\"" - type: string - message: - description: A human-readable description of the cause - of the error. This field may be presented as-is - to a reader. - type: string - reason: - description: A machine-readable description of the - cause of the error. If this value is empty there - is no information available. - type: string - type: object - type: array - group: - description: The group attribute of the resource associated - with the status StatusReason. - type: string - kind: - description: 'The kind attribute of the resource associated - with the status StatusReason. On some operations may differ - from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - name: - description: The name attribute of the resource associated - with the status StatusReason (when there is a single name - which can be described). - type: string - retryAfterSeconds: - description: If specified, the time in seconds before the - operation should be retried. Some errors may indicate - the client must take an alternate action - for those errors - this field may indicate how long to wait before taking - the alternate action. - format: int32 - type: integer - uid: - description: 'UID of the resource. (when there is a single - resource which can be described). More info: http://kubernetes.io/docs/user-guide/identifiers#uids' - type: string - type: object - kind: - description: 'Kind is a string value representing the REST resource - this object represents. Servers may infer this from the endpoint - the client submits requests to. Cannot be updated. In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - message: - description: A human-readable description of the status of this - operation. - type: string - metadata: - description: 'Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - properties: - continue: - description: continue may be set if the user set a limit - on the number of items returned, and indicates that the - server has more data available. The value is opaque and - may be used to issue another request to the endpoint that - served this list to retrieve the next set of available - objects. Continuing a consistent list may not be possible - if the server configuration has changed or more than a - few minutes have passed. The resourceVersion field returned - when using this continue value will be identical to the - value in the first response, unless you have received - this token from an error message. - type: string - resourceVersion: - description: 'String that identifies the server''s internal - version of this object that can be used by clients to - determine when objects have changed. Value must be treated - as opaque by clients and passed unmodified back to the - server. Populated by the system. Read-only. More info: - https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency' - type: string - selfLink: - description: selfLink is a URL representing this object. - Populated by the system. Read-only. - type: string - type: object - reason: - description: A machine-readable description of why this operation - is in the "Failure" status. If this value is empty there is - no information available. A Reason clarifies an HTTP status - code but does not override it. - type: string - status: - description: 'Status of the operation. One of: "Success" or - "Failure". More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status' - type: string - type: object - required: - - pending - type: object - labels: - additionalProperties: - type: string - description: 'Map of string keys and values that can be used to organize - and categorize (scope and select) objects. May match selectors of - replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels' - type: object - name: - description: 'Name must be unique within a namespace. Is required when - creating resources, although some resources may allow a client to - request the generation of an appropriate name automatically. Name - is primarily intended for creation idempotence and configuration definition. - Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names' - type: string - namespace: - description: "Namespace defines the space within each name must be unique. - An empty namespace is equivalent to the \"default\" namespace, but - \"default\" is the canonical representation. Not all objects are required - to be scoped to a namespace - the value of this field for those objects - will be empty. \n Must be a DNS_LABEL. Cannot be updated. More info: - http://kubernetes.io/docs/user-guide/namespaces" - type: string - ownerReferences: - description: List of objects depended by this object. If ALL objects - in the list have been deleted, this object will be garbage collected. - If this object is managed by a controller, then an entry in this list - will point to this controller, with the controller field set to true. - There cannot be more than one managing controller. - items: - properties: - apiVersion: - description: API version of the referent. - type: string - blockOwnerDeletion: - description: If true, AND if the owner has the "foregroundDeletion" - finalizer, then the owner cannot be deleted from the key-value - store until this reference is removed. Defaults to false. To - set this field, a user needs "delete" permission of the owner, - otherwise 422 (Unprocessable Entity) will be returned. - type: boolean - controller: - description: If true, this reference points to the managing controller. - type: boolean - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - name: - description: 'Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names' - type: string - uid: - description: 'UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids' - type: string - required: - - apiVersion - - kind - - name - - uid - type: object - type: array - resourceVersion: - description: "An opaque value that represents the internal version of - this object that can be used by clients to determine when objects - have changed. May be used for optimistic concurrency, change detection, - and the watch operation on a resource or set of resources. Clients - must treat these values as opaque and passed unmodified back to the - server. They may only be valid for a particular resource or set of - resources. \n Populated by the system. Read-only. Value must be treated - as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency" - type: string - selfLink: - description: SelfLink is a URL representing this object. Populated by - the system. Read-only. - type: string - uid: - description: "UID is the unique in time and space value for this object. - It is typically generated by the server on successful creation of - a resource and is not allowed to change on PUT operations. \n Populated - by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids" - type: string - type: object - spec: - type: object - status: - type: object - type: object + scope: Namespaced versions: - name: v1 served: true storage: true + schema: + openAPIV3Schema: + description: Driver is the Schema for the drivers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + type: object + status: + type: object + type: object - name: v2 served: true storage: false + schema: + openAPIV3Schema: + description: Driver is the Schema for the drivers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + type: object + status: + type: object + type: object - name: v3 served: true storage: false + schema: + openAPIV3Schema: + description: Driver is the Schema for the drivers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + type: object + status: + type: object + type: object status: acceptedNames: kind: "" diff --git a/pkg/envtest/testdata/example_multiversion_crd1.yaml b/pkg/envtest/testdata/example_multiversion_crd1.yaml index 1999d1e02e..5bb2d73f69 100644 --- a/pkg/envtest/testdata/example_multiversion_crd1.yaml +++ b/pkg/envtest/testdata/example_multiversion_crd1.yaml @@ -1,4 +1,4 @@ -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: creationTimestamp: null @@ -8,352 +8,52 @@ spec: names: kind: Driver plural: drivers - scope: "" - validation: - openAPIV3Schema: - description: Driver is the Schema for the drivers API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - metadata: - properties: - annotations: - additionalProperties: - type: string - description: 'Annotations is an unstructured key value map stored with - a resource that may be set by external tools to store and retrieve - arbitrary metadata. They are not queryable and should be preserved - when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations' - type: object - clusterName: - description: The name of the cluster which the object belongs to. This - is used to distinguish resources with same name and namespace in different - clusters. This field is not set anywhere right now and apiserver is - going to ignore it if set in create or update request. - type: string - creationTimestamp: - description: "CreationTimestamp is a timestamp representing the server - time when this object was created. It is not guaranteed to be set - in happens-before order across separate operations. Clients may not - set this value. It is represented in RFC3339 form and is in UTC. \n - Populated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata" - format: date-time - type: string - deletionGracePeriodSeconds: - description: Number of seconds allowed for this object to gracefully - terminate before it will be removed from the system. Only set when - deletionTimestamp is also set. May only be shortened. Read-only. - format: int64 - type: integer - deletionTimestamp: - description: "DeletionTimestamp is RFC 3339 date and time at which this - resource will be deleted. This field is set by the server when a graceful - deletion is requested by the user, and is not directly settable by - a client. The resource is expected to be deleted (no longer visible - from resource lists, and not reachable by name) after the time in - this field, once the finalizers list is empty. As long as the finalizers - list contains items, deletion is blocked. Once the deletionTimestamp - is set, this value may not be unset or be set further into the future, - although it may be shortened or the resource may be deleted prior - to this time. For example, a user may request that a pod is deleted - in 30 seconds. The Kubelet will react by sending a graceful termination - signal to the containers in the pod. After that 30 seconds, the Kubelet - will send a hard termination signal (SIGKILL) to the container and - after cleanup, remove the pod from the API. In the presence of network - partitions, this object may still exist after this timestamp, until - an administrator or automated process can determine the resource is - fully terminated. If not set, graceful deletion of the object has - not been requested. \n Populated by the system when a graceful deletion - is requested. Read-only. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata" - format: date-time - type: string - finalizers: - description: Must be empty before the object is deleted from the registry. - Each entry is an identifier for the responsible component that will - remove the entry from the list. If the deletionTimestamp of the object - is non-nil, entries in this list can only be removed. - items: - type: string - type: array - generateName: - description: "GenerateName is an optional prefix, used by the server, - to generate a unique name ONLY IF the Name field has not been provided. - If this field is used, the name returned to the client will be different - than the name passed. This value will also be combined with a unique - suffix. The provided value has the same validation rules as the Name - field, and may be truncated by the length of the suffix required to - make the value unique on the server. \n If this field is specified - and the generated name exists, the server will NOT return a 409 - - instead, it will either return 201 Created or 500 with Reason ServerTimeout - indicating a unique name could not be found in the time allotted, - and the client should retry (optionally after the time indicated in - the Retry-After header). \n Applied only if Name is not specified. - More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#idempotency" - type: string - generation: - description: A sequence number representing a specific generation of - the desired state. Populated by the system. Read-only. - format: int64 - type: integer - initializers: - description: "An initializer is a controller which enforces some system - invariant at object creation time. This field is a list of initializers - that have not yet acted on this object. If nil or empty, this object - has been completely initialized. Otherwise, the object is considered - uninitialized and is hidden (in list/watch and get calls) from clients - that haven't explicitly asked to observe uninitialized objects. \n - When an object is created, the system will populate this list with - the current set of initializers. Only privileged users may set or - modify this list. Once it is empty, it may not be modified further - by any user." - properties: - pending: - description: Pending is a list of initializers that must execute - in order before this object is visible. When the last pending - initializer is removed, and no failing result is set, the initializers - struct will be set to nil and the object is considered as initialized - and visible to all clients. - items: - properties: - name: - description: name of the process that is responsible for initializing - this object. - type: string - required: - - name - type: object - type: array - result: - description: If result is set with the Failure field, the object - will be persisted to storage and then deleted, ensuring that other - clients can observe the deletion. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this - representation of an object. Servers should convert recognized - schemas to the latest internal value, and may reject unrecognized - values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' - type: string - code: - description: Suggested HTTP return code for this status, 0 if - not set. - format: int32 - type: integer - details: - description: Extended data associated with the reason. Each - reason may define its own extended details. This field is - optional and the data returned is not guaranteed to conform - to any schema except that defined by the reason type. - properties: - causes: - description: The Causes array includes more details associated - with the StatusReason failure. Not all StatusReasons may - provide detailed causes. - items: - properties: - field: - description: "The field of the resource that has caused - this error, as named by its JSON serialization. - May include dot and postfix notation for nested - attributes. Arrays are zero-indexed. Fields may - appear more than once in an array of causes due - to fields having multiple errors. Optional. \n Examples: - \ \"name\" - the field \"name\" on the current - resource \"items[0].name\" - the field \"name\" - on the first array entry in \"items\"" - type: string - message: - description: A human-readable description of the cause - of the error. This field may be presented as-is - to a reader. - type: string - reason: - description: A machine-readable description of the - cause of the error. If this value is empty there - is no information available. - type: string - type: object - type: array - group: - description: The group attribute of the resource associated - with the status StatusReason. - type: string - kind: - description: 'The kind attribute of the resource associated - with the status StatusReason. On some operations may differ - from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - name: - description: The name attribute of the resource associated - with the status StatusReason (when there is a single name - which can be described). - type: string - retryAfterSeconds: - description: If specified, the time in seconds before the - operation should be retried. Some errors may indicate - the client must take an alternate action - for those errors - this field may indicate how long to wait before taking - the alternate action. - format: int32 - type: integer - uid: - description: 'UID of the resource. (when there is a single - resource which can be described). More info: http://kubernetes.io/docs/user-guide/identifiers#uids' - type: string - type: object - kind: - description: 'Kind is a string value representing the REST resource - this object represents. Servers may infer this from the endpoint - the client submits requests to. Cannot be updated. In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - message: - description: A human-readable description of the status of this - operation. - type: string - metadata: - description: 'Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - properties: - continue: - description: continue may be set if the user set a limit - on the number of items returned, and indicates that the - server has more data available. The value is opaque and - may be used to issue another request to the endpoint that - served this list to retrieve the next set of available - objects. Continuing a consistent list may not be possible - if the server configuration has changed or more than a - few minutes have passed. The resourceVersion field returned - when using this continue value will be identical to the - value in the first response, unless you have received - this token from an error message. - type: string - resourceVersion: - description: 'String that identifies the server''s internal - version of this object that can be used by clients to - determine when objects have changed. Value must be treated - as opaque by clients and passed unmodified back to the - server. Populated by the system. Read-only. More info: - https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency' - type: string - selfLink: - description: selfLink is a URL representing this object. - Populated by the system. Read-only. - type: string - type: object - reason: - description: A machine-readable description of why this operation - is in the "Failure" status. If this value is empty there is - no information available. A Reason clarifies an HTTP status - code but does not override it. - type: string - status: - description: 'Status of the operation. One of: "Success" or - "Failure". More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status' - type: string - type: object - required: - - pending - type: object - labels: - additionalProperties: - type: string - description: 'Map of string keys and values that can be used to organize - and categorize (scope and select) objects. May match selectors of - replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels' - type: object - name: - description: 'Name must be unique within a namespace. Is required when - creating resources, although some resources may allow a client to - request the generation of an appropriate name automatically. Name - is primarily intended for creation idempotence and configuration definition. - Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names' - type: string - namespace: - description: "Namespace defines the space within each name must be unique. - An empty namespace is equivalent to the \"default\" namespace, but - \"default\" is the canonical representation. Not all objects are required - to be scoped to a namespace - the value of this field for those objects - will be empty. \n Must be a DNS_LABEL. Cannot be updated. More info: - http://kubernetes.io/docs/user-guide/namespaces" - type: string - ownerReferences: - description: List of objects depended by this object. If ALL objects - in the list have been deleted, this object will be garbage collected. - If this object is managed by a controller, then an entry in this list - will point to this controller, with the controller field set to true. - There cannot be more than one managing controller. - items: - properties: - apiVersion: - description: API version of the referent. - type: string - blockOwnerDeletion: - description: If true, AND if the owner has the "foregroundDeletion" - finalizer, then the owner cannot be deleted from the key-value - store until this reference is removed. Defaults to false. To - set this field, a user needs "delete" permission of the owner, - otherwise 422 (Unprocessable Entity) will be returned. - type: boolean - controller: - description: If true, this reference points to the managing controller. - type: boolean - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - name: - description: 'Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names' - type: string - uid: - description: 'UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids' - type: string - required: - - apiVersion - - kind - - name - - uid - type: object - type: array - resourceVersion: - description: "An opaque value that represents the internal version of - this object that can be used by clients to determine when objects - have changed. May be used for optimistic concurrency, change detection, - and the watch operation on a resource or set of resources. Clients - must treat these values as opaque and passed unmodified back to the - server. They may only be valid for a particular resource or set of - resources. \n Populated by the system. Read-only. Value must be treated - as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency" - type: string - selfLink: - description: SelfLink is a URL representing this object. Populated by - the system. Read-only. - type: string - uid: - description: "UID is the unique in time and space value for this object. - It is typically generated by the server on successful creation of - a resource and is not allowed to change on PUT operations. \n Populated - by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids" - type: string - type: object - spec: - type: object - status: - type: object - type: object + scope: Namespaced versions: - name: v1 served: true storage: true + schema: + openAPIV3Schema: + description: Driver is the Schema for the drivers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + type: object + status: + type: object + type: object - name: v2 served: true storage: false + schema: + openAPIV3Schema: + description: Driver is the Schema for the drivers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + spec: + type: object + status: + type: object + type: object status: acceptedNames: kind: "" diff --git a/pkg/envtest/testdata/examplecrd.yaml b/pkg/envtest/testdata/examplecrd.yaml index f66835ae20..f1638f8310 100644 --- a/pkg/envtest/testdata/examplecrd.yaml +++ b/pkg/envtest/testdata/examplecrd.yaml @@ -1,4 +1,4 @@ -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: bazs.qux.example.com @@ -8,4 +8,10 @@ spec: kind: Baz plural: bazs scope: Namespaced - version: "v1beta1" \ No newline at end of file + versions: + - name: "v1beta1" + storage: true + served: true + schema: + openAPIV3Schema: + type: object diff --git a/pkg/envtest/testdata/multiplecrds.yaml b/pkg/envtest/testdata/multiplecrds.yaml index 2148eca204..a855140ead 100644 --- a/pkg/envtest/testdata/multiplecrds.yaml +++ b/pkg/envtest/testdata/multiplecrds.yaml @@ -1,5 +1,5 @@ --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: captains.crew.example.com @@ -9,9 +9,15 @@ spec: kind: Captain plural: captains scope: Namespaced - version: "v1beta1" + versions: + - name: "v1beta1" + storage: true + served: true + schema: + openAPIV3Schema: + type: object --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: firstmates.crew.example.com @@ -21,5 +27,11 @@ spec: kind: FirstMate plural: firstmates scope: Namespaced - version: "v1beta1" ---- \ No newline at end of file + versions: + - name: "v1beta1" + storage: true + served: true + schema: + openAPIV3Schema: + type: object +--- diff --git a/pkg/envtest/testdata/webhooks/manifests.yaml b/pkg/envtest/testdata/webhooks/manifests.yaml index 312128bd2d..72437905cd 100644 --- a/pkg/envtest/testdata/webhooks/manifests.yaml +++ b/pkg/envtest/testdata/webhooks/manifests.yaml @@ -1,5 +1,5 @@ --- -apiVersion: admissionregistration.k8s.io/v1beta1 +apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: creationTimestamp: null @@ -10,7 +10,7 @@ webhooks: service: name: webhook-service namespace: system - path: /mutate-v1beta1 + path: /mutate-v1 failurePolicy: Fail name: mpods.kb.io rules: @@ -49,7 +49,7 @@ webhooks: resources: - pods --- -apiVersion: admissionregistration.k8s.io/v1beta1 +apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: creationTimestamp: null @@ -60,7 +60,7 @@ webhooks: service: name: webhook-service namespace: system - path: /validate-v1beta1 + path: /validate-v1 failurePolicy: Fail name: vpods.kb.io rules: diff --git a/pkg/envtest/webhook.go b/pkg/envtest/webhook.go index 567f732e41..49d8773588 100644 --- a/pkg/envtest/webhook.go +++ b/pkg/envtest/webhook.go @@ -15,26 +15,27 @@ package envtest import ( "context" - "encoding/base64" "fmt" - "io/ioutil" "net" "os" "path/filepath" "time" + admissionv1 "k8s.io/api/admissionregistration/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/yaml" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/internal/testing/addr" "sigs.k8s.io/controller-runtime/pkg/internal/testing/certs" - "sigs.k8s.io/yaml" ) // WebhookInstallOptions are the options for installing mutating or validating webhooks. @@ -43,10 +44,10 @@ type WebhookInstallOptions struct { Paths []string // MutatingWebhooks is a list of MutatingWebhookConfigurations to install - MutatingWebhooks []client.Object + MutatingWebhooks []*admissionv1.MutatingWebhookConfiguration // ValidatingWebhooks is a list of ValidatingWebhookConfigurations to install - ValidatingWebhooks []client.Object + ValidatingWebhooks []*admissionv1.ValidatingWebhookConfiguration // IgnoreErrorIfPathMissing will ignore an error if a DirectoryPath does not exist when set to true IgnoreErrorIfPathMissing bool @@ -88,64 +89,34 @@ func (o *WebhookInstallOptions) ModifyWebhookDefinitions() error { return err } - for i, unstructuredHook := range runtimeListToUnstructured(o.MutatingWebhooks) { - webhooks, found, err := unstructured.NestedSlice(unstructuredHook.Object, "webhooks") - if !found || err != nil { - return fmt.Errorf("unexpected object, %v", err) - } - for j := range webhooks { - webhook, err := modifyWebhook(webhooks[j].(map[string]interface{}), caData, hostPort) - if err != nil { - return err - } - webhooks[j] = webhook - unstructuredHook.Object["webhooks"] = webhooks - o.MutatingWebhooks[i] = unstructuredHook + for i := range o.MutatingWebhooks { + for j := range o.MutatingWebhooks[i].Webhooks { + updateClientConfig(&o.MutatingWebhooks[i].Webhooks[j].ClientConfig, hostPort, caData) } } - for i, unstructuredHook := range runtimeListToUnstructured(o.ValidatingWebhooks) { - webhooks, found, err := unstructured.NestedSlice(unstructuredHook.Object, "webhooks") - if !found || err != nil { - return fmt.Errorf("unexpected object, %v", err) - } - for j := range webhooks { - webhook, err := modifyWebhook(webhooks[j].(map[string]interface{}), caData, hostPort) - if err != nil { - return err - } - webhooks[j] = webhook - unstructuredHook.Object["webhooks"] = webhooks - o.ValidatingWebhooks[i] = unstructuredHook + for i := range o.ValidatingWebhooks { + for j := range o.ValidatingWebhooks[i].Webhooks { + updateClientConfig(&o.ValidatingWebhooks[i].Webhooks[j].ClientConfig, hostPort, caData) } } return nil } -func modifyWebhook(webhook map[string]interface{}, caData []byte, hostPort string) (map[string]interface{}, error) { - clientConfig, found, err := unstructured.NestedMap(webhook, "clientConfig") - if !found || err != nil { - return nil, fmt.Errorf("cannot find clientconfig: %v", err) - } - clientConfig["caBundle"] = base64.StdEncoding.EncodeToString(caData) - servicePath, found, err := unstructured.NestedString(clientConfig, "service", "path") - if found && err == nil { - // we cannot use service in integration tests since we're running controller outside cluster - // the intent here is that we swap out service for raw address because we don't have an actually standard kube service network. - // We want to users to be able to use your standard config though - url := fmt.Sprintf("https://%s/%s", hostPort, servicePath) - clientConfig["url"] = url - clientConfig["service"] = nil +func updateClientConfig(cc *admissionv1.WebhookClientConfig, hostPort string, caData []byte) { + cc.CABundle = caData + if cc.Service != nil && cc.Service.Path != nil { + url := fmt.Sprintf("https://%s/%s", hostPort, *cc.Service.Path) + cc.URL = &url + cc.Service = nil } - webhook["clientConfig"] = clientConfig - return webhook, nil } func (o *WebhookInstallOptions) generateHostPort() (string, error) { if o.LocalServingPort == 0 { port, host, err := addr.Suggest(o.LocalServingHost) if err != nil { - return "", fmt.Errorf("unable to grab random port for serving webhooks on: %v", err) + return "", fmt.Errorf("unable to grab random port for serving webhooks on: %w", err) } o.LocalServingPort = port o.LocalServingHost = host @@ -199,16 +170,35 @@ func (o *WebhookInstallOptions) Cleanup() error { // WaitForWebhooks waits for the Webhooks to be available through API server. func WaitForWebhooks(config *rest.Config, - mutatingWebhooks []client.Object, - validatingWebhooks []client.Object, + mutatingWebhooks []*admissionv1.MutatingWebhookConfiguration, + validatingWebhooks []*admissionv1.ValidatingWebhookConfiguration, options WebhookInstallOptions) error { - waitingFor := map[schema.GroupVersionKind]*sets.String{} + waitingFor := map[schema.GroupVersionKind]*sets.Set[string]{} + + for _, hook := range mutatingWebhooks { + h := hook + gvk, err := apiutil.GVKForObject(h, scheme.Scheme) + if err != nil { + return fmt.Errorf("unable to get gvk for MutatingWebhookConfiguration %s: %w", hook.GetName(), err) + } + + if _, ok := waitingFor[gvk]; !ok { + waitingFor[gvk] = &sets.Set[string]{} + } + waitingFor[gvk].Insert(h.GetName()) + } + + for _, hook := range validatingWebhooks { + h := hook + gvk, err := apiutil.GVKForObject(h, scheme.Scheme) + if err != nil { + return fmt.Errorf("unable to get gvk for ValidatingWebhookConfiguration %s: %w", hook.GetName(), err) + } - for _, hook := range runtimeListToUnstructured(append(validatingWebhooks, mutatingWebhooks...)) { - if _, ok := waitingFor[hook.GroupVersionKind()]; !ok { - waitingFor[hook.GroupVersionKind()] = &sets.String{} + if _, ok := waitingFor[gvk]; !ok { + waitingFor[gvk] = &sets.Set[string]{} } - waitingFor[hook.GroupVersionKind()].Insert(hook.GetName()) + waitingFor[gvk].Insert(hook.GetName()) } // Poll until all resources are found in discovery @@ -222,7 +212,7 @@ type webhookPoller struct { config *rest.Config // waitingFor is the map of resources keyed by group version that have not yet been found in discovery - waitingFor map[schema.GroupVersionKind]*sets.String + waitingFor map[schema.GroupVersionKind]*sets.Set[string] } // poll checks if all the resources have been found in discovery, and returns false if not. @@ -239,7 +229,7 @@ func (p *webhookPoller) poll() (done bool, err error) { delete(p.waitingFor, gvk) continue } - for _, name := range names.List() { + for _, name := range names.UnsortedList() { var obj = &unstructured.Unstructured{} obj.SetGroupVersionKind(gvk) err := c.Get(context.Background(), client.ObjectKey{ @@ -266,51 +256,53 @@ func (p *webhookPoller) poll() (done bool, err error) { func (o *WebhookInstallOptions) setupCA() error { hookCA, err := certs.NewTinyCA() if err != nil { - return fmt.Errorf("unable to set up webhook CA: %v", err) + return fmt.Errorf("unable to set up webhook CA: %w", err) } names := []string{"localhost", o.LocalServingHost, o.LocalServingHostExternalName} hookCert, err := hookCA.NewServingCert(names...) if err != nil { - return fmt.Errorf("unable to set up webhook serving certs: %v", err) + return fmt.Errorf("unable to set up webhook serving certs: %w", err) } - localServingCertsDir, err := ioutil.TempDir("", "envtest-serving-certs-") + localServingCertsDir, err := os.MkdirTemp("", "envtest-serving-certs-") o.LocalServingCertDir = localServingCertsDir if err != nil { - return fmt.Errorf("unable to create directory for webhook serving certs: %v", err) + return fmt.Errorf("unable to create directory for webhook serving certs: %w", err) } certData, keyData, err := hookCert.AsBytes() if err != nil { - return fmt.Errorf("unable to marshal webhook serving certs: %v", err) + return fmt.Errorf("unable to marshal webhook serving certs: %w", err) } - if err := ioutil.WriteFile(filepath.Join(localServingCertsDir, "tls.crt"), certData, 0640); err != nil { //nolint:gosec - return fmt.Errorf("unable to write webhook serving cert to disk: %v", err) + if err := os.WriteFile(filepath.Join(localServingCertsDir, "tls.crt"), certData, 0640); err != nil { //nolint:gosec + return fmt.Errorf("unable to write webhook serving cert to disk: %w", err) } - if err := ioutil.WriteFile(filepath.Join(localServingCertsDir, "tls.key"), keyData, 0640); err != nil { //nolint:gosec - return fmt.Errorf("unable to write webhook serving key to disk: %v", err) + if err := os.WriteFile(filepath.Join(localServingCertsDir, "tls.key"), keyData, 0640); err != nil { //nolint:gosec + return fmt.Errorf("unable to write webhook serving key to disk: %w", err) } o.LocalServingCAData = certData return err } -func createWebhooks(config *rest.Config, mutHooks []client.Object, valHooks []client.Object) error { +func createWebhooks(config *rest.Config, mutHooks []*admissionv1.MutatingWebhookConfiguration, valHooks []*admissionv1.ValidatingWebhookConfiguration) error { cs, err := client.New(config, client.Options{}) if err != nil { return err } // Create each webhook - for _, hook := range runtimeListToUnstructured(mutHooks) { + for _, hook := range mutHooks { + hook := hook log.V(1).Info("installing mutating webhook", "webhook", hook.GetName()) if err := ensureCreated(cs, hook); err != nil { return err } } - for _, hook := range runtimeListToUnstructured(valHooks) { + for _, hook := range valHooks { + hook := hook log.V(1).Info("installing validating webhook", "webhook", hook.GetName()) if err := ensureCreated(cs, hook); err != nil { return err @@ -320,8 +312,8 @@ func createWebhooks(config *rest.Config, mutHooks []client.Object, valHooks []cl } // ensureCreated creates or update object if already exists in the cluster. -func ensureCreated(cs client.Client, obj *unstructured.Unstructured) error { - existing := obj.DeepCopy() +func ensureCreated(cs client.Client, obj client.Object) error { + existing := obj.DeepCopyObject().(client.Object) err := cs.Get(context.Background(), client.ObjectKey{Name: obj.GetName()}, existing) switch { case apierrors.IsNotFound(err): @@ -364,9 +356,9 @@ func parseWebhook(options *WebhookInstallOptions) error { // readWebhooks reads the Webhooks from files and Unmarshals them into structs // returns slice of mutating and validating webhook configurations. -func readWebhooks(path string) ([]client.Object, []client.Object, error) { +func readWebhooks(path string) ([]*admissionv1.MutatingWebhookConfiguration, []*admissionv1.ValidatingWebhookConfiguration, error) { // Get the webhook files - var files []os.FileInfo + var files []string var err error log.V(1).Info("reading Webhooks from path", "path", path) info, err := os.Stat(path) @@ -374,24 +366,30 @@ func readWebhooks(path string) ([]client.Object, []client.Object, error) { return nil, nil, err } if !info.IsDir() { - path, files = filepath.Dir(path), []os.FileInfo{info} - } else if files, err = ioutil.ReadDir(path); err != nil { - return nil, nil, err + path, files = filepath.Dir(path), []string{info.Name()} + } else { + entries, err := os.ReadDir(path) + if err != nil { + return nil, nil, err + } + for _, e := range entries { + files = append(files, e.Name()) + } } // file extensions that may contain Webhooks resourceExtensions := sets.NewString(".json", ".yaml", ".yml") - var mutHooks []client.Object - var valHooks []client.Object + var mutHooks []*admissionv1.MutatingWebhookConfiguration + var valHooks []*admissionv1.ValidatingWebhookConfiguration for _, file := range files { // Only parse allowlisted file types - if !resourceExtensions.Has(filepath.Ext(file.Name())) { + if !resourceExtensions.Has(filepath.Ext(file)) { continue } // Unmarshal Webhooks from file into structs - docs, err := readDocuments(filepath.Join(path, file.Name())) + docs, err := readDocuments(filepath.Join(path, file)) if err != nil { return nil, nil, err } @@ -403,25 +401,24 @@ func readWebhooks(path string) ([]client.Object, []client.Object, error) { } const ( - admissionregv1 = "admissionregistration.k8s.io/v1" - admissionregv1beta1 = "admissionregistration.k8s.io/v1beta1" + admissionregv1 = "admissionregistration.k8s.io/v1" ) switch { case generic.Kind == "MutatingWebhookConfiguration": - if generic.APIVersion != admissionregv1beta1 && generic.APIVersion != admissionregv1 { - return nil, nil, fmt.Errorf("only v1beta1 and v1 are supported right now for MutatingWebhookConfiguration (name: %s)", generic.Name) + if generic.APIVersion != admissionregv1 { + return nil, nil, fmt.Errorf("only v1 is supported right now for MutatingWebhookConfiguration (name: %s)", generic.Name) } - hook := &unstructured.Unstructured{} - if err := yaml.Unmarshal(doc, &hook); err != nil { + hook := &admissionv1.MutatingWebhookConfiguration{} + if err := yaml.Unmarshal(doc, hook); err != nil { return nil, nil, err } mutHooks = append(mutHooks, hook) case generic.Kind == "ValidatingWebhookConfiguration": - if generic.APIVersion != admissionregv1beta1 && generic.APIVersion != admissionregv1 { - return nil, nil, fmt.Errorf("only v1beta1 and v1 are supported right now for ValidatingWebhookConfiguration (name: %s)", generic.Name) + if generic.APIVersion != admissionregv1 { + return nil, nil, fmt.Errorf("only v1 is supported right now for ValidatingWebhookConfiguration (name: %s)", generic.Name) } - hook := &unstructured.Unstructured{} - if err := yaml.Unmarshal(doc, &hook); err != nil { + hook := &admissionv1.ValidatingWebhookConfiguration{} + if err := yaml.Unmarshal(doc, hook); err != nil { return nil, nil, err } valHooks = append(valHooks, hook) @@ -430,21 +427,7 @@ func readWebhooks(path string) ([]client.Object, []client.Object, error) { } } - log.V(1).Info("read webhooks from file", "file", file.Name()) + log.V(1).Info("read webhooks from file", "file", file) } return mutHooks, valHooks, nil } - -func runtimeListToUnstructured(l []client.Object) []*unstructured.Unstructured { - res := []*unstructured.Unstructured{} - for _, obj := range l { - m, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj.DeepCopyObject()) - if err != nil { - continue - } - res = append(res, &unstructured.Unstructured{ - Object: m, - }) - } - return res -} diff --git a/pkg/envtest/webhook_test.go b/pkg/envtest/webhook_test.go index 97080f288c..2cbc9ab9c8 100644 --- a/pkg/envtest/webhook_test.go +++ b/pkg/envtest/webhook_test.go @@ -18,10 +18,12 @@ package envtest import ( "context" + "crypto/tls" "path/filepath" + "strings" "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -36,11 +38,14 @@ import ( var _ = Describe("Test", func() { Describe("Webhook", func() { - It("should reject create request for webhook that rejects all requests", func(done Done) { + It("should reject create request for webhook that rejects all requests", func() { m, err := manager.New(env.Config, manager.Options{ Port: env.WebhookInstallOptions.LocalServingPort, Host: env.WebhookInstallOptions.LocalServingHost, CertDir: env.WebhookInstallOptions.LocalServingCertDir, + TLSOpts: []func(*tls.Config){ + func(config *tls.Config) {}, + }, }) // we need manager here just to leverage manager.SetFields Expect(err).NotTo(HaveOccurred()) server := m.GetWebhookServer() @@ -83,11 +88,10 @@ var _ = Describe("Test", func() { Eventually(func() bool { err = c.Create(context.TODO(), obj) - return apierrors.ReasonForError(err) == metav1.StatusReason("Always denied") + return err != nil && strings.HasSuffix(err.Error(), "Always denied") && apierrors.ReasonForError(err) == metav1.StatusReasonForbidden }, 1*time.Second).Should(BeTrue()) cancel() - close(done) }) It("should load webhooks from directory", func() { diff --git a/pkg/finalizer/finalizer.go b/pkg/finalizer/finalizer.go index 1f627f8c49..10c5645dbe 100644 --- a/pkg/finalizer/finalizer.go +++ b/pkg/finalizer/finalizer.go @@ -64,7 +64,7 @@ func (f finalizers) Finalize(ctx context.Context, obj client.Object) (Result, er // object (e.g. it may set a condition and need a status update). res.Updated = res.Updated || finalizerRes.Updated res.StatusUpdated = res.StatusUpdated || finalizerRes.StatusUpdated - errList = append(errList, fmt.Errorf("finalizer %q failed: %v", key, err)) + errList = append(errList, fmt.Errorf("finalizer %q failed: %w", key, err)) } else { // If the finalizer succeeds, we remove the finalizer from the primary // object's metadata, so we know it will need an update. diff --git a/pkg/finalizer/finalizer_test.go b/pkg/finalizer/finalizer_test.go index 944acd595a..eb85bd020a 100644 --- a/pkg/finalizer/finalizer_test.go +++ b/pkg/finalizer/finalizer_test.go @@ -5,12 +5,11 @@ import ( "fmt" "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) type mockFinalizer struct { @@ -21,10 +20,10 @@ type mockFinalizer struct { func (f mockFinalizer) Finalize(context.Context, client.Object) (Result, error) { return f.result, f.err } + func TestFinalizer(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Finalizer Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Finalizer Suite") } var _ = Describe("TestFinalizer", func() { diff --git a/pkg/handler/doc.go b/pkg/handler/doc.go index 3b5b79048d..e5fd177aff 100644 --- a/pkg/handler/doc.go +++ b/pkg/handler/doc.go @@ -21,7 +21,7 @@ Controller.Watch in order to generate and enqueue reconcile.Request work items. Generally, following premade event handlers should be sufficient for most use cases: -EventHandlers +EventHandlers: EnqueueRequestForObject - Enqueues a reconcile.Request containing the Name and Namespace of the object in the Event. This will cause the object that was the source of the Event (e.g. the created / deleted / updated object) to be diff --git a/pkg/handler/enqueue.go b/pkg/handler/enqueue.go index e6d3a4eaab..c72b2e1ebb 100644 --- a/pkg/handler/enqueue.go +++ b/pkg/handler/enqueue.go @@ -17,6 +17,8 @@ limitations under the License. package handler import ( + "context" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/event" @@ -36,7 +38,7 @@ var _ EventHandler = &EnqueueRequestForObject{} type EnqueueRequestForObject struct{} // Create implements EventHandler. -func (e *EnqueueRequestForObject) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) { +func (e *EnqueueRequestForObject) Create(ctx context.Context, evt event.CreateEvent, q workqueue.RateLimitingInterface) { if evt.Object == nil { enqueueLog.Error(nil, "CreateEvent received with no metadata", "event", evt) return @@ -48,7 +50,7 @@ func (e *EnqueueRequestForObject) Create(evt event.CreateEvent, q workqueue.Rate } // Update implements EventHandler. -func (e *EnqueueRequestForObject) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) { +func (e *EnqueueRequestForObject) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.RateLimitingInterface) { switch { case evt.ObjectNew != nil: q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ @@ -66,7 +68,7 @@ func (e *EnqueueRequestForObject) Update(evt event.UpdateEvent, q workqueue.Rate } // Delete implements EventHandler. -func (e *EnqueueRequestForObject) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) { +func (e *EnqueueRequestForObject) Delete(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) { if evt.Object == nil { enqueueLog.Error(nil, "DeleteEvent received with no metadata", "event", evt) return @@ -78,7 +80,7 @@ func (e *EnqueueRequestForObject) Delete(evt event.DeleteEvent, q workqueue.Rate } // Generic implements EventHandler. -func (e *EnqueueRequestForObject) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) { +func (e *EnqueueRequestForObject) Generic(ctx context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) { if evt.Object == nil { enqueueLog.Error(nil, "GenericEvent received with no metadata", "event", evt) return diff --git a/pkg/handler/enqueue_mapped.go b/pkg/handler/enqueue_mapped.go index 17401b1fdb..b55fdde6ba 100644 --- a/pkg/handler/enqueue_mapped.go +++ b/pkg/handler/enqueue_mapped.go @@ -17,16 +17,17 @@ limitations under the License. package handler import ( + "context" + "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" ) // MapFunc is the signature required for enqueueing requests from a generic function. // This type is usually used with EnqueueRequestsFromMapFunc when registering an event handler. -type MapFunc func(client.Object) []reconcile.Request +type MapFunc func(context.Context, client.Object) []reconcile.Request // EnqueueRequestsFromMapFunc enqueues Requests by running a transformation function that outputs a collection // of reconcile.Requests on each Event. The reconcile.Requests may be for an arbitrary set of objects @@ -52,32 +53,32 @@ type enqueueRequestsFromMapFunc struct { } // Create implements EventHandler. -func (e *enqueueRequestsFromMapFunc) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) { +func (e *enqueueRequestsFromMapFunc) Create(ctx context.Context, evt event.CreateEvent, q workqueue.RateLimitingInterface) { reqs := map[reconcile.Request]empty{} - e.mapAndEnqueue(q, evt.Object, reqs) + e.mapAndEnqueue(ctx, q, evt.Object, reqs) } // Update implements EventHandler. -func (e *enqueueRequestsFromMapFunc) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) { +func (e *enqueueRequestsFromMapFunc) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.RateLimitingInterface) { reqs := map[reconcile.Request]empty{} - e.mapAndEnqueue(q, evt.ObjectOld, reqs) - e.mapAndEnqueue(q, evt.ObjectNew, reqs) + e.mapAndEnqueue(ctx, q, evt.ObjectOld, reqs) + e.mapAndEnqueue(ctx, q, evt.ObjectNew, reqs) } // Delete implements EventHandler. -func (e *enqueueRequestsFromMapFunc) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) { +func (e *enqueueRequestsFromMapFunc) Delete(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) { reqs := map[reconcile.Request]empty{} - e.mapAndEnqueue(q, evt.Object, reqs) + e.mapAndEnqueue(ctx, q, evt.Object, reqs) } // Generic implements EventHandler. -func (e *enqueueRequestsFromMapFunc) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) { +func (e *enqueueRequestsFromMapFunc) Generic(ctx context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) { reqs := map[reconcile.Request]empty{} - e.mapAndEnqueue(q, evt.Object, reqs) + e.mapAndEnqueue(ctx, q, evt.Object, reqs) } -func (e *enqueueRequestsFromMapFunc) mapAndEnqueue(q workqueue.RateLimitingInterface, object client.Object, reqs map[reconcile.Request]empty) { - for _, req := range e.toRequests(object) { +func (e *enqueueRequestsFromMapFunc) mapAndEnqueue(ctx context.Context, q workqueue.RateLimitingInterface, object client.Object, reqs map[reconcile.Request]empty) { + for _, req := range e.toRequests(ctx, object) { _, ok := reqs[req] if !ok { q.Add(req) @@ -85,13 +86,3 @@ func (e *enqueueRequestsFromMapFunc) mapAndEnqueue(q workqueue.RateLimitingInter } } } - -// EnqueueRequestsFromMapFunc can inject fields into the mapper. - -// InjectFunc implements inject.Injector. -func (e *enqueueRequestsFromMapFunc) InjectFunc(f inject.Func) error { - if f == nil { - return nil - } - return f(e.toRequests) -} diff --git a/pkg/handler/enqueue_owner.go b/pkg/handler/enqueue_owner.go index 63699893fc..02e7d756f8 100644 --- a/pkg/handler/enqueue_owner.go +++ b/pkg/handler/enqueue_owner.go @@ -17,6 +17,7 @@ limitations under the License. package handler import ( + "context" "fmt" "k8s.io/apimachinery/pkg/api/meta" @@ -25,15 +26,18 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" logf "sigs.k8s.io/controller-runtime/pkg/internal/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" ) -var _ EventHandler = &EnqueueRequestForOwner{} +var _ EventHandler = &enqueueRequestForOwner{} -var log = logf.RuntimeLog.WithName("eventhandler").WithName("EnqueueRequestForOwner") +var log = logf.RuntimeLog.WithName("eventhandler").WithName("enqueueRequestForOwner") + +// OwnerOption modifies an EnqueueRequestForOwner EventHandler. +type OwnerOption func(e *enqueueRequestForOwner) // EnqueueRequestForOwner enqueues Requests for the Owners of an object. E.g. the object that created // the object that was the source of the Event. @@ -42,13 +46,34 @@ var log = logf.RuntimeLog.WithName("eventhandler").WithName("EnqueueRequestForOw // // - a source.Kind Source with Type of Pod. // -// - a handler.EnqueueRequestForOwner EventHandler with an OwnerType of ReplicaSet and IsController set to true. -type EnqueueRequestForOwner struct { - // OwnerType is the type of the Owner object to look for in OwnerReferences. Only Group and Kind are compared. - OwnerType runtime.Object +// - a handler.enqueueRequestForOwner EventHandler with an OwnerType of ReplicaSet and OnlyControllerOwner set to true. +func EnqueueRequestForOwner(scheme *runtime.Scheme, mapper meta.RESTMapper, ownerType client.Object, opts ...OwnerOption) EventHandler { + e := &enqueueRequestForOwner{ + ownerType: ownerType, + mapper: mapper, + } + if err := e.parseOwnerTypeGroupKind(scheme); err != nil { + panic(err) + } + for _, opt := range opts { + opt(e) + } + return e +} + +// OnlyControllerOwner if provided will only look at the first OwnerReference with Controller: true. +func OnlyControllerOwner() OwnerOption { + return func(e *enqueueRequestForOwner) { + e.isController = true + } +} - // IsController if set will only look at the first OwnerReference with Controller: true. - IsController bool +type enqueueRequestForOwner struct { + // ownerType is the type of the Owner object to look for in OwnerReferences. Only Group and Kind are compared. + ownerType runtime.Object + + // isController if set will only look at the first OwnerReference with Controller: true. + isController bool // groupKind is the cached Group and Kind from OwnerType groupKind schema.GroupKind @@ -58,7 +83,7 @@ type EnqueueRequestForOwner struct { } // Create implements EventHandler. -func (e *EnqueueRequestForOwner) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) { +func (e *enqueueRequestForOwner) Create(ctx context.Context, evt event.CreateEvent, q workqueue.RateLimitingInterface) { reqs := map[reconcile.Request]empty{} e.getOwnerReconcileRequest(evt.Object, reqs) for req := range reqs { @@ -67,7 +92,7 @@ func (e *EnqueueRequestForOwner) Create(evt event.CreateEvent, q workqueue.RateL } // Update implements EventHandler. -func (e *EnqueueRequestForOwner) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) { +func (e *enqueueRequestForOwner) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.RateLimitingInterface) { reqs := map[reconcile.Request]empty{} e.getOwnerReconcileRequest(evt.ObjectOld, reqs) e.getOwnerReconcileRequest(evt.ObjectNew, reqs) @@ -77,7 +102,7 @@ func (e *EnqueueRequestForOwner) Update(evt event.UpdateEvent, q workqueue.RateL } // Delete implements EventHandler. -func (e *EnqueueRequestForOwner) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) { +func (e *enqueueRequestForOwner) Delete(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) { reqs := map[reconcile.Request]empty{} e.getOwnerReconcileRequest(evt.Object, reqs) for req := range reqs { @@ -86,7 +111,7 @@ func (e *EnqueueRequestForOwner) Delete(evt event.DeleteEvent, q workqueue.RateL } // Generic implements EventHandler. -func (e *EnqueueRequestForOwner) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) { +func (e *enqueueRequestForOwner) Generic(ctx context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) { reqs := map[reconcile.Request]empty{} e.getOwnerReconcileRequest(evt.Object, reqs) for req := range reqs { @@ -96,17 +121,17 @@ func (e *EnqueueRequestForOwner) Generic(evt event.GenericEvent, q workqueue.Rat // parseOwnerTypeGroupKind parses the OwnerType into a Group and Kind and caches the result. Returns false // if the OwnerType could not be parsed using the scheme. -func (e *EnqueueRequestForOwner) parseOwnerTypeGroupKind(scheme *runtime.Scheme) error { +func (e *enqueueRequestForOwner) parseOwnerTypeGroupKind(scheme *runtime.Scheme) error { // Get the kinds of the type - kinds, _, err := scheme.ObjectKinds(e.OwnerType) + kinds, _, err := scheme.ObjectKinds(e.ownerType) if err != nil { - log.Error(err, "Could not get ObjectKinds for OwnerType", "owner type", fmt.Sprintf("%T", e.OwnerType)) + log.Error(err, "Could not get ObjectKinds for OwnerType", "owner type", fmt.Sprintf("%T", e.ownerType)) return err } // Expect only 1 kind. If there is more than one kind this is probably an edge case such as ListOptions. if len(kinds) != 1 { - err := fmt.Errorf("expected exactly 1 kind for OwnerType %T, but found %s kinds", e.OwnerType, kinds) - log.Error(nil, "expected exactly 1 kind for OwnerType", "owner type", fmt.Sprintf("%T", e.OwnerType), "kinds", kinds) + err := fmt.Errorf("expected exactly 1 kind for OwnerType %T, but found %s kinds", e.ownerType, kinds) + log.Error(nil, "expected exactly 1 kind for OwnerType", "owner type", fmt.Sprintf("%T", e.ownerType), "kinds", kinds) return err } // Cache the Group and Kind for the OwnerType @@ -116,7 +141,7 @@ func (e *EnqueueRequestForOwner) parseOwnerTypeGroupKind(scheme *runtime.Scheme) // getOwnerReconcileRequest looks at object and builds a map of reconcile.Request to reconcile // owners of object that match e.OwnerType. -func (e *EnqueueRequestForOwner) getOwnerReconcileRequest(object metav1.Object, result map[reconcile.Request]empty) { +func (e *enqueueRequestForOwner) getOwnerReconcileRequest(object metav1.Object, result map[reconcile.Request]empty) { // Iterate through the OwnerReferences looking for a match on Group and Kind against what was requested // by the user for _, ref := range e.getOwnersReferences(object) { @@ -138,7 +163,7 @@ func (e *EnqueueRequestForOwner) getOwnerReconcileRequest(object metav1.Object, Name: ref.Name, }} - // if owner is not namespaced then we should set the namespace to the empty + // if owner is not namespaced then we should not set the namespace mapping, err := e.mapper.RESTMapping(e.groupKind, refGV.Version) if err != nil { log.Error(err, "Could not retrieve rest mapping", "kind", e.groupKind) @@ -153,16 +178,16 @@ func (e *EnqueueRequestForOwner) getOwnerReconcileRequest(object metav1.Object, } } -// getOwnersReferences returns the OwnerReferences for an object as specified by the EnqueueRequestForOwner +// getOwnersReferences returns the OwnerReferences for an object as specified by the enqueueRequestForOwner // - if IsController is true: only take the Controller OwnerReference (if found) // - if IsController is false: take all OwnerReferences. -func (e *EnqueueRequestForOwner) getOwnersReferences(object metav1.Object) []metav1.OwnerReference { +func (e *enqueueRequestForOwner) getOwnersReferences(object metav1.Object) []metav1.OwnerReference { if object == nil { return nil } // If not filtered as Controller only, then use all the OwnerReferences - if !e.IsController { + if !e.isController { return object.GetOwnerReferences() } // If filtered to a Controller, only take the Controller OwnerReference @@ -172,18 +197,3 @@ func (e *EnqueueRequestForOwner) getOwnersReferences(object metav1.Object) []met // No Controller OwnerReference found return nil } - -var _ inject.Scheme = &EnqueueRequestForOwner{} - -// InjectScheme is called by the Controller to provide a singleton scheme to the EnqueueRequestForOwner. -func (e *EnqueueRequestForOwner) InjectScheme(s *runtime.Scheme) error { - return e.parseOwnerTypeGroupKind(s) -} - -var _ inject.Mapper = &EnqueueRequestForOwner{} - -// InjectMapper is called by the Controller to provide the rest mapper used by the manager. -func (e *EnqueueRequestForOwner) InjectMapper(m meta.RESTMapper) error { - e.mapper = m - return nil -} diff --git a/pkg/handler/eventhandler.go b/pkg/handler/eventhandler.go index 8652d22d72..2f380f4fc4 100644 --- a/pkg/handler/eventhandler.go +++ b/pkg/handler/eventhandler.go @@ -17,6 +17,8 @@ limitations under the License. package handler import ( + "context" + "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/event" ) @@ -41,17 +43,17 @@ import ( // Most users shouldn't need to implement their own EventHandler. type EventHandler interface { // Create is called in response to an create event - e.g. Pod Creation. - Create(event.CreateEvent, workqueue.RateLimitingInterface) + Create(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) // Update is called in response to an update event - e.g. Pod Updated. - Update(event.UpdateEvent, workqueue.RateLimitingInterface) + Update(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) // Delete is called in response to a delete event - e.g. Pod Deleted. - Delete(event.DeleteEvent, workqueue.RateLimitingInterface) + Delete(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) // Generic is called in response to an event of an unknown type or a synthetic event triggered as a cron or // external trigger request - e.g. reconcile Autoscaling, or a Webhook. - Generic(event.GenericEvent, workqueue.RateLimitingInterface) + Generic(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) } var _ EventHandler = Funcs{} @@ -60,45 +62,45 @@ var _ EventHandler = Funcs{} type Funcs struct { // Create is called in response to an add event. Defaults to no-op. // RateLimitingInterface is used to enqueue reconcile.Requests. - CreateFunc func(event.CreateEvent, workqueue.RateLimitingInterface) + CreateFunc func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) // Update is called in response to an update event. Defaults to no-op. // RateLimitingInterface is used to enqueue reconcile.Requests. - UpdateFunc func(event.UpdateEvent, workqueue.RateLimitingInterface) + UpdateFunc func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) // Delete is called in response to a delete event. Defaults to no-op. // RateLimitingInterface is used to enqueue reconcile.Requests. - DeleteFunc func(event.DeleteEvent, workqueue.RateLimitingInterface) + DeleteFunc func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) // GenericFunc is called in response to a generic event. Defaults to no-op. // RateLimitingInterface is used to enqueue reconcile.Requests. - GenericFunc func(event.GenericEvent, workqueue.RateLimitingInterface) + GenericFunc func(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) } // Create implements EventHandler. -func (h Funcs) Create(e event.CreateEvent, q workqueue.RateLimitingInterface) { +func (h Funcs) Create(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { if h.CreateFunc != nil { - h.CreateFunc(e, q) + h.CreateFunc(ctx, e, q) } } // Delete implements EventHandler. -func (h Funcs) Delete(e event.DeleteEvent, q workqueue.RateLimitingInterface) { +func (h Funcs) Delete(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { if h.DeleteFunc != nil { - h.DeleteFunc(e, q) + h.DeleteFunc(ctx, e, q) } } // Update implements EventHandler. -func (h Funcs) Update(e event.UpdateEvent, q workqueue.RateLimitingInterface) { +func (h Funcs) Update(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) { if h.UpdateFunc != nil { - h.UpdateFunc(e, q) + h.UpdateFunc(ctx, e, q) } } // Generic implements EventHandler. -func (h Funcs) Generic(e event.GenericEvent, q workqueue.RateLimitingInterface) { +func (h Funcs) Generic(ctx context.Context, e event.GenericEvent, q workqueue.RateLimitingInterface) { if h.GenericFunc != nil { - h.GenericFunc(e, q) + h.GenericFunc(ctx, e, q) } } diff --git a/pkg/handler/eventhandler_suite_test.go b/pkg/handler/eventhandler_suite_test.go index ebcc993915..3f6b17f337 100644 --- a/pkg/handler/eventhandler_suite_test.go +++ b/pkg/handler/eventhandler_suite_test.go @@ -19,19 +19,17 @@ package handler_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestEventhandler(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Eventhandler Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Eventhandler Suite") } var testenv *envtest.Environment diff --git a/pkg/handler/eventhandler_test.go b/pkg/handler/eventhandler_test.go index 7ba3b8aa84..17f975ea7d 100644 --- a/pkg/handler/eventhandler_test.go +++ b/pkg/handler/eventhandler_test.go @@ -17,7 +17,9 @@ package handler_test import ( - . "github.com/onsi/ginkgo" + "context" + + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" autoscalingv1 "k8s.io/api/autoscaling/v1" @@ -26,7 +28,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" "k8s.io/client-go/util/workqueue" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/controller/controllertest" @@ -36,11 +40,11 @@ import ( ) var _ = Describe("Eventhandler", func() { + var ctx = context.Background() var q workqueue.RateLimitingInterface var instance handler.EnqueueRequestForObject var pod *corev1.Pod var mapper meta.RESTMapper - t := true BeforeEach(func() { q = controllertest.Queue{Interface: workqueue.New()} pod = &corev1.Pod{ @@ -48,17 +52,18 @@ var _ = Describe("Eventhandler", func() { } Expect(cfg).NotTo(BeNil()) - var err error - mapper, err = apiutil.NewDiscoveryRESTMapper(cfg) + httpClient, err := rest.HTTPClientFor(cfg) + Expect(err).ShouldNot(HaveOccurred()) + mapper, err = apiutil.NewDiscoveryRESTMapper(cfg, httpClient) Expect(err).ShouldNot(HaveOccurred()) }) Describe("EnqueueRequestForObject", func() { - It("should enqueue a Request with the Name / Namespace of the object in the CreateEvent.", func(done Done) { + It("should enqueue a Request with the Name / Namespace of the object in the CreateEvent.", func() { evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(1)) i, _ := q.Get() @@ -66,15 +71,13 @@ var _ = Describe("Eventhandler", func() { req, ok := i.(reconcile.Request) Expect(ok).To(BeTrue()) Expect(req.NamespacedName).To(Equal(types.NamespacedName{Namespace: "biz", Name: "baz"})) - - close(done) }) - It("should enqueue a Request with the Name / Namespace of the object in the DeleteEvent.", func(done Done) { + It("should enqueue a Request with the Name / Namespace of the object in the DeleteEvent.", func() { evt := event.DeleteEvent{ Object: pod, } - instance.Delete(evt, q) + instance.Delete(ctx, evt, q) Expect(q.Len()).To(Equal(1)) i, _ := q.Get() @@ -82,12 +85,10 @@ var _ = Describe("Eventhandler", func() { req, ok := i.(reconcile.Request) Expect(ok).To(BeTrue()) Expect(req.NamespacedName).To(Equal(types.NamespacedName{Namespace: "biz", Name: "baz"})) - - close(done) }) It("should enqueue a Request with the Name / Namespace of one object in the UpdateEvent.", - func(done Done) { + func() { newPod := pod.DeepCopy() newPod.Name = "baz2" newPod.Namespace = "biz2" @@ -96,7 +97,7 @@ var _ = Describe("Eventhandler", func() { ObjectOld: pod, ObjectNew: newPod, } - instance.Update(evt, q) + instance.Update(ctx, evt, q) Expect(q.Len()).To(Equal(1)) i, _ := q.Get() @@ -104,36 +105,31 @@ var _ = Describe("Eventhandler", func() { req, ok := i.(reconcile.Request) Expect(ok).To(BeTrue()) Expect(req.NamespacedName).To(Equal(types.NamespacedName{Namespace: "biz2", Name: "baz2"})) - - close(done) }) - It("should enqueue a Request with the Name / Namespace of the object in the GenericEvent.", func(done Done) { + It("should enqueue a Request with the Name / Namespace of the object in the GenericEvent.", func() { evt := event.GenericEvent{ Object: pod, } - instance.Generic(evt, q) + instance.Generic(ctx, evt, q) Expect(q.Len()).To(Equal(1)) i, _ := q.Get() Expect(i).NotTo(BeNil()) req, ok := i.(reconcile.Request) Expect(ok).To(BeTrue()) Expect(req.NamespacedName).To(Equal(types.NamespacedName{Namespace: "biz", Name: "baz"})) - - close(done) }) Context("for a runtime.Object without Object", func() { - It("should do nothing if the Object is missing for a CreateEvent.", func(done Done) { + It("should do nothing if the Object is missing for a CreateEvent.", func() { evt := event.CreateEvent{ Object: nil, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(0)) - close(done) }) - It("should do nothing if the Object is missing for a UpdateEvent.", func(done Done) { + It("should do nothing if the Object is missing for a UpdateEvent.", func() { newPod := pod.DeepCopy() newPod.Name = "baz2" newPod.Namespace = "biz2" @@ -142,7 +138,7 @@ var _ = Describe("Eventhandler", func() { ObjectNew: newPod, ObjectOld: nil, } - instance.Update(evt, q) + instance.Update(ctx, evt, q) Expect(q.Len()).To(Equal(1)) i, _ := q.Get() Expect(i).NotTo(BeNil()) @@ -152,33 +148,29 @@ var _ = Describe("Eventhandler", func() { evt.ObjectNew = nil evt.ObjectOld = pod - instance.Update(evt, q) + instance.Update(ctx, evt, q) Expect(q.Len()).To(Equal(1)) i, _ = q.Get() Expect(i).NotTo(BeNil()) req, ok = i.(reconcile.Request) Expect(ok).To(BeTrue()) Expect(req.NamespacedName).To(Equal(types.NamespacedName{Namespace: "biz", Name: "baz"})) - - close(done) }) - It("should do nothing if the Object is missing for a DeleteEvent.", func(done Done) { + It("should do nothing if the Object is missing for a DeleteEvent.", func() { evt := event.DeleteEvent{ Object: nil, } - instance.Delete(evt, q) + instance.Delete(ctx, evt, q) Expect(q.Len()).To(Equal(0)) - close(done) }) - It("should do nothing if the Object is missing for a GenericEvent.", func(done Done) { + It("should do nothing if the Object is missing for a GenericEvent.", func() { evt := event.GenericEvent{ Object: nil, } - instance.Generic(evt, q) + instance.Generic(ctx, evt, q) Expect(q.Len()).To(Equal(0)) - close(done) }) }) }) @@ -186,7 +178,7 @@ var _ = Describe("Eventhandler", func() { Describe("EnqueueRequestsFromMapFunc", func() { It("should enqueue a Request with the function applied to the CreateEvent.", func() { req := []reconcile.Request{} - instance := handler.EnqueueRequestsFromMapFunc(func(a client.Object) []reconcile.Request { + instance := handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, a client.Object) []reconcile.Request { defer GinkgoRecover() Expect(a).To(Equal(pod)) req = []reconcile.Request{ @@ -203,7 +195,7 @@ var _ = Describe("Eventhandler", func() { evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(2)) i1, _ := q.Get() @@ -218,7 +210,7 @@ var _ = Describe("Eventhandler", func() { It("should enqueue a Request with the function applied to the DeleteEvent.", func() { req := []reconcile.Request{} - instance := handler.EnqueueRequestsFromMapFunc(func(a client.Object) []reconcile.Request { + instance := handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, a client.Object) []reconcile.Request { defer GinkgoRecover() Expect(a).To(Equal(pod)) req = []reconcile.Request{ @@ -235,7 +227,7 @@ var _ = Describe("Eventhandler", func() { evt := event.DeleteEvent{ Object: pod, } - instance.Delete(evt, q) + instance.Delete(ctx, evt, q) Expect(q.Len()).To(Equal(2)) i1, _ := q.Get() @@ -254,7 +246,7 @@ var _ = Describe("Eventhandler", func() { req := []reconcile.Request{} - instance := handler.EnqueueRequestsFromMapFunc(func(a client.Object) []reconcile.Request { + instance := handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, a client.Object) []reconcile.Request { defer GinkgoRecover() req = []reconcile.Request{ { @@ -271,7 +263,7 @@ var _ = Describe("Eventhandler", func() { ObjectOld: pod, ObjectNew: newPod, } - instance.Update(evt, q) + instance.Update(ctx, evt, q) Expect(q.Len()).To(Equal(2)) i, _ := q.Get() @@ -283,7 +275,7 @@ var _ = Describe("Eventhandler", func() { It("should enqueue a Request with the function applied to the GenericEvent.", func() { req := []reconcile.Request{} - instance := handler.EnqueueRequestsFromMapFunc(func(a client.Object) []reconcile.Request { + instance := handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, a client.Object) []reconcile.Request { defer GinkgoRecover() Expect(a).To(Equal(pod)) req = []reconcile.Request{ @@ -300,7 +292,7 @@ var _ = Describe("Eventhandler", func() { evt := event.GenericEvent{ Object: pod, } - instance.Generic(evt, q) + instance.Generic(ctx, evt, q) Expect(q.Len()).To(Equal(2)) i1, _ := q.Get() @@ -316,11 +308,7 @@ var _ = Describe("Eventhandler", func() { Describe("EnqueueRequestForOwner", func() { It("should enqueue a Request with the Owner of the object in the CreateEvent.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) pod.OwnerReferences = []metav1.OwnerReference{ { @@ -332,7 +320,7 @@ var _ = Describe("Eventhandler", func() { evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(1)) i, _ := q.Get() @@ -341,11 +329,7 @@ var _ = Describe("Eventhandler", func() { }) It("should enqueue a Request with the Owner of the object in the DeleteEvent.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) pod.OwnerReferences = []metav1.OwnerReference{ { @@ -357,7 +341,7 @@ var _ = Describe("Eventhandler", func() { evt := event.DeleteEvent{ Object: pod, } - instance.Delete(evt, q) + instance.Delete(ctx, evt, q) Expect(q.Len()).To(Equal(1)) i, _ := q.Get() @@ -370,11 +354,7 @@ var _ = Describe("Eventhandler", func() { newPod.Name = pod.Name + "2" newPod.Namespace = pod.Namespace + "2" - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) pod.OwnerReferences = []metav1.OwnerReference{ { @@ -394,7 +374,7 @@ var _ = Describe("Eventhandler", func() { ObjectOld: pod, ObjectNew: newPod, } - instance.Update(evt, q) + instance.Update(ctx, evt, q) Expect(q.Len()).To(Equal(2)) i1, _ := q.Get() @@ -411,11 +391,7 @@ var _ = Describe("Eventhandler", func() { newPod := pod.DeepCopy() newPod.Name = pod.Name + "2" - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) pod.OwnerReferences = []metav1.OwnerReference{ { @@ -435,7 +411,7 @@ var _ = Describe("Eventhandler", func() { ObjectOld: pod, ObjectNew: newPod, } - instance.Update(evt, q) + instance.Update(ctx, evt, q) Expect(q.Len()).To(Equal(1)) i, _ := q.Get() @@ -444,12 +420,7 @@ var _ = Describe("Eventhandler", func() { }) It("should enqueue a Request with the Owner of the object in the GenericEvent.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) - + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) pod.OwnerReferences = []metav1.OwnerReference{ { Name: "foo-parent", @@ -460,7 +431,7 @@ var _ = Describe("Eventhandler", func() { evt := event.GenericEvent{ Object: pod, } - instance.Generic(evt, q) + instance.Generic(ctx, evt, q) Expect(q.Len()).To(Equal(1)) i, _ := q.Get() @@ -469,12 +440,7 @@ var _ = Describe("Eventhandler", func() { }) It("should not enqueue a Request if there are no owners matching Group and Kind.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - IsController: t, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}, handler.OnlyControllerOwner()) pod.OwnerReferences = []metav1.OwnerReference{ { // Wrong group Name: "foo1-parent", @@ -490,17 +456,13 @@ var _ = Describe("Eventhandler", func() { evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(0)) }) It("should enqueue a Request if there are owners matching Group "+ "and Kind with a different version.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &autoscalingv1.HorizontalPodAutoscaler{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &autoscalingv1.HorizontalPodAutoscaler{}) pod.OwnerReferences = []metav1.OwnerReference{ { Name: "foo-parent", @@ -511,7 +473,7 @@ var _ = Describe("Eventhandler", func() { evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(1)) i, _ := q.Get() @@ -520,11 +482,7 @@ var _ = Describe("Eventhandler", func() { }) It("should enqueue a Request for a owner that is cluster scoped", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &corev1.Node{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &corev1.Node{}) pod.OwnerReferences = []metav1.OwnerReference{ { Name: "node-1", @@ -535,7 +493,7 @@ var _ = Describe("Eventhandler", func() { evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(1)) i, _ := q.Get() @@ -545,27 +503,18 @@ var _ = Describe("Eventhandler", func() { }) It("should not enqueue a Request if there are no owners.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(0)) }) Context("with the Controller field set to true", func() { It("should enqueue reconcile.Requests for only the first the Controller if there are "+ "multiple Controller owners.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - IsController: t, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}, handler.OnlyControllerOwner()) pod.OwnerReferences = []metav1.OwnerReference{ { Name: "foo1-parent", @@ -576,7 +525,7 @@ var _ = Describe("Eventhandler", func() { Name: "foo2-parent", Kind: "ReplicaSet", APIVersion: "apps/v1", - Controller: &t, + Controller: pointer.Bool(true), }, { Name: "foo3-parent", @@ -587,7 +536,7 @@ var _ = Describe("Eventhandler", func() { Name: "foo4-parent", Kind: "ReplicaSet", APIVersion: "apps/v1", - Controller: &t, + Controller: pointer.Bool(true), }, { Name: "foo5-parent", @@ -598,7 +547,7 @@ var _ = Describe("Eventhandler", func() { evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(1)) i, _ := q.Get() Expect(i).To(Equal(reconcile.Request{ @@ -606,12 +555,7 @@ var _ = Describe("Eventhandler", func() { }) It("should not enqueue reconcile.Requests if there are no Controller owners.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - IsController: t, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}, handler.OnlyControllerOwner()) pod.OwnerReferences = []metav1.OwnerReference{ { Name: "foo1-parent", @@ -632,32 +576,23 @@ var _ = Describe("Eventhandler", func() { evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(0)) }) It("should not enqueue reconcile.Requests if there are no owners.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - IsController: t, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}, handler.OnlyControllerOwner()) evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(0)) }) }) Context("with the Controller field set to false", func() { It("should enqueue a reconcile.Requests for all owners.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) pod.OwnerReferences = []metav1.OwnerReference{ { Name: "foo1-parent", @@ -678,7 +613,7 @@ var _ = Describe("Eventhandler", func() { evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(3)) i1, _ := q.Get() @@ -697,11 +632,7 @@ var _ = Describe("Eventhandler", func() { Context("with a nil object", func() { It("should do nothing.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) pod.OwnerReferences = []metav1.OwnerReference{ { Name: "foo1-parent", @@ -712,81 +643,22 @@ var _ = Describe("Eventhandler", func() { evt := event.CreateEvent{ Object: nil, } - instance.Create(evt, q) - Expect(q.Len()).To(Equal(0)) - }) - }) - - Context("with a multiple matching kinds", func() { - It("should do nothing.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &metav1.ListOptions{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).NotTo(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) - pod.OwnerReferences = []metav1.OwnerReference{ - { - Name: "foo1-parent", - Kind: "ListOptions", - APIVersion: "meta/v1", - }, - } - evt := event.CreateEvent{ - Object: pod, - } - instance.Create(evt, q) - Expect(q.Len()).To(Equal(0)) - }) - }) - Context("with an OwnerType that cannot be resolved", func() { - It("should do nothing.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &controllertest.ErrorType{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).NotTo(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) - pod.OwnerReferences = []metav1.OwnerReference{ - { - Name: "foo1-parent", - Kind: "ListOptions", - APIVersion: "meta/v1", - }, - } - evt := event.CreateEvent{ - Object: pod, - } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(0)) }) }) Context("with a nil OwnerType", func() { - It("should do nothing.", func() { - instance := handler.EnqueueRequestForOwner{} - Expect(instance.InjectScheme(scheme.Scheme)).NotTo(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) - pod.OwnerReferences = []metav1.OwnerReference{ - { - Name: "foo1-parent", - Kind: "OwnerType", - APIVersion: "meta/v1", - }, - } - evt := event.CreateEvent{ - Object: pod, - } - instance.Create(evt, q) - Expect(q.Len()).To(Equal(0)) + It("should panic", func() { + Expect(func() { + handler.EnqueueRequestForOwner(nil, nil, nil) + }).To(Panic()) }) }) Context("with an invalid APIVersion in the OwnerReference", func() { It("should do nothing.", func() { - instance := handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.ReplicaSet{}, - } - Expect(instance.InjectScheme(scheme.Scheme)).To(Succeed()) - Expect(instance.InjectMapper(mapper)).To(Succeed()) + instance := handler.EnqueueRequestForOwner(scheme.Scheme, mapper, &appsv1.ReplicaSet{}) pod.OwnerReferences = []metav1.OwnerReference{ { Name: "foo1-parent", @@ -797,7 +669,7 @@ var _ = Describe("Eventhandler", func() { evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) + instance.Create(ctx, evt, q) Expect(q.Len()).To(Equal(0)) }) }) @@ -805,49 +677,47 @@ var _ = Describe("Eventhandler", func() { Describe("Funcs", func() { failingFuncs := handler.Funcs{ - CreateFunc: func(event.CreateEvent, workqueue.RateLimitingInterface) { + CreateFunc: func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Did not expect CreateEvent to be called.") }, - DeleteFunc: func(event.DeleteEvent, workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Did not expect DeleteEvent to be called.") }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Did not expect UpdateEvent to be called.") }, - GenericFunc: func(event.GenericEvent, workqueue.RateLimitingInterface) { + GenericFunc: func(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Did not expect GenericEvent to be called.") }, } - It("should call CreateFunc for a CreateEvent if provided.", func(done Done) { + It("should call CreateFunc for a CreateEvent if provided.", func() { instance := failingFuncs evt := event.CreateEvent{ Object: pod, } - instance.CreateFunc = func(evt2 event.CreateEvent, q2 workqueue.RateLimitingInterface) { + instance.CreateFunc = func(ctx context.Context, evt2 event.CreateEvent, q2 workqueue.RateLimitingInterface) { defer GinkgoRecover() Expect(q2).To(Equal(q)) Expect(evt2).To(Equal(evt)) } - instance.Create(evt, q) - close(done) + instance.Create(ctx, evt, q) }) - It("should NOT call CreateFunc for a CreateEvent if NOT provided.", func(done Done) { + It("should NOT call CreateFunc for a CreateEvent if NOT provided.", func() { instance := failingFuncs instance.CreateFunc = nil evt := event.CreateEvent{ Object: pod, } - instance.Create(evt, q) - close(done) + instance.Create(ctx, evt, q) }) - It("should call UpdateFunc for an UpdateEvent if provided.", func(done Done) { + It("should call UpdateFunc for an UpdateEvent if provided.", func() { newPod := pod.DeepCopy() newPod.Name = pod.Name + "2" newPod.Namespace = pod.Namespace + "2" @@ -857,17 +727,16 @@ var _ = Describe("Eventhandler", func() { } instance := failingFuncs - instance.UpdateFunc = func(evt2 event.UpdateEvent, q2 workqueue.RateLimitingInterface) { + instance.UpdateFunc = func(ctx context.Context, evt2 event.UpdateEvent, q2 workqueue.RateLimitingInterface) { defer GinkgoRecover() Expect(q2).To(Equal(q)) Expect(evt2).To(Equal(evt)) } - instance.Update(evt, q) - close(done) + instance.Update(ctx, evt, q) }) - It("should NOT call UpdateFunc for an UpdateEvent if NOT provided.", func(done Done) { + It("should NOT call UpdateFunc for an UpdateEvent if NOT provided.", func() { newPod := pod.DeepCopy() newPod.Name = pod.Name + "2" newPod.Namespace = pod.Namespace + "2" @@ -875,56 +744,51 @@ var _ = Describe("Eventhandler", func() { ObjectOld: pod, ObjectNew: newPod, } - instance.Update(evt, q) - close(done) + instance.Update(ctx, evt, q) }) - It("should call DeleteFunc for a DeleteEvent if provided.", func(done Done) { + It("should call DeleteFunc for a DeleteEvent if provided.", func() { instance := failingFuncs evt := event.DeleteEvent{ Object: pod, } - instance.DeleteFunc = func(evt2 event.DeleteEvent, q2 workqueue.RateLimitingInterface) { + instance.DeleteFunc = func(ctx context.Context, evt2 event.DeleteEvent, q2 workqueue.RateLimitingInterface) { defer GinkgoRecover() Expect(q2).To(Equal(q)) Expect(evt2).To(Equal(evt)) } - instance.Delete(evt, q) - close(done) + instance.Delete(ctx, evt, q) }) - It("should NOT call DeleteFunc for a DeleteEvent if NOT provided.", func(done Done) { + It("should NOT call DeleteFunc for a DeleteEvent if NOT provided.", func() { instance := failingFuncs instance.DeleteFunc = nil evt := event.DeleteEvent{ Object: pod, } - instance.Delete(evt, q) - close(done) + instance.Delete(ctx, evt, q) }) - It("should call GenericFunc for a GenericEvent if provided.", func(done Done) { + It("should call GenericFunc for a GenericEvent if provided.", func() { instance := failingFuncs evt := event.GenericEvent{ Object: pod, } - instance.GenericFunc = func(evt2 event.GenericEvent, q2 workqueue.RateLimitingInterface) { + instance.GenericFunc = func(ctx context.Context, evt2 event.GenericEvent, q2 workqueue.RateLimitingInterface) { defer GinkgoRecover() Expect(q2).To(Equal(q)) Expect(evt2).To(Equal(evt)) } - instance.Generic(evt, q) - close(done) + instance.Generic(ctx, evt, q) }) - It("should NOT call GenericFunc for a GenericEvent if NOT provided.", func(done Done) { + It("should NOT call GenericFunc for a GenericEvent if NOT provided.", func() { instance := failingFuncs instance.GenericFunc = nil evt := event.GenericEvent{ Object: pod, } - instance.Generic(evt, q) - close(done) + instance.Generic(ctx, evt, q) }) }) }) diff --git a/pkg/handler/example_test.go b/pkg/handler/example_test.go index dbfab46157..ad07e4e31d 100644 --- a/pkg/handler/example_test.go +++ b/pkg/handler/example_test.go @@ -17,6 +17,8 @@ limitations under the License. package handler_test import ( + "context" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -25,10 +27,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" ) +var mgr manager.Manager var c controller.Controller // This example watches Pods and enqueues Requests with the Name and Namespace of the Pod from @@ -36,7 +40,7 @@ var c controller.Controller func ExampleEnqueueRequestForObject() { // controller is a controller.controller err := c.Watch( - &source.Kind{Type: &corev1.Pod{}}, + source.Kind(mgr.GetCache(), &corev1.Pod{}), &handler.EnqueueRequestForObject{}, ) if err != nil { @@ -49,11 +53,8 @@ func ExampleEnqueueRequestForObject() { func ExampleEnqueueRequestForOwner() { // controller is a controller.controller err := c.Watch( - &source.Kind{Type: &appsv1.ReplicaSet{}}, - &handler.EnqueueRequestForOwner{ - OwnerType: &appsv1.Deployment{}, - IsController: true, - }, + source.Kind(mgr.GetCache(), &appsv1.ReplicaSet{}), + handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &appsv1.Deployment{}, handler.OnlyControllerOwner()), ) if err != nil { // handle it @@ -65,8 +66,8 @@ func ExampleEnqueueRequestForOwner() { func ExampleEnqueueRequestsFromMapFunc() { // controller is a controller.controller err := c.Watch( - &source.Kind{Type: &appsv1.Deployment{}}, - handler.EnqueueRequestsFromMapFunc(func(a client.Object) []reconcile.Request { + source.Kind(mgr.GetCache(), &appsv1.Deployment{}), + handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, a client.Object) []reconcile.Request { return []reconcile.Request{ {NamespacedName: types.NamespacedName{ Name: a.GetName() + "-1", @@ -88,27 +89,27 @@ func ExampleEnqueueRequestsFromMapFunc() { func ExampleFuncs() { // controller is a controller.controller err := c.Watch( - &source.Kind{Type: &corev1.Pod{}}, + source.Kind(mgr.GetCache(), &corev1.Pod{}), handler.Funcs{ - CreateFunc: func(e event.CreateEvent, q workqueue.RateLimitingInterface) { + CreateFunc: func(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) { q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ Name: e.Object.GetName(), Namespace: e.Object.GetNamespace(), }}) }, - UpdateFunc: func(e event.UpdateEvent, q workqueue.RateLimitingInterface) { + UpdateFunc: func(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) { q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ Name: e.ObjectNew.GetName(), Namespace: e.ObjectNew.GetNamespace(), }}) }, - DeleteFunc: func(e event.DeleteEvent, q workqueue.RateLimitingInterface) { + DeleteFunc: func(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) { q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ Name: e.Object.GetName(), Namespace: e.Object.GetNamespace(), }}) }, - GenericFunc: func(e event.GenericEvent, q workqueue.RateLimitingInterface) { + GenericFunc: func(ctx context.Context, e event.GenericEvent, q workqueue.RateLimitingInterface) { q.Add(reconcile.Request{NamespacedName: types.NamespacedName{ Name: e.Object.GetName(), Namespace: e.Object.GetNamespace(), diff --git a/pkg/healthz/healthz.go b/pkg/healthz/healthz.go index bd1cc151af..cfb5dc8d02 100644 --- a/pkg/healthz/healthz.go +++ b/pkg/healthz/healthz.go @@ -70,7 +70,7 @@ func (h *Handler) serveAggregated(resp http.ResponseWriter, req *http.Request) { parts = append(parts, checkStatus{name: "ping", healthy: true}) } - for _, c := range excluded.List() { + for _, c := range excluded.UnsortedList() { log.V(1).Info("cannot exclude health check, no matches for it", "checker", c) } @@ -88,7 +88,7 @@ func (h *Handler) serveAggregated(resp http.ResponseWriter, req *http.Request) { // any checks that the user requested to have excluded, but weren't actually // known checks. writeStatusAsText is always verbose on failure, and can be // forced to be verbose on success using the given argument. -func writeStatusesAsText(resp http.ResponseWriter, parts []checkStatus, unknownExcludes sets.String, failed, forceVerbose bool) { +func writeStatusesAsText(resp http.ResponseWriter, parts []checkStatus, unknownExcludes sets.Set[string], failed, forceVerbose bool) { resp.Header().Set("Content-Type", "text/plain; charset=utf-8") resp.Header().Set("X-Content-Type-Options", "nosniff") @@ -121,7 +121,7 @@ func writeStatusesAsText(resp http.ResponseWriter, parts []checkStatus, unknownE } if unknownExcludes.Len() > 0 { - fmt.Fprintf(resp, "warn: some health checks cannot be excluded: no matches for %s\n", formatQuoted(unknownExcludes.List()...)) + fmt.Fprintf(resp, "warn: some health checks cannot be excluded: no matches for %s\n", formatQuoted(unknownExcludes.UnsortedList()...)) } if failed { @@ -187,12 +187,12 @@ type Checker func(req *http.Request) error var Ping Checker = func(_ *http.Request) error { return nil } // getExcludedChecks extracts the health check names to be excluded from the query param. -func getExcludedChecks(r *http.Request) sets.String { +func getExcludedChecks(r *http.Request) sets.Set[string] { checks, found := r.URL.Query()["exclude"] if found { - return sets.NewString(checks...) + return sets.New[string](checks...) } - return sets.NewString() + return sets.New[string]() } // formatQuoted returns a formatted string of the health check names, diff --git a/pkg/healthz/healthz_suite_test.go b/pkg/healthz/healthz_suite_test.go index b51fcb3605..8e16a58aa0 100644 --- a/pkg/healthz/healthz_suite_test.go +++ b/pkg/healthz/healthz_suite_test.go @@ -19,17 +19,15 @@ package healthz_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestHealthz(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Healthz Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Healthz Suite") } var _ = BeforeSuite(func() { diff --git a/pkg/healthz/healthz_test.go b/pkg/healthz/healthz_test.go index e0413103f7..639a7575f3 100644 --- a/pkg/healthz/healthz_test.go +++ b/pkg/healthz/healthz_test.go @@ -21,7 +21,7 @@ import ( "net/http" "net/http/httptest" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "sigs.k8s.io/controller-runtime/pkg/healthz" ) diff --git a/pkg/internal/controller/controller.go b/pkg/internal/controller/controller.go index 224d300b89..969eeeb7d2 100644 --- a/pkg/internal/controller/controller.go +++ b/pkg/internal/controller/controller.go @@ -24,19 +24,18 @@ import ( "time" "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/handler" ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/source" ) -var _ inject.Injector = &Controller{} - // Controller implements controller.Controller. type Controller struct { // Name is used to uniquely identify a Controller in tracing, logging and monitoring. Name is required. @@ -59,10 +58,6 @@ type Controller struct { // the Queue for processing Queue workqueue.RateLimitingInterface - // SetFields is used to inject dependencies into other objects such as Sources, EventHandlers and Predicates - // Deprecated: the caller should handle injected fields itself. - SetFields func(i interface{}) error - // mu is used to synchronize Controller setup mu sync.Mutex @@ -83,8 +78,17 @@ type Controller struct { // startWatches maintains a list of sources, handlers, and predicates to start when the controller is started. startWatches []watchDescription - // Log is used to log messages to users during reconciliation, or for example when a watch is started. - Log logr.Logger + // LogConstructor is used to construct a logger to then log messages to users during reconciliation, + // or for example when a watch is started. + // Note: LogConstructor has to be able to handle nil requests as we are also using it + // outside the context of a reconciliation. + LogConstructor func(request *reconcile.Request) logr.Logger + + // RecoverPanic indicates whether the panic caused by reconcile should be recovered. + RecoverPanic *bool + + // LeaderElected indicates whether the controller is leader elected or always running. + LeaderElected *bool } // watchDescription contains all the information necessary to start a watch. @@ -95,9 +99,22 @@ type watchDescription struct { } // Reconcile implements reconcile.Reconciler. -func (c *Controller) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - log := c.Log.WithValues("name", req.Name, "namespace", req.Namespace) - ctx = logf.IntoContext(ctx, log) +func (c *Controller) Reconcile(ctx context.Context, req reconcile.Request) (_ reconcile.Result, err error) { + defer func() { + if r := recover(); r != nil { + if c.RecoverPanic != nil && *c.RecoverPanic { + for _, fn := range utilruntime.PanicHandlers { + fn(r) + } + err = fmt.Errorf("panic: %v [recovered]", r) + return + } + + log := logf.FromContext(ctx) + log.Info(fmt.Sprintf("Observed a panic in reconciler: %v", r)) + panic(r) + } + }() return c.Do.Reconcile(ctx, req) } @@ -106,19 +123,6 @@ func (c *Controller) Watch(src source.Source, evthdler handler.EventHandler, prc c.mu.Lock() defer c.mu.Unlock() - // Inject Cache into arguments - if err := c.SetFields(src); err != nil { - return err - } - if err := c.SetFields(evthdler); err != nil { - return err - } - for _, pr := range prct { - if err := c.SetFields(pr); err != nil { - return err - } - } - // Controller hasn't started yet, store the watches locally and return. // // These watches are going to be held on the controller struct until the manager or user calls Start(...). @@ -127,10 +131,18 @@ func (c *Controller) Watch(src source.Source, evthdler handler.EventHandler, prc return nil } - c.Log.Info("Starting EventSource", "source", src) + c.LogConstructor(nil).Info("Starting EventSource", "source", src) return src.Start(c.ctx, evthdler, c.Queue, prct...) } +// NeedLeaderElection implements the manager.LeaderElectionRunnable interface. +func (c *Controller) NeedLeaderElection() bool { + if c.LeaderElected == nil { + return true + } + return *c.LeaderElected +} + // Start implements controller.Controller. func (c *Controller) Start(ctx context.Context) error { // use an IIFE to get proper lock handling @@ -162,7 +174,7 @@ func (c *Controller) Start(ctx context.Context) error { // caches to sync so that they have a chance to register their intendeded // caches. for _, watch := range c.startWatches { - c.Log.Info("Starting EventSource", "source", watch.src) + c.LogConstructor(nil).Info("Starting EventSource", "source", fmt.Sprintf("%s", watch.src)) if err := watch.src.Start(ctx, watch.handler, c.Queue, watch.predicates...); err != nil { return err @@ -170,7 +182,7 @@ func (c *Controller) Start(ctx context.Context) error { } // Start the SharedIndexInformer factories to begin populating the SharedIndexInformer caches - c.Log.Info("Starting Controller") + c.LogConstructor(nil).Info("Starting Controller") for _, watch := range c.startWatches { syncingSource, ok := watch.src.(source.SyncingSource) @@ -187,7 +199,7 @@ func (c *Controller) Start(ctx context.Context) error { // is an error or a timeout if err := syncingSource.WaitForSync(sourceStartCtx); err != nil { err := fmt.Errorf("failed to wait for %s caches to sync: %w", c.Name, err) - c.Log.Error(err, "Could not wait for Cache to sync") + c.LogConstructor(nil).Error(err, "Could not wait for Cache to sync") return err } @@ -204,7 +216,7 @@ func (c *Controller) Start(ctx context.Context) error { c.startWatches = nil // Launch workers to process resources - c.Log.Info("Starting workers", "worker count", c.MaxConcurrentReconciles) + c.LogConstructor(nil).Info("Starting workers", "worker count", c.MaxConcurrentReconciles) wg.Add(c.MaxConcurrentReconciles) for i := 0; i < c.MaxConcurrentReconciles; i++ { go func() { @@ -224,9 +236,9 @@ func (c *Controller) Start(ctx context.Context) error { } <-ctx.Done() - c.Log.Info("Shutdown signal received, waiting for all workers to finish") + c.LogConstructor(nil).Info("Shutdown signal received, waiting for all workers to finish") wg.Wait() - c.Log.Info("All workers finished") + c.LogConstructor(nil).Info("All workers finished") return nil } @@ -278,24 +290,28 @@ func (c *Controller) reconcileHandler(ctx context.Context, obj interface{}) { c.updateMetrics(time.Since(reconcileStartTS)) }() - // Make sure that the the object is a valid request. + // Make sure that the object is a valid request. req, ok := obj.(reconcile.Request) if !ok { // As the item in the workqueue is actually invalid, we call // Forget here else we'd go into a loop of attempting to // process a work item that is invalid. c.Queue.Forget(obj) - c.Log.Error(nil, "Queue item was not a Request", "type", fmt.Sprintf("%T", obj), "value", obj) + c.LogConstructor(nil).Error(nil, "Queue item was not a Request", "type", fmt.Sprintf("%T", obj), "value", obj) // Return true, don't take a break return } - log := c.Log.WithValues("name", req.Name, "namespace", req.Namespace) + log := c.LogConstructor(&req) + reconcileID := uuid.NewUUID() + + log = log.WithValues("reconcileID", reconcileID) ctx = logf.IntoContext(ctx, log) + ctx = addReconcileID(ctx, reconcileID) // RunInformersAndControllers the syncHandler, passing it the Namespace/Name string of the // resource to be synced. - result, err := c.Do.Reconcile(ctx, req) + result, err := c.Reconcile(ctx, req) switch { case err != nil: c.Queue.AddRateLimited(req) @@ -323,16 +339,28 @@ func (c *Controller) reconcileHandler(ctx context.Context, obj interface{}) { // GetLogger returns this controller's logger. func (c *Controller) GetLogger() logr.Logger { - return c.Log -} - -// InjectFunc implement SetFields.Injector. -func (c *Controller) InjectFunc(f inject.Func) error { - c.SetFields = f - return nil + return c.LogConstructor(nil) } // updateMetrics updates prometheus metrics within the controller. func (c *Controller) updateMetrics(reconcileTime time.Duration) { ctrlmetrics.ReconcileTime.WithLabelValues(c.Name).Observe(reconcileTime.Seconds()) } + +// ReconcileIDFromContext gets the reconcileID from the current context. +func ReconcileIDFromContext(ctx context.Context) types.UID { + r, ok := ctx.Value(reconcileIDKey{}).(types.UID) + if !ok { + return "" + } + + return r +} + +// reconcileIDKey is a context.Context Value key. Its associated value should +// be a types.UID. +type reconcileIDKey struct{} + +func addReconcileID(ctx context.Context, reconcileID types.UID) context.Context { + return context.WithValue(ctx, reconcileIDKey{}, reconcileID) +} diff --git a/pkg/internal/controller/controller_suite_test.go b/pkg/internal/controller/controller_suite_test.go index 31567e66a5..3143d3dd74 100644 --- a/pkg/internal/controller/controller_suite_test.go +++ b/pkg/internal/controller/controller_suite_test.go @@ -19,27 +19,25 @@ package controller import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Controller internal Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Controller internal Suite") } var testenv *envtest.Environment var cfg *rest.Config var clientset *kubernetes.Clientset -var _ = BeforeSuite(func(done Done) { +var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) testenv = &envtest.Environment{} @@ -50,9 +48,7 @@ var _ = BeforeSuite(func(done Done) { clientset, err = kubernetes.NewForConfig(cfg) Expect(err).NotTo(HaveOccurred()) - - close(done) -}, 60) +}) var _ = AfterSuite(func() { Expect(testenv.Stop()).To(Succeed()) diff --git a/pkg/internal/controller/controller_test.go b/pkg/internal/controller/controller_test.go index 5bed7696ac..9024556fd0 100644 --- a/pkg/internal/controller/controller_test.go +++ b/pkg/internal/controller/controller_test.go @@ -23,7 +23,8 @@ import ( "sync" "time" - . "github.com/onsi/ginkgo" + "github.com/go-logr/logr" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" @@ -32,6 +33,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/cache/informertest" "sigs.k8s.io/controller-runtime/pkg/client" @@ -42,7 +44,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/internal/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/source" ) @@ -50,7 +51,6 @@ var _ = Describe("controller", func() { var fakeReconcile *fakeReconciler var ctrl *Controller var queue *controllertest.Queue - var informers *informertest.FakeInformers var reconciled chan reconcile.Request var request = reconcile.Request{ NamespacedName: types.NamespacedName{Namespace: "foo", Name: "bar"}, @@ -65,14 +65,14 @@ var _ = Describe("controller", func() { queue = &controllertest.Queue{ Interface: workqueue.New(), } - informers = &informertest.FakeInformers{} ctrl = &Controller{ MaxConcurrentReconciles: 1, Do: fakeReconcile, MakeQueue: func() workqueue.RateLimitingInterface { return queue }, - Log: log.RuntimeLog.WithName("controller").WithName("test"), + LogConstructor: func(_ *reconcile.Request) logr.Logger { + return log.RuntimeLog.WithName("controller").WithName("test") + }, } - Expect(ctrl.InjectFunc(func(interface{}) error { return nil })).To(Succeed()) }) Describe("Reconciler", func() { @@ -88,13 +88,46 @@ var _ = Describe("controller", func() { Expect(err).NotTo(HaveOccurred()) Expect(result).To(Equal(reconcile.Result{Requeue: true})) }) + + It("should not recover panic if RecoverPanic is false by default", func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + defer func() { + Expect(recover()).ShouldNot(BeNil()) + }() + ctrl.Do = reconcile.Func(func(context.Context, reconcile.Request) (reconcile.Result, error) { + var res *reconcile.Result + return *res, nil + }) + _, _ = ctrl.Reconcile(ctx, + reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "foo", Name: "bar"}}) + }) + + It("should recover panic if RecoverPanic is true", func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + defer func() { + Expect(recover()).To(BeNil()) + }() + ctrl.RecoverPanic = pointer.Bool(true) + ctrl.Do = reconcile.Func(func(context.Context, reconcile.Request) (reconcile.Result, error) { + var res *reconcile.Result + return *res, nil + }) + _, err := ctrl.Reconcile(ctx, + reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "foo", Name: "bar"}}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("[recovered]")) + }) }) Describe("Start", func() { - It("should return an error if there is an error waiting for the informers", func(done Done) { + It("should return an error if there is an error waiting for the informers", func() { f := false ctrl.startWatches = []watchDescription{{ - src: source.NewKindWithCache(&corev1.Pod{}, &informertest.FakeInformers{Synced: &f}), + src: source.Kind(&informertest.FakeInformers{Synced: &f}, &corev1.Pod{}), }} ctrl.Name = "foo" ctx, cancel := context.WithCancel(context.Background()) @@ -102,11 +135,9 @@ var _ = Describe("controller", func() { err := ctrl.Start(ctx) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("failed to wait for foo caches to sync")) - - close(done) }) - It("should error when cache sync timeout occurs", func(done Done) { + It("should error when cache sync timeout occurs", func() { ctrl.CacheSyncTimeout = 10 * time.Nanosecond c, err := cache.New(cfg, cache.Options{}) @@ -114,18 +145,42 @@ var _ = Describe("controller", func() { c = &cacheWithIndefinitelyBlockingGetInformer{c} ctrl.startWatches = []watchDescription{{ - src: source.NewKindWithCache(&appsv1.Deployment{}, c), + src: source.Kind(c, &appsv1.Deployment{}), }} ctrl.Name = "testcontroller" err = ctrl.Start(context.TODO()) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("failed to wait for testcontroller caches to sync: timed out waiting for cache to be synced")) + }) + + It("should not error when context cancelled", func() { + ctrl.CacheSyncTimeout = 1 * time.Second + + sourceSynced := make(chan struct{}) + c, err := cache.New(cfg, cache.Options{}) + Expect(err).NotTo(HaveOccurred()) + c = &cacheWithIndefinitelyBlockingGetInformer{c} + ctrl.startWatches = []watchDescription{{ + src: &singnallingSourceWrapper{ + SyncingSource: source.Kind(c, &appsv1.Deployment{}), + cacheSyncDone: sourceSynced, + }, + }} + ctrl.Name = "testcontroller" - close(done) + ctx, cancel := context.WithCancel(context.TODO()) + go func() { + defer GinkgoRecover() + err = ctrl.Start(ctx) + Expect(err).To(Succeed()) + }() + + cancel() + <-sourceSynced }) - It("should not error when cache sync timeout is of sufficiently high", func(done Done) { + It("should not error when cache sync timeout is of sufficiently high", func() { ctrl.CacheSyncTimeout = 1 * time.Second ctx, cancel := context.WithCancel(context.Background()) @@ -136,7 +191,7 @@ var _ = Describe("controller", func() { Expect(err).NotTo(HaveOccurred()) ctrl.startWatches = []watchDescription{{ src: &singnallingSourceWrapper{ - SyncingSource: source.NewKindWithCache(&appsv1.Deployment{}, c), + SyncingSource: source.Kind(c, &appsv1.Deployment{}), cacheSyncDone: sourceSynced, }, }} @@ -152,13 +207,12 @@ var _ = Describe("controller", func() { }() <-sourceSynced - close(done) - }, 10.0) + }) - It("should process events from source.Channel", func(done Done) { + It("should process events from source.Channel", func() { // channel to be closed when event is processed processed := make(chan struct{}) - // source channel to be injected + // source channel ch := make(chan event.GenericEvent, 1) ctx, cancel := context.WithCancel(context.TODO()) @@ -174,7 +228,6 @@ var _ = Describe("controller", func() { ins := &source.Channel{Source: ch} ins.DestBufferSize = 1 - Expect(inject.StopChannelInto(ctx.Done(), ins)).To(BeTrue()) // send the event to the channel ch <- evt @@ -182,7 +235,7 @@ var _ = Describe("controller", func() { ctrl.startWatches = []watchDescription{{ src: ins, handler: handler.Funcs{ - GenericFunc: func(evt event.GenericEvent, q workqueue.RateLimitingInterface) { + GenericFunc: func(ctx context.Context, evt event.GenericEvent, q workqueue.RateLimitingInterface) { defer GinkgoRecover() close(processed) }, @@ -194,42 +247,20 @@ var _ = Describe("controller", func() { Expect(ctrl.Start(ctx)).To(Succeed()) }() <-processed - close(done) }) - It("should error when channel is passed as a source but stop channel is not injected", func(done Done) { - ch := make(chan event.GenericEvent) - ctx, cancel := context.WithCancel(context.TODO()) - defer cancel() - - ins := &source.Channel{Source: ch} - ctrl.startWatches = []watchDescription{{ - src: ins, - }} - - e := ctrl.Start(ctx) - - Expect(e).NotTo(BeNil()) - Expect(e.Error()).To(ContainSubstring("must call InjectStop on Channel before calling Start")) - close(done) - }) - - It("should error when channel source is not specified", func(done Done) { + It("should error when channel source is not specified", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() ins := &source.Channel{} - Expect(inject.StopChannelInto(make(<-chan struct{}), ins)).To(BeTrue()) - ctrl.startWatches = []watchDescription{{ - src: &source.Channel{}, + src: ins, }} e := ctrl.Start(ctx) Expect(e).NotTo(BeNil()) Expect(e.Error()).To(ContainSubstring("must specify Channel.Source")) - - close(done) }) It("should call Start on sources with the appropriate EventHandler, Queue, and Predicates", func() { @@ -282,128 +313,8 @@ var _ = Describe("controller", func() { }) - Describe("Watch", func() { - It("should inject dependencies into the Source", func() { - src := &source.Kind{Type: &corev1.Pod{}} - Expect(src.InjectCache(informers)).To(Succeed()) - evthdl := &handler.EnqueueRequestForObject{} - found := false - ctrl.SetFields = func(i interface{}) error { - defer GinkgoRecover() - if i == src { - found = true - } - return nil - } - Expect(ctrl.Watch(src, evthdl)).NotTo(HaveOccurred()) - Expect(found).To(BeTrue(), "Source not injected") - }) - - It("should return an error if there is an error injecting into the Source", func() { - src := &source.Kind{Type: &corev1.Pod{}} - Expect(src.InjectCache(informers)).To(Succeed()) - evthdl := &handler.EnqueueRequestForObject{} - expected := fmt.Errorf("expect fail source") - ctrl.SetFields = func(i interface{}) error { - defer GinkgoRecover() - if i == src { - return expected - } - return nil - } - Expect(ctrl.Watch(src, evthdl)).To(Equal(expected)) - }) - - It("should inject dependencies into the EventHandler", func() { - src := &source.Kind{Type: &corev1.Pod{}} - Expect(src.InjectCache(informers)).To(Succeed()) - evthdl := &handler.EnqueueRequestForObject{} - found := false - ctrl.SetFields = func(i interface{}) error { - defer GinkgoRecover() - if i == evthdl { - found = true - } - return nil - } - Expect(ctrl.Watch(src, evthdl)).NotTo(HaveOccurred()) - Expect(found).To(BeTrue(), "EventHandler not injected") - }) - - It("should return an error if there is an error injecting into the EventHandler", func() { - src := &source.Kind{Type: &corev1.Pod{}} - evthdl := &handler.EnqueueRequestForObject{} - expected := fmt.Errorf("expect fail eventhandler") - ctrl.SetFields = func(i interface{}) error { - defer GinkgoRecover() - if i == evthdl { - return expected - } - return nil - } - Expect(ctrl.Watch(src, evthdl)).To(Equal(expected)) - }) - - PIt("should inject dependencies into the Reconciler", func() { - // TODO(community): Write this - }) - - PIt("should return an error if there is an error injecting into the Reconciler", func() { - // TODO(community): Write this - }) - - It("should inject dependencies into all of the Predicates", func() { - src := &source.Kind{Type: &corev1.Pod{}} - Expect(src.InjectCache(informers)).To(Succeed()) - evthdl := &handler.EnqueueRequestForObject{} - pr1 := &predicate.Funcs{} - pr2 := &predicate.Funcs{} - found1 := false - found2 := false - ctrl.SetFields = func(i interface{}) error { - defer GinkgoRecover() - if i == pr1 { - found1 = true - } - if i == pr2 { - found2 = true - } - return nil - } - Expect(ctrl.Watch(src, evthdl, pr1, pr2)).NotTo(HaveOccurred()) - Expect(found1).To(BeTrue(), "First Predicated not injected") - Expect(found2).To(BeTrue(), "Second Predicated not injected") - }) - - It("should return an error if there is an error injecting into any of the Predicates", func() { - src := &source.Kind{Type: &corev1.Pod{}} - Expect(src.InjectCache(informers)).To(Succeed()) - evthdl := &handler.EnqueueRequestForObject{} - pr1 := &predicate.Funcs{} - pr2 := &predicate.Funcs{} - expected := fmt.Errorf("expect fail predicate") - ctrl.SetFields = func(i interface{}) error { - defer GinkgoRecover() - if i == pr1 { - return expected - } - return nil - } - Expect(ctrl.Watch(src, evthdl, pr1, pr2)).To(Equal(expected)) - - ctrl.SetFields = func(i interface{}) error { - defer GinkgoRecover() - if i == pr2 { - return expected - } - return nil - } - Expect(ctrl.Watch(src, evthdl, pr1, pr2)).To(Equal(expected)) - }) - }) - Describe("Processing queue items from a Controller", func() { - It("should call Reconciler if an item is enqueued", func(done Done) { + It("should call Reconciler if an item is enqueued", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() go func() { @@ -419,11 +330,9 @@ var _ = Describe("controller", func() { By("Removing the item from the queue") Eventually(queue.Len).Should(Equal(0)) Eventually(func() int { return queue.NumRequeues(request) }).Should(Equal(0)) - - close(done) }) - It("should continue to process additional queue items after the first", func(done Done) { + It("should continue to process additional queue items after the first", func() { ctrl.Do = reconcile.Func(func(context.Context, reconcile.Request) (reconcile.Result, error) { defer GinkgoRecover() Fail("Reconciler should not have been called") @@ -443,15 +352,13 @@ var _ = Describe("controller", func() { By("expecting both of them to be skipped") Eventually(queue.Len).Should(Equal(0)) Eventually(func() int { return queue.NumRequeues(request) }).Should(Equal(0)) - - close(done) }) PIt("should forget an item if it is not a Request and continue processing items", func() { // TODO(community): write this test }) - It("should requeue a Request if there is an error and continue processing items", func(done Done) { + It("should requeue a Request if there is an error and continue processing items", func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -472,10 +379,8 @@ var _ = Describe("controller", func() { By("Removing the item from the queue") Eventually(queue.Len).Should(Equal(0)) - Eventually(func() int { return queue.NumRequeues(request) }).Should(Equal(0)) - - close(done) - }, 1.0) + Eventually(func() int { return queue.NumRequeues(request) }, 1.0).Should(Equal(0)) + }) // TODO(directxman12): we should ensure that backoff occurrs with error requeue @@ -624,7 +529,7 @@ var _ = Describe("controller", func() { reconcileTotal.Reset() }) - It("should get updated on successful reconciliation", func(done Done) { + It("should get updated on successful reconciliation", func() { Expect(func() error { Expect(ctrlmetrics.ReconcileTotal.WithLabelValues(ctrl.Name, "success").Write(&reconcileTotal)).To(Succeed()) if reconcileTotal.GetCounter().GetValue() != 0.0 { @@ -651,11 +556,9 @@ var _ = Describe("controller", func() { } return nil }, 2.0).Should(Succeed()) + }) - close(done) - }, 2.0) - - It("should get updated on reconcile errors", func(done Done) { + It("should get updated on reconcile errors", func() { Expect(func() error { Expect(ctrlmetrics.ReconcileTotal.WithLabelValues(ctrl.Name, "error").Write(&reconcileTotal)).To(Succeed()) if reconcileTotal.GetCounter().GetValue() != 0.0 { @@ -682,11 +585,9 @@ var _ = Describe("controller", func() { } return nil }, 2.0).Should(Succeed()) + }) - close(done) - }, 2.0) - - It("should get updated when reconcile returns with retry enabled", func(done Done) { + It("should get updated when reconcile returns with retry enabled", func() { Expect(func() error { Expect(ctrlmetrics.ReconcileTotal.WithLabelValues(ctrl.Name, "retry").Write(&reconcileTotal)).To(Succeed()) if reconcileTotal.GetCounter().GetValue() != 0.0 { @@ -714,11 +615,9 @@ var _ = Describe("controller", func() { } return nil }, 2.0).Should(Succeed()) + }) - close(done) - }, 2.0) - - It("should get updated when reconcile returns with retryAfter enabled", func(done Done) { + It("should get updated when reconcile returns with retryAfter enabled", func() { Expect(func() error { Expect(ctrlmetrics.ReconcileTotal.WithLabelValues(ctrl.Name, "retry_after").Write(&reconcileTotal)).To(Succeed()) if reconcileTotal.GetCounter().GetValue() != 0.0 { @@ -745,13 +644,11 @@ var _ = Describe("controller", func() { } return nil }, 2.0).Should(Succeed()) - - close(done) - }, 2.0) + }) }) Context("should update prometheus metrics", func() { - It("should requeue a Request if there is an error and continue processing items", func(done Done) { + It("should requeue a Request if there is an error and continue processing items", func() { var reconcileErrs dto.Metric ctrlmetrics.ReconcileErrors.Reset() Expect(func() error { @@ -788,11 +685,9 @@ var _ = Describe("controller", func() { By("Removing the item from the queue") Eventually(queue.Len).Should(Equal(0)) Eventually(func() int { return queue.NumRequeues(request) }).Should(Equal(0)) + }) - close(done) - }, 2.0) - - It("should add a reconcile time to the reconcile time histogram", func(done Done) { + It("should add a reconcile time to the reconcile time histogram", func() { var reconcileTime dto.Metric ctrlmetrics.ReconcileTime.Reset() @@ -831,13 +726,28 @@ var _ = Describe("controller", func() { } return nil }, 2.0).Should(Succeed()) - - close(done) - }, 4.0) + }) }) }) }) +var _ = Describe("ReconcileIDFromContext function", func() { + It("should return an empty string if there is nothing in the context", func() { + ctx := context.Background() + reconcileID := ReconcileIDFromContext(ctx) + + Expect(reconcileID).To(Equal(types.UID(""))) + }) + + It("should return the correct reconcileID from context", func() { + const expectedReconcileID = types.UID("uuid") + ctx := addReconcileID(context.Background(), expectedReconcileID) + reconcileID := ReconcileIDFromContext(ctx) + + Expect(reconcileID).To(Equal(expectedReconcileID)) + }) +}) + type DelegatingQueue struct { workqueue.RateLimitingInterface mu sync.Mutex diff --git a/pkg/internal/field/selector/utils.go b/pkg/internal/field/selector/utils.go new file mode 100644 index 0000000000..4f6d084318 --- /dev/null +++ b/pkg/internal/field/selector/utils.go @@ -0,0 +1,35 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package selector + +import ( + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/selection" +) + +// RequiresExactMatch checks if the given field selector is of the form `k=v` or `k==v`. +func RequiresExactMatch(sel fields.Selector) (field, val string, required bool) { + reqs := sel.Requirements() + if len(reqs) != 1 { + return "", "", false + } + req := reqs[0] + if req.Operator != selection.Equals && req.Operator != selection.DoubleEquals { + return "", "", false + } + return req.Field, req.Value, true +} diff --git a/pkg/patterns/operator/doc.go b/pkg/internal/field/selector/utils_suite_test.go similarity index 67% rename from pkg/patterns/operator/doc.go rename to pkg/internal/field/selector/utils_suite_test.go index 5ccd0791af..dd42f1d1ac 100644 --- a/pkg/patterns/operator/doc.go +++ b/pkg/internal/field/selector/utils_suite_test.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2022 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,10 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. */ -/* -Package operator serves to redirect users to the application package. +package selector_test -Operators are the common name for Kubernetes APIs which manage specific applications. e.g. Spark Operator, -Etcd Operator. -*/ -package operator +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestSource(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Fields Selector Utils Suite") +} diff --git a/pkg/internal/field/selector/utils_test.go b/pkg/internal/field/selector/utils_test.go new file mode 100644 index 0000000000..fba214ff16 --- /dev/null +++ b/pkg/internal/field/selector/utils_test.go @@ -0,0 +1,88 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package selector_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/fields" + + . "sigs.k8s.io/controller-runtime/pkg/internal/field/selector" +) + +var _ = Describe("RequiresExactMatch function", func() { + + It("Returns false when the selector matches everything", func() { + _, _, requiresExactMatch := RequiresExactMatch(fields.Everything()) + Expect(requiresExactMatch).To(BeFalse()) + }) + + It("Returns false when the selector matches nothing", func() { + _, _, requiresExactMatch := RequiresExactMatch(fields.Nothing()) + Expect(requiresExactMatch).To(BeFalse()) + }) + + It("Returns false when the selector has the form key!=val", func() { + _, _, requiresExactMatch := RequiresExactMatch(fields.ParseSelectorOrDie("key!=val")) + Expect(requiresExactMatch).To(BeFalse()) + }) + + It("Returns false when the selector has the form key1==val1,key2==val2", func() { + _, _, requiresExactMatch := RequiresExactMatch(fields.ParseSelectorOrDie("key1==val1,key2==val2")) + Expect(requiresExactMatch).To(BeFalse()) + }) + + It("Returns true when the selector has the form key==val", func() { + _, _, requiresExactMatch := RequiresExactMatch(fields.ParseSelectorOrDie("key==val")) + Expect(requiresExactMatch).To(BeTrue()) + }) + + It("Returns true when the selector has the form key=val", func() { + _, _, requiresExactMatch := RequiresExactMatch(fields.ParseSelectorOrDie("key=val")) + Expect(requiresExactMatch).To(BeTrue()) + }) + + It("Returns empty key and value when the selector matches everything", func() { + key, val, _ := RequiresExactMatch(fields.Everything()) + Expect(key).To(Equal("")) + Expect(val).To(Equal("")) + }) + + It("Returns empty key and value when the selector matches nothing", func() { + key, val, _ := RequiresExactMatch(fields.Nothing()) + Expect(key).To(Equal("")) + Expect(val).To(Equal("")) + }) + + It("Returns empty key and value when the selector has the form key!=val", func() { + key, val, _ := RequiresExactMatch(fields.ParseSelectorOrDie("key!=val")) + Expect(key).To(Equal("")) + Expect(val).To(Equal("")) + }) + + It("Returns key and value when the selector has the form key==val", func() { + key, val, _ := RequiresExactMatch(fields.ParseSelectorOrDie("key==val")) + Expect(key).To(Equal("key")) + Expect(val).To(Equal("val")) + }) + + It("Returns key and value when the selector has the form key=val", func() { + key, val, _ := RequiresExactMatch(fields.ParseSelectorOrDie("key=val")) + Expect(key).To(Equal("key")) + Expect(val).To(Equal("val")) + }) +}) diff --git a/pkg/runtime/doc.go b/pkg/internal/flock/doc.go similarity index 66% rename from pkg/runtime/doc.go rename to pkg/internal/flock/doc.go index 34101b3fa4..11e39823ed 100644 --- a/pkg/runtime/doc.go +++ b/pkg/internal/flock/doc.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,8 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package runtime contains not-quite-internal mechanisms for -// controller-runtime, plus some deprecated exports of functionality -// moved elsewhere. Most users should not need to import anything in -// pkg/runtime. -package runtime +// Package flock is copied from k8s.io/kubernetes/pkg/util/flock to avoid +// importing k8s.io/kubernetes as a dependency. +// +// Provides file locking functionalities on unix systems. +package flock diff --git a/pkg/runtime/inject/doc.go b/pkg/internal/flock/errors.go similarity index 61% rename from pkg/runtime/inject/doc.go rename to pkg/internal/flock/errors.go index 17c60895f0..ee7a434372 100644 --- a/pkg/runtime/inject/doc.go +++ b/pkg/internal/flock/errors.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Kubernetes Authors. +Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,9 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -/* -Package inject defines interfaces and functions for propagating dependencies from a ControllerManager to -the components registered with it. Dependencies are propagated to Reconciler, Source, EventHandler and Predicate -objects which implement the Injectable interfaces. -*/ -package inject +package flock + +import "errors" + +var ( + // ErrAlreadyLocked is returned when the file is already locked. + ErrAlreadyLocked = errors.New("the file is already locked") +) diff --git a/pkg/internal/flock/flock_other.go b/pkg/internal/flock/flock_other.go new file mode 100644 index 0000000000..069a5b3a2c --- /dev/null +++ b/pkg/internal/flock/flock_other.go @@ -0,0 +1,24 @@ +// +build !linux,!darwin,!freebsd,!openbsd,!netbsd,!dragonfly + +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flock + +// Acquire is not implemented on non-unix systems. +func Acquire(path string) error { + return nil +} diff --git a/pkg/internal/flock/flock_unix.go b/pkg/internal/flock/flock_unix.go new file mode 100644 index 0000000000..71ec576df2 --- /dev/null +++ b/pkg/internal/flock/flock_unix.go @@ -0,0 +1,48 @@ +//go:build linux || darwin || freebsd || openbsd || netbsd || dragonfly +// +build linux darwin freebsd openbsd netbsd dragonfly + +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flock + +import ( + "errors" + "fmt" + "os" + + "golang.org/x/sys/unix" +) + +// Acquire acquires a lock on a file for the duration of the process. This method +// is reentrant. +func Acquire(path string) error { + fd, err := unix.Open(path, unix.O_CREAT|unix.O_RDWR|unix.O_CLOEXEC, 0600) + if err != nil { + if errors.Is(err, os.ErrExist) { + return fmt.Errorf("cannot lock file %q: %w", path, ErrAlreadyLocked) + } + return err + } + + // We don't need to close the fd since we should hold + // it until the process exits. + err = unix.Flock(fd, unix.LOCK_NB|unix.LOCK_EX) + if errors.Is(err, unix.EWOULDBLOCK) { // This condition requires LOCK_NB. + return fmt.Errorf("cannot lock file %q: %w", path, ErrAlreadyLocked) + } + return err +} diff --git a/pkg/internal/httpserver/server.go b/pkg/internal/httpserver/server.go new file mode 100644 index 0000000000..b5f91f18e0 --- /dev/null +++ b/pkg/internal/httpserver/server.go @@ -0,0 +1,16 @@ +package httpserver + +import ( + "net/http" + "time" +) + +// New returns a new server with sane defaults. +func New(handler http.Handler) *http.Server { + return &http.Server{ + Handler: handler, + MaxHeaderBytes: 1 << 20, + IdleTimeout: 90 * time.Second, // matches http.DefaultTransport keep-alive timeout + ReadHeaderTimeout: 32 * time.Second, + } +} diff --git a/pkg/internal/objectutil/objectutil.go b/pkg/internal/objectutil/objectutil.go index 7057f3dbe4..0189c04323 100644 --- a/pkg/internal/objectutil/objectutil.go +++ b/pkg/internal/objectutil/objectutil.go @@ -17,14 +17,9 @@ limitations under the License. package objectutil import ( - "errors" - "fmt" - apimeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" ) // FilterWithLabels returns a copy of the items in objs matching labelSel. @@ -45,34 +40,3 @@ func FilterWithLabels(objs []runtime.Object, labelSel labels.Selector) ([]runtim } return outItems, nil } - -// IsAPINamespaced returns true if the object is namespace scoped. -// For unstructured objects the gvk is found from the object itself. -func IsAPINamespaced(obj runtime.Object, scheme *runtime.Scheme, restmapper apimeta.RESTMapper) (bool, error) { - gvk, err := apiutil.GVKForObject(obj, scheme) - if err != nil { - return false, err - } - - return IsAPINamespacedWithGVK(gvk, scheme, restmapper) -} - -// IsAPINamespacedWithGVK returns true if the object having the provided -// GVK is namespace scoped. -func IsAPINamespacedWithGVK(gk schema.GroupVersionKind, scheme *runtime.Scheme, restmapper apimeta.RESTMapper) (bool, error) { - restmapping, err := restmapper.RESTMapping(schema.GroupKind{Group: gk.Group, Kind: gk.Kind}) - if err != nil { - return false, fmt.Errorf("failed to get restmapping: %w", err) - } - - scope := restmapping.Scope.Name() - - if scope == "" { - return false, errors.New("scope cannot be identified, empty scope returned") - } - - if scope != apimeta.RESTScopeNameRoot { - return true, nil - } - return false, nil -} diff --git a/pkg/internal/recorder/recorder.go b/pkg/internal/recorder/recorder.go index cb8b7b6d63..21f0146ba3 100644 --- a/pkg/internal/recorder/recorder.go +++ b/pkg/internal/recorder/recorder.go @@ -19,13 +19,13 @@ package recorder import ( "context" "fmt" + "net/http" "sync" "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/kubernetes" - typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" ) @@ -45,7 +45,7 @@ type Provider struct { scheme *runtime.Scheme // logger is the logger to use when logging diagnostic event info logger logr.Logger - evtClient typedcorev1.EventInterface + evtClient corev1client.EventInterface makeBroadcaster EventBroadcasterProducer broadcasterOnce sync.Once @@ -98,10 +98,10 @@ func (p *Provider) getBroadcaster() record.EventBroadcaster { p.broadcasterOnce.Do(func() { broadcaster, stop := p.makeBroadcaster() - broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: p.evtClient}) + broadcaster.StartRecordingToSink(&corev1client.EventSinkImpl{Interface: p.evtClient}) broadcaster.StartEventWatcher( func(e *corev1.Event) { - p.logger.V(1).Info(e.Type, "object", e.InvolvedObject, "reason", e.Reason, "message", e.Message) + p.logger.V(1).Info(e.Message, "type", e.Type, "object", e.InvolvedObject, "reason", e.Reason) }) p.broadcaster = broadcaster p.stopBroadcaster = stop @@ -111,13 +111,17 @@ func (p *Provider) getBroadcaster() record.EventBroadcaster { } // NewProvider create a new Provider instance. -func NewProvider(config *rest.Config, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster EventBroadcasterProducer) (*Provider, error) { - clientSet, err := kubernetes.NewForConfig(config) +func NewProvider(config *rest.Config, httpClient *http.Client, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster EventBroadcasterProducer) (*Provider, error) { + if httpClient == nil { + panic("httpClient must not be nil") + } + + corev1Client, err := corev1client.NewForConfigAndClient(config, httpClient) if err != nil { - return nil, fmt.Errorf("failed to init clientSet: %w", err) + return nil, fmt.Errorf("failed to init client: %w", err) } - p := &Provider{scheme: scheme, logger: logger, makeBroadcaster: makeBroadcaster, evtClient: clientSet.CoreV1().Events("")} + p := &Provider{scheme: scheme, logger: logger, makeBroadcaster: makeBroadcaster, evtClient: corev1Client.Events("")} return p, nil } diff --git a/pkg/internal/recorder/recorder_integration_test.go b/pkg/internal/recorder/recorder_integration_test.go index a67d0e1ed5..130a306053 100644 --- a/pkg/internal/recorder/recorder_integration_test.go +++ b/pkg/internal/recorder/recorder_integration_test.go @@ -31,13 +31,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) var _ = Describe("recorder", func() { Describe("recorder", func() { - It("should publish events", func(done Done) { + It("should publish events", func() { By("Creating the Manager") cm, err := manager.New(cfg, manager.Options{}) Expect(err).NotTo(HaveOccurred()) @@ -56,7 +56,7 @@ var _ = Describe("recorder", func() { Expect(err).NotTo(HaveOccurred()) By("Watching Resources") - err = instance.Watch(&source.Kind{Type: &appsv1.Deployment{}}, &handler.EnqueueRequestForObject{}) + err = instance.Watch(source.Kind(cm.GetCache(), &appsv1.Deployment{}), &handler.EnqueueRequestForObject{}) Expect(err).NotTo(HaveOccurred()) By("Starting the Manager") @@ -108,8 +108,6 @@ var _ = Describe("recorder", func() { Expect(evt.Type).To(Equal(corev1.EventTypeNormal)) Expect(evt.Reason).To(Equal("test-reason")) Expect(evt.Message).To(Equal("test-msg")) - - close(done) }) }) }) diff --git a/pkg/internal/recorder/recorder_suite_test.go b/pkg/internal/recorder/recorder_suite_test.go index ed4a5c4140..e5b5836d58 100644 --- a/pkg/internal/recorder/recorder_suite_test.go +++ b/pkg/internal/recorder/recorder_suite_test.go @@ -17,29 +17,29 @@ limitations under the License. package recorder_test import ( + "net/http" "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestRecorder(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Recorder Integration Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Recorder Integration Suite") } var testenv *envtest.Environment var cfg *rest.Config +var httpClient *http.Client var clientset *kubernetes.Clientset -var _ = BeforeSuite(func(done Done) { +var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) testenv = &envtest.Environment{} @@ -48,11 +48,12 @@ var _ = BeforeSuite(func(done Done) { cfg, err = testenv.Start() Expect(err).NotTo(HaveOccurred()) + httpClient, err = rest.HTTPClientFor(cfg) + Expect(err).ToNot(HaveOccurred()) + clientset, err = kubernetes.NewForConfig(cfg) Expect(err).NotTo(HaveOccurred()) - - close(done) -}, 60) +}) var _ = AfterSuite(func() { Expect(testenv.Stop()).To(Succeed()) diff --git a/pkg/internal/recorder/recorder_test.go b/pkg/internal/recorder/recorder_test.go index ea7aaf8d43..804bdb3d21 100644 --- a/pkg/internal/recorder/recorder_test.go +++ b/pkg/internal/recorder/recorder_test.go @@ -18,7 +18,7 @@ package recorder_test import ( "github.com/go-logr/logr" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" @@ -29,23 +29,23 @@ var _ = Describe("recorder.Provider", func() { makeBroadcaster := func() (record.EventBroadcaster, bool) { return record.NewBroadcaster(), true } Describe("NewProvider", func() { It("should return a provider instance and a nil error.", func() { - provider, err := recorder.NewProvider(cfg, scheme.Scheme, logr.DiscardLogger{}, makeBroadcaster) + provider, err := recorder.NewProvider(cfg, httpClient, scheme.Scheme, logr.Discard(), makeBroadcaster) Expect(provider).NotTo(BeNil()) Expect(err).NotTo(HaveOccurred()) }) - It("should return an error if failed to init clientSet.", func() { + It("should return an error if failed to init client.", func() { // Invalid the config cfg1 := *cfg cfg1.Host = "invalid host" - _, err := recorder.NewProvider(&cfg1, scheme.Scheme, logr.DiscardLogger{}, makeBroadcaster) + _, err := recorder.NewProvider(&cfg1, httpClient, scheme.Scheme, logr.Discard(), makeBroadcaster) Expect(err).NotTo(BeNil()) - Expect(err.Error()).To(ContainSubstring("failed to init clientSet")) + Expect(err.Error()).To(ContainSubstring("failed to init client")) }) }) Describe("GetEventRecorder", func() { It("should return a recorder instance.", func() { - provider, err := recorder.NewProvider(cfg, scheme.Scheme, logr.DiscardLogger{}, makeBroadcaster) + provider, err := recorder.NewProvider(cfg, httpClient, scheme.Scheme, logr.Discard(), makeBroadcaster) Expect(err).NotTo(HaveOccurred()) recorder := provider.GetEventRecorderFor("test") diff --git a/pkg/source/internal/eventsource.go b/pkg/internal/source/event_handler.go similarity index 72% rename from pkg/source/internal/eventsource.go rename to pkg/internal/source/event_handler.go index f0cfe212ed..8449a9dc75 100644 --- a/pkg/source/internal/eventsource.go +++ b/pkg/internal/source/event_handler.go @@ -17,6 +17,7 @@ limitations under the License. package internal import ( + "context" "fmt" "k8s.io/client-go/tools/cache" @@ -31,17 +32,31 @@ import ( var log = logf.RuntimeLog.WithName("source").WithName("EventHandler") -var _ cache.ResourceEventHandler = EventHandler{} +var _ cache.ResourceEventHandler = &EventHandler{} + +// NewEventHandler creates a new EventHandler. +func NewEventHandler(ctx context.Context, queue workqueue.RateLimitingInterface, handler handler.EventHandler, predicates []predicate.Predicate) *EventHandler { + return &EventHandler{ + ctx: ctx, + handler: handler, + queue: queue, + predicates: predicates, + } +} // EventHandler adapts a handler.EventHandler interface to a cache.ResourceEventHandler interface. type EventHandler struct { - EventHandler handler.EventHandler - Queue workqueue.RateLimitingInterface - Predicates []predicate.Predicate + // ctx stores the context that created the event handler + // that is used to propagate cancellation signals to each handler function. + ctx context.Context + + handler handler.EventHandler + queue workqueue.RateLimitingInterface + predicates []predicate.Predicate } // OnAdd creates CreateEvent and calls Create on EventHandler. -func (e EventHandler) OnAdd(obj interface{}) { +func (e *EventHandler) OnAdd(obj interface{}) { c := event.CreateEvent{} // Pull Object out of the object @@ -53,18 +68,20 @@ func (e EventHandler) OnAdd(obj interface{}) { return } - for _, p := range e.Predicates { + for _, p := range e.predicates { if !p.Create(c) { return } } // Invoke create handler - e.EventHandler.Create(c, e.Queue) + ctx, cancel := context.WithCancel(e.ctx) + defer cancel() + e.handler.Create(ctx, c, e.queue) } // OnUpdate creates UpdateEvent and calls Update on EventHandler. -func (e EventHandler) OnUpdate(oldObj, newObj interface{}) { +func (e *EventHandler) OnUpdate(oldObj, newObj interface{}) { u := event.UpdateEvent{} if o, ok := oldObj.(client.Object); ok { @@ -84,18 +101,20 @@ func (e EventHandler) OnUpdate(oldObj, newObj interface{}) { return } - for _, p := range e.Predicates { + for _, p := range e.predicates { if !p.Update(u) { return } } // Invoke update handler - e.EventHandler.Update(u, e.Queue) + ctx, cancel := context.WithCancel(e.ctx) + defer cancel() + e.handler.Update(ctx, u, e.queue) } // OnDelete creates DeleteEvent and calls Delete on EventHandler. -func (e EventHandler) OnDelete(obj interface{}) { +func (e *EventHandler) OnDelete(obj interface{}) { d := event.DeleteEvent{} // Deal with tombstone events by pulling the object out. Tombstone events wrap the object in a @@ -127,12 +146,14 @@ func (e EventHandler) OnDelete(obj interface{}) { return } - for _, p := range e.Predicates { + for _, p := range e.predicates { if !p.Delete(d) { return } } // Invoke delete handler - e.EventHandler.Delete(d, e.Queue) + ctx, cancel := context.WithCancel(e.ctx) + defer cancel() + e.handler.Delete(ctx, d, e.queue) } diff --git a/pkg/source/internal/internal_suite_test.go b/pkg/internal/source/internal_suite_test.go similarity index 78% rename from pkg/source/internal/internal_suite_test.go rename to pkg/internal/source/internal_suite_test.go index 21dd5ee6b4..eeee8b22cd 100644 --- a/pkg/source/internal/internal_suite_test.go +++ b/pkg/internal/source/internal_suite_test.go @@ -19,17 +19,15 @@ package internal_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestInternal(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Source Internal Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Source Internal Suite") } var _ = BeforeSuite(func() { diff --git a/pkg/source/internal/internal_test.go b/pkg/internal/source/internal_test.go similarity index 64% rename from pkg/source/internal/internal_test.go rename to pkg/internal/source/internal_test.go index 7b9ab7a223..9203879ac8 100644 --- a/pkg/source/internal/internal_test.go +++ b/pkg/internal/source/internal_test.go @@ -17,13 +17,15 @@ limitations under the License. package internal_test import ( - . "github.com/onsi/ginkgo" + "context" + + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/source/internal" + internal "sigs.k8s.io/controller-runtime/pkg/internal/source" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,48 +36,45 @@ import ( ) var _ = Describe("Internal", func() { - - var instance internal.EventHandler + var ctx = context.Background() + var instance *internal.EventHandler var funcs, setfuncs *handler.Funcs var set bool BeforeEach(func() { funcs = &handler.Funcs{ - CreateFunc: func(event.CreateEvent, workqueue.RateLimitingInterface) { + CreateFunc: func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Did not expect CreateEvent to be called.") }, - DeleteFunc: func(e event.DeleteEvent, q workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Did not expect DeleteEvent to be called.") }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Did not expect UpdateEvent to be called.") }, - GenericFunc: func(event.GenericEvent, workqueue.RateLimitingInterface) { + GenericFunc: func(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Did not expect GenericEvent to be called.") }, } setfuncs = &handler.Funcs{ - CreateFunc: func(event.CreateEvent, workqueue.RateLimitingInterface) { + CreateFunc: func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) { set = true }, - DeleteFunc: func(e event.DeleteEvent, q workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { set = true }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { set = true }, - GenericFunc: func(event.GenericEvent, workqueue.RateLimitingInterface) { + GenericFunc: func(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) { set = true }, } - instance = internal.EventHandler{ - Queue: controllertest.Queue{}, - EventHandler: funcs, - } + instance = internal.NewEventHandler(ctx, controllertest.Queue{}, funcs, nil) }) Describe("EventHandler", func() { @@ -91,237 +90,198 @@ var _ = Describe("Internal", func() { newPod.Labels = map[string]string{"foo": "bar"} }) - It("should create a CreateEvent", func(done Done) { - funcs.CreateFunc = func(evt event.CreateEvent, q workqueue.RateLimitingInterface) { + It("should create a CreateEvent", func() { + funcs.CreateFunc = func(ctx context.Context, evt event.CreateEvent, q workqueue.RateLimitingInterface) { defer GinkgoRecover() - Expect(q).To(Equal(instance.Queue)) Expect(evt.Object).To(Equal(pod)) } instance.OnAdd(pod) - close(done) }) - It("should used Predicates to filter CreateEvents", func(done Done) { - instance = internal.EventHandler{ - Queue: controllertest.Queue{}, - EventHandler: setfuncs, - } - - set = false - instance.Predicates = []predicate.Predicate{ + It("should used Predicates to filter CreateEvents", func() { + instance = internal.NewEventHandler(ctx, controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return false }}, - } + }) + set = false instance.OnAdd(pod) Expect(set).To(BeFalse()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return true }}, - } + }) instance.OnAdd(pod) Expect(set).To(BeTrue()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return true }}, predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return false }}, - } + }) instance.OnAdd(pod) Expect(set).To(BeFalse()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return false }}, predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return true }}, - } + }) instance.OnAdd(pod) Expect(set).To(BeFalse()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return true }}, predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return true }}, - } + }) instance.OnAdd(pod) Expect(set).To(BeTrue()) - - close(done) }) - It("should not call Create EventHandler if the object is not a runtime.Object", func(done Done) { + It("should not call Create EventHandler if the object is not a runtime.Object", func() { instance.OnAdd(&metav1.ObjectMeta{}) - close(done) }) - It("should not call Create EventHandler if the object does not have metadata", func(done Done) { + It("should not call Create EventHandler if the object does not have metadata", func() { instance.OnAdd(FooRuntimeObject{}) - close(done) }) - It("should create an UpdateEvent", func(done Done) { - funcs.UpdateFunc = func(evt event.UpdateEvent, q workqueue.RateLimitingInterface) { + It("should create an UpdateEvent", func() { + funcs.UpdateFunc = func(ctx context.Context, evt event.UpdateEvent, q workqueue.RateLimitingInterface) { defer GinkgoRecover() - Expect(q).To(Equal(instance.Queue)) - Expect(evt.ObjectOld).To(Equal(pod)) Expect(evt.ObjectNew).To(Equal(newPod)) } instance.OnUpdate(pod, newPod) - close(done) }) - It("should used Predicates to filter UpdateEvents", func(done Done) { - instance = internal.EventHandler{ - Queue: controllertest.Queue{}, - EventHandler: setfuncs, - } - + It("should used Predicates to filter UpdateEvents", func() { set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{UpdateFunc: func(updateEvent event.UpdateEvent) bool { return false }}, - } + }) instance.OnUpdate(pod, newPod) Expect(set).To(BeFalse()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{UpdateFunc: func(event.UpdateEvent) bool { return true }}, - } + }) instance.OnUpdate(pod, newPod) Expect(set).To(BeTrue()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{UpdateFunc: func(event.UpdateEvent) bool { return true }}, predicate.Funcs{UpdateFunc: func(event.UpdateEvent) bool { return false }}, - } + }) instance.OnUpdate(pod, newPod) Expect(set).To(BeFalse()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{UpdateFunc: func(event.UpdateEvent) bool { return false }}, predicate.Funcs{UpdateFunc: func(event.UpdateEvent) bool { return true }}, - } + }) instance.OnUpdate(pod, newPod) Expect(set).To(BeFalse()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return true }}, predicate.Funcs{CreateFunc: func(event.CreateEvent) bool { return true }}, - } + }) instance.OnUpdate(pod, newPod) Expect(set).To(BeTrue()) - - close(done) }) - It("should not call Update EventHandler if the object is not a runtime.Object", func(done Done) { + It("should not call Update EventHandler if the object is not a runtime.Object", func() { instance.OnUpdate(&metav1.ObjectMeta{}, &corev1.Pod{}) instance.OnUpdate(&corev1.Pod{}, &metav1.ObjectMeta{}) - close(done) }) - It("should not call Update EventHandler if the object does not have metadata", func(done Done) { + It("should not call Update EventHandler if the object does not have metadata", func() { instance.OnUpdate(FooRuntimeObject{}, &corev1.Pod{}) instance.OnUpdate(&corev1.Pod{}, FooRuntimeObject{}) - close(done) }) - It("should create a DeleteEvent", func(done Done) { - funcs.DeleteFunc = func(evt event.DeleteEvent, q workqueue.RateLimitingInterface) { + It("should create a DeleteEvent", func() { + funcs.DeleteFunc = func(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) { defer GinkgoRecover() - Expect(q).To(Equal(instance.Queue)) - Expect(evt.Object).To(Equal(pod)) } instance.OnDelete(pod) - close(done) }) - It("should used Predicates to filter DeleteEvents", func(done Done) { - instance = internal.EventHandler{ - Queue: controllertest.Queue{}, - EventHandler: setfuncs, - } - + It("should used Predicates to filter DeleteEvents", func() { set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{DeleteFunc: func(event.DeleteEvent) bool { return false }}, - } + }) instance.OnDelete(pod) Expect(set).To(BeFalse()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{DeleteFunc: func(event.DeleteEvent) bool { return true }}, - } + }) instance.OnDelete(pod) Expect(set).To(BeTrue()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{DeleteFunc: func(event.DeleteEvent) bool { return true }}, predicate.Funcs{DeleteFunc: func(event.DeleteEvent) bool { return false }}, - } + }) instance.OnDelete(pod) Expect(set).To(BeFalse()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{DeleteFunc: func(event.DeleteEvent) bool { return false }}, predicate.Funcs{DeleteFunc: func(event.DeleteEvent) bool { return true }}, - } + }) instance.OnDelete(pod) Expect(set).To(BeFalse()) set = false - instance.Predicates = []predicate.Predicate{ + instance = internal.NewEventHandler(ctx, controllertest.Queue{}, setfuncs, []predicate.Predicate{ predicate.Funcs{DeleteFunc: func(event.DeleteEvent) bool { return true }}, predicate.Funcs{DeleteFunc: func(event.DeleteEvent) bool { return true }}, - } + }) instance.OnDelete(pod) Expect(set).To(BeTrue()) - - close(done) }) - It("should not call Delete EventHandler if the object is not a runtime.Object", func(done Done) { + It("should not call Delete EventHandler if the object is not a runtime.Object", func() { instance.OnDelete(&metav1.ObjectMeta{}) - close(done) }) - It("should not call Delete EventHandler if the object does not have metadata", func(done Done) { + It("should not call Delete EventHandler if the object does not have metadata", func() { instance.OnDelete(FooRuntimeObject{}) - close(done) }) - It("should create a DeleteEvent from a tombstone", func(done Done) { + It("should create a DeleteEvent from a tombstone", func() { tombstone := cache.DeletedFinalStateUnknown{ Obj: pod, } - funcs.DeleteFunc = func(evt event.DeleteEvent, q workqueue.RateLimitingInterface) { + funcs.DeleteFunc = func(ctx context.Context, evt event.DeleteEvent, q workqueue.RateLimitingInterface) { defer GinkgoRecover() - Expect(q).To(Equal(instance.Queue)) Expect(evt.Object).To(Equal(pod)) } instance.OnDelete(tombstone) - close(done) }) - It("should ignore tombstone objects without meta", func(done Done) { + It("should ignore tombstone objects without meta", func() { tombstone := cache.DeletedFinalStateUnknown{Obj: Foo{}} instance.OnDelete(tombstone) - close(done) }) - It("should ignore objects without meta", func(done Done) { + It("should ignore objects without meta", func() { instance.OnAdd(Foo{}) instance.OnUpdate(Foo{}, Foo{}) instance.OnDelete(Foo{}) - close(done) }) }) }) diff --git a/pkg/internal/source/kind.go b/pkg/internal/source/kind.go new file mode 100644 index 0000000000..2e765acce8 --- /dev/null +++ b/pkg/internal/source/kind.go @@ -0,0 +1,117 @@ +package internal + +import ( + "context" + "errors" + "fmt" + "time" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +// Kind is used to provide a source of events originating inside the cluster from Watches (e.g. Pod Create). +type Kind struct { + // Type is the type of object to watch. e.g. &v1.Pod{} + Type client.Object + + // Cache used to watch APIs + Cache cache.Cache + + // started may contain an error if one was encountered during startup. If its closed and does not + // contain an error, startup and syncing finished. + started chan error + startCancel func() +} + +// Start is internal and should be called only by the Controller to register an EventHandler with the Informer +// to enqueue reconcile.Requests. +func (ks *Kind) Start(ctx context.Context, handler handler.EventHandler, queue workqueue.RateLimitingInterface, + prct ...predicate.Predicate) error { + if ks.Type == nil { + return fmt.Errorf("must create Kind with a non-nil object") + } + if ks.Cache == nil { + return fmt.Errorf("must create Kind with a non-nil cache") + } + + // cache.GetInformer will block until its context is cancelled if the cache was already started and it can not + // sync that informer (most commonly due to RBAC issues). + ctx, ks.startCancel = context.WithCancel(ctx) + ks.started = make(chan error) + go func() { + var ( + i cache.Informer + lastErr error + ) + + // Tries to get an informer until it returns true, + // an error or the specified context is cancelled or expired. + if err := wait.PollImmediateUntilWithContext(ctx, 10*time.Second, func(ctx context.Context) (bool, error) { + // Lookup the Informer from the Cache and add an EventHandler which populates the Queue + i, lastErr = ks.Cache.GetInformer(ctx, ks.Type) + if lastErr != nil { + kindMatchErr := &meta.NoKindMatchError{} + switch { + case errors.As(lastErr, &kindMatchErr): + log.Error(lastErr, "if kind is a CRD, it should be installed before calling Start", + "kind", kindMatchErr.GroupKind) + case runtime.IsNotRegisteredError(lastErr): + log.Error(lastErr, "kind must be registered to the Scheme") + default: + log.Error(lastErr, "failed to get informer from cache") + } + return false, nil // Retry. + } + return true, nil + }); err != nil { + if lastErr != nil { + ks.started <- fmt.Errorf("failed to get informer from cache: %w", lastErr) + return + } + ks.started <- err + return + } + + _, err := i.AddEventHandler(NewEventHandler(ctx, queue, handler, prct)) + if err != nil { + ks.started <- err + return + } + if !ks.Cache.WaitForCacheSync(ctx) { + // Would be great to return something more informative here + ks.started <- errors.New("cache did not sync") + } + close(ks.started) + }() + + return nil +} + +func (ks *Kind) String() string { + if ks.Type != nil { + return fmt.Sprintf("kind source: %T", ks.Type) + } + return "kind source: unknown type" +} + +// WaitForSync implements SyncingSource to allow controllers to wait with starting +// workers until the cache is synced. +func (ks *Kind) WaitForSync(ctx context.Context) error { + select { + case err := <-ks.started: + return err + case <-ctx.Done(): + ks.startCancel() + if errors.Is(ctx.Err(), context.Canceled) { + return nil + } + return errors.New("timed out waiting for cache to be synced") + } +} diff --git a/pkg/internal/testing/addr/addr_suite_test.go b/pkg/internal/testing/addr/addr_suite_test.go index b18c62def9..3869bb0207 100644 --- a/pkg/internal/testing/addr/addr_suite_test.go +++ b/pkg/internal/testing/addr/addr_suite_test.go @@ -19,15 +19,12 @@ package addr_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) func TestAddr(t *testing.T) { t.Parallel() RegisterFailHandler(Fail) - suiteName := "Addr Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Addr Suite") } diff --git a/pkg/internal/testing/addr/manager.go b/pkg/internal/testing/addr/manager.go index 15f36fe2d0..ffa33a8861 100644 --- a/pkg/internal/testing/addr/manager.go +++ b/pkg/internal/testing/addr/manager.go @@ -17,79 +17,126 @@ limitations under the License. package addr import ( + "errors" "fmt" + "io/fs" "net" - "sync" + "os" + "path/filepath" + "strings" "time" + + "sigs.k8s.io/controller-runtime/pkg/internal/flock" ) // TODO(directxman12): interface / release functionality for external port managers const ( - portReserveTime = 1 * time.Minute + portReserveTime = 2 * time.Minute portConflictRetry = 100 + portFilePrefix = "port-" +) + +var ( + cacheDir string ) -type portCache struct { - lock sync.Mutex - ports map[int]time.Time +func init() { + baseDir, err := os.UserCacheDir() + if err == nil { + cacheDir = filepath.Join(baseDir, "kubebuilder-envtest") + err = os.MkdirAll(cacheDir, 0o750) + } + if err != nil { + // Either we didn't get a cache directory, or we can't use it + baseDir = os.TempDir() + cacheDir = filepath.Join(baseDir, "kubebuilder-envtest") + err = os.MkdirAll(cacheDir, 0o750) + } + if err != nil { + panic(err) + } } -func (c *portCache) add(port int) bool { - c.lock.Lock() - defer c.lock.Unlock() - // remove outdated port - for p, t := range c.ports { - if time.Since(t) > portReserveTime { - delete(c.ports, p) +type portCache struct{} + +func (c *portCache) add(port int) (bool, error) { + // Remove outdated ports. + if err := fs.WalkDir(os.DirFS(cacheDir), ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err } + if d.IsDir() || !d.Type().IsRegular() || !strings.HasPrefix(path, portFilePrefix) { + return nil + } + info, err := d.Info() + if err != nil { + // No-op if file no longer exists; may have been deleted by another + // process/thread trying to allocate ports. + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err + } + if time.Since(info.ModTime()) > portReserveTime { + if err := os.Remove(filepath.Join(cacheDir, path)); err != nil { + // No-op if file no longer exists; may have been deleted by another + // process/thread trying to allocate ports. + if os.IsNotExist(err) { + return nil + } + return err + } + } + return nil + }); err != nil { + return false, err } - // try allocating new port - if _, ok := c.ports[port]; ok { - return false + // Try allocating new port, by acquiring a file. + path := fmt.Sprintf("%s/%s%d", cacheDir, portFilePrefix, port) + if err := flock.Acquire(path); errors.Is(err, flock.ErrAlreadyLocked) { + return false, nil + } else if err != nil { + return false, err } - c.ports[port] = time.Now() - return true + return true, nil } -var cache = &portCache{ - ports: make(map[int]time.Time), -} +var cache = &portCache{} -func suggest(listenHost string) (port int, resolvedHost string, err error) { +func suggest(listenHost string) (*net.TCPListener, int, string, error) { if listenHost == "" { listenHost = "localhost" } addr, err := net.ResolveTCPAddr("tcp", net.JoinHostPort(listenHost, "0")) if err != nil { - return + return nil, -1, "", err } l, err := net.ListenTCP("tcp", addr) if err != nil { - return + return nil, -1, "", err } - port = l.Addr().(*net.TCPAddr).Port - defer func() { - err = l.Close() - }() - resolvedHost = addr.IP.String() - return + return l, l.Addr().(*net.TCPAddr).Port, + addr.IP.String(), + nil } // Suggest suggests an address a process can listen on. It returns // a tuple consisting of a free port and the hostname resolved to its IP. // It makes sure that new port allocated does not conflict with old ports // allocated within 1 minute. -func Suggest(listenHost string) (port int, resolvedHost string, err error) { +func Suggest(listenHost string) (int, string, error) { for i := 0; i < portConflictRetry; i++ { - port, resolvedHost, err = suggest(listenHost) + listener, port, resolvedHost, err := suggest(listenHost) if err != nil { - return + return -1, "", err } - if cache.add(port) { - return + defer listener.Close() + if ok, err := cache.add(port); ok { + return port, resolvedHost, nil + } else if err != nil { + return -1, "", err } } - err = fmt.Errorf("no free ports found after %d retries", portConflictRetry) - return + return -1, "", fmt.Errorf("no free ports found after %d retries", portConflictRetry) } diff --git a/pkg/internal/testing/addr/manager_test.go b/pkg/internal/testing/addr/manager_test.go index cf95c36115..065e847dc5 100644 --- a/pkg/internal/testing/addr/manager_test.go +++ b/pkg/internal/testing/addr/manager_test.go @@ -20,7 +20,7 @@ import ( "net" "strconv" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "sigs.k8s.io/controller-runtime/pkg/internal/testing/addr" diff --git a/pkg/internal/testing/certs/certs_suite_test.go b/pkg/internal/testing/certs/certs_suite_test.go index 5b63fc4f55..3b3008c294 100644 --- a/pkg/internal/testing/certs/certs_suite_test.go +++ b/pkg/internal/testing/certs/certs_suite_test.go @@ -19,15 +19,12 @@ package certs_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) func TestInternal(t *testing.T) { t.Parallel() RegisterFailHandler(Fail) - suiteName := "TinyCA (Internal Certs) Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "TinyCA (Internal Certs) Suite") } diff --git a/pkg/internal/testing/certs/tinyca.go b/pkg/internal/testing/certs/tinyca.go index ed3c2da65d..b4188237e6 100644 --- a/pkg/internal/testing/certs/tinyca.go +++ b/pkg/internal/testing/certs/tinyca.go @@ -24,8 +24,9 @@ package certs import ( "crypto" + "crypto/ecdsa" + "crypto/elliptic" crand "crypto/rand" - "crypto/rsa" "crypto/x509" "crypto/x509/pkix" "encoding/pem" @@ -38,8 +39,8 @@ import ( ) var ( - rsaKeySize = 2048 // a decent number, as of 2019 - bigOne = big.NewInt(1) + ellipticCurve = elliptic.P256() + bigOne = big.NewInt(1) ) // CertPair is a private key and certificate for use for client auth, as a CA, or serving. @@ -63,7 +64,7 @@ func (k CertPair) AsBytes() (cert []byte, key []byte, err error) { rawKeyData, err := x509.MarshalPKCS8PrivateKey(k.Key) if err != nil { - return nil, nil, fmt.Errorf("unable to encode private key: %v", err) + return nil, nil, fmt.Errorf("unable to encode private key: %w", err) } key = pem.EncodeToMemory(&pem.Block{ @@ -86,7 +87,7 @@ type TinyCA struct { // newPrivateKey generates a new private key of a relatively sane size (see // rsaKeySize). func newPrivateKey() (crypto.Signer, error) { - return rsa.GenerateKey(crand.Reader, rsaKeySize) + return ecdsa.GenerateKey(ellipticCurve, crand.Reader) } // NewTinyCA creates a new a tiny CA utility for provisioning serving certs and client certs FOR TESTING ONLY. @@ -94,12 +95,12 @@ func newPrivateKey() (crypto.Signer, error) { func NewTinyCA() (*TinyCA, error) { caPrivateKey, err := newPrivateKey() if err != nil { - return nil, fmt.Errorf("unable to generate private key for CA: %v", err) + return nil, fmt.Errorf("unable to generate private key for CA: %w", err) } caCfg := certutil.Config{CommonName: "envtest-environment", Organization: []string{"envtest"}} caCert, err := certutil.NewSelfSignedCACert(caCfg, caPrivateKey) if err != nil { - return nil, fmt.Errorf("unable to generate certificate for CA: %v", err) + return nil, fmt.Errorf("unable to generate certificate for CA: %w", err) } return &TinyCA{ @@ -114,7 +115,7 @@ func (c *TinyCA) makeCert(cfg certutil.Config) (CertPair, error) { key, err := newPrivateKey() if err != nil { - return CertPair{}, fmt.Errorf("unable to create private key: %v", err) + return CertPair{}, fmt.Errorf("unable to create private key: %w", err) } serial := new(big.Int).Set(c.nextSerial) @@ -139,12 +140,12 @@ func (c *TinyCA) makeCert(cfg certutil.Config) (CertPair, error) { certRaw, err := x509.CreateCertificate(crand.Reader, &template, c.CA.Cert, key.Public(), c.CA.Key) if err != nil { - return CertPair{}, fmt.Errorf("unable to create certificate: %v", err) + return CertPair{}, fmt.Errorf("unable to create certificate: %w", err) } cert, err := x509.ParseCertificate(certRaw) if err != nil { - return CertPair{}, fmt.Errorf("generated invalid certificate, could not parse: %v", err) + return CertPair{}, fmt.Errorf("generated invalid certificate, could not parse: %w", err) } return CertPair{ diff --git a/pkg/internal/testing/certs/tinyca_test.go b/pkg/internal/testing/certs/tinyca_test.go index e3f2513210..6e0540ba9f 100644 --- a/pkg/internal/testing/certs/tinyca_test.go +++ b/pkg/internal/testing/certs/tinyca_test.go @@ -24,7 +24,7 @@ import ( "sort" "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" diff --git a/pkg/internal/testing/controlplane/apiserver.go b/pkg/internal/testing/controlplane/apiserver.go index ece5efd651..c9a1a232ea 100644 --- a/pkg/internal/testing/controlplane/apiserver.go +++ b/pkg/internal/testing/controlplane/apiserver.go @@ -19,7 +19,6 @@ package controlplane import ( "fmt" "io" - "io/ioutil" "net/url" "os" "path/filepath" @@ -385,10 +384,10 @@ func (s *APIServer) populateAPIServerCerts() error { return err } - if err := ioutil.WriteFile(filepath.Join(s.CertDir, "apiserver.crt"), certData, 0640); err != nil { //nolint:gosec + if err := os.WriteFile(filepath.Join(s.CertDir, "apiserver.crt"), certData, 0640); err != nil { //nolint:gosec return err } - if err := ioutil.WriteFile(filepath.Join(s.CertDir, "apiserver.key"), keyData, 0640); err != nil { //nolint:gosec + if err := os.WriteFile(filepath.Join(s.CertDir, "apiserver.key"), keyData, 0640); err != nil { //nolint:gosec return err } @@ -405,19 +404,19 @@ func (s *APIServer) populateAPIServerCerts() error { return err } - if err := ioutil.WriteFile(filepath.Join(s.CertDir, saCertFile), saCert, 0640); err != nil { //nolint:gosec + if err := os.WriteFile(filepath.Join(s.CertDir, saCertFile), saCert, 0640); err != nil { //nolint:gosec return err } - return ioutil.WriteFile(filepath.Join(s.CertDir, saKeyFile), saKey, 0640) //nolint:gosec + return os.WriteFile(filepath.Join(s.CertDir, saKeyFile), saKey, 0640) //nolint:gosec } // Stop stops this process gracefully, waits for its termination, and cleans up // the CertDir if necessary. func (s *APIServer) Stop() error { - if s.processState.DirNeedsCleaning { - s.CertDir = "" // reset the directory if it was randomly allocated, so that we can safely restart - } if s.processState != nil { + if s.processState.DirNeedsCleaning { + s.CertDir = "" // reset the directory if it was randomly allocated, so that we can safely restart + } if err := s.processState.Stop(); err != nil { return err } diff --git a/pkg/internal/testing/controlplane/apiserver_test.go b/pkg/internal/testing/controlplane/apiserver_test.go index b857220203..6ce1577d45 100644 --- a/pkg/internal/testing/controlplane/apiserver_test.go +++ b/pkg/internal/testing/controlplane/apiserver_test.go @@ -20,7 +20,7 @@ import ( "errors" "net/url" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/rest" diff --git a/pkg/internal/testing/controlplane/auth.go b/pkg/internal/testing/controlplane/auth.go index b2cd4e5e04..16c86a712c 100644 --- a/pkg/internal/testing/controlplane/auth.go +++ b/pkg/internal/testing/controlplane/auth.go @@ -18,7 +18,7 @@ package controlplane import ( "fmt" - "io/ioutil" + "os" "path/filepath" "k8s.io/client-go/rest" @@ -128,7 +128,7 @@ func (c *CertAuthn) Start() error { return fmt.Errorf("start called before configure") } caCrt := c.ca.CA.CertBytes() - if err := ioutil.WriteFile(c.caCrtPath(), caCrt, 0640); err != nil { //nolint:gosec + if err := os.WriteFile(c.caCrtPath(), caCrt, 0640); err != nil { //nolint:gosec return fmt.Errorf("unable to save the client certificate CA to %s: %w", c.caCrtPath(), err) } diff --git a/pkg/internal/testing/controlplane/auth_test.go b/pkg/internal/testing/controlplane/auth_test.go index 3acbc3d3c4..9891c6f2e2 100644 --- a/pkg/internal/testing/controlplane/auth_test.go +++ b/pkg/internal/testing/controlplane/auth_test.go @@ -22,7 +22,7 @@ import ( "os" "path/filepath" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/rest" kcert "k8s.io/client-go/util/cert" diff --git a/pkg/internal/testing/controlplane/controlplane_suite_test.go b/pkg/internal/testing/controlplane/controlplane_suite_test.go index 067b0c40ce..9ac69047f0 100644 --- a/pkg/internal/testing/controlplane/controlplane_suite_test.go +++ b/pkg/internal/testing/controlplane/controlplane_suite_test.go @@ -19,15 +19,12 @@ package controlplane_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) func TestIntegration(t *testing.T) { t.Parallel() RegisterFailHandler(Fail) - suiteName := "Control Plane Standup Unit Tests" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Control Plane Standup Unit Tests") } diff --git a/pkg/internal/testing/controlplane/etcd.go b/pkg/internal/testing/controlplane/etcd.go index 00d8254777..c30d213295 100644 --- a/pkg/internal/testing/controlplane/etcd.go +++ b/pkg/internal/testing/controlplane/etcd.go @@ -84,10 +84,14 @@ type Etcd struct { // args contains the structured arguments to use for running etcd. // Lazily initialized by .Configure(), Defaulted eventually with .defaultArgs() args *process.Arguments + + // listenPeerURL is the address the Etcd should listen on for peer connections. + // It's automatically generated and a random port is picked during execution. + listenPeerURL *url.URL } // Start starts the etcd, waits for it to come up, and returns an error, if one -// occoured. +// occurred. func (e *Etcd) Start() error { if err := e.setProcessState(); err != nil { return err @@ -111,6 +115,7 @@ func (e *Etcd) setProcessState() error { return err } + // Set the listen url. if e.URL == nil { port, host, err := addr.Suggest("") if err != nil { @@ -122,6 +127,18 @@ func (e *Etcd) setProcessState() error { } } + // Set the listen peer URL. + { + port, host, err := addr.Suggest("") + if err != nil { + return err + } + e.listenPeerURL = &url.URL{ + Scheme: "http", + Host: net.JoinHostPort(host, strconv.Itoa(port)), + } + } + // can use /health as of etcd 3.3.0 e.processState.HealthCheck.URL = *e.URL e.processState.HealthCheck.Path = "/health" @@ -150,13 +167,18 @@ func (e *Etcd) Stop() error { func (e *Etcd) defaultArgs() map[string][]string { args := map[string][]string{ - "listen-peer-urls": {"http://localhost:0"}, + "listen-peer-urls": {e.listenPeerURL.String()}, "data-dir": {e.DataDir}, } if e.URL != nil { args["advertise-client-urls"] = []string{e.URL.String()} args["listen-client-urls"] = []string{e.URL.String()} } + + // Add unsafe no fsync, available from etcd 3.5 + if ok, _ := e.processState.CheckFlag("unsafe-no-fsync"); ok { + args["unsafe-no-fsync"] = []string{"true"} + } return args } diff --git a/pkg/internal/testing/controlplane/etcd_test.go b/pkg/internal/testing/controlplane/etcd_test.go index e9a1f7a181..7c7c7561ff 100644 --- a/pkg/internal/testing/controlplane/etcd_test.go +++ b/pkg/internal/testing/controlplane/etcd_test.go @@ -17,7 +17,7 @@ limitations under the License. package controlplane_test import ( - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "sigs.k8s.io/controller-runtime/pkg/internal/testing/controlplane" diff --git a/pkg/internal/testing/controlplane/kubectl_test.go b/pkg/internal/testing/controlplane/kubectl_test.go index d7105b29f2..5484bc31a1 100644 --- a/pkg/internal/testing/controlplane/kubectl_test.go +++ b/pkg/internal/testing/controlplane/kubectl_test.go @@ -17,9 +17,9 @@ limitations under the License. package controlplane_test import ( - "io/ioutil" + "io" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" "k8s.io/client-go/rest" @@ -36,7 +36,7 @@ var _ = Describe("Kubectl", func() { stdout, stderr, err := k.Run(args...) Expect(err).NotTo(HaveOccurred()) Expect(stdout).To(ContainSubstring("something")) - bytes, err := ioutil.ReadAll(stderr) + bytes, err := io.ReadAll(stderr) Expect(err).NotTo(HaveOccurred()) Expect(bytes).To(BeEmpty()) }) diff --git a/pkg/internal/testing/controlplane/plane.go b/pkg/internal/testing/controlplane/plane.go index 36fd3c6306..456183a7a3 100644 --- a/pkg/internal/testing/controlplane/plane.go +++ b/pkg/internal/testing/controlplane/plane.go @@ -47,13 +47,18 @@ type ControlPlane struct { } // Start will start your control plane processes. To stop them, call Stop(). -func (f *ControlPlane) Start() error { +func (f *ControlPlane) Start() (retErr error) { if f.Etcd == nil { f.Etcd = &Etcd{} } if err := f.Etcd.Start(); err != nil { return err } + defer func() { + if retErr != nil { + _ = f.Etcd.Stop() + } + }() if f.APIServer == nil { f.APIServer = &APIServer{} @@ -62,6 +67,11 @@ func (f *ControlPlane) Start() error { if err := f.APIServer.Start(); err != nil { return err } + defer func() { + if retErr != nil { + _ = f.APIServer.Stop() + } + }() // provision the default user -- can be removed when the related // methods are removed. The default user has admin permissions to @@ -88,6 +98,7 @@ func (f *ControlPlane) Stop() error { errList = append(errList, err) } } + if f.Etcd != nil { if err := f.Etcd.Stop(); err != nil { errList = append(errList, err) diff --git a/pkg/internal/testing/controlplane/plane_test.go b/pkg/internal/testing/controlplane/plane_test.go index 714e76e8a4..cd0359dbca 100644 --- a/pkg/internal/testing/controlplane/plane_test.go +++ b/pkg/internal/testing/controlplane/plane_test.go @@ -19,7 +19,7 @@ package controlplane_test import ( "context" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" kauthn "k8s.io/api/authorization/v1" diff --git a/pkg/internal/testing/process/arguments.go b/pkg/internal/testing/process/arguments.go index 6c2c91e144..391eec1fac 100644 --- a/pkg/internal/testing/process/arguments.go +++ b/pkg/internal/testing/process/arguments.go @@ -93,12 +93,12 @@ type TemplateDefaults struct { // TemplateAndArguments joins structured arguments and non-structured arguments, preserving existing // behavior. Namely: // -// 1. if templ has len > 0, it will be rendered against data -// 2. the rendered template values that look like `--foo=bar` will be split -// and appended to args, the rest will be kept around -// 3. the given args will be rendered as string form. If a template is given, -// no defaults will be used, otherwise defaults will be used -// 4. a result of [args..., rest...] will be returned +// 1. if templ has len > 0, it will be rendered against data +// 2. the rendered template values that look like `--foo=bar` will be split +// and appended to args, the rest will be kept around +// 3. the given args will be rendered as string form. If a template is given, +// no defaults will be used, otherwise defaults will be used +// 4. a result of [args..., rest...] will be returned // // It returns the resulting rendered arguments, plus the arguments that were // not transferred to `args` during rendering. @@ -215,9 +215,9 @@ var ( // for passing to exec.Command and friends, making use of the given defaults // as indicated for each particular argument. // -// - Any flag in defaults that's not in Arguments will be present in the output -// - Any flag that's present in Arguments will be passed the corresponding -// defaults to do with as it will (ignore, append-to, suppress, etc). +// - Any flag in defaults that's not in Arguments will be present in the output +// - Any flag that's present in Arguments will be passed the corresponding +// defaults to do with as it will (ignore, append-to, suppress, etc). func (a *Arguments) AsStrings(defaults map[string][]string) []string { // sort for deterministic ordering keysInOrder := make([]string, 0, len(defaults)+len(a.values)) @@ -323,9 +323,9 @@ func (a *Arguments) SetRaw(key string, val Arg) *Arguments { // used in conjunction with SetRaw. For example, to set `--some-flag` to the // API server's CertDir, you could do: // -// server.Configure().SetRaw("--some-flag", FuncArg(func(defaults []string) []string { -// return []string{server.CertDir} -// })) +// server.Configure().SetRaw("--some-flag", FuncArg(func(defaults []string) []string { +// return []string{server.CertDir} +// })) // // FuncArg ignores Appends; if you need to support appending values too, consider implementing // Arg directly. diff --git a/pkg/internal/testing/process/arguments_test.go b/pkg/internal/testing/process/arguments_test.go index 2534eb8446..b513cbdf86 100644 --- a/pkg/internal/testing/process/arguments_test.go +++ b/pkg/internal/testing/process/arguments_test.go @@ -20,7 +20,7 @@ import ( "net/url" "strings" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "sigs.k8s.io/controller-runtime/pkg/internal/testing/process" @@ -225,12 +225,6 @@ var _ = Describe("Arguments Templates", func() { }) }) -type plainDefaults map[string][]string - -func (d plainDefaults) DefaultArgs() map[string][]string { - return d -} - var _ = Describe("Arguments", func() { Context("when appending", func() { It("should copy from defaults when appending for the first time", func() { diff --git a/pkg/internal/testing/process/bin_path_finder_test.go b/pkg/internal/testing/process/bin_path_finder_test.go index c933478811..1b15941840 100644 --- a/pkg/internal/testing/process/bin_path_finder_test.go +++ b/pkg/internal/testing/process/bin_path_finder_test.go @@ -19,7 +19,7 @@ package process import ( "os" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) diff --git a/pkg/internal/testing/process/process.go b/pkg/internal/testing/process/process.go index 5a0f7b8565..af83c70a2f 100644 --- a/pkg/internal/testing/process/process.go +++ b/pkg/internal/testing/process/process.go @@ -20,7 +20,6 @@ import ( "crypto/tls" "fmt" "io" - "io/ioutil" "net" "net/http" "net/url" @@ -84,7 +83,7 @@ type State struct { DirNeedsCleaning bool Path string - // ready holds wether the process is currently in ready state (hit the ready condition) or not. + // ready holds whether the process is currently in ready state (hit the ready condition) or not. // It will be set to true on a successful `Start()` and set to false on a successful `Stop()` ready bool @@ -109,7 +108,7 @@ func (ps *State) Init(name string) error { } if ps.Dir == "" { - newDir, err := ioutil.TempDir("", "k8s_test_framework_") + newDir, err := os.MkdirTemp("", "k8s_test_framework_") if err != nil { return err } @@ -185,16 +184,12 @@ func (ps *State) Start(stdout, stderr io.Writer) (err error) { ps.ready = true return nil case <-ps.waitDone: - if pollerStopCh != nil { - close(pollerStopCh) - } + close(pollerStopCh) return fmt.Errorf("timeout waiting for process %s to start successfully "+ "(it may have failed to start, or stopped unexpectedly before becoming ready)", path.Base(ps.Path)) case <-timedOut: - if pollerStopCh != nil { - close(pollerStopCh) - } + close(pollerStopCh) if ps.Cmd != nil { // intentionally ignore this -- we might've crashed, failed to start, etc ps.Cmd.Process.Signal(syscall.SIGTERM) //nolint:errcheck @@ -248,6 +243,12 @@ func pollURLUntilOK(url url.URL, interval time.Duration, ready chan bool, stopCh // Stop stops this process gracefully, waits for its termination, and cleans up // the CertDir if necessary. func (ps *State) Stop() error { + // Always clear the directory if we need to. + defer func() { + if ps.DirNeedsCleaning { + _ = os.RemoveAll(ps.Dir) + } + }() if ps.Cmd == nil { return nil } @@ -267,9 +268,5 @@ func (ps *State) Stop() error { return fmt.Errorf("timeout waiting for process %s to stop", path.Base(ps.Path)) } ps.ready = false - if ps.DirNeedsCleaning { - return os.RemoveAll(ps.Dir) - } - return nil } diff --git a/pkg/internal/testing/process/process_suite_test.go b/pkg/internal/testing/process/process_suite_test.go index 4b9d7ab198..5a64e9d2f0 100644 --- a/pkg/internal/testing/process/process_suite_test.go +++ b/pkg/internal/testing/process/process_suite_test.go @@ -19,15 +19,12 @@ package process_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) func TestInternal(t *testing.T) { t.Parallel() RegisterFailHandler(Fail) - suiteName := "Envtest Process Launcher Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Envtest Process Launcher Suite") } diff --git a/pkg/internal/testing/process/process_test.go b/pkg/internal/testing/process/process_test.go index 446d06707c..5b0708227a 100644 --- a/pkg/internal/testing/process/process_test.go +++ b/pkg/internal/testing/process/process_test.go @@ -18,7 +18,6 @@ package process_test import ( "bytes" - "io/ioutil" "net" "net/http" "net/url" @@ -26,7 +25,7 @@ import ( "strconv" "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/onsi/gomega/ghttp" "sigs.k8s.io/controller-runtime/pkg/internal/testing/addr" @@ -139,7 +138,7 @@ var _ = Describe("Start method", func() { echo 'i started' >&2 `, } - processState.StartTimeout = 1 * time.Second + processState.StartTimeout = 5 * time.Second Expect(processState.Start(stdout, stderr)).To(Succeed()) Eventually(processState.Exited).Should(BeTrue()) @@ -297,7 +296,7 @@ var _ = Describe("Stop method", func() { var err error Expect(processState.Start(nil, nil)).To(Succeed()) - processState.Dir, err = ioutil.TempDir("", "k8s_test_framework_") + processState.Dir, err = os.MkdirTemp("", "k8s_test_framework_") Expect(err).NotTo(HaveOccurred()) processState.DirNeedsCleaning = true processState.StopTimeout = 400 * time.Millisecond diff --git a/pkg/leaderelection/leader_election.go b/pkg/leaderelection/leader_election.go index 55fd228690..ee4fcf4cbe 100644 --- a/pkg/leaderelection/leader_election.go +++ b/pkg/leaderelection/leader_election.go @@ -19,13 +19,14 @@ package leaderelection import ( "errors" "fmt" - "io/ioutil" "os" "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/client-go/kubernetes" + coordinationv1client "k8s.io/client-go/kubernetes/typed/coordination/v1" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/rest" "k8s.io/client-go/tools/leaderelection/resourcelock" + "sigs.k8s.io/controller-runtime/pkg/recorder" ) @@ -38,7 +39,7 @@ type Options struct { LeaderElection bool // LeaderElectionResourceLock determines which resource lock to use for leader election, - // defaults to "configmapsleases". + // defaults to "leases". LeaderElectionResourceLock string // LeaderElectionNamespace determines the namespace in which the leader @@ -56,11 +57,12 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op return nil, nil } - // Default resource lock to "configmapsleases". We must keep this default until we are sure all controller-runtime - // users have upgraded from the original default ConfigMap lock to a controller-runtime version that has this new - // default. Many users of controller-runtime skip versions, so we should be extremely conservative here. + // Default resource lock to "leases". The previous default (from v0.7.0 to v0.11.x) was configmapsleases, which was + // used to migrate from configmaps to leases. Since the default was "configmapsleases" for over a year, spanning + // five minor releases, any actively maintained operators are very likely to have a released version that uses + // "configmapsleases". Therefore defaulting to "leases" should be safe. if options.LeaderElectionResourceLock == "" { - options.LeaderElectionResourceLock = resourcelock.ConfigMapsLeasesResourceLock + options.LeaderElectionResourceLock = resourcelock.LeasesResourceLock } // LeaderElectionID must be provided to prevent clashes @@ -84,8 +86,14 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op } id = id + "_" + string(uuid.NewUUID()) - // Construct client for leader election - client, err := kubernetes.NewForConfig(rest.AddUserAgent(config, "leader-election")) + // Construct clients for leader election + rest.AddUserAgent(config, "leader-election") + corev1Client, err := corev1client.NewForConfig(config) + if err != nil { + return nil, err + } + + coordinationClient, err := coordinationv1client.NewForConfig(config) if err != nil { return nil, err } @@ -93,8 +101,8 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op return resourcelock.New(options.LeaderElectionResourceLock, options.LeaderElectionNamespace, options.LeaderElectionID, - client.CoreV1(), - client.CoordinationV1(), + corev1Client, + coordinationClient, resourcelock.ResourceLockConfig{ Identity: id, EventRecorder: recorderProvider.GetEventRecorderFor(id), @@ -111,7 +119,7 @@ func getInClusterNamespace() (string, error) { } // Load the namespace file and return its content - namespace, err := ioutil.ReadFile(inClusterNamespacePath) + namespace, err := os.ReadFile(inClusterNamespacePath) if err != nil { return "", fmt.Errorf("error reading namespace file: %w", err) } diff --git a/pkg/log/deleg.go b/pkg/log/deleg.go index bbd9c9c756..c82447d919 100644 --- a/pkg/log/deleg.go +++ b/pkg/log/deleg.go @@ -25,16 +25,15 @@ import ( // loggerPromise knows how to populate a concrete logr.Logger // with options, given an actual base logger later on down the line. type loggerPromise struct { - logger *DelegatingLogger + logger *DelegatingLogSink childPromises []*loggerPromise promisesLock sync.Mutex - name *string - tags []interface{} - level int + name *string + tags []interface{} } -func (p *loggerPromise) WithName(l *DelegatingLogger, name string) *loggerPromise { +func (p *loggerPromise) WithName(l *DelegatingLogSink, name string) *loggerPromise { res := &loggerPromise{ logger: l, name: &name, @@ -48,7 +47,7 @@ func (p *loggerPromise) WithName(l *DelegatingLogger, name string) *loggerPromis } // WithValues provides a new Logger with the tags appended. -func (p *loggerPromise) WithValues(l *DelegatingLogger, tags ...interface{}) *loggerPromise { +func (p *loggerPromise) WithValues(l *DelegatingLogSink, tags ...interface{}) *loggerPromise { res := &loggerPromise{ logger: l, tags: tags, @@ -61,61 +60,56 @@ func (p *loggerPromise) WithValues(l *DelegatingLogger, tags ...interface{}) *lo return res } -func (p *loggerPromise) V(l *DelegatingLogger, level int) *loggerPromise { - res := &loggerPromise{ - logger: l, - level: level, - promisesLock: sync.Mutex{}, - } - - p.promisesLock.Lock() - defer p.promisesLock.Unlock() - p.childPromises = append(p.childPromises, res) - return res -} - // Fulfill instantiates the Logger with the provided logger. -func (p *loggerPromise) Fulfill(parentLogger logr.Logger) { - var logger = parentLogger +func (p *loggerPromise) Fulfill(parentLogSink logr.LogSink) { + sink := parentLogSink if p.name != nil { - logger = logger.WithName(*p.name) + sink = sink.WithName(*p.name) } if p.tags != nil { - logger = logger.WithValues(p.tags...) - } - if p.level != 0 { - logger = logger.V(p.level) + sink = sink.WithValues(p.tags...) } p.logger.lock.Lock() - p.logger.logger = logger + p.logger.logger = sink + if withCallDepth, ok := sink.(logr.CallDepthLogSink); ok { + p.logger.logger = withCallDepth.WithCallDepth(1) + } p.logger.promise = nil p.logger.lock.Unlock() for _, childPromise := range p.childPromises { - childPromise.Fulfill(logger) + childPromise.Fulfill(sink) } } -// DelegatingLogger is a logr.Logger that delegates to another logr.Logger. +// DelegatingLogSink is a logsink that delegates to another logr.LogSink. // If the underlying promise is not nil, it registers calls to sub-loggers with // the logging factory to be populated later, and returns a new delegating // logger. It expects to have *some* logr.Logger set at all times (generally // a no-op logger before the promises are fulfilled). -type DelegatingLogger struct { +type DelegatingLogSink struct { lock sync.RWMutex - logger logr.Logger + logger logr.LogSink promise *loggerPromise + info logr.RuntimeInfo +} + +// Init implements logr.LogSink. +func (l *DelegatingLogSink) Init(info logr.RuntimeInfo) { + l.lock.Lock() + defer l.lock.Unlock() + l.info = info } // Enabled tests whether this Logger is enabled. For example, commandline // flags might be used to set the logging verbosity and disable some info // logs. -func (l *DelegatingLogger) Enabled() bool { +func (l *DelegatingLogSink) Enabled(level int) bool { l.lock.RLock() defer l.lock.RUnlock() - return l.logger.Enabled() + return l.logger.Enabled(level) } // Info logs a non-error message with the given key/value pairs as context. @@ -124,10 +118,10 @@ func (l *DelegatingLogger) Enabled() bool { // the log line. The key/value pairs can then be used to add additional // variable information. The key/value pairs should alternate string // keys and arbitrary values. -func (l *DelegatingLogger) Info(msg string, keysAndValues ...interface{}) { +func (l *DelegatingLogSink) Info(level int, msg string, keysAndValues ...interface{}) { l.lock.RLock() defer l.lock.RUnlock() - l.logger.Info(msg, keysAndValues...) + l.logger.Info(level, msg, keysAndValues...) } // Error logs an error, with the given message and key/value pairs as context. @@ -138,41 +132,26 @@ func (l *DelegatingLogger) Info(msg string, keysAndValues ...interface{}) { // The msg field should be used to add context to any underlying error, // while the err field should be used to attach the actual error that // triggered this log line, if present. -func (l *DelegatingLogger) Error(err error, msg string, keysAndValues ...interface{}) { +func (l *DelegatingLogSink) Error(err error, msg string, keysAndValues ...interface{}) { l.lock.RLock() defer l.lock.RUnlock() l.logger.Error(err, msg, keysAndValues...) } -// V returns an Logger value for a specific verbosity level, relative to -// this Logger. In other words, V values are additive. V higher verbosity -// level means a log message is less important. It's illegal to pass a log -// level less than zero. -func (l *DelegatingLogger) V(level int) logr.Logger { - l.lock.RLock() - defer l.lock.RUnlock() - - if l.promise == nil { - return l.logger.V(level) - } - - res := &DelegatingLogger{logger: l.logger} - promise := l.promise.V(res, level) - res.promise = promise - - return res -} - // WithName provides a new Logger with the name appended. -func (l *DelegatingLogger) WithName(name string) logr.Logger { +func (l *DelegatingLogSink) WithName(name string) logr.LogSink { l.lock.RLock() defer l.lock.RUnlock() if l.promise == nil { - return l.logger.WithName(name) + sink := l.logger.WithName(name) + if withCallDepth, ok := sink.(logr.CallDepthLogSink); ok { + sink = withCallDepth.WithCallDepth(-1) + } + return sink } - res := &DelegatingLogger{logger: l.logger} + res := &DelegatingLogSink{logger: l.logger} promise := l.promise.WithName(res, name) res.promise = promise @@ -180,15 +159,19 @@ func (l *DelegatingLogger) WithName(name string) logr.Logger { } // WithValues provides a new Logger with the tags appended. -func (l *DelegatingLogger) WithValues(tags ...interface{}) logr.Logger { +func (l *DelegatingLogSink) WithValues(tags ...interface{}) logr.LogSink { l.lock.RLock() defer l.lock.RUnlock() if l.promise == nil { - return l.logger.WithValues(tags...) + sink := l.logger.WithValues(tags...) + if withCallDepth, ok := sink.(logr.CallDepthLogSink); ok { + sink = withCallDepth.WithCallDepth(-1) + } + return sink } - res := &DelegatingLogger{logger: l.logger} + res := &DelegatingLogSink{logger: l.logger} promise := l.promise.WithValues(res, tags...) res.promise = promise @@ -198,16 +181,16 @@ func (l *DelegatingLogger) WithValues(tags ...interface{}) logr.Logger { // Fulfill switches the logger over to use the actual logger // provided, instead of the temporary initial one, if this method // has not been previously called. -func (l *DelegatingLogger) Fulfill(actual logr.Logger) { +func (l *DelegatingLogSink) Fulfill(actual logr.LogSink) { if l.promise != nil { l.promise.Fulfill(actual) } } -// NewDelegatingLogger constructs a new DelegatingLogger which uses -// the given logger before it's promise is fulfilled. -func NewDelegatingLogger(initial logr.Logger) *DelegatingLogger { - l := &DelegatingLogger{ +// NewDelegatingLogSink constructs a new DelegatingLogSink which uses +// the given logger before its promise is fulfilled. +func NewDelegatingLogSink(initial logr.LogSink) *DelegatingLogSink { + l := &DelegatingLogSink{ logger: initial, promise: &loggerPromise{promisesLock: sync.Mutex{}}, } diff --git a/pkg/log/log.go b/pkg/log/log.go index 229ac7ec35..082dce3adb 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -17,7 +17,7 @@ limitations under the License. // Package log contains utilities for fetching a new logger // when one is not already available. // -// The Log Handle +// # The Log Handle // // This package contains a root logr.Logger Log. It may be used to // get a handle to whatever the root logging implementation is. By @@ -25,11 +25,11 @@ limitations under the License. // to loggers. When the implementation is set using SetLogger, these // "promises" will be converted over to real loggers. // -// Logr +// # Logr // // All logging in controller-runtime is structured, using a set of interfaces // defined by a package called logr -// (https://godoc.org/github.com/go-logr/logr). The sub-package zap provides +// (https://pkg.go.dev/github.com/go-logr/logr). The sub-package zap provides // helpers for setting up logr backed by Zap (go.uber.org/zap). package log @@ -47,14 +47,14 @@ func SetLogger(l logr.Logger) { defer loggerWasSetLock.Unlock() loggerWasSet = true - Log.Fulfill(l) + dlog.Fulfill(l.GetSink()) } // It is safe to assume that if this wasn't set within the first 30 seconds of a binaries -// lifetime, it will never get set. The DelegatingLogger causes a high number of memory -// allocations when not given an actual Logger, so we set a NullLogger to avoid that. +// lifetime, it will never get set. The DelegatingLogSink causes a high number of memory +// allocations when not given an actual Logger, so we set a NullLogSink to avoid that. // -// We need to keep the DelegatingLogger because we have various inits() that get a logger from +// We need to keep the DelegatingLogSink because we have various inits() that get a logger from // here. They will always get executed before any code that imports controller-runtime // has a chance to run and hence to set an actual logger. func init() { @@ -64,7 +64,7 @@ func init() { loggerWasSetLock.Lock() defer loggerWasSetLock.Unlock() if !loggerWasSet { - Log.Fulfill(NullLogger{}) + dlog.Fulfill(NullLogSink{}) } }() } @@ -78,14 +78,17 @@ var ( // to another logr.Logger. You *must* call SetLogger to // get any actual logging. If SetLogger is not called within // the first 30 seconds of a binaries lifetime, it will get -// set to a NullLogger. -var Log = NewDelegatingLogger(NullLogger{}) +// set to a NullLogSink. +var ( + dlog = NewDelegatingLogSink(NullLogSink{}) + Log = logr.New(dlog) +) // FromContext returns a logger with predefined values from a context.Context. func FromContext(ctx context.Context, keysAndValues ...interface{}) logr.Logger { - var log logr.Logger = Log + log := Log if ctx != nil { - if logger := logr.FromContext(ctx); logger != nil { + if logger, err := logr.FromContext(ctx); err == nil { log = logger } } diff --git a/pkg/log/log_suite_test.go b/pkg/log/log_suite_test.go index bf8e967cb7..f0e349aa86 100644 --- a/pkg/log/log_suite_test.go +++ b/pkg/log/log_suite_test.go @@ -19,13 +19,11 @@ package log import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Log Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Log Suite") } diff --git a/pkg/log/log_test.go b/pkg/log/log_test.go index be3ad87997..a4f4d895ab 100644 --- a/pkg/log/log_test.go +++ b/pkg/log/log_test.go @@ -21,11 +21,11 @@ import ( "errors" "github.com/go-logr/logr" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) -var _ logr.Logger = &DelegatingLogger{} +var _ logr.LogSink = &DelegatingLogSink{} // logInfo is the information for a particular fakeLogger message. type logInfo struct { @@ -49,7 +49,10 @@ type fakeLogger struct { root *fakeLoggerRoot } -func (f *fakeLogger) WithName(name string) logr.Logger { +func (f *fakeLogger) Init(info logr.RuntimeInfo) { +} + +func (f *fakeLogger) WithName(name string) logr.LogSink { names := append([]string(nil), f.name...) names = append(names, name) return &fakeLogger{ @@ -59,7 +62,7 @@ func (f *fakeLogger) WithName(name string) logr.Logger { } } -func (f *fakeLogger) WithValues(vals ...interface{}) logr.Logger { +func (f *fakeLogger) WithValues(vals ...interface{}) logr.LogSink { tags := append([]interface{}(nil), f.tags...) tags = append(tags, vals...) return &fakeLogger{ @@ -80,7 +83,7 @@ func (f *fakeLogger) Error(err error, msg string, vals ...interface{}) { }) } -func (f *fakeLogger) Info(msg string, vals ...interface{}) { +func (f *fakeLogger) Info(level int, msg string, vals ...interface{}) { tags := append([]interface{}(nil), f.tags...) tags = append(tags, vals...) f.root.messages = append(f.root.messages, logInfo{ @@ -90,8 +93,7 @@ func (f *fakeLogger) Info(msg string, vals ...interface{}) { }) } -func (f *fakeLogger) Enabled() bool { return true } -func (f *fakeLogger) V(lvl int) logr.Logger { return f } +func (f *fakeLogger) Enabled(level int) bool { return true } var _ = Describe("logging", func() { @@ -103,7 +105,7 @@ var _ = Describe("logging", func() { By("actually setting the logger") logger := &fakeLogger{root: &fakeLoggerRoot{}} - SetLogger(logger) + SetLogger(logr.New(logger)) By("grabbing another sub-logger and logging to both loggers") l2 := Log.WithName("runtimeLog").WithValues("newtag", "newvalue2") @@ -121,24 +123,24 @@ var _ = Describe("logging", func() { Describe("lazy logger initialization", func() { var ( root *fakeLoggerRoot - baseLog logr.Logger - delegLog *DelegatingLogger + baseLog logr.LogSink + delegLog *DelegatingLogSink ) BeforeEach(func() { root = &fakeLoggerRoot{} baseLog = &fakeLogger{root: root} - delegLog = NewDelegatingLogger(NullLogger{}) + delegLog = NewDelegatingLogSink(NullLogSink{}) }) It("should delegate with name", func() { By("asking for a logger with a name before fulfill, and logging") - befFulfill1 := delegLog.WithName("before-fulfill") + befFulfill1 := logr.New(delegLog).WithName("before-fulfill") befFulfill2 := befFulfill1.WithName("two") befFulfill1.Info("before fulfill") By("logging on the base logger before fulfill") - delegLog.Info("before fulfill base") + logr.New(delegLog).Info("before fulfill base") By("ensuring that no messages were actually recorded") Expect(root.messages).To(BeEmpty()) @@ -154,7 +156,7 @@ var _ = Describe("logging", func() { befFulfill1.WithName("after-from-before").Info("after 3") By("logging with new loggers") - delegLog.WithName("after-fulfill").Info("after 4") + logr.New(delegLog).WithName("after-fulfill").Info("after 4") By("ensuring that the messages are appropriately named") Expect(root.messages).To(ConsistOf( @@ -179,10 +181,10 @@ var _ = Describe("logging", func() { // Constructing the child in the goroutine does not reliably // trigger the race detector - child := delegLog.WithName("child") + child := logr.New(delegLog).WithName("child") go func() { defer GinkgoRecover() - delegLog.Fulfill(NullLogger{}) + delegLog.Fulfill(NullLogSink{}) close(fulfillDone) }() go func() { @@ -202,12 +204,12 @@ var _ = Describe("logging", func() { }() go func() { defer GinkgoRecover() - delegLog.Enabled() + logr.New(delegLog).Enabled() close(logEnabledDone) }() go func() { defer GinkgoRecover() - delegLog.Info("hello world") + logr.New(delegLog).Info("hello world") close(logInfoDone) }() go func() { @@ -217,7 +219,7 @@ var _ = Describe("logging", func() { }() go func() { defer GinkgoRecover() - delegLog.V(1) + logr.New(delegLog).V(1) close(logVDone) }() @@ -233,12 +235,12 @@ var _ = Describe("logging", func() { It("should delegate with tags", func() { By("asking for a logger with a name before fulfill, and logging") - befFulfill1 := delegLog.WithValues("tag1", "val1") + befFulfill1 := logr.New(delegLog).WithValues("tag1", "val1") befFulfill2 := befFulfill1.WithValues("tag2", "val2") befFulfill1.Info("before fulfill") By("logging on the base logger before fulfill") - delegLog.Info("before fulfill base") + logr.New(delegLog).Info("before fulfill base") By("ensuring that no messages were actually recorded") Expect(root.messages).To(BeEmpty()) @@ -254,7 +256,7 @@ var _ = Describe("logging", func() { befFulfill1.WithValues("tag3", "val3").Info("after 3") By("logging with new loggers") - delegLog.WithValues("tag3", "val3").Info("after 4") + logr.New(delegLog).WithValues("tag3", "val3").Info("after 4") By("ensuring that the messages are appropriately named") Expect(root.messages).To(ConsistOf( @@ -270,13 +272,13 @@ var _ = Describe("logging", func() { delegLog.Fulfill(baseLog) By("logging a bit") - delegLog.Info("msg 1") + logr.New(delegLog).Info("msg 1") By("fulfilling with a new logger") delegLog.Fulfill(&fakeLogger{}) By("logging some more") - delegLog.Info("msg 2") + logr.New(delegLog).Info("msg 2") By("checking that all log messages are present") Expect(root.messages).To(ConsistOf( @@ -296,7 +298,7 @@ var _ = Describe("logging", func() { root := &fakeLoggerRoot{} baseLog := &fakeLogger{root: root} - wantLog := baseLog.WithName("my-logger") + wantLog := logr.New(baseLog).WithName("my-logger") ctx := IntoContext(context.Background(), wantLog) gotLog := FromContext(ctx) @@ -312,7 +314,7 @@ var _ = Describe("logging", func() { root := &fakeLoggerRoot{} baseLog := &fakeLogger{root: root} - wantLog := baseLog.WithName("my-logger") + wantLog := logr.New(baseLog).WithName("my-logger") ctx := IntoContext(context.Background(), wantLog) gotLog := FromContext(ctx, "tag1", "value1") diff --git a/pkg/log/null.go b/pkg/log/null.go index 09a5a02eb6..f3e81074fe 100644 --- a/pkg/log/null.go +++ b/pkg/log/null.go @@ -24,37 +24,36 @@ import ( // but avoids accidentally adding the testing flags to // all binaries. -// NullLogger is a logr.Logger that does nothing. -type NullLogger struct{} +// NullLogSink is a logr.Logger that does nothing. +type NullLogSink struct{} -var _ logr.Logger = NullLogger{} +var _ logr.LogSink = NullLogSink{} + +// Init implements logr.LogSink. +func (log NullLogSink) Init(logr.RuntimeInfo) { +} // Info implements logr.InfoLogger. -func (NullLogger) Info(_ string, _ ...interface{}) { +func (NullLogSink) Info(_ int, _ string, _ ...interface{}) { // Do nothing. } // Enabled implements logr.InfoLogger. -func (NullLogger) Enabled() bool { +func (NullLogSink) Enabled(level int) bool { return false } // Error implements logr.Logger. -func (NullLogger) Error(_ error, _ string, _ ...interface{}) { +func (NullLogSink) Error(_ error, _ string, _ ...interface{}) { // Do nothing. } -// V implements logr.Logger. -func (log NullLogger) V(_ int) logr.Logger { - return log -} - // WithName implements logr.Logger. -func (log NullLogger) WithName(_ string) logr.Logger { +func (log NullLogSink) WithName(_ string) logr.LogSink { return log } // WithValues implements logr.Logger. -func (log NullLogger) WithValues(_ ...interface{}) logr.Logger { +func (log NullLogSink) WithValues(_ ...interface{}) logr.LogSink { return log } diff --git a/pkg/log/warning_handler.go b/pkg/log/warning_handler.go index 3012fdd411..e9522632d3 100644 --- a/pkg/log/warning_handler.go +++ b/pkg/log/warning_handler.go @@ -47,7 +47,7 @@ type KubeAPIWarningLogger struct { } // HandleWarningHeader handles logging for responses from API server that are -// warnings with code being 299 and uses a logr.Logger for it's logging purposes. +// warnings with code being 299 and uses a logr.Logger for its logging purposes. func (l *KubeAPIWarningLogger) HandleWarningHeader(code int, agent string, message string) { if code != 299 || len(message) == 0 { return diff --git a/pkg/log/zap/flags.go b/pkg/log/zap/flags.go index 3339655075..fb492b14da 100644 --- a/pkg/log/zap/flags.go +++ b/pkg/log/zap/flags.go @@ -128,3 +128,41 @@ func (ev *stackTraceFlag) String() string { func (ev *stackTraceFlag) Type() string { return "level" } + +type timeEncodingFlag struct { + setFunc func(zapcore.TimeEncoder) + value string +} + +var _ flag.Value = &timeEncodingFlag{} + +func (ev *timeEncodingFlag) String() string { + return ev.value +} + +func (ev *timeEncodingFlag) Type() string { + return "time-encoding" +} + +func (ev *timeEncodingFlag) Set(flagValue string) error { + val := strings.ToLower(flagValue) + switch val { + case "rfc3339nano": + ev.setFunc(zapcore.RFC3339NanoTimeEncoder) + case "rfc3339": + ev.setFunc(zapcore.RFC3339TimeEncoder) + case "iso8601": + ev.setFunc(zapcore.ISO8601TimeEncoder) + case "millis": + ev.setFunc(zapcore.EpochMillisTimeEncoder) + case "nanos": + ev.setFunc(zapcore.EpochNanosTimeEncoder) + case "epoch": + ev.setFunc(zapcore.EpochTimeEncoder) + default: + return fmt.Errorf("invalid time-encoding value \"%s\"", flagValue) + } + + ev.value = flagValue + return nil +} diff --git a/pkg/log/zap/kube_helpers.go b/pkg/log/zap/kube_helpers.go index 765327d623..9824470240 100644 --- a/pkg/log/zap/kube_helpers.go +++ b/pkg/log/zap/kube_helpers.go @@ -18,6 +18,7 @@ package zap import ( "fmt" + "reflect" "go.uber.org/zap/buffer" "go.uber.org/zap/zapcore" @@ -64,6 +65,11 @@ type kubeObjectWrapper struct { func (w kubeObjectWrapper) MarshalLogObject(enc zapcore.ObjectEncoder) error { // TODO(directxman12): log kind and apiversion if not set explicitly (common case) // -- needs an a scheme to convert to the GVK. + + if reflect.ValueOf(w.obj).IsNil() { + return fmt.Errorf("got nil for runtime.Object") + } + if gvk := w.obj.GetObjectKind().GroupVersionKind(); gvk.Version != "" { enc.AddString("apiVersion", gvk.GroupVersion().String()) enc.AddString("kind", gvk.Kind) diff --git a/pkg/log/zap/zap.go b/pkg/log/zap/zap.go index 22eb5d771a..ee89a7c6a4 100644 --- a/pkg/log/zap/zap.go +++ b/pkg/log/zap/zap.go @@ -101,7 +101,7 @@ func newConsoleEncoder(opts ...EncoderConfigOption) zapcore.Encoder { return zapcore.NewConsoleEncoder(encoderConfig) } -// Level sets Options.Level, which configures the the minimum enabled logging level e.g Debug, Info. +// Level sets Options.Level, which configures the minimum enabled logging level e.g Debug, Info. // A zap log level should be multiplied by -1 to get the logr verbosity. // For example, to get logr verbosity of 3, pass zapcore.Level(-3) to this Opts. // See https://pkg.go.dev/github.com/go-logr/zapr for how zap level relates to logr verbosity. @@ -138,7 +138,7 @@ type Options struct { // console when Development is true and JSON otherwise Encoder zapcore.Encoder // EncoderConfigOptions can modify the EncoderConfig needed to initialize an Encoder. - // See https://godoc.org/go.uber.org/zap/zapcore#EncoderConfig for the list of options + // See https://pkg.go.dev/go.uber.org/zap/zapcore#EncoderConfig for the list of options // that can be configured. // Note that the EncoderConfigOptions are not applied when the Encoder option is already set. EncoderConfigOptions []EncoderConfigOption @@ -167,6 +167,9 @@ type Options struct { // ZapOpts allows passing arbitrary zap.Options to configure on the // underlying Zap logger. ZapOpts []zap.Option + // TimeEncoder specifies the encoder for the timestamps in log messages. + // Defaults to RFC3339TimeEncoder. + TimeEncoder zapcore.TimeEncoder } // addDefaults adds defaults to the Options. @@ -212,6 +215,16 @@ func (o *Options) addDefaults() { })) } } + + if o.TimeEncoder == nil { + o.TimeEncoder = zapcore.RFC3339TimeEncoder + } + f := func(ecfg *zapcore.EncoderConfig) { + ecfg.EncodeTime = o.TimeEncoder + } + // prepend instead of append it in case someone adds a time encoder option in it + o.EncoderConfigOptions = append([]EncoderConfigOption{f}, o.EncoderConfigOptions...) + if o.Encoder == nil { o.Encoder = o.NewEncoder(o.EncoderConfigOptions...) } @@ -231,19 +244,22 @@ func NewRaw(opts ...Opts) *zap.Logger { // this basically mimics NewConfig, but with a custom sink sink := zapcore.AddSync(o.DestWriter) - o.ZapOpts = append(o.ZapOpts, zap.AddCallerSkip(1), zap.ErrorOutput(sink)) + o.ZapOpts = append(o.ZapOpts, zap.ErrorOutput(sink)) log := zap.New(zapcore.NewCore(&KubeAwareEncoder{Encoder: o.Encoder, Verbose: o.Development}, sink, o.Level)) log = log.WithOptions(o.ZapOpts...) return log } -// BindFlags will parse the given flagset for zap option flags and set the log options accordingly -// zap-devel: Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn) -// Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) -// zap-encoder: Zap log encoding (one of 'json' or 'console') -// zap-log-level: Zap Level to configure the verbosity of logging. Can be one of 'debug', 'info', 'error', -// or any integer value > 0 which corresponds to custom debug levels of increasing verbosity") -// zap-stacktrace-level: Zap Level at and above which stacktraces are captured (one of 'info', 'error' or 'panic') +// BindFlags will parse the given flagset for zap option flags and set the log options accordingly: +// - zap-devel: +// Development Mode defaults(encoder=consoleEncoder,logLevel=Debug,stackTraceLevel=Warn) +// Production Mode defaults(encoder=jsonEncoder,logLevel=Info,stackTraceLevel=Error) +// - zap-encoder: Zap log encoding (one of 'json' or 'console') +// - zap-log-level: Zap Level to configure the verbosity of logging. Can be one of 'debug', 'info', 'error', +// or any integer value > 0 which corresponds to custom debug levels of increasing verbosity"). +// - zap-stacktrace-level: Zap Level at and above which stacktraces are captured (one of 'info', 'error' or 'panic') +// - zap-time-encoding: Zap time encoding (one of 'epoch', 'millis', 'nano', 'iso8601', 'rfc3339' or 'rfc3339nano'), +// Defaults to 'epoch'. func (o *Options) BindFlags(fs *flag.FlagSet) { // Set Development mode value fs.BoolVar(&o.Development, "zap-devel", o.Development, @@ -273,13 +289,21 @@ func (o *Options) BindFlags(fs *flag.FlagSet) { } fs.Var(&stackVal, "zap-stacktrace-level", "Zap Level at and above which stacktraces are captured (one of 'info', 'error', 'panic').") + + // Set the time encoding + var timeEncoderVal timeEncodingFlag + timeEncoderVal.setFunc = func(fromFlag zapcore.TimeEncoder) { + o.TimeEncoder = fromFlag + } + fs.Var(&timeEncoderVal, "zap-time-encoding", "Zap time encoding (one of 'epoch', 'millis', 'nano', 'iso8601', 'rfc3339' or 'rfc3339nano'). Defaults to 'epoch'.") } // UseFlagOptions configures the logger to use the Options set by parsing zap option flags from the CLI. -// opts := zap.Options{} -// opts.BindFlags(flag.CommandLine) -// flag.Parse() -// log := zap.New(zap.UseFlagOptions(&opts)) +// +// opts := zap.Options{} +// opts.BindFlags(flag.CommandLine) +// flag.Parse() +// log := zap.New(zap.UseFlagOptions(&opts)) func UseFlagOptions(in *Options) Opts { return func(o *Options) { *o = *in diff --git a/pkg/log/zap/zap_suite_test.go b/pkg/log/zap/zap_suite_test.go index 43044d8066..d7a7f22866 100644 --- a/pkg/log/zap/zap_suite_test.go +++ b/pkg/log/zap/zap_suite_test.go @@ -19,13 +19,11 @@ package zap import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Zap Log Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Zap Log Suite") } diff --git a/pkg/log/zap/zap_test.go b/pkg/log/zap/zap_test.go index 095fc54598..1a8b3995c2 100644 --- a/pkg/log/zap/zap_test.go +++ b/pkg/log/zap/zap_test.go @@ -21,9 +21,10 @@ import ( "encoding/json" "flag" "os" + "reflect" "github.com/go-logr/logr" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "go.uber.org/zap/zapcore" corev1 "k8s.io/api/core/v1" @@ -61,7 +62,7 @@ type fakeLoggerRoot struct { messages []logInfo } -var _ logr.Logger = &fakeLogger{} +var _ logr.LogSink = &fakeLogger{} // fakeLogger is a fake implementation of logr.Logger that records // messages, tags, and names, @@ -73,7 +74,10 @@ type fakeLogger struct { root *fakeLoggerRoot } -func (f *fakeLogger) WithName(name string) logr.Logger { +func (f *fakeLogger) Init(info logr.RuntimeInfo) { +} + +func (f *fakeLogger) WithName(name string) logr.LogSink { names := append([]string(nil), f.name...) names = append(names, name) return &fakeLogger{ @@ -83,7 +87,7 @@ func (f *fakeLogger) WithName(name string) logr.Logger { } } -func (f *fakeLogger) WithValues(vals ...interface{}) logr.Logger { +func (f *fakeLogger) WithValues(vals ...interface{}) logr.LogSink { tags := append([]interface{}(nil), f.tags...) tags = append(tags, vals...) return &fakeLogger{ @@ -104,7 +108,7 @@ func (f *fakeLogger) Error(err error, msg string, vals ...interface{}) { }) } -func (f *fakeLogger) Info(msg string, vals ...interface{}) { +func (f *fakeLogger) Info(level int, msg string, vals ...interface{}) { tags := append([]interface{}(nil), f.tags...) tags = append(tags, vals...) f.root.messages = append(f.root.messages, logInfo{ @@ -114,8 +118,8 @@ func (f *fakeLogger) Info(msg string, vals ...interface{}) { }) } -func (f *fakeLogger) Enabled() bool { return true } -func (f *fakeLogger) V(lvl int) logr.Logger { return f } +func (f *fakeLogger) Enabled(level int) bool { return true } +func (f *fakeLogger) V(lvl int) logr.LogSink { return f } var _ = Describe("Zap options setup", func() { var opts *Options @@ -251,6 +255,14 @@ var _ = Describe("Zap logger setup", func() { "namespace": name.Namespace, })) }) + + It("should not panic with nil obj", func() { + var pod *corev1.Pod + logger.Info("here's a kubernetes object", "thing", pod) + + outRaw := logOut.Bytes() + Expect(string(outRaw)).Should(ContainSubstring("got nil for runtime.Object")) + }) } Context("with logger created using New", func() { @@ -472,35 +484,76 @@ var _ = Describe("Zap log level flag options setup", func() { }) }) - Context("with encoder options provided programmatically", func() { + Context("with zap-time-encoding flag provided", func() { + + It("Should set time encoder in options", func() { + args := []string{"--zap-time-encoding=rfc3339"} + fromFlags.BindFlags(&fs) + err := fs.Parse(args) + Expect(err).ToNot(HaveOccurred()) + + opt := Options{} + UseFlagOptions(&fromFlags)(&opt) + opt.addDefaults() - It("Should set Console Encoder, with given Nanos TimeEncoder option.", func() { + optVal := reflect.ValueOf(opt.TimeEncoder) + expVal := reflect.ValueOf(zapcore.RFC3339TimeEncoder) + + Expect(optVal.Pointer()).To(Equal(expVal.Pointer())) + }) + + It("Should default to 'rfc3339' time encoding", func() { + args := []string{""} + fromFlags.BindFlags(&fs) + err := fs.Parse(args) + Expect(err).ToNot(HaveOccurred()) + + opt := Options{} + UseFlagOptions(&fromFlags)(&opt) + opt.addDefaults() + + optVal := reflect.ValueOf(opt.TimeEncoder) + expVal := reflect.ValueOf(zapcore.RFC3339TimeEncoder) + + Expect(optVal.Pointer()).To(Equal(expVal.Pointer())) + }) + + It("Should return an error message, with unknown time-encoding", func() { + fs = *flag.NewFlagSet(os.Args[0], flag.ContinueOnError) + args := []string{"--zap-time-encoding=foobar"} + fromFlags.BindFlags(&fs) + err := fs.Parse(args) + Expect(err).To(HaveOccurred()) + }) + + It("Should propagate time encoder to logger", func() { + // zaps ISO8601TimeEncoder uses 2006-01-02T15:04:05.000Z0700 as pattern for iso8601 encoding + iso8601Pattern := `^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}([-+][0-9]{4}|Z)` + + args := []string{"--zap-time-encoding=iso8601"} + fromFlags.BindFlags(&fs) + err := fs.Parse(args) + Expect(err).ToNot(HaveOccurred()) logOut := new(bytes.Buffer) - f := func(ec *zapcore.EncoderConfig) { - if err := ec.EncodeTime.UnmarshalText([]byte("nanos")); err != nil { - Expect(err).ToNot(HaveOccurred()) - } - } - opts := func(o *Options) { - o.EncoderConfigOptions = append(o.EncoderConfigOptions, f) - } - log := New(UseDevMode(true), WriteTo(logOut), opts) - log.Info("This is a test message") + + logger := New(UseFlagOptions(&fromFlags), WriteTo(logOut)) + logger.Info("This is a test message") + outRaw := logOut.Bytes() - // Assert for Console Encoder - res := map[string]interface{}{} - Expect(json.Unmarshal(outRaw, &res)).ToNot(Succeed()) - // Assert for Epoch Nanos TimeEncoder - Expect(string(outRaw)).ShouldNot(ContainSubstring(".")) + res := map[string]interface{}{} + Expect(json.Unmarshal(outRaw, &res)).To(Succeed()) + Expect(res["ts"]).Should(MatchRegexp(iso8601Pattern)) }) + + }) + + Context("with encoder options provided programmatically", func() { + It("Should set JSON Encoder, with given Millis TimeEncoder option, and MessageKey", func() { logOut := new(bytes.Buffer) f := func(ec *zapcore.EncoderConfig) { ec.MessageKey = "MillisTimeFormat" - if err := ec.EncodeTime.UnmarshalText([]byte("millis")); err != nil { - Expect(err).ToNot(HaveOccurred()) - } } opts := func(o *Options) { o.EncoderConfigOptions = append(o.EncoderConfigOptions, f) @@ -511,8 +564,6 @@ var _ = Describe("Zap log level flag options setup", func() { // Assert for JSON Encoder res := map[string]interface{}{} Expect(json.Unmarshal(outRaw, &res)).To(Succeed()) - // Assert for Epoch Nanos TimeEncoder - Expect(string(outRaw)).Should(ContainSubstring(".")) // Assert for MessageKey Expect(string(outRaw)).Should(ContainSubstring("MillisTimeFormat")) }) @@ -535,8 +586,8 @@ var _ = Describe("Zap log level flag options setup", func() { logOut.Truncate(0) logger.V(4).Info("test 4") // Should not be logged Expect(logOut.String()).To(BeEmpty()) - logger.V(-3).Info("test -3") // Log a panic, since V(-1*N) for all N > 0 is not permitted. - Expect(logOut.String()).To(ContainSubstring(`"level":"dpanic"`)) + logger.V(-3).Info("test -3") + Expect(logOut.String()).To(ContainSubstring("test -3")) }) It("does not log with positive logr level", func() { By("setting up the logger") diff --git a/pkg/manager/example_test.go b/pkg/manager/example_test.go index 17557d1817..06712d7171 100644 --- a/pkg/manager/example_test.go +++ b/pkg/manager/example_test.go @@ -20,6 +20,7 @@ import ( "context" "os" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client/config" conf "sigs.k8s.io/controller-runtime/pkg/config" @@ -51,7 +52,7 @@ func ExampleNew() { } // This example creates a new Manager that has a cache scoped to a list of namespaces. -func ExampleNew_multinamespaceCache() { +func ExampleNew_limitToNamespaces() { cfg, err := config.GetConfig() if err != nil { log.Error(err, "unable to get kubeconfig") @@ -59,8 +60,11 @@ func ExampleNew_multinamespaceCache() { } mgr, err := manager.New(cfg, manager.Options{ - NewCache: cache.MultiNamespacedCacheBuilder([]string{"namespace1", "namespace2"}), - }) + NewCache: func(config *rest.Config, opts cache.Options) (cache.Cache, error) { + opts.Namespaces = []string{"namespace1", "namespace2"} + return cache.New(config, opts) + }}, + ) if err != nil { log.Error(err, "unable to set up manager") os.Exit(1) diff --git a/pkg/manager/internal.go b/pkg/manager/internal.go index 5f85e10c90..86dddf088a 100644 --- a/pkg/manager/internal.go +++ b/pkg/manager/internal.go @@ -18,11 +18,13 @@ package manager import ( "context" + "crypto/tls" "errors" "fmt" "net" "net/http" "sync" + "sync/atomic" "time" "github.com/go-logr/logr" @@ -30,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" kerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/rest" "k8s.io/client-go/tools/leaderelection" "k8s.io/client-go/tools/leaderelection/resourcelock" @@ -38,16 +41,16 @@ import ( "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" - "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/internal/httpserver" intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder" "sigs.k8s.io/controller-runtime/pkg/metrics" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/webhook" ) const ( - // Values taken from: https://github.com/kubernetes/apiserver/blob/master/pkg/apis/config/v1alpha1/defaults.go + // Values taken from: https://github.com/kubernetes/component-base/blob/master/config/v1alpha1/defaults.go defaultLeaseDuration = 15 * time.Second defaultRenewDeadline = 10 * time.Second defaultRetryPeriod = 2 * time.Second @@ -61,16 +64,15 @@ const ( var _ Runnable = &controllerManager{} type controllerManager struct { - // cluster holds a variety of methods to interact with a cluster. Required. - cluster cluster.Cluster + sync.Mutex + started bool - // leaderElectionRunnables is the set of Controllers that the controllerManager injects deps into and Starts. - // These Runnables are managed by lead election. - leaderElectionRunnables []Runnable + stopProcedureEngaged *int64 + errChan chan error + runnables *runnables - // nonLeaderElectionRunnables is the set of webhook servers that the controllerManager injects deps into and Starts. - // These Runnables will not be blocked by lead election. - nonLeaderElectionRunnables []Runnable + // cluster holds a variety of methods to interact with a cluster. Required. + cluster cluster.Cluster // recorderProvider is used to generate event recorders that will be injected into Controllers // (and EventHandlers, Sources and Predicates). @@ -104,38 +106,27 @@ type controllerManager struct { // Healthz probe handler healthzHandler *healthz.Handler - mu sync.Mutex - started bool - startedLeader bool - healthzStarted bool - errChan chan error - - // controllerOptions are the global controller options. - controllerOptions v1alpha1.ControllerConfigurationSpec + // controllerConfig are the global controller options. + controllerConfig config.Controller // Logger is the logger that should be used by this manager. // If none is set, it defaults to log.Log global logger. logger logr.Logger - // leaderElectionCancel is used to cancel the leader election. It is distinct from internalStopper, - // because for safety reasons we need to os.Exit() when we lose the leader election, meaning that - // it must be deferred until after gracefulShutdown is done. - leaderElectionCancel context.CancelFunc - // leaderElectionStopped is an internal channel used to signal the stopping procedure that the // LeaderElection.Run(...) function has returned and the shutdown can proceed. leaderElectionStopped chan struct{} - // stop procedure engaged. In other words, we should not add anything else to the manager - stopProcedureEngaged bool + // leaderElectionCancel is used to cancel the leader election. It is distinct from internalStopper, + // because for safety reasons we need to os.Exit() when we lose the leader election, meaning that + // it must be deferred until after gracefulShutdown is done. + leaderElectionCancel context.CancelFunc // elected is closed when this manager becomes the leader of a group of // managers, either because it won a leader election or because no leader // election was configured. elected chan struct{} - caches []hasCache - // port is the port that the webhook server serves at. port int // host is the hostname that the webhook server binds to. @@ -144,12 +135,17 @@ type controllerManager struct { // if not set, webhook server would look up the server key and certificate in // {TempDir}/k8s-webhook-server/serving-certs certDir string + // tlsOpts is used to allow configuring the TLS config used for the webhook server. + tlsOpts []func(*tls.Config) webhookServer *webhook.Server // webhookServerOnce will be called in GetWebhookServer() to optionally initialize // webhookServer if unset, and Add() it to controllerManager. webhookServerOnce sync.Once + // leaderElectionID is the name of the resource that leader election + // will use for holding the leader lock. + leaderElectionID string // leaseDuration is the duration that non-leader candidates will // wait to force acquire leadership. leaseDuration time.Duration @@ -160,10 +156,6 @@ type controllerManager struct { // between tries of actions. retryPeriod time.Duration - // waitForRunnable is holding the number of runnables currently running so that - // we can wait for them to exit before quitting the manager - waitForRunnable sync.WaitGroup - // gracefulShutdownTimeout is the duration given to runnable to stop // before the manager actually returns on stop. gracefulShutdownTimeout time.Duration @@ -192,65 +184,28 @@ type hasCache interface { // Add sets dependencies on i, and adds it to the list of Runnables to start. func (cm *controllerManager) Add(r Runnable) error { - cm.mu.Lock() - defer cm.mu.Unlock() - if cm.stopProcedureEngaged { - return errors.New("can't accept new runnable as stop procedure is already engaged") - } - - // Set dependencies on the object - if err := cm.SetFields(r); err != nil { - return err - } - - var shouldStart bool - - // Add the runnable to the leader election or the non-leaderelection list - if leRunnable, ok := r.(LeaderElectionRunnable); ok && !leRunnable.NeedLeaderElection() { - shouldStart = cm.started - cm.nonLeaderElectionRunnables = append(cm.nonLeaderElectionRunnables, r) - } else if hasCache, ok := r.(hasCache); ok { - cm.caches = append(cm.caches, hasCache) - } else { - shouldStart = cm.startedLeader - cm.leaderElectionRunnables = append(cm.leaderElectionRunnables, r) - } - - if shouldStart { - // If already started, start the controller - cm.startRunnable(r) - } - - return nil + cm.Lock() + defer cm.Unlock() + return cm.add(r) } -// Deprecated: use the equivalent Options field to set a field. This method will be removed in v0.10. -func (cm *controllerManager) SetFields(i interface{}) error { - if err := cm.cluster.SetFields(i); err != nil { - return err - } - if _, err := inject.InjectorInto(cm.SetFields, i); err != nil { - return err - } - if _, err := inject.StopChannelInto(cm.internalProceduresStop, i); err != nil { - return err - } - if _, err := inject.LoggerInto(cm.logger, i); err != nil { - return err - } - - return nil +func (cm *controllerManager) add(r Runnable) error { + return cm.runnables.Add(r) } // AddMetricsExtraHandler adds extra handler served on path to the http server that serves metrics. func (cm *controllerManager) AddMetricsExtraHandler(path string, handler http.Handler) error { + cm.Lock() + defer cm.Unlock() + + if cm.started { + return fmt.Errorf("unable to add new metrics handler because metrics endpoint has already been created") + } + if path == defaultMetricsEndpoint { return fmt.Errorf("overriding builtin %s endpoint is not allowed", defaultMetricsEndpoint) } - cm.mu.Lock() - defer cm.mu.Unlock() - if _, found := cm.metricsExtraHandlers[path]; found { return fmt.Errorf("can't register extra handler by duplicate path %q on metrics http server", path) } @@ -262,14 +217,10 @@ func (cm *controllerManager) AddMetricsExtraHandler(path string, handler http.Ha // AddHealthzCheck allows you to add Healthz checker. func (cm *controllerManager) AddHealthzCheck(name string, check healthz.Checker) error { - cm.mu.Lock() - defer cm.mu.Unlock() + cm.Lock() + defer cm.Unlock() - if cm.stopProcedureEngaged { - return errors.New("can't accept new healthCheck as stop procedure is already engaged") - } - - if cm.healthzStarted { + if cm.started { return fmt.Errorf("unable to add new checker because healthz endpoint has already been created") } @@ -283,15 +234,11 @@ func (cm *controllerManager) AddHealthzCheck(name string, check healthz.Checker) // AddReadyzCheck allows you to add Readyz checker. func (cm *controllerManager) AddReadyzCheck(name string, check healthz.Checker) error { - cm.mu.Lock() - defer cm.mu.Unlock() - - if cm.stopProcedureEngaged { - return errors.New("can't accept new ready check as stop procedure is already engaged") - } + cm.Lock() + defer cm.Unlock() - if cm.healthzStarted { - return fmt.Errorf("unable to add new checker because readyz endpoint has already been created") + if cm.started { + return fmt.Errorf("unable to add new checker because healthz endpoint has already been created") } if cm.readyzHandler == nil { @@ -302,6 +249,10 @@ func (cm *controllerManager) AddReadyzCheck(name string, check healthz.Checker) return nil } +func (cm *controllerManager) GetHTTPClient() *http.Client { + return cm.cluster.GetHTTPClient() +} + func (cm *controllerManager) GetConfig() *rest.Config { return cm.cluster.GetConfig() } @@ -341,10 +292,11 @@ func (cm *controllerManager) GetWebhookServer() *webhook.Server { Port: cm.port, Host: cm.host, CertDir: cm.certDir, + TLSOpts: cm.tlsOpts, } } if err := cm.Add(cm.webhookServer); err != nil { - panic("unable to add webhook server to the controller manager") + panic(fmt.Sprintf("unable to add webhook server to the controller manager: %s", err)) } }) return cm.webhookServer @@ -354,8 +306,8 @@ func (cm *controllerManager) GetLogger() logr.Logger { return cm.logger } -func (cm *controllerManager) GetControllerOptions() v1alpha1.ControllerConfigurationSpec { - return cm.controllerOptions +func (cm *controllerManager) GetControllerOptions() config.Controller { + return cm.controllerConfig } func (cm *controllerManager) serveMetrics() { @@ -365,77 +317,91 @@ func (cm *controllerManager) serveMetrics() { // TODO(JoelSpeed): Use existing Kubernetes machinery for serving metrics mux := http.NewServeMux() mux.Handle(defaultMetricsEndpoint, handler) - - func() { - cm.mu.Lock() - defer cm.mu.Unlock() - - for path, extraHandler := range cm.metricsExtraHandlers { - mux.Handle(path, extraHandler) - } - }() - - server := http.Server{ - Handler: mux, + for path, extraHandler := range cm.metricsExtraHandlers { + mux.Handle(path, extraHandler) } - // Run the server - cm.startRunnable(RunnableFunc(func(_ context.Context) error { - cm.logger.Info("starting metrics server", "path", defaultMetricsEndpoint) - if err := server.Serve(cm.metricsListener); err != nil && err != http.ErrServerClosed { - return err - } - return nil - })) - // Shutdown the server when stop is closed - <-cm.internalProceduresStop - if err := server.Shutdown(cm.shutdownCtx); err != nil { - cm.errChan <- err - } + server := httpserver.New(mux) + go cm.httpServe("metrics", cm.logger.WithValues("path", defaultMetricsEndpoint), server, cm.metricsListener) } func (cm *controllerManager) serveHealthProbes() { mux := http.NewServeMux() - server := http.Server{ - Handler: mux, + server := httpserver.New(mux) + + if cm.readyzHandler != nil { + mux.Handle(cm.readinessEndpointName, http.StripPrefix(cm.readinessEndpointName, cm.readyzHandler)) + // Append '/' suffix to handle subpaths + mux.Handle(cm.readinessEndpointName+"/", http.StripPrefix(cm.readinessEndpointName, cm.readyzHandler)) + } + if cm.healthzHandler != nil { + mux.Handle(cm.livenessEndpointName, http.StripPrefix(cm.livenessEndpointName, cm.healthzHandler)) + // Append '/' suffix to handle subpaths + mux.Handle(cm.livenessEndpointName+"/", http.StripPrefix(cm.livenessEndpointName, cm.healthzHandler)) } - func() { - cm.mu.Lock() - defer cm.mu.Unlock() + go cm.httpServe("health probe", cm.logger, server, cm.healthProbeListener) +} - if cm.readyzHandler != nil { - mux.Handle(cm.readinessEndpointName, http.StripPrefix(cm.readinessEndpointName, cm.readyzHandler)) - // Append '/' suffix to handle subpaths - mux.Handle(cm.readinessEndpointName+"/", http.StripPrefix(cm.readinessEndpointName, cm.readyzHandler)) - } - if cm.healthzHandler != nil { - mux.Handle(cm.livenessEndpointName, http.StripPrefix(cm.livenessEndpointName, cm.healthzHandler)) - // Append '/' suffix to handle subpaths - mux.Handle(cm.livenessEndpointName+"/", http.StripPrefix(cm.livenessEndpointName, cm.healthzHandler)) - } +func (cm *controllerManager) httpServe(kind string, log logr.Logger, server *http.Server, ln net.Listener) { + log = log.WithValues("kind", kind, "addr", ln.Addr()) - // Run server - cm.startRunnable(RunnableFunc(func(_ context.Context) error { - if err := server.Serve(cm.healthProbeListener); err != nil && err != http.ErrServerClosed { - return err + go func() { + log.Info("Starting server") + if err := server.Serve(ln); err != nil { + if errors.Is(err, http.ErrServerClosed) { + return } - return nil - })) - cm.healthzStarted = true + if atomic.LoadInt64(cm.stopProcedureEngaged) > 0 { + // There might be cases where connections are still open and we try to shutdown + // but not having enough time to close the connection causes an error in Serve + // + // In that case we want to avoid returning an error to the main error channel. + log.Error(err, "error on Serve after stop has been engaged") + return + } + cm.errChan <- err + } }() - // Shutdown the server when stop is closed + // Shutdown the server when stop is closed. <-cm.internalProceduresStop if err := server.Shutdown(cm.shutdownCtx); err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + // Avoid logging context related errors. + return + } + if atomic.LoadInt64(cm.stopProcedureEngaged) > 0 { + cm.logger.Error(err, "error on Shutdown after stop has been engaged") + return + } cm.errChan <- err } } +// Start starts the manager and waits indefinitely. +// There is only two ways to have start return: +// An error has occurred during in one of the internal operations, +// such as leader election, cache start, webhooks, and so on. +// Or, the context is cancelled. func (cm *controllerManager) Start(ctx context.Context) (err error) { - if err := cm.Add(cm.cluster); err != nil { - return fmt.Errorf("failed to add cluster to runnables: %w", err) + cm.Lock() + if cm.started { + cm.Unlock() + return errors.New("manager already started") } + cm.started = true + + var ready bool + defer func() { + // Only unlock the manager if we haven't reached + // the internal readiness condition. + if !ready { + cm.Unlock() + } + }() + + // Initialize the internal context. cm.internalCtx, cm.internalCancel = context.WithCancel(ctx) // This chan indicates that stop is complete, in other words all runnables have returned or timeout on stop request @@ -457,40 +423,70 @@ func (cm *controllerManager) Start(ctx context.Context) (err error) { } }() - // initialize this here so that we reset the signal channel state on every start - // Everything that might write into this channel must be started in a new goroutine, - // because otherwise we might block this routine trying to write into the full channel - // and will not be able to enter the deferred cm.engageStopProcedure() which drains - // it. - cm.errChan = make(chan error) + // Add the cluster runnable. + if err := cm.add(cm.cluster); err != nil { + return fmt.Errorf("failed to add cluster to runnables: %w", err) + } // Metrics should be served whether the controller is leader or not. // (If we don't serve metrics for non-leaders, prometheus will still scrape - // the pod but will get a connection refused) + // the pod but will get a connection refused). if cm.metricsListener != nil { - go cm.serveMetrics() + cm.serveMetrics() } - // Serve health probes + // Serve health probes. if cm.healthProbeListener != nil { - go cm.serveHealthProbes() + cm.serveHealthProbes() } - go cm.startNonLeaderElectionRunnables() + // First start any webhook servers, which includes conversion, validation, and defaulting + // webhooks that are registered. + // + // WARNING: Webhooks MUST start before any cache is populated, otherwise there is a race condition + // between conversion webhooks and the cache sync (usually initial list) which causes the webhooks + // to never start because no cache can be populated. + if err := cm.runnables.Webhooks.Start(cm.internalCtx); err != nil { + if !errors.Is(err, wait.ErrWaitTimeout) { + return err + } + } - go func() { - if cm.resourceLock != nil { - err := cm.startLeaderElection() - if err != nil { - cm.errChan <- err - } - } else { - // Treat not having leader election enabled the same as being elected. - cm.startLeaderElectionRunnables() - close(cm.elected) + // Start and wait for caches. + if err := cm.runnables.Caches.Start(cm.internalCtx); err != nil { + if !errors.Is(err, wait.ErrWaitTimeout) { + return err } - }() + } + + // Start the non-leaderelection Runnables after the cache has synced. + if err := cm.runnables.Others.Start(cm.internalCtx); err != nil { + if !errors.Is(err, wait.ErrWaitTimeout) { + return err + } + } + // Start the leader election and all required runnables. + { + ctx, cancel := context.WithCancel(context.Background()) + cm.leaderElectionCancel = cancel + go func() { + if cm.resourceLock != nil { + if err := cm.startLeaderElection(ctx); err != nil { + cm.errChan <- err + } + } else { + // Treat not having leader election enabled the same as being elected. + if err := cm.startLeaderElectionRunnables(); err != nil { + cm.errChan <- err + } + close(cm.elected) + } + }() + } + + ready = true + cm.Unlock() select { case <-ctx.Done(): // We are done @@ -504,24 +500,36 @@ func (cm *controllerManager) Start(ctx context.Context) (err error) { // engageStopProcedure signals all runnables to stop, reads potential errors // from the errChan and waits for them to end. It must not be called more than once. func (cm *controllerManager) engageStopProcedure(stopComplete <-chan struct{}) error { - // Populate the shutdown context. + if !atomic.CompareAndSwapInt64(cm.stopProcedureEngaged, 0, 1) { + return errors.New("stop procedure already engaged") + } + + // Populate the shutdown context, this operation MUST be done before + // closing the internalProceduresStop channel. + // + // The shutdown context immediately expires if the gracefulShutdownTimeout is not set. var shutdownCancel context.CancelFunc - if cm.gracefulShutdownTimeout > 0 { - cm.shutdownCtx, shutdownCancel = context.WithTimeout(context.Background(), cm.gracefulShutdownTimeout) - } else { + if cm.gracefulShutdownTimeout < 0 { + // We want to wait forever for the runnables to stop. cm.shutdownCtx, shutdownCancel = context.WithCancel(context.Background()) + } else { + cm.shutdownCtx, shutdownCancel = context.WithTimeout(context.Background(), cm.gracefulShutdownTimeout) } defer shutdownCancel() - // Cancel the internal stop channel and wait for the procedures to stop and complete. - close(cm.internalProceduresStop) - cm.internalCancel() - // Start draining the errors before acquiring the lock to make sure we don't deadlock // if something that has the lock is blocked on trying to write into the unbuffered // channel after something else already wrote into it. + var closeOnce sync.Once go func() { for { + // Closing in the for loop is required to avoid race conditions between + // the closure of all internal procedures and making sure to have a reader off the error channel. + closeOnce.Do(func() { + // Cancel the internal stop channel and wait for the procedures to stop and complete. + close(cm.internalProceduresStop) + cm.internalCancel() + }) select { case err, ok := <-cm.errChan: if ok { @@ -532,26 +540,14 @@ func (cm *controllerManager) engageStopProcedure(stopComplete <-chan struct{}) e } } }() - if cm.gracefulShutdownTimeout == 0 { - return nil - } - cm.mu.Lock() - defer cm.mu.Unlock() - cm.stopProcedureEngaged = true - // we want to close this after the other runnables stop, because we don't + // We want to close this after the other runnables stop, because we don't // want things like leader election to try and emit events on a closed // channel defer cm.recorderProvider.Stop(cm.shutdownCtx) - return cm.waitForRunnableToEnd(shutdownCancel) -} - -// waitForRunnableToEnd blocks until all runnables ended or the -// tearDownTimeout was reached. In the latter case, an error is returned. -func (cm *controllerManager) waitForRunnableToEnd(shutdownCancel context.CancelFunc) (retErr error) { - // Cancel leader election only after we waited. It will os.Exit() the app for safety. defer func() { - if retErr == nil && cm.leaderElectionCancel != nil { + // Cancel leader election only after we waited. It will os.Exit() the app for safety. + if cm.resourceLock != nil { // After asking the context to be cancelled, make sure // we wait for the leader stopped channel to be closed, otherwise // we might encounter race conditions between this code @@ -562,102 +558,49 @@ func (cm *controllerManager) waitForRunnableToEnd(shutdownCancel context.CancelF }() go func() { - cm.waitForRunnable.Wait() + // First stop the non-leader election runnables. + cm.logger.Info("Stopping and waiting for non leader election runnables") + cm.runnables.Others.StopAndWait(cm.shutdownCtx) + + // Stop all the leader election runnables, which includes reconcilers. + cm.logger.Info("Stopping and waiting for leader election runnables") + cm.runnables.LeaderElection.StopAndWait(cm.shutdownCtx) + + // Stop the caches before the leader election runnables, this is an important + // step to make sure that we don't race with the reconcilers by receiving more events + // from the API servers and enqueueing them. + cm.logger.Info("Stopping and waiting for caches") + cm.runnables.Caches.StopAndWait(cm.shutdownCtx) + + // Webhooks should come last, as they might be still serving some requests. + cm.logger.Info("Stopping and waiting for webhooks") + cm.runnables.Webhooks.StopAndWait(cm.shutdownCtx) + + // Proceed to close the manager and overall shutdown context. + cm.logger.Info("Wait completed, proceeding to shutdown the manager") shutdownCancel() }() <-cm.shutdownCtx.Done() - if err := cm.shutdownCtx.Err(); err != nil && err != context.Canceled { - return fmt.Errorf("failed waiting for all runnables to end within grace period of %s: %w", cm.gracefulShutdownTimeout, err) - } - return nil -} - -func (cm *controllerManager) startNonLeaderElectionRunnables() { - cm.mu.Lock() - defer cm.mu.Unlock() - - // First start any webhook servers, which includes conversion, validation, and defaulting - // webhooks that are registered. - // - // WARNING: Webhooks MUST start before any cache is populated, otherwise there is a race condition - // between conversion webhooks and the cache sync (usually initial list) which causes the webhooks - // to never start because no cache can be populated. - for _, c := range cm.nonLeaderElectionRunnables { - if _, ok := c.(*webhook.Server); ok { - cm.startRunnable(c) - } - } - - // Start and wait for caches. - cm.waitForCache(cm.internalCtx) - - // Start the non-leaderelection Runnables after the cache has synced - for _, c := range cm.nonLeaderElectionRunnables { - if _, ok := c.(*webhook.Server); ok { - continue + if err := cm.shutdownCtx.Err(); err != nil && !errors.Is(err, context.Canceled) { + if errors.Is(err, context.DeadlineExceeded) { + if cm.gracefulShutdownTimeout > 0 { + return fmt.Errorf("failed waiting for all runnables to end within grace period of %s: %w", cm.gracefulShutdownTimeout, err) + } + return nil } - - // Controllers block, but we want to return an error if any have an error starting. - // Write any Start errors to a channel so we can return them - cm.startRunnable(c) - } -} - -func (cm *controllerManager) startLeaderElectionRunnables() { - cm.mu.Lock() - defer cm.mu.Unlock() - - cm.waitForCache(cm.internalCtx) - - // Start the leader election Runnables after the cache has synced - for _, c := range cm.leaderElectionRunnables { - // Controllers block, but we want to return an error if any have an error starting. - // Write any Start errors to a channel so we can return them - cm.startRunnable(c) + // For any other error, return the error. + return err } - cm.startedLeader = true + return nil } -func (cm *controllerManager) waitForCache(ctx context.Context) { - if cm.started { - return - } - - for _, cache := range cm.caches { - cm.startRunnable(cache) - } - - // Wait for the caches to sync. - // TODO(community): Check the return value and write a test - for _, cache := range cm.caches { - cache.GetCache().WaitForCacheSync(ctx) - } - // TODO: This should be the return value of cm.cache.WaitForCacheSync but we abuse - // cm.started as check if we already started the cache so it must always become true. - // Making sure that the cache doesn't get started twice is needed to not get a "close - // of closed channel" panic - cm.started = true +func (cm *controllerManager) startLeaderElectionRunnables() error { + return cm.runnables.LeaderElection.Start(cm.internalCtx) } -func (cm *controllerManager) startLeaderElection() (err error) { - ctx, cancel := context.WithCancel(context.Background()) - cm.mu.Lock() - cm.leaderElectionCancel = cancel - cm.mu.Unlock() - - if cm.onStoppedLeading == nil { - cm.onStoppedLeading = func() { - // Make sure graceful shutdown is skipped if we lost the leader lock without - // intending to. - cm.gracefulShutdownTimeout = time.Duration(0) - // Most implementations of leader election log.Fatal() here. - // Since Start is wrapped in log.Fatal when called, we can just return - // an error here which will cause the program to exit. - cm.errChan <- errors.New("leader election lost") - } - } +func (cm *controllerManager) startLeaderElection(ctx context.Context) (err error) { l, err := leaderelection.NewLeaderElector(leaderelection.LeaderElectionConfig{ Lock: cm.resourceLock, LeaseDuration: cm.leaseDuration, @@ -665,12 +608,27 @@ func (cm *controllerManager) startLeaderElection() (err error) { RetryPeriod: cm.retryPeriod, Callbacks: leaderelection.LeaderCallbacks{ OnStartedLeading: func(_ context.Context) { - cm.startLeaderElectionRunnables() + if err := cm.startLeaderElectionRunnables(); err != nil { + cm.errChan <- err + return + } close(cm.elected) }, - OnStoppedLeading: cm.onStoppedLeading, + OnStoppedLeading: func() { + if cm.onStoppedLeading != nil { + cm.onStoppedLeading() + } + // Make sure graceful shutdown is skipped if we lost the leader lock without + // intending to. + cm.gracefulShutdownTimeout = time.Duration(0) + // Most implementations of leader election log.Fatal() here. + // Since Start is wrapped in log.Fatal when called, we can just return + // an error here which will cause the program to exit. + cm.errChan <- errors.New("leader election lost") + }, }, ReleaseOnCancel: cm.leaderElectionReleaseOnCancel, + Name: cm.leaderElectionID, }) if err != nil { return err @@ -688,13 +646,3 @@ func (cm *controllerManager) startLeaderElection() (err error) { func (cm *controllerManager) Elected() <-chan struct{} { return cm.elected } - -func (cm *controllerManager) startRunnable(r Runnable) { - cm.waitForRunnable.Add(1) - go func() { - defer cm.waitForRunnable.Done() - if err := r.Start(cm.internalCtx); err != nil { - cm.errChan <- err - } - }() -} diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 903e3e47f9..93f9f60d8e 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -18,6 +18,7 @@ package manager import ( "context" + "crypto/tls" "fmt" "net" "net/http" @@ -31,18 +32,18 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/client-go/tools/record" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/healthz" - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder" "sigs.k8s.io/controller-runtime/pkg/leaderelection" + "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/recorder" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/webhook" ) @@ -53,8 +54,7 @@ type Manager interface { cluster.Cluster // Add will set requested dependencies on the component, and cause the component to be - // started when Start is called. Add will inject any dependencies for which the argument - // implements the inject interface - e.g. inject.Client. + // started when Start is called. // Depending on if a Runnable implements LeaderElectionRunnable interface, a Runnable can be run in either // non-leaderelection mode (always running) or leader election mode (managed by leader election if enabled). Add(Runnable) error @@ -92,18 +92,18 @@ type Manager interface { GetLogger() logr.Logger // GetControllerOptions returns controller global configuration options. - GetControllerOptions() v1alpha1.ControllerConfigurationSpec + GetControllerOptions() config.Controller } // Options are the arguments for creating a new Manager. type Options struct { - // Scheme is the scheme used to resolve runtime.Objects to GroupVersionKinds / Resources + // Scheme is the scheme used to resolve runtime.Objects to GroupVersionKinds / Resources. // Defaults to the kubernetes/client-go scheme.Scheme, but it's almost always better - // idea to pass your own scheme in. See the documentation in pkg/scheme for more information. + // to pass your own scheme in. See the documentation in pkg/scheme for more information. Scheme *runtime.Scheme // MapperProvider provides the rest mapper used to map go types to Kubernetes APIs - MapperProvider func(c *rest.Config) (meta.RESTMapper, error) + MapperProvider func(c *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) // SyncPeriod determines the minimum frequency at which watched resources are // reconciled. A lower period will correct entropy more quickly, but reduce @@ -141,18 +141,36 @@ type Options struct { LeaderElection bool // LeaderElectionResourceLock determines which resource lock to use for leader election, - // defaults to "configmapsleases". Change this value only if you know what you are doing. - // Otherwise, users of your controller might end up with multiple running instances that + // defaults to "leases". Change this value only if you know what you are doing. + // + // If you are using `configmaps`/`endpoints` resource lock and want to migrate to "leases", + // you might do so by migrating to the respective multilock first ("configmapsleases" or "endpointsleases"), + // which will acquire a leader lock on both resources. + // After all your users have migrated to the multilock, you can go ahead and migrate to "leases". + // Please also keep in mind, that users might skip versions of your controller. + // + // Note: before controller-runtime version v0.7, it was set to "configmaps". + // And from v0.7 to v0.11, the default was "configmapsleases", which was + // used to migrate from configmaps to leases. + // Since the default was "configmapsleases" for over a year, spanning five minor releases, + // any actively maintained operators are very likely to have a released version that uses + // "configmapsleases". Therefore defaulting to "leases" should be safe since v0.12. + // + // So, what do you have to do when you are updating your controller-runtime dependency + // from a lower version to v0.12 or newer? + // - If your operator matches at least one of these conditions: + // - the LeaderElectionResourceLock in your operator has already been explicitly set to "leases" + // - the old controller-runtime version is between v0.7.0 and v0.11.x and the + // LeaderElectionResourceLock wasn't set or was set to "leases"/"configmapsleases"/"endpointsleases" + // feel free to update controller-runtime to v0.12 or newer. + // - Otherwise, you may have to take these steps: + // 1. update controller-runtime to v0.12 or newer in your go.mod + // 2. set LeaderElectionResourceLock to "configmapsleases" (or "endpointsleases") + // 3. package your operator and upgrade it in all your clusters + // 4. only if you have finished 3, you can remove the LeaderElectionResourceLock to use the default "leases" + // Otherwise, your operator might end up with multiple running instances that // each acquired leadership through different resource locks during upgrades and thus // act on the same resources concurrently. - // If you want to migrate to the "leases" resource lock, you might do so by migrating to the - // respective multilock first ("configmapsleases" or "endpointsleases"), which will acquire a - // leader lock on both resources. After all your users have migrated to the multilock, you can - // go ahead and migrate to "leases". Please also keep in mind, that users might skip versions - // of your controller. - // - // Note: before controller-runtime version v0.7, the resource lock was set to "configmaps". - // Please keep this in mind, when planning a proper migration path for your controller. LeaderElectionResourceLock string // LeaderElectionNamespace determines the namespace in which the leader @@ -174,6 +192,12 @@ type Options struct { // LeaseDuration time first. LeaderElectionReleaseOnCancel bool + // LeaderElectionResourceLockInterface allows to provide a custom resourcelock.Interface that was created outside + // of the controller-runtime. If this value is set the options LeaderElectionID, LeaderElectionNamespace, + // LeaderElectionResourceLock, LeaseDuration, RenewDeadline and RetryPeriod will be ignored. This can be useful if you + // want to use a locking mechanism that is currently not supported, like a MultiLock across two Kubernetes clusters. + LeaderElectionResourceLockInterface resourcelock.Interface + // LeaseDuration is the duration that non-leader candidates will // wait to force acquire leadership. This is measured against time of // last observed ack. Default is 15 seconds. @@ -185,11 +209,11 @@ type Options struct { // between tries of actions. Default is 2 seconds. RetryPeriod *time.Duration - // Namespace if specified restricts the manager's cache to watch objects in - // the desired namespace Defaults to all namespaces + // Namespace, if specified, restricts the manager's cache to watch objects in + // the desired namespace. Defaults to all namespaces. // // Note: If a namespace is specified, controllers can still Watch for a - // cluster-scoped resource (e.g Node). For namespaced resources the cache + // cluster-scoped resource (e.g Node). For namespaced resources, the cache // will only hold objects from the desired namespace. Namespace string @@ -200,6 +224,7 @@ type Options struct { // HealthProbeBindAddress is the TCP address that the controller should bind to // for serving health probes + // It can be set to "0" or "" to disable serving the health probe. HealthProbeBindAddress string // Readiness probe endpoint name, defaults to "readyz" @@ -222,21 +247,29 @@ type Options struct { // It is used to set webhook.Server.CertDir if WebhookServer is not set. CertDir string + // TLSOpts is used to allow configuring the TLS config used for the webhook server. + TLSOpts []func(*tls.Config) + // WebhookServer is an externally configured webhook.Server. By default, // a Manager will create a default server using Port, Host, and CertDir; // if this is set, the Manager will use this server instead. WebhookServer *webhook.Server - // Functions to all for a user to customize the values that will be injected. + // Functions to allow for a user to customize values that will be injected. // NewCache is the function that will create the cache to be used // by the manager. If not set this will use the default new cache function. NewCache cache.NewCacheFunc // NewClient is the func that creates the client to be used by the manager. - // If not set this will create the default DelegatingClient that will - // use the cache for reads and the client for writes. - NewClient cluster.NewClientFunc + // If not set this will create a Client backed by a Cache for read operations + // and a direct Client for write operations. + NewClient client.NewClientFunc + + // BaseContext is the function that provides Context values to Runnables + // managed by the Manager. If a BaseContext function isn't provided, Runnables + // will receive a new Background Context instead. + BaseContext BaseContextFunc // ClientDisableCacheFor tells the client that, if any cache is used, to bypass it // for the given objects. @@ -262,7 +295,7 @@ type Options struct { // Controller contains global configuration options for controllers // registered within this manager. // +optional - Controller v1alpha1.ControllerConfigurationSpec + Controller config.Controller // makeBroadcaster allows deferring the creation of the broadcaster to // avoid leaking goroutines if we never call Start on this manager. It also @@ -271,12 +304,16 @@ type Options struct { makeBroadcaster intrec.EventBroadcasterProducer // Dependency injection for testing - newRecorderProvider func(config *rest.Config, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster intrec.EventBroadcasterProducer) (*intrec.Provider, error) + newRecorderProvider func(config *rest.Config, httpClient *http.Client, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster intrec.EventBroadcasterProducer) (*intrec.Provider, error) newResourceLock func(config *rest.Config, recorderProvider recorder.Provider, options leaderelection.Options) (resourcelock.Interface, error) newMetricsListener func(addr string) (net.Listener, error) newHealthProbeListener func(addr string) (net.Listener, error) } +// BaseContextFunc is a function used to provide a base Context to Runnables +// managed by a Manager. +type BaseContextFunc func() context.Context + // Runnable allows a component to be started. // It's very important that Start blocks until // it's done running. @@ -311,15 +348,15 @@ func New(config *rest.Config, options Options) (Manager, error) { cluster, err := cluster.New(config, func(clusterOptions *cluster.Options) { clusterOptions.Scheme = options.Scheme - clusterOptions.MapperProvider = options.MapperProvider + clusterOptions.MapperProvider = options.MapperProvider //nolint:staticcheck clusterOptions.Logger = options.Logger clusterOptions.SyncPeriod = options.SyncPeriod - clusterOptions.Namespace = options.Namespace + clusterOptions.Namespace = options.Namespace //nolint:staticcheck clusterOptions.NewCache = options.NewCache clusterOptions.NewClient = options.NewClient - clusterOptions.ClientDisableCacheFor = options.ClientDisableCacheFor - clusterOptions.DryRunClient = options.DryRunClient - clusterOptions.EventBroadcaster = options.EventBroadcaster //nolint:staticcheck + clusterOptions.ClientDisableCacheFor = options.ClientDisableCacheFor //nolint:staticcheck + clusterOptions.DryRunClient = options.DryRunClient //nolint:staticcheck + clusterOptions.EventBroadcaster = options.EventBroadcaster //nolint:staticcheck }) if err != nil { return nil, err @@ -328,24 +365,39 @@ func New(config *rest.Config, options Options) (Manager, error) { // Create the recorder provider to inject event recorders for the components. // TODO(directxman12): the log for the event provider should have a context (name, tags, etc) specific // to the particular controller that it's being injected into, rather than a generic one like is here. - recorderProvider, err := options.newRecorderProvider(config, cluster.GetScheme(), options.Logger.WithName("events"), options.makeBroadcaster) + recorderProvider, err := options.newRecorderProvider(config, cluster.GetHTTPClient(), cluster.GetScheme(), options.Logger.WithName("events"), options.makeBroadcaster) if err != nil { return nil, err } // Create the resource lock to enable leader election) - leaderConfig := options.LeaderElectionConfig - if leaderConfig == nil { + var leaderConfig *rest.Config + var leaderRecorderProvider *intrec.Provider + + if options.LeaderElectionConfig == nil { leaderConfig = rest.CopyConfig(config) + leaderRecorderProvider = recorderProvider + } else { + leaderConfig = rest.CopyConfig(options.LeaderElectionConfig) + leaderRecorderProvider, err = options.newRecorderProvider(leaderConfig, cluster.GetHTTPClient(), cluster.GetScheme(), options.Logger.WithName("events"), options.makeBroadcaster) + if err != nil { + return nil, err + } } - resourceLock, err := options.newResourceLock(leaderConfig, recorderProvider, leaderelection.Options{ - LeaderElection: options.LeaderElection, - LeaderElectionResourceLock: options.LeaderElectionResourceLock, - LeaderElectionID: options.LeaderElectionID, - LeaderElectionNamespace: options.LeaderElectionNamespace, - }) - if err != nil { - return nil, err + + var resourceLock resourcelock.Interface + if options.LeaderElectionResourceLockInterface != nil && options.LeaderElection { + resourceLock = options.LeaderElectionResourceLockInterface + } else { + resourceLock, err = options.newResourceLock(leaderConfig, leaderRecorderProvider, leaderelection.Options{ + LeaderElection: options.LeaderElection, + LeaderElectionResourceLock: options.LeaderElectionResourceLock, + LeaderElectionID: options.LeaderElectionID, + LeaderElectionNamespace: options.LeaderElectionNamespace, + }) + if err != nil { + return nil, err + } } // Create the metrics listener. This will throw an error if the metrics bind @@ -365,19 +417,27 @@ func New(config *rest.Config, options Options) (Manager, error) { return nil, err } + errChan := make(chan error) + runnables := newRunnables(options.BaseContext, errChan) + return &controllerManager{ + stopProcedureEngaged: pointer.Int64(0), cluster: cluster, + runnables: runnables, + errChan: errChan, recorderProvider: recorderProvider, resourceLock: resourceLock, metricsListener: metricsListener, metricsExtraHandlers: metricsExtraHandlers, - controllerOptions: options.Controller, + controllerConfig: options.Controller, logger: options.Logger, elected: make(chan struct{}), port: options.Port, host: options.Host, certDir: options.CertDir, + tlsOpts: options.TLSOpts, webhookServer: options.WebhookServer, + leaderElectionID: options.LeaderElectionID, leaseDuration: *options.LeaseDuration, renewDeadline: *options.RenewDeadline, retryPeriod: *options.RetryPeriod, @@ -394,14 +454,9 @@ func New(config *rest.Config, options Options) (Manager, error) { // AndFrom will use a supplied type and convert to Options // any options already set on Options will be ignored, this is used to allow // cli flags to override anything specified in the config file. +// +// Deprecated: This function has been deprecated and will be removed in a future release. func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options, error) { - if inj, wantsScheme := loader.(inject.Scheme); wantsScheme { - err := inj.InjectScheme(o.Scheme) - if err != nil { - return o, err - } - } - newObj, err := loader.Complete() if err != nil { return o, err @@ -446,8 +501,8 @@ func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options, } if newObj.Controller != nil { - if o.Controller.CacheSyncTimeout == nil && newObj.Controller.CacheSyncTimeout != nil { - o.Controller.CacheSyncTimeout = newObj.Controller.CacheSyncTimeout + if o.Controller.CacheSyncTimeout == 0 && newObj.Controller.CacheSyncTimeout != nil { + o.Controller.CacheSyncTimeout = *newObj.Controller.CacheSyncTimeout } if len(o.Controller.GroupKindConcurrency) == 0 && len(newObj.Controller.GroupKindConcurrency) > 0 { @@ -459,6 +514,8 @@ func (o Options) AndFrom(loader config.ControllerManagerConfiguration) (Options, } // AndFromOrDie will use options.AndFrom() and will panic if there are errors. +// +// Deprecated: This function has been deprecated and will be removed in a future release. func (o Options) AndFromOrDie(loader config.ControllerManagerConfiguration) Options { o, err := o.AndFrom(loader) if err != nil { @@ -468,6 +525,11 @@ func (o Options) AndFromOrDie(loader config.ControllerManagerConfiguration) Opti } func (o Options) setLeaderElectionConfig(obj v1alpha1.ControllerManagerConfigurationSpec) Options { + if obj.LeaderElection == nil { + // The source does not have any configuration; noop + return o + } + if !o.LeaderElection && obj.LeaderElection.LeaderElect != nil { o.LeaderElection = *obj.LeaderElection.LeaderElect } @@ -507,11 +569,17 @@ func defaultHealthProbeListener(addr string) (net.Listener, error) { ln, err := net.Listen("tcp", addr) if err != nil { - return nil, fmt.Errorf("error listening on %s: %v", addr, err) + return nil, fmt.Errorf("error listening on %s: %w", addr, err) } return ln, nil } +// defaultBaseContext is used as the BaseContext value in Options if one +// has not already been set. +func defaultBaseContext() context.Context { + return context.Background() +} + // setOptionsDefaults set default values for Options fields. func setOptionsDefaults(options Options) Options { // Allow newResourceLock to be mocked @@ -571,8 +639,12 @@ func setOptionsDefaults(options Options) Options { options.GracefulShutdownTimeout = &gracefulShutdownTimeout } - if options.Logger == nil { - options.Logger = logf.RuntimeLog.WithName("manager") + if options.Logger.GetSink() == nil { + options.Logger = log.Log + } + + if options.BaseContext == nil { + options.BaseContext = defaultBaseContext } return options diff --git a/pkg/manager/manager_options_test.go b/pkg/manager/manager_options_test.go new file mode 100644 index 0000000000..3718bedcbe --- /dev/null +++ b/pkg/manager/manager_options_test.go @@ -0,0 +1,54 @@ +package manager + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "sigs.k8s.io/controller-runtime/pkg/config" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + configv1alpha1 "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" +) + +var _ = Describe("manager.Options", func() { + Describe("AndFrom", func() { + Describe("reading custom type using OfKind", func() { + var ( + o Options + c customConfig + err error + ) + + JustBeforeEach(func() { + s := runtime.NewScheme() + o = Options{Scheme: s} + c = customConfig{} + + _, err = o.AndFrom(config.File().AtPath("./testdata/custom-config.yaml").OfKind(&c)) + }) + + It("should not panic or fail", func() { + Expect(err).To(Succeed()) + }) + It("should set custom properties", func() { + Expect(c.CustomValue).To(Equal("foo")) + }) + }) + }) +}) + +type customConfig struct { + metav1.TypeMeta `json:",inline"` + configv1alpha1.ControllerManagerConfigurationSpec `json:",inline"` + CustomValue string `json:"customValue"` +} + +func (in *customConfig) DeepCopyObject() runtime.Object { + out := &customConfig{} + *out = *in + + in.ControllerManagerConfigurationSpec.DeepCopyInto(&out.ControllerManagerConfigurationSpec) + + return out +} diff --git a/pkg/manager/manager_suite_test.go b/pkg/manager/manager_suite_test.go index 6723d85149..ab514ef1e9 100644 --- a/pkg/manager/manager_suite_test.go +++ b/pkg/manager/manager_suite_test.go @@ -21,12 +21,11 @@ import ( "net/http" "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics" @@ -34,8 +33,7 @@ import ( func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Manager Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Manager Suite") } var testenv *envtest.Environment @@ -45,7 +43,7 @@ var clientset *kubernetes.Clientset // clientTransport is used to force-close keep-alives in tests that check for leaks. var clientTransport *http.Transport -var _ = BeforeSuite(func(done Done) { +var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) testenv = &envtest.Environment{} @@ -72,9 +70,7 @@ var _ = BeforeSuite(func(done Done) { // Prevent the metrics listener being created metrics.DefaultBindAddress = "0" - - close(done) -}, 60) +}) var _ = AfterSuite(func() { Expect(testenv.Stop()).To(Succeed()) diff --git a/pkg/manager/manager_test.go b/pkg/manager/manager_test.go index e930f26f92..6581bfc5ec 100644 --- a/pkg/manager/manager_test.go +++ b/pkg/manager/manager_test.go @@ -18,18 +18,20 @@ package manager import ( "context" + "crypto/tls" "errors" "fmt" - "io/ioutil" + "io" "net" "net/http" "path" + "reflect" "sync" "sync/atomic" "time" "github.com/go-logr/logr" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/prometheus/client_golang/prometheus" "go.uber.org/goleak" @@ -39,20 +41,18 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" "k8s.io/client-go/tools/leaderelection/resourcelock" + configv1alpha1 "k8s.io/component-base/config/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/cache/informertest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder" "sigs.k8s.io/controller-runtime/pkg/leaderelection" fakeleaderelection "sigs.k8s.io/controller-runtime/pkg/leaderelection/fake" "sigs.k8s.io/controller-runtime/pkg/metrics" - "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/recorder" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/webhook" ) @@ -68,27 +68,25 @@ var _ = Describe("manger.Manager", func() { It("should return an error if it can't create a RestMapper", func() { expected := fmt.Errorf("expected error: RestMapper") m, err := New(cfg, Options{ - MapperProvider: func(c *rest.Config) (meta.RESTMapper, error) { return nil, expected }, + MapperProvider: func(c *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) { return nil, expected }, }) Expect(m).To(BeNil()) Expect(err).To(Equal(expected)) }) - It("should return an error it can't create a client.Client", func(done Done) { + It("should return an error it can't create a client.Client", func() { m, err := New(cfg, Options{ - NewClient: func(cache cache.Cache, config *rest.Config, options client.Options, uncachedObjects ...client.Object) (client.Client, error) { + NewClient: func(config *rest.Config, options client.Options) (client.Client, error) { return nil, errors.New("expected error") }, }) Expect(m).To(BeNil()) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("expected error")) - - close(done) }) - It("should return an error it can't create a cache.Cache", func(done Done) { + It("should return an error it can't create a cache.Cache", func() { m, err := New(cfg, Options{ NewCache: func(config *rest.Config, opts cache.Options) (cache.Cache, error) { return nil, fmt.Errorf("expected error") @@ -97,37 +95,31 @@ var _ = Describe("manger.Manager", func() { Expect(m).To(BeNil()) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("expected error")) - - close(done) }) - It("should create a client defined in by the new client function", func(done Done) { + It("should create a client defined in by the new client function", func() { m, err := New(cfg, Options{ - NewClient: func(cache cache.Cache, config *rest.Config, options client.Options, uncachedObjects ...client.Object) (client.Client, error) { + NewClient: func(config *rest.Config, options client.Options) (client.Client, error) { return nil, nil }, }) Expect(m).ToNot(BeNil()) Expect(err).ToNot(HaveOccurred()) Expect(m.GetClient()).To(BeNil()) - - close(done) }) - It("should return an error it can't create a recorder.Provider", func(done Done) { + It("should return an error it can't create a recorder.Provider", func() { m, err := New(cfg, Options{ - newRecorderProvider: func(_ *rest.Config, _ *runtime.Scheme, _ logr.Logger, _ intrec.EventBroadcasterProducer) (*intrec.Provider, error) { + newRecorderProvider: func(_ *rest.Config, _ *http.Client, _ *runtime.Scheme, _ logr.Logger, _ intrec.EventBroadcasterProducer) (*intrec.Provider, error) { return nil, fmt.Errorf("expected error") }, }) Expect(m).To(BeNil()) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("expected error")) - - close(done) }) - It("should be able to load Options from cfg.ControllerManagerConfiguration type", func(done Done) { + It("should be able to load Options from cfg.ControllerManagerConfiguration type", func() { duration := metav1.Duration{Duration: 48 * time.Hour} port := int(6090) leaderElect := false @@ -180,11 +172,9 @@ var _ = Describe("manger.Manager", func() { Expect(m.Port).To(Equal(port)) Expect(m.Host).To(Equal("localhost")) Expect(m.CertDir).To(Equal("/certs")) - - close(done) }) - It("should be able to keep Options when cfg.ControllerManagerConfiguration set", func(done Done) { + It("should be able to keep Options when cfg.ControllerManagerConfiguration set", func() { optDuration := time.Duration(2) duration := metav1.Duration{Duration: 48 * time.Hour} port := int(6090) @@ -219,6 +209,9 @@ var _ = Describe("manger.Manager", func() { }, } + optionsTlSOptsFuncs := []func(*tls.Config){ + func(config *tls.Config) {}, + } m, err := Options{ SyncPeriod: &optDuration, LeaderElection: true, @@ -236,6 +229,7 @@ var _ = Describe("manger.Manager", func() { Port: 8080, Host: "example.com", CertDir: "/pki", + TLSOpts: optionsTlSOptsFuncs, }.AndFrom(&fakeDeferredLoader{ccfg}) Expect(err).To(BeNil()) @@ -255,11 +249,10 @@ var _ = Describe("manger.Manager", func() { Expect(m.Port).To(Equal(8080)) Expect(m.Host).To(Equal("example.com")) Expect(m.CertDir).To(Equal("/pki")) - - close(done) + Expect(m.TLSOpts).To(Equal(optionsTlSOptsFuncs)) }) - It("should lazily initialize a webhook server if needed", func(done Done) { + It("should lazily initialize a webhook server if needed", func() { By("creating a manager with options") m, err := New(cfg, Options{Port: 9440, Host: "foo.com"}) Expect(err).NotTo(HaveOccurred()) @@ -270,11 +263,9 @@ var _ = Describe("manger.Manager", func() { Expect(svr).NotTo(BeNil()) Expect(svr.Port).To(Equal(9440)) Expect(svr.Host).To(Equal("foo.com")) - - close(done) }) - It("should not initialize a webhook server if Options.WebhookServer is set", func(done Done) { + It("should not initialize a webhook server if Options.WebhookServer is set", func() { By("creating a manager with options") m, err := New(cfg, Options{Port: 9441, WebhookServer: &webhook.Server{Port: 9440}}) Expect(err).NotTo(HaveOccurred()) @@ -284,8 +275,6 @@ var _ = Describe("manger.Manager", func() { svr := m.GetWebhookServer() Expect(svr).NotTo(BeNil()) Expect(svr.Port).To(Equal(9440)) - - close(done) }) Context("with leader election enabled", func() { @@ -322,7 +311,7 @@ var _ = Describe("manger.Manager", func() { Expect(m.Start(ctx)).To(BeNil()) close(mgrDone) }() - <-cm.elected + <-cm.Elected() cancel() select { case <-leaderElectionDone: @@ -351,7 +340,7 @@ var _ = Describe("manger.Manager", func() { defer GinkgoRecover() err := m.Start(ctx) Expect(err).ToNot(BeNil()) - Expect(err.Error()).To(Equal("leader election lost")) + Expect(err.Error()).To(ContainSubstring("leader election lost")) close(mgrDone) }() cm := m.(*controllerManager) @@ -417,8 +406,8 @@ var _ = Describe("manger.Manager", func() { defer GinkgoRecover() Expect(m1.Elected()).ShouldNot(BeClosed()) Expect(m1.Start(ctx1)).NotTo(HaveOccurred()) - Expect(m1.Elected()).Should(BeClosed()) }() + <-m1.Elected() <-c1 c2 := make(chan struct{}) @@ -451,6 +440,7 @@ var _ = Describe("manger.Manager", func() { Expect(m).To(BeNil()) Expect(err).To(MatchError(ContainSubstring("expected error"))) }) + It("should return an error if namespace not set and not running in cluster", func() { m, err := New(cfg, Options{LeaderElection: true, LeaderElectionID: "controller-runtime"}) Expect(m).To(BeNil()) @@ -461,24 +451,20 @@ var _ = Describe("manger.Manager", func() { // We must keep this default until we are sure all controller-runtime users have upgraded from the original default // ConfigMap lock to a controller-runtime version that has this new default. Many users of controller-runtime skip // versions, so we should be extremely conservative here. - It("should default to ConfigMapsLeasesResourceLock", func() { + It("should default to LeasesResourceLock", func() { m, err := New(cfg, Options{LeaderElection: true, LeaderElectionID: "controller-runtime", LeaderElectionNamespace: "my-ns"}) Expect(m).ToNot(BeNil()) Expect(err).ToNot(HaveOccurred()) cm, ok := m.(*controllerManager) Expect(ok).To(BeTrue()) - multilock, isMultiLock := cm.resourceLock.(*resourcelock.MultiLock) - Expect(isMultiLock).To(BeTrue()) - _, primaryIsConfigMapLock := multilock.Primary.(*resourcelock.ConfigMapLock) - Expect(primaryIsConfigMapLock).To(BeTrue()) - _, secondaryIsLeaseLock := multilock.Secondary.(*resourcelock.LeaseLock) - Expect(secondaryIsLeaseLock).To(BeTrue()) + _, isLeaseLock := cm.resourceLock.(*resourcelock.LeaseLock) + Expect(isLeaseLock).To(BeTrue()) }) It("should use the specified ResourceLock", func() { m, err := New(cfg, Options{ LeaderElection: true, - LeaderElectionResourceLock: resourcelock.LeasesResourceLock, + LeaderElectionResourceLock: resourcelock.ConfigMapsLeasesResourceLock, LeaderElectionID: "controller-runtime", LeaderElectionNamespace: "my-ns", }) @@ -486,8 +472,14 @@ var _ = Describe("manger.Manager", func() { Expect(err).ToNot(HaveOccurred()) cm, ok := m.(*controllerManager) Expect(ok).To(BeTrue()) - _, isLeaseLock := cm.resourceLock.(*resourcelock.LeaseLock) - Expect(isLeaseLock).To(BeTrue()) + multilock, isMultiLock := cm.resourceLock.(*resourcelock.MultiLock) + Expect(isMultiLock).To(BeTrue()) + primaryLockType := reflect.TypeOf(multilock.Primary) + Expect(primaryLockType.Kind()).To(Equal(reflect.Ptr)) + Expect(primaryLockType.Elem().PkgPath()).To(Equal("k8s.io/client-go/tools/leaderelection/resourcelock")) + Expect(primaryLockType.Elem().Name()).To(Equal("configMapLock")) + _, secondaryIsLeaseLock := multilock.Secondary.(*resourcelock.LeaseLock) + Expect(secondaryIsLeaseLock).To(BeTrue()) }) It("should release lease if ElectionReleaseOnCancel is true", func() { var rl resourcelock.Interface @@ -522,6 +514,25 @@ var _ = Describe("manger.Manager", func() { Expect(err).To(BeNil()) Expect(record.HolderIdentity).To(BeEmpty()) }) + When("using a custom LeaderElectionResourceLockInterface", func() { + It("should use the custom LeaderElectionResourceLockInterface", func() { + rl, err := fakeleaderelection.NewResourceLock(nil, nil, leaderelection.Options{}) + Expect(err).NotTo(HaveOccurred()) + + m, err := New(cfg, Options{ + LeaderElection: true, + LeaderElectionResourceLockInterface: rl, + newResourceLock: func(config *rest.Config, recorderProvider recorder.Provider, options leaderelection.Options) (resourcelock.Interface, error) { + return nil, fmt.Errorf("this should not be called") + }, + }) + Expect(m).ToNot(BeNil()) + Expect(err).ToNot(HaveOccurred()) + cm, ok := m.(*controllerManager) + Expect(ok).To(BeTrue()) + Expect(cm.resourceLock).To(Equal(rl)) + }) + }) }) It("should create a listener for the metrics if a valid address is provided", func() { @@ -599,7 +610,7 @@ var _ = Describe("manger.Manager", func() { Describe("Start", func() { var startSuite = func(options Options, callbacks ...func(Manager)) { - It("should Start each Component", func(done Done) { + It("should Start each Component", func() { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -625,11 +636,10 @@ var _ = Describe("manger.Manager", func() { defer GinkgoRecover() Expect(m.Elected()).ShouldNot(BeClosed()) Expect(m.Start(ctx)).NotTo(HaveOccurred()) - Expect(m.Elected()).Should(BeClosed()) }() + <-m.Elected() wgRunnableStarted.Wait() - close(done) }) It("should not manipulate the provided config", func() { @@ -651,7 +661,7 @@ var _ = Describe("manger.Manager", func() { Expect(m.GetConfig()).To(Equal(originalCfg)) }) - It("should stop when context is cancelled", func(done Done) { + It("should stop when context is cancelled", func() { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -660,11 +670,9 @@ var _ = Describe("manger.Manager", func() { ctx, cancel := context.WithCancel(context.Background()) cancel() Expect(m.Start(ctx)).NotTo(HaveOccurred()) - - close(done) }) - It("should return an error if it can't start the cache", func(done Done) { + It("should return an error if it can't start the cache", func() { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -672,16 +680,16 @@ var _ = Describe("manger.Manager", func() { } mgr, ok := m.(*controllerManager) Expect(ok).To(BeTrue()) - mgr.caches = []hasCache{&cacheProvider{cache: &informertest.FakeInformers{Error: fmt.Errorf("expected error")}}} + Expect(mgr.Add( + &cacheProvider{cache: &informertest.FakeInformers{Error: fmt.Errorf("expected error")}}, + )).To(Succeed()) ctx, cancel := context.WithCancel(context.Background()) defer cancel() Expect(m.Start(ctx)).To(MatchError(ContainSubstring("expected error"))) - - close(done) }) - It("should start the cache before starting anything else", func(done Done) { + It("should start the cache before starting anything else", func() { fakeCache := &startSignalingInformer{Cache: &informertest.FakeInformers{}} options.NewCache = func(_ *rest.Config, _ cache.Options) (cache.Cache, error) { return fakeCache, nil @@ -693,14 +701,15 @@ var _ = Describe("manger.Manager", func() { } runnableWasStarted := make(chan struct{}) - Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + runnable := RunnableFunc(func(ctx context.Context) error { defer GinkgoRecover() if !fakeCache.wasSynced { return errors.New("runnable got started before cache was synced") } close(runnableWasStarted) return nil - }))).To(Succeed()) + }) + Expect(m.Add(runnable)).To(Succeed()) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -710,10 +719,9 @@ var _ = Describe("manger.Manager", func() { }() <-runnableWasStarted - close(done) }) - It("should start additional clusters before anything else", func(done Done) { + It("should start additional clusters before anything else", func() { fakeCache := &startSignalingInformer{Cache: &informertest.FakeInformers{}} options.NewCache = func(_ *rest.Config, _ cache.Options) (cache.Cache, error) { return fakeCache, nil @@ -754,10 +762,9 @@ var _ = Describe("manger.Manager", func() { }() <-runnableWasStarted - close(done) }) - It("should return an error if any Components fail to Start", func(done Done) { + It("should return an error if any Components fail to Start", func() { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -786,11 +793,52 @@ var _ = Describe("manger.Manager", func() { err = m.Start(ctx) Expect(err).ToNot(BeNil()) Expect(err.Error()).To(Equal("expected error")) + }) + + It("should start caches added after Manager has started", func() { + fakeCache := &startSignalingInformer{Cache: &informertest.FakeInformers{}} + options.NewCache = func(_ *rest.Config, _ cache.Options) (cache.Cache, error) { + return fakeCache, nil + } + m, err := New(cfg, options) + Expect(err).NotTo(HaveOccurred()) + for _, cb := range callbacks { + cb(m) + } + + runnableWasStarted := make(chan struct{}) + Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + defer GinkgoRecover() + if !fakeCache.wasSynced { + return errors.New("WaitForCacheSyncCalled wasn't called before Runnable got started") + } + close(runnableWasStarted) + return nil + }))).To(Succeed()) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + defer GinkgoRecover() + Expect(m.Start(ctx)).ToNot(HaveOccurred()) + }() + + <-runnableWasStarted + + additionalClusterCache := &startSignalingInformer{Cache: &informertest.FakeInformers{}} + fakeCluster := &startClusterAfterManager{informer: additionalClusterCache} + + Expect(err).NotTo(HaveOccurred()) + Expect(m.Add(fakeCluster)).NotTo(HaveOccurred()) - close(done) + Eventually(func() bool { + fakeCluster.informer.mu.Lock() + defer fakeCluster.informer.mu.Unlock() + return fakeCluster.informer.wasStarted && fakeCluster.informer.wasSynced + }).Should(BeTrue()) }) - It("should wait for runnables to stop", func(done Done) { + It("should wait for runnables to stop", func() { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -840,10 +888,9 @@ var _ = Describe("manger.Manager", func() { cancel() wgManagerRunning.Wait() - close(done) }) - It("should return an error if any Components fail to Start and wait for runnables to stop", func(done Done) { + It("should return an error if any Components fail to Start and wait for runnables to stop", func() { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -875,11 +922,9 @@ var _ = Describe("manger.Manager", func() { defer cancel() Expect(m.Start(ctx)).To(HaveOccurred()) Expect(runnableDoneCount).To(Equal(2)) - - close(done) }) - It("should refuse to add runnable if stop procedure is already engaged", func(done Done) { + It("should refuse to add runnable if stop procedure is already engaged", func() { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -907,11 +952,9 @@ var _ = Describe("manger.Manager", func() { defer GinkgoRecover() return nil }))).NotTo(Succeed()) - - close(done) }) - It("should return both runnables and stop errors when both error", func(done Done) { + It("should return both runnables and stop errors when both error", func() { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -942,11 +985,9 @@ var _ = Describe("manger.Manager", func() { Expect(err.Error()).To(Equal(eMsg)) Expect(errors.Is(err, context.DeadlineExceeded)).To(BeTrue()) Expect(errors.Is(err, runnableError{})).To(BeTrue()) - - close(done) }) - It("should return only stop errors if runnables dont error", func(done Done) { + It("should return only stop errors if runnables dont error", func() { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -982,11 +1023,9 @@ var _ = Describe("manger.Manager", func() { Expect(err.Error()).To(Equal("failed waiting for all runnables to end within grace period of 1ns: context deadline exceeded")) Expect(errors.Is(err, context.DeadlineExceeded)).To(BeTrue()) Expect(errors.Is(err, runnableError{})).ToNot(BeTrue()) - - close(done) }) - It("should return only runnables error if stop doesn't error", func(done Done) { + It("should return only runnables error if stop doesn't error", func() { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -1002,11 +1041,9 @@ var _ = Describe("manger.Manager", func() { Expect(err.Error()).To(Equal("not feeling like that")) Expect(errors.Is(err, context.DeadlineExceeded)).ToNot(BeTrue()) Expect(errors.Is(err, runnableError{})).To(BeTrue()) - - close(done) }) - It("should not wait for runnables if gracefulShutdownTimeout is 0", func(done Done) { + It("should not wait for runnables if gracefulShutdownTimeout is 0", func() { m, err := New(cfg, options) Expect(err).NotTo(HaveOccurred()) for _, cb := range callbacks { @@ -1025,15 +1062,59 @@ var _ = Describe("manger.Manager", func() { ctx, cancel := context.WithCancel(context.Background()) managerStopDone := make(chan struct{}) go func() { + defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) close(managerStopDone) }() - <-m.(*controllerManager).elected + <-m.Elected() cancel() <-managerStopDone <-runnableStopped - close(done) + }) + + It("should wait forever for runnables if gracefulShutdownTimeout is <0 (-1)", func() { + m, err := New(cfg, options) + Expect(err).NotTo(HaveOccurred()) + for _, cb := range callbacks { + cb(m) + } + m.(*controllerManager).gracefulShutdownTimeout = time.Duration(-1) + + Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + <-ctx.Done() + time.Sleep(100 * time.Millisecond) + return nil + }))).ToNot(HaveOccurred()) + Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + <-ctx.Done() + time.Sleep(200 * time.Millisecond) + return nil + }))).ToNot(HaveOccurred()) + Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + <-ctx.Done() + time.Sleep(500 * time.Millisecond) + return nil + }))).ToNot(HaveOccurred()) + Expect(m.Add(RunnableFunc(func(ctx context.Context) error { + <-ctx.Done() + time.Sleep(1500 * time.Millisecond) + return nil + }))).ToNot(HaveOccurred()) + + ctx, cancel := context.WithCancel(context.Background()) + managerStopDone := make(chan struct{}) + go func() { + defer GinkgoRecover() + Expect(m.Start(ctx)).NotTo(HaveOccurred()) + close(managerStopDone) + }() + <-m.Elected() + cancel() + + beforeDone := time.Now() + <-managerStopDone + Expect(time.Since(beforeDone)).To(BeNumerically(">=", 1500*time.Millisecond)) }) } @@ -1079,7 +1160,7 @@ var _ = Describe("manger.Manager", func() { } }) - It("should stop serving metrics when stop is called", func(done Done) { + It("should stop serving metrics when stop is called", func() { opts.MetricsBindAddress = ":0" m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) @@ -1088,7 +1169,6 @@ var _ = Describe("manger.Manager", func() { go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) - close(done) }() // Check the metrics started @@ -1106,7 +1186,7 @@ var _ = Describe("manger.Manager", func() { }).ShouldNot(Succeed()) }) - It("should serve metrics endpoint", func(done Done) { + It("should serve metrics endpoint", func() { opts.MetricsBindAddress = ":0" m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) @@ -1116,8 +1196,8 @@ var _ = Describe("manger.Manager", func() { go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) - close(done) }() + <-m.Elected() metricsEndpoint := fmt.Sprintf("http://%s/metrics", listener.Addr().String()) resp, err := http.Get(metricsEndpoint) @@ -1125,7 +1205,7 @@ var _ = Describe("manger.Manager", func() { Expect(resp.StatusCode).To(Equal(200)) }) - It("should not serve anything other than metrics endpoint by default", func(done Done) { + It("should not serve anything other than metrics endpoint by default", func() { opts.MetricsBindAddress = ":0" m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) @@ -1135,16 +1215,17 @@ var _ = Describe("manger.Manager", func() { go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) - close(done) }() + <-m.Elected() endpoint := fmt.Sprintf("http://%s/should-not-exist", listener.Addr().String()) resp, err := http.Get(endpoint) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(404)) }) - It("should serve metrics in its registry", func(done Done) { + It("should serve metrics in its registry", func() { one := prometheus.NewCounter(prometheus.CounterOpts{ Name: "test_one", Help: "test metric for testing", @@ -1162,15 +1243,16 @@ var _ = Describe("manger.Manager", func() { go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) - close(done) }() + <-m.Elected() metricsEndpoint := fmt.Sprintf("http://%s/metrics", listener.Addr().String()) resp, err := http.Get(metricsEndpoint) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(200)) - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) Expect(err).NotTo(HaveOccurred()) Expect(string(data)).To(ContainSubstring("%s\n%s\n%s\n", `# HELP test_one test metric for testing`, @@ -1183,7 +1265,7 @@ var _ = Describe("manger.Manager", func() { Expect(ok).To(BeTrue()) }) - It("should serve extra endpoints", func(done Done) { + It("should serve extra endpoints", func() { opts.MetricsBindAddress = ":0" m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) @@ -1204,15 +1286,16 @@ var _ = Describe("manger.Manager", func() { go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) - close(done) }() + <-m.Elected() endpoint := fmt.Sprintf("http://%s/debug", listener.Addr().String()) resp, err := http.Get(endpoint) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(http.StatusOK)) - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) Expect(err).NotTo(HaveOccurred()) Expect(string(body)).To(Equal("Some debug info")) }) @@ -1240,7 +1323,7 @@ var _ = Describe("manger.Manager", func() { } }) - It("should stop serving health probes when stop is called", func(done Done) { + It("should stop serving health probes when stop is called", func() { opts.HealthProbeBindAddress = ":0" m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) @@ -1249,8 +1332,8 @@ var _ = Describe("manger.Manager", func() { go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) - close(done) }() + <-m.Elected() // Check the health probes started endpoint := fmt.Sprintf("http://%s", listener.Addr().String()) @@ -1264,10 +1347,10 @@ var _ = Describe("manger.Manager", func() { Eventually(func() error { _, err = http.Get(endpoint) return err - }).ShouldNot(Succeed()) + }, 10*time.Second).ShouldNot(Succeed()) }) - It("should serve readiness endpoint", func(done Done) { + It("should serve readiness endpoint", func() { opts.HealthProbeBindAddress = ":0" m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) @@ -1282,20 +1365,22 @@ var _ = Describe("manger.Manager", func() { go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) - close(done) }() + <-m.Elected() readinessEndpoint := fmt.Sprint("http://", listener.Addr().String(), defaultReadinessEndpoint) // Controller is not ready resp, err := http.Get(readinessEndpoint) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(http.StatusInternalServerError)) // Controller is ready res = nil resp, err = http.Get(readinessEndpoint) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(http.StatusOK)) // Check readiness path without trailing slash without redirect @@ -1308,6 +1393,7 @@ var _ = Describe("manger.Manager", func() { } resp, err = httpClient.Get(readinessEndpoint) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(http.StatusOK)) // Check readiness path for individual check @@ -1315,10 +1401,11 @@ var _ = Describe("manger.Manager", func() { res = nil resp, err = http.Get(readinessEndpoint) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(http.StatusOK)) }) - It("should serve liveness endpoint", func(done Done) { + It("should serve liveness endpoint", func() { opts.HealthProbeBindAddress = ":0" m, err := New(cfg, opts) Expect(err).NotTo(HaveOccurred()) @@ -1333,20 +1420,22 @@ var _ = Describe("manger.Manager", func() { go func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) - close(done) }() + <-m.Elected() livenessEndpoint := fmt.Sprint("http://", listener.Addr().String(), defaultLivenessEndpoint) // Controller is not ready resp, err := http.Get(livenessEndpoint) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(http.StatusInternalServerError)) // Controller is ready res = nil resp, err = http.Get(livenessEndpoint) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(http.StatusOK)) // Check liveness path without trailing slash without redirect @@ -1359,6 +1448,7 @@ var _ = Describe("manger.Manager", func() { } resp, err = httpClient.Get(livenessEndpoint) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(http.StatusOK)) // Check readiness path for individual check @@ -1366,13 +1456,14 @@ var _ = Describe("manger.Manager", func() { res = nil resp, err = http.Get(livenessEndpoint) Expect(err).NotTo(HaveOccurred()) + defer resp.Body.Close() Expect(resp.StatusCode).To(Equal(http.StatusOK)) }) }) Describe("Add", func() { It("should immediately start the Component if the Manager has already Started another Component", - func(done Done) { + func() { m, err := New(cfg, Options{}) Expect(err).NotTo(HaveOccurred()) mgr, ok := m.(*controllerManager) @@ -1392,12 +1483,11 @@ var _ = Describe("manger.Manager", func() { defer GinkgoRecover() Expect(m.Start(ctx)).NotTo(HaveOccurred()) }() + <-m.Elected() // Wait for the Manager to start Eventually(func() bool { - mgr.mu.Lock() - defer mgr.mu.Unlock() - return mgr.started + return mgr.runnables.Caches.Started() }).Should(BeTrue()) // Add another component after starting @@ -1409,11 +1499,9 @@ var _ = Describe("manger.Manager", func() { }))).To(Succeed()) <-c1 <-c2 - - close(done) }) - It("should immediately start the Component if the Manager has already Started", func(done Done) { + It("should immediately start the Component if the Manager has already Started", func() { m, err := New(cfg, Options{}) Expect(err).NotTo(HaveOccurred()) mgr, ok := m.(*controllerManager) @@ -1428,9 +1516,7 @@ var _ = Describe("manger.Manager", func() { // Wait for the Manager to start Eventually(func() bool { - mgr.mu.Lock() - defer mgr.mu.Unlock() - return mgr.started + return mgr.runnables.Caches.Started() }).Should(BeTrue()) c1 := make(chan struct{}) @@ -1440,110 +1526,29 @@ var _ = Describe("manger.Manager", func() { return nil }))).To(Succeed()) <-c1 - - close(done) }) - It("should fail if SetFields fails", func() { + It("should fail if attempted to start a second time", func() { m, err := New(cfg, Options{}) Expect(err).NotTo(HaveOccurred()) - Expect(m.Add(&failRec{})).To(HaveOccurred()) - }) - }) - Describe("SetFields", func() { - It("should inject field values", func(done Done) { - m, err := New(cfg, Options{ - NewCache: func(_ *rest.Config, _ cache.Options) (cache.Cache, error) { - return &informertest.FakeInformers{}, nil - }, - }) - Expect(err).NotTo(HaveOccurred()) - By("Injecting the dependencies") - err = m.SetFields(&injectable{ - scheme: func(scheme *runtime.Scheme) error { - defer GinkgoRecover() - Expect(scheme).To(Equal(m.GetScheme())) - return nil - }, - config: func(config *rest.Config) error { - defer GinkgoRecover() - Expect(config).To(Equal(m.GetConfig())) - return nil - }, - client: func(client client.Client) error { - defer GinkgoRecover() - Expect(client).To(Equal(m.GetClient())) - return nil - }, - cache: func(c cache.Cache) error { - defer GinkgoRecover() - Expect(c).To(Equal(m.GetCache())) - return nil - }, - stop: func(stop <-chan struct{}) error { - defer GinkgoRecover() - Expect(stop).NotTo(BeNil()) - return nil - }, - f: func(f inject.Func) error { - defer GinkgoRecover() - Expect(f).NotTo(BeNil()) - return nil - }, - log: func(logger logr.Logger) error { - defer GinkgoRecover() - Expect(logger).To(Equal(logf.RuntimeLog.WithName("manager"))) - return nil - }, - }) - Expect(err).NotTo(HaveOccurred()) - - By("Returning an error if dependency injection fails") - - expected := fmt.Errorf("expected error") - err = m.SetFields(&injectable{ - client: func(client client.Client) error { - return expected - }, - }) - Expect(err).To(Equal(expected)) - - err = m.SetFields(&injectable{ - scheme: func(scheme *runtime.Scheme) error { - return expected - }, - }) - Expect(err).To(Equal(expected)) - - err = m.SetFields(&injectable{ - config: func(config *rest.Config) error { - return expected - }, - }) - Expect(err).To(Equal(expected)) - - err = m.SetFields(&injectable{ - cache: func(c cache.Cache) error { - return expected - }, - }) - Expect(err).To(Equal(expected)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + defer GinkgoRecover() + Expect(m.Start(ctx)).NotTo(HaveOccurred()) + }() + // Wait for the Manager to start + Eventually(func() bool { + mgr, ok := m.(*controllerManager) + Expect(ok).To(BeTrue()) + return mgr.runnables.Caches.Started() + }).Should(BeTrue()) - err = m.SetFields(&injectable{ - f: func(c inject.Func) error { - return expected - }, - }) - Expect(err).To(Equal(expected)) + err = m.Start(ctx) + Expect(err).ToNot(BeNil()) + Expect(err.Error()).To(Equal("manager already started")) - err = m.SetFields(&injectable{ - stop: func(<-chan struct{}) error { - return expected - }, - }) - Expect(err).To(Equal(expected)) - close(done) }) }) @@ -1587,6 +1592,8 @@ var _ = Describe("manger.Manager", func() { defer close(doneCh) Expect(m.Start(ctx)).To(Succeed()) }() + <-m.Elected() + Eventually(func() *corev1.Event { evts, err := clientset.CoreV1().Events("").Search(m.GetScheme(), &ns) Expect(err).NotTo(HaveOccurred()) @@ -1653,94 +1660,6 @@ var _ = Describe("manger.Manager", func() { }) }) -var _ reconcile.Reconciler = &failRec{} -var _ inject.Client = &failRec{} - -type failRec struct{} - -func (*failRec) Reconcile(context.Context, reconcile.Request) (reconcile.Result, error) { - return reconcile.Result{}, nil -} - -func (*failRec) Start(context.Context) error { - return nil -} - -func (*failRec) InjectClient(client.Client) error { - return fmt.Errorf("expected error") -} - -var _ inject.Injector = &injectable{} -var _ inject.Cache = &injectable{} -var _ inject.Client = &injectable{} -var _ inject.Scheme = &injectable{} -var _ inject.Config = &injectable{} -var _ inject.Stoppable = &injectable{} -var _ inject.Logger = &injectable{} - -type injectable struct { - scheme func(scheme *runtime.Scheme) error - client func(client.Client) error - config func(config *rest.Config) error - cache func(cache.Cache) error - f func(inject.Func) error - stop func(<-chan struct{}) error - log func(logger logr.Logger) error -} - -func (i *injectable) InjectCache(c cache.Cache) error { - if i.cache == nil { - return nil - } - return i.cache(c) -} - -func (i *injectable) InjectConfig(config *rest.Config) error { - if i.config == nil { - return nil - } - return i.config(config) -} - -func (i *injectable) InjectClient(c client.Client) error { - if i.client == nil { - return nil - } - return i.client(c) -} - -func (i *injectable) InjectScheme(scheme *runtime.Scheme) error { - if i.scheme == nil { - return nil - } - return i.scheme(scheme) -} - -func (i *injectable) InjectFunc(f inject.Func) error { - if i.f == nil { - return nil - } - return i.f(f) -} - -func (i *injectable) InjectStopChannel(stop <-chan struct{}) error { - if i.stop == nil { - return nil - } - return i.stop(stop) -} - -func (i *injectable) InjectLogger(log logr.Logger) error { - if i.log == nil { - return nil - } - return i.log(log) -} - -func (i *injectable) Start(<-chan struct{}) error { - return nil -} - type runnableError struct { } @@ -1748,18 +1667,6 @@ func (runnableError) Error() string { return "not feeling like that" } -type fakeDeferredLoader struct { - *v1alpha1.ControllerManagerConfiguration -} - -func (f *fakeDeferredLoader) Complete() (v1alpha1.ControllerManagerConfigurationSpec, error) { - return f.ControllerManagerConfiguration.ControllerManagerConfigurationSpec, nil -} - -func (f *fakeDeferredLoader) InjectScheme(scheme *runtime.Scheme) error { - return nil -} - var _ Runnable = &cacheProvider{} type cacheProvider struct { @@ -1775,36 +1682,54 @@ func (c *cacheProvider) Start(ctx context.Context) error { } type startSignalingInformer struct { + mu sync.Mutex + // The manager calls Start and WaitForCacheSync in // parallel, so we have to protect wasStarted with a Mutex // and block in WaitForCacheSync until it is true. - wasStartedLock sync.Mutex - wasStarted bool + wasStarted bool // was synced will be true once Start was called and // WaitForCacheSync returned, just like a real cache. wasSynced bool cache.Cache } -func (c *startSignalingInformer) started() bool { - c.wasStartedLock.Lock() - defer c.wasStartedLock.Unlock() - return c.wasStarted -} - func (c *startSignalingInformer) Start(ctx context.Context) error { - c.wasStartedLock.Lock() + c.mu.Lock() c.wasStarted = true - c.wasStartedLock.Unlock() + c.mu.Unlock() return c.Cache.Start(ctx) } func (c *startSignalingInformer) WaitForCacheSync(ctx context.Context) bool { defer func() { - for !c.started() { - continue - } + c.mu.Lock() c.wasSynced = true + c.mu.Unlock() }() return c.Cache.WaitForCacheSync(ctx) } + +type startClusterAfterManager struct { + informer *startSignalingInformer +} + +func (c *startClusterAfterManager) Start(ctx context.Context) error { + return c.informer.Start(ctx) +} + +func (c *startClusterAfterManager) GetCache() cache.Cache { + return c.informer +} + +type fakeDeferredLoader struct { + *v1alpha1.ControllerManagerConfiguration +} + +func (f *fakeDeferredLoader) Complete() (v1alpha1.ControllerManagerConfigurationSpec, error) { + return f.ControllerManagerConfiguration.ControllerManagerConfigurationSpec, nil +} + +func (f *fakeDeferredLoader) InjectScheme(scheme *runtime.Scheme) error { + return nil +} diff --git a/pkg/manager/runnable_group.go b/pkg/manager/runnable_group.go new file mode 100644 index 0000000000..f7b91a209f --- /dev/null +++ b/pkg/manager/runnable_group.go @@ -0,0 +1,297 @@ +package manager + +import ( + "context" + "errors" + "sync" + + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var ( + errRunnableGroupStopped = errors.New("can't accept new runnable as stop procedure is already engaged") +) + +// readyRunnable encapsulates a runnable with +// a ready check. +type readyRunnable struct { + Runnable + Check runnableCheck + signalReady bool +} + +// runnableCheck can be passed to Add() to let the runnable group determine that a +// runnable is ready. A runnable check should block until a runnable is ready, +// if the returned result is false, the runnable is considered not ready and failed. +type runnableCheck func(ctx context.Context) bool + +// runnables handles all the runnables for a manager by grouping them accordingly to their +// type (webhooks, caches etc.). +type runnables struct { + Webhooks *runnableGroup + Caches *runnableGroup + LeaderElection *runnableGroup + Others *runnableGroup +} + +// newRunnables creates a new runnables object. +func newRunnables(baseContext BaseContextFunc, errChan chan error) *runnables { + return &runnables{ + Webhooks: newRunnableGroup(baseContext, errChan), + Caches: newRunnableGroup(baseContext, errChan), + LeaderElection: newRunnableGroup(baseContext, errChan), + Others: newRunnableGroup(baseContext, errChan), + } +} + +// Add adds a runnable to closest group of runnable that they belong to. +// +// Add should be able to be called before and after Start, but not after StopAndWait. +// Add should return an error when called during StopAndWait. +// The runnables added before Start are started when Start is called. +// The runnables added after Start are started directly. +func (r *runnables) Add(fn Runnable) error { + switch runnable := fn.(type) { + case hasCache: + return r.Caches.Add(fn, func(ctx context.Context) bool { + return runnable.GetCache().WaitForCacheSync(ctx) + }) + case *webhook.Server: + return r.Webhooks.Add(fn, nil) + case LeaderElectionRunnable: + if !runnable.NeedLeaderElection() { + return r.Others.Add(fn, nil) + } + return r.LeaderElection.Add(fn, nil) + default: + return r.LeaderElection.Add(fn, nil) + } +} + +// runnableGroup manages a group of runnables that are +// meant to be running together until StopAndWait is called. +// +// Runnables can be added to a group after the group has started +// but not after it's stopped or while shutting down. +type runnableGroup struct { + ctx context.Context + cancel context.CancelFunc + + start sync.Mutex + startOnce sync.Once + started bool + startQueue []*readyRunnable + startReadyCh chan *readyRunnable + + stop sync.RWMutex + stopOnce sync.Once + stopped bool + + // errChan is the error channel passed by the caller + // when the group is created. + // All errors are forwarded to this channel once they occur. + errChan chan error + + // ch is the internal channel where the runnables are read off from. + ch chan *readyRunnable + + // wg is an internal sync.WaitGroup that allows us to properly stop + // and wait for all the runnables to finish before returning. + wg *sync.WaitGroup +} + +func newRunnableGroup(baseContext BaseContextFunc, errChan chan error) *runnableGroup { + r := &runnableGroup{ + startReadyCh: make(chan *readyRunnable), + errChan: errChan, + ch: make(chan *readyRunnable), + wg: new(sync.WaitGroup), + } + + r.ctx, r.cancel = context.WithCancel(baseContext()) + return r +} + +// Started returns true if the group has started. +func (r *runnableGroup) Started() bool { + r.start.Lock() + defer r.start.Unlock() + return r.started +} + +// Start starts the group and waits for all +// initially registered runnables to start. +// It can only be called once, subsequent calls have no effect. +func (r *runnableGroup) Start(ctx context.Context) error { + var retErr error + + r.startOnce.Do(func() { + defer close(r.startReadyCh) + + // Start the internal reconciler. + go r.reconcile() + + // Start the group and queue up all + // the runnables that were added prior. + r.start.Lock() + r.started = true + for _, rn := range r.startQueue { + rn.signalReady = true + r.ch <- rn + } + r.start.Unlock() + + // If we don't have any queue, return. + if len(r.startQueue) == 0 { + return + } + + // Wait for all runnables to signal. + for { + select { + case <-ctx.Done(): + if err := ctx.Err(); !errors.Is(err, context.Canceled) { + retErr = err + } + case rn := <-r.startReadyCh: + for i, existing := range r.startQueue { + if existing == rn { + // Remove the item from the start queue. + r.startQueue = append(r.startQueue[:i], r.startQueue[i+1:]...) + break + } + } + // We're done waiting if the queue is empty, return. + if len(r.startQueue) == 0 { + return + } + } + } + }) + + return retErr +} + +// reconcile is our main entrypoint for every runnable added +// to this group. Its primary job is to read off the internal channel +// and schedule runnables while tracking their state. +func (r *runnableGroup) reconcile() { + for runnable := range r.ch { + // Handle stop. + // If the shutdown has been called we want to avoid + // adding new goroutines to the WaitGroup because Wait() + // panics if Add() is called after it. + { + r.stop.RLock() + if r.stopped { + // Drop any runnables if we're stopped. + r.errChan <- errRunnableGroupStopped + r.stop.RUnlock() + continue + } + + // Why is this here? + // When StopAndWait is called, if a runnable is in the process + // of being added, we could end up in a situation where + // the WaitGroup is incremented while StopAndWait has called Wait(), + // which would result in a panic. + r.wg.Add(1) + r.stop.RUnlock() + } + + // Start the runnable. + go func(rn *readyRunnable) { + go func() { + if rn.Check(r.ctx) { + if rn.signalReady { + r.startReadyCh <- rn + } + } + }() + + // If we return, the runnable ended cleanly + // or returned an error to the channel. + // + // We should always decrement the WaitGroup here. + defer r.wg.Done() + + // Start the runnable. + if err := rn.Start(r.ctx); err != nil { + r.errChan <- err + } + }(runnable) + } +} + +// Add should be able to be called before and after Start, but not after StopAndWait. +// Add should return an error when called during StopAndWait. +func (r *runnableGroup) Add(rn Runnable, ready runnableCheck) error { + r.stop.RLock() + if r.stopped { + r.stop.RUnlock() + return errRunnableGroupStopped + } + r.stop.RUnlock() + + if ready == nil { + ready = func(_ context.Context) bool { return true } + } + + readyRunnable := &readyRunnable{ + Runnable: rn, + Check: ready, + } + + // Handle start. + // If the overall runnable group isn't started yet + // we want to buffer the runnables and let Start() + // queue them up again later. + { + r.start.Lock() + + // Check if we're already started. + if !r.started { + // Store the runnable in the internal if not. + r.startQueue = append(r.startQueue, readyRunnable) + r.start.Unlock() + return nil + } + r.start.Unlock() + } + + // Enqueue the runnable. + r.ch <- readyRunnable + return nil +} + +// StopAndWait waits for all the runnables to finish before returning. +func (r *runnableGroup) StopAndWait(ctx context.Context) { + r.stopOnce.Do(func() { + // Close the reconciler channel once we're done. + defer close(r.ch) + + _ = r.Start(ctx) + r.stop.Lock() + // Store the stopped variable so we don't accept any new + // runnables for the time being. + r.stopped = true + r.stop.Unlock() + + // Cancel the internal channel. + r.cancel() + + done := make(chan struct{}) + go func() { + defer close(done) + // Wait for all the runnables to finish. + r.wg.Wait() + }() + + select { + case <-done: + // We're done, exit. + case <-ctx.Done(): + // Calling context has expired, exit. + } + }) +} diff --git a/pkg/manager/runnable_group_test.go b/pkg/manager/runnable_group_test.go new file mode 100644 index 0000000000..2122f23656 --- /dev/null +++ b/pkg/manager/runnable_group_test.go @@ -0,0 +1,182 @@ +package manager + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/cache/informertest" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var _ = Describe("runnables", func() { + errCh := make(chan error) + + It("should be able to create a new runnables object", func() { + Expect(newRunnables(defaultBaseContext, errCh)).ToNot(BeNil()) + }) + + It("should add caches to the appropriate group", func() { + cache := &cacheProvider{cache: &informertest.FakeInformers{Error: fmt.Errorf("expected error")}} + r := newRunnables(defaultBaseContext, errCh) + Expect(r.Add(cache)).To(Succeed()) + Expect(r.Caches.startQueue).To(HaveLen(1)) + }) + + It("should add webhooks to the appropriate group", func() { + webhook := &webhook.Server{} + r := newRunnables(defaultBaseContext, errCh) + Expect(r.Add(webhook)).To(Succeed()) + Expect(r.Webhooks.startQueue).To(HaveLen(1)) + }) + + It("should add any runnable to the leader election group", func() { + err := errors.New("runnable func") + runnable := RunnableFunc(func(c context.Context) error { + return err + }) + + r := newRunnables(defaultBaseContext, errCh) + Expect(r.Add(runnable)).To(Succeed()) + Expect(r.LeaderElection.startQueue).To(HaveLen(1)) + }) +}) + +var _ = Describe("runnableGroup", func() { + errCh := make(chan error) + + It("should be able to add new runnables before it starts", func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rg := newRunnableGroup(defaultBaseContext, errCh) + Expect(rg.Add(RunnableFunc(func(c context.Context) error { + <-ctx.Done() + return nil + }), nil)).To(Succeed()) + + Expect(rg.Started()).To(BeFalse()) + }) + + It("should be able to add new runnables before and after start", func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rg := newRunnableGroup(defaultBaseContext, errCh) + Expect(rg.Add(RunnableFunc(func(c context.Context) error { + <-ctx.Done() + return nil + }), nil)).To(Succeed()) + Expect(rg.Start(ctx)).To(Succeed()) + Expect(rg.Started()).To(BeTrue()) + Expect(rg.Add(RunnableFunc(func(c context.Context) error { + <-ctx.Done() + return nil + }), nil)).To(Succeed()) + }) + + It("should be able to add new runnables before and after start concurrently", func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rg := newRunnableGroup(defaultBaseContext, errCh) + + go func() { + defer GinkgoRecover() + <-time.After(50 * time.Millisecond) + Expect(rg.Start(ctx)).To(Succeed()) + }() + + for i := 0; i < 20; i++ { + go func(i int) { + defer GinkgoRecover() + + <-time.After(time.Duration(i) * 10 * time.Millisecond) + Expect(rg.Add(RunnableFunc(func(c context.Context) error { + <-ctx.Done() + return nil + }), nil)).To(Succeed()) + }(i) + } + }) + + It("should be able to close the group and wait for all runnables to finish", func() { + ctx, cancel := context.WithCancel(context.Background()) + + exited := pointer.Int64(0) + rg := newRunnableGroup(defaultBaseContext, errCh) + for i := 0; i < 10; i++ { + Expect(rg.Add(RunnableFunc(func(c context.Context) error { + defer atomic.AddInt64(exited, 1) + <-ctx.Done() + <-time.After(time.Duration(i) * 10 * time.Millisecond) + return nil + }), nil)).To(Succeed()) + } + Expect(rg.Start(ctx)).To(Succeed()) + + // Cancel the context, asking the runnables to exit. + cancel() + rg.StopAndWait(context.Background()) + + Expect(rg.Add(RunnableFunc(func(c context.Context) error { + return nil + }), nil)).ToNot(Succeed()) + + Expect(atomic.LoadInt64(exited)).To(BeNumerically("==", 10)) + }) + + It("should be able to wait for all runnables to be ready at different intervals", func() { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + rg := newRunnableGroup(defaultBaseContext, errCh) + + go func() { + defer GinkgoRecover() + <-time.After(50 * time.Millisecond) + Expect(rg.Start(ctx)).To(Succeed()) + }() + + for i := 0; i < 20; i++ { + go func(i int) { + defer GinkgoRecover() + + Expect(rg.Add(RunnableFunc(func(c context.Context) error { + <-ctx.Done() + return nil + }), func(_ context.Context) bool { + <-time.After(time.Duration(i) * 10 * time.Millisecond) + return true + })).To(Succeed()) + }(i) + } + }) + + It("should not turn ready if some readiness check fail", func() { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + rg := newRunnableGroup(defaultBaseContext, errCh) + + go func() { + defer GinkgoRecover() + <-time.After(50 * time.Millisecond) + Expect(rg.Start(ctx)).To(Succeed()) + }() + + for i := 0; i < 20; i++ { + go func(i int) { + defer GinkgoRecover() + + Expect(rg.Add(RunnableFunc(func(c context.Context) error { + <-ctx.Done() + return nil + }), func(_ context.Context) bool { + <-time.After(time.Duration(i) * 10 * time.Millisecond) + return i%2 == 0 // Return false readiness all uneven indexes. + })).To(Succeed()) + }(i) + } + }) +}) diff --git a/pkg/manager/signals/signal.go b/pkg/manager/signals/signal.go index 9a85558f82..a79cfb42df 100644 --- a/pkg/manager/signals/signal.go +++ b/pkg/manager/signals/signal.go @@ -24,8 +24,8 @@ import ( var onlyOneSignalHandler = make(chan struct{}) -// SetupSignalHandler registers for SIGTERM and SIGINT. A stop channel is returned -// which is closed on one of these signals. If a second signal is caught, the program +// SetupSignalHandler registers for SIGTERM and SIGINT. A context is returned +// which is canceled on one of these signals. If a second signal is caught, the program // is terminated with exit code 1. func SetupSignalHandler() context.Context { close(onlyOneSignalHandler) // panics when called twice diff --git a/pkg/manager/signals/signal_posix.go b/pkg/manager/signals/signal_posix.go index 9bdb4e7418..a0f00a7321 100644 --- a/pkg/manager/signals/signal_posix.go +++ b/pkg/manager/signals/signal_posix.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows /* diff --git a/pkg/manager/signals/signal_test.go b/pkg/manager/signals/signal_test.go index 2776e13a6d..134937e012 100644 --- a/pkg/manager/signals/signal_test.go +++ b/pkg/manager/signals/signal_test.go @@ -23,7 +23,7 @@ import ( "sync" "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) diff --git a/pkg/manager/signals/signals_suite_test.go b/pkg/manager/signals/signals_suite_test.go index 770df0ca9c..bae6d72ed5 100644 --- a/pkg/manager/signals/signals_suite_test.go +++ b/pkg/manager/signals/signals_suite_test.go @@ -20,15 +20,13 @@ import ( "os/signal" "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Runtime Signal Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Runtime Signal Suite") } var _ = BeforeSuite(func() { diff --git a/pkg/manager/testdata/custom-config.yaml b/pkg/manager/testdata/custom-config.yaml new file mode 100644 index 0000000000..a15c9f8e5c --- /dev/null +++ b/pkg/manager/testdata/custom-config.yaml @@ -0,0 +1,3 @@ +apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 +kind: CustomControllerManagerConfiguration +customValue: foo diff --git a/pkg/metrics/client_go_adapter.go b/pkg/metrics/client_go_adapter.go index 90754269dd..dc805a9d04 100644 --- a/pkg/metrics/client_go_adapter.go +++ b/pkg/metrics/client_go_adapter.go @@ -22,7 +22,6 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" - reflectormetrics "k8s.io/client-go/tools/cache" clientmetrics "k8s.io/client-go/tools/metrics" ) @@ -37,22 +36,26 @@ const ( ResultKey = "requests_total" ) -// Metrics subsystem and all keys used by the reflectors. -const ( - ReflectorSubsystem = "reflector" - ListsTotalKey = "lists_total" - ListsDurationKey = "list_duration_seconds" - ItemsPerListKey = "items_per_list" - WatchesTotalKey = "watches_total" - ShortWatchesTotalKey = "short_watches_total" - WatchDurationKey = "watch_duration_seconds" - ItemsPerWatchKey = "items_per_watch" - LastResourceVersionKey = "last_resource_version" -) - var ( // client metrics. - requestLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + + // RequestLatency reports the request latency in seconds per verb/URL. + // Deprecated: This metric is deprecated for removal in a future release: using the URL as a + // dimension results in cardinality explosion for some consumers. It was deprecated upstream + // in k8s v1.14 and hidden in v1.17 via https://github.com/kubernetes/kubernetes/pull/83836. + // It is not registered by default. To register: + // import ( + // clientmetrics "k8s.io/client-go/tools/metrics" + // clmetrics "sigs.k8s.io/controller-runtime/metrics" + // ) + // + // func init() { + // clmetrics.Registry.MustRegister(clmetrics.RequestLatency) + // clientmetrics.Register(clientmetrics.RegisterOpts{ + // RequestLatency: clmetrics.LatencyAdapter + // }) + // } + RequestLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Subsystem: RestClientSubsystem, Name: LatencyKey, Help: "Request latency in seconds. Broken down by verb and URL.", @@ -64,93 +67,23 @@ var ( Name: ResultKey, Help: "Number of HTTP requests, partitioned by status code, method, and host.", }, []string{"code", "method", "host"}) - - // reflector metrics. - - // TODO(directxman12): update these to be histograms once the metrics overhaul KEP - // PRs start landing. - - listsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: ReflectorSubsystem, - Name: ListsTotalKey, - Help: "Total number of API lists done by the reflectors", - }, []string{"name"}) - - listsDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Subsystem: ReflectorSubsystem, - Name: ListsDurationKey, - Help: "How long an API list takes to return and decode for the reflectors", - }, []string{"name"}) - - itemsPerList = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Subsystem: ReflectorSubsystem, - Name: ItemsPerListKey, - Help: "How many items an API list returns to the reflectors", - }, []string{"name"}) - - watchesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: ReflectorSubsystem, - Name: WatchesTotalKey, - Help: "Total number of API watches done by the reflectors", - }, []string{"name"}) - - shortWatchesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Subsystem: ReflectorSubsystem, - Name: ShortWatchesTotalKey, - Help: "Total number of short API watches done by the reflectors", - }, []string{"name"}) - - watchDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Subsystem: ReflectorSubsystem, - Name: WatchDurationKey, - Help: "How long an API watch takes to return and decode for the reflectors", - }, []string{"name"}) - - itemsPerWatch = prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Subsystem: ReflectorSubsystem, - Name: ItemsPerWatchKey, - Help: "How many items an API watch returns to the reflectors", - }, []string{"name"}) - - lastResourceVersion = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Subsystem: ReflectorSubsystem, - Name: LastResourceVersionKey, - Help: "Last resource version seen for the reflectors", - }, []string{"name"}) ) func init() { registerClientMetrics() - registerReflectorMetrics() } // registerClientMetrics sets up the client latency metrics from client-go. func registerClientMetrics() { // register the metrics with our registry - Registry.MustRegister(requestLatency) Registry.MustRegister(requestResult) // register the metrics with client-go clientmetrics.Register(clientmetrics.RegisterOpts{ - RequestLatency: &latencyAdapter{metric: requestLatency}, - RequestResult: &resultAdapter{metric: requestResult}, + RequestResult: &resultAdapter{metric: requestResult}, }) } -// registerReflectorMetrics sets up reflector (reconcile) loop metrics. -func registerReflectorMetrics() { - Registry.MustRegister(listsTotal) - Registry.MustRegister(listsDuration) - Registry.MustRegister(itemsPerList) - Registry.MustRegister(watchesTotal) - Registry.MustRegister(shortWatchesTotal) - Registry.MustRegister(watchDuration) - Registry.MustRegister(itemsPerWatch) - Registry.MustRegister(lastResourceVersion) - - reflectormetrics.SetReflectorMetricsProvider(reflectorMetricsProvider{}) -} - // this section contains adapters, implementations, and other sundry organic, artisanally // hand-crafted syntax trees required to convince client-go that it actually wants to let // someone use its metrics. @@ -159,11 +92,13 @@ func registerReflectorMetrics() { // copied (more-or-less directly) from k8s.io/kubernetes setup code // (which isn't anywhere in an easily-importable place). -type latencyAdapter struct { +// LatencyAdapter implements LatencyMetric. +type LatencyAdapter struct { metric *prometheus.HistogramVec } -func (l *latencyAdapter) Observe(_ context.Context, verb string, u url.URL, latency time.Duration) { +// Observe increments the request latency metric for the given verb/URL. +func (l *LatencyAdapter) Observe(_ context.Context, verb string, u url.URL, latency time.Duration) { l.metric.WithLabelValues(verb, u.String()).Observe(latency.Seconds()) } @@ -174,41 +109,3 @@ type resultAdapter struct { func (r *resultAdapter) Increment(_ context.Context, code, method, host string) { r.metric.WithLabelValues(code, method, host).Inc() } - -// Reflector metrics provider (method #2 for client-go metrics), -// copied (more-or-less directly) from k8s.io/kubernetes setup code -// (which isn't anywhere in an easily-importable place). - -type reflectorMetricsProvider struct{} - -func (reflectorMetricsProvider) NewListsMetric(name string) reflectormetrics.CounterMetric { - return listsTotal.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewListDurationMetric(name string) reflectormetrics.SummaryMetric { - return listsDuration.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewItemsInListMetric(name string) reflectormetrics.SummaryMetric { - return itemsPerList.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewWatchesMetric(name string) reflectormetrics.CounterMetric { - return watchesTotal.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewShortWatchesMetric(name string) reflectormetrics.CounterMetric { - return shortWatchesTotal.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewWatchDurationMetric(name string) reflectormetrics.SummaryMetric { - return watchDuration.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewItemsInWatchMetric(name string) reflectormetrics.SummaryMetric { - return itemsPerWatch.WithLabelValues(name) -} - -func (reflectorMetricsProvider) NewLastResourceVersionMetric(name string) reflectormetrics.GaugeMetric { - return lastResourceVersion.WithLabelValues(name) -} diff --git a/pkg/metrics/leaderelection.go b/pkg/metrics/leaderelection.go new file mode 100644 index 0000000000..a19c099602 --- /dev/null +++ b/pkg/metrics/leaderelection.go @@ -0,0 +1,40 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "k8s.io/client-go/tools/leaderelection" +) + +// This file is copied and adapted from k8s.io/component-base/metrics/prometheus/clientgo/leaderelection +// which registers metrics to the k8s legacy Registry. We require very +// similar functionality, but must register metrics to a different Registry. + +var ( + leaderGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "leader_election_master_status", + Help: "Gauge of if the reporting system is master of the relevant lease, 0 indicates backup, 1 indicates master. 'name' is the string used to identify the lease. Please make sure to group by name.", + }, []string{"name"}) +) + +func init() { + Registry.MustRegister(leaderGauge) + leaderelection.SetProvider(leaderelectionMetricsProvider{}) +} + +type leaderelectionMetricsProvider struct{} + +func (leaderelectionMetricsProvider) NewLeaderMetric() leaderelection.SwitchMetric { + return &switchAdapter{gauge: leaderGauge} +} + +type switchAdapter struct { + gauge *prometheus.GaugeVec +} + +func (s *switchAdapter) On(name string) { + s.gauge.WithLabelValues(name).Set(1.0) +} + +func (s *switchAdapter) Off(name string) { + s.gauge.WithLabelValues(name).Set(0.0) +} diff --git a/pkg/metrics/listener.go b/pkg/metrics/listener.go index d32ae58186..123d8c15f9 100644 --- a/pkg/metrics/listener.go +++ b/pkg/metrics/listener.go @@ -41,7 +41,7 @@ func NewListener(addr string) (net.Listener, error) { return nil, nil } - log.Info("metrics server is starting to listen", "addr", addr) + log.Info("Metrics server is starting to listen", "addr", addr) ln, err := net.Listen("tcp", addr) if err != nil { er := fmt.Errorf("error listening on %s: %w", addr, err) diff --git a/pkg/metrics/workqueue.go b/pkg/metrics/workqueue.go index 8ca47235da..277b878810 100644 --- a/pkg/metrics/workqueue.go +++ b/pkg/metrics/workqueue.go @@ -21,8 +21,8 @@ import ( "k8s.io/client-go/util/workqueue" ) -// This file is copied and adapted from k8s.io/kubernetes/pkg/util/workqueue/prometheus -// which registers metrics to the default prometheus Registry. We require very +// This file is copied and adapted from k8s.io/component-base/metrics/prometheus/workqueue +// which registers metrics to the k8s legacy Registry. We require very // similar functionality, but must register metrics to a different Registry. // Metrics subsystem and all keys used by the workqueue. diff --git a/pkg/patterns/application/doc.go b/pkg/patterns/application/doc.go deleted file mode 100644 index 5784051b96..0000000000 --- a/pkg/patterns/application/doc.go +++ /dev/null @@ -1,28 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package application documents patterns for building Controllers to manage specific applications. -// -// -// An application is a Controller and Resource that together implement the operational logic for an application. -// They are often used to take off-the-shelf OSS applications, and make them Kubernetes native. -// -// A typical application Controller may use builder.ControllerManagedBy() to create a Controller -// for a single API type that manages other objects it creates. -// -// Application Controllers are most useful for stateful applications such as Cassandra, Etcd and MySQL -// which contain operation logic for sharding, backup and restore, upgrade / downgrade, etc. -package application diff --git a/pkg/predicate/predicate.go b/pkg/predicate/predicate.go index fc59d89ba3..314635875e 100644 --- a/pkg/predicate/predicate.go +++ b/pkg/predicate/predicate.go @@ -49,6 +49,7 @@ var _ Predicate = GenerationChangedPredicate{} var _ Predicate = AnnotationChangedPredicate{} var _ Predicate = or{} var _ Predicate = and{} +var _ Predicate = not{} // Funcs is a function that implements Predicate. type Funcs struct { @@ -175,9 +176,9 @@ func (GenerationChangedPredicate) Update(e event.UpdateEvent) bool { // This predicate will skip update events that have no change in the object's annotation. // It is intended to be used in conjunction with the GenerationChangedPredicate, as in the following example: // -// Controller.Watch( +// Controller.Watch( // &source.Kind{Type: v1.MyCustomKind}, -// &handler.EnqueueRequestForObject{}, +// &handler.EnqueueRequestForObject{}, // predicate.Or(predicate.GenerationChangedPredicate{}, predicate.AnnotationChangedPredicate{})) // // This is mostly useful for controllers that needs to trigger both when the resource's generation is incremented @@ -206,9 +207,10 @@ func (AnnotationChangedPredicate) Update(e event.UpdateEvent) bool { // It is intended to be used in conjunction with the GenerationChangedPredicate, as in the following example: // // Controller.Watch( -// &source.Kind{Type: v1.MyCustomKind}, -// &handler.EnqueueRequestForObject{}, -// predicate.Or(predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{})) +// +// &source.Kind{Type: v1.MyCustomKind}, +// &handler.EnqueueRequestForObject{}, +// predicate.Or(predicate.GenerationChangedPredicate{}, predicate.LabelChangedPredicate{})) // // This will be helpful when object's labels is carrying some extra specification information beyond object's spec, // and the controller will be triggered if any valid spec change (not only in spec, but also in labels) happens. @@ -320,6 +322,31 @@ func (o or) Generic(e event.GenericEvent) bool { return false } +// Not returns a predicate that implements a logical NOT of the predicate passed to it. +func Not(predicate Predicate) Predicate { + return not{predicate} +} + +type not struct { + predicate Predicate +} + +func (n not) Create(e event.CreateEvent) bool { + return !n.predicate.Create(e) +} + +func (n not) Update(e event.UpdateEvent) bool { + return !n.predicate.Update(e) +} + +func (n not) Delete(e event.DeleteEvent) bool { + return !n.predicate.Delete(e) +} + +func (n not) Generic(e event.GenericEvent) bool { + return !n.predicate.Generic(e) +} + // LabelSelectorPredicate constructs a Predicate from a LabelSelector. // Only objects matching the LabelSelector will be admitted. func LabelSelectorPredicate(s metav1.LabelSelector) (Predicate, error) { diff --git a/pkg/predicate/predicate_suite_test.go b/pkg/predicate/predicate_suite_test.go index a03d94b17d..170594ca52 100644 --- a/pkg/predicate/predicate_suite_test.go +++ b/pkg/predicate/predicate_suite_test.go @@ -19,17 +19,15 @@ package predicate_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestPredicate(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Predicate Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Predicate Suite") } var _ = BeforeSuite(func() { diff --git a/pkg/predicate/predicate_test.go b/pkg/predicate/predicate_test.go index f8fb15bc0c..6bbf21adf0 100644 --- a/pkg/predicate/predicate_test.go +++ b/pkg/predicate/predicate_test.go @@ -17,7 +17,7 @@ limitations under the License. package predicate_test import ( - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -59,7 +59,7 @@ var _ = Describe("Predicate", func() { }, } - It("should call Create", func(done Done) { + It("should call Create", func() { instance := failingFuncs instance.CreateFunc = func(evt event.CreateEvent) bool { defer GinkgoRecover() @@ -80,10 +80,9 @@ var _ = Describe("Predicate", func() { instance.CreateFunc = nil Expect(instance.Create(evt)).To(BeTrue()) - close(done) }) - It("should call Update", func(done Done) { + It("should call Update", func() { newPod := pod.DeepCopy() newPod.Name = "baz2" newPod.Namespace = "biz2" @@ -111,10 +110,9 @@ var _ = Describe("Predicate", func() { instance.UpdateFunc = nil Expect(instance.Update(evt)).To(BeTrue()) - close(done) }) - It("should call Delete", func(done Done) { + It("should call Delete", func() { instance := failingFuncs instance.DeleteFunc = func(evt event.DeleteEvent) bool { defer GinkgoRecover() @@ -135,10 +133,9 @@ var _ = Describe("Predicate", func() { instance.DeleteFunc = nil Expect(instance.Delete(evt)).To(BeTrue()) - close(done) }) - It("should call Generic", func(done Done) { + It("should call Generic", func() { instance := failingFuncs instance.GenericFunc = func(evt event.GenericEvent) bool { defer GinkgoRecover() @@ -159,7 +156,6 @@ var _ = Describe("Predicate", func() { instance.GenericFunc = nil Expect(instance.Generic(evt)).To(BeTrue()) - close(done) }) }) @@ -416,7 +412,6 @@ var _ = Describe("Predicate", func() { // AnnotationChangedPredicate has almost identical test cases as LabelChangedPredicates, // so the duplication linter should be muted on both two test suites. - // nolint:dupl Describe("When checking an AnnotationChangedPredicate", func() { instance := predicate.AnnotationChangedPredicate{} Context("Where the old object is missing", func() { @@ -615,7 +610,6 @@ var _ = Describe("Predicate", func() { // LabelChangedPredicates has almost identical test cases as AnnotationChangedPredicates, // so the duplication linter should be muted on both two test suites. - // nolint:dupl Describe("When checking a LabelChangedPredicate", func() { instance := predicate.LabelChangedPredicate{} Context("Where the old object is missing", func() { @@ -831,6 +825,7 @@ var _ = Describe("Predicate", func() { } passFuncs := funcs(true) failFuncs := funcs(false) + Describe("When checking an And predicate", func() { It("should return false when one of its predicates returns false", func() { a := predicate.And(passFuncs, failFuncs) @@ -863,6 +858,22 @@ var _ = Describe("Predicate", func() { Expect(o.Generic(event.GenericEvent{})).To(BeFalse()) }) }) + Describe("When checking a Not predicate", func() { + It("should return false when its predicate returns true", func() { + n := predicate.Not(passFuncs) + Expect(n.Create(event.CreateEvent{})).To(BeFalse()) + Expect(n.Update(event.UpdateEvent{})).To(BeFalse()) + Expect(n.Delete(event.DeleteEvent{})).To(BeFalse()) + Expect(n.Generic(event.GenericEvent{})).To(BeFalse()) + }) + It("should return true when its predicate returns false", func() { + n := predicate.Not(failFuncs) + Expect(n.Create(event.CreateEvent{})).To(BeTrue()) + Expect(n.Update(event.UpdateEvent{})).To(BeTrue()) + Expect(n.Delete(event.DeleteEvent{})).To(BeTrue()) + Expect(n.Generic(event.GenericEvent{})).To(BeTrue()) + }) + }) }) Describe("NewPredicateFuncs with a namespace filter function", func() { diff --git a/pkg/reconcile/reconcile.go b/pkg/reconcile/reconcile.go index b2159c531f..8285e2ca9b 100644 --- a/pkg/reconcile/reconcile.go +++ b/pkg/reconcile/reconcile.go @@ -61,24 +61,24 @@ Deleting Kubernetes objects) or external Events (GitHub Webhooks, polling extern Example reconcile Logic: - * Read an object and all the Pods it owns. - * Observe that the object spec specifies 5 replicas but actual cluster contains only 1 Pod replica. - * Create 4 Pods and set their OwnerReferences to the object. +* Read an object and all the Pods it owns. +* Observe that the object spec specifies 5 replicas but actual cluster contains only 1 Pod replica. +* Create 4 Pods and set their OwnerReferences to the object. reconcile may be implemented as either a type: - type reconcile struct {} + type reconciler struct {} - func (reconcile) reconcile(controller.Request) (controller.Result, error) { + func (reconciler) Reconcile(ctx context.Context, o reconcile.Request) (reconcile.Result, error) { // Implement business logic of reading and writing objects here - return controller.Result{}, nil + return reconcile.Result{}, nil } Or as a function: - controller.Func(func(o controller.Request) (controller.Result, error) { + reconcile.Func(func(ctx context.Context, o reconcile.Request) (reconcile.Result, error) { // Implement business logic of reading and writing objects here - return controller.Result{}, nil + return reconcile.Result{}, nil }) Reconciliation is level-based, meaning action isn't driven off changes in individual Events, but instead is @@ -87,7 +87,7 @@ For example if responding to a Pod Delete Event, the Request won't contain that instead the reconcile function observes this when reading the cluster state and seeing the Pod as missing. */ type Reconciler interface { - // Reconciler performs a full reconciliation for the object referred to by the Request. + // Reconcile performs a full reconciliation for the object referred to by the Request. // The Controller will requeue the Request to be processed again if an error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. Reconcile(context.Context, Request) (Result, error) diff --git a/pkg/reconcile/reconcile_suite_test.go b/pkg/reconcile/reconcile_suite_test.go index 179fb10de4..9bab444ebd 100644 --- a/pkg/reconcile/reconcile_suite_test.go +++ b/pkg/reconcile/reconcile_suite_test.go @@ -19,17 +19,15 @@ package reconcile_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestReconcile(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Reconcile Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Reconcile Suite") } var _ = BeforeSuite(func() { diff --git a/pkg/reconcile/reconcile_test.go b/pkg/reconcile/reconcile_test.go index 26924c8fa9..125b80936d 100644 --- a/pkg/reconcile/reconcile_test.go +++ b/pkg/reconcile/reconcile_test.go @@ -21,7 +21,7 @@ import ( "fmt" "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/reconcile" diff --git a/pkg/recorder/example_test.go b/pkg/recorder/example_test.go index cf1beb40c8..969420d817 100644 --- a/pkg/recorder/example_test.go +++ b/pkg/recorder/example_test.go @@ -19,6 +19,7 @@ package recorder_test import ( corev1 "k8s.io/api/core/v1" + _ "github.com/onsi/ginkgo/v2" "sigs.k8s.io/controller-runtime/pkg/recorder" ) diff --git a/pkg/runtime/inject/inject.go b/pkg/runtime/inject/inject.go deleted file mode 100644 index c8c56ba817..0000000000 --- a/pkg/runtime/inject/inject.go +++ /dev/null @@ -1,164 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package inject is used by a Manager to inject types into Sources, EventHandlers, Predicates, and Reconciles. -// Deprecated: Use manager.Options fields directly. This package will be removed in v0.10. -package inject - -import ( - "github.com/go-logr/logr" - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/rest" - - "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// Cache is used by the ControllerManager to inject Cache into Sources, EventHandlers, Predicates, and -// Reconciles. -type Cache interface { - InjectCache(cache cache.Cache) error -} - -// CacheInto will set informers on i and return the result if it implements Cache. Returns -// false if i does not implement Cache. -func CacheInto(c cache.Cache, i interface{}) (bool, error) { - if s, ok := i.(Cache); ok { - return true, s.InjectCache(c) - } - return false, nil -} - -// APIReader is used by the Manager to inject the APIReader into necessary types. -type APIReader interface { - InjectAPIReader(client.Reader) error -} - -// APIReaderInto will set APIReader on i and return the result if it implements APIReaderInto. -// Returns false if i does not implement APIReader. -func APIReaderInto(reader client.Reader, i interface{}) (bool, error) { - if s, ok := i.(APIReader); ok { - return true, s.InjectAPIReader(reader) - } - return false, nil -} - -// Config is used by the ControllerManager to inject Config into Sources, EventHandlers, Predicates, and -// Reconciles. -type Config interface { - InjectConfig(*rest.Config) error -} - -// ConfigInto will set config on i and return the result if it implements Config. Returns -// false if i does not implement Config. -func ConfigInto(config *rest.Config, i interface{}) (bool, error) { - if s, ok := i.(Config); ok { - return true, s.InjectConfig(config) - } - return false, nil -} - -// Client is used by the ControllerManager to inject client into Sources, EventHandlers, Predicates, and -// Reconciles. -type Client interface { - InjectClient(client.Client) error -} - -// ClientInto will set client on i and return the result if it implements Client. Returns -// false if i does not implement Client. -func ClientInto(client client.Client, i interface{}) (bool, error) { - if s, ok := i.(Client); ok { - return true, s.InjectClient(client) - } - return false, nil -} - -// Scheme is used by the ControllerManager to inject Scheme into Sources, EventHandlers, Predicates, and -// Reconciles. -type Scheme interface { - InjectScheme(scheme *runtime.Scheme) error -} - -// SchemeInto will set scheme and return the result on i if it implements Scheme. Returns -// false if i does not implement Scheme. -func SchemeInto(scheme *runtime.Scheme, i interface{}) (bool, error) { - if is, ok := i.(Scheme); ok { - return true, is.InjectScheme(scheme) - } - return false, nil -} - -// Stoppable is used by the ControllerManager to inject stop channel into Sources, -// EventHandlers, Predicates, and Reconciles. -type Stoppable interface { - InjectStopChannel(<-chan struct{}) error -} - -// StopChannelInto will set stop channel on i and return the result if it implements Stoppable. -// Returns false if i does not implement Stoppable. -func StopChannelInto(stop <-chan struct{}, i interface{}) (bool, error) { - if s, ok := i.(Stoppable); ok { - return true, s.InjectStopChannel(stop) - } - return false, nil -} - -// Mapper is used to inject the rest mapper to components that may need it. -type Mapper interface { - InjectMapper(meta.RESTMapper) error -} - -// MapperInto will set the rest mapper on i and return the result if it implements Mapper. -// Returns false if i does not implement Mapper. -func MapperInto(mapper meta.RESTMapper, i interface{}) (bool, error) { - if m, ok := i.(Mapper); ok { - return true, m.InjectMapper(mapper) - } - return false, nil -} - -// Func injects dependencies into i. -type Func func(i interface{}) error - -// Injector is used by the ControllerManager to inject Func into Controllers. -type Injector interface { - InjectFunc(f Func) error -} - -// InjectorInto will set f and return the result on i if it implements Injector. Returns -// false if i does not implement Injector. -func InjectorInto(f Func, i interface{}) (bool, error) { - if ii, ok := i.(Injector); ok { - return true, ii.InjectFunc(f) - } - return false, nil -} - -// Logger is used to inject Loggers into components that need them -// and don't otherwise have opinions. -type Logger interface { - InjectLogger(l logr.Logger) error -} - -// LoggerInto will set the logger on the given object if it implements inject.Logger, -// returning true if a InjectLogger was called, and false otherwise. -func LoggerInto(l logr.Logger, i interface{}) (bool, error) { - if injectable, wantsLogger := i.(Logger); wantsLogger { - return true, injectable.InjectLogger(l) - } - return false, nil -} diff --git a/pkg/runtime/inject/inject_test.go b/pkg/runtime/inject/inject_test.go deleted file mode 100644 index bffc34ec27..0000000000 --- a/pkg/runtime/inject/inject_test.go +++ /dev/null @@ -1,331 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package inject - -import ( - "fmt" - "reflect" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/cache/informertest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -var instance *testSource -var uninjectable *failSource -var errInjectFail = fmt.Errorf("injection fails") -var expectedFalse = false - -var _ = Describe("runtime inject", func() { - - BeforeEach(func() { - instance = &testSource{} - uninjectable = &failSource{} - }) - - It("should set informers", func() { - injectedCache := &informertest.FakeInformers{} - - By("Validating injecting the informer") - res, err := CacheInto(injectedCache, instance) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(true)) - Expect(injectedCache).To(Equal(instance.GetCache())) - - By("Returning false if the type does not implement inject.Cache") - res, err = CacheInto(injectedCache, uninjectable) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(expectedFalse)) - Expect(uninjectable.GetCache()).To(BeNil()) - - By("Returning an error if informer injection fails") - res, err = CacheInto(nil, instance) - Expect(err).To(Equal(errInjectFail)) - Expect(res).To(Equal(true)) - - }) - - It("should set config", func() { - - cfg := &rest.Config{} - - By("Validating injecting config") - res, err := ConfigInto(cfg, instance) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(true)) - Expect(cfg).To(Equal(instance.GetConfig())) - - By("Returning false if the type does not implement inject.Config") - res, err = ConfigInto(cfg, uninjectable) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(false)) - Expect(uninjectable.GetConfig()).To(BeNil()) - - By("Returning an error if config injection fails") - res, err = ConfigInto(nil, instance) - Expect(err).To(Equal(errInjectFail)) - Expect(res).To(Equal(true)) - }) - - It("should set client", func() { - client, err := client.NewDelegatingClient(client.NewDelegatingClientInput{Client: fake.NewClientBuilder().Build()}) - Expect(err).NotTo(HaveOccurred()) - - By("Validating injecting client") - res, err := ClientInto(client, instance) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(true)) - Expect(client).To(Equal(instance.GetClient())) - - By("Returning false if the type does not implement inject.Client") - res, err = ClientInto(client, uninjectable) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(false)) - Expect(uninjectable.GetClient()).To(BeNil()) - - By("Returning an error if client injection fails") - res, err = ClientInto(nil, instance) - Expect(err).To(Equal(errInjectFail)) - Expect(res).To(Equal(true)) - }) - - It("should set scheme", func() { - - scheme := runtime.NewScheme() - - By("Validating injecting scheme") - res, err := SchemeInto(scheme, instance) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(true)) - Expect(scheme).To(Equal(instance.GetScheme())) - - By("Returning false if the type does not implement inject.Scheme") - res, err = SchemeInto(scheme, uninjectable) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(false)) - Expect(uninjectable.GetScheme()).To(BeNil()) - - By("Returning an error if scheme injection fails") - res, err = SchemeInto(nil, instance) - Expect(err).To(Equal(errInjectFail)) - Expect(res).To(Equal(true)) - }) - - It("should set stop channel", func() { - - stop := make(<-chan struct{}) - - By("Validating injecting stop channel") - res, err := StopChannelInto(stop, instance) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(true)) - Expect(stop).To(Equal(instance.GetStop())) - - By("Returning false if the type does not implement inject.Stoppable") - res, err = StopChannelInto(stop, uninjectable) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(false)) - Expect(uninjectable.GetStop()).To(BeNil()) - - By("Returning an error if stop channel injection fails") - res, err = StopChannelInto(nil, instance) - Expect(err).To(Equal(errInjectFail)) - Expect(res).To(Equal(true)) - }) - - It("should set api reader", func() { - apiReader, err := client.NewDelegatingClient(client.NewDelegatingClientInput{Client: fake.NewClientBuilder().Build()}) - Expect(err).NotTo(HaveOccurred()) - - By("Validating injecting client") - res, err := APIReaderInto(apiReader, instance) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(true)) - Expect(apiReader).To(Equal(instance.GetAPIReader())) - - By("Returning false if the type does not implement inject.Client") - res, err = APIReaderInto(apiReader, uninjectable) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(false)) - Expect(uninjectable.GetAPIReader()).To(BeNil()) - - By("Returning an error if client injection fails") - res, err = APIReaderInto(nil, instance) - Expect(err).To(Equal(errInjectFail)) - Expect(res).To(Equal(true)) - }) - - It("should set dependencies", func() { - - f := func(interface{}) error { return nil } - - By("Validating injecting dependencies") - res, err := InjectorInto(f, instance) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(true)) - Expect(reflect.ValueOf(f).Pointer()).To(Equal(reflect.ValueOf(instance.GetFunc()).Pointer())) - - By("Returning false if the type does not implement inject.Injector") - res, err = InjectorInto(f, uninjectable) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(false)) - Expect(uninjectable.GetFunc()).To(BeNil()) - - By("Returning an error if dependencies injection fails") - res, err = InjectorInto(nil, instance) - Expect(err).To(Equal(errInjectFail)) - Expect(res).To(Equal(true)) - }) - -}) - -type testSource struct { - scheme *runtime.Scheme - cache cache.Cache - config *rest.Config - client client.Client - apiReader client.Reader - f Func - stop <-chan struct{} -} - -func (s *testSource) InjectCache(c cache.Cache) error { - if c != nil { - s.cache = c - return nil - } - return fmt.Errorf("injection fails") -} - -func (s *testSource) InjectConfig(config *rest.Config) error { - if config != nil { - s.config = config - return nil - } - return fmt.Errorf("injection fails") -} - -func (s *testSource) InjectClient(client client.Client) error { - if client != nil { - s.client = client - return nil - } - return fmt.Errorf("injection fails") -} - -func (s *testSource) InjectScheme(scheme *runtime.Scheme) error { - if scheme != nil { - s.scheme = scheme - return nil - } - return fmt.Errorf("injection fails") -} - -func (s *testSource) InjectStopChannel(stop <-chan struct{}) error { - if stop != nil { - s.stop = stop - return nil - } - return fmt.Errorf("injection fails") -} - -func (s *testSource) InjectAPIReader(reader client.Reader) error { - if reader != nil { - s.apiReader = reader - return nil - } - return fmt.Errorf("injection fails") -} - -func (s *testSource) InjectFunc(f Func) error { - if f != nil { - s.f = f - return nil - } - return fmt.Errorf("injection fails") -} - -func (s *testSource) GetCache() cache.Cache { - return s.cache -} - -func (s *testSource) GetConfig() *rest.Config { - return s.config -} - -func (s *testSource) GetScheme() *runtime.Scheme { - return s.scheme -} - -func (s *testSource) GetClient() client.Client { - return s.client -} - -func (s *testSource) GetAPIReader() client.Reader { - return s.apiReader -} - -func (s *testSource) GetFunc() Func { - return s.f -} - -func (s *testSource) GetStop() <-chan struct{} { - return s.stop -} - -type failSource struct { - scheme *runtime.Scheme - cache cache.Cache - config *rest.Config - client client.Client - apiReader client.Reader - f Func - stop <-chan struct{} -} - -func (s *failSource) GetCache() cache.Cache { - return s.cache -} - -func (s *failSource) GetConfig() *rest.Config { - return s.config -} - -func (s *failSource) GetScheme() *runtime.Scheme { - return s.scheme -} - -func (s *failSource) GetClient() client.Client { - return s.client -} - -func (s *failSource) GetAPIReader() client.Reader { - return s.apiReader -} - -func (s *failSource) GetFunc() Func { - return s.f -} - -func (s *failSource) GetStop() <-chan struct{} { - return s.stop -} diff --git a/pkg/scheme/scheme.go b/pkg/scheme/scheme.go index 9dc93a9b21..55ebe21773 100644 --- a/pkg/scheme/scheme.go +++ b/pkg/scheme/scheme.go @@ -21,37 +21,36 @@ limitations under the License. // Each API group should define a utility function // called AddToScheme for adding its types to a Scheme: // -// // in package myapigroupv1... -// var ( -// SchemeGroupVersion = schema.GroupVersion{Group: "my.api.group", Version: "v1"} -// SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion} -// AddToScheme = SchemeBuilder.AddToScheme -// ) +// // in package myapigroupv1... +// var ( +// SchemeGroupVersion = schema.GroupVersion{Group: "my.api.group", Version: "v1"} +// SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion} +// AddToScheme = SchemeBuilder.AddToScheme +// ) // -// func init() { -// SchemeBuilder.Register(&MyType{}, &MyTypeList) -// } -// var ( -// scheme *runtime.Scheme = runtime.NewScheme() -// ) +// func init() { +// SchemeBuilder.Register(&MyType{}, &MyTypeList) +// } +// var ( +// scheme *runtime.Scheme = runtime.NewScheme() +// ) // // This also true of the built-in Kubernetes types. Then, in the entrypoint for // your manager, assemble the scheme containing exactly the types you need, // panicing if scheme registration failed. For instance, if our controller needs // types from the core/v1 API group (e.g. Pod), plus types from my.api.group/v1: // -// func init() { -// utilruntime.Must(myapigroupv1.AddToScheme(scheme)) -// utilruntime.Must(kubernetesscheme.AddToScheme(scheme)) -// } -// -// func main() { -// mgr := controllers.NewManager(context.Background(), controllers.GetConfigOrDie(), manager.Options{ -// Scheme: scheme, -// }) -// // ... -// } +// func init() { +// utilruntime.Must(myapigroupv1.AddToScheme(scheme)) +// utilruntime.Must(kubernetesscheme.AddToScheme(scheme)) +// } // +// func main() { +// mgr := controllers.NewManager(context.Background(), controllers.GetConfigOrDie(), manager.Options{ +// Scheme: scheme, +// }) +// // ... +// } package scheme import ( diff --git a/pkg/scheme/scheme_suite_test.go b/pkg/scheme/scheme_suite_test.go index a11e08fa5c..36ddd9decc 100644 --- a/pkg/scheme/scheme_suite_test.go +++ b/pkg/scheme/scheme_suite_test.go @@ -19,14 +19,11 @@ package scheme_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" ) func TestScheme(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Scheme Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Scheme Suite") } diff --git a/pkg/scheme/scheme_test.go b/pkg/scheme/scheme_test.go index 72f083ad4b..37c6766e6f 100644 --- a/pkg/scheme/scheme_test.go +++ b/pkg/scheme/scheme_test.go @@ -19,7 +19,7 @@ package scheme_test import ( "reflect" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" appsv1 "k8s.io/api/apps/v1" diff --git a/pkg/source/example_test.go b/pkg/source/example_test.go index d306eaf583..77857729de 100644 --- a/pkg/source/example_test.go +++ b/pkg/source/example_test.go @@ -21,15 +21,17 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/source" ) +var mgr manager.Manager var ctrl controller.Controller // This example Watches for Pod Events (e.g. Create / Update / Delete) and enqueues a reconcile.Request // with the Name and Namespace of the Pod. func ExampleKind() { - err := ctrl.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForObject{}) + err := ctrl.Watch(source.Kind(mgr.GetCache(), &corev1.Pod{}), &handler.EnqueueRequestForObject{}) if err != nil { // handle it } diff --git a/pkg/source/source.go b/pkg/source/source.go index a63b37c443..5fb7c439b6 100644 --- a/pkg/source/source.go +++ b/pkg/source/source.go @@ -18,25 +18,19 @@ package source import ( "context" - "errors" "fmt" "sync" - "k8s.io/apimachinery/pkg/api/meta" "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" - "sigs.k8s.io/controller-runtime/pkg/source/internal" + internal "sigs.k8s.io/controller-runtime/pkg/internal/source" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/predicate" ) -var log = logf.RuntimeLog.WithName("source") - const ( // defaultBufferSize is the default number of event notifications that can be buffered. defaultBufferSize = 1024 @@ -49,8 +43,7 @@ const ( // // * Use Channel for events originating outside the cluster (eh.g. GitHub Webhook callback, Polling external urls). // -// Users may build their own Source implementations. If their implementations implement any of the inject package -// interfaces, the dependencies will be injected by the Controller when Watch is called. +// Users may build their own Source implementations. type Source interface { // Start is internal and should be called only by the Controller to register an EventHandler with the Informer // to enqueue reconcile.Requests. @@ -64,111 +57,9 @@ type SyncingSource interface { WaitForSync(ctx context.Context) error } -// NewKindWithCache creates a Source without InjectCache, so that it is assured that the given cache is used -// and not overwritten. It can be used to watch objects in a different cluster by passing the cache -// from that other cluster. -func NewKindWithCache(object client.Object, cache cache.Cache) SyncingSource { - return &kindWithCache{kind: Kind{Type: object, cache: cache}} -} - -type kindWithCache struct { - kind Kind -} - -func (ks *kindWithCache) Start(ctx context.Context, handler handler.EventHandler, queue workqueue.RateLimitingInterface, - prct ...predicate.Predicate) error { - return ks.kind.Start(ctx, handler, queue, prct...) -} - -func (ks *kindWithCache) WaitForSync(ctx context.Context) error { - return ks.kind.WaitForSync(ctx) -} - -// Kind is used to provide a source of events originating inside the cluster from Watches (e.g. Pod Create). -type Kind struct { - // Type is the type of object to watch. e.g. &v1.Pod{} - Type client.Object - - // cache used to watch APIs - cache cache.Cache - - // started may contain an error if one was encountered during startup. If its closed and does not - // contain an error, startup and syncing finished. - started chan error - startCancel func() -} - -var _ SyncingSource = &Kind{} - -// Start is internal and should be called only by the Controller to register an EventHandler with the Informer -// to enqueue reconcile.Requests. -func (ks *Kind) Start(ctx context.Context, handler handler.EventHandler, queue workqueue.RateLimitingInterface, - prct ...predicate.Predicate) error { - // Type should have been specified by the user. - if ks.Type == nil { - return fmt.Errorf("must specify Kind.Type") - } - - // cache should have been injected before Start was called - if ks.cache == nil { - return fmt.Errorf("must call CacheInto on Kind before calling Start") - } - - // cache.GetInformer will block until its context is cancelled if the cache was already started and it can not - // sync that informer (most commonly due to RBAC issues). - ctx, ks.startCancel = context.WithCancel(ctx) - ks.started = make(chan error) - go func() { - // Lookup the Informer from the Cache and add an EventHandler which populates the Queue - i, err := ks.cache.GetInformer(ctx, ks.Type) - if err != nil { - kindMatchErr := &meta.NoKindMatchError{} - if errors.As(err, &kindMatchErr) { - log.Error(err, "if kind is a CRD, it should be installed before calling Start", - "kind", kindMatchErr.GroupKind) - } - ks.started <- err - return - } - i.AddEventHandler(internal.EventHandler{Queue: queue, EventHandler: handler, Predicates: prct}) - if !ks.cache.WaitForCacheSync(ctx) { - // Would be great to return something more informative here - ks.started <- errors.New("cache did not sync") - } - close(ks.started) - }() - - return nil -} - -func (ks *Kind) String() string { - if ks.Type != nil && ks.Type.GetObjectKind() != nil { - return fmt.Sprintf("kind source: %v", ks.Type.GetObjectKind().GroupVersionKind().String()) - } - return "kind source: unknown GVK" -} - -// WaitForSync implements SyncingSource to allow controllers to wait with starting -// workers until the cache is synced. -func (ks *Kind) WaitForSync(ctx context.Context) error { - select { - case err := <-ks.started: - return err - case <-ctx.Done(): - ks.startCancel() - return errors.New("timed out waiting for cache to be synced") - } -} - -var _ inject.Cache = &Kind{} - -// InjectCache is internal should be called only by the Controller. InjectCache is used to inject -// the Cache dependency initialized by the ControllerManager. -func (ks *Kind) InjectCache(c cache.Cache) error { - if ks.cache == nil { - ks.cache = c - } - return nil +// Kind creates a KindSource with the given cache provider. +func Kind(cache cache.Cache, object client.Object) SyncingSource { + return &internal.Kind{Type: object, Cache: cache} } var _ Source = &Channel{} @@ -201,18 +92,6 @@ func (cs *Channel) String() string { return fmt.Sprintf("channel source: %p", cs) } -var _ inject.Stoppable = &Channel{} - -// InjectStopChannel is internal should be called only by the Controller. -// It is used to inject the stop channel initialized by the ControllerManager. -func (cs *Channel) InjectStopChannel(stop <-chan struct{}) error { - if cs.stop == nil { - cs.stop = stop - } - - return nil -} - // Start implements Source and should only be called by the Controller. func (cs *Channel) Start( ctx context.Context, @@ -224,10 +103,8 @@ func (cs *Channel) Start( return fmt.Errorf("must specify Channel.Source") } - // stop should have been injected before Start was called - if cs.stop == nil { - return fmt.Errorf("must call InjectStop on Channel before calling Start") - } + // set the stop channel to be the context. + cs.stop = ctx.Done() // use default value if DestBufferSize not specified if cs.DestBufferSize == 0 { @@ -256,7 +133,11 @@ func (cs *Channel) Start( } if shouldHandle { - handler.Generic(evt, queue) + func() { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + handler.Generic(ctx, evt, queue) + }() } } }() @@ -323,7 +204,10 @@ func (is *Informer) Start(ctx context.Context, handler handler.EventHandler, que return fmt.Errorf("must specify Informer.Informer") } - is.Informer.AddEventHandler(internal.EventHandler{Queue: queue, EventHandler: handler, Predicates: prct}) + _, err := is.Informer.AddEventHandler(internal.NewEventHandler(ctx, queue, handler, prct)) + if err != nil { + return err + } return nil } diff --git a/pkg/source/source_integration_test.go b/pkg/source/source_integration_test.go index 087cdbcb4c..594d3c9a9c 100644 --- a/pkg/source/source_integration_test.go +++ b/pkg/source/source_integration_test.go @@ -17,16 +17,16 @@ limitations under the License. package source_test import ( + "context" "fmt" "time" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/source" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -37,14 +37,14 @@ import ( ) var _ = Describe("Source", func() { - var instance1, instance2 *source.Kind + var instance1, instance2 source.Source var obj client.Object var q workqueue.RateLimitingInterface var c1, c2 chan interface{} var ns string count := 0 - BeforeEach(func(done Done) { + BeforeEach(func() { // Create the namespace for the test ns = fmt.Sprintf("controller-source-kindsource-%v", count) count++ @@ -56,32 +56,25 @@ var _ = Describe("Source", func() { q = workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") c1 = make(chan interface{}) c2 = make(chan interface{}) - - close(done) }) JustBeforeEach(func() { - instance1 = &source.Kind{Type: obj} - Expect(inject.CacheInto(icache, instance1)).To(BeTrue()) - - instance2 = &source.Kind{Type: obj} - Expect(inject.CacheInto(icache, instance2)).To(BeTrue()) + instance1 = source.Kind(icache, obj) + instance2 = source.Kind(icache, obj) }) - AfterEach(func(done Done) { + AfterEach(func() { err := clientset.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}) Expect(err).NotTo(HaveOccurred()) close(c1) close(c2) - - close(done) }) Describe("Kind", func() { Context("for a Deployment resource", func() { obj = &appsv1.Deployment{} - It("should provide Deployment Events", func(done Done) { + It("should provide Deployment Events", func() { var created, updated, deleted *appsv1.Deployment var err error @@ -110,17 +103,17 @@ var _ = Describe("Source", func() { // Create an event handler to verify the events newHandler := func(c chan interface{}) handler.Funcs { return handler.Funcs{ - CreateFunc: func(evt event.CreateEvent, rli workqueue.RateLimitingInterface) { + CreateFunc: func(ctx context.Context, evt event.CreateEvent, rli workqueue.RateLimitingInterface) { defer GinkgoRecover() Expect(rli).To(Equal(q)) c <- evt }, - UpdateFunc: func(evt event.UpdateEvent, rli workqueue.RateLimitingInterface) { + UpdateFunc: func(ctx context.Context, evt event.UpdateEvent, rli workqueue.RateLimitingInterface) { defer GinkgoRecover() Expect(rli).To(Equal(q)) c <- evt }, - DeleteFunc: func(evt event.DeleteEvent, rli workqueue.RateLimitingInterface) { + DeleteFunc: func(ctx context.Context, evt event.DeleteEvent, rli workqueue.RateLimitingInterface) { defer GinkgoRecover() Expect(rli).To(Equal(q)) c <- evt @@ -192,9 +185,7 @@ var _ = Describe("Source", func() { Expect(ok).To(BeTrue(), fmt.Sprintf("expect %T to be %T", evt, event.DeleteEvent{})) deleteEvt.Object.SetResourceVersion("") Expect(deleteEvt.Object).To(Equal(deleted)) - - close(done) - }, 5) + }) }) // TODO(pwittrock): Write this test @@ -212,7 +203,7 @@ var _ = Describe("Source", func() { var informerFactory kubeinformers.SharedInformerFactory var stopTest chan struct{} - BeforeEach(func(done Done) { + BeforeEach(func() { stopTest = make(chan struct{}) informerFactory = kubeinformers.NewSharedInformerFactory(clientset, time.Second*30) depInformer = informerFactory.Apps().V1().ReplicaSets().Informer() @@ -239,22 +230,20 @@ var _ = Describe("Source", func() { }, }, } - close(done) }) - AfterEach(func(done Done) { + AfterEach(func() { close(stopTest) - close(done) }) Context("for a ReplicaSet resource", func() { - It("should provide a ReplicaSet CreateEvent", func(done Done) { + It("should provide a ReplicaSet CreateEvent", func() { c := make(chan struct{}) q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") instance := &source.Informer{Informer: depInformer} err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(evt event.CreateEvent, q2 workqueue.RateLimitingInterface) { + CreateFunc: func(ctx context.Context, evt event.CreateEvent, q2 workqueue.RateLimitingInterface) { defer GinkgoRecover() var err error rs, err := clientset.AppsV1().ReplicaSets("default").Get(ctx, rs.Name, metav1.GetOptions{}) @@ -264,15 +253,15 @@ var _ = Describe("Source", func() { Expect(evt.Object).To(Equal(rs)) close(c) }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected UpdateEvent") }, - DeleteFunc: func(event.DeleteEvent, workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected DeleteEvent") }, - GenericFunc: func(event.GenericEvent, workqueue.RateLimitingInterface) { + GenericFunc: func(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected GenericEvent") }, @@ -282,10 +271,9 @@ var _ = Describe("Source", func() { _, err = clientset.AppsV1().ReplicaSets("default").Create(ctx, rs, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) <-c - close(done) - }, 30) + }) - It("should provide a ReplicaSet UpdateEvent", func(done Done) { + It("should provide a ReplicaSet UpdateEvent", func() { var err error rs, err = clientset.AppsV1().ReplicaSets("default").Get(ctx, rs.Name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) @@ -296,9 +284,9 @@ var _ = Describe("Source", func() { q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") instance := &source.Informer{Informer: depInformer} err = instance.Start(ctx, handler.Funcs{ - CreateFunc: func(evt event.CreateEvent, q2 workqueue.RateLimitingInterface) { + CreateFunc: func(ctx context.Context, evt event.CreateEvent, q2 workqueue.RateLimitingInterface) { }, - UpdateFunc: func(evt event.UpdateEvent, q2 workqueue.RateLimitingInterface) { + UpdateFunc: func(ctx context.Context, evt event.UpdateEvent, q2 workqueue.RateLimitingInterface) { defer GinkgoRecover() var err error rs2, err := clientset.AppsV1().ReplicaSets("default").Get(ctx, rs.Name, metav1.GetOptions{}) @@ -311,11 +299,11 @@ var _ = Describe("Source", func() { close(c) }, - DeleteFunc: func(event.DeleteEvent, workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected DeleteEvent") }, - GenericFunc: func(event.GenericEvent, workqueue.RateLimitingInterface) { + GenericFunc: func(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected GenericEvent") }, @@ -325,26 +313,25 @@ var _ = Describe("Source", func() { _, err = clientset.AppsV1().ReplicaSets("default").Update(ctx, rs2, metav1.UpdateOptions{}) Expect(err).NotTo(HaveOccurred()) <-c - close(done) }) - It("should provide a ReplicaSet DeletedEvent", func(done Done) { + It("should provide a ReplicaSet DeletedEvent", func() { c := make(chan struct{}) q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") instance := &source.Informer{Informer: depInformer} err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(event.CreateEvent, workqueue.RateLimitingInterface) { + CreateFunc: func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) { }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { }, - DeleteFunc: func(evt event.DeleteEvent, q2 workqueue.RateLimitingInterface) { + DeleteFunc: func(ctx context.Context, evt event.DeleteEvent, q2 workqueue.RateLimitingInterface) { defer GinkgoRecover() Expect(q2).To(Equal(q)) Expect(evt.Object.GetName()).To(Equal(rs.Name)) close(c) }, - GenericFunc: func(event.GenericEvent, workqueue.RateLimitingInterface) { + GenericFunc: func(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected GenericEvent") }, @@ -354,7 +341,6 @@ var _ = Describe("Source", func() { err = clientset.AppsV1().ReplicaSets("default").Delete(ctx, rs.Name, metav1.DeleteOptions{}) Expect(err).NotTo(HaveOccurred()) <-c - close(done) }) }) }) diff --git a/pkg/source/source_suite_test.go b/pkg/source/source_suite_test.go index 7be654bf0a..131099f0b9 100644 --- a/pkg/source/source_suite_test.go +++ b/pkg/source/source_suite_test.go @@ -20,21 +20,19 @@ import ( "context" "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Source Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Source Suite") } var testenv *envtest.Environment @@ -44,7 +42,7 @@ var icache cache.Cache var ctx context.Context var cancel context.CancelFunc -var _ = BeforeSuite(func(done Done) { +var _ = BeforeSuite(func() { ctx, cancel = context.WithCancel(context.Background()) logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) @@ -64,13 +62,9 @@ var _ = BeforeSuite(func(done Done) { defer GinkgoRecover() Expect(icache.Start(ctx)).NotTo(HaveOccurred()) }() +}) - close(done) -}, 60) - -var _ = AfterSuite(func(done Done) { +var _ = AfterSuite(func() { cancel() Expect(testenv.Stop()).To(Succeed()) - - close(done) -}, 5) +}) diff --git a/pkg/source/source_test.go b/pkg/source/source_test.go index 9b0a1c9744..1e0e3afed3 100644 --- a/pkg/source/source_test.go +++ b/pkg/source/source_test.go @@ -19,14 +19,14 @@ package source_test import ( "context" "fmt" + "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "sigs.k8s.io/controller-runtime/pkg/cache/informertest" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" "sigs.k8s.io/controller-runtime/pkg/source" corev1 "k8s.io/api/core/v1" @@ -40,7 +40,7 @@ var _ = Describe("Source", func() { var p *corev1.Pod var ic *informertest.FakeInformers - BeforeEach(func(done Done) { + BeforeEach(func() { ic = &informertest.FakeInformers{} c = make(chan struct{}) p = &corev1.Pod{ @@ -50,11 +50,10 @@ var _ = Describe("Source", func() { }, }, } - close(done) }) Context("for a Pod resource", func() { - It("should provide a Pod CreateEvent", func(done Done) { + It("should provide a Pod CreateEvent", func() { c := make(chan struct{}) p := &corev1.Pod{ Spec: corev1.PodSpec{ @@ -65,26 +64,23 @@ var _ = Describe("Source", func() { } q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := &source.Kind{ - Type: &corev1.Pod{}, - } - Expect(inject.CacheInto(ic, instance)).To(BeTrue()) + instance := source.Kind(ic, &corev1.Pod{}) err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(evt event.CreateEvent, q2 workqueue.RateLimitingInterface) { + CreateFunc: func(ctx context.Context, evt event.CreateEvent, q2 workqueue.RateLimitingInterface) { defer GinkgoRecover() Expect(q2).To(Equal(q)) Expect(evt.Object).To(Equal(p)) close(c) }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected UpdateEvent") }, - DeleteFunc: func(event.DeleteEvent, workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected DeleteEvent") }, - GenericFunc: func(event.GenericEvent, workqueue.RateLimitingInterface) { + GenericFunc: func(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected GenericEvent") }, @@ -97,25 +93,21 @@ var _ = Describe("Source", func() { i.Add(p) <-c - close(done) }) - It("should provide a Pod UpdateEvent", func(done Done) { + It("should provide a Pod UpdateEvent", func() { p2 := p.DeepCopy() p2.SetLabels(map[string]string{"biz": "baz"}) ic := &informertest.FakeInformers{} q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := &source.Kind{ - Type: &corev1.Pod{}, - } - Expect(instance.InjectCache(ic)).To(Succeed()) + instance := source.Kind(ic, &corev1.Pod{}) err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(evt event.CreateEvent, q2 workqueue.RateLimitingInterface) { + CreateFunc: func(ctx context.Context, evt event.CreateEvent, q2 workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected CreateEvent") }, - UpdateFunc: func(evt event.UpdateEvent, q2 workqueue.RateLimitingInterface) { + UpdateFunc: func(ctx context.Context, evt event.UpdateEvent, q2 workqueue.RateLimitingInterface) { defer GinkgoRecover() Expect(q2).To(BeIdenticalTo(q)) Expect(evt.ObjectOld).To(Equal(p)) @@ -124,11 +116,11 @@ var _ = Describe("Source", func() { close(c) }, - DeleteFunc: func(event.DeleteEvent, workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected DeleteEvent") }, - GenericFunc: func(event.GenericEvent, workqueue.RateLimitingInterface) { + GenericFunc: func(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected GenericEvent") }, @@ -141,10 +133,9 @@ var _ = Describe("Source", func() { i.Update(p, p2) <-c - close(done) }) - It("should provide a Pod DeletedEvent", func(done Done) { + It("should provide a Pod DeletedEvent", func() { c := make(chan struct{}) p := &corev1.Pod{ Spec: corev1.PodSpec{ @@ -155,26 +146,23 @@ var _ = Describe("Source", func() { } q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := &source.Kind{ - Type: &corev1.Pod{}, - } - Expect(inject.CacheInto(ic, instance)).To(BeTrue()) + instance := source.Kind(ic, &corev1.Pod{}) err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(event.CreateEvent, workqueue.RateLimitingInterface) { + CreateFunc: func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected DeleteEvent") }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected UpdateEvent") }, - DeleteFunc: func(evt event.DeleteEvent, q2 workqueue.RateLimitingInterface) { + DeleteFunc: func(ctx context.Context, evt event.DeleteEvent, q2 workqueue.RateLimitingInterface) { defer GinkgoRecover() Expect(q2).To(BeIdenticalTo(q)) Expect(evt.Object).To(Equal(p)) close(c) }, - GenericFunc: func(event.GenericEvent, workqueue.RateLimitingInterface) { + GenericFunc: func(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected GenericEvent") }, @@ -187,83 +175,61 @@ var _ = Describe("Source", func() { i.Delete(p) <-c - close(done) }) }) - It("should return an error from Start if informers were not injected", func(done Done) { - instance := source.Kind{Type: &corev1.Pod{}} + It("should return an error from Start cache was not provided", func() { + instance := source.Kind(nil, &corev1.Pod{}) err := instance.Start(ctx, nil, nil) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("must call CacheInto on Kind before calling Start")) - - close(done) + Expect(err.Error()).To(ContainSubstring("must create Kind with a non-nil cache")) }) - It("should return an error from Start if a type was not provided", func(done Done) { - instance := source.Kind{} - Expect(instance.InjectCache(&informertest.FakeInformers{})).To(Succeed()) + It("should return an error from Start if a type was not provided", func() { + instance := source.Kind(ic, nil) err := instance.Start(ctx, nil, nil) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("must specify Kind.Type")) - - close(done) + Expect(err.Error()).To(ContainSubstring("must create Kind with a non-nil object")) }) - It("should return an error if syncing fails", func(done Done) { - instance := source.Kind{Type: &corev1.Pod{}} + It("should return an error if syncing fails", func() { f := false - Expect(instance.InjectCache(&informertest.FakeInformers{Synced: &f})).To(Succeed()) + instance := source.Kind(&informertest.FakeInformers{Synced: &f}, &corev1.Pod{}) Expect(instance.Start(context.Background(), nil, nil)).NotTo(HaveOccurred()) err := instance.WaitForSync(context.Background()) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("cache did not sync")) - close(done) - }) Context("for a Kind not in the cache", func() { - It("should return an error when WaitForSync is called", func(done Done) { + It("should return an error when WaitForSync is called", func() { ic.Error = fmt.Errorf("test error") q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := &source.Kind{ - Type: &corev1.Pod{}, - } - Expect(instance.InjectCache(ic)).To(Succeed()) + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + + instance := source.Kind(ic, &corev1.Pod{}) err := instance.Start(ctx, handler.Funcs{}, q) Expect(err).NotTo(HaveOccurred()) - Expect(instance.WaitForSync(context.Background())).To(HaveOccurred()) - - close(done) + Eventually(instance.WaitForSync(context.Background())).Should(HaveOccurred()) }) }) - }) - Describe("KindWithCache", func() { - It("should not allow injecting a cache", func() { - instance := source.NewKindWithCache(nil, nil) - injected, err := inject.CacheInto(&informertest.FakeInformers{}, instance) - Expect(err).To(BeNil()) - Expect(injected).To(BeFalse()) - }) - - It("should return an error if syncing fails", func(done Done) { + It("should return an error if syncing fails", func() { f := false - instance := source.NewKindWithCache(&corev1.Pod{}, &informertest.FakeInformers{Synced: &f}) + instance := source.Kind(&informertest.FakeInformers{Synced: &f}, &corev1.Pod{}) Expect(instance.Start(context.Background(), nil, nil)).NotTo(HaveOccurred()) err := instance.WaitForSync(context.Background()) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(Equal("cache did not sync")) - close(done) - }) }) Describe("Func", func() { - It("should be called from Start", func(done Done) { + It("should be called from Start", func() { run := false instance := source.Func(func( context.Context, @@ -283,8 +249,6 @@ var _ = Describe("Source", func() { return expected }) Expect(instance.Start(ctx, nil, nil)).To(Equal(expected)) - - close(done) }) }) @@ -304,7 +268,7 @@ var _ = Describe("Source", func() { }) Context("for a source", func() { - It("should provide a GenericEvent", func(done Done) { + It("should provide a GenericEvent", func() { ch := make(chan event.GenericEvent) c := make(chan struct{}) p := &corev1.Pod{ @@ -325,21 +289,20 @@ var _ = Describe("Source", func() { q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") instance := &source.Channel{Source: ch} - Expect(inject.StopChannelInto(ctx.Done(), instance)).To(BeTrue()) err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(event.CreateEvent, workqueue.RateLimitingInterface) { + CreateFunc: func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected CreateEvent") }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected UpdateEvent") }, - DeleteFunc: func(event.DeleteEvent, workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected DeleteEvent") }, - GenericFunc: func(evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { + GenericFunc: func(ctx context.Context, evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { defer GinkgoRecover() // The empty event should have been filtered out by the predicates, // and will not be passed to the handler. @@ -353,9 +316,8 @@ var _ = Describe("Source", func() { ch <- invalidEvt ch <- evt <-c - close(done) }) - It("should get pending events processed once channel unblocked", func(done Done) { + It("should get pending events processed once channel unblocked", func() { ch := make(chan event.GenericEvent) unblock := make(chan struct{}) processed := make(chan struct{}) @@ -366,21 +328,20 @@ var _ = Describe("Source", func() { // Add a handler to get distribution blocked instance := &source.Channel{Source: ch} instance.DestBufferSize = 1 - Expect(inject.StopChannelInto(ctx.Done(), instance)).To(BeTrue()) err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(event.CreateEvent, workqueue.RateLimitingInterface) { + CreateFunc: func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected CreateEvent") }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected UpdateEvent") }, - DeleteFunc: func(event.DeleteEvent, workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected DeleteEvent") }, - GenericFunc: func(evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { + GenericFunc: func(ctx context.Context, evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { defer GinkgoRecover() // Block for the first time if eventCount == 0 { @@ -412,10 +373,8 @@ var _ = Describe("Source", func() { // Validate all of the events have been processed. Expect(eventCount).To(Equal(3)) - - close(done) }) - It("should be able to cope with events in the channel before the source is started", func(done Done) { + It("should be able to cope with events in the channel before the source is started", func() { ch := make(chan event.GenericEvent, 1) processed := make(chan struct{}) evt := event.GenericEvent{} @@ -425,22 +384,21 @@ var _ = Describe("Source", func() { // Add a handler to get distribution blocked instance := &source.Channel{Source: ch} instance.DestBufferSize = 1 - Expect(inject.StopChannelInto(ctx.Done(), instance)).To(BeTrue()) err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(event.CreateEvent, workqueue.RateLimitingInterface) { + CreateFunc: func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected CreateEvent") }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected UpdateEvent") }, - DeleteFunc: func(event.DeleteEvent, workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected DeleteEvent") }, - GenericFunc: func(evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { + GenericFunc: func(ctx context.Context, evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { defer GinkgoRecover() close(processed) @@ -449,8 +407,6 @@ var _ = Describe("Source", func() { Expect(err).NotTo(HaveOccurred()) <-processed - - close(done) }) It("should stop when the source channel is closed", func() { q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") @@ -466,25 +422,24 @@ var _ = Describe("Source", func() { By("feeding that channel to a channel source") src := &source.Channel{Source: ch} - Expect(inject.StopChannelInto(ctx.Done(), src)).To(BeTrue()) processed := make(chan struct{}) defer close(processed) err := src.Start(ctx, handler.Funcs{ - CreateFunc: func(event.CreateEvent, workqueue.RateLimitingInterface) { + CreateFunc: func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected CreateEvent") }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected UpdateEvent") }, - DeleteFunc: func(event.DeleteEvent, workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected DeleteEvent") }, - GenericFunc: func(evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { + GenericFunc: func(ctx context.Context, evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { defer GinkgoRecover() processed <- struct{}{} @@ -496,24 +451,15 @@ var _ = Describe("Source", func() { Eventually(processed).Should(Receive()) Consistently(processed).ShouldNot(Receive()) }) - It("should get error if no source specified", func(done Done) { + It("should get error if no source specified", func() { q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") instance := &source.Channel{ /*no source specified*/ } - Expect(inject.StopChannelInto(ctx.Done(), instance)).To(BeTrue()) err := instance.Start(ctx, handler.Funcs{}, q) Expect(err).To(Equal(fmt.Errorf("must specify Channel.Source"))) - close(done) - }) - It("should get error if no stop channel injected", func(done Done) { - q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") - instance := &source.Channel{Source: ch} - err := instance.Start(ctx, handler.Funcs{}, q) - Expect(err).To(Equal(fmt.Errorf("must call InjectStop on Channel before calling Start"))) - close(done) }) }) Context("for multi sources (handlers)", func() { - It("should provide GenericEvents for all handlers", func(done Done) { + It("should provide GenericEvents for all handlers", func() { ch := make(chan event.GenericEvent) p := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "bar"}, @@ -528,21 +474,20 @@ var _ = Describe("Source", func() { q := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test") instance := &source.Channel{Source: ch} - Expect(inject.StopChannelInto(ctx.Done(), instance)).To(BeTrue()) err := instance.Start(ctx, handler.Funcs{ - CreateFunc: func(event.CreateEvent, workqueue.RateLimitingInterface) { + CreateFunc: func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected CreateEvent") }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected UpdateEvent") }, - DeleteFunc: func(event.DeleteEvent, workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected DeleteEvent") }, - GenericFunc: func(evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { + GenericFunc: func(ctx context.Context, evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { defer GinkgoRecover() Expect(q2).To(BeIdenticalTo(q)) Expect(evt.Object).To(Equal(p)) @@ -553,19 +498,19 @@ var _ = Describe("Source", func() { Expect(err).NotTo(HaveOccurred()) err = instance.Start(ctx, handler.Funcs{ - CreateFunc: func(event.CreateEvent, workqueue.RateLimitingInterface) { + CreateFunc: func(context.Context, event.CreateEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected CreateEvent") }, - UpdateFunc: func(event.UpdateEvent, workqueue.RateLimitingInterface) { + UpdateFunc: func(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected UpdateEvent") }, - DeleteFunc: func(event.DeleteEvent, workqueue.RateLimitingInterface) { + DeleteFunc: func(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) { defer GinkgoRecover() Fail("Unexpected DeleteEvent") }, - GenericFunc: func(evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { + GenericFunc: func(ctx context.Context, evt event.GenericEvent, q2 workqueue.RateLimitingInterface) { defer GinkgoRecover() Expect(q2).To(BeIdenticalTo(q)) Expect(evt.Object).To(Equal(p)) @@ -581,7 +526,6 @@ var _ = Describe("Source", func() { // Validate the two handlers received same event Expect(resEvent1).To(Equal(resEvent2)) - close(done) }) }) }) diff --git a/pkg/webhook/admission/admission_suite_test.go b/pkg/webhook/admission/admission_suite_test.go index 339c7d83c3..f4e561b1b8 100644 --- a/pkg/webhook/admission/admission_suite_test.go +++ b/pkg/webhook/admission/admission_suite_test.go @@ -19,22 +19,18 @@ package admission import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestAdmissionWebhook(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Admission Webhook Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Admission Webhook Suite") } -var _ = BeforeSuite(func(done Done) { +var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) - - close(done) -}, 60) +}) diff --git a/pkg/webhook/admission/admissiontest/util.go b/pkg/webhook/admission/admissiontest/util.go index 685e8274d8..6e35a73907 100644 --- a/pkg/webhook/admission/admissiontest/util.go +++ b/pkg/webhook/admission/admissiontest/util.go @@ -27,7 +27,7 @@ import ( // or passes if ErrorToReturn is nil. type FakeValidator struct { // ErrorToReturn is the error for which the FakeValidator rejects all requests - ErrorToReturn error `json:"ErrorToReturn,omitempty"` + ErrorToReturn error `json:"errorToReturn,omitempty"` // GVKToReturn is the GroupVersionKind that the webhook operates on GVKToReturn schema.GroupVersionKind } diff --git a/pkg/webhook/admission/decode.go b/pkg/webhook/admission/decode.go index c7cb71b755..f14f130f7b 100644 --- a/pkg/webhook/admission/decode.go +++ b/pkg/webhook/admission/decode.go @@ -19,7 +19,6 @@ package admission import ( "fmt" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/json" @@ -32,8 +31,11 @@ type Decoder struct { } // NewDecoder creates a Decoder given the runtime.Scheme. -func NewDecoder(scheme *runtime.Scheme) (*Decoder, error) { - return &Decoder{codecs: serializer.NewCodecFactory(scheme)}, nil +func NewDecoder(scheme *runtime.Scheme) *Decoder { + if scheme == nil { + panic("scheme should never be nil") + } + return &Decoder{codecs: serializer.NewCodecFactory(scheme)} } // Decode decodes the inlined object in the AdmissionRequest into the passed-in runtime.Object. @@ -62,9 +64,13 @@ func (d *Decoder) DecodeRaw(rawObj runtime.RawExtension, into runtime.Object) er if len(rawObj.Raw) == 0 { return fmt.Errorf("there is no content to decode") } - if unstructuredInto, isUnstructured := into.(*unstructured.Unstructured); isUnstructured { + if unstructuredInto, isUnstructured := into.(runtime.Unstructured); isUnstructured { // unmarshal into unstructured's underlying object to avoid calling the decoder - return json.Unmarshal(rawObj.Raw, &unstructuredInto.Object) + var object map[string]interface{} + if err := json.Unmarshal(rawObj.Raw, &object); err != nil { + return err + } + unstructuredInto.SetUnstructuredContent(object) } deserializer := d.codecs.UniversalDeserializer() diff --git a/pkg/webhook/admission/decode_test.go b/pkg/webhook/admission/decode_test.go index c167c51026..66586da790 100644 --- a/pkg/webhook/admission/decode_test.go +++ b/pkg/webhook/admission/decode_test.go @@ -17,7 +17,7 @@ limitations under the License. package admission import ( - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" admissionv1 "k8s.io/api/admission/v1" @@ -32,9 +32,7 @@ var _ = Describe("Admission Webhook Decoder", func() { var decoder *Decoder BeforeEach(func() { By("creating a new decoder for a scheme") - var err error - decoder, err = NewDecoder(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) + decoder = NewDecoder(scheme.Scheme) Expect(decoder).NotTo(BeNil()) }) diff --git a/pkg/webhook/admission/defaulter.go b/pkg/webhook/admission/defaulter.go index 0d9aa7a838..a3b7207168 100644 --- a/pkg/webhook/admission/defaulter.go +++ b/pkg/webhook/admission/defaulter.go @@ -21,6 +21,8 @@ import ( "encoding/json" "net/http" + admissionv1 "k8s.io/api/admission/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -31,9 +33,9 @@ type Defaulter interface { } // DefaultingWebhookFor creates a new Webhook for Defaulting the provided type. -func DefaultingWebhookFor(defaulter Defaulter) *Webhook { +func DefaultingWebhookFor(scheme *runtime.Scheme, defaulter Defaulter) *Webhook { return &Webhook{ - Handler: &mutatingHandler{defaulter: defaulter}, + Handler: &mutatingHandler{defaulter: defaulter, decoder: NewDecoder(scheme)}, } } @@ -42,20 +44,26 @@ type mutatingHandler struct { decoder *Decoder } -var _ DecoderInjector = &mutatingHandler{} - -// InjectDecoder injects the decoder into a mutatingHandler. -func (h *mutatingHandler) InjectDecoder(d *Decoder) error { - h.decoder = d - return nil -} - // Handle handles admission requests. func (h *mutatingHandler) Handle(ctx context.Context, req Request) Response { + if h.decoder == nil { + panic("decoder should never be nil") + } if h.defaulter == nil { panic("defaulter should never be nil") } + // always skip when a DELETE operation received in mutation handler + // describe in https://github.com/kubernetes-sigs/controller-runtime/issues/1762 + if req.Operation == admissionv1.Delete { + return Response{AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: true, + Result: &metav1.Status{ + Code: http.StatusOK, + }, + }} + } + // Get the object in the request obj := h.defaulter.DeepCopyObject().(Defaulter) if err := h.decoder.Decode(req, obj); err != nil { diff --git a/pkg/webhook/admission/defaulter_custom.go b/pkg/webhook/admission/defaulter_custom.go new file mode 100644 index 0000000000..5f697e7dce --- /dev/null +++ b/pkg/webhook/admission/defaulter_custom.go @@ -0,0 +1,94 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package admission + +import ( + "context" + "encoding/json" + "errors" + "net/http" + + admissionv1 "k8s.io/api/admission/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// CustomDefaulter defines functions for setting defaults on resources. +type CustomDefaulter interface { + Default(ctx context.Context, obj runtime.Object) error +} + +// WithCustomDefaulter creates a new Webhook for a CustomDefaulter interface. +func WithCustomDefaulter(scheme *runtime.Scheme, obj runtime.Object, defaulter CustomDefaulter) *Webhook { + return &Webhook{ + Handler: &defaulterForType{object: obj, defaulter: defaulter, decoder: NewDecoder(scheme)}, + } +} + +type defaulterForType struct { + defaulter CustomDefaulter + object runtime.Object + decoder *Decoder +} + +// Handle handles admission requests. +func (h *defaulterForType) Handle(ctx context.Context, req Request) Response { + if h.decoder == nil { + panic("decoder should never be nil") + } + if h.defaulter == nil { + panic("defaulter should never be nil") + } + if h.object == nil { + panic("object should never be nil") + } + + // Always skip when a DELETE operation received in custom mutation handler. + if req.Operation == admissionv1.Delete { + return Response{AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: true, + Result: &metav1.Status{ + Code: http.StatusOK, + }, + }} + } + + ctx = NewContextWithRequest(ctx, req) + + // Get the object in the request + obj := h.object.DeepCopyObject() + if err := h.decoder.Decode(req, obj); err != nil { + return Errored(http.StatusBadRequest, err) + } + + // Default the object + if err := h.defaulter.Default(ctx, obj); err != nil { + var apiStatus apierrors.APIStatus + if errors.As(err, &apiStatus) { + return validationResponseFromStatus(false, apiStatus.Status()) + } + return Denied(err.Error()) + } + + // Create the patch + marshalled, err := json.Marshal(obj) + if err != nil { + return Errored(http.StatusInternalServerError, err) + } + return PatchResponseFromRaw(req.Object.Raw, marshalled) +} diff --git a/pkg/webhook/admission/defaulter_test.go b/pkg/webhook/admission/defaulter_test.go new file mode 100644 index 0000000000..cf7571663c --- /dev/null +++ b/pkg/webhook/admission/defaulter_test.go @@ -0,0 +1,68 @@ +package admission + +import ( + "context" + "net/http" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + admissionv1 "k8s.io/api/admission/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var _ = Describe("Defaulter Handler", func() { + + It("should return ok if received delete verb in defaulter handler", func() { + obj := &TestDefaulter{} + handler := DefaultingWebhookFor(admissionScheme, obj) + + resp := handler.Handle(context.TODO(), Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + Operation: admissionv1.Delete, + OldObject: runtime.RawExtension{ + Raw: []byte("{}"), + }, + }, + }) + Expect(resp.Allowed).Should(BeTrue()) + Expect(resp.Result.Code).Should(Equal(int32(http.StatusOK))) + }) + +}) + +// TestDefaulter. +var _ runtime.Object = &TestDefaulter{} + +type TestDefaulter struct { + Replica int `json:"replica,omitempty"` +} + +var testDefaulterGVK = schema.GroupVersionKind{Group: "foo.test.org", Version: "v1", Kind: "TestDefaulter"} + +func (d *TestDefaulter) GetObjectKind() schema.ObjectKind { return d } +func (d *TestDefaulter) DeepCopyObject() runtime.Object { + return &TestDefaulter{ + Replica: d.Replica, + } +} + +func (d *TestDefaulter) GroupVersionKind() schema.GroupVersionKind { + return testDefaulterGVK +} + +func (d *TestDefaulter) SetGroupVersionKind(gvk schema.GroupVersionKind) {} + +var _ runtime.Object = &TestDefaulterList{} + +type TestDefaulterList struct{} + +func (*TestDefaulterList) GetObjectKind() schema.ObjectKind { return nil } +func (*TestDefaulterList) DeepCopyObject() runtime.Object { return nil } + +func (d *TestDefaulter) Default() { + if d.Replica < 2 { + d.Replica = 2 + } +} diff --git a/pkg/webhook/admission/doc.go b/pkg/webhook/admission/doc.go index 0b274dd02b..8dc0cbec6f 100644 --- a/pkg/webhook/admission/doc.go +++ b/pkg/webhook/admission/doc.go @@ -20,9 +20,3 @@ Package admission provides implementation for admission webhook and methods to i See examples/mutatingwebhook.go and examples/validatingwebhook.go for examples of admission webhooks. */ package admission - -import ( - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" -) - -var log = logf.RuntimeLog.WithName("admission") diff --git a/pkg/webhook/admission/http.go b/pkg/webhook/admission/http.go index 3fa8872ff2..c3b7a5cc61 100644 --- a/pkg/webhook/admission/http.go +++ b/pkg/webhook/admission/http.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" v1 "k8s.io/api/admission/v1" @@ -53,15 +52,15 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { var reviewResponse Response if r.Body == nil { err = errors.New("request body is empty") - wh.log.Error(err, "bad request") + wh.getLogger(nil).Error(err, "bad request") reviewResponse = Errored(http.StatusBadRequest, err) wh.writeResponse(w, reviewResponse) return } defer r.Body.Close() - if body, err = ioutil.ReadAll(r.Body); err != nil { - wh.log.Error(err, "unable to read the body from the incoming request") + if body, err = io.ReadAll(r.Body); err != nil { + wh.getLogger(nil).Error(err, "unable to read the body from the incoming request") reviewResponse = Errored(http.StatusBadRequest, err) wh.writeResponse(w, reviewResponse) return @@ -70,7 +69,7 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { // verify the content type is accurate if contentType := r.Header.Get("Content-Type"); contentType != "application/json" { err = fmt.Errorf("contentType=%s, expected application/json", contentType) - wh.log.Error(err, "unable to process a request with an unknown content type", "content type", contentType) + wh.getLogger(nil).Error(err, "unable to process a request with an unknown content type", "content type", contentType) reviewResponse = Errored(http.StatusBadRequest, err) wh.writeResponse(w, reviewResponse) return @@ -89,12 +88,12 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { ar.SetGroupVersionKind(v1.SchemeGroupVersion.WithKind("AdmissionReview")) _, actualAdmRevGVK, err := admissionCodecs.UniversalDeserializer().Decode(body, nil, &ar) if err != nil { - wh.log.Error(err, "unable to decode the request") + wh.getLogger(nil).Error(err, "unable to decode the request") reviewResponse = Errored(http.StatusBadRequest, err) wh.writeResponse(w, reviewResponse) return } - wh.log.V(1).Info("received request", "UID", req.UID, "kind", req.Kind, "resource", req.Resource) + wh.getLogger(nil).V(1).Info("received request", "UID", req.UID, "kind", req.Kind, "resource", req.Resource) reviewResponse = wh.Handle(ctx, req) wh.writeResponseTyped(w, reviewResponse, actualAdmRevGVK) @@ -125,13 +124,21 @@ func (wh *Webhook) writeResponseTyped(w io.Writer, response Response, admRevGVK // writeAdmissionResponse writes ar to w. func (wh *Webhook) writeAdmissionResponse(w io.Writer, ar v1.AdmissionReview) { if err := json.NewEncoder(w).Encode(ar); err != nil { - wh.log.Error(err, "unable to encode the response") - wh.writeResponse(w, Errored(http.StatusInternalServerError, err)) + wh.getLogger(nil).Error(err, "unable to encode and write the response") + // Since the `ar v1.AdmissionReview` is a clear and legal object, + // it should not have problem to be marshalled into bytes. + // The error here is probably caused by the abnormal HTTP connection, + // e.g., broken pipe, so we can only write the error response once, + // to avoid endless circular calling. + serverError := Errored(http.StatusInternalServerError, err) + if err = json.NewEncoder(w).Encode(v1.AdmissionReview{Response: &serverError.AdmissionResponse}); err != nil { + wh.getLogger(nil).Error(err, "still unable to encode and write the InternalServerError response") + } } else { res := ar.Response - if log := wh.log; log.V(1).Enabled() { + if log := wh.getLogger(nil); log.V(1).Enabled() { if res.Result != nil { - log = log.WithValues("code", res.Result.Code, "reason", res.Result.Reason) + log = log.WithValues("code", res.Result.Code, "reason", res.Result.Reason, "message", res.Result.Message) } log.V(1).Info("wrote response", "UID", res.UID, "allowed", res.Allowed) } diff --git a/pkg/webhook/admission/http_test.go b/pkg/webhook/admission/http_test.go index 7dd2d5bcfc..be10aea459 100644 --- a/pkg/webhook/admission/http_test.go +++ b/pkg/webhook/admission/http_test.go @@ -23,14 +23,12 @@ import ( "io" "net/http" "net/http/httptest" + "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" admissionv1 "k8s.io/api/admission/v1" - - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" ) var _ = Describe("Admission Webhooks", func() { @@ -49,8 +47,6 @@ var _ = Describe("Admission Webhooks", func() { respRecorder = &httptest.ResponseRecorder{ Body: bytes.NewBuffer(nil), } - _, err := inject.LoggerInto(log.WithName("test-webhook"), webhook) - Expect(err).NotTo(HaveOccurred()) }) It("should return bad-request when given an empty body", func() { @@ -95,7 +91,6 @@ var _ = Describe("Admission Webhooks", func() { } webhook := &Webhook{ Handler: &fakeHandler{}, - log: logf.RuntimeLog.WithName("webhook"), } expected := fmt.Sprintf(`{%s,"response":{"uid":"","allowed":true,"status":{"metadata":{},"code":200}}} @@ -111,7 +106,6 @@ var _ = Describe("Admission Webhooks", func() { } webhook := &Webhook{ Handler: &fakeHandler{}, - log: logf.RuntimeLog.WithName("webhook"), } expected := fmt.Sprintf(`{%s,"response":{"uid":"","allowed":true,"status":{"metadata":{},"code":200}}} @@ -127,7 +121,6 @@ var _ = Describe("Admission Webhooks", func() { } webhook := &Webhook{ Handler: &fakeHandler{}, - log: logf.RuntimeLog.WithName("webhook"), } expected := fmt.Sprintf(`{%s,"response":{"uid":"","allowed":true,"status":{"metadata":{},"code":200}}} @@ -151,10 +144,9 @@ var _ = Describe("Admission Webhooks", func() { return Allowed(ctx.Value(key).(string)) }, }, - log: logf.RuntimeLog.WithName("webhook"), } - expected := fmt.Sprintf(`{%s,"response":{"uid":"","allowed":true,"status":{"metadata":{},"reason":%q,"code":200}}} + expected := fmt.Sprintf(`{%s,"response":{"uid":"","allowed":true,"status":{"metadata":{},"message":%q,"code":200}}} `, gvkJSONv1, value) ctx, cancel := context.WithCancel(context.WithValue(context.Background(), key, value)) @@ -179,10 +171,9 @@ var _ = Describe("Admission Webhooks", func() { WithContextFunc: func(ctx context.Context, r *http.Request) context.Context { return context.WithValue(ctx, key, r.Header["Content-Type"][0]) }, - log: logf.RuntimeLog.WithName("webhook"), } - expected := fmt.Sprintf(`{%s,"response":{"uid":"","allowed":true,"status":{"metadata":{},"reason":%q,"code":200}}} + expected := fmt.Sprintf(`{%s,"response":{"uid":"","allowed":true,"status":{"metadata":{},"message":%q,"code":200}}} `, gvkJSONv1, "application/json") ctx, cancel := context.WithCancel(context.Background()) @@ -190,6 +181,23 @@ var _ = Describe("Admission Webhooks", func() { webhook.ServeHTTP(respRecorder, req.WithContext(ctx)) Expect(respRecorder.Body.String()).To(Equal(expected)) }) + + It("should never run into circular calling if the writer has broken", func() { + req := &http.Request{ + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: nopCloser{Reader: bytes.NewBufferString(fmt.Sprintf(`{%s,"request":{}}`, gvkJSONv1))}, + } + webhook := &Webhook{ + Handler: &fakeHandler{}, + } + + bw := &brokenWriter{ResponseWriter: respRecorder} + Eventually(func() int { + // This should not be blocked by the circular calling of writeResponse and writeAdmissionResponse + webhook.ServeHTTP(bw, req) + return respRecorder.Body.Len() + }, time.Second*3).Should(Equal(0)) + }) }) }) @@ -200,20 +208,8 @@ type nopCloser struct { func (nopCloser) Close() error { return nil } type fakeHandler struct { - invoked bool - fn func(context.Context, Request) Response - decoder *Decoder - injectedString string -} - -func (h *fakeHandler) InjectDecoder(d *Decoder) error { - h.decoder = d - return nil -} - -func (h *fakeHandler) InjectString(s string) error { - h.injectedString = s - return nil + invoked bool + fn func(context.Context, Request) Response } func (h *fakeHandler) Handle(ctx context.Context, req Request) Response { @@ -225,3 +221,11 @@ func (h *fakeHandler) Handle(ctx context.Context, req Request) Response { Allowed: true, }} } + +type brokenWriter struct { + http.ResponseWriter +} + +func (bw *brokenWriter) Write(buf []byte) (int, error) { + return 0, fmt.Errorf("mock: write: broken pipe") +} diff --git a/pkg/webhook/admission/inject.go b/pkg/webhook/admission/inject.go deleted file mode 100644 index d5af0d598f..0000000000 --- a/pkg/webhook/admission/inject.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright 2019 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package admission - -// DecoderInjector is used by the ControllerManager to inject decoder into webhook handlers. -type DecoderInjector interface { - InjectDecoder(*Decoder) error -} - -// InjectDecoderInto will set decoder on i and return the result if it implements Decoder. Returns -// false if i does not implement Decoder. -func InjectDecoderInto(decoder *Decoder, i interface{}) (bool, error) { - if s, ok := i.(DecoderInjector); ok { - return true, s.InjectDecoder(decoder) - } - return false, nil -} diff --git a/pkg/webhook/admission/multi.go b/pkg/webhook/admission/multi.go index 26900cf2eb..2f7820d04b 100644 --- a/pkg/webhook/admission/multi.go +++ b/pkg/webhook/admission/multi.go @@ -25,8 +25,6 @@ import ( jsonpatch "gomodules.xyz/jsonpatch/v2" admissionv1 "k8s.io/api/admission/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" ) type multiMutating []Handler @@ -62,31 +60,6 @@ func (hs multiMutating) Handle(ctx context.Context, req Request) Response { } } -// InjectFunc injects the field setter into the handlers. -func (hs multiMutating) InjectFunc(f inject.Func) error { - // inject directly into the handlers. It would be more correct - // to do this in a sync.Once in Handle (since we don't have some - // other start/finalize-type method), but it's more efficient to - // do it here, presumably. - for _, handler := range hs { - if err := f(handler); err != nil { - return err - } - } - - return nil -} - -// InjectDecoder injects the decoder into the handlers. -func (hs multiMutating) InjectDecoder(d *Decoder) error { - for _, handler := range hs { - if _, err := InjectDecoderInto(d, handler); err != nil { - return err - } - } - return nil -} - // MultiMutatingHandler combines multiple mutating webhook handlers into a single // mutating webhook handler. Handlers are called in sequential order, and the first // `allowed: false` response may short-circuit the rest. Users must take care to @@ -120,28 +93,3 @@ func (hs multiValidating) Handle(ctx context.Context, req Request) Response { func MultiValidatingHandler(handlers ...Handler) Handler { return multiValidating(handlers) } - -// InjectFunc injects the field setter into the handlers. -func (hs multiValidating) InjectFunc(f inject.Func) error { - // inject directly into the handlers. It would be more correct - // to do this in a sync.Once in Handle (since we don't have some - // other start/finalize-type method), but it's more efficient to - // do it here, presumably. - for _, handler := range hs { - if err := f(handler); err != nil { - return err - } - } - - return nil -} - -// InjectDecoder injects the decoder into the handlers. -func (hs multiValidating) InjectDecoder(d *Decoder) error { - for _, handler := range hs { - if _, err := InjectDecoderInto(d, handler); err != nil { - return err - } - } - return nil -} diff --git a/pkg/webhook/admission/multi_test.go b/pkg/webhook/admission/multi_test.go index a8b51872a2..da85a52e42 100644 --- a/pkg/webhook/admission/multi_test.go +++ b/pkg/webhook/admission/multi_test.go @@ -19,7 +19,7 @@ package admission import ( "context" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" jsonpatch "gomodules.xyz/jsonpatch/v2" diff --git a/pkg/webhook/admission/response.go b/pkg/webhook/admission/response.go index 24ff1dee3c..ec1c88c989 100644 --- a/pkg/webhook/admission/response.go +++ b/pkg/webhook/admission/response.go @@ -26,21 +26,21 @@ import ( // Allowed constructs a response indicating that the given operation // is allowed (without any patches). -func Allowed(reason string) Response { - return ValidationResponse(true, reason) +func Allowed(message string) Response { + return ValidationResponse(true, message) } // Denied constructs a response indicating that the given operation // is not allowed. -func Denied(reason string) Response { - return ValidationResponse(false, reason) +func Denied(message string) Response { + return ValidationResponse(false, message) } // Patched constructs a response indicating that the given operation is // allowed, and that the target object should be modified by the given // JSONPatch operations. -func Patched(reason string, patches ...jsonpatch.JsonPatchOperation) Response { - resp := Allowed(reason) +func Patched(message string, patches ...jsonpatch.JsonPatchOperation) Response { + resp := Allowed(message) resp.Patches = patches return resp @@ -60,21 +60,24 @@ func Errored(code int32, err error) Response { } // ValidationResponse returns a response for admitting a request. -func ValidationResponse(allowed bool, reason string) Response { +func ValidationResponse(allowed bool, message string) Response { code := http.StatusForbidden + reason := metav1.StatusReasonForbidden if allowed { code = http.StatusOK + reason = "" } resp := Response{ AdmissionResponse: admissionv1.AdmissionResponse{ Allowed: allowed, Result: &metav1.Status{ - Code: int32(code), + Code: int32(code), + Reason: reason, }, }, } - if len(reason) > 0 { - resp.Result.Reason = metav1.StatusReason(reason) + if len(message) > 0 { + resp.Result.Message = message } return resp } diff --git a/pkg/webhook/admission/response_test.go b/pkg/webhook/admission/response_test.go index e96b0e6ca7..03107c92f5 100644 --- a/pkg/webhook/admission/response_test.go +++ b/pkg/webhook/admission/response_test.go @@ -20,7 +20,7 @@ import ( "errors" "net/http" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" jsonpatch "gomodules.xyz/jsonpatch/v2" @@ -49,8 +49,8 @@ var _ = Describe("Admission Webhook Response Helpers", func() { AdmissionResponse: admissionv1.AdmissionResponse{ Allowed: true, Result: &metav1.Status{ - Code: http.StatusOK, - Reason: "acceptable", + Code: http.StatusOK, + Message: "acceptable", }, }, }, @@ -65,7 +65,8 @@ var _ = Describe("Admission Webhook Response Helpers", func() { AdmissionResponse: admissionv1.AdmissionResponse{ Allowed: false, Result: &metav1.Status{ - Code: http.StatusForbidden, + Code: http.StatusForbidden, + Reason: metav1.StatusReasonForbidden, }, }, }, @@ -78,8 +79,9 @@ var _ = Describe("Admission Webhook Response Helpers", func() { AdmissionResponse: admissionv1.AdmissionResponse{ Allowed: false, Result: &metav1.Status{ - Code: http.StatusForbidden, - Reason: "UNACCEPTABLE!", + Code: http.StatusForbidden, + Reason: metav1.StatusReasonForbidden, + Message: "UNACCEPTABLE!", }, }, }, @@ -118,8 +120,8 @@ var _ = Describe("Admission Webhook Response Helpers", func() { AdmissionResponse: admissionv1.AdmissionResponse{ Allowed: true, Result: &metav1.Status{ - Code: http.StatusOK, - Reason: "some changes", + Code: http.StatusOK, + Message: "some changes", }, }, Patches: ops, @@ -146,15 +148,15 @@ var _ = Describe("Admission Webhook Response Helpers", func() { }) Describe("ValidationResponse", func() { - It("should populate a status with a reason when a reason is given", func() { + It("should populate a status with a message when a message is given", func() { By("checking that a message is populated for 'allowed' responses") Expect(ValidationResponse(true, "acceptable")).To(Equal( Response{ AdmissionResponse: admissionv1.AdmissionResponse{ Allowed: true, Result: &metav1.Status{ - Code: http.StatusOK, - Reason: "acceptable", + Code: http.StatusOK, + Message: "acceptable", }, }, }, @@ -166,8 +168,9 @@ var _ = Describe("Admission Webhook Response Helpers", func() { AdmissionResponse: admissionv1.AdmissionResponse{ Allowed: false, Result: &metav1.Status{ - Code: http.StatusForbidden, - Reason: "UNACCEPTABLE!", + Code: http.StatusForbidden, + Reason: metav1.StatusReasonForbidden, + Message: "UNACCEPTABLE!", }, }, }, @@ -193,7 +196,8 @@ var _ = Describe("Admission Webhook Response Helpers", func() { AdmissionResponse: admissionv1.AdmissionResponse{ Allowed: false, Result: &metav1.Status{ - Code: http.StatusForbidden, + Code: http.StatusForbidden, + Reason: metav1.StatusReasonForbidden, }, }, }, diff --git a/pkg/webhook/admission/validator.go b/pkg/webhook/admission/validator.go index 4b27e75ede..43ea3ee65f 100644 --- a/pkg/webhook/admission/validator.go +++ b/pkg/webhook/admission/validator.go @@ -35,9 +35,9 @@ type Validator interface { } // ValidatingWebhookFor creates a new Webhook for validating the provided type. -func ValidatingWebhookFor(validator Validator) *Webhook { +func ValidatingWebhookFor(scheme *runtime.Scheme, validator Validator) *Webhook { return &Webhook{ - Handler: &validatingHandler{validator: validator}, + Handler: &validatingHandler{validator: validator, decoder: NewDecoder(scheme)}, } } @@ -46,16 +46,11 @@ type validatingHandler struct { decoder *Decoder } -var _ DecoderInjector = &validatingHandler{} - -// InjectDecoder injects the decoder into a validatingHandler. -func (h *validatingHandler) InjectDecoder(d *Decoder) error { - h.decoder = d - return nil -} - // Handle handles admission requests. func (h *validatingHandler) Handle(ctx context.Context, req Request) Response { + if h.decoder == nil { + panic("decoder should never be nil") + } if h.validator == nil { panic("validator should never be nil") } diff --git a/pkg/webhook/admission/validator_custom.go b/pkg/webhook/admission/validator_custom.go new file mode 100644 index 0000000000..755e464a91 --- /dev/null +++ b/pkg/webhook/admission/validator_custom.go @@ -0,0 +1,111 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package admission + +import ( + "context" + "errors" + "fmt" + "net/http" + + v1 "k8s.io/api/admission/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" +) + +// CustomValidator defines functions for validating an operation. +type CustomValidator interface { + ValidateCreate(ctx context.Context, obj runtime.Object) error + ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) error + ValidateDelete(ctx context.Context, obj runtime.Object) error +} + +// WithCustomValidator creates a new Webhook for validating the provided type. +func WithCustomValidator(scheme *runtime.Scheme, obj runtime.Object, validator CustomValidator) *Webhook { + return &Webhook{ + Handler: &validatorForType{object: obj, validator: validator, decoder: NewDecoder(scheme)}, + } +} + +type validatorForType struct { + validator CustomValidator + object runtime.Object + decoder *Decoder +} + +// Handle handles admission requests. +func (h *validatorForType) Handle(ctx context.Context, req Request) Response { + if h.decoder == nil { + panic("decoder should never be nil") + } + if h.validator == nil { + panic("validator should never be nil") + } + if h.object == nil { + panic("object should never be nil") + } + + ctx = NewContextWithRequest(ctx, req) + + // Get the object in the request + obj := h.object.DeepCopyObject() + + var err error + switch req.Operation { + case v1.Connect: + // No validation for connect requests. + // TODO(vincepri): Should we validate CONNECT requests? In what cases? + case v1.Create: + if err := h.decoder.Decode(req, obj); err != nil { + return Errored(http.StatusBadRequest, err) + } + + err = h.validator.ValidateCreate(ctx, obj) + case v1.Update: + oldObj := obj.DeepCopyObject() + if err := h.decoder.DecodeRaw(req.Object, obj); err != nil { + return Errored(http.StatusBadRequest, err) + } + if err := h.decoder.DecodeRaw(req.OldObject, oldObj); err != nil { + return Errored(http.StatusBadRequest, err) + } + + err = h.validator.ValidateUpdate(ctx, oldObj, obj) + case v1.Delete: + // In reference to PR: https://github.com/kubernetes/kubernetes/pull/76346 + // OldObject contains the object being deleted + if err := h.decoder.DecodeRaw(req.OldObject, obj); err != nil { + return Errored(http.StatusBadRequest, err) + } + + err = h.validator.ValidateDelete(ctx, obj) + default: + return Errored(http.StatusBadRequest, fmt.Errorf("unknown operation request %q", req.Operation)) + } + + // Check the error message first. + if err != nil { + var apiStatus apierrors.APIStatus + if errors.As(err, &apiStatus) { + return validationResponseFromStatus(false, apiStatus.Status()) + } + return Denied(err.Error()) + } + + // Return allowed if everything succeeded. + return Allowed("") +} diff --git a/pkg/webhook/admission/validator_test.go b/pkg/webhook/admission/validator_test.go index 7fe19268d9..3da8f0b46a 100644 --- a/pkg/webhook/admission/validator_test.go +++ b/pkg/webhook/admission/validator_test.go @@ -21,7 +21,7 @@ import ( goerrors "errors" "net/http" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "sigs.k8s.io/controller-runtime/pkg/webhook/admission/admissiontest" @@ -37,7 +37,7 @@ var fakeValidatorVK = schema.GroupVersionKind{Group: "foo.test.org", Version: "v var _ = Describe("validatingHandler", func() { - decoder, _ := NewDecoder(scheme.Scheme) + decoder := NewDecoder(scheme.Scheme) Context("when dealing with successful results", func() { @@ -185,7 +185,7 @@ var _ = Describe("validatingHandler", func() { }) Expect(response.Allowed).Should(BeFalse()) Expect(response.Result.Code).Should(Equal(int32(http.StatusForbidden))) - Expect(string(response.Result.Reason)).Should(Equal(expectedError.Error())) + Expect(response.Result.Message).Should(Equal(expectedError.Error())) }) @@ -206,7 +206,8 @@ var _ = Describe("validatingHandler", func() { }) Expect(response.Allowed).Should(BeFalse()) Expect(response.Result.Code).Should(Equal(int32(http.StatusForbidden))) - Expect(string(response.Result.Reason)).Should(Equal(expectedError.Error())) + Expect(response.Result.Reason).Should(Equal(metav1.StatusReasonForbidden)) + Expect(response.Result.Message).Should(Equal(expectedError.Error())) }) @@ -223,8 +224,8 @@ var _ = Describe("validatingHandler", func() { }) Expect(response.Allowed).Should(BeFalse()) Expect(response.Result.Code).Should(Equal(int32(http.StatusForbidden))) - Expect(string(response.Result.Reason)).Should(Equal(expectedError.Error())) - + Expect(response.Result.Reason).Should(Equal(metav1.StatusReasonForbidden)) + Expect(response.Result.Message).Should(Equal(expectedError.Error())) }) }) diff --git a/pkg/webhook/admission/webhook.go b/pkg/webhook/admission/webhook.go index cf7dbcf68d..93b11f18ad 100644 --- a/pkg/webhook/admission/webhook.go +++ b/pkg/webhook/admission/webhook.go @@ -19,18 +19,18 @@ package admission import ( "context" "errors" + "fmt" "net/http" + "sync" "github.com/go-logr/logr" - jsonpatch "gomodules.xyz/jsonpatch/v2" + "gomodules.xyz/jsonpatch/v2" admissionv1 "k8s.io/api/admission/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/json" - "k8s.io/client-go/kubernetes/scheme" - - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/klog/v2" + logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics" ) @@ -121,101 +121,88 @@ type Webhook struct { // and potentially patches to apply to the handler. Handler Handler + // RecoverPanic indicates whether the panic caused by webhook should be recovered. + RecoverPanic bool + // WithContextFunc will allow you to take the http.Request.Context() and // add any additional information such as passing the request path or // headers thus allowing you to read them from within the handler WithContextFunc func(context.Context, *http.Request) context.Context - // decoder is constructed on receiving a scheme and passed down to then handler - decoder *Decoder + // LogConstructor is used to construct a logger for logging messages during webhook calls + // based on the given base logger (which might carry more values like the webhook's path). + // Note: LogConstructor has to be able to handle nil requests as we are also using it + // outside the context of requests. + LogConstructor func(base logr.Logger, req *Request) logr.Logger - log logr.Logger + setupLogOnce sync.Once + log logr.Logger } -// InjectLogger gets a handle to a logging instance, hopefully with more info about this particular webhook. -func (wh *Webhook) InjectLogger(l logr.Logger) error { - wh.log = l - return nil +// WithRecoverPanic takes a bool flag which indicates whether the panic caused by webhook should be recovered. +func (wh *Webhook) WithRecoverPanic(recoverPanic bool) *Webhook { + wh.RecoverPanic = recoverPanic + return wh } // Handle processes AdmissionRequest. // If the webhook is mutating type, it delegates the AdmissionRequest to each handler and merge the patches. // If the webhook is validating type, it delegates the AdmissionRequest to each handler and // deny the request if anyone denies. -func (wh *Webhook) Handle(ctx context.Context, req Request) Response { +func (wh *Webhook) Handle(ctx context.Context, req Request) (response Response) { + if wh.RecoverPanic { + defer func() { + if r := recover(); r != nil { + for _, fn := range utilruntime.PanicHandlers { + fn(r) + } + response = Errored(http.StatusInternalServerError, fmt.Errorf("panic: %v [recovered]", r)) + return + } + }() + } + + reqLog := wh.getLogger(&req) + reqLog = reqLog.WithValues("requestID", req.UID) + ctx = logf.IntoContext(ctx, reqLog) + resp := wh.Handler.Handle(ctx, req) if err := resp.Complete(req); err != nil { - wh.log.Error(err, "unable to encode response") + reqLog.Error(err, "unable to encode response") return Errored(http.StatusInternalServerError, errUnableToEncodeResponse) } return resp } -// InjectScheme injects a scheme into the webhook, in order to construct a Decoder. -func (wh *Webhook) InjectScheme(s *runtime.Scheme) error { - // TODO(directxman12): we should have a better way to pass this down - - var err error - wh.decoder, err = NewDecoder(s) - if err != nil { - return err - } - - // inject the decoder here too, just in case the order of calling this is not - // scheme first, then inject func - if wh.Handler != nil { - if _, err := InjectDecoderInto(wh.GetDecoder(), wh.Handler); err != nil { - return err +// getLogger constructs a logger from the injected log and LogConstructor. +func (wh *Webhook) getLogger(req *Request) logr.Logger { + wh.setupLogOnce.Do(func() { + if wh.log.GetSink() == nil { + wh.log = logf.Log.WithName("admission") } - } + }) - return nil -} - -// GetDecoder returns a decoder to decode the objects embedded in admission requests. -// It may be nil if we haven't received a scheme to use to determine object types yet. -func (wh *Webhook) GetDecoder() *Decoder { - return wh.decoder + logConstructor := wh.LogConstructor + if logConstructor == nil { + logConstructor = DefaultLogConstructor + } + return logConstructor(wh.log, req) } -// InjectFunc injects the field setter into the webhook. -func (wh *Webhook) InjectFunc(f inject.Func) error { - // inject directly into the handlers. It would be more correct - // to do this in a sync.Once in Handle (since we don't have some - // other start/finalize-type method), but it's more efficient to - // do it here, presumably. - - // also inject a decoder, and wrap this so that we get a setFields - // that injects a decoder (hopefully things don't ignore the duplicate - // InjectorInto call). - - var setFields inject.Func - setFields = func(target interface{}) error { - if err := f(target); err != nil { - return err - } - - if _, err := inject.InjectorInto(setFields, target); err != nil { - return err - } - - if _, err := InjectDecoderInto(wh.GetDecoder(), target); err != nil { - return err - } - - return nil +// DefaultLogConstructor adds some commonly interesting fields to the given logger. +func DefaultLogConstructor(base logr.Logger, req *Request) logr.Logger { + if req != nil { + return base.WithValues("object", klog.KRef(req.Namespace, req.Name), + "namespace", req.Namespace, "name", req.Name, + "resource", req.Resource, "user", req.UserInfo.Username, + ) } - - return setFields(wh.Handler) + return base } // StandaloneOptions let you configure a StandaloneWebhook. type StandaloneOptions struct { - // Scheme is the scheme used to resolve runtime.Objects to GroupVersionKinds / Resources - // Defaults to the kubernetes/client-go scheme.Scheme, but it's almost always better - // idea to pass your own scheme in. See the documentation in pkg/scheme for more information. - Scheme *runtime.Scheme // Logger to be used by the webhook. // If none is set, it defaults to log.Log global logger. Logger logr.Logger @@ -235,21 +222,29 @@ type StandaloneOptions struct { // in your own server/mux. In order to be accessed by a kubernetes cluster, // all webhook servers require TLS. func StandaloneWebhook(hook *Webhook, opts StandaloneOptions) (http.Handler, error) { - if opts.Scheme == nil { - opts.Scheme = scheme.Scheme - } - - if err := hook.InjectScheme(opts.Scheme); err != nil { - return nil, err - } - - if opts.Logger == nil { - opts.Logger = logf.RuntimeLog.WithName("webhook") + if opts.Logger.GetSink() != nil { + hook.log = opts.Logger } - hook.log = opts.Logger - if opts.MetricsPath == "" { return hook, nil } return metrics.InstrumentedHook(opts.MetricsPath, hook), nil } + +// requestContextKey is how we find the admission.Request in a context.Context. +type requestContextKey struct{} + +// RequestFromContext returns an admission.Request from ctx. +func RequestFromContext(ctx context.Context) (Request, error) { + if v, ok := ctx.Value(requestContextKey{}).(Request); ok { + return v, nil + } + + return Request{}, errors.New("admission.Request not found in context") +} + +// NewContextWithRequest returns a new Context, derived from ctx, which carries the +// provided admission.Request. +func NewContextWithRequest(ctx context.Context, req Request) context.Context { + return context.WithValue(ctx, requestContextKey{}, req) +} diff --git a/pkg/webhook/admission/webhook_test.go b/pkg/webhook/admission/webhook_test.go index 73b0be1694..5d603a5e6f 100644 --- a/pkg/webhook/admission/webhook_test.go +++ b/pkg/webhook/admission/webhook_test.go @@ -18,22 +18,34 @@ package admission import ( "context" + "io" "net/http" - . "github.com/onsi/ginkgo" + "github.com/go-logr/logr" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - jsonpatch "gomodules.xyz/jsonpatch/v2" + "github.com/onsi/gomega/gbytes" + "gomodules.xyz/jsonpatch/v2" admissionv1 "k8s.io/api/admission/v1" + authenticationv1 "k8s.io/api/authentication/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" machinerytypes "k8s.io/apimachinery/pkg/types" - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" ) var _ = Describe("Admission Webhooks", func() { + var ( + logBuffer *gbytes.Buffer + testLogger logr.Logger + ) + + BeforeEach(func() { + logBuffer = gbytes.NewBuffer() + testLogger = zap.New(zap.JSONEncoder(), zap.WriteTo(io.MultiWriter(logBuffer, GinkgoWriter))) + }) + allowHandler := func() *Webhook { handler := &fakeHandler{ fn: func(ctx context.Context, req Request) Response { @@ -46,7 +58,6 @@ var _ = Describe("Admission Webhooks", func() { } webhook := &Webhook{ Handler: handler, - log: logf.RuntimeLog.WithName("webhook"), } return webhook @@ -96,7 +107,6 @@ var _ = Describe("Admission Webhooks", func() { }, } }), - log: logf.RuntimeLog.WithName("webhook"), } By("invoking the webhook") @@ -113,7 +123,6 @@ var _ = Describe("Admission Webhooks", func() { Handler: HandlerFunc(func(ctx context.Context, req Request) Response { return Patched("", jsonpatch.Operation{Operation: "add", Path: "/a", Value: 2}, jsonpatch.Operation{Operation: "replace", Path: "/b", Value: 4}) }), - log: logf.RuntimeLog.WithName("webhook"), } By("invoking the webhook") @@ -125,93 +134,135 @@ var _ = Describe("Admission Webhooks", func() { Expect(resp.Patch).To(Equal([]byte(`[{"op":"add","path":"/a","value":2},{"op":"replace","path":"/b","value":4}]`))) }) - Describe("dependency injection", func() { - It("should set dependencies passed in on the handler", func() { - By("setting up a webhook and injecting it with a injection func that injects a string") - setFields := func(target interface{}) error { - inj, ok := target.(stringInjector) - if !ok { - return nil + It("should pass a request logger via the context", func() { + By("setting up a webhook that uses the request logger") + webhook := &Webhook{ + Handler: HandlerFunc(func(ctx context.Context, req Request) Response { + logf.FromContext(ctx).Info("Received request") + + return Response{ + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: true, + }, } + }), + log: testLogger, + } - return inj.InjectString("something") - } - handler := &fakeHandler{} - webhook := &Webhook{ - Handler: handler, - log: logf.RuntimeLog.WithName("webhook"), - } - Expect(setFields(webhook)).To(Succeed()) - Expect(inject.InjectorInto(setFields, webhook)).To(BeTrue()) + By("invoking the webhook") + resp := webhook.Handle(context.Background(), Request{AdmissionRequest: admissionv1.AdmissionRequest{ + UID: "test123", + Name: "foo", + Namespace: "bar", + Resource: metav1.GroupVersionResource{ + Group: "apps", + Version: "v1", + Resource: "deployments", + }, + UserInfo: authenticationv1.UserInfo{ + Username: "tim", + }, + }}) + Expect(resp.Allowed).To(BeTrue()) - By("checking that the string was injected") - Expect(handler.injectedString).To(Equal("something")) - }) + By("checking that the log message contains the request fields") + Eventually(logBuffer).Should(gbytes.Say(`"msg":"Received request","object":{"name":"foo","namespace":"bar"},"namespace":"bar","name":"foo","resource":{"group":"apps","version":"v1","resource":"deployments"},"user":"tim","requestID":"test123"}`)) + }) - It("should inject a decoder into the handler", func() { - By("setting up a webhook and injecting it with a injection func that injects a scheme") - setFields := func(target interface{}) error { - if _, err := inject.SchemeInto(runtime.NewScheme(), target); err != nil { - return err + It("should pass a request logger created by LogConstructor via the context", func() { + By("setting up a webhook that uses the request logger") + webhook := &Webhook{ + Handler: HandlerFunc(func(ctx context.Context, req Request) Response { + logf.FromContext(ctx).Info("Received request") + + return Response{ + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: true, + }, } - return nil - } - handler := &fakeHandler{} - webhook := &Webhook{ - Handler: handler, - log: logf.RuntimeLog.WithName("webhook"), + }), + LogConstructor: func(base logr.Logger, req *Request) logr.Logger { + return base.WithValues("operation", req.Operation) + }, + log: testLogger, + } + + By("invoking the webhook") + resp := webhook.Handle(context.Background(), Request{AdmissionRequest: admissionv1.AdmissionRequest{ + UID: "test123", + Operation: admissionv1.Create, + }}) + Expect(resp.Allowed).To(BeTrue()) + + By("checking that the log message contains the request fields") + Eventually(logBuffer).Should(gbytes.Say(`"msg":"Received request","operation":"CREATE","requestID":"test123"}`)) + }) + + Describe("panic recovery", func() { + It("should recover panic if RecoverPanic is true", func() { + panicHandler := func() *Webhook { + handler := &fakeHandler{ + fn: func(ctx context.Context, req Request) Response { + panic("fake panic test") + }, + } + webhook := &Webhook{ + Handler: handler, + RecoverPanic: true, + } + + return webhook } - Expect(setFields(webhook)).To(Succeed()) - Expect(inject.InjectorInto(setFields, webhook)).To(BeTrue()) - By("checking that the decoder was injected") - Expect(handler.decoder).NotTo(BeNil()) + By("setting up a webhook with a panicking handler") + webhook := panicHandler() + + By("invoking the webhook") + resp := webhook.Handle(context.Background(), Request{}) + + By("checking that it errored the request") + Expect(resp.Allowed).To(BeFalse()) + Expect(resp.Result.Code).To(Equal(int32(http.StatusInternalServerError))) + Expect(resp.Result.Message).To(Equal("panic: fake panic test [recovered]")) }) - It("should pass a setFields that also injects a decoder into sub-dependencies", func() { - By("setting up a webhook and injecting it with a injection func that injects a scheme") - setFields := func(target interface{}) error { - if _, err := inject.SchemeInto(runtime.NewScheme(), target); err != nil { - return err + It("should not recover panic if RecoverPanic is false by default", func() { + panicHandler := func() *Webhook { + handler := &fakeHandler{ + fn: func(ctx context.Context, req Request) Response { + panic("fake panic test") + }, } - return nil - } - handler := &handlerWithSubDependencies{ - Handler: HandlerFunc(func(ctx context.Context, req Request) Response { - return Response{} - }), - dep: &subDep{}, - } - webhook := &Webhook{ - Handler: handler, + webhook := &Webhook{ + Handler: handler, + } + + return webhook } - Expect(setFields(webhook)).To(Succeed()) - Expect(inject.InjectorInto(setFields, webhook)).To(BeTrue()) - By("checking that setFields sets the decoder as well") - Expect(handler.dep.decoder).NotTo(BeNil()) + By("setting up a webhook with a panicking handler") + defer func() { + Expect(recover()).ShouldNot(BeNil()) + }() + webhook := panicHandler() + + By("invoking the webhook") + webhook.Handle(context.Background(), Request{}) }) }) }) -type stringInjector interface { - InjectString(s string) error -} - -type handlerWithSubDependencies struct { - Handler - dep *subDep -} - -func (h *handlerWithSubDependencies) InjectFunc(f inject.Func) error { - return f(h.dep) -} +var _ = Describe("Should be able to write/read admission.Request to/from context", func() { + ctx := context.Background() + testRequest := Request{ + admissionv1.AdmissionRequest{ + UID: "test-uid", + }, + } -type subDep struct { - decoder *Decoder -} + ctx = NewContextWithRequest(ctx, testRequest) -func (d *subDep) InjectDecoder(dec *Decoder) error { - d.decoder = dec - return nil -} + gotRequest, err := RequestFromContext(ctx) + Expect(err).To(Not(HaveOccurred())) + Expect(gotRequest).To(Equal(testRequest)) +}) diff --git a/pkg/webhook/alias.go b/pkg/webhook/alias.go index 1a831016af..293137db49 100644 --- a/pkg/webhook/alias.go +++ b/pkg/webhook/alias.go @@ -29,6 +29,12 @@ type Defaulter = admission.Defaulter // Validator defines functions for validating an operation. type Validator = admission.Validator +// CustomDefaulter defines functions for setting defaults on resources. +type CustomDefaulter = admission.CustomDefaulter + +// CustomValidator defines functions for validating an operation. +type CustomValidator = admission.CustomValidator + // AdmissionRequest defines the input for an admission handler. // It contains information to identify the object in // question (group, version, kind, resource, subresource, diff --git a/pkg/webhook/authentication/authentication_suite_test.go b/pkg/webhook/authentication/authentication_suite_test.go index 0988f81285..29f7b3e17e 100644 --- a/pkg/webhook/authentication/authentication_suite_test.go +++ b/pkg/webhook/authentication/authentication_suite_test.go @@ -19,22 +19,18 @@ package authentication import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestAuthenticationWebhook(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Authentication Webhook Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Authentication Webhook Suite") } -var _ = BeforeSuite(func(done Done) { +var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) - - close(done) -}, 60) +}) diff --git a/pkg/webhook/authentication/doc.go b/pkg/webhook/authentication/doc.go index a1b45c1aef..d2b85f378c 100644 --- a/pkg/webhook/authentication/doc.go +++ b/pkg/webhook/authentication/doc.go @@ -21,9 +21,3 @@ methods to implement authentication webhook handlers. See examples/tokenreview/ for an example of authentication webhooks. */ package authentication - -import ( - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" -) - -var log = logf.RuntimeLog.WithName("authentication") diff --git a/pkg/webhook/authentication/http.go b/pkg/webhook/authentication/http.go index 19f2f9e51c..dfed3efea0 100644 --- a/pkg/webhook/authentication/http.go +++ b/pkg/webhook/authentication/http.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" authenticationv1 "k8s.io/api/authentication/v1" @@ -53,15 +52,15 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { var reviewResponse Response if r.Body == nil { err = errors.New("request body is empty") - wh.log.Error(err, "bad request") + wh.getLogger(nil).Error(err, "bad request") reviewResponse = Errored(err) wh.writeResponse(w, reviewResponse) return } defer r.Body.Close() - if body, err = ioutil.ReadAll(r.Body); err != nil { - wh.log.Error(err, "unable to read the body from the incoming request") + if body, err = io.ReadAll(r.Body); err != nil { + wh.getLogger(nil).Error(err, "unable to read the body from the incoming request") reviewResponse = Errored(err) wh.writeResponse(w, reviewResponse) return @@ -70,7 +69,7 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { // verify the content type is accurate if contentType := r.Header.Get("Content-Type"); contentType != "application/json" { err = fmt.Errorf("contentType=%s, expected application/json", contentType) - wh.log.Error(err, "unable to process a request with an unknown content type", "content type", contentType) + wh.getLogger(nil).Error(err, "unable to process a request with an unknown content type", "content type", contentType) reviewResponse = Errored(err) wh.writeResponse(w, reviewResponse) return @@ -90,16 +89,16 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { ar.SetGroupVersionKind(authenticationv1.SchemeGroupVersion.WithKind("TokenReview")) _, actualTokRevGVK, err := authenticationCodecs.UniversalDeserializer().Decode(body, nil, &ar) if err != nil { - wh.log.Error(err, "unable to decode the request") + wh.getLogger(&req).Error(err, "unable to decode the request") reviewResponse = Errored(err) wh.writeResponse(w, reviewResponse) return } - wh.log.V(1).Info("received request", "UID", req.UID, "kind", req.Kind) + wh.getLogger(&req).V(1).Info("received request", "UID", req.UID, "kind", req.Kind) if req.Spec.Token == "" { err = errors.New("token is empty") - wh.log.Error(err, "bad request") + wh.getLogger(&req).Error(err, "bad request") reviewResponse = Errored(err) wh.writeResponse(w, reviewResponse) return @@ -132,12 +131,12 @@ func (wh *Webhook) writeResponseTyped(w io.Writer, response Response, tokRevGVK // writeTokenResponse writes ar to w. func (wh *Webhook) writeTokenResponse(w io.Writer, ar authenticationv1.TokenReview) { if err := json.NewEncoder(w).Encode(ar); err != nil { - wh.log.Error(err, "unable to encode the response") + wh.getLogger(nil).Error(err, "unable to encode the response") wh.writeResponse(w, Errored(err)) } res := ar - if log := wh.log; log.V(1).Enabled() { - log.V(1).Info("wrote response", "UID", res.UID, "authenticated", res.Status.Authenticated) + if wh.getLogger(nil).V(1).Enabled() { + wh.getLogger(nil).V(1).Info("wrote response", "UID", res.UID, "authenticated", res.Status.Authenticated) } } diff --git a/pkg/webhook/authentication/http_test.go b/pkg/webhook/authentication/http_test.go index 3007ddda78..86bd5d0153 100644 --- a/pkg/webhook/authentication/http_test.go +++ b/pkg/webhook/authentication/http_test.go @@ -24,19 +24,15 @@ import ( "net/http" "net/http/httptest" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" authenticationv1 "k8s.io/api/authentication/v1" - - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" ) var _ = Describe("Authentication Webhooks", func() { const ( - gvkJSONv1 = `"kind":"TokenReview","apiVersion":"authentication.k8s.io/v1"` - gvkJSONv1beta1 = `"kind":"TokenReview","apiVersion":"authentication.k8s.io/v1beta1"` + gvkJSONv1 = `"kind":"TokenReview","apiVersion":"authentication.k8s.io/v1"` ) Describe("HTTP Handler", func() { @@ -48,8 +44,6 @@ var _ = Describe("Authentication Webhooks", func() { respRecorder = &httptest.ResponseRecorder{ Body: bytes.NewBuffer(nil), } - _, err := inject.LoggerInto(log.WithName("test-webhook"), webhook) - Expect(err).NotTo(HaveOccurred()) }) It("should return bad-request when given an empty body", func() { @@ -108,7 +102,6 @@ var _ = Describe("Authentication Webhooks", func() { } webhook := &Webhook{ Handler: &fakeHandler{}, - log: logf.RuntimeLog.WithName("webhook"), } expected := fmt.Sprintf(`{%s,"metadata":{"creationTimestamp":null},"spec":{},"status":{"authenticated":true,"user":{}}} @@ -126,7 +119,6 @@ var _ = Describe("Authentication Webhooks", func() { } webhook := &Webhook{ Handler: &fakeHandler{}, - log: logf.RuntimeLog.WithName("webhook"), } expected := fmt.Sprintf(`{%s,"metadata":{"creationTimestamp":null},"spec":{},"status":{"authenticated":true,"user":{}}} @@ -135,23 +127,6 @@ var _ = Describe("Authentication Webhooks", func() { Expect(respRecorder.Body.String()).To(Equal(expected)) }) - It("should return the v1beta1 response given by the handler", func() { - req := &http.Request{ - Header: http.Header{"Content-Type": []string{"application/json"}}, - Method: http.MethodPost, - Body: nopCloser{Reader: bytes.NewBufferString(fmt.Sprintf(`{%s,"spec":{"token":"foobar"}}`, gvkJSONv1beta1))}, - } - webhook := &Webhook{ - Handler: &fakeHandler{}, - log: logf.RuntimeLog.WithName("webhook"), - } - - expected := fmt.Sprintf(`{%s,"metadata":{"creationTimestamp":null},"spec":{},"status":{"authenticated":true,"user":{}}} -`, gvkJSONv1beta1) - webhook.ServeHTTP(respRecorder, req) - Expect(respRecorder.Body.String()).To(Equal(expected)) - }) - It("should present the Context from the HTTP request, if any", func() { req := &http.Request{ Header: http.Header{"Content-Type": []string{"application/json"}}, @@ -168,7 +143,6 @@ var _ = Describe("Authentication Webhooks", func() { return Authenticated(ctx.Value(key).(string), authenticationv1.UserInfo{}) }, }, - log: logf.RuntimeLog.WithName("webhook"), } expected := fmt.Sprintf(`{%s,"metadata":{"creationTimestamp":null},"spec":{},"status":{"authenticated":true,"user":{},"error":%q}} @@ -197,7 +171,6 @@ var _ = Describe("Authentication Webhooks", func() { WithContextFunc: func(ctx context.Context, r *http.Request) context.Context { return context.WithValue(ctx, key, r.Header["Content-Type"][0]) }, - log: logf.RuntimeLog.WithName("webhook"), } expected := fmt.Sprintf(`{%s,"metadata":{"creationTimestamp":null},"spec":{},"status":{"authenticated":true,"user":{},"error":%q}} @@ -218,14 +191,8 @@ type nopCloser struct { func (nopCloser) Close() error { return nil } type fakeHandler struct { - invoked bool - fn func(context.Context, Request) Response - injectedString string -} - -func (h *fakeHandler) InjectString(s string) error { - h.injectedString = s - return nil + invoked bool + fn func(context.Context, Request) Response } func (h *fakeHandler) Handle(ctx context.Context, req Request) Response { diff --git a/pkg/webhook/authentication/response_test.go b/pkg/webhook/authentication/response_test.go index 22c1ee3ba7..6eeef87c11 100644 --- a/pkg/webhook/authentication/response_test.go +++ b/pkg/webhook/authentication/response_test.go @@ -19,7 +19,7 @@ package authentication import ( "errors" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" authenticationv1 "k8s.io/api/authentication/v1" diff --git a/pkg/webhook/authentication/webhook.go b/pkg/webhook/authentication/webhook.go index b1229e422e..7c5e627aa5 100644 --- a/pkg/webhook/authentication/webhook.go +++ b/pkg/webhook/authentication/webhook.go @@ -20,11 +20,13 @@ import ( "context" "errors" "net/http" + "sync" "github.com/go-logr/logr" authenticationv1 "k8s.io/api/authentication/v1" + "k8s.io/klog/v2" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" + logf "sigs.k8s.io/controller-runtime/pkg/log" ) var ( @@ -85,45 +87,39 @@ type Webhook struct { // headers thus allowing you to read them from within the handler WithContextFunc func(context.Context, *http.Request) context.Context - log logr.Logger -} - -// InjectLogger gets a handle to a logging instance, hopefully with more info about this particular webhook. -func (wh *Webhook) InjectLogger(l logr.Logger) error { - wh.log = l - return nil + setupLogOnce sync.Once + log logr.Logger } // Handle processes TokenReview. func (wh *Webhook) Handle(ctx context.Context, req Request) Response { resp := wh.Handler.Handle(ctx, req) if err := resp.Complete(req); err != nil { - wh.log.Error(err, "unable to encode response") + wh.getLogger(&req).Error(err, "unable to encode response") return Errored(errUnableToEncodeResponse) } return resp } -// InjectFunc injects the field setter into the webhook. -func (wh *Webhook) InjectFunc(f inject.Func) error { - // inject directly into the handlers. It would be more correct - // to do this in a sync.Once in Handle (since we don't have some - // other start/finalize-type method), but it's more efficient to - // do it here, presumably. - - var setFields inject.Func - setFields = func(target interface{}) error { - if err := f(target); err != nil { - return err +// getLogger constructs a logger from the injected log and LogConstructor. +func (wh *Webhook) getLogger(req *Request) logr.Logger { + wh.setupLogOnce.Do(func() { + if wh.log.GetSink() == nil { + wh.log = logf.Log.WithName("authentication") } + }) - if _, err := inject.InjectorInto(setFields, target); err != nil { - return err - } + return logConstructor(wh.log, req) +} - return nil +// logConstructor adds some commonly interesting fields to the given logger. +func logConstructor(base logr.Logger, req *Request) logr.Logger { + if req != nil { + return base.WithValues("object", klog.KRef(req.Namespace, req.Name), + "namespace", req.Namespace, "name", req.Name, + "user", req.Status.User.Username, + ) } - - return setFields(wh.Handler) + return base } diff --git a/pkg/webhook/authentication/webhook_test.go b/pkg/webhook/authentication/webhook_test.go index 55849ece32..3df446d898 100644 --- a/pkg/webhook/authentication/webhook_test.go +++ b/pkg/webhook/authentication/webhook_test.go @@ -19,15 +19,12 @@ package authentication import ( "context" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" authenticationv1 "k8s.io/api/authentication/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" machinerytypes "k8s.io/apimachinery/pkg/types" - - logf "sigs.k8s.io/controller-runtime/pkg/internal/log" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" ) var _ = Describe("Authentication Webhooks", func() { @@ -45,7 +42,6 @@ var _ = Describe("Authentication Webhooks", func() { } webhook := &Webhook{ Handler: handler, - log: logf.RuntimeLog.WithName("webhook"), } return webhook @@ -97,7 +93,6 @@ var _ = Describe("Authentication Webhooks", func() { }, } }), - log: logf.RuntimeLog.WithName("webhook"), } By("invoking the webhook") @@ -108,33 +103,4 @@ var _ = Describe("Authentication Webhooks", func() { Expect(resp.Status.Authenticated).To(BeTrue()) Expect(resp.Status.Error).To(Equal("Ground Control to Major Tom")) }) - - Describe("dependency injection", func() { - It("should set dependencies passed in on the handler", func() { - By("setting up a webhook and injecting it with a injection func that injects a string") - setFields := func(target interface{}) error { - inj, ok := target.(stringInjector) - if !ok { - return nil - } - - return inj.InjectString("something") - } - handler := &fakeHandler{} - webhook := &Webhook{ - Handler: handler, - log: logf.RuntimeLog.WithName("webhook"), - } - Expect(setFields(webhook)).To(Succeed()) - Expect(inject.InjectorInto(setFields, webhook)).To(BeTrue()) - - By("checking that the string was injected") - Expect(handler.injectedString).To(Equal("something")) - }) - - }) }) - -type stringInjector interface { - InjectString(s string) error -} diff --git a/pkg/webhook/conversion/conversion.go b/pkg/webhook/conversion/conversion.go index af9e673ccb..249a364b38 100644 --- a/pkg/webhook/conversion/conversion.go +++ b/pkg/webhook/conversion/conversion.go @@ -26,7 +26,7 @@ import ( "fmt" "net/http" - apix "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apix "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -39,28 +39,20 @@ var ( log = logf.Log.WithName("conversion-webhook") ) -// Webhook implements a CRD conversion webhook HTTP handler. -type Webhook struct { - scheme *runtime.Scheme - decoder *Decoder +func NewWebhookHandler(scheme *runtime.Scheme) http.Handler { + return &webhook{scheme: scheme, decoder: NewDecoder(scheme)} } -// InjectScheme injects a scheme into the webhook, in order to construct a Decoder. -func (wh *Webhook) InjectScheme(s *runtime.Scheme) error { - var err error - wh.scheme = s - wh.decoder, err = NewDecoder(s) - if err != nil { - return err - } - - return nil +// webhook implements a CRD conversion webhook HTTP handler. +type webhook struct { + scheme *runtime.Scheme + decoder *Decoder } // ensure Webhook implements http.Handler -var _ http.Handler = &Webhook{} +var _ http.Handler = &webhook{} -func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { +func (wh *webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { convertReview := &apix.ConversionReview{} err := json.NewDecoder(r.Body).Decode(convertReview) if err != nil { @@ -69,6 +61,12 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + if convertReview.Request == nil { + log.Error(nil, "conversion request is nil") + w.WriteHeader(http.StatusBadRequest) + return + } + // TODO(droot): may be move the conversion logic to a separate module to // decouple it from the http layer ? resp, err := wh.handleConvertRequest(convertReview.Request) @@ -89,7 +87,7 @@ func (wh *Webhook) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // handles a version conversion request. -func (wh *Webhook) handleConvertRequest(req *apix.ConversionRequest) (*apix.ConversionResponse, error) { +func (wh *webhook) handleConvertRequest(req *apix.ConversionRequest) (*apix.ConversionResponse, error) { if req == nil { return nil, fmt.Errorf("conversion request is nil") } @@ -122,7 +120,7 @@ func (wh *Webhook) handleConvertRequest(req *apix.ConversionRequest) (*apix.Conv // convertObject will convert given a src object to dst object. // Note(droot): couldn't find a way to reduce the cyclomatic complexity under 10 // without compromising readability, so disabling gocyclo linter -func (wh *Webhook) convertObject(src, dst runtime.Object) error { +func (wh *webhook) convertObject(src, dst runtime.Object) error { srcGVK := src.GetObjectKind().GroupVersionKind() dstGVK := dst.GetObjectKind().GroupVersionKind() @@ -149,7 +147,7 @@ func (wh *Webhook) convertObject(src, dst runtime.Object) error { } } -func (wh *Webhook) convertViaHub(src, dst conversion.Convertible) error { +func (wh *webhook) convertViaHub(src, dst conversion.Convertible) error { hub, err := wh.getHub(src) if err != nil { return err @@ -173,7 +171,7 @@ func (wh *Webhook) convertViaHub(src, dst conversion.Convertible) error { } // getHub returns an instance of the Hub for passed-in object's group/kind. -func (wh *Webhook) getHub(obj runtime.Object) (conversion.Hub, error) { +func (wh *webhook) getHub(obj runtime.Object) (conversion.Hub, error) { gvks, err := objectGVKs(wh.scheme, obj) if err != nil { return nil, err @@ -201,7 +199,7 @@ func (wh *Webhook) getHub(obj runtime.Object) (conversion.Hub, error) { } // allocateDstObject returns an instance for a given GVK. -func (wh *Webhook) allocateDstObject(apiVersion, kind string) (runtime.Object, error) { +func (wh *webhook) allocateDstObject(apiVersion, kind string) (runtime.Object, error) { gvk := schema.FromAPIVersionAndKind(apiVersion, kind) obj, err := wh.scheme.New(gvk) diff --git a/pkg/webhook/conversion/conversion_suite_test.go b/pkg/webhook/conversion/conversion_suite_test.go index 76bbf505ff..7ca3c48ba2 100644 --- a/pkg/webhook/conversion/conversion_suite_test.go +++ b/pkg/webhook/conversion/conversion_suite_test.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -18,22 +18,18 @@ package conversion import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestConversionWebhook(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "CRD conversion Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "CRD conversion Suite") } -var _ = BeforeSuite(func(done Done) { +var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) - - close(done) }) diff --git a/pkg/webhook/conversion/conversion_test.go b/pkg/webhook/conversion/conversion_test.go index 64ca7b575d..be984e232b 100644 --- a/pkg/webhook/conversion/conversion_test.go +++ b/pkg/webhook/conversion/conversion_test.go @@ -14,25 +14,26 @@ See the License for the specific language governing permissions and limitations under the License. */ -package conversion +package conversion_test import ( "bytes" "encoding/json" - "io/ioutil" + "io" "net/http" "net/http/httptest" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1beta1 "k8s.io/api/apps/v1beta1" - apix "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apix "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" kscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/webhook/conversion" jobsv1 "sigs.k8s.io/controller-runtime/pkg/webhook/conversion/testdata/api/v1" jobsv2 "sigs.k8s.io/controller-runtime/pkg/webhook/conversion/testdata/api/v2" jobsv3 "sigs.k8s.io/controller-runtime/pkg/webhook/conversion/testdata/api/v3" @@ -41,9 +42,9 @@ import ( var _ = Describe("Conversion Webhook", func() { var respRecorder *httptest.ResponseRecorder - var decoder *Decoder + var decoder *conversion.Decoder var scheme *runtime.Scheme - webhook := Webhook{} + var wh http.Handler BeforeEach(func() { respRecorder = &httptest.ResponseRecorder{ @@ -55,12 +56,9 @@ var _ = Describe("Conversion Webhook", func() { Expect(jobsv1.AddToScheme(scheme)).To(Succeed()) Expect(jobsv2.AddToScheme(scheme)).To(Succeed()) Expect(jobsv3.AddToScheme(scheme)).To(Succeed()) - Expect(webhook.InjectScheme(scheme)).To(Succeed()) - - var err error - decoder, err = NewDecoder(scheme) - Expect(err).NotTo(HaveOccurred()) + decoder = conversion.NewDecoder(scheme) + wh = conversion.NewWebhookHandler(scheme) }) doRequest := func(convReq *apix.ConversionReview) *apix.ConversionReview { @@ -70,9 +68,9 @@ var _ = Describe("Conversion Webhook", func() { convReview := &apix.ConversionReview{} req := &http.Request{ - Body: ioutil.NopCloser(bytes.NewReader(payload.Bytes())), + Body: io.NopCloser(bytes.NewReader(payload.Bytes())), } - webhook.ServeHTTP(respRecorder, req) + wh.ServeHTTP(respRecorder, req) Expect(json.NewDecoder(respRecorder.Result().Body).Decode(convReview)).To(Succeed()) return convReview } @@ -316,7 +314,7 @@ var _ = Describe("IsConvertible", func() { It("should not error for uninitialized types", func() { obj := &jobsv2.ExternalJob{} - ok, err := IsConvertible(scheme, obj) + ok, err := conversion.IsConvertible(scheme, obj) Expect(err).NotTo(HaveOccurred()) Expect(ok).To(BeTrue()) }) @@ -329,7 +327,7 @@ var _ = Describe("IsConvertible", func() { }, } - ok, err := IsConvertible(scheme, obj) + ok, err := conversion.IsConvertible(scheme, obj) Expect(err).NotTo(HaveOccurred()) Expect(ok).To(BeTrue()) }) @@ -342,7 +340,7 @@ var _ = Describe("IsConvertible", func() { }, } - ok, err := IsConvertible(scheme, obj) + ok, err := conversion.IsConvertible(scheme, obj) Expect(err).NotTo(HaveOccurred()) Expect(ok).To(BeTrue()) }) @@ -355,7 +353,7 @@ var _ = Describe("IsConvertible", func() { }, } - ok, err := IsConvertible(scheme, obj) + ok, err := conversion.IsConvertible(scheme, obj) Expect(err).NotTo(HaveOccurred()) Expect(ok).ToNot(BeTrue()) }) diff --git a/pkg/webhook/conversion/decoder.go b/pkg/webhook/conversion/decoder.go index 6a9e9c2365..b6bb8bd938 100644 --- a/pkg/webhook/conversion/decoder.go +++ b/pkg/webhook/conversion/decoder.go @@ -30,8 +30,11 @@ type Decoder struct { } // NewDecoder creates a Decoder given the runtime.Scheme -func NewDecoder(scheme *runtime.Scheme) (*Decoder, error) { - return &Decoder{codecs: serializer.NewCodecFactory(scheme)}, nil +func NewDecoder(scheme *runtime.Scheme) *Decoder { + if scheme == nil { + panic("scheme should never be nil") + } + return &Decoder{codecs: serializer.NewCodecFactory(scheme)} } // Decode decodes the inlined object. diff --git a/pkg/webhook/example_test.go b/pkg/webhook/example_test.go index e1f2bbee6c..7c9fbfe24b 100644 --- a/pkg/webhook/example_test.go +++ b/pkg/webhook/example_test.go @@ -20,7 +20,6 @@ import ( "context" "net/http" - "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/internal/log" "sigs.k8s.io/controller-runtime/pkg/manager/signals" @@ -87,7 +86,7 @@ func Example() { // Note that this assumes and requires a valid TLS // cert and key at the default locations // tls.crt and tls.key. -func ExampleServer_StartStandalone() { +func ExampleServer_Start() { // Create a webhook server hookServer := &Server{ Port: 8443, @@ -98,7 +97,7 @@ func ExampleServer_StartStandalone() { hookServer.Register("/validating", validatingHook) // Start the server without a manger - err := hookServer.StartStandalone(signals.SetupSignalHandler(), scheme.Scheme) + err := hookServer.Start(signals.SetupSignalHandler()) if err != nil { // handle error panic(err) @@ -118,7 +117,6 @@ func ExampleStandaloneWebhook() { // Create the standalone HTTP handlers from our webhooks mutatingHookHandler, err := admission.StandaloneWebhook(mutatingHook, admission.StandaloneOptions{ - Scheme: scheme.Scheme, // Logger let's you optionally pass // a custom logger (defaults to log.Log global Logger) Logger: logf.RuntimeLog.WithName("mutating-webhook"), @@ -134,7 +132,6 @@ func ExampleStandaloneWebhook() { } validatingHookHandler, err := admission.StandaloneWebhook(validatingHook, admission.StandaloneOptions{ - Scheme: scheme.Scheme, Logger: logf.RuntimeLog.WithName("validating-webhook"), MetricsPath: "/validating", }) @@ -148,7 +145,7 @@ func ExampleStandaloneWebhook() { mux.Handle("/validating", validatingHookHandler) // Run your handler - if err := http.ListenAndServe(port, mux); err != nil { + if err := http.ListenAndServe(port, mux); err != nil { //nolint:gosec // it's fine to not set timeouts here panic(err) } } diff --git a/pkg/webhook/server.go b/pkg/webhook/server.go index 99b6ae9eb4..6b68ae3ed5 100644 --- a/pkg/webhook/server.go +++ b/pkg/webhook/server.go @@ -21,18 +21,17 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "io/ioutil" "net" "net/http" "os" "path/filepath" "strconv" "sync" + "time" - "k8s.io/apimachinery/pkg/runtime" - kscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/certwatcher" - "sigs.k8s.io/controller-runtime/pkg/runtime/inject" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/internal/httpserver" "sigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics" ) @@ -72,21 +71,25 @@ type Server struct { // TLSVersion is the minimum version of TLS supported. Accepts // "", "1.0", "1.1", "1.2" and "1.3" only ("" is equivalent to "1.0" for backwards compatibility) + // Deprecated: Use TLSOpts instead. TLSMinVersion string + // TLSOpts is used to allow configuring the TLS config used for the server + TLSOpts []func(*tls.Config) + // WebhookMux is the multiplexer that handles different webhooks. WebhookMux *http.ServeMux - // webhooks keep track of all registered webhooks for dependency injection, - // and to provide better panic messages on duplicate webhook registration. + // webhooks keep track of all registered webhooks webhooks map[string]http.Handler - // setFields allows injecting dependencies from an external source - setFields inject.Func - // defaultingOnce ensures that the default fields are only ever set once. defaultingOnce sync.Once + // started is set to true immediately before the server is started + // and thus can be used to check if the server has been started + started bool + // mu protects access to the webhook map & setFields for Start, Register, etc mu sync.Mutex } @@ -131,51 +134,11 @@ func (s *Server) Register(path string, hook http.Handler) { if _, found := s.webhooks[path]; found { panic(fmt.Errorf("can't register duplicate path: %v", path)) } - // TODO(directxman12): call setfields if we've already started the server s.webhooks[path] = hook s.WebhookMux.Handle(path, metrics.InstrumentedHook(path, hook)) regLog := log.WithValues("path", path) - regLog.Info("registering webhook") - - // we've already been "started", inject dependencies here. - // Otherwise, InjectFunc will do this for us later. - if s.setFields != nil { - if err := s.setFields(hook); err != nil { - // TODO(directxman12): swallowing this error isn't great, but we'd have to - // change the signature to fix that - regLog.Error(err, "unable to inject fields into webhook during registration") - } - - baseHookLog := log.WithName("webhooks") - - // NB(directxman12): we don't propagate this further by wrapping setFields because it's - // unclear if this is how we want to deal with log propagation. In this specific instance, - // we want to be able to pass a logger to webhooks because they don't know their own path. - if _, err := inject.LoggerInto(baseHookLog.WithValues("webhook", path), hook); err != nil { - regLog.Error(err, "unable to logger into webhook during registration") - } - } -} - -// StartStandalone runs a webhook server without -// a controller manager. -func (s *Server) StartStandalone(ctx context.Context, scheme *runtime.Scheme) error { - // Use the Kubernetes client-go scheme if none is specified - if scheme == nil { - scheme = kscheme.Scheme - } - - if err := s.InjectFunc(func(i interface{}) error { - if _, err := inject.SchemeInto(scheme, i); err != nil { - return err - } - return nil - }); err != nil { - return err - } - - return s.Start(ctx) + regLog.Info("Registering webhook") } // tlsVersion converts from human-readable TLS version (for example "1.1") @@ -204,7 +167,7 @@ func (s *Server) Start(ctx context.Context) error { s.defaultingOnce.Do(s.setDefaults) baseHookLog := log.WithName("webhooks") - baseHookLog.Info("starting webhook server") + baseHookLog.Info("Starting webhook server") certPath := filepath.Join(s.CertDir, s.CertName) keyPath := filepath.Join(s.CertDir, s.KeyName) @@ -234,9 +197,9 @@ func (s *Server) Start(ctx context.Context) error { // load CA to verify client certificate if s.ClientCAName != "" { certPool := x509.NewCertPool() - clientCABytes, err := ioutil.ReadFile(filepath.Join(s.CertDir, s.ClientCAName)) + clientCABytes, err := os.ReadFile(filepath.Join(s.CertDir, s.ClientCAName)) if err != nil { - return fmt.Errorf("failed to read client CA cert: %v", err) + return fmt.Errorf("failed to read client CA cert: %w", err) } ok := certPool.AppendCertsFromPEM(clientCABytes) @@ -248,30 +211,37 @@ func (s *Server) Start(ctx context.Context) error { cfg.ClientAuth = tls.RequireAndVerifyClientCert } + // fallback TLS config ready, will now mutate if passer wants full control over it + for _, op := range s.TLSOpts { + op(cfg) + } + listener, err := tls.Listen("tcp", net.JoinHostPort(s.Host, strconv.Itoa(s.Port)), cfg) if err != nil { return err } - log.Info("serving webhook server", "host", s.Host, "port", s.Port) + log.Info("Serving webhook server", "host", s.Host, "port", s.Port) - srv := &http.Server{ - Handler: s.WebhookMux, - } + srv := httpserver.New(s.WebhookMux) idleConnsClosed := make(chan struct{}) go func() { <-ctx.Done() - log.Info("shutting down webhook server") + log.Info("Shutting down webhook server with timeout of 1 minute") - // TODO: use a context with reasonable timeout - if err := srv.Shutdown(context.Background()); err != nil { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + if err := srv.Shutdown(ctx); err != nil { // Error from closing listeners, or context timeout log.Error(err, "error shutting down the HTTP server") } close(idleConnsClosed) }() + s.mu.Lock() + s.started = true + s.mu.Unlock() if err := srv.Serve(listener); err != nil && err != http.ErrServerClosed { return err } @@ -280,23 +250,30 @@ func (s *Server) Start(ctx context.Context) error { return nil } -// InjectFunc injects the field setter into the server. -func (s *Server) InjectFunc(f inject.Func) error { - s.setFields = f +// StartedChecker returns an healthz.Checker which is healthy after the +// server has been started. +func (s *Server) StartedChecker() healthz.Checker { + config := &tls.Config{ + InsecureSkipVerify: true, //nolint:gosec // config is used to connect to our own webhook port. + } + return func(req *http.Request) error { + s.mu.Lock() + defer s.mu.Unlock() + + if !s.started { + return fmt.Errorf("webhook server has not been started yet") + } - // inject fields here that weren't injected in Register because we didn't have setFields yet. - baseHookLog := log.WithName("webhooks") - for hookPath, webhook := range s.webhooks { - if err := s.setFields(webhook); err != nil { - return err + d := &net.Dialer{Timeout: 10 * time.Second} + conn, err := tls.DialWithDialer(d, "tcp", net.JoinHostPort(s.Host, strconv.Itoa(s.Port)), config) + if err != nil { + return fmt.Errorf("webhook server is not reachable: %w", err) } - // NB(directxman12): we don't propagate this further by wrapping setFields because it's - // unclear if this is how we want to deal with log propagation. In this specific instance, - // we want to be able to pass a logger to webhooks because they don't know their own path. - if _, err := inject.LoggerInto(baseHookLog.WithValues("webhook", hookPath), webhook); err != nil { - return err + if err := conn.Close(); err != nil { + return fmt.Errorf("webhook server is not reachable: closing connection: %w", err) } + + return nil } - return nil } diff --git a/pkg/webhook/server_test.go b/pkg/webhook/server_test.go index ca7da4ce49..e9b40a1542 100644 --- a/pkg/webhook/server_test.go +++ b/pkg/webhook/server_test.go @@ -18,14 +18,14 @@ package webhook_test import ( "context" + "crypto/tls" "fmt" - "io/ioutil" + "io" "net" "net/http" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -82,15 +82,6 @@ var _ = Describe("Webhook Server", func() { return err }).Should(Succeed()) - // this is normally called before Start by the manager - Expect(server.InjectFunc(func(i interface{}) error { - boolInj, canInj := i.(interface{ InjectBool(bool) error }) - if !canInj { - return nil - } - return boolInj.InjectBool(true) - })).To(Succeed()) - return doneCh } @@ -128,25 +119,18 @@ var _ = Describe("Webhook Server", func() { It("should serve a webhook on the requested path", func() { server.Register("/somepath", &testHandler{}) + Expect(server.StartedChecker()(nil)).ToNot(Succeed()) + doneCh := startServer() Eventually(func() ([]byte, error) { resp, err := client.Get(fmt.Sprintf("https://%s/somepath", testHostPort)) Expect(err).NotTo(HaveOccurred()) defer resp.Body.Close() - return ioutil.ReadAll(resp.Body) + return io.ReadAll(resp.Body) }).Should(Equal([]byte("gadzooks!"))) - ctxCancel() - Eventually(doneCh, "4s").Should(BeClosed()) - }) - - It("should inject dependencies eventually, given an inject func is eventually provided", func() { - handler := &testHandler{} - server.Register("/somepath", handler) - doneCh := startServer() - - Eventually(func() bool { return handler.injectedField }).Should(BeTrue()) + Expect(server.StartedChecker()(nil)).To(Succeed()) ctxCancel() Eventually(doneCh, "4s").Should(BeClosed()) @@ -172,33 +156,45 @@ var _ = Describe("Webhook Server", func() { Expect(err).NotTo(HaveOccurred()) defer resp.Body.Close() - Expect(ioutil.ReadAll(resp.Body)).To(Equal([]byte("gadzooks!"))) - }) - - It("should inject dependencies, if an inject func has been provided already", func() { - handler := &testHandler{} - server.Register("/somepath", handler) - Expect(handler.injectedField).To(BeTrue()) + Expect(io.ReadAll(resp.Body)).To(Equal([]byte("gadzooks!"))) }) }) - It("should serve be able to serve in unmanaged mode", func() { + It("should respect passed in TLS configurations", func() { + var finalCfg *tls.Config + tlsCfgFunc := func(cfg *tls.Config) { + cfg.CipherSuites = []uint16{ + tls.TLS_AES_128_GCM_SHA256, + tls.TLS_AES_256_GCM_SHA384, + } + // save cfg after changes to test against + finalCfg = cfg + } server = &webhook.Server{ - Host: servingOpts.LocalServingHost, - Port: servingOpts.LocalServingPort, - CertDir: servingOpts.LocalServingCertDir, + Host: servingOpts.LocalServingHost, + Port: servingOpts.LocalServingPort, + CertDir: servingOpts.LocalServingCertDir, + TLSMinVersion: "1.2", + TLSOpts: []func(*tls.Config){ + tlsCfgFunc, + }, } server.Register("/somepath", &testHandler{}) doneCh := genericStartServer(func(ctx context.Context) { - Expect(server.StartStandalone(ctx, scheme.Scheme)) + Expect(server.Start(ctx)) }) Eventually(func() ([]byte, error) { resp, err := client.Get(fmt.Sprintf("https://%s/somepath", testHostPort)) Expect(err).NotTo(HaveOccurred()) defer resp.Body.Close() - return ioutil.ReadAll(resp.Body) + return io.ReadAll(resp.Body) }).Should(Equal([]byte("gadzooks!"))) + Expect(finalCfg.MinVersion).To(Equal(uint16(tls.VersionTLS12))) + Expect(finalCfg.CipherSuites).To(ContainElements( + tls.TLS_AES_128_GCM_SHA256, + tls.TLS_AES_256_GCM_SHA384, + )) ctxCancel() Eventually(doneCh, "4s").Should(BeClosed()) @@ -206,13 +202,8 @@ var _ = Describe("Webhook Server", func() { }) type testHandler struct { - injectedField bool } -func (t *testHandler) InjectBool(val bool) error { - t.injectedField = val - return nil -} func (t *testHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) { if _, err := resp.Write([]byte("gadzooks!")); err != nil { panic("unable to write http response!") diff --git a/pkg/webhook/webhook_integration_test.go b/pkg/webhook/webhook_integration_test.go index 3ed9713157..60f6b2d6be 100644 --- a/pkg/webhook/webhook_integration_test.go +++ b/pkg/webhook/webhook_integration_test.go @@ -19,27 +19,19 @@ package webhook_test import ( "context" "crypto/tls" - "errors" - "net" - "net/http" - "path/filepath" - "strconv" + "strings" "time" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" - "sigs.k8s.io/controller-runtime/pkg/webhook/admission/admissiontest" ) var _ = Describe("Webhook", func() { @@ -79,39 +71,40 @@ var _ = Describe("Webhook", func() { } }) Context("when running a webhook server with a manager", func() { - It("should reject create request for webhook that rejects all requests", func(done Done) { + It("should reject create request for webhook that rejects all requests", func() { m, err := manager.New(cfg, manager.Options{ Port: testenv.WebhookInstallOptions.LocalServingPort, Host: testenv.WebhookInstallOptions.LocalServingHost, CertDir: testenv.WebhookInstallOptions.LocalServingCertDir, + TLSOpts: []func(*tls.Config){func(config *tls.Config) {}}, }) // we need manager here just to leverage manager.SetFields Expect(err).NotTo(HaveOccurred()) server := m.GetWebhookServer() - server.Register("/failing", &webhook.Admission{Handler: &rejectingValidator{}}) + server.Register("/failing", &webhook.Admission{Handler: &rejectingValidator{d: admission.NewDecoder(testenv.Scheme)}}) ctx, cancel := context.WithCancel(context.Background()) go func() { - err = server.Start(ctx) + err := server.Start(ctx) Expect(err).NotTo(HaveOccurred()) }() Eventually(func() bool { - err = c.Create(context.TODO(), obj) - return apierrors.ReasonForError(err) == metav1.StatusReason("Always denied") + err := c.Create(context.TODO(), obj) + return err != nil && strings.HasSuffix(err.Error(), "Always denied") && apierrors.ReasonForError(err) == metav1.StatusReasonForbidden }, 1*time.Second).Should(BeTrue()) cancel() - close(done) }) - It("should reject create request for multi-webhook that rejects all requests", func(done Done) { + It("should reject create request for multi-webhook that rejects all requests", func() { m, err := manager.New(cfg, manager.Options{ Port: testenv.WebhookInstallOptions.LocalServingPort, Host: testenv.WebhookInstallOptions.LocalServingHost, CertDir: testenv.WebhookInstallOptions.LocalServingCertDir, + TLSOpts: []func(*tls.Config){func(config *tls.Config) {}}, }) // we need manager here just to leverage manager.SetFields Expect(err).NotTo(HaveOccurred()) server := m.GetWebhookServer() - server.Register("/failing", &webhook.Admission{Handler: admission.MultiValidatingHandler(&rejectingValidator{})}) + server.Register("/failing", &webhook.Admission{Handler: admission.MultiValidatingHandler(&rejectingValidator{d: admission.NewDecoder(testenv.Scheme)})}) ctx, cancel := context.WithCancel(context.Background()) go func() { @@ -121,92 +114,33 @@ var _ = Describe("Webhook", func() { Eventually(func() bool { err = c.Create(context.TODO(), obj) - return apierrors.ReasonForError(err) == metav1.StatusReason("Always denied") + return err != nil && strings.HasSuffix(err.Error(), "Always denied") && apierrors.ReasonForError(err) == metav1.StatusReasonForbidden }, 1*time.Second).Should(BeTrue()) cancel() - close(done) }) }) Context("when running a webhook server without a manager", func() { - It("should reject create request for webhook that rejects all requests", func(done Done) { + It("should reject create request for webhook that rejects all requests", func() { server := webhook.Server{ Port: testenv.WebhookInstallOptions.LocalServingPort, Host: testenv.WebhookInstallOptions.LocalServingHost, CertDir: testenv.WebhookInstallOptions.LocalServingCertDir, } - server.Register("/failing", &webhook.Admission{Handler: &rejectingValidator{}}) + server.Register("/failing", &webhook.Admission{Handler: &rejectingValidator{d: admission.NewDecoder(testenv.Scheme)}}) ctx, cancel := context.WithCancel(context.Background()) go func() { - err := server.StartStandalone(ctx, scheme.Scheme) + err := server.Start(ctx) Expect(err).NotTo(HaveOccurred()) }() Eventually(func() bool { err := c.Create(context.TODO(), obj) - return apierrors.ReasonForError(err) == metav1.StatusReason("Always denied") - }, 1*time.Second).Should(BeTrue()) - - cancel() - close(done) - }) - }) - Context("when running a standalone webhook", func() { - It("should reject create request for webhook that rejects all requests", func(done Done) { - ctx, cancel := context.WithCancel(context.Background()) - - By("generating the TLS config") - certPath := filepath.Join(testenv.WebhookInstallOptions.LocalServingCertDir, "tls.crt") - keyPath := filepath.Join(testenv.WebhookInstallOptions.LocalServingCertDir, "tls.key") - - certWatcher, err := certwatcher.New(certPath, keyPath) - Expect(err).NotTo(HaveOccurred()) - go func() { - Expect(certWatcher.Start(ctx)).NotTo(HaveOccurred()) - }() - - cfg := &tls.Config{ - NextProtos: []string{"h2"}, - GetCertificate: certWatcher.GetCertificate, - MinVersion: tls.VersionTLS12, - } - - By("generating the listener") - listener, err := tls.Listen("tcp", - net.JoinHostPort(testenv.WebhookInstallOptions.LocalServingHost, - strconv.Itoa(testenv.WebhookInstallOptions.LocalServingPort)), cfg) - Expect(err).NotTo(HaveOccurred()) - - By("creating and registering the standalone webhook") - hook, err := admission.StandaloneWebhook(admission.ValidatingWebhookFor( - &admissiontest.FakeValidator{ - ErrorToReturn: errors.New("Always denied"), - GVKToReturn: schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, - }), admission.StandaloneOptions{}) - Expect(err).NotTo(HaveOccurred()) - http.Handle("/failing", hook) - - By("running the http server") - srv := &http.Server{} - go func() { - idleConnsClosed := make(chan struct{}) - go func() { - <-ctx.Done() - Expect(srv.Shutdown(context.Background())).NotTo(HaveOccurred()) - close(idleConnsClosed) - }() - _ = srv.Serve(listener) - <-idleConnsClosed - }() - - Eventually(func() bool { - err = c.Create(context.TODO(), obj) - return apierrors.ReasonForError(err) == metav1.StatusReason("Always denied") + return err != nil && strings.HasSuffix(err.Error(), "Always denied") && apierrors.ReasonForError(err) == metav1.StatusReasonForbidden }, 1*time.Second).Should(BeTrue()) cancel() - close(done) }) }) }) @@ -215,11 +149,6 @@ type rejectingValidator struct { d *admission.Decoder } -func (v *rejectingValidator) InjectDecoder(d *admission.Decoder) error { - v.d = d - return nil -} - func (v *rejectingValidator) Handle(ctx context.Context, req admission.Request) admission.Response { var obj appsv1.Deployment if err := v.d.Decode(req, &obj); err != nil { diff --git a/pkg/webhook/webhook_suite_test.go b/pkg/webhook/webhook_suite_test.go index fb2b02f195..ee9c1f4057 100644 --- a/pkg/webhook/webhook_suite_test.go +++ b/pkg/webhook/webhook_suite_test.go @@ -20,29 +20,26 @@ import ( "fmt" "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" admissionv1 "k8s.io/api/admissionregistration/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" - "sigs.k8s.io/controller-runtime/pkg/envtest/printer" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) func TestSource(t *testing.T) { RegisterFailHandler(Fail) - suiteName := "Webhook Integration Suite" - RunSpecsWithDefaultAndCustomReporters(t, suiteName, []Reporter{printer.NewlineReporter{}, printer.NewProwReporter(suiteName)}) + RunSpecs(t, "Webhook Integration Suite") } var testenv *envtest.Environment var cfg *rest.Config -var _ = BeforeSuite(func(done Done) { +var _ = BeforeSuite(func() { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) testenv = &envtest.Environment{} @@ -51,13 +48,12 @@ var _ = BeforeSuite(func(done Done) { var err error cfg, err = testenv.Start() Expect(err).NotTo(HaveOccurred()) - close(done) -}, 60) +}) var _ = AfterSuite(func() { fmt.Println("stopping?") Expect(testenv.Stop()).To(Succeed()) -}, 60) +}) func initializeWebhookInEnvironment() { namespacedScopeV1 := admissionv1.NamespacedScope @@ -67,14 +63,14 @@ func initializeWebhookInEnvironment() { webhookPathV1 := "/failing" testenv.WebhookInstallOptions = envtest.WebhookInstallOptions{ - ValidatingWebhooks: []client.Object{ - &admissionv1.ValidatingWebhookConfiguration{ + ValidatingWebhooks: []*admissionv1.ValidatingWebhookConfiguration{ + { ObjectMeta: metav1.ObjectMeta{ Name: "deployment-validation-webhook-config", }, TypeMeta: metav1.TypeMeta{ Kind: "ValidatingWebhookConfiguration", - APIVersion: "admissionregistration.k8s.io/v1beta1", + APIVersion: "admissionregistration.k8s.io/v1", }, Webhooks: []admissionv1.ValidatingWebhook{ { @@ -100,6 +96,7 @@ func initializeWebhookInEnvironment() { Path: &webhookPathV1, }, }, + AdmissionReviewVersions: []string{"v1"}, }, }, }, diff --git a/tools/setup-envtest/README.md b/tools/setup-envtest/README.md index 0a497c3ec4..40379c9b8c 100644 --- a/tools/setup-envtest/README.md +++ b/tools/setup-envtest/README.md @@ -91,7 +91,7 @@ Then, you have a few options for managing your binaries: `--use-env` makes the command unconditionally use the value of KUBEBUILDER_ASSETS as long as it contains the required binaries, and `-i` indicates that we only ever want to work with installed binaries - (no reaching out the the remote GCS storage). + (no reaching out the remote GCS storage). As noted about, you can use `ENVTEST_INSTALLED_ONLY=true` to switch `-i` on by default, and you can use `ENVTEST_USE_ENV=true` to switch diff --git a/tools/setup-envtest/env/env_suite_test.go b/tools/setup-envtest/env/env_suite_test.go index 7d9fe9c179..3400dd91aa 100644 --- a/tools/setup-envtest/env/env_suite_test.go +++ b/tools/setup-envtest/env/env_suite_test.go @@ -19,7 +19,7 @@ package env_test import ( "testing" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/go-logr/logr" diff --git a/tools/setup-envtest/env/env_test.go b/tools/setup-envtest/env/env_test.go index 874ac7a736..fd6e7633bd 100644 --- a/tools/setup-envtest/env/env_test.go +++ b/tools/setup-envtest/env/env_test.go @@ -19,7 +19,7 @@ package env_test import ( "bytes" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/spf13/afero" diff --git a/tools/setup-envtest/go.mod b/tools/setup-envtest/go.mod index cdcee04e08..29592281a0 100644 --- a/tools/setup-envtest/go.mod +++ b/tools/setup-envtest/go.mod @@ -1,13 +1,24 @@ module sigs.k8s.io/controller-runtime/tools/setup-envtest -go 1.16 +go 1.17 require ( - github.com/go-logr/logr v0.4.0 - github.com/go-logr/zapr v0.4.0 - github.com/onsi/ginkgo v1.16.1 - github.com/onsi/gomega v1.11.0 + github.com/go-logr/logr v1.2.0 + github.com/go-logr/zapr v1.2.0 + github.com/onsi/ginkgo/v2 v2.1.4 + github.com/onsi/gomega v1.19.0 github.com/spf13/afero v1.6.0 github.com/spf13/pflag v1.0.5 - go.uber.org/zap v1.16.0 + go.uber.org/zap v1.19.1 +) + +require ( + github.com/golang/protobuf v1.5.2 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect + golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 // indirect + golang.org/x/text v0.3.7 // indirect + google.golang.org/protobuf v1.26.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/tools/setup-envtest/go.sum b/tools/setup-envtest/go.sum index 0d9a66c095..42347d8c1d 100644 --- a/tools/setup-envtest/go.sum +++ b/tools/setup-envtest/go.sum @@ -1,15 +1,17 @@ -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM= -github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= +github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= +github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= @@ -18,15 +20,17 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -34,22 +38,25 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.1 h1:foqVmeWDD6yYpK+Yz3fHyNIxFYNxswxqNFjSKe+vI54= -github.com/onsi/ginkgo v1.16.1/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.11.0 h1:+CqWgvj0OZycCaqclBD1pxKHAU+tOkHmQIWvDHq2aug= -github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQLTUg= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -57,59 +64,80 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= -go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091 h1:DMyOG0U+gKfu8JZzg2UQe9MeaC1X+xQWlAKcRnjxjCw= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e h1:4nW4NLDYnU28ojHaHO8OVxFHk/aQ33U01a9cjED+pzE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -120,19 +148,21 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/setup-envtest/main.go b/tools/setup-envtest/main.go index 9c0ab78057..8dca774157 100644 --- a/tools/setup-envtest/main.go +++ b/tools/setup-envtest/main.go @@ -192,7 +192,7 @@ Versions: Z may also be '*' or 'x' to match a wildcard. You may also just write X.Y, which means X.Y.*. - A version may be prefixed with '~' to match the the most recent Z release + A version may be prefixed with '~' to match the most recent Z release in the given Y release ( [X.Y.Z, X.Y+1.0) ). Finally, you may suffix the version with '!' to force checking the @@ -203,7 +203,7 @@ Versions: 1.16.x / 1.16.* / 1.16 # any 1.16 version ~1.19.3 # any 1.19 version that's at least 1.19.3 <1.17 # any release 1.17.x or below - 1.22.x! # the latest one 1.22 release avaible remotely + 1.22.x! # the latest one 1.22 release available remotely Output: diff --git a/tools/setup-envtest/remote/client.go b/tools/setup-envtest/remote/client.go index 00e4840813..be82532583 100644 --- a/tools/setup-envtest/remote/client.go +++ b/tools/setup-envtest/remote/client.go @@ -8,6 +8,7 @@ import ( "crypto/md5" //nolint:gosec "encoding/base64" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -160,7 +161,7 @@ func (c *Client) GetVersion(ctx context.Context, version versions.Concrete, plat checksum := md5.New() //nolint:gosec for cont := true; cont; { amt, err := resp.Body.Read(buf) - if err != nil && err != io.EOF { + if err != nil && !errors.Is(err, io.EOF) { return fmt.Errorf("unable read next chunk of %s: %w", itemName, err) } if amt > 0 { @@ -170,7 +171,7 @@ func (c *Client) GetVersion(ctx context.Context, version versions.Concrete, plat return fmt.Errorf("unable write next chunk of %s: %w", itemName, err) } } - cont = amt > 0 && err != io.EOF + cont = amt > 0 && !errors.Is(err, io.EOF) } sum := base64.StdEncoding.EncodeToString(checksum.Sum(nil)) diff --git a/tools/setup-envtest/store/store.go b/tools/setup-envtest/store/store.go index e3c9ca018f..e6f258e4ac 100644 --- a/tools/setup-envtest/store/store.go +++ b/tools/setup-envtest/store/store.go @@ -68,7 +68,12 @@ func NewAt(path string) *Store { // Initialize ensures that the store is all set up on disk, etc. func (s *Store) Initialize(ctx context.Context) error { - logr.FromContext(ctx).V(1).Info("ensuring base binaries dir exists") + log, err := logr.FromContext(ctx) + if err != nil { + return err + } + + log.V(1).Info("ensuring base binaries dir exists") if err := s.unpackedBase().MkdirAll("", 0755); err != nil { return fmt.Errorf("unable to make sure base binaries dir exists: %w", err) } @@ -109,7 +114,11 @@ func (s *Store) List(ctx context.Context, matching Filter) ([]Item, error) { // Add adds this item to the store, with the given contents (a .tar.gz file). func (s *Store) Add(ctx context.Context, item Item, contents io.Reader) (resErr error) { - log := logr.FromContext(ctx) + log, err := logr.FromContext(ctx) + if err != nil { + return err + } + itemName := item.dirName() log = log.WithValues("version-platform", itemName) itemPath := s.unpackedPath(itemName) @@ -126,7 +135,7 @@ func (s *Store) Add(ctx context.Context, item Item, contents io.Reader) (resErr }() log.V(1).Info("ensuring version-platform binaries dir exists and is empty & writable") - _, err := itemPath.Stat("") + _, err = itemPath.Stat("") if err != nil && !errors.Is(err, afero.ErrFileNotFound) { return fmt.Errorf("unable to ensure version-platform binaries dir %s exists", itemName) } @@ -173,7 +182,7 @@ func (s *Store) Add(ctx context.Context, item Item, contents io.Reader) (resErr return err } } - if err != nil && err != io.EOF { + if err != nil && !errors.Is(err, io.EOF) { return fmt.Errorf("unable to finish un-tar-ing the downloaded archive: %w", err) } log.V(1).Info("unpacked archive") @@ -191,7 +200,11 @@ func (s *Store) Add(ctx context.Context, item Item, contents io.Reader) (resErr // It returns a list of the successfully removed items (even in the case // of an error). func (s *Store) Remove(ctx context.Context, matching Filter) ([]Item, error) { - log := logr.FromContext(ctx) + log, err := logr.FromContext(ctx) + if err != nil { + return nil, err + } + var removed []Item var savedErr error if err := s.eachItem(ctx, matching, func(name string, item Item) { @@ -216,7 +229,7 @@ func (s *Store) Remove(ctx context.Context, matching Filter) ([]Item, error) { func (s *Store) Path(item Item) (string, error) { path := s.unpackedPath(item.dirName()) // NB(directxman12): we need root's realpath because RealPath only - // looks at it's own path, and so thus doesn't prepend the underlying + // looks at its own path, and so thus doesn't prepend the underlying // root's base path. // // Technically, if we're fed something that's double wrapped as root, @@ -237,7 +250,11 @@ func (s *Store) unpackedPath(name string) afero.Fs { // eachItem iterates through the on-disk versions that match our version & platform selector, // calling the callback for each. func (s *Store) eachItem(ctx context.Context, filter Filter, cb func(name string, item Item)) error { - log := logr.FromContext(ctx) + log, err := logr.FromContext(ctx) + if err != nil { + return err + } + entries, err := afero.ReadDir(s.unpackedBase(), "") if err != nil { return fmt.Errorf("unable to list folders in store's unpacked directory: %w", err) diff --git a/tools/setup-envtest/store/store_suite_test.go b/tools/setup-envtest/store/store_suite_test.go index 2eb909af6b..c2795a3227 100644 --- a/tools/setup-envtest/store/store_suite_test.go +++ b/tools/setup-envtest/store/store_suite_test.go @@ -25,7 +25,7 @@ import ( "go.uber.org/zap" "go.uber.org/zap/zapcore" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) diff --git a/tools/setup-envtest/store/store_test.go b/tools/setup-envtest/store/store_test.go index 862996abfa..723eada3cd 100644 --- a/tools/setup-envtest/store/store_test.go +++ b/tools/setup-envtest/store/store_test.go @@ -25,7 +25,7 @@ import ( "math/rand" "path/filepath" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/spf13/afero" diff --git a/tools/setup-envtest/versions/misc_test.go b/tools/setup-envtest/versions/misc_test.go index 3429211f27..8a97de0410 100644 --- a/tools/setup-envtest/versions/misc_test.go +++ b/tools/setup-envtest/versions/misc_test.go @@ -17,7 +17,7 @@ limitations under the License. package versions_test import ( - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "sigs.k8s.io/controller-runtime/tools/setup-envtest/versions" diff --git a/tools/setup-envtest/versions/parse.go b/tools/setup-envtest/versions/parse.go index 4b3c5bb5ba..c053bf8757 100644 --- a/tools/setup-envtest/versions/parse.go +++ b/tools/setup-envtest/versions/parse.go @@ -25,15 +25,14 @@ var ( // where X, Y, and Z may also be wildcards ('*', 'x'), // and pre-release names & numbers may also be wildcards. The prerelease section is slightly // restricted to match what k8s does. -// The the whole string is a version selector as follows: -// -// - X.Y.Z matches version X.Y.Z where x, y, and z are -// are ints >= 0, and Z may be '*' or 'x' -// - X.Y is equivalent to X.Y.* -// - ~X.Y.Z means >= X.Y.Z && < X.Y+1.0 -// - = 0, and Z may be '*' or 'x' +// - X.Y is equivalent to X.Y.* +// - ~X.Y.Z means >= X.Y.Z && < X.Y+1.0 +// -