diff --git a/Makefile b/Makefile index adc3a4da..e2cbe204 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,7 @@ format: # run test on all modules test: cd simulator/ && make test + cd scenario/ && make test .PHONY: mod-download mod-download: @@ -53,6 +54,11 @@ docker_build_server: docker_build_scheduler: docker $(BUILD) -f simulator/cmd/scheduler/Dockerfile -t simulator-scheduler simulator +.PHONY: docker_build_scenario +docker_build_scenario: + docker $(BUILD) -f scenario/Dockerfile -t scenario-controller scenario + docker $(BUILD) -f scenario/bootstrap/Dockerfile -t bootstrap scenario/bootstrap + .PHONY: docker_build_front docker_build_front: docker $(BUILD) -t simulator-frontend ./web/ @@ -65,6 +71,10 @@ docker_up: docker_up_local: docker compose -f compose.yml -f compose.local.yml up -d +.PHONY: docker_up_scenario +docker_up_scenario: + docker compose -f compose.yml -f compose.local.yml --profile scenario up -d + .PHONY: docker_build_and_up docker_build_and_up: docker_build docker_up_local @@ -75,3 +85,7 @@ docker_down: .PHONY: docker_down_local docker_down_local: docker compose -f compose.yml -f compose.local.yml down --volumes + +.PHONY: docker_down_scenario +docker_down_scenario: + docker compose -f compose.yml -f compose.local.yml --profile scenario down --volumes \ No newline at end of file diff --git a/compose.local.yml b/compose.local.yml index dc4ccc92..f911e95c 100644 --- a/compose.local.yml +++ b/compose.local.yml @@ -32,9 +32,50 @@ services: - simulator-internal-network profiles: - externalImportEnabled + + scenario-controller: + image: scenario-controller + profiles: ["scenario"] + command: + - "--kubeconfig=/config/kubeconfig.yaml" + - "--webhook-cert-path=/tmp/k8s-webhook-server/serving-certs" + volumes: + - ./simulator/cmd/scheduler/kubeconfig.yaml:/config/kubeconfig.yaml:ro + - scenario-webhook-certs:/tmp/k8s-webhook-server/serving-certs:ro + ports: + - "9443:9443" + depends_on: + bootstrap: + condition: service_healthy + networks: + - simulator-internal-network + + # Bootstrap container: prepares CRDs and generates webhook certificates + bootstrap: + image: bootstrap + profiles: ["scenario"] + depends_on: + - simulator-cluster + volumes: + - ./simulator/cmd/scheduler/kubeconfig.yaml:/config/kubeconfig.yaml:ro + - ./scenario/config/webhook:/tmp/webhook:ro + - ./scenario/config/crd:/manifests/crd:ro + - scenario-webhook-certs:/manifests/webhook/certs + - scenario-webhook-work:/manifests/webhook + networks: + - simulator-internal-network + healthcheck: + test: ["CMD", "sh", "-c", "test -f /manifests/webhook/certs/tls.crt"] + interval: 5s + timeout: 3s + retries: 5 + start_period: 5s + networks: simulator-internal-network: driver: bridge volumes: simulator-etcd-data: + scenario-webhook-certs: + scenario-webhook-work: conf: diff --git a/scenario/PROJECT b/scenario/PROJECT index 2495f03a..d6c0c0b8 100644 --- a/scenario/PROJECT +++ b/scenario/PROJECT @@ -17,4 +17,7 @@ resources: kind: Scenario path: sigs.k8s.io/kube-scheduler-simulator/scenario/api/v1alpha1 version: v1alpha1 + webhooks: + validation: true + webhookVersion: v1 version: "3" diff --git a/scenario/api/v1alpha1/run_suite_test.go b/scenario/api/v1alpha1/run_suite_test.go new file mode 100644 index 00000000..edd20e78 --- /dev/null +++ b/scenario/api/v1alpha1/run_suite_test.go @@ -0,0 +1,57 @@ +package v1alpha1 + +import ( + "context" + "path/filepath" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var ( + cfg *rest.Config + k8sCli client.Client + testEnv *envtest.Environment + ctx context.Context + cancel context.CancelFunc +) + +func TestRun(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "ScenarioOperation Run() API Suite") +} + +var _ = BeforeSuite(func() { + ctrl.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + ctx, cancel = context.WithCancel(context.TODO()) + + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{ + filepath.Join("..", "..", "..", "config", "crd", "bases"), + }, + } + var err error + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + Expect(AddToScheme(scheme)).To(Succeed()) + + k8sCli, err = client.New(cfg, client.Options{Scheme: scheme}) + Expect(err).NotTo(HaveOccurred()) +}) + +var _ = AfterSuite(func() { + cancel() + Expect(testEnv.Stop()).To(Succeed()) +}) diff --git a/scenario/api/v1alpha1/scenario_operation.go b/scenario/api/v1alpha1/scenario_operation.go new file mode 100644 index 00000000..3236f347 --- /dev/null +++ b/scenario/api/v1alpha1/scenario_operation.go @@ -0,0 +1,117 @@ +package v1alpha1 + +import ( + "context" + "errors" + "fmt" + + "golang.org/x/xerrors" + runtime "k8s.io/apimachinery/pkg/runtime" + runtimeschema "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" + memory "k8s.io/client-go/discovery/cached" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" +) + +func (s *ScenarioOperation) Run(ctx context.Context, cfg *rest.Config) (bool, error) { + switch { + case s.Create != nil: + ope := s.Create + gvk := ope.Object.GetObjectKind().GroupVersionKind() + client, err := buildClient(gvk, cfg) + if err != nil { + return true, xerrors.Errorf("build client failed for id: %s error: %w", s.ID, err) + } + _, err = client.Namespace(ope.Object.GetNamespace()).Create(ctx, ope.Object, ope.CreateOptions) + if err != nil { + return true, xerrors.Errorf("run create operation: id: %s error: %w", s.ID, err) + } + case s.Patch != nil: + ope := s.Patch + gvk := ope.TypeMeta.GroupVersionKind() + client, err := buildClient(gvk, cfg) + if err != nil { + return true, xerrors.Errorf("build client failed for id: %s error: %w", s.ID, err) + } + _, err = client.Namespace(ope.ObjectMeta.Namespace).Patch(ctx, ope.ObjectMeta.Name, ope.PatchType, []byte(ope.Patch), ope.PatchOptions) + if err != nil { + return true, xerrors.Errorf("run patch operation: id: %s error: %w", s.ID, err) + } + case s.Delete != nil: + ope := s.Delete + gvk := ope.TypeMeta.GroupVersionKind() + client, err := buildClient(gvk, cfg) + if err != nil { + return true, xerrors.Errorf("build client failed for id: %s error: %w", s.ID, err) + } + err = client.Namespace(ope.ObjectMeta.Namespace).Delete(ctx, ope.ObjectMeta.Name, ope.DeleteOptions) + if err != nil { + return true, xerrors.Errorf("run delete operation: id: %s error: %w", s.ID, err) + } + case s.Done != nil: + return true, nil + default: + return true, ErrUnknownOperation + } + + return false, nil +} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type. +func (s *ScenarioOperation) ValidateCreate() error { + return s.validateOperations() +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type. +func (s *ScenarioOperation) ValidateUpdate(old runtime.Object) error { + return s.validateOperations() +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type. +func (s *ScenarioOperation) ValidateDelete() error { + return nil +} + +// validateOperations checks that exactly one operation is set. +func (s *ScenarioOperation) validateOperations() error { + var count int + if s.Create != nil { + count++ + } + if s.Patch != nil { + count++ + } + if s.Delete != nil { + count++ + } + if s.Done != nil { + count++ + } + if count != 1 { + return fmt.Errorf("exactly one operation type must be specified, but found %d", count) + } + return nil +} + +var ErrUnknownOperation = errors.New("unknown operation") + +func buildClient(gvk runtimeschema.GroupVersionKind, cfg *rest.Config) (dynamic.NamespaceableResourceInterface, error) { + cli, err := dynamic.NewForConfig(cfg) + if err != nil { + return nil, xerrors.Errorf("build dynamic client: %w", err) + } + + dc, err := discovery.NewDiscoveryClientForConfig(cfg) + if err != nil { + return nil, xerrors.Errorf("build discovery client: %w", err) + } + mapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(dc)) + mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + return nil, xerrors.Errorf("build mapping from RESTMapper: %w", err) + } + + return cli.Resource(mapping.Resource), nil +} diff --git a/scenario/api/v1alpha1/scenario_operation_run_test.go b/scenario/api/v1alpha1/scenario_operation_run_test.go new file mode 100644 index 00000000..69b120c1 --- /dev/null +++ b/scenario/api/v1alpha1/scenario_operation_run_test.go @@ -0,0 +1,122 @@ +package v1alpha1 + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/uuid" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +const ns = "default" + +func podToUnstructured(p *corev1.Pod) *unstructured.Unstructured { + objMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(p) + Expect(err).NotTo(HaveOccurred()) + return &unstructured.Unstructured{Object: objMap} +} + +var _ = Describe("ScenarioOperation.Run()", func() { + + It("executes CreateOperation", func() { + pod := &corev1.Pod{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Pod"}, + ObjectMeta: metav1.ObjectMeta{Name: "run-create-pod", Namespace: ns}, + Spec: corev1.PodSpec{Containers: []corev1.Container{ + {Name: "pause", Image: "registry.k8s.io/pause:3.9"}, + }}, + } + + op := &ScenarioOperation{ + ID: "create-" + string(uuid.NewUUID()), + Create: &CreateOperation{ + Object: podToUnstructured(pod), + }, + } + + done, err := op.Run(context.TODO(), cfg) + Expect(err).NotTo(HaveOccurred()) + Expect(done).To(BeFalse()) + + Eventually(func() error { + return k8sCli.Get(context.TODO(), + types.NamespacedName{Name: "run-create-pod", Namespace: ns}, + &corev1.Pod{}) + }).Should(Succeed()) + }) + + It("executes PatchOperation", func() { + base := &corev1.Pod{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Pod"}, + ObjectMeta: metav1.ObjectMeta{Name: "run-patch-pod", Namespace: ns}, + Spec: corev1.PodSpec{Containers: []corev1.Container{ + {Name: "pause", Image: "registry.k8s.io/pause:3.9"}, + }}, + } + Expect(k8sCli.Create(context.TODO(), base)).To(Succeed()) + + op := &ScenarioOperation{ + ID: "patch-" + string(uuid.NewUUID()), + Patch: &PatchOperation{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Pod"}, + ObjectMeta: metav1.ObjectMeta{Name: "run-patch-pod", Namespace: ns}, + PatchType: types.MergePatchType, + Patch: `{"metadata":{"labels":{"patched":"true"}}}`, + }, + } + + _, err := op.Run(context.TODO(), cfg) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() map[string]string { + tmp := &corev1.Pod{} + _ = k8sCli.Get(context.TODO(), + types.NamespacedName{Name: "run-patch-pod", Namespace: ns}, tmp) + return tmp.Labels + }).Should(HaveKeyWithValue("patched", "true")) + }) + + It("executes DeleteOperation", func() { + p := &corev1.Pod{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Pod"}, + ObjectMeta: metav1.ObjectMeta{Name: "run-delete-pod", Namespace: ns}, + Spec: corev1.PodSpec{Containers: []corev1.Container{ + {Name: "pause", Image: "registry.k8s.io/pause:3.9"}, + }}, + } + Expect(k8sCli.Create(context.TODO(), p)).To(Succeed()) + + op := &ScenarioOperation{ + ID: "delete-" + string(uuid.NewUUID()), + Delete: &DeleteOperation{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Pod"}, + ObjectMeta: metav1.ObjectMeta{Name: "run-delete-pod", Namespace: ns}, + }, + } + + _, err := op.Run(context.TODO(), cfg) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() error { + return k8sCli.Get(context.TODO(), + types.NamespacedName{Name: "run-delete-pod", Namespace: ns}, + &corev1.Pod{}) + }).Should(MatchError(ContainSubstring("not found"))) + }) + + It("executes DoneOperation", func() { + op := &ScenarioOperation{ + ID: "done-" + string(uuid.NewUUID()), + Done: &DoneOperation{}, + } + done, err := op.Run(context.TODO(), cfg) + Expect(err).NotTo(HaveOccurred()) + Expect(done).To(BeTrue()) + }) +}) diff --git a/scenario/api/v1alpha1/scenario_operation_test.go b/scenario/api/v1alpha1/scenario_operation_test.go new file mode 100644 index 00000000..f7b871cc --- /dev/null +++ b/scenario/api/v1alpha1/scenario_operation_test.go @@ -0,0 +1,23 @@ +package v1alpha1 + +import ( + "testing" +) + +func TestValidateOperations(t *testing.T) { + cases := []struct { + name string + op ScenarioOperation + wantErr bool + }{ + {"one Op", ScenarioOperation{Create: &CreateOperation{}}, false}, + {"two Ops", ScenarioOperation{Create: &CreateOperation{}, Delete: &DeleteOperation{}}, true}, + {"zero Op", ScenarioOperation{}, true}, + } + for _, tc := range cases { + got := tc.op.ValidateCreate() + if (got != nil) != tc.wantErr { + t.Errorf("%s: unexpected error=%v", tc.name, got) + } + } +} diff --git a/scenario/api/v1alpha1/scenario_types.go b/scenario/api/v1alpha1/scenario_types.go index d91c1af4..71f865cf 100644 --- a/scenario/api/v1alpha1/scenario_types.go +++ b/scenario/api/v1alpha1/scenario_types.go @@ -18,6 +18,8 @@ package v1alpha1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" ) // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! @@ -30,8 +32,72 @@ type ScenarioSpec struct { // Foo is an example field of Scenario. Edit scenario_types.go to remove/update Foo string `json:"foo,omitempty"` + + // Operations is a list of ScenarioOperation that define the actions to perform. + // +optional + Operations []*ScenarioOperation `json:"operations,omitempty"` +} + +type ScenarioOperation struct { + // ID for this operation. Normally, the system sets this field for you. + ID string `json:"id"` + // MajorStep indicates when the operation should be done. + MajorStep int32 `json:"step"` + + // One of the following four fields must be specified. + // If more than one is set or all are empty, the operation is invalid, and the scenario will fail. + + // Create is the operation to create a new resource. + // + // +optional + Create *CreateOperation `json:"createOperation,omitempty"` + // Patch is the operation to patch a resource. + // + // +optional + Patch *PatchOperation `json:"patchOperation,omitempty"` + // Delete indicates the operation to delete a resource. + // + // +optional + Delete *DeleteOperation `json:"deleteOperation,omitempty"` + // Done indicates the operation to mark the scenario as Succeeded. + // When finish the step DoneOperation belongs, this Scenario changes its status to Succeeded. + // + // +optional + Done *DoneOperation `json:"doneOperation,omitempty"` +} + +type CreateOperation struct { + // Object is the Object to be created. + // +kubebuilder:pruning:PreserveUnknownFields + Object *unstructured.Unstructured `json:"object"` + + // +optional + CreateOptions metav1.CreateOptions `json:"createOptions,omitempty"` } +type PatchOperation struct { + TypeMeta metav1.TypeMeta `json:"typeMeta"` + // +kubebuilder:pruning:PreserveUnknownFields + ObjectMeta metav1.ObjectMeta `json:"objectMeta"` + // Patch is the patch for target. + Patch string `json:"patch"` + // PatchType + PatchType types.PatchType `json:"patchType"` + + // +optional + PatchOptions metav1.PatchOptions `json:"patchOptions,omitempty"` +} + +type DeleteOperation struct { + TypeMeta metav1.TypeMeta `json:"typeMeta"` + ObjectMeta metav1.ObjectMeta `json:"objectMeta"` + + // +optional + DeleteOptions metav1.DeleteOptions `json:"deleteOptions,omitempty"` +} + +type DoneOperation struct{} + // ScenarioStatus defines the observed state of Scenario. type ScenarioStatus struct { // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster diff --git a/scenario/api/v1alpha1/zz_generated.deepcopy.go b/scenario/api/v1alpha1/zz_generated.deepcopy.go index b255cde3..cd83651e 100644 --- a/scenario/api/v1alpha1/zz_generated.deepcopy.go +++ b/scenario/api/v1alpha1/zz_generated.deepcopy.go @@ -21,15 +21,86 @@ limitations under the License. package v1alpha1 import ( - runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CreateOperation) DeepCopyInto(out *CreateOperation) { + *out = *in + if in.Object != nil { + in, out := &in.Object, &out.Object + *out = (*in).DeepCopy() + } + in.CreateOptions.DeepCopyInto(&out.CreateOptions) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CreateOperation. +func (in *CreateOperation) DeepCopy() *CreateOperation { + if in == nil { + return nil + } + out := new(CreateOperation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeleteOperation) DeepCopyInto(out *DeleteOperation) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.DeleteOptions.DeepCopyInto(&out.DeleteOptions) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeleteOperation. +func (in *DeleteOperation) DeepCopy() *DeleteOperation { + if in == nil { + return nil + } + out := new(DeleteOperation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DoneOperation) DeepCopyInto(out *DoneOperation) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DoneOperation. +func (in *DoneOperation) DeepCopy() *DoneOperation { + if in == nil { + return nil + } + out := new(DoneOperation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PatchOperation) DeepCopyInto(out *PatchOperation) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.PatchOptions.DeepCopyInto(&out.PatchOptions) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PatchOperation. +func (in *PatchOperation) DeepCopy() *PatchOperation { + if in == nil { + return nil + } + out := new(PatchOperation) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Scenario) DeepCopyInto(out *Scenario) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) out.Status = in.Status } @@ -83,9 +154,55 @@ func (in *ScenarioList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ScenarioOperation) DeepCopyInto(out *ScenarioOperation) { + *out = *in + if in.Create != nil { + in, out := &in.Create, &out.Create + *out = new(CreateOperation) + (*in).DeepCopyInto(*out) + } + if in.Patch != nil { + in, out := &in.Patch, &out.Patch + *out = new(PatchOperation) + (*in).DeepCopyInto(*out) + } + if in.Delete != nil { + in, out := &in.Delete, &out.Delete + *out = new(DeleteOperation) + (*in).DeepCopyInto(*out) + } + if in.Done != nil { + in, out := &in.Done, &out.Done + *out = new(DoneOperation) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScenarioOperation. +func (in *ScenarioOperation) DeepCopy() *ScenarioOperation { + if in == nil { + return nil + } + out := new(ScenarioOperation) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ScenarioSpec) DeepCopyInto(out *ScenarioSpec) { *out = *in + if in.Operations != nil { + in, out := &in.Operations, &out.Operations + *out = make([]*ScenarioOperation, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(ScenarioOperation) + (*in).DeepCopyInto(*out) + } + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScenarioSpec. diff --git a/scenario/bootstrap/Dockerfile b/scenario/bootstrap/Dockerfile new file mode 100644 index 00000000..f8a75252 --- /dev/null +++ b/scenario/bootstrap/Dockerfile @@ -0,0 +1,12 @@ +# Dockerfile +FROM bitnami/kubectl:latest + +USER root +RUN install_packages openssl curl ca-certificates \ + && curl -Lo /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 \ + && chmod +x /usr/local/bin/yq + +COPY bootstrap.sh /usr/local/bin/bootstrap.sh +RUN chmod +x /usr/local/bin/bootstrap.sh + +ENTRYPOINT ["bootstrap.sh"] diff --git a/scenario/bootstrap/bootstrap.sh b/scenario/bootstrap/bootstrap.sh new file mode 100644 index 00000000..653ef6b4 --- /dev/null +++ b/scenario/bootstrap/bootstrap.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +set -euo pipefail +trap 'echo "[Error] on line $LINENO"; exit 1' ERR + +readonly KUBECONFIG_PATH=/config/kubeconfig.yaml +readonly TMP_MANIFEST=/tmp/webhook +readonly WORK_MANIFEST=/manifests/webhook +readonly CRD_DIR=/manifests/crd +readonly CAFILE_DIR=$WORK_MANIFEST/ca +readonly CERT_DIR=$WORK_MANIFEST/certs +readonly NAMESPACE=scenario-system + +sync_manifests() { + mkdir -p "$WORK_MANIFEST" + find "$WORK_MANIFEST" -mindepth 1 -maxdepth 1 \ + ! -name certs -exec rm -rf {} + + cp -a "$TMP_MANIFEST"/. "$WORK_MANIFEST"/ +} + +wait_for_k8s() { + echo ">>> Waiting for Kubernetes API server…" + until kubectl get nodes &>/dev/null; do sleep 1; done + echo ">>> API server is up!" +} + +generate_ca() { + mkdir -p "$CAFILE_DIR" + cat >"$CAFILE_DIR/openssl-ca.cnf" <<'EOF' +[req] +distinguished_name = req_distinguished_name +x509_extensions = v3_ca + +[req_distinguished_name] + +[v3_ca] +basicConstraints = critical,CA:TRUE,pathlen:0 +keyUsage = critical,keyCertSign,cRLSign +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always +EOF + + openssl req -x509 -nodes -newkey rsa:2048 -days 3650 \ + -subj "/CN=${NAMESPACE}-webhook-ca" \ + -keyout "$CAFILE_DIR/ca.key" \ + -out "$CAFILE_DIR/ca.crt" \ + -config "$CAFILE_DIR/openssl-ca.cnf" \ + -extensions v3_ca +} + +generate_server_cert() { + mkdir -p "$CERT_DIR" + cat >"$CERT_DIR/openssl-server.cnf" <>> Bootstrap complete" + tail -f /dev/null +} + +main "$@" diff --git a/scenario/cmd/main.go b/scenario/cmd/main.go index d48efb6f..5b85b4bb 100644 --- a/scenario/cmd/main.go +++ b/scenario/cmd/main.go @@ -25,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" @@ -35,8 +36,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" + simulationv1alpha1 "sigs.k8s.io/kube-scheduler-simulator/scenario/api/v1alpha1" "sigs.k8s.io/kube-scheduler-simulator/scenario/internal/controller" + webhooksimulationv1alpha1 "sigs.k8s.io/kube-scheduler-simulator/scenario/internal/webhook/v1alpha1" ) var ( @@ -206,6 +209,13 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "Scenario") os.Exit(1) } + // nolint:goconst + if os.Getenv("ENABLE_WEBHOOKS") != "false" { + if err = webhooksimulationv1alpha1.SetupScenarioWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "Scenario") + os.Exit(1) + } + } // +kubebuilder:scaffold:builder if metricsCertWatcher != nil { diff --git a/scenario/config/crd/bases/simulation.kube-scheduler-simulator.x-k8s.io_scenarios.yaml b/scenario/config/crd/bases/simulation.kube-scheduler-simulator.x-k8s.io_scenarios.yaml index c4fc9b99..e93d6da4 100644 --- a/scenario/config/crd/bases/simulation.kube-scheduler-simulator.x-k8s.io_scenarios.yaml +++ b/scenario/config/crd/bases/simulation.kube-scheduler-simulator.x-k8s.io_scenarios.yaml @@ -43,6 +43,319 @@ spec: description: Foo is an example field of Scenario. Edit scenario_types.go to remove/update type: string + operations: + description: Operations is a list of ScenarioOperation that define + the actions to perform. + items: + properties: + createOperation: + description: Create is the operation to create a new resource. + properties: + createOptions: + description: CreateOptions may be provided when creating + an API object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + dryRun: + description: |- + When present, indicates that modifications should not be + persisted. An invalid or unrecognized dryRun directive will + result in an error response and no further processing of the + request. Valid values are: + - All: all dry run stages will be processed + items: + type: string + type: array + x-kubernetes-list-type: atomic + fieldManager: + description: |- + fieldManager is a name associated with the actor or entity + that is making these changes. The value must be less than or + 128 characters long, and only contain printable characters, + as defined by https://golang.org/pkg/unicode/#IsPrint. + type: string + fieldValidation: + description: |- + fieldValidation instructs the server on how to handle + objects in the request (POST/PUT/PATCH) containing unknown + or duplicate fields. Valid values are: + - Ignore: This will ignore any unknown fields that are silently + dropped from the object, and will ignore all but the last duplicate + field that the decoder encounters. This is the default behavior + prior to v1.23. + - Warn: This will send a warning via the standard warning response + header for each unknown field that is dropped from the object, and + for each duplicate field that is encountered. The request will + still succeed if there are no other errors, and will only persist + the last of any duplicate fields. This is the default in v1.23+ + - Strict: This will fail the request with a BadRequest error if + any unknown fields would be dropped from the object, or if any + duplicate fields are present. The error returned from the server + will contain all unknown and duplicate fields encountered. + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + type: object + object: + description: Object is the Object to be created. + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - object + type: object + deleteOperation: + description: Delete indicates the operation to delete a resource. + properties: + deleteOptions: + description: DeleteOptions may be provided when deleting + an API object. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + dryRun: + description: |- + When present, indicates that modifications should not be + persisted. An invalid or unrecognized dryRun directive will + result in an error response and no further processing of the + request. Valid values are: + - All: all dry run stages will be processed + items: + type: string + type: array + x-kubernetes-list-type: atomic + gracePeriodSeconds: + description: |- + The duration in seconds before the object should be deleted. Value must be non-negative integer. + The value zero indicates delete immediately. If this value is nil, the default grace period for the + specified type will be used. + Defaults to a per object value if not specified. zero means delete immediately. + format: int64 + type: integer + ignoreStoreReadErrorWithClusterBreakingPotential: + description: |- + if set to true, it will trigger an unsafe deletion of the resource in + case the normal deletion flow fails with a corrupt object error. + A resource is considered corrupt if it can not be retrieved from + the underlying storage successfully because of a) its data can + not be transformed e.g. decryption failure, or b) it fails + to decode into an object. + NOTE: unsafe deletion ignores finalizer constraints, skips + precondition checks, and removes the object from the storage. + WARNING: This may potentially break the cluster if the workload + associated with the resource being unsafe-deleted relies on normal + deletion flow. Use only if you REALLY know what you are doing. + The default value is false, and the user must opt in to enable it + type: boolean + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + orphanDependents: + description: |- + Deprecated: please use the PropagationPolicy, this field will be deprecated in 1.7. + Should the dependent objects be orphaned. If true/false, the "orphan" + finalizer will be added to/removed from the object's finalizers list. + Either this field or PropagationPolicy may be set, but not both. + type: boolean + preconditions: + description: |- + Must be fulfilled before a deletion is carried out. If not possible, a 409 Conflict status will be + returned. + properties: + resourceVersion: + description: Specifies the target ResourceVersion + type: string + uid: + description: Specifies the target UID. + type: string + type: object + propagationPolicy: + description: |- + Whether and how garbage collection will be performed. + Either this field or OrphanDependents may be set, but not both. + The default policy is decided by the existing finalizer set in the + metadata.finalizers and the resource-specific default policy. + Acceptable values are: 'Orphan' - orphan the dependents; 'Background' - + allow the garbage collector to delete the dependents in the background; + 'Foreground' - a cascading policy that deletes all dependents in the + foreground. + type: string + type: object + objectMeta: + type: object + typeMeta: + description: |- + TypeMeta describes an individual object in an API response or request + with strings representing the type of the object and its API schema version. + Structures that are versioned or persisted should inline TypeMeta. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + type: object + required: + - objectMeta + - typeMeta + type: object + doneOperation: + description: |- + Done indicates the operation to mark the scenario as Succeeded. + When finish the step DoneOperation belongs, this Scenario changes its status to Succeeded. + type: object + id: + description: ID for this operation. Normally, the system sets + this field for you. + type: string + patchOperation: + description: Patch is the operation to patch a resource. + properties: + objectMeta: + type: object + x-kubernetes-preserve-unknown-fields: true + patch: + description: Patch is the patch for target. + type: string + patchOptions: + description: |- + PatchOptions may be provided when patching an API object. + PatchOptions is meant to be a superset of UpdateOptions. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + dryRun: + description: |- + When present, indicates that modifications should not be + persisted. An invalid or unrecognized dryRun directive will + result in an error response and no further processing of the + request. Valid values are: + - All: all dry run stages will be processed + items: + type: string + type: array + x-kubernetes-list-type: atomic + fieldManager: + description: |- + fieldManager is a name associated with the actor or entity + that is making these changes. The value must be less than or + 128 characters long, and only contain printable characters, + as defined by https://golang.org/pkg/unicode/#IsPrint. This + field is required for apply requests + (application/apply-patch) but optional for non-apply patch + types (JsonPatch, MergePatch, StrategicMergePatch). + type: string + fieldValidation: + description: |- + fieldValidation instructs the server on how to handle + objects in the request (POST/PUT/PATCH) containing unknown + or duplicate fields. Valid values are: + - Ignore: This will ignore any unknown fields that are silently + dropped from the object, and will ignore all but the last duplicate + field that the decoder encounters. This is the default behavior + prior to v1.23. + - Warn: This will send a warning via the standard warning response + header for each unknown field that is dropped from the object, and + for each duplicate field that is encountered. The request will + still succeed if there are no other errors, and will only persist + the last of any duplicate fields. This is the default in v1.23+ + - Strict: This will fail the request with a BadRequest error if + any unknown fields would be dropped from the object, or if any + duplicate fields are present. The error returned from the server + will contain all unknown and duplicate fields encountered. + type: string + force: + description: |- + Force is going to "force" Apply requests. It means user will + re-acquire conflicting fields owned by other people. Force + flag must be unset for non-apply patch requests. + type: boolean + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + type: object + patchType: + description: PatchType + type: string + typeMeta: + description: |- + TypeMeta describes an individual object in an API response or request + with strings representing the type of the object and its API schema version. + Structures that are versioned or persisted should inline TypeMeta. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + type: object + required: + - objectMeta + - patch + - patchType + - typeMeta + type: object + step: + description: MajorStep indicates when the operation should be + done. + format: int32 + type: integer + required: + - id + - step + type: object + type: array type: object status: description: ScenarioStatus defines the observed state of Scenario. diff --git a/scenario/config/webhook/kustomization.yaml b/scenario/config/webhook/kustomization.yaml new file mode 100644 index 00000000..36d4cc6e --- /dev/null +++ b/scenario/config/webhook/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- manifests.yaml +- service.yaml diff --git a/scenario/config/webhook/manifests.yaml b/scenario/config/webhook/manifests.yaml new file mode 100644 index 00000000..ee430a1c --- /dev/null +++ b/scenario/config/webhook/manifests.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-webhook-configuration +webhooks: + - admissionReviewVersions: + - v1 + clientConfig: + service: + name: scenario-webhook-service + namespace: scenario-system + path: /validate-simulation-kube-scheduler-simulator-x-k8s-io-v1alpha1-scenario + port: 9443 + caBundle: "" + failurePolicy: Fail + name: vscenario-v1alpha1.kb.io + rules: + - apiGroups: + - simulation.kube-scheduler-simulator.x-k8s.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - scenarios + sideEffects: None diff --git a/scenario/config/webhook/service.yaml b/scenario/config/webhook/service.yaml new file mode 100644 index 00000000..e97ec373 --- /dev/null +++ b/scenario/config/webhook/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: scenario + app.kubernetes.io/managed-by: kustomize + name: scenario-webhook-service + namespace: scenario-system +spec: + type: ExternalName + externalName: host.docker.internal + ports: + - port: 9443 + protocol: TCP + targetPort: 9443 diff --git a/scenario/go.mod b/scenario/go.mod index e0dc6255..8df0257b 100644 --- a/scenario/go.mod +++ b/scenario/go.mod @@ -7,6 +7,7 @@ godebug default=go1.23 require ( github.com/onsi/ginkgo/v2 v2.21.0 github.com/onsi/gomega v1.35.1 + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 k8s.io/apimachinery v0.32.0 k8s.io/client-go v0.32.0 sigs.k8s.io/controller-runtime v0.20.0 diff --git a/scenario/go.sum b/scenario/go.sum index 72516376..21645dd0 100644 --- a/scenario/go.sum +++ b/scenario/go.sum @@ -196,6 +196,7 @@ golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= diff --git a/scenario/internal/webhook/v1alpha1/scenario_webhook.go b/scenario/internal/webhook/v1alpha1/scenario_webhook.go new file mode 100644 index 00000000..56fb16ae --- /dev/null +++ b/scenario/internal/webhook/v1alpha1/scenario_webhook.go @@ -0,0 +1,109 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "context" + "fmt" + + "golang.org/x/xerrors" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + simulationv1alpha1 "sigs.k8s.io/kube-scheduler-simulator/scenario/api/v1alpha1" +) + +// nolint:unused +// log is for logging in this package. +var scenariolog = logf.Log.WithName("scenario-resource") + +// SetupScenarioWebhookWithManager registers the webhook for Scenario in the manager. +func SetupScenarioWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr).For(&simulationv1alpha1.Scenario{}). + WithValidator(&ScenarioCustomValidator{}). + Complete() +} + +// TODO(user): EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! + +// TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation. +// NOTE: The 'path' attribute must follow a specific pattern and should not be modified directly here. +// Modifying the path for an invalid path can cause API server errors; failing to locate the webhook. +// +kubebuilder:webhook:path=/validate-simulation-kube-scheduler-simulator-x-k8s-io-v1alpha1-scenario,mutating=false,failurePolicy=fail,sideEffects=None,groups=simulation.kube-scheduler-simulator.x-k8s.io,resources=scenarios,verbs=create;update,versions=v1alpha1,name=vscenario-v1alpha1.kb.io,admissionReviewVersions=v1 + +// ScenarioCustomValidator struct is responsible for validating the Scenario resource +// when it is created, updated, or deleted. +// +// NOTE: The +kubebuilder:object:generate=false marker prevents controller-gen from generating DeepCopy methods, +// as this struct is used only for temporary operations and does not need to be deeply copied. +type ScenarioCustomValidator struct { + // TODO(user): Add more fields as needed for validation +} + +var _ webhook.CustomValidator = &ScenarioCustomValidator{} + +// ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type Scenario. +func (v *ScenarioCustomValidator) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + scenario, ok := obj.(*simulationv1alpha1.Scenario) + if !ok { + return nil, fmt.Errorf("expected a Scenario object but got %T", obj) + } + scenariolog.Info("Validation for Scenario upon creation", "name", scenario.GetName()) + + for _, op := range scenario.Spec.Operations { + err := op.ValidateCreate() + if err != nil { + return nil, xerrors.Errorf("scenario webhook ValidateCreate: %w", err) + } + } + + return nil, nil +} + +// ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type Scenario. +func (v *ScenarioCustomValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + scenario, ok := newObj.(*simulationv1alpha1.Scenario) + if !ok { + return nil, fmt.Errorf("expected a Scenario object for the newObj but got %T", newObj) + } + scenariolog.Info("Validation for Scenario upon update", "name", scenario.GetName()) + + for _, op := range scenario.Spec.Operations { + err := op.ValidateUpdate(oldObj) + if err != nil { + return nil, xerrors.Errorf("scenario webhook ValidateUpdate: %w", err) + } + } + + return nil, nil +} + +// ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type Scenario. +func (v *ScenarioCustomValidator) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + scenario, ok := obj.(*simulationv1alpha1.Scenario) + if !ok { + return nil, fmt.Errorf("expected a Scenario object but got %T", obj) + } + scenariolog.Info("Validation for Scenario upon deletion", "name", scenario.GetName()) + + // TODO(user): fill in your validation logic upon object deletion. + + return nil, nil +} diff --git a/scenario/internal/webhook/v1alpha1/scenario_webhook_test.go b/scenario/internal/webhook/v1alpha1/scenario_webhook_test.go new file mode 100644 index 00000000..00358cc5 --- /dev/null +++ b/scenario/internal/webhook/v1alpha1/scenario_webhook_test.go @@ -0,0 +1,92 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + simulationv1alpha1 "sigs.k8s.io/kube-scheduler-simulator/scenario/api/v1alpha1" + // TODO (user): Add any additional imports if needed +) + +var _ = Describe("Scenario Webhook", func() { + var ( + obj *simulationv1alpha1.Scenario + oldObj *simulationv1alpha1.Scenario + validator ScenarioCustomValidator + ) + + BeforeEach(func() { + obj = &simulationv1alpha1.Scenario{} + oldObj = &simulationv1alpha1.Scenario{} + validator = ScenarioCustomValidator{} + Expect(validator).NotTo(BeNil(), "Expected validator to be initialized") + Expect(oldObj).NotTo(BeNil(), "Expected oldObj to be initialized") + Expect(obj).NotTo(BeNil(), "Expected obj to be initialized") + // TODO (user): Add any setup logic common to all tests + }) + + AfterEach(func() { + // TODO (user): Add any teardown logic common to all tests + }) + + Context("When creating or updating Scenario under Validating Webhook", func() { + It("Should deny creation if there are some operation fields", func() { + By("simulating an invalid creation scenario") + obj.Spec.Operations = []*simulationv1alpha1.ScenarioOperation{ + { + Create: &simulationv1alpha1.CreateOperation{}, + Delete: &simulationv1alpha1.DeleteOperation{}, + }, + } + Expect(validator.ValidateCreate(ctx, obj)).Error().To(HaveOccurred()) + }) + + It("Should validate create correctly", func() { + By("simulating a valid creation scenario") + obj.Spec.Operations = []*simulationv1alpha1.ScenarioOperation{ + { + Create: &simulationv1alpha1.CreateOperation{}, + }, + } + Expect(validator.ValidateCreate(ctx, obj)).To(BeNil()) + }) + + It("Should deny update if there are some operation fields", func() { + By("simulating an invalid creation scenario") + obj.Spec.Operations = []*simulationv1alpha1.ScenarioOperation{ + { + Create: &simulationv1alpha1.CreateOperation{}, + Delete: &simulationv1alpha1.DeleteOperation{}, + }, + } + Expect(validator.ValidateUpdate(ctx, oldObj, obj)).Error().To(HaveOccurred()) + }) + + It("Should validate updates correctly", func() { + By("simulating a valid update scenario") + obj.Spec.Operations = []*simulationv1alpha1.ScenarioOperation{ + { + Create: &simulationv1alpha1.CreateOperation{}, + }, + } + Expect(validator.ValidateUpdate(ctx, oldObj, obj)).To(BeNil()) + }) + }) + +}) diff --git a/scenario/internal/webhook/v1alpha1/webhook_suite_test.go b/scenario/internal/webhook/v1alpha1/webhook_suite_test.go new file mode 100644 index 00000000..1ad54ba1 --- /dev/null +++ b/scenario/internal/webhook/v1alpha1/webhook_suite_test.go @@ -0,0 +1,164 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "os" + "path/filepath" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + simulationv1alpha1 "sigs.k8s.io/kube-scheduler-simulator/scenario/api/v1alpha1" + // +kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var ( + ctx context.Context + cancel context.CancelFunc + k8sClient client.Client + cfg *rest.Config + testEnv *envtest.Environment +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Webhook Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + var err error + err = simulationv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:scheme + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: false, + + WebhookInstallOptions: envtest.WebhookInstallOptions{ + Paths: []string{filepath.Join("..", "..", "..", "config", "webhook")}, + }, + } + + // Retrieve the first found binary directory to allow running tests from IDEs + if getFirstFoundEnvTestBinaryDir() != "" { + testEnv.BinaryAssetsDirectory = getFirstFoundEnvTestBinaryDir() + } + + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + // start webhook server using Manager. + webhookInstallOptions := &testEnv.WebhookInstallOptions + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme.Scheme, + WebhookServer: webhook.NewServer(webhook.Options{ + Host: webhookInstallOptions.LocalServingHost, + Port: webhookInstallOptions.LocalServingPort, + CertDir: webhookInstallOptions.LocalServingCertDir, + }), + LeaderElection: false, + Metrics: metricsserver.Options{BindAddress: "0"}, + }) + Expect(err).NotTo(HaveOccurred()) + + err = SetupScenarioWebhookWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:webhook + + go func() { + defer GinkgoRecover() + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + }() + + // wait for the webhook server to get ready. + dialer := &net.Dialer{Timeout: time.Second} + addrPort := fmt.Sprintf("%s:%d", webhookInstallOptions.LocalServingHost, webhookInstallOptions.LocalServingPort) + Eventually(func() error { + conn, err := tls.DialWithDialer(dialer, "tcp", addrPort, &tls.Config{InsecureSkipVerify: true}) + if err != nil { + return err + } + + return conn.Close() + }).Should(Succeed()) +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + cancel() + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) + +// getFirstFoundEnvTestBinaryDir locates the first binary in the specified path. +// ENVTEST-based tests depend on specific binaries, usually located in paths set by +// controller-runtime. When running tests directly (e.g., via an IDE) without using +// Makefile targets, the 'BinaryAssetsDirectory' must be explicitly configured. +// +// This function streamlines the process by finding the required binaries, similar to +// setting the 'KUBEBUILDER_ASSETS' environment variable. To ensure the binaries are +// properly set up, run 'make setup-envtest' beforehand. +func getFirstFoundEnvTestBinaryDir() string { + basePath := filepath.Join("..", "..", "..", "bin", "k8s") + entries, err := os.ReadDir(basePath) + if err != nil { + logf.Log.Error(err, "Failed to read directory", "path", basePath) + return "" + } + for _, entry := range entries { + if entry.IsDir() { + return filepath.Join(basePath, entry.Name()) + } + } + return "" +} diff --git a/scenario/test/e2e/e2e_test.go b/scenario/test/e2e/e2e_test.go index 112279c2..3b09d47d 100644 --- a/scenario/test/e2e/e2e_test.go +++ b/scenario/test/e2e/e2e_test.go @@ -26,6 +26,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "sigs.k8s.io/kube-scheduler-simulator/scenario/test/utils" ) @@ -260,6 +261,30 @@ var _ = Describe("Manager", Ordered, func() { )) }) + It("should provisioned cert-manager", func() { + By("validating that cert-manager has the certificate Secret") + verifyCertManager := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "secrets", "webhook-server-cert", "-n", namespace) + _, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + } + Eventually(verifyCertManager).Should(Succeed()) + }) + + It("should have CA injection for validating webhooks", func() { + By("checking CA injection for validating webhooks") + verifyCAInjection := func(g Gomega) { + cmd := exec.Command("kubectl", "get", + "validatingwebhookconfigurations.admissionregistration.k8s.io", + "scenario-validating-webhook-configuration", + "-o", "go-template={{ range .webhooks }}{{ .clientConfig.caBundle }}{{ end }}") + vwhOutput, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(vwhOutput)).To(BeNumerically(">", 10)) + } + Eventually(verifyCAInjection).Should(Succeed()) + }) + // +kubebuilder:scaffold:e2e-webhooks-checks // TODO: Customize the e2e test suite with scenarios specific to your project.