diff --git a/PROJECT b/PROJECT index 89c2c00a5..9efb75bbd 100644 --- a/PROJECT +++ b/PROJECT @@ -1,3 +1,7 @@ +# Code generated by tool. DO NOT EDIT. +# This file is used to track the info used to scaffold your project +# and allow the plugins properly work. +# More info: https://book.kubebuilder.io/reference/project-config.html domain: openstack.org layout: - go.kubebuilder.io/v3 @@ -111,4 +115,17 @@ resources: defaulting: true validation: true webhookVersion: v1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: openstack.org + group: nova + kind: NovaPlacementAPI + path: github.com/openstack-k8s-operators/nova-operator/api/v1beta1 + version: v1beta1 + webhooks: + defaulting: true + validation: true + webhookVersion: v1 version: "3" diff --git a/api/bases/nova.openstack.org_novaplacementapis.yaml b/api/bases/nova.openstack.org_novaplacementapis.yaml new file mode 100644 index 000000000..76033ea5a --- /dev/null +++ b/api/bases/nova.openstack.org_novaplacementapis.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: novaplacementapis.nova.openstack.org +spec: + group: nova.openstack.org + names: + kind: NovaPlacementAPI + listKind: NovaPlacementAPIList + plural: novaplacementapis + singular: novaplacementapi + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: NovaPlacementAPI is the Schema for the novaplacementapis API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NovaPlacementAPISpec defines the desired state of NovaPlacementAPI + properties: + foo: + description: Foo is an example field of NovaPlacementAPI. Edit novaplacementapi_types.go + to remove/update + type: string + type: object + status: + description: NovaPlacementAPIStatus defines the observed state of NovaPlacementAPI + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/api/go.mod b/api/go.mod index 4deafae72..f4b7f07ef 100644 --- a/api/go.mod +++ b/api/go.mod @@ -4,11 +4,14 @@ go 1.21 require ( github.com/google/go-cmp v0.7.0 + github.com/onsi/ginkgo/v2 v2.20.1 + github.com/onsi/gomega v1.34.1 github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20250513115636-b549982a5d8f github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20250508141203-be026d3164f7 github.com/robfig/cron/v3 v3.0.1 k8s.io/api v0.29.15 k8s.io/apimachinery v0.29.15 + k8s.io/client-go v0.29.15 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 sigs.k8s.io/controller-runtime v0.17.6 ) @@ -21,14 +24,17 @@ require ( github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect github.com/google/uuid v1.6.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -43,6 +49,8 @@ require ( github.com/prometheus/common v0.51.1 // indirect github.com/prometheus/procfs v0.13.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/net v0.28.0 // indirect golang.org/x/oauth2 v0.18.0 // indirect @@ -50,6 +58,7 @@ require ( golang.org/x/term v0.23.0 // indirect golang.org/x/text v0.17.0 // indirect golang.org/x/time v0.5.0 // indirect + golang.org/x/tools v0.24.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.34.1 // indirect @@ -57,7 +66,6 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.29.15 // indirect - k8s.io/client-go v0.29.15 // indirect k8s.io/component-base v0.29.15 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240322212309-b815d8309940 // indirect diff --git a/api/go.sum b/api/go.sum index 29d00fc5f..9ae665e67 100644 --- a/api/go.sum +++ b/api/go.sum @@ -24,7 +24,6 @@ github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= diff --git a/api/v1beta1/common_types.go b/api/v1beta1/common_types.go index 73d580c5a..046f7c8f8 100644 --- a/api/v1beta1/common_types.go +++ b/api/v1beta1/common_types.go @@ -25,12 +25,13 @@ import ( // Container image fall-back defaults const ( - NovaAPIContainerImage = "quay.io/podified-antelope-centos9/openstack-nova-api:current-podified" - NovaConductorContainerImage = "quay.io/podified-antelope-centos9/openstack-nova-conductor:current-podified" - NovaMetadataContainerImage = "quay.io/podified-antelope-centos9/openstack-nova-api:current-podified" - NovaNoVNCContainerImage = "quay.io/podified-antelope-centos9/openstack-nova-novncproxy:current-podified" - NovaSchedulerContainerImage = "quay.io/podified-antelope-centos9/openstack-nova-scheduler:current-podified" - NovaComputeContainerImage = "quay.io/podified-antelope-centos9/openstack-nova-compute:current-podified" + NovaAPIContainerImage = "quay.io/podified-antelope-centos9/openstack-nova-api:current-podified" + NovaConductorContainerImage = "quay.io/podified-antelope-centos9/openstack-nova-conductor:current-podified" + NovaMetadataContainerImage = "quay.io/podified-antelope-centos9/openstack-nova-api:current-podified" + NovaNoVNCContainerImage = "quay.io/podified-antelope-centos9/openstack-nova-novncproxy:current-podified" + NovaSchedulerContainerImage = "quay.io/podified-antelope-centos9/openstack-nova-scheduler:current-podified" + NovaComputeContainerImage = "quay.io/podified-antelope-centos9/openstack-nova-compute:current-podified" + NovaPlacementAPIContainerImage = "quay.io/podified-antelope-centos9/openstack-placement-api:current-podified" ) // Compute drivers names diff --git a/api/v1beta1/novaplacementapi_types.go b/api/v1beta1/novaplacementapi_types.go new file mode 100644 index 000000000..33cec54fa --- /dev/null +++ b/api/v1beta1/novaplacementapi_types.go @@ -0,0 +1,239 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + service "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// NovaPlacementAPITemplate defines the input parameters specified by the user to +// create a NovaPlacement via higher level CRDs. + +// NovaPlacementAPISpec defines the desired state of NovaPlacementAPI +type NovaPlacementAPISpec struct { + NovaPlacementAPISpecCore `json:",inline"` + + // +kubebuilder:validation:Required + //NovaPlacementAPI Container Image URL (will be set to environmental default if empty) + ContainerImage string `json:"containerImage"` +} + +// NovaPlacementAPISpecCore - +type NovaPlacementAPISpecCore struct { + // +kubebuilder:validation:Optional + // +kubebuilder:default=60 + // +kubebuilder:validation:Minimum=10 + // APITimeout for HAProxy, Apache + APITimeout int `json:"apiTimeout"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=placement + // ServiceUser - optional username used for this service to register in keystone + ServiceUser string `json:"serviceUser"` + + // +kubebuilder:validation:Required + // MariaDB instance name + // Right now required by the maridb-operator to get the credentials from the instance to create the DB + // Might not be required in future + DatabaseInstance string `json:"databaseInstance"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=placement + // DatabaseAccount - name of MariaDBAccount which will be used to connect. + DatabaseAccount string `json:"databaseAccount"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=1 + // +kubebuilder:validation:Maximum=32 + // +kubebuilder:validation:Minimum=0 + // Replicas of placement API to run + Replicas *int32 `json:"replicas"` + + // +kubebuilder:validation:Required + // Secret containing OpenStack password information for placement PlacementPassword + Secret string `json:"secret"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default={service: PlacementPassword} + // PasswordSelectors - Selectors to identify the DB and ServiceUser password from the Secret + PasswordSelectors PasswordSelector `json:"passwordSelectors"` + + // +kubebuilder:validation:Optional + // NodeSelector to target subset of worker nodes running this service + NodeSelector *map[string]string `json:"nodeSelector,omitempty"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // PreserveJobs - do not delete jobs after they finished e.g. to check logs + PreserveJobs bool `json:"preserveJobs"` + + // +kubebuilder:validation:Optional + // CustomServiceConfig - customize the service config using this parameter to change service defaults, + // or overwrite rendered information using raw OpenStack config format. The content gets added to + // to /etc//.conf.d directory as custom.conf file. + CustomServiceConfig string `json:"customServiceConfig"` + + // +kubebuilder:validation:Optional + // DefaultConfigOverwrite - interface to overwrite default config files like policy.yaml. + DefaultConfigOverwrite map[string]string `json:"defaultConfigOverwrite,omitempty"` + + // +kubebuilder:validation:Optional + // Resources - Compute Resources required by this service (Limits/Requests). + // https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + + // +kubebuilder:validation:Optional + // NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network + NetworkAttachments []string `json:"networkAttachments,omitempty"` + + // +kubebuilder:validation:Optional + // Override, provides the ability to override the generated manifest of several child resources. + Override APIOverrideSpec `json:"override,omitempty"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.API `json:"tls,omitempty"` + + // +kubebuilder:validation:Optional + // TopologyRef to apply the Topology defined by the associated CR referenced + // by name + TopologyRef *topologyv1.TopoRef `json:"topologyRef,omitempty"` +} + +// APIOverrideSpec to override the generated manifest of several child resources. +type APIOverrideSpec struct { + // Override configuration for the Service created to serve traffic to the cluster. + // The key must be the endpoint type (public, internal) + Service map[service.Endpoint]service.RoutedOverrideSpec `json:"service,omitempty"` +} + +// PasswordSelector to identify the DB and AdminUser password from the Secret +type PasswordSelector struct { + // +kubebuilder:validation:Optional + // +kubebuilder:default="PlacementPassword" + // Service - Selector to get the service user password from the Secret + Service string `json:"service"` +} + +// NovaPlacementAPIStatus defines the observed state of NovaPlacementAPI +type NovaPlacementAPIStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file + // ReadyCount of placement API instances + ReadyCount int32 `json:"readyCount,omitempty"` + + // Map of hashes to track e.g. job status + Hash map[string]string `json:"hash,omitempty"` + + // Conditions + Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` + + // Placement Database Hostname + DatabaseHostname string `json:"databaseHostname,omitempty"` + + // NetworkAttachments status of the deployment pods + NetworkAttachments map[string][]string `json:"networkAttachments,omitempty"` + + //ObservedGeneration - the most recent generation observed for this service. If the observed generation is less than the spec generation, then the controller has not processed the latest changes. + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // LastAppliedTopology - the last applied Topology + LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` +} + +// NovaPlacementAPI is the Schema for the novaplacementapis API +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="NetworkAttachments",type="string",JSONPath=".spec.networkAttachments",description="NetworkAttachments" +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[0].status",description="Status" +// +kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.conditions[0].message",description="Message" +type NovaPlacementAPI struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NovaPlacementAPISpec `json:"spec,omitempty"` + Status NovaPlacementAPIStatus `json:"status,omitempty"` +} + +// NovaPlacementAPIList contains a list of NovaPlacementAPI +//+kubebuilder:object:root=true +type NovaPlacementAPIList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NovaPlacementAPI `json:"items"` +} + +func init() { + SchemeBuilder.Register(&NovaPlacementAPI{}, &NovaPlacementAPIList{}) +} + +// IsReady - returns true if NovaPlacementAPI is reconciled successfully +func (instance NovaPlacementAPI) IsReady() bool { + return instance.Status.Conditions.IsTrue(condition.ReadyCondition) +} + +// RbacConditionsSet - set the conditions for the rbac object +func (instance NovaPlacementAPI) RbacConditionsSet(c *condition.Condition) { + instance.Status.Conditions.Set(c) +} + +// RbacNamespace - return the namespace +func (instance NovaPlacementAPI) RbacNamespace() string { + return instance.Namespace +} + +// RbacResourceName - return the name to be used for rbac objects (serviceaccount, role, rolebinding) +func (instance NovaPlacementAPI) RbacResourceName() string { + return "placement-" + instance.Name +} + +// SetupDefaults - initializes any CRD field defaults based on environment variables (the defaulting mechanism itself is implemented via webhooks) +func SetupDefaults() { + // Acquire environmental defaults and initialize NovaPlacement defaults with them + placementDefaults := NovaPlacementAPIDefaults{ + ContainerImageURL: util.GetEnvVar("RELATED_IMAGE_PLACEMENT_API_IMAGE_URL_DEFAULT", NovaPlacementAPIContainerImage), + APITimeout: 60, + } + + SetupNovaPlacementAPIDefaults(placementDefaults) +} + +// GetSecret returns the value of the Nova.Spec.Secret +func (instance NovaPlacementAPI) GetSecret() string { + return instance.Spec.Secret +} + +// ValidateTopology - +func (instance *NovaPlacementAPISpecCore) ValidateTopology( + basePath *field.Path, + namespace string, +) field.ErrorList { + var allErrs field.ErrorList + allErrs = append(allErrs, topologyv1.ValidateTopologyRef( + instance.TopologyRef, + *basePath.Child("topologyRef"), namespace)...) + return allErrs +} diff --git a/api/v1beta1/novaplacementapi_webhook.go b/api/v1beta1/novaplacementapi_webhook.go new file mode 100644 index 000000000..7703bc23c --- /dev/null +++ b/api/v1beta1/novaplacementapi_webhook.go @@ -0,0 +1,211 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "fmt" + + "github.com/openstack-k8s-operators/lib-common/modules/common/service" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +type NovaPlacementAPIDefaults struct { + ContainerImageURL string + APITimeout int +} + +var novaPlacementAPIDefaults NovaPlacementAPIDefaults + +// log is for logging in this package. +var novaplacementapilog = logf.Log.WithName("novaplacementapi-resource") + +// SetupNovaPlacementAPIDefaults - initialize NovaPlacementAPI spec defaults for use with either internal or external webhooks +func SetupNovaPlacementAPIDefaults(defaults NovaPlacementAPIDefaults) { + novaPlacementAPIDefaults = defaults + novaplacementapilog.Info("NovaPlacementAPI defaults initialized", "defaults", defaults) +} + +// SetupWebhookWithManager sets up the webhook with the Manager +func (r *NovaPlacementAPI) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +// TODO(user): EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! + +//+kubebuilder:webhook:path=/mutate-nova-openstack-org-v1beta1-novaplacementapi,mutating=true,failurePolicy=fail,sideEffects=None,groups=nova.openstack.org,resources=novaplacementapis,verbs=create;update,versions=v1beta1,name=mnovaplacementapi.kb.io,admissionReviewVersions=v1 + +var _ webhook.Defaulter = &NovaPlacementAPI{} + +// Default implements webhook.Defaulter so a webhook will be registered for the type +func (r *NovaPlacementAPI) Default() { + novaplacementapilog.Info("default", "name", r.Name) + + r.Spec.Default() + +} + +// Default - set defaults for this NovaPlacementAPI spec +func (spec *NovaPlacementAPISpec) Default() { + if spec.ContainerImage == "" { + spec.ContainerImage = novaplacementAPIDefaults.ContainerImageURL + } + if spec.APITimeout == 0 { + spec.APITimeout = novaPlacementAPIDefaults.APITimeout + } + +} + +// Default - set defaults for this NovaPlacementAPI core spec (this version is used by the OpenStackControlplane webhook) +func (spec *NovaPlacementAPISpecCore) Default() { + // nothing here yet +} + + +// TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation. +//+kubebuilder:webhook:path=/validate-nova-openstack-org-v1beta1-novaplacementapi,mutating=false,failurePolicy=fail,sideEffects=None,groups=nova.openstack.org,resources=novaplacementapis,verbs=create;update,versions=v1beta1,name=vnovaplacementapi.kb.io,admissionReviewVersions=v1 + +var _ webhook.Validator = &NovaPlacementAPI{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type +func (r *NovaPlacementAPI) ValidateCreate() (admission.Warnings, error) { + novaplacementapilog.Info("validate create", "name", r.Name) + + errors := r.Spec.ValidateCreate(field.NewPath("spec"), r.Namespace) + if len(errors) != 0 { + placementapilog.Info("validation failed", "name", r.Name) + return nil, apierrors.NewInvalid( + schema.GroupKind{Group: "nova.openstack.org", Kind: "NovaPlacementAPI"}, + r.Name, errors) + } + return nil, nil +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type +func (r *NovaPlacementAPI) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { + novaplacementapilog.Info("validate update", "name", r.Name) + oldPlacement, ok := old.(*NovaPlacementAPI) + if !ok || oldPlacement == nil { + return nil, apierrors.NewInternalError(fmt.Errorf("unable to convert existing object")) + } + + errors := r.Spec.ValidateUpdate(oldPlacement.Spec, field.NewPath("spec"), r.Namespace) + if len(errors) != 0 { + placementapilog.Info("validation failed", "name", r.Name) + return nil, apierrors.NewInvalid( + schema.GroupKind{Group: "nova.openstack.org", Kind: "NovaPlacementAPI"}, + r.Name, errors) + } + return nil, nil +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type +func (r *NovaPlacementAPI) ValidateDelete() (admission.Warnings, error) { + novaplacementapilog.Info("validate delete", "name", r.Name) + + // TODO(user): fill in your validation logic upon object deletion. + return nil +} + +func (r NovaPlacementAPISpec) ValidateCreate(basePath *field.Path, namespace string) field.ErrorList { + return r.NovaPlacementAPISpecCore.ValidateCreate(basePath, namespace) +} + +func (r NovaPlacementAPISpec) ValidateUpdate(old NovaPlacementAPISpec, basePath *field.Path, namespace string) field.ErrorList { + return r.NovaPlacementAPISpecCore.ValidateCreate(basePath, namespace) +} + +func (r NovaPlacementAPISpecCore) ValidateCreate(basePath *field.Path, namespace string) field.ErrorList { + var allErrs field.ErrorList + + // validate the service override key is valid + allErrs = append(allErrs, service.ValidateRoutedOverrides(basePath.Child("override").Child("service"), r.Override.Service)...) + + allErrs = append(allErrs, ValidateDefaultConfigOverwrite(basePath, r.DefaultConfigOverwrite)...) + + // When a TopologyRef CR is referenced, fail if a different Namespace is + // referenced because is not supported + allErrs = append(allErrs, r.ValidateTopology(basePath, namespace)...) + + return allErrs +} + +func (r NovaPlacementAPISpecCore) ValidateUpdate(old NovaPlacementAPISpecCore, basePath *field.Path, namespace string) field.ErrorList { + var allErrs field.ErrorList + + // validate the service override key is valid + allErrs = append(allErrs, service.ValidateRoutedOverrides(basePath.Child("override").Child("service"), r.Override.Service)...) + + allErrs = append(allErrs, ValidateDefaultConfigOverwrite(basePath, r.DefaultConfigOverwrite)...) + + // When a TopologyRef CR is referenced, fail if a different Namespace is + // referenced because is not supported + allErrs = append(allErrs, r.ValidateTopology(basePath, namespace)...) + + return allErrs +} + +func ValidateDefaultConfigOverwrite( + basePath *field.Path, + validateConfigOverwrite map[string]string, +) field.ErrorList { + var errors field.ErrorList + for requested := range validateConfigOverwrite { + if requested != "policy.yaml" { + errors = append( + errors, + field.Invalid( + basePath.Child("defaultConfigOverwrite"), + requested, + "Only the following keys are valid: policy.yaml", + ), + ) + } + } + return errors +} + +// SetDefaultRouteAnnotations sets HAProxy timeout values of the route +func (spec *NovaPlacementAPISpecCore) SetDefaultRouteAnnotations(annotations map[string]string) { + const haProxyAnno = "haproxy.router.openshift.io/timeout" + // Use a custom annotation to flag when the operator has set the default HAProxy timeout + // With the annotation func determines when to overwrite existing HAProxy timeout with the APITimeout + const placementAnno = "api.nova.openstack.org/timeout" + valPlacementAPI, okPlacementAPI := annotations[placementAnno] + valHAProxy, okHAProxy := annotations[haProxyAnno] + // Human operator set the HAProxy timeout manually + if !okPlacementAPI && okHAProxy { + return + } + // Human operator modified the HAProxy timeout manually without removing the Placemen flag + if okPlacementAPI && okHAProxy && valPlacementAPI != valHAProxy { + delete(annotations, placementAnno) + placementapilog.Info("Human operator modified the HAProxy timeout manually without removing the Placement flag. Deleting the Placement flag to ensure proper configuration.") + return + } + timeout := fmt.Sprintf("%ds", spec.APITimeout) + annotations[placementAnno] = timeout + annotations[haProxyAnno] = timeout +} + diff --git a/api/v1beta1/webhook_suite_test.go b/api/v1beta1/webhook_suite_test.go new file mode 100644 index 000000000..de9e506e3 --- /dev/null +++ b/api/v1beta1/webhook_suite_test.go @@ -0,0 +1,132 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "path/filepath" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + admissionv1beta1 "k8s.io/api/admission/v1beta1" + //+kubebuilder:scaffold:imports + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment +var ctx context.Context +var cancel context.CancelFunc + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Webhook Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: false, + WebhookInstallOptions: envtest.WebhookInstallOptions{ + Paths: []string{filepath.Join("..", "..", "config", "webhook")}, + }, + } + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + scheme := runtime.NewScheme() + err = AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + + err = admissionv1beta1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + + // start webhook server using Manager + webhookInstallOptions := &testEnv.WebhookInstallOptions + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ + Scheme: scheme, + Host: webhookInstallOptions.LocalServingHost, + Port: webhookInstallOptions.LocalServingPort, + CertDir: webhookInstallOptions.LocalServingCertDir, + LeaderElection: false, + MetricsBindAddress: "0", + }) + Expect(err).NotTo(HaveOccurred()) + + err = (&NovaPlacementAPI{}).SetupWebhookWithManager(mgr) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:webhook + + go func() { + defer GinkgoRecover() + err = mgr.Start(ctx) + Expect(err).NotTo(HaveOccurred()) + }() + + // wait for the webhook server to get ready + dialer := &net.Dialer{Timeout: time.Second} + addrPort := fmt.Sprintf("%s:%d", webhookInstallOptions.LocalServingHost, webhookInstallOptions.LocalServingPort) + Eventually(func() error { + conn, err := tls.DialWithDialer(dialer, "tcp", addrPort, &tls.Config{InsecureSkipVerify: true}) + if err != nil { + return err + } + conn.Close() + return nil + }).Should(Succeed()) + +}) + +var _ = AfterSuite(func() { + cancel() + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index e493aca03..45f9b347e 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1407,6 +1407,95 @@ func (in *NovaNoVNCProxyTemplate) DeepCopy() *NovaNoVNCProxyTemplate { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NovaPlacementAPI) DeepCopyInto(out *NovaPlacementAPI) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NovaPlacementAPI. +func (in *NovaPlacementAPI) DeepCopy() *NovaPlacementAPI { + if in == nil { + return nil + } + out := new(NovaPlacementAPI) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NovaPlacementAPI) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NovaPlacementAPIList) DeepCopyInto(out *NovaPlacementAPIList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NovaPlacementAPI, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NovaPlacementAPIList. +func (in *NovaPlacementAPIList) DeepCopy() *NovaPlacementAPIList { + if in == nil { + return nil + } + out := new(NovaPlacementAPIList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NovaPlacementAPIList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NovaPlacementAPISpec) DeepCopyInto(out *NovaPlacementAPISpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NovaPlacementAPISpec. +func (in *NovaPlacementAPISpec) DeepCopy() *NovaPlacementAPISpec { + if in == nil { + return nil + } + out := new(NovaPlacementAPISpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NovaPlacementAPIStatus) DeepCopyInto(out *NovaPlacementAPIStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NovaPlacementAPIStatus. +func (in *NovaPlacementAPIStatus) DeepCopy() *NovaPlacementAPIStatus { + if in == nil { + return nil + } + out := new(NovaPlacementAPIStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NovaScheduler) DeepCopyInto(out *NovaScheduler) { *out = *in diff --git a/config/crd/bases/nova.openstack.org_novaplacementapis.yaml b/config/crd/bases/nova.openstack.org_novaplacementapis.yaml new file mode 100644 index 000000000..167de93b1 --- /dev/null +++ b/config/crd/bases/nova.openstack.org_novaplacementapis.yaml @@ -0,0 +1,493 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: novaplacementapis.nova.openstack.org +spec: + group: nova.openstack.org + names: + kind: NovaPlacementAPI + listKind: NovaPlacementAPIList + plural: novaplacementapis + singular: novaplacementapi + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: NetworkAttachments + jsonPath: .spec.networkAttachments + name: NetworkAttachments + type: string + - description: Status + jsonPath: .status.conditions[0].status + name: Status + type: string + - description: Message + jsonPath: .status.conditions[0].message + name: Message + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: NovaPlacementAPI is the Schema for the novaplacementapis API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NovaPlacementAPISpec defines the desired state of NovaPlacementAPI + properties: + apiTimeout: + default: 60 + description: APITimeout for HAProxy, Apache + minimum: 10 + type: integer + containerImage: + description: NovaPlacementAPI Container Image URL (will be set to + environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as custom.conf file. + type: string + databaseAccount: + default: placement + description: DatabaseAccount - name of MariaDBAccount which will be + used to connect. + type: string + databaseInstance: + description: |- + MariaDB instance name + Right now required by the maridb-operator to get the credentials from the instance to create the DB + Might not be required in future + type: string + defaultConfigOverwrite: + additionalProperties: + type: string + description: DefaultConfigOverwrite - interface to overwrite default + config files like policy.yaml. + type: object + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment resource + names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: NodeSelector to target subset of worker nodes running + this service + type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + passwordSelectors: + default: + service: PlacementPassword + description: PasswordSelectors - Selectors to identify the DB and + ServiceUser password from the Secret + properties: + service: + default: PlacementPassword + description: Service - Selector to get the service user password + from the Secret + type: string + type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean + replicas: + default: 1 + description: Replicas of placement API to run + format: int32 + maximum: 32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + secret: + description: Secret containing OpenStack password information for + placement PlacementPassword + type: string + serviceUser: + default: placement + description: ServiceUser - optional username used for this service + to register in keystone + type: string + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret for + the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + public: + description: Public GenericService - holds the secret for + the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + - databaseInstance + - secret + type: object + status: + description: NovaPlacementAPIStatus defines the observed state of NovaPlacementAPI + properties: + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + databaseHostname: + description: Placement Database Hostname + type: string + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object + lastAppliedTopology: + description: LastAppliedTopology - the last applied Topology + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + networkAttachments: + additionalProperties: + items: + type: string + type: array + description: NetworkAttachments status of the deployment pods + type: object + observedGeneration: + description: ObservedGeneration - the most recent generation observed + for this service. If the observed generation is less than the spec + generation, then the controller has not processed the latest changes. + format: int64 + type: integer + readyCount: + description: |- + INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + Important: Run "make" to regenerate code after modifying this file + ReadyCount of placement API instances + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 5df6d225d..be45367f0 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -10,6 +10,7 @@ resources: - bases/nova.openstack.org_novacells.yaml - bases/nova.openstack.org_nova.yaml - bases/nova.openstack.org_novacomputes.yaml +- bases/nova.openstack.org_novaplacementapis.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -22,6 +23,7 @@ patchesStrategicMerge: #- patches/webhook_in_novanovncproxies.yaml #- patches/webhook_in_novacells.yaml #- patches/webhook_in_nova.yaml +#- patches/webhook_in_novaplacementapis.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. @@ -33,6 +35,7 @@ patchesStrategicMerge: #- patches/cainjection_in_novanovncproxies.yaml #- patches/cainjection_in_novacells.yaml #- patches/cainjection_in_nova.yaml +#- patches/cainjection_in_novaplacementapis.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/crd/patches/cainjection_in_novaplacementapis.yaml b/config/crd/patches/cainjection_in_novaplacementapis.yaml new file mode 100644 index 000000000..7ee66b841 --- /dev/null +++ b/config/crd/patches/cainjection_in_novaplacementapis.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: novaplacementapis.nova.openstack.org diff --git a/config/crd/patches/webhook_in_novaplacementapis.yaml b/config/crd/patches/webhook_in_novaplacementapis.yaml new file mode 100644 index 000000000..8acb32872 --- /dev/null +++ b/config/crd/patches/webhook_in_novaplacementapis.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: novaplacementapis.nova.openstack.org +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/default/manager_default_images.yaml b/config/default/manager_default_images.yaml index 1c98fc6c2..c56e29978 100644 --- a/config/default/manager_default_images.yaml +++ b/config/default/manager_default_images.yaml @@ -21,3 +21,5 @@ spec: value: quay.io/podified-antelope-centos9/openstack-nova-scheduler:current-podified - name: RELATED_IMAGE_NOVA_COMPUTE_IMAGE_URL_DEFAULT value: quay.io/podified-antelope-centos9/openstack-nova-compute:current-podified + - name: RELATED_IMAGE_NOVA_PLACEMENT_IMAGE_URL_DEFAULT + value: quay.io/podified-antelope-centos9/openstack-placement-api:current-podified diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml index fef8a1c9e..78ce9e9c9 100644 --- a/config/default/manager_webhook_patch.yaml +++ b/config/default/manager_webhook_patch.yaml @@ -9,7 +9,7 @@ spec: containers: - name: manager ports: - - containerPort: 9444 + - containerPort: 9444 # (TODO - ratailor) containerPort in placement-opertor is 9443 else everything is same name: webhook-server protocol: TCP volumeMounts: diff --git a/config/manager/controller_manager_config.yaml b/config/manager/controller_manager_config.yaml index a92f8355f..5e095d83b 100644 --- a/config/manager/controller_manager_config.yaml +++ b/config/manager/controller_manager_config.yaml @@ -5,7 +5,7 @@ health: metrics: bindAddress: 127.0.0.1:8080 webhook: - port: 9444 + port: 9444 (TODO- ratailor) port in placement-operator is 9443 leaderElection: leaderElect: true resourceName: f33036c1.openstack.org diff --git a/config/manifests/bases/nova-operator.clusterserviceversion.yaml b/config/manifests/bases/nova-operator.clusterserviceversion.yaml index 49d5abebb..b763f4aac 100644 --- a/config/manifests/bases/nova-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/nova-operator.clusterserviceversion.yaml @@ -136,6 +136,15 @@ spec: displayName: TLS path: tls version: v1beta1 + - description: NovaPlacementAPI is the Schema for the placementapis API + displayName: Placement API + kind: NovaPlacementAPI + name: novaplacementapis.nova.openstack.org + specDescriptors: + - description: TLS - Parameters related to the TLS + displayName: TLS + path: tls + version: v1beta1 description: Nova Operator displayName: Nova Operator install: diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml index 76a567cb9..de42eb966 100644 --- a/config/rbac/leader_election_role_binding.yaml +++ b/config/rbac/leader_election_role_binding.yaml @@ -9,4 +9,4 @@ roleRef: subjects: - kind: ServiceAccount name: controller-manager - namespace: openstack + namespace: openstack # (TODO - ratailor) it value is system in placement-operator diff --git a/config/rbac/novaplacementapi_editor_role.yaml b/config/rbac/novaplacementapi_editor_role.yaml new file mode 100644 index 000000000..56894ea67 --- /dev/null +++ b/config/rbac/novaplacementapi_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit novaplacementapis. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: novaplacementapi-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: nova-operator + app.kubernetes.io/part-of: nova-operator + app.kubernetes.io/managed-by: kustomize + name: novaplacementapi-editor-role +rules: +- apiGroups: + - nova.openstack.org + resources: + - novaplacementapis + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - nova.openstack.org + resources: + - novaplacementapis/status + verbs: + - get diff --git a/config/rbac/novaplacementapi_viewer_role.yaml b/config/rbac/novaplacementapi_viewer_role.yaml new file mode 100644 index 000000000..d310e7fa3 --- /dev/null +++ b/config/rbac/novaplacementapi_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view novaplacementapis. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: novaplacementapi-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: nova-operator + app.kubernetes.io/part-of: nova-operator + app.kubernetes.io/managed-by: kustomize + name: novaplacementapi-viewer-role +rules: +- apiGroups: + - nova.openstack.org + resources: + - novaplacementapis + verbs: + - get + - list + - watch +- apiGroups: + - nova.openstack.org + resources: + - novaplacementapis/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index b23be6242..a70c19ff9 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -378,6 +378,32 @@ rules: - get - patch - update +- apiGroups: + - nova.openstack.org + resources: + - novaplacementapis + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - nova.openstack.org + resources: + - novaplacementapis/finalizers + verbs: + - update +- apiGroups: + - nova.openstack.org + resources: + - novaplacementapis/status + verbs: + - get + - patch + - update - apiGroups: - nova.openstack.org resources: diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml index d67e1cf4d..f22ac2c85 100644 --- a/config/rbac/service_account.yaml +++ b/config/rbac/service_account.yaml @@ -2,4 +2,4 @@ apiVersion: v1 kind: ServiceAccount metadata: name: controller-manager - namespace: openstack + namespace: openstack # (TODO - ratailor) this is set to system in placement-operator diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index 84a35fd9b..39c096983 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -8,4 +8,5 @@ resources: - nova_v1beta1_novacell1-upcall.yaml - nova_v1beta1_nova.yaml - nova_v1beta1_novacompute-ironic.yaml +- nova_v1beta1_novaplacementapi.yaml #+kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/nova_v1beta1_novaplacementapi-tls.yaml b/config/samples/nova_v1beta1_novaplacementapi-tls.yaml new file mode 100644 index 000000000..5d0c50428 --- /dev/null +++ b/config/samples/nova_v1beta1_novaplacementapi-tls.yaml @@ -0,0 +1,22 @@ +apiVersion: nova.openstack.org/v1beta1 +kind: NovaPlacementAPI +metadata: + name: placement +spec: + serviceUser: placement + customServiceConfig: | + [DEFAULT] + debug = true + databaseInstance: openstack + databaseAccount: placement + preserveJobs: false + replicas: 1 + secret: placement-secret + tls: + api: + internal: + secretName: cert-internal-svc + public: + secretName: cert-public-svc + caBundleSecretName: combined-ca-bundle + diff --git a/config/samples/nova_v1beta1_novaplacementapi.yaml b/config/samples/nova_v1beta1_novaplacementapi.yaml new file mode 100644 index 000000000..eae1d01b8 --- /dev/null +++ b/config/samples/nova_v1beta1_novaplacementapi.yaml @@ -0,0 +1,24 @@ +apiVersion: nova.openstack.org/v1beta1 +kind: NovaPlacementAPI +metadata: + labels: + app.kubernetes.io/name: novaplacementapi + app.kubernetes.io/instance: novaplacementapi-sample + app.kubernetes.io/part-of: nova-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: nova-operator + name: novaplacementapi-sample +spec: + serviceUser: placement + customServiceConfig: | + [DEFAULT] + debug = true + databaseInstance: openstack + databaseAccount: placement + preserveJobs: false + replicas: 1 + secret: placement-secret + #resources: + # requests: + # memory: "500Mi" + # cpu: "1.0" diff --git a/config/samples/placement_db.yaml b/config/samples/placement_db.yaml new file mode 100644 index 000000000..02e2e4dd3 --- /dev/null +++ b/config/samples/placement_db.yaml @@ -0,0 +1,14 @@ +# Please edit the object below. Lines beginning with a '#' will be ignored, +# and an empty file will abort the edit. If an error occurs while saving this file will be +# reopened with the relevant failures. +# +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBDatabase +metadata: + labels: + dbName: openstack + name: placement + namespace: openstack +spec: + name: placement + secret: placement-secret diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index ac5f1acc1..83b6bbbc3 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -144,6 +144,26 @@ webhooks: resources: - novanovncproxies sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-nova-openstack-org-v1beta1-novaplacementapi + failurePolicy: Fail + name: mnovaplacementapi.kb.io + rules: + - apiGroups: + - nova.openstack.org + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - novaplacementapis + sideEffects: None - admissionReviewVersions: - v1 clientConfig: @@ -310,6 +330,26 @@ webhooks: resources: - novanovncproxies sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-nova-openstack-org-v1beta1-novaplacementapi + failurePolicy: Fail + name: vnovaplacementapi.kb.io + rules: + - apiGroups: + - nova.openstack.org + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - novaplacementapis + sideEffects: None - admissionReviewVersions: - v1 clientConfig: diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml index 74ca7ae9f..fe95d0c1c 100644 --- a/config/webhook/service.yaml +++ b/config/webhook/service.yaml @@ -15,6 +15,6 @@ spec: ports: - port: 443 protocol: TCP - targetPort: 9444 + targetPort: 9444 # (TODO - ratailor) this is set to 9443 in placement-operator selector: openstack.org/operator-name: nova diff --git a/controllers/novaplacementapi_controller.go b/controllers/novaplacementapi_controller.go new file mode 100644 index 000000000..e1ce3dc5f --- /dev/null +++ b/controllers/novaplacementapi_controller.go @@ -0,0 +1,157 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + novav1beta1 "github.com/openstack-k8s-operators/nova-operator/api/v1beta1" +) + +// NovaPlacementAPIReconciler reconciles a NovaPlacementAPI object +type NovaPlacementAPIReconciler struct { + ReconcilerBase +} + +// GetLog returns a logger object with a prefix of "controller.name" and additional controller context fields +func (r *NovaPlacementAPIReconciler) GetLogger(ctx context.Context) logr.Logger { + return log.FromContext(ctx).WithName("Controllers").WithName("NovaPlacementAPI") +} + +// +kubebuilder:rbac:groups=nova.openstack.org,resources=novaplacementapis,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=nova.openstack.org,resources=novaplacementapis/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=nova.openstack.org,resources=novaplacementapis/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list; +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbdatabases,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbdatabases/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbaccounts,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbaccounts/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneapis,verbs=get;list;watch; +// +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneservices,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneendpoints,verbs=get;list;watch;create;update;patch;delete; +// +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch + +// service account, role, rolebinding +// +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;update;patch +// service account permissions that are needed to grant permission to the above +// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=anyuid,resources=securitycontextconstraints,verbs=use +// +kubebuilder:rbac:groups="",resources=pods,verbs=create;delete;get;list;patch;update;watch +// +kubebuilder:rbac:groups=topology.openstack.org,resources=topologies,verbs=get;list;watch;update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the NovaPlacementAPI object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.14.1/pkg/reconcile +func (r *NovaPlacementAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + // TODO(user): your logic here + //Fetch the NovaPlacementAPI instance + instance := &novav1.NovaPlacementAPI{} + err := r.Client.Get(ctx, req.NamespacedName, instance) + if err != nil { + if k8s_errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. + // For additional cleanup logic use finalizers. Return and don't requeue. + Log.Info("Placement instance not found, probably deleted before reconciled. Nothing to do.") + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + Log.Error(err, "Failed to read the Placement instance.") + return ctrl.Result{}, err + } + + h, err := helper.NewHelper( + instance, + r.Client, + r.Kclient, + r.Scheme, + Log + ) + if err != nil { + Log.Error(err, "Failed to create lib-common Helper") + return ctrl.Result{}, err + } + + // Save a copy of the conditions so that we can restore the LastTransitionTime + // when a condition's status doesn't change. + savedConditions := instance.Status.Conditions.DeepCopy() + //initialize status fields + if err = r.initStatus(instance); err != nil { + return ctrl.Result{}, err + } + instance.Status.ObservedGeneration = instance.Generation + + // Always patch the instance when exiting this function so we can persist any changes. + defer func() { + // Don't update the status, if reconciler Panics + if r := recover(); r != nil { + Log.Info(fmt.Sprintf("panic during reconcile %v\n", r)) + panic(r) + } + // update the Ready condition based on the sub conditions + if instance.Status.Conditions.AllSubConditionIsTrue() { + instance.Status.Conditions.MarkTrue( + condition.ReadyCondition, condition.ReadyMessage) + } else { + // something is not ready so reset the Ready condition + instance.Status.Conditions.MarkUnknown( + condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage) + // and recalculate it based on the state of the rest of the conditions + instance.Status.Conditions.Set( + instance.Status.Conditions.Mirror(condition.ReadyCondition)) + } + condition.RestoreLastTransitionTimes(&instance.Status.Conditions, savedConditions) + err := h.PatchInstance(ctx, instance) + if err != nil { + _err = err + return + } + }() + + // Handle service delete + if !instance.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, h, instance) + } + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *NovaPlacementAPIReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&novav1beta1.NovaPlacementAPI{}). + Complete(r) +} diff --git a/controllers/suite_test.go b/controllers/suite_test.go new file mode 100644 index 000000000..0d90afb3d --- /dev/null +++ b/controllers/suite_test.go @@ -0,0 +1,80 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + novav1beta1 "github.com/openstack-k8s-operators/nova-operator/api/v1beta1" + //+kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Controller Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + err = novav1beta1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + //+kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) + +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/main.go b/main.go index a9914f8fb..d07ef72ff 100644 --- a/main.go +++ b/main.go @@ -51,8 +51,10 @@ import ( mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + novav1 "github.com/openstack-k8s-operators/nova-operator/api/v1beta1" + novav1beta1 "github.com/openstack-k8s-operators/nova-operator/api/v1beta1" "github.com/openstack-k8s-operators/nova-operator/controllers" //+kubebuilder:scaffold:imports ) @@ -73,6 +75,7 @@ func init() { utilruntime.Must(networkv1.AddToScheme(scheme)) utilruntime.Must(memcachedv1.AddToScheme(scheme)) utilruntime.Must(topologyv1.AddToScheme(scheme)) + utilruntime.Must(novav1beta1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } @@ -190,8 +193,23 @@ func main() { setupLog.Error(err, "unable to create webhook", "webhook", "NovaCompute") os.Exit(1) } + if err = (&novav1.NovaPlacementAPI{}).SetupWebhookWithmanager(mg); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "NovaPlacementAPI") + os.Exit(1) + } } + if err = (&controllers.NovaPlacementAPIReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "NovaPlacementAPI") + os.Exit(1) + } + if err = (&novav1beta1.NovaPlacementAPI{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "NovaPlacementAPI") + os.Exit(1) + } //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", checker); err != nil { diff --git a/pkg/nova/volumes.go b/pkg/nova/volumes.go index 5caf2025a..b2ba853e1 100644 --- a/pkg/nova/volumes.go +++ b/pkg/nova/volumes.go @@ -29,8 +29,14 @@ const ( var ( configMode int32 = 0640 scriptMode int32 = 0740 + // copied from placement-operator + // check if placement-api needs below mode + //or can use above mode (common for nova) + //scriptsVolumeDefaultMode int32 = 0755 + //configMode int32 = 0640 ) + func GetConfigVolumeMount() corev1.VolumeMount { return corev1.VolumeMount{ Name: configVolume, @@ -68,6 +74,14 @@ func GetLogVolumeMount() corev1.VolumeMount { } } +func GetPlacementLogVolumeMount() corev1.VolumeMount { + return corev1.VolumeMount{ + Name: logVolume, + MountPath: "/var/log/placement", + ReadOnly: false, + } +} + func GetLogVolume() corev1.Volume { return corev1.Volume{ Name: logVolume, @@ -85,6 +99,14 @@ func GetScriptVolumeMount() corev1.VolumeMount { } } +func GetPlacementScriptVolumeMount() corev1.VolumeMount { + return corev1.VolumeMount{ + Name: scriptVolume + MountPath: "/usr/local/bin/container-scripts", + ReadOnly: true, + } +} + func GetScriptVolume(secretName string) corev1.Volume { return corev1.Volume{ Name: scriptVolume, diff --git a/pkg/novaplacement/const.go b/pkg/novaplacement/const.go new file mode 100644 index 000000000..155a995a8 --- /dev/null +++ b/pkg/novaplacement/const.go @@ -0,0 +1,35 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nova + +const ( + // ServiceName - + ServiceName = "placement" + // DatabaseName - + DatabaseName = "placement" + + //config secret name + ConfigSecretName = "placement-config-data" + + // PlacementPublicPort - + PlacementPublicPort int32 = 8778 + // PlacementInternalPort - + PlacementInternalPort int32 = 8778 + + // PlacementUserID is the linux user ID used by Kolla for the placement + // user in the service containers + PlacementUserID int64 = 42482 +) diff --git a/pkg/novaplacement/dbsync.go b/pkg/novaplacement/dbsync.go new file mode 100644 index 000000000..2953f0de9 --- /dev/null +++ b/pkg/novaplacement/dbsync.go @@ -0,0 +1,101 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nova + +import ( + novav1 "github.com/openstack-k8s-operators/nova-operator/api/v1beta1" + "github.com/openstack-k8s-operators/nova-operator/pkg/nova" + + env "github.com/openstack-k8s-operators/lib-common/modules/common/env" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +// DbSyncJob func +func DbSyncJob( + instance *novav1.NovaPlacementAPI, + labels map[string]string, + annotations map[string]string, +) *batchv1.Job { + args := []string{"-c", nova.KollaServiceCommand} + + envVars := map[string]env.Setter{} + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["KOLLA_BOOTSTRAP"] = env.SetValue("true") + + // create Volume and VolumeMounts + volumes := []corev1.Volume{ + nova.GetConfigVolume(nova.GetServiceConfigSecretName(instance.Name)), + nova.GetScriptVolume(nova.GetScriptSecretName(instance.Name)), + nova.GetLogVolume(), + } + volumeMounts := []corev1.VolumeMount{ + nova.GetConfigVolumeMount(), + nova.GetPlacementScriptVolumeMount(), + nova.GetKollaConfigVolumeMount("placement-dbsync") + nova.GetPlacementLogVolumeMount(), + } + + // add CA cert if defined + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Name + "-db-sync", + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + ServiceAccountName: instance.RbacResourceName(), + Containers: []corev1.Container{ + { + Name: instance.Name + "-db-sync", + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: instance.Spec.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(PlacementUserID), + }, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + }, + }, + Volumes: volumes, + }, + }, + }, + } + + if instance.Spec.NodeSelector != nil { + job.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector + } + + return job +} diff --git a/pkg/novaplacement/deployment.go b/pkg/novaplacement/deployment.go new file mode 100644 index 000000000..c798ef037 --- /dev/null +++ b/pkg/novaplacement/deployment.go @@ -0,0 +1,195 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nova + +import ( + common "github.com/openstack-k8s-operators/lib-common/modules/common" + affinity "github.com/openstack-k8s-operators/lib-common/modules/common/affinity" + env "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + novav1 "github.com/openstack-k8s-operators/nova-operator/api/v1beta1" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" +) + +// Deployment func +func Deployment( + instance *novav1.NovaPlacementAPI, + configHash string, + labels map[string]string, + annotations map[string]string, + topology *topologyv1.Topology, +) (*appsv1.Deployment, error) { + livenessProbe := &corev1.Probe{ + // TODO might need tuning + TimeoutSeconds: 30, + PeriodSeconds: 30, + InitialDelaySeconds: 5, + } + readinessProbe := &corev1.Probe{ + // TODO might need tuning + TimeoutSeconds: 30, + PeriodSeconds: 30, + InitialDelaySeconds: 5, + } + + args := []string{"-c", KollaServiceCommand} + // + // https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ + // + livenessProbe.HTTPGet = &corev1.HTTPGetAction{ + Port: intstr.IntOrString{Type: intstr.Int, IntVal: int32(PlacementPublicPort)}, + } + readinessProbe.HTTPGet = &corev1.HTTPGetAction{ + Port: intstr.IntOrString{Type: intstr.Int, IntVal: int32(PlacementPublicPort)}, + } + + if instance.Spec.TLS.API.Enabled(service.EndpointPublic) { + livenessProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS + readinessProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS + } + + envVars := map[string]env.Setter{} + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["CONFIG_HASH"] = env.SetValue(configHash) + + // create Volume and VolumeMounts + volumes := []corev1.Volume{ + nova.GetConfigVolume(nova.GetServiceConfigSecretName(instace.name)), + nova.GetScriptVolume(nova.GetScriptSecretName(instance.name)), + nova.GetLogVolume(), + } + volumeMounts := []corev1.VolumeMount{ + nova.GetConfigVolumeMount(), + nova.GetPlacementScriptVolumeMount(), + nova.GetKollaConfigVolumeMount("api"), + nova.GetPlacementLogVolumeMount(), + } + + // add CA cert if defined + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + + for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} { + if instance.Spec.TLS.API.Enabled(endpt) { + var tlsEndptCfg tls.GenericService + switch endpt { + case service.EndpointPublic: + tlsEndptCfg = instance.Spec.TLS.API.Public + case service.EndpointInternal: + tlsEndptCfg = instance.Spec.TLS.API.Internal + } + + svc, err := tlsEndptCfg.ToService() + if err != nil { + return nil, err + } + volumes = append(volumes, svc.CreateVolume(endpt.String())) + volumeMounts = append(volumeMounts, svc.CreateVolumeMounts(endpt.String())...) + } + } + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Name, + Namespace: instance.Namespace, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Replicas: instance.Spec.Replicas, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + Labels: labels, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: instance.RbacResourceName(), + Volumes: volumes, + Containers: []corev1.Container{ + { + Name: instance.Name + "-log", + Command: []string{ + "/usr/bin/dumb-init", + }, + Args: []string{ + "--single-child", + "--", + "/usr/bin/tail", + "-n+1", + "-F", + "/var/log/placement/placement-api.log", + }, + Image: instance.Spec.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(PlacementUserID), + }, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + Resources: instance.Spec.Resources, + ReadinessProbe: readinessProbe, + LivenessProbe: livenessProbe, + }, + { + Name: instance.Name + "-api", + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: instance.Spec.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To(PlacementUserID), + }, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + Resources: instance.Spec.Resources, + ReadinessProbe: readinessProbe, + LivenessProbe: livenessProbe, + }, + }, + }, + }, + }, + } + if instance.Spec.NodeSelector != nil { + deployment.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector + } + if topology != nil { + topology.ApplyTo(&deployment.Spec.Template) + } else { + // If possible two pods of the same service should not + // run on the same worker node. If this is not possible + // the get still created on the same worker node. + deployment.Spec.Template.Spec.Affinity = affinity.DistributePods( + common.AppSelector, + []string{ + ServiceName, + }, + corev1.LabelHostname, + ) + } + return deployment, nil +}