From a25975abf534c1b22334ca775a5f4a8721d95423 Mon Sep 17 00:00:00 2001 From: jlarriba Date: Fri, 20 Jun 2025 14:22:07 +0200 Subject: [PATCH 01/12] Cloudkitty deployment --- PROJECT | 31 +- .../telemetry.openstack.org_autoscalings.yaml | 5 + .../telemetry.openstack.org_ceilometers.yaml | 5 + .../telemetry.openstack.org_cloudkitties.yaml | 665 ++++++++++ ...elemetry.openstack.org_cloudkittyapis.yaml | 499 ++++++++ ...lemetry.openstack.org_cloudkittyprocs.yaml | 321 +++++ .../telemetry.openstack.org_telemetries.yaml | 546 ++++++++ api/v1beta1/autoscaling_types.go | 12 +- api/v1beta1/cloudkitty_types.go | 270 ++++ api/v1beta1/cloudkitty_webhook.go | 102 ++ api/v1beta1/cloudkittyapi_types.go | 155 +++ api/v1beta1/cloudkittyproc_types.go | 148 +++ api/v1beta1/conditions.go | 51 + api/v1beta1/telemetry_types.go | 33 +- api/v1beta1/zz_generated.deepcopy.go | 630 ++++++++++ .../telemetry.openstack.org_autoscalings.yaml | 5 + .../telemetry.openstack.org_ceilometers.yaml | 5 + .../telemetry.openstack.org_cloudkitties.yaml | 665 ++++++++++ ...elemetry.openstack.org_cloudkittyapis.yaml | 499 ++++++++ ...lemetry.openstack.org_cloudkittyprocs.yaml | 321 +++++ .../telemetry.openstack.org_telemetries.yaml | 546 ++++++++ config/crd/kustomization.yaml | 9 + .../patches/cainjection_in_cloudkitties.yaml | 7 + .../cainjection_in_cloudkittyapis.yaml | 7 + .../cainjection_in_cloudkittyprocs.yaml | 7 + .../crd/patches/webhook_in_cloudkitties.yaml | 16 + .../patches/webhook_in_cloudkittyapis.yaml | 16 + .../patches/webhook_in_cloudkittyprocs.yaml | 16 + config/rbac/cloudkitty_editor_role.yaml | 31 + config/rbac/cloudkitty_viewer_role.yaml | 27 + config/rbac/cloudkittyapi_editor_role.yaml | 31 + config/rbac/cloudkittyapi_viewer_role.yaml | 27 + config/rbac/cloudkittyproc_editor_role.yaml | 31 + config/rbac/cloudkittyproc_viewer_role.yaml | 27 + config/rbac/role.yaml | 130 ++ config/samples/kustomization.yaml | 3 + .../samples/telemetry_v1beta1_cloudkitty.yaml | 12 + .../telemetry_v1beta1_cloudkittyapi.yaml | 12 + .../telemetry_v1beta1_cloudkittyproc.yaml | 12 + config/webhook/manifests.yaml | 40 + controllers/cloudkitty_controller.go | 1085 ++++++++++++++++ controllers/cloudkittyapi_controller.go | 1119 +++++++++++++++++ controllers/cloudkittyproc_controller.go | 759 +++++++++++ controllers/telemetry_controller.go | 116 ++ main.go | 32 + pkg/cloudkitty/common.go | 161 +++ pkg/cloudkitty/const.go | 54 + pkg/cloudkitty/dbsync.go | 107 ++ pkg/cloudkitty/funcs.go | 49 + pkg/cloudkitty/volumes.go | 57 + pkg/cloudkittyapi/const.go | 24 + pkg/cloudkittyapi/statefulset.go | 188 +++ pkg/cloudkittyapi/volumes.go | 54 + pkg/cloudkittyproc/const.go | 21 + pkg/cloudkittyproc/statefulset.go | 153 +++ pkg/cloudkittyproc/volumes.go | 38 + templates/cloudkitty/bin/healthcheck.py | 94 ++ templates/cloudkitty/bin/run-on-host | 2 + .../cloudkitty/config/10-cloudkitty_wsgi.conf | 40 + .../config/cloudkitty-api-config-httpd.json | 51 + .../config/cloudkitty-api-config.json | 11 + .../config/cloudkitty-api-uwsgi.ini | 17 + .../config/cloudkitty-dbsync-config.json | 11 + .../config/cloudkitty-proc-config.json | 17 + templates/cloudkitty/config/cloudkitty.conf | 78 ++ templates/cloudkitty/config/httpd.conf | 27 + templates/cloudkitty/config/metrics.yaml | 77 ++ templates/cloudkitty/config/ssl.conf | 21 + 68 files changed, 10423 insertions(+), 15 deletions(-) create mode 100644 api/bases/telemetry.openstack.org_cloudkitties.yaml create mode 100644 api/bases/telemetry.openstack.org_cloudkittyapis.yaml create mode 100644 api/bases/telemetry.openstack.org_cloudkittyprocs.yaml create mode 100644 api/v1beta1/cloudkitty_types.go create mode 100644 api/v1beta1/cloudkitty_webhook.go create mode 100644 api/v1beta1/cloudkittyapi_types.go create mode 100644 api/v1beta1/cloudkittyproc_types.go create mode 100644 config/crd/bases/telemetry.openstack.org_cloudkitties.yaml create mode 100644 config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml create mode 100644 config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml create mode 100644 config/crd/patches/cainjection_in_cloudkitties.yaml create mode 100644 config/crd/patches/cainjection_in_cloudkittyapis.yaml create mode 100644 config/crd/patches/cainjection_in_cloudkittyprocs.yaml create mode 100644 config/crd/patches/webhook_in_cloudkitties.yaml create mode 100644 config/crd/patches/webhook_in_cloudkittyapis.yaml create mode 100644 config/crd/patches/webhook_in_cloudkittyprocs.yaml create mode 100644 config/rbac/cloudkitty_editor_role.yaml create mode 100644 config/rbac/cloudkitty_viewer_role.yaml create mode 100644 config/rbac/cloudkittyapi_editor_role.yaml create mode 100644 config/rbac/cloudkittyapi_viewer_role.yaml create mode 100644 config/rbac/cloudkittyproc_editor_role.yaml create mode 100644 config/rbac/cloudkittyproc_viewer_role.yaml create mode 100644 config/samples/telemetry_v1beta1_cloudkitty.yaml create mode 100644 config/samples/telemetry_v1beta1_cloudkittyapi.yaml create mode 100644 config/samples/telemetry_v1beta1_cloudkittyproc.yaml create mode 100644 controllers/cloudkitty_controller.go create mode 100644 controllers/cloudkittyapi_controller.go create mode 100644 controllers/cloudkittyproc_controller.go create mode 100644 pkg/cloudkitty/common.go create mode 100644 pkg/cloudkitty/const.go create mode 100644 pkg/cloudkitty/dbsync.go create mode 100644 pkg/cloudkitty/funcs.go create mode 100644 pkg/cloudkitty/volumes.go create mode 100644 pkg/cloudkittyapi/const.go create mode 100644 pkg/cloudkittyapi/statefulset.go create mode 100644 pkg/cloudkittyapi/volumes.go create mode 100644 pkg/cloudkittyproc/const.go create mode 100644 pkg/cloudkittyproc/statefulset.go create mode 100644 pkg/cloudkittyproc/volumes.go create mode 100755 templates/cloudkitty/bin/healthcheck.py create mode 100755 templates/cloudkitty/bin/run-on-host create mode 100644 templates/cloudkitty/config/10-cloudkitty_wsgi.conf create mode 100644 templates/cloudkitty/config/cloudkitty-api-config-httpd.json create mode 100644 templates/cloudkitty/config/cloudkitty-api-config.json create mode 100644 templates/cloudkitty/config/cloudkitty-api-uwsgi.ini create mode 100644 templates/cloudkitty/config/cloudkitty-dbsync-config.json create mode 100644 templates/cloudkitty/config/cloudkitty-proc-config.json create mode 100644 templates/cloudkitty/config/cloudkitty.conf create mode 100644 templates/cloudkitty/config/httpd.conf create mode 100644 templates/cloudkitty/config/metrics.yaml create mode 100644 templates/cloudkitty/config/ssl.conf diff --git a/PROJECT b/PROJECT index 5230867a..58516761 100644 --- a/PROJECT +++ b/PROJECT @@ -1,7 +1,3 @@ -# Code generated by tool. DO NOT EDIT. -# This file is used to track the info used to scaffold your project -# and allow the plugins properly work. -# More info: https://book.kubebuilder.io/reference/project-config.html domain: openstack.org layout: - go.kubebuilder.io/v3 @@ -69,4 +65,31 @@ resources: defaulting: true validation: true webhookVersion: v1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: openstack.org + group: telemetry + kind: CloudKittyApi + path: github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1 + version: v1beta1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: openstack.org + group: telemetry + kind: CloudKittyProc + path: github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1 + version: v1beta1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: openstack.org + group: telemetry + kind: CloudKitty + path: github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1 + version: v1beta1 version: "3" diff --git a/api/bases/telemetry.openstack.org_autoscalings.yaml b/api/bases/telemetry.openstack.org_autoscalings.yaml index 4cfabab5..a4ef6ccf 100644 --- a/api/bases/telemetry.openstack.org_autoscalings.yaml +++ b/api/bases/telemetry.openstack.org_autoscalings.yaml @@ -294,6 +294,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object preserveJobs: default: false diff --git a/api/bases/telemetry.openstack.org_ceilometers.yaml b/api/bases/telemetry.openstack.org_ceilometers.yaml index 824f5daa..3307bc27 100644 --- a/api/bases/telemetry.openstack.org_ceilometers.yaml +++ b/api/bases/telemetry.openstack.org_ceilometers.yaml @@ -210,6 +210,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object proxyImage: type: string diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml new file mode 100644 index 00000000..bc5b7567 --- /dev/null +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -0,0 +1,665 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: cloudkitties.telemetry.openstack.org +spec: + group: telemetry.openstack.org + names: + kind: CloudKitty + listKind: CloudKittyList + plural: cloudkitties + singular: cloudkitty + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: CloudKitty is the Schema for the cloudkitties API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CloudKittySpec defines the desired state of CloudKitty + properties: + apiTimeout: + default: 60 + description: APITimeout for HAProxy, Apache, and rpc_response_timeout + minimum: 10 + type: integer + cloudKittyAPI: + description: CloudKittyAPI - Spec definition for the API service of + this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the + configurations of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret + for the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object + public: + description: Public GenericService - holds the secret + for the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in + a pre-created bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + cloudKittyProc: + description: CloudKittyProc - Spec definition for the Scheduler service + of this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config for all CloudKitty services using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for cloudkitty + DB, defaults to cloudkitty + type: string + databaseInstance: + description: |- + MariaDB instance name + Right now required by the maridb-operator to get the credentials from the instance to create the DB + Might not be required in future + type: string + memcachedInstance: + default: memcached + description: Memcached instance name. + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting + NodeSelector here acts as a default value and can be overridden by service + specific NodeSelector Settings. + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service password + from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean + rabbitMqClusterName: + default: rabbitmq + description: |- + RabbitMQ instance name + Needed to request a transportURL that is created and used in CloudKitty + type: string + secret: + description: Secret containing OpenStack password information + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - cloudKittyAPI + - cloudKittyProc + - databaseInstance + - memcachedInstance + - rabbitMqClusterName + - secret + type: object + status: + description: CloudKittyStatus defines the observed state of CloudKitty + properties: + apiEndpoints: + additionalProperties: + additionalProperties: + type: string + type: object + description: API endpoints + type: object + cloudKittyAPIReadyCount: + default: 0 + description: ReadyCount of CloudKitty API instance + format: int32 + minimum: 0 + type: integer + cloudKittyProcReadyCounts: + default: 0 + description: ReadyCount of CloudKitty Processor instances + format: int32 + minimum: 0 + type: integer + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + databaseHostname: + description: CloudKitty Database Hostname + type: string + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object + observedGeneration: + description: |- + ObservedGeneration - the most recent generation observed for this service. + If the observed generation is different than the spec generation, then the + controller has not started processing the latest changes, and the status + and its conditions are likely stale. + format: int64 + type: integer + serviceIDs: + additionalProperties: + type: string + description: ServiceIDs + type: object + transportURLSecret: + description: TransportURLSecret - Secret containing RabbitMQ transportURL + type: string + required: + - cloudKittyAPIReadyCount + - cloudKittyProcReadyCounts + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/api/bases/telemetry.openstack.org_cloudkittyapis.yaml b/api/bases/telemetry.openstack.org_cloudkittyapis.yaml new file mode 100644 index 00000000..d33b713c --- /dev/null +++ b/api/bases/telemetry.openstack.org_cloudkittyapis.yaml @@ -0,0 +1,499 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: cloudkittyapis.telemetry.openstack.org +spec: + group: telemetry.openstack.org + names: + kind: CloudKittyAPI + listKind: CloudKittyAPIList + plural: cloudkittyapis + singular: cloudkittyapi + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: CloudKittyAPI is the Schema for the cloudkittyapis API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CloudKittyAPISpec defines the desired state of CloudKittyAPI + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for cloudkitty + DB, defaults to cloudkitty + type: string + databaseHostname: + description: DatabaseHostname - CloudKitty Database Hostname + type: string + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment resource + names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service password + from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + secret: + description: Secret containing OpenStack password information + type: string + serviceAccount: + description: ServiceAccount - service account name used internally + to provide CloudKitty services the default SA name + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret for + the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + public: + description: Public GenericService - holds the secret for + the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + transportURLSecret: + description: Secret containing RabbitMq transport URL + type: string + required: + - containerImage + - databaseHostname + - secret + - serviceAccount + - transportURLSecret + type: object + status: + description: CloudKittyAPIStatus defines the observed state of CloudKittyAPI + properties: + apiEndpoints: + additionalProperties: + additionalProperties: + type: string + type: object + description: API endpoints + type: object + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object + lastAppliedTopology: + description: LastAppliedTopology - the last applied Topology + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + networkAttachments: + additionalProperties: + items: + type: string + type: array + description: NetworkAttachments status of the deployment pods + type: object + observedGeneration: + description: |- + ObservedGeneration - the most recent generation observed for this service. + If the observed generation is different than the spec generation, then the + controller has not started processing the latest changes, and the status + and its conditions are likely stale. + format: int64 + type: integer + readyCount: + default: 0 + description: ReadyCount of CloudKitty API instances + format: int32 + minimum: 0 + type: integer + serviceIDs: + additionalProperties: + type: string + description: ServiceIDs + type: object + required: + - readyCount + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml new file mode 100644 index 00000000..9cd9b6ef --- /dev/null +++ b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -0,0 +1,321 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: cloudkittyprocs.telemetry.openstack.org +spec: + group: telemetry.openstack.org + names: + kind: CloudKittyProc + listKind: CloudKittyProcList + plural: cloudkittyprocs + singular: cloudkittyproc + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: NetworkAttachments + jsonPath: .status.networkAttachments + name: NetworkAttachments + type: string + - description: Status + jsonPath: .status.conditions[0].status + name: Status + type: string + - description: Message + jsonPath: .status.conditions[0].message + name: Message + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: CloudKittyProc is the Schema for the cloudkittprocs API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CloudKittyProcSpec defines the desired state of CloudKitty + Processor + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for cloudkitty + DB, defaults to cloudkitty + type: string + databaseHostname: + description: DatabaseHostname - CloudKitty Database Hostname + type: string + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment resource + names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service password + from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + secret: + description: Secret containing OpenStack password information + type: string + serviceAccount: + description: ServiceAccount - service account name used internally + to provide CloudKitty services the default SA name + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + transportURLSecret: + description: Secret containing RabbitMq transport URL + type: string + required: + - containerImage + - databaseHostname + - secret + - serviceAccount + - transportURLSecret + type: object + status: + description: CloudKittyProcStatus defines the observed state of CloudKitty + Processor + properties: + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object + lastAppliedTopology: + description: LastAppliedTopology - the last applied Topology + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + networkAttachments: + additionalProperties: + items: + type: string + type: array + description: NetworkAttachments status of the deployment pods + type: object + observedGeneration: + description: |- + ObservedGeneration - the most recent generation observed for this service. + If the observed generation is different than the spec generation, then the + controller has not started processing the latest changes, and the status + and its conditions are likely stale. + format: int64 + type: integer + readyCount: + default: 0 + description: ReadyCount of CloudKitty Processor instances + format: int32 + minimum: 0 + type: integer + required: + - readyCount + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index 6c18bdfc..0d8bf4d3 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -297,6 +297,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object preserveJobs: default: false @@ -525,6 +530,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object proxyImage: type: string @@ -584,6 +594,542 @@ spec: - secret - sgCoreImage type: object + cloudkitty: + description: CloudKitty - Parameters related to the cloudkitty service + properties: + apiTimeout: + default: 60 + description: APITimeout for HAProxy, Apache, and rpc_response_timeout + minimum: 10 + type: integer + cloudKittyAPI: + description: CloudKittyAPI - Spec definition for the API service + of this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL + (will be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + override: + description: Override, provides the ability to override the + generated manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains + the configurations of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret + for the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key + for the service + type: string + type: object + public: + description: Public GenericService - holds the secret + for the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key + for the service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs + in a pre-created bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + cloudKittyProc: + description: CloudKittyProc - Spec definition for the Scheduler + service of this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL + (will be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config for all CloudKitty services using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for + cloudkitty DB, defaults to cloudkitty + type: string + databaseInstance: + description: |- + MariaDB instance name + Right now required by the maridb-operator to get the credentials from the instance to create the DB + Might not be required in future + type: string + enabled: + default: true + description: Enabled - Whether OpenStack CloudKitty service should + be deployed and managed + type: boolean + memcachedInstance: + default: memcached + description: Memcached instance name. + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting + NodeSelector here acts as a default value and can be overridden by service + specific NodeSelector Settings. + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service + password from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean + rabbitMqClusterName: + default: rabbitmq + description: |- + RabbitMQ instance name + Needed to request a transportURL that is created and used in CloudKitty + type: string + secret: + description: Secret containing OpenStack password information + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - cloudKittyAPI + - cloudKittyProc + - databaseInstance + - memcachedInstance + - rabbitMqClusterName + - secret + type: object logging: description: Logging - Parameters related to the logging properties: diff --git a/api/v1beta1/autoscaling_types.go b/api/v1beta1/autoscaling_types.go index d6f3b99b..f541155e 100644 --- a/api/v1beta1/autoscaling_types.go +++ b/api/v1beta1/autoscaling_types.go @@ -17,13 +17,12 @@ limitations under the License. package v1beta1 import ( + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" - "github.com/openstack-k8s-operators/lib-common/modules/common/service" - "github.com/openstack-k8s-operators/lib-common/modules/common/util" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -139,13 +138,6 @@ type AodhCore struct { TopologyRef *topologyv1.TopoRef `json:"topologyRef,omitempty"` } -// APIOverrideSpec to override the generated manifest of several child resources. -type APIOverrideSpec struct { - // Override configuration for the Service created to serve traffic to the cluster. - // The key must be the endpoint type (public, internal) - Service map[service.Endpoint]service.RoutedOverrideSpec `json:"service,omitempty"` -} - // AutoscalingSpec defines the desired state of Autoscaling type AutoscalingSpec struct { AutoscalingSpecBase `json:",inline"` diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go new file mode 100644 index 00000000..08181d6c --- /dev/null +++ b/api/v1beta1/cloudkitty_types.go @@ -0,0 +1,270 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // CloudKittyUserID - Kolla's cloudkitty UID comes from the 'cloudkitty-user' in + // https://github.com/openstack/kolla/blob/master/kolla/common/users.py + CloudKittyUserID = 42408 + // CloudKittyGroupID - Kolla's cloudkitty GID + CloudKittyGroupID = 42408 + + // CloudKittyAPIContainerImage - default fall-back image for CloudKitty API + CloudKittyAPIContainerImage = "quay.io/podified-master-centos9/cloudkitty-api:current-podified" + // CloudKittyProcContainerImage - default fall-back image for CloudKitty Processor + CloudKittyProcContainerImage = "quay.io/podified-master-centos9/cloudkitty-processor:current-podified" + // CloudKittyDbSyncHash hash + CKDbSyncHash = "ckdbsync" + // CloudKittyReplicas - The number of replicas per each service deployed + CloudKittyReplicas = 1 +) + +type CloudKittySpecBase struct { + CloudKittyTemplate `json:",inline"` + + // +kubebuilder:validation:Required + // MariaDB instance name + // Right now required by the maridb-operator to get the credentials from the instance to create the DB + // Might not be required in future + DatabaseInstance string `json:"databaseInstance"` + + // +kubebuilder:validation:Required + // +kubebuilder:default=rabbitmq + // RabbitMQ instance name + // Needed to request a transportURL that is created and used in CloudKitty + RabbitMqClusterName string `json:"rabbitMqClusterName"` + + // +kubebuilder:validation:Required + // +kubebuilder:default=memcached + // Memcached instance name. + MemcachedInstance string `json:"memcachedInstance"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // PreserveJobs - do not delete jobs after they finished e.g. to check logs + PreserveJobs bool `json:"preserveJobs"` + + // +kubebuilder:validation:Optional + // CustomServiceConfig - customize the service config for all CloudKitty services using this parameter to change service defaults, + // or overwrite rendered information using raw OpenStack config format. The content gets added to + // to /etc//.conf.d directory as a custom config file. + CustomServiceConfig string `json:"customServiceConfig,omitempty"` + + // +kubebuilder:validation:Optional + // NodeSelector to target subset of worker nodes running this service. Setting + // NodeSelector here acts as a default value and can be overridden by service + // specific NodeSelector Settings. + NodeSelector *map[string]string `json:"nodeSelector,omitempty"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=60 + // +kubebuilder:validation:Minimum=10 + // APITimeout for HAProxy, Apache, and rpc_response_timeout + APITimeout int `json:"apiTimeout"` + + // +kubebuilder:validation:Optional + // TopologyRef to apply the Topology defined by the associated CR referenced + // by name + TopologyRef *topologyv1.TopoRef `json:"topologyRef,omitempty"` +} + +// CloudKittySpecCore the same as CloudKittySpec without ContainerImage references +type CloudKittySpecCore struct { + CloudKittySpecBase `json:",inline"` + + // +kubebuilder:validation:Required + // CloudKittyAPI - Spec definition for the API service of this CloudKitty deployment + CloudKittyAPI CloudKittyAPITemplateCore `json:"cloudKittyAPI"` + + // +kubebuilder:validation:Required + // CloudKittyProc - Spec definition for the Scheduler service of this CloudKitty deployment + CloudKittyProc CloudKittyProcTemplateCore `json:"cloudKittyProc"` +} + +// CloudKittySpec defines the desired state of CloudKitty +type CloudKittySpec struct { + CloudKittySpecBase `json:",inline"` + + // +kubebuilder:validation:Required + // CloudKittyAPI - Spec definition for the API service of this CloudKitty deployment + CloudKittyAPI CloudKittyAPITemplate `json:"cloudKittyAPI"` + + // +kubebuilder:validation:Required + // CloudKittyProc - Spec definition for the Scheduler service of this CloudKitty deployment + CloudKittyProc CloudKittyProcTemplate `json:"cloudKittyProc"` +} + +// CloudKittyTemplate defines common input parameters used by all CloudKitty services +type CloudKittyTemplate struct { + // +kubebuilder:validation:Optional + // +kubebuilder:default=cloudkitty + // ServiceUser - optional username used for this service to register in cloudkitty + ServiceUser string `json:"serviceUser"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=cloudkitty + // DatabaseAccount - optional MariaDBAccount used for cloudkitty DB, defaults to cloudkitty + DatabaseAccount string `json:"databaseAccount"` + + // +kubebuilder:validation:Required + // Secret containing OpenStack password information + Secret string `json:"secret"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default={cloudKittyService: CloudKittyPassword} + // PasswordsSelectors - Selectors to identify the ServiceUser password from the Secret + PasswordSelectors PasswordsSelector `json:"passwordSelector"` +} + +// CloudKittyServiceTemplate defines the input parameters that can be defined for a given +// CloudKitty service +type CloudKittyServiceTemplate struct { + + // +kubebuilder:validation:Optional + // NodeSelector to target subset of worker nodes running this service. Setting here overrides + // any global NodeSelector settings within the CloudKitty CR. + NodeSelector *map[string]string `json:"nodeSelector,omitempty"` + + // +kubebuilder:validation:Optional + // CustomServiceConfig - customize the service config using this parameter to change service defaults, + // or overwrite rendered information using raw OpenStack config format. The content gets added to + // to /etc//.conf.d directory as a custom config file. + CustomServiceConfig string `json:"customServiceConfig,omitempty"` + + // +kubebuilder:validation:Optional + // CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + // that contain sensitive service config data. The content of each Secret gets added to the + // /etc//.conf.d directory as a custom config file. + CustomServiceConfigSecrets []string `json:"customServiceConfigSecrets,omitempty"` + + // +kubebuilder:validation:Optional + // Resources - Compute Resources required by this service (Limits/Requests). + // https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + + // +kubebuilder:validation:Optional + // NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network + NetworkAttachments []string `json:"networkAttachments,omitempty"` + + // +kubebuilder:validation:Optional + // TopologyRef to apply the Topology defined by the associated CR referenced + // by name + TopologyRef *topologyv1.TopoRef `json:"topologyRef,omitempty"` +} + +// CloudKittyStatus defines the observed state of CloudKitty +type CloudKittyStatus struct { + // Map of hashes to track e.g. job status + Hash map[string]string `json:"hash,omitempty"` + + // Conditions + Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` + + // CloudKitty Database Hostname + DatabaseHostname string `json:"databaseHostname,omitempty"` + + // TransportURLSecret - Secret containing RabbitMQ transportURL + TransportURLSecret string `json:"transportURLSecret,omitempty"` + + // API endpoints + APIEndpoints map[string]map[string]string `json:"apiEndpoints,omitempty"` + + // ServiceIDs + ServiceIDs map[string]string `json:"serviceIDs,omitempty"` + + // ReadyCount of CloudKitty API instance + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:default=0 + CloudKittyAPIReadyCount int32 `json:"cloudKittyAPIReadyCount"` + + // ReadyCount of CloudKitty Processor instances + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:default=0 + CloudKittyProcReadyCount int32 `json:"cloudKittyProcReadyCounts"` + + // ObservedGeneration - the most recent generation observed for this service. + // If the observed generation is different than the spec generation, then the + // controller has not started processing the latest changes, and the status + // and its conditions are likely stale. + ObservedGeneration int64 `json:"observedGeneration,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// CloudKitty is the Schema for the cloudkitties API +type CloudKitty struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CloudKittySpec `json:"spec,omitempty"` + Status CloudKittyStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// CloudKittyList contains a list of CloudKitty +type CloudKittyList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CloudKitty `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CloudKitty{}, &CloudKittyList{}) +} + +// SetupDefaultsCloudKitty - initializes any CRD field defaults based on environment variables (the defaulting mechanism itself is implemented via webhooks) +func SetupDefaultsCloudKitty() { + // Acquire environmental defaults and initialize Telemetry defaults with them + cloudKittyDefaults := CloudKittyDefaults{ + APIContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CLOUDKITTY_API_IMAGE_URL_DEFAULT", CloudKittyAPIContainerImage), + ProcContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CLOUDKITTY_PROCESSOR_IMAGE_URL_DEFAULT", CloudKittyProcContainerImage), + } + + SetupCloudKittyDefaults(cloudKittyDefaults) +} + +// IsReady - returns true if all subresources Ready condition is true +func (instance CloudKitty) IsReady() bool { + return instance.Generation == instance.Status.ObservedGeneration && + instance.Status.Conditions.IsTrue(CloudKittyAPIReadyCondition) && + instance.Status.Conditions.IsTrue(CloudKittyProcReadyCondition) +} + +// RbacConditionsSet - set the conditions for the rbac object +func (instance CloudKitty) RbacConditionsSet(c *condition.Condition) { + instance.Status.Conditions.Set(c) +} + +// RbacNamespace - return the namespace +func (instance CloudKitty) RbacNamespace() string { + return instance.Namespace +} + +// RbacResourceName - return the name to be used for rbac objects (serviceaccount, role, rolebinding) +func (instance CloudKitty) RbacResourceName() string { + return "cloudkitty-" + instance.Name +} diff --git a/api/v1beta1/cloudkitty_webhook.go b/api/v1beta1/cloudkitty_webhook.go new file mode 100644 index 00000000..5ebd644a --- /dev/null +++ b/api/v1beta1/cloudkitty_webhook.go @@ -0,0 +1,102 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// CloudKittyDefaults - +type CloudKittyDefaults struct { + APIContainerImageURL string + ProcContainerImageURL string +} + +var cloudKittyDefaults CloudKittyDefaults + +// log is for logging in this package. +var cloudKittyLog = logf.Log.WithName("cloudkitty-resource") + +// SetupCloudKittyDefaults - initialize CloudKitty spec defaults for use with either internal or external webhooks +func SetupCloudKittyDefaults(defaults CloudKittyDefaults) { + cloudKittyDefaults = defaults + cloudKittyLog.Info("CloudKitty defaults initialized", "defaults", defaults) +} + +// SetupWebhookWithManager - setups webhook with the adequate manager +func (r *CloudKitty) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +// TODO(user): EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! + +//+kubebuilder:webhook:path=/mutate-telemetry-openstack-org-v1beta1-cloudkitty,mutating=true,failurePolicy=fail,sideEffects=None,groups=telemetry.openstack.org,resources=cloudkitties,verbs=create;update,versions=v1beta1,name=mcloudkitty.kb.io,admissionReviewVersions=v1 + +var _ webhook.Defaulter = &CloudKitty{} + +// Default implements webhook.Defaulter so a webhook will be registered for the type +func (r *CloudKitty) Default() { + cloudKittyLog.Info("default", "name", r.Name) + + r.Spec.Default() +} + +// Default - set defaults for this CloudKitty spec +func (spec *CloudKittySpec) Default() { + if spec.CloudKittyAPI.ContainerImage == "" { + spec.CloudKittyAPI.ContainerImage = cloudKittyDefaults.APIContainerImageURL + } + if spec.CloudKittyProc.ContainerImage == "" { + spec.CloudKittyProc.ContainerImage = cloudKittyDefaults.ProcContainerImageURL + } + +} + +// TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation. +//+kubebuilder:webhook:path=/validate-telemetry-openstack-org-v1beta1-cloudkitty,mutating=false,failurePolicy=fail,sideEffects=None,groups=telemetry.openstack.org,resources=cloudkitties,verbs=create;update,versions=v1beta1,name=vcloudkitty.kb.io,admissionReviewVersions=v1 + +var _ webhook.Validator = &CloudKitty{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type +func (r *CloudKitty) ValidateCreate() (admission.Warnings, error) { + cloudKittyLog.Info("validate create", "name", r.Name) + + // TODO(user): fill in your validation logic upon object creation. + return nil, nil +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type +func (r *CloudKitty) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { + cloudKittyLog.Info("validate update", "name", r.Name) + + // TODO(user): fill in your validation logic upon object update. + return nil, nil +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type +func (r *CloudKitty) ValidateDelete() (admission.Warnings, error) { + cloudKittyLog.Info("validate delete", "name", r.Name) + + // TODO(user): fill in your validation logic upon object deletion. + return nil, nil +} diff --git a/api/v1beta1/cloudkittyapi_types.go b/api/v1beta1/cloudkittyapi_types.go new file mode 100644 index 00000000..42660c7e --- /dev/null +++ b/api/v1beta1/cloudkittyapi_types.go @@ -0,0 +1,155 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// CloudKittyAPITemplateCore defines the input parameters for the CloudKitty API service +type CloudKittyAPITemplateCore struct { + // Common input parameters for the CloudKitty API service + CloudKittyServiceTemplate `json:",inline"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=1 + // +kubebuilder:validation:Minimum=0 + // Replicas - CloudKitty API Replicas + Replicas *int32 `json:"replicas"` + + // +kubebuilder:validation:Optional + // Override, provides the ability to override the generated manifest of several child resources. + Override APIOverrideSpec `json:"override,omitempty"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.API `json:"tls,omitempty"` +} + +// CloudKittyAPITemplate defines the input parameters for the CloudKitty API service +type CloudKittyAPITemplate struct { + // +kubebuilder:validation:Required + // ContainerImage - CloudKitty Container Image URL (will be set to environmental default if empty) + ContainerImage string `json:"containerImage"` + + CloudKittyAPITemplateCore `json:",inline"` +} + +// CloudKittyAPISpec defines the desired state of CloudKittyAPI +type CloudKittyAPISpec struct { + // Common input parameters for all CloudKitty services + CloudKittyTemplate `json:",inline"` + + // Input parameters for the CloudKitty API service + CloudKittyAPITemplate `json:",inline"` + + // +kubebuilder:validation:Required + // DatabaseHostname - CloudKitty Database Hostname + DatabaseHostname string `json:"databaseHostname"` + + // +kubebuilder:validation:Required + // Secret containing RabbitMq transport URL + TransportURLSecret string `json:"transportURLSecret"` + + // +kubebuilder:validation:Required + // ServiceAccount - service account name used internally to provide CloudKitty services the default SA name + ServiceAccount string `json:"serviceAccount"` +} + +// CloudKittyAPIStatus defines the observed state of CloudKittyAPI +type CloudKittyAPIStatus struct { + // Map of hashes to track e.g. job status + Hash map[string]string `json:"hash,omitempty"` + + // API endpoints + APIEndpoints map[string]map[string]string `json:"apiEndpoints,omitempty"` + + // Conditions + Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` + + // ReadyCount of CloudKitty API instances + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:default=0 + ReadyCount int32 `json:"readyCount"` + + // ServiceIDs + ServiceIDs map[string]string `json:"serviceIDs,omitempty"` + + // NetworkAttachments status of the deployment pods + NetworkAttachments map[string][]string `json:"networkAttachments,omitempty"` + + // ObservedGeneration - the most recent generation observed for this service. + // If the observed generation is different than the spec generation, then the + // controller has not started processing the latest changes, and the status + // and its conditions are likely stale. + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // LastAppliedTopology - the last applied Topology + LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// CloudKittyAPI is the Schema for the cloudkittyapis API +type CloudKittyAPI struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CloudKittyAPISpec `json:"spec,omitempty"` + Status CloudKittyAPIStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// CloudKittyAPIList contains a list of CloudKittyAPI +type CloudKittyAPIList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CloudKittyAPI `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CloudKittyAPI{}, &CloudKittyAPIList{}) +} + +// IsReady - returns true if service is ready to serve requests +func (instance CloudKittyAPI) IsReady() bool { + return instance.Generation == instance.Status.ObservedGeneration && + instance.Status.ReadyCount == *instance.Spec.Replicas && + (instance.Status.Conditions.IsTrue(condition.DeploymentReadyCondition) || + (instance.Status.Conditions.IsFalse(condition.DeploymentReadyCondition) && *instance.Spec.Replicas == 0)) +} + +// GetSpecTopologyRef - Returns the LastAppliedTopology Set in the Status +func (instance *CloudKittyAPI) GetSpecTopologyRef() *topologyv1.TopoRef { + return instance.Spec.TopologyRef +} + +// GetLastAppliedTopology - Returns the LastAppliedTopology Set in the Status +func (instance *CloudKittyAPI) GetLastAppliedTopology() *topologyv1.TopoRef { + return instance.Status.LastAppliedTopology +} + +// SetLastAppliedTopology - Sets the LastAppliedTopology value in the Status +func (instance *CloudKittyAPI) SetLastAppliedTopology(topologyRef *topologyv1.TopoRef) { + instance.Status.LastAppliedTopology = topologyRef +} diff --git a/api/v1beta1/cloudkittyproc_types.go b/api/v1beta1/cloudkittyproc_types.go new file mode 100644 index 00000000..d2459afd --- /dev/null +++ b/api/v1beta1/cloudkittyproc_types.go @@ -0,0 +1,148 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// CloudKittyProcTemplateCore defines the input parameters for the CloudKitty Processor service +type CloudKittyProcTemplateCore struct { + // Common input parameters for the CloudKitty Processor service + CloudKittyServiceTemplate `json:",inline"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=1 + // +kubebuilder:validation:Minimum=0 + // Replicas - CloudKitty API Replicas + Replicas *int32 `json:"replicas"` +} + +// CloudKittyProcTemplate defines the input parameters for the CloudKitty Processor service +type CloudKittyProcTemplate struct { + // +kubebuilder:validation:Required + // ContainerImage - CloudKitty Container Image URL (will be set to environmental default if empty) + ContainerImage string `json:"containerImage"` + + CloudKittyProcTemplateCore `json:",inline"` +} + +// CloudKittyProcSpec defines the desired state of CloudKitty Processor +type CloudKittyProcSpec struct { + // Common input parameters for all CloudKitty services + CloudKittyTemplate `json:",inline"` + + // Input parameters for the CloudKitty Processor service + CloudKittyProcTemplate `json:",inline"` + + // +kubebuilder:validation:Required + // DatabaseHostname - CloudKitty Database Hostname + DatabaseHostname string `json:"databaseHostname"` + + // +kubebuilder:validation:Required + // Secret containing RabbitMq transport URL + TransportURLSecret string `json:"transportURLSecret"` + + // +kubebuilder:validation:Required + // ServiceAccount - service account name used internally to provide CloudKitty services the default SA name + ServiceAccount string `json:"serviceAccount"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.Ca `json:"tls,omitempty"` +} + +// CloudKittyProcStatus defines the observed state of CloudKitty Processor +type CloudKittyProcStatus struct { + // Map of hashes to track e.g. job status + Hash map[string]string `json:"hash,omitempty"` + + // Conditions + Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` + + // ReadyCount of CloudKitty Processor instances + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:default=0 + ReadyCount int32 `json:"readyCount"` + + // NetworkAttachments status of the deployment pods + NetworkAttachments map[string][]string `json:"networkAttachments,omitempty"` + + // ObservedGeneration - the most recent generation observed for this service. + // If the observed generation is different than the spec generation, then the + // controller has not started processing the latest changes, and the status + // and its conditions are likely stale. + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // LastAppliedTopology - the last applied Topology + LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:printcolumn:name="NetworkAttachments",type="string",JSONPath=".status.networkAttachments",description="NetworkAttachments" +//+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[0].status",description="Status" +//+kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.conditions[0].message",description="Message" + +// CloudKittyProc is the Schema for the cloudkittprocs API +type CloudKittyProc struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CloudKittyProcSpec `json:"spec,omitempty"` + Status CloudKittyProcStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// CloudKittyProcList contains a list of CloudKittyProc +type CloudKittyProcList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CloudKittyProc `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CloudKittyProc{}, &CloudKittyProcList{}) +} + +// IsReady - returns true if service is ready to serve requests +func (instance CloudKittyProc) IsReady() bool { + return instance.Generation == instance.Status.ObservedGeneration && + instance.Status.ReadyCount == *instance.Spec.Replicas && + (instance.Status.Conditions.IsTrue(condition.DeploymentReadyCondition) || + (instance.Status.Conditions.IsFalse(condition.DeploymentReadyCondition) && *instance.Spec.Replicas == 0)) +} + +// GetSpecTopologyRef - Returns the LastAppliedTopology Set in the Status +func (instance *CloudKittyProc) GetSpecTopologyRef() *topologyv1.TopoRef { + return instance.Spec.TopologyRef +} + +// GetLastAppliedTopology - Returns the LastAppliedTopology Set in the Status +func (instance *CloudKittyProc) GetLastAppliedTopology() *topologyv1.TopoRef { + return instance.Status.LastAppliedTopology +} + +// SetLastAppliedTopology - Sets the LastAppliedTopology value in the Status +func (instance *CloudKittyProc) SetLastAppliedTopology(topologyRef *topologyv1.TopoRef) { + instance.Status.LastAppliedTopology = topologyRef +} diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index f9db6290..f8e0b4b7 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -45,6 +45,15 @@ const ( // LoggingReadyCondition Status=True condition which indicates if the Logging is configured and operational LoggingReadyCondition condition.Type = "LoggingReady" + // CloudKittyReadyCondition Status=True condition which indicates if the CloudKitty is configured and operational + CloudKittyReadyCondition condition.Type = "CloudKittyReady" + + // CloudKittyAPIReadyCondition Status=True condition which indicates if the CloudKitty API is configured and operational + CloudKittyAPIReadyCondition condition.Type = "CloudKittyAPIReady" + + // CloudKittyProcReadyCondition Status=True condition which indicates if the CloudKitty Processor is configured and operational + CloudKittyProcReadyCondition condition.Type = "CloudKittyProcReady" + // LoggingCLONamespaceReadyCondition Status=True condition which indicates if the cluster-logging-operator namespace is created LoggingCLONamespaceReadyCondition condition.Type = "LoggingCLONamespaceReady" @@ -202,6 +211,48 @@ const ( // LoggingCLONamespaceFailedMessage LoggingCLONamespaceFailedMessage = "CLO Namespace %s does not exist" + // + // CloudKittyReady condition messages + // + // CloudKittyReadyInitMessage + CloudKittyReadyInitMessage = "CloudKitty not started" + + // CloudKittyReadyMessage + CloudKittyReadyMessage = "CloudKitty completed" + + // CloudKittyReadyErrorMessage + CloudKittyReadyErrorMessage = "CloudKitty error occured %s" + + // + // CloudKittyAPIReady condition messages + // + // CloudKittyAPIReadyInitMessage + CloudKittyAPIReadyInitMessage = "CloudKittyAPI not started" + + // CloudKittyAPIReadyMessage + CloudKittyAPIReadyMessage = "CloudKittyAPI completed" + + // CloudKittyAPIReadyErrorMessage + CloudKittyAPIReadyErrorMessage = "CloudKittyAPI error occured %s" + + // CloudKittyAPIReadyRunningMessage + CloudKittyAPIReadyRunningMessage = "CloudKittyAPI in progress" + + // + // CloudKittyProcReady condition messages + // + // CloudKittyProcReadyInitMessage + CloudKittyProcReadyInitMessage = "CloudKittyProc not started" + + // CloudKittyProcReadyMessage + CloudKittyProcReadyMessage = "CloudKittyProc completed" + + // CloudKittyProcReadyErrorMessage + CloudKittyProcReadyErrorMessage = "CloudKittyProc error occured %s" + + // CloudKittyProcReadyRunningMessage + CloudKittyProcReadyRunningMessage = "CloudKittyProc in progress" + DashboardsNotEnabledMessage = "Dashboarding was not enabled, so no actions required" DashboardPrometheusRuleReadyInitMessage = "Dashboard PrometheusRule not started" diff --git a/api/v1beta1/telemetry_types.go b/api/v1beta1/telemetry_types.go index 7fb6a930..07621b5e 100644 --- a/api/v1beta1/telemetry_types.go +++ b/api/v1beta1/telemetry_types.go @@ -17,13 +17,21 @@ limitations under the License. package v1beta1 import ( + "github.com/openstack-k8s-operators/lib-common/modules/common/service" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/util" - topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" ) +// APIOverrideSpec to override the generated manifest of several child resources. +type APIOverrideSpec struct { + // Override configuration for the Service created to serve traffic to the cluster. + // The key must be the endpoint type (public, internal) + Service map[service.Endpoint]service.RoutedOverrideSpec `json:"service,omitempty"` +} + // PasswordsSelector to identify the Service password from the Secret type PasswordsSelector struct { // CeilometerService - Selector to get the ceilometer service password from the Secret @@ -35,6 +43,11 @@ type PasswordsSelector struct { // +kubebuilder:validation:Optional // +kubebuilder:default:=AodhPassword AodhService string `json:"aodhService"` + + // CloudKittyService - Selector to get the CloudKitty service password from the Secret + // +kubebuilder:validation:Optional + // +kubebuilder:default:=CloudKittyPassword + CloudKittyService string `json:"cloudKittyService"` } // TelemetrySpec defines the desired state of Telemetry @@ -73,6 +86,10 @@ type TelemetrySpecBase struct { // Logging - Parameters related to the logging Logging LoggingSection `json:"logging,omitempty"` + // +kubebuilder:validation:Optional + // CloudKitty - Parameters related to the cloudkitty service + CloudKitty CloudKittySection `json:"cloudkitty,omitempty"` + // +kubebuilder:validation:Optional // NodeSelector to target subset of worker nodes running this service NodeSelector *map[string]string `json:"nodeSelector,omitempty"` @@ -167,6 +184,20 @@ type LoggingSection struct { LoggingSpec `json:",inline"` } +// CloudKittySpec defines the desired state of the cloudkitty service +type CloudKittySection struct { + // +kubebuilder:validation:Optional + // +kubebuilder:default=true + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:booleanSwitch"} + // Enabled - Whether OpenStack CloudKitty service should be deployed and managed + Enabled *bool `json:"enabled"` + + // +kubebuilder:validation:Optional + //+operator-sdk:csv:customresourcedefinitions:type=spec + // Template - Overrides to use when creating the OpenStack CloudKitty service + CloudKittySpec `json:",inline"` +} + // TelemetryStatus defines the observed state of Telemetry type TelemetryStatus struct { // Map of hashes to track e.g. job status diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index da8d5649..1c5006d6 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -578,6 +578,635 @@ func (in *CeilometerStatus) DeepCopy() *CeilometerStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKitty) DeepCopyInto(out *CloudKitty) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKitty. +func (in *CloudKitty) DeepCopy() *CloudKitty { + if in == nil { + return nil + } + out := new(CloudKitty) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CloudKitty) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyAPI) DeepCopyInto(out *CloudKittyAPI) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyAPI. +func (in *CloudKittyAPI) DeepCopy() *CloudKittyAPI { + if in == nil { + return nil + } + out := new(CloudKittyAPI) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CloudKittyAPI) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyAPIList) DeepCopyInto(out *CloudKittyAPIList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CloudKittyAPI, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyAPIList. +func (in *CloudKittyAPIList) DeepCopy() *CloudKittyAPIList { + if in == nil { + return nil + } + out := new(CloudKittyAPIList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CloudKittyAPIList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyAPISpec) DeepCopyInto(out *CloudKittyAPISpec) { + *out = *in + out.CloudKittyTemplate = in.CloudKittyTemplate + in.CloudKittyAPITemplate.DeepCopyInto(&out.CloudKittyAPITemplate) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyAPISpec. +func (in *CloudKittyAPISpec) DeepCopy() *CloudKittyAPISpec { + if in == nil { + return nil + } + out := new(CloudKittyAPISpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyAPIStatus) DeepCopyInto(out *CloudKittyAPIStatus) { + *out = *in + if in.Hash != nil { + in, out := &in.Hash, &out.Hash + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.APIEndpoints != nil { + in, out := &in.APIEndpoints, &out.APIEndpoints + *out = make(map[string]map[string]string, len(*in)) + for key, val := range *in { + var outVal map[string]string + if val == nil { + (*out)[key] = nil + } else { + inVal := (*in)[key] + in, out := &inVal, &outVal + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + (*out)[key] = outVal + } + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(condition.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ServiceIDs != nil { + in, out := &in.ServiceIDs, &out.ServiceIDs + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.NetworkAttachments != nil { + in, out := &in.NetworkAttachments, &out.NetworkAttachments + *out = make(map[string][]string, len(*in)) + for key, val := range *in { + var outVal []string + if val == nil { + (*out)[key] = nil + } else { + inVal := (*in)[key] + in, out := &inVal, &outVal + *out = make([]string, len(*in)) + copy(*out, *in) + } + (*out)[key] = outVal + } + } + if in.LastAppliedTopology != nil { + in, out := &in.LastAppliedTopology, &out.LastAppliedTopology + *out = new(topologyv1beta1.TopoRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyAPIStatus. +func (in *CloudKittyAPIStatus) DeepCopy() *CloudKittyAPIStatus { + if in == nil { + return nil + } + out := new(CloudKittyAPIStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyAPITemplate) DeepCopyInto(out *CloudKittyAPITemplate) { + *out = *in + in.CloudKittyAPITemplateCore.DeepCopyInto(&out.CloudKittyAPITemplateCore) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyAPITemplate. +func (in *CloudKittyAPITemplate) DeepCopy() *CloudKittyAPITemplate { + if in == nil { + return nil + } + out := new(CloudKittyAPITemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyAPITemplateCore) DeepCopyInto(out *CloudKittyAPITemplateCore) { + *out = *in + in.CloudKittyServiceTemplate.DeepCopyInto(&out.CloudKittyServiceTemplate) + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + in.Override.DeepCopyInto(&out.Override) + in.TLS.DeepCopyInto(&out.TLS) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyAPITemplateCore. +func (in *CloudKittyAPITemplateCore) DeepCopy() *CloudKittyAPITemplateCore { + if in == nil { + return nil + } + out := new(CloudKittyAPITemplateCore) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyDefaults) DeepCopyInto(out *CloudKittyDefaults) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyDefaults. +func (in *CloudKittyDefaults) DeepCopy() *CloudKittyDefaults { + if in == nil { + return nil + } + out := new(CloudKittyDefaults) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyList) DeepCopyInto(out *CloudKittyList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CloudKitty, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyList. +func (in *CloudKittyList) DeepCopy() *CloudKittyList { + if in == nil { + return nil + } + out := new(CloudKittyList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CloudKittyList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyProc) DeepCopyInto(out *CloudKittyProc) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProc. +func (in *CloudKittyProc) DeepCopy() *CloudKittyProc { + if in == nil { + return nil + } + out := new(CloudKittyProc) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CloudKittyProc) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyProcList) DeepCopyInto(out *CloudKittyProcList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CloudKittyProc, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProcList. +func (in *CloudKittyProcList) DeepCopy() *CloudKittyProcList { + if in == nil { + return nil + } + out := new(CloudKittyProcList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CloudKittyProcList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyProcSpec) DeepCopyInto(out *CloudKittyProcSpec) { + *out = *in + out.CloudKittyTemplate = in.CloudKittyTemplate + in.CloudKittyProcTemplate.DeepCopyInto(&out.CloudKittyProcTemplate) + out.TLS = in.TLS +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProcSpec. +func (in *CloudKittyProcSpec) DeepCopy() *CloudKittyProcSpec { + if in == nil { + return nil + } + out := new(CloudKittyProcSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyProcStatus) DeepCopyInto(out *CloudKittyProcStatus) { + *out = *in + if in.Hash != nil { + in, out := &in.Hash, &out.Hash + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(condition.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.NetworkAttachments != nil { + in, out := &in.NetworkAttachments, &out.NetworkAttachments + *out = make(map[string][]string, len(*in)) + for key, val := range *in { + var outVal []string + if val == nil { + (*out)[key] = nil + } else { + inVal := (*in)[key] + in, out := &inVal, &outVal + *out = make([]string, len(*in)) + copy(*out, *in) + } + (*out)[key] = outVal + } + } + if in.LastAppliedTopology != nil { + in, out := &in.LastAppliedTopology, &out.LastAppliedTopology + *out = new(topologyv1beta1.TopoRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProcStatus. +func (in *CloudKittyProcStatus) DeepCopy() *CloudKittyProcStatus { + if in == nil { + return nil + } + out := new(CloudKittyProcStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyProcTemplate) DeepCopyInto(out *CloudKittyProcTemplate) { + *out = *in + in.CloudKittyProcTemplateCore.DeepCopyInto(&out.CloudKittyProcTemplateCore) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProcTemplate. +func (in *CloudKittyProcTemplate) DeepCopy() *CloudKittyProcTemplate { + if in == nil { + return nil + } + out := new(CloudKittyProcTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyProcTemplateCore) DeepCopyInto(out *CloudKittyProcTemplateCore) { + *out = *in + in.CloudKittyServiceTemplate.DeepCopyInto(&out.CloudKittyServiceTemplate) + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProcTemplateCore. +func (in *CloudKittyProcTemplateCore) DeepCopy() *CloudKittyProcTemplateCore { + if in == nil { + return nil + } + out := new(CloudKittyProcTemplateCore) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittySection) DeepCopyInto(out *CloudKittySection) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + in.CloudKittySpec.DeepCopyInto(&out.CloudKittySpec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittySection. +func (in *CloudKittySection) DeepCopy() *CloudKittySection { + if in == nil { + return nil + } + out := new(CloudKittySection) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyServiceTemplate) DeepCopyInto(out *CloudKittyServiceTemplate) { + *out = *in + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = new(map[string]string) + if **in != nil { + in, out := *in, *out + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + } + if in.CustomServiceConfigSecrets != nil { + in, out := &in.CustomServiceConfigSecrets, &out.CustomServiceConfigSecrets + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.Resources.DeepCopyInto(&out.Resources) + if in.NetworkAttachments != nil { + in, out := &in.NetworkAttachments, &out.NetworkAttachments + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.TopologyRef != nil { + in, out := &in.TopologyRef, &out.TopologyRef + *out = new(topologyv1beta1.TopoRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyServiceTemplate. +func (in *CloudKittyServiceTemplate) DeepCopy() *CloudKittyServiceTemplate { + if in == nil { + return nil + } + out := new(CloudKittyServiceTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittySpec) DeepCopyInto(out *CloudKittySpec) { + *out = *in + in.CloudKittySpecBase.DeepCopyInto(&out.CloudKittySpecBase) + in.CloudKittyAPI.DeepCopyInto(&out.CloudKittyAPI) + in.CloudKittyProc.DeepCopyInto(&out.CloudKittyProc) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittySpec. +func (in *CloudKittySpec) DeepCopy() *CloudKittySpec { + if in == nil { + return nil + } + out := new(CloudKittySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittySpecBase) DeepCopyInto(out *CloudKittySpecBase) { + *out = *in + out.CloudKittyTemplate = in.CloudKittyTemplate + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = new(map[string]string) + if **in != nil { + in, out := *in, *out + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + } + if in.TopologyRef != nil { + in, out := &in.TopologyRef, &out.TopologyRef + *out = new(topologyv1beta1.TopoRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittySpecBase. +func (in *CloudKittySpecBase) DeepCopy() *CloudKittySpecBase { + if in == nil { + return nil + } + out := new(CloudKittySpecBase) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittySpecCore) DeepCopyInto(out *CloudKittySpecCore) { + *out = *in + in.CloudKittySpecBase.DeepCopyInto(&out.CloudKittySpecBase) + in.CloudKittyAPI.DeepCopyInto(&out.CloudKittyAPI) + in.CloudKittyProc.DeepCopyInto(&out.CloudKittyProc) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittySpecCore. +func (in *CloudKittySpecCore) DeepCopy() *CloudKittySpecCore { + if in == nil { + return nil + } + out := new(CloudKittySpecCore) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyStatus) DeepCopyInto(out *CloudKittyStatus) { + *out = *in + if in.Hash != nil { + in, out := &in.Hash, &out.Hash + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(condition.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.APIEndpoints != nil { + in, out := &in.APIEndpoints, &out.APIEndpoints + *out = make(map[string]map[string]string, len(*in)) + for key, val := range *in { + var outVal map[string]string + if val == nil { + (*out)[key] = nil + } else { + inVal := (*in)[key] + in, out := &inVal, &outVal + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + (*out)[key] = outVal + } + } + if in.ServiceIDs != nil { + in, out := &in.ServiceIDs, &out.ServiceIDs + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyStatus. +func (in *CloudKittyStatus) DeepCopy() *CloudKittyStatus { + if in == nil { + return nil + } + out := new(CloudKittyStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyTemplate) DeepCopyInto(out *CloudKittyTemplate) { + *out = *in + out.PasswordSelectors = in.PasswordSelectors +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyTemplate. +func (in *CloudKittyTemplate) DeepCopy() *CloudKittyTemplate { + if in == nil { + return nil + } + out := new(CloudKittyTemplate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KSMStatus) DeepCopyInto(out *KSMStatus) { *out = *in @@ -1047,6 +1676,7 @@ func (in *TelemetrySpecBase) DeepCopyInto(out *TelemetrySpecBase) { *out = *in in.MetricStorage.DeepCopyInto(&out.MetricStorage) in.Logging.DeepCopyInto(&out.Logging) + in.CloudKitty.DeepCopyInto(&out.CloudKitty) if in.NodeSelector != nil { in, out := &in.NodeSelector, &out.NodeSelector *out = new(map[string]string) diff --git a/config/crd/bases/telemetry.openstack.org_autoscalings.yaml b/config/crd/bases/telemetry.openstack.org_autoscalings.yaml index 4cfabab5..a4ef6ccf 100644 --- a/config/crd/bases/telemetry.openstack.org_autoscalings.yaml +++ b/config/crd/bases/telemetry.openstack.org_autoscalings.yaml @@ -294,6 +294,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object preserveJobs: default: false diff --git a/config/crd/bases/telemetry.openstack.org_ceilometers.yaml b/config/crd/bases/telemetry.openstack.org_ceilometers.yaml index 824f5daa..3307bc27 100644 --- a/config/crd/bases/telemetry.openstack.org_ceilometers.yaml +++ b/config/crd/bases/telemetry.openstack.org_ceilometers.yaml @@ -210,6 +210,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object proxyImage: type: string diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml new file mode 100644 index 00000000..bc5b7567 --- /dev/null +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -0,0 +1,665 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: cloudkitties.telemetry.openstack.org +spec: + group: telemetry.openstack.org + names: + kind: CloudKitty + listKind: CloudKittyList + plural: cloudkitties + singular: cloudkitty + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: CloudKitty is the Schema for the cloudkitties API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CloudKittySpec defines the desired state of CloudKitty + properties: + apiTimeout: + default: 60 + description: APITimeout for HAProxy, Apache, and rpc_response_timeout + minimum: 10 + type: integer + cloudKittyAPI: + description: CloudKittyAPI - Spec definition for the API service of + this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the + configurations of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret + for the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object + public: + description: Public GenericService - holds the secret + for the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in + a pre-created bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + cloudKittyProc: + description: CloudKittyProc - Spec definition for the Scheduler service + of this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config for all CloudKitty services using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for cloudkitty + DB, defaults to cloudkitty + type: string + databaseInstance: + description: |- + MariaDB instance name + Right now required by the maridb-operator to get the credentials from the instance to create the DB + Might not be required in future + type: string + memcachedInstance: + default: memcached + description: Memcached instance name. + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting + NodeSelector here acts as a default value and can be overridden by service + specific NodeSelector Settings. + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service password + from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean + rabbitMqClusterName: + default: rabbitmq + description: |- + RabbitMQ instance name + Needed to request a transportURL that is created and used in CloudKitty + type: string + secret: + description: Secret containing OpenStack password information + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - cloudKittyAPI + - cloudKittyProc + - databaseInstance + - memcachedInstance + - rabbitMqClusterName + - secret + type: object + status: + description: CloudKittyStatus defines the observed state of CloudKitty + properties: + apiEndpoints: + additionalProperties: + additionalProperties: + type: string + type: object + description: API endpoints + type: object + cloudKittyAPIReadyCount: + default: 0 + description: ReadyCount of CloudKitty API instance + format: int32 + minimum: 0 + type: integer + cloudKittyProcReadyCounts: + default: 0 + description: ReadyCount of CloudKitty Processor instances + format: int32 + minimum: 0 + type: integer + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + databaseHostname: + description: CloudKitty Database Hostname + type: string + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object + observedGeneration: + description: |- + ObservedGeneration - the most recent generation observed for this service. + If the observed generation is different than the spec generation, then the + controller has not started processing the latest changes, and the status + and its conditions are likely stale. + format: int64 + type: integer + serviceIDs: + additionalProperties: + type: string + description: ServiceIDs + type: object + transportURLSecret: + description: TransportURLSecret - Secret containing RabbitMQ transportURL + type: string + required: + - cloudKittyAPIReadyCount + - cloudKittyProcReadyCounts + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml new file mode 100644 index 00000000..d33b713c --- /dev/null +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml @@ -0,0 +1,499 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: cloudkittyapis.telemetry.openstack.org +spec: + group: telemetry.openstack.org + names: + kind: CloudKittyAPI + listKind: CloudKittyAPIList + plural: cloudkittyapis + singular: cloudkittyapi + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: CloudKittyAPI is the Schema for the cloudkittyapis API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CloudKittyAPISpec defines the desired state of CloudKittyAPI + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for cloudkitty + DB, defaults to cloudkitty + type: string + databaseHostname: + description: DatabaseHostname - CloudKitty Database Hostname + type: string + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment resource + names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service password + from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + secret: + description: Secret containing OpenStack password information + type: string + serviceAccount: + description: ServiceAccount - service account name used internally + to provide CloudKitty services the default SA name + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret for + the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + public: + description: Public GenericService - holds the secret for + the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + transportURLSecret: + description: Secret containing RabbitMq transport URL + type: string + required: + - containerImage + - databaseHostname + - secret + - serviceAccount + - transportURLSecret + type: object + status: + description: CloudKittyAPIStatus defines the observed state of CloudKittyAPI + properties: + apiEndpoints: + additionalProperties: + additionalProperties: + type: string + type: object + description: API endpoints + type: object + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object + lastAppliedTopology: + description: LastAppliedTopology - the last applied Topology + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + networkAttachments: + additionalProperties: + items: + type: string + type: array + description: NetworkAttachments status of the deployment pods + type: object + observedGeneration: + description: |- + ObservedGeneration - the most recent generation observed for this service. + If the observed generation is different than the spec generation, then the + controller has not started processing the latest changes, and the status + and its conditions are likely stale. + format: int64 + type: integer + readyCount: + default: 0 + description: ReadyCount of CloudKitty API instances + format: int32 + minimum: 0 + type: integer + serviceIDs: + additionalProperties: + type: string + description: ServiceIDs + type: object + required: + - readyCount + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml new file mode 100644 index 00000000..9cd9b6ef --- /dev/null +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -0,0 +1,321 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: cloudkittyprocs.telemetry.openstack.org +spec: + group: telemetry.openstack.org + names: + kind: CloudKittyProc + listKind: CloudKittyProcList + plural: cloudkittyprocs + singular: cloudkittyproc + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: NetworkAttachments + jsonPath: .status.networkAttachments + name: NetworkAttachments + type: string + - description: Status + jsonPath: .status.conditions[0].status + name: Status + type: string + - description: Message + jsonPath: .status.conditions[0].message + name: Message + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: CloudKittyProc is the Schema for the cloudkittprocs API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CloudKittyProcSpec defines the desired state of CloudKitty + Processor + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for cloudkitty + DB, defaults to cloudkitty + type: string + databaseHostname: + description: DatabaseHostname - CloudKitty Database Hostname + type: string + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment resource + names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service password + from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + secret: + description: Secret containing OpenStack password information + type: string + serviceAccount: + description: ServiceAccount - service account name used internally + to provide CloudKitty services the default SA name + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + transportURLSecret: + description: Secret containing RabbitMq transport URL + type: string + required: + - containerImage + - databaseHostname + - secret + - serviceAccount + - transportURLSecret + type: object + status: + description: CloudKittyProcStatus defines the observed state of CloudKitty + Processor + properties: + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object + lastAppliedTopology: + description: LastAppliedTopology - the last applied Topology + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + networkAttachments: + additionalProperties: + items: + type: string + type: array + description: NetworkAttachments status of the deployment pods + type: object + observedGeneration: + description: |- + ObservedGeneration - the most recent generation observed for this service. + If the observed generation is different than the spec generation, then the + controller has not started processing the latest changes, and the status + and its conditions are likely stale. + format: int64 + type: integer + readyCount: + default: 0 + description: ReadyCount of CloudKitty Processor instances + format: int32 + minimum: 0 + type: integer + required: + - readyCount + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index 6c18bdfc..0d8bf4d3 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -297,6 +297,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object preserveJobs: default: false @@ -525,6 +530,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object proxyImage: type: string @@ -584,6 +594,542 @@ spec: - secret - sgCoreImage type: object + cloudkitty: + description: CloudKitty - Parameters related to the cloudkitty service + properties: + apiTimeout: + default: 60 + description: APITimeout for HAProxy, Apache, and rpc_response_timeout + minimum: 10 + type: integer + cloudKittyAPI: + description: CloudKittyAPI - Spec definition for the API service + of this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL + (will be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + override: + description: Override, provides the ability to override the + generated manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains + the configurations of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret + for the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key + for the service + type: string + type: object + public: + description: Public GenericService - holds the secret + for the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key + for the service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs + in a pre-created bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + cloudKittyProc: + description: CloudKittyProc - Spec definition for the Scheduler + service of this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL + (will be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config for all CloudKitty services using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for + cloudkitty DB, defaults to cloudkitty + type: string + databaseInstance: + description: |- + MariaDB instance name + Right now required by the maridb-operator to get the credentials from the instance to create the DB + Might not be required in future + type: string + enabled: + default: true + description: Enabled - Whether OpenStack CloudKitty service should + be deployed and managed + type: boolean + memcachedInstance: + default: memcached + description: Memcached instance name. + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting + NodeSelector here acts as a default value and can be overridden by service + specific NodeSelector Settings. + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service + password from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean + rabbitMqClusterName: + default: rabbitmq + description: |- + RabbitMQ instance name + Needed to request a transportURL that is created and used in CloudKitty + type: string + secret: + description: Secret containing OpenStack password information + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - cloudKittyAPI + - cloudKittyProc + - databaseInstance + - memcachedInstance + - rabbitMqClusterName + - secret + type: object logging: description: Logging - Parameters related to the logging properties: diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 703cccdd..e6a95c46 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -7,6 +7,9 @@ resources: - bases/telemetry.openstack.org_ceilometers.yaml - bases/telemetry.openstack.org_loggings.yaml - bases/telemetry.openstack.org_metricstorages.yaml +- bases/telemetry.openstack.org_cloudkittyapis.yaml +- bases/telemetry.openstack.org_cloudkittyprocs.yaml +- bases/telemetry.openstack.org_cloudkitties.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -17,6 +20,9 @@ patchesStrategicMerge: #- patches/webhook_in_autoscalings.yaml #- patches/webhook_in_loggings.yaml #- patches/webhook_in_metricstorages.yaml +#- patches/webhook_in_cloudkittyapis.yaml +#- patches/webhook_in_cloudkittyprocs.yaml +#- patches/webhook_in_cloudkitties.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. @@ -26,6 +32,9 @@ patchesStrategicMerge: #- patches/cainjection_in_autoscalings.yaml #- patches/cainjection_in_loggings.yaml #- patches/cainjection_in_metricstorages.yaml +#- patches/cainjection_in_cloudkittyapis.yaml +#- patches/cainjection_in_cloudkittyprocs.yaml +#- patches/cainjection_in_cloudkitties.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/crd/patches/cainjection_in_cloudkitties.yaml b/config/crd/patches/cainjection_in_cloudkitties.yaml new file mode 100644 index 00000000..b0360335 --- /dev/null +++ b/config/crd/patches/cainjection_in_cloudkitties.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: cloudkitties.telemetry.openstack.org diff --git a/config/crd/patches/cainjection_in_cloudkittyapis.yaml b/config/crd/patches/cainjection_in_cloudkittyapis.yaml new file mode 100644 index 00000000..35526ad6 --- /dev/null +++ b/config/crd/patches/cainjection_in_cloudkittyapis.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: cloudkittyapis.telemetry.openstack.org diff --git a/config/crd/patches/cainjection_in_cloudkittyprocs.yaml b/config/crd/patches/cainjection_in_cloudkittyprocs.yaml new file mode 100644 index 00000000..bfc383ec --- /dev/null +++ b/config/crd/patches/cainjection_in_cloudkittyprocs.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: cloudkittyprocs.telemetry.openstack.org diff --git a/config/crd/patches/webhook_in_cloudkitties.yaml b/config/crd/patches/webhook_in_cloudkitties.yaml new file mode 100644 index 00000000..18a2c841 --- /dev/null +++ b/config/crd/patches/webhook_in_cloudkitties.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cloudkitties.telemetry.openstack.org +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/crd/patches/webhook_in_cloudkittyapis.yaml b/config/crd/patches/webhook_in_cloudkittyapis.yaml new file mode 100644 index 00000000..cad85de9 --- /dev/null +++ b/config/crd/patches/webhook_in_cloudkittyapis.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cloudkittyapis.telemetry.openstack.org +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/crd/patches/webhook_in_cloudkittyprocs.yaml b/config/crd/patches/webhook_in_cloudkittyprocs.yaml new file mode 100644 index 00000000..90c0d18e --- /dev/null +++ b/config/crd/patches/webhook_in_cloudkittyprocs.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cloudkittyprocs.telemetry.openstack.org +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/rbac/cloudkitty_editor_role.yaml b/config/rbac/cloudkitty_editor_role.yaml new file mode 100644 index 00000000..940f3121 --- /dev/null +++ b/config/rbac/cloudkitty_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit cloudkitties. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: cloudkitty-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: telemetry-operator + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + name: cloudkitty-editor-role +rules: +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkitties + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkitties/status + verbs: + - get diff --git a/config/rbac/cloudkitty_viewer_role.yaml b/config/rbac/cloudkitty_viewer_role.yaml new file mode 100644 index 00000000..253ecd75 --- /dev/null +++ b/config/rbac/cloudkitty_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view cloudkitties. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: cloudkitty-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: telemetry-operator + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + name: cloudkitty-viewer-role +rules: +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkitties + verbs: + - get + - list + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkitties/status + verbs: + - get diff --git a/config/rbac/cloudkittyapi_editor_role.yaml b/config/rbac/cloudkittyapi_editor_role.yaml new file mode 100644 index 00000000..2a0e0027 --- /dev/null +++ b/config/rbac/cloudkittyapi_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit cloudkittyapis. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: cloudkittyapi-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: telemetry-operator + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + name: cloudkittyapi-editor-role +rules: +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyapis + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyapis/status + verbs: + - get diff --git a/config/rbac/cloudkittyapi_viewer_role.yaml b/config/rbac/cloudkittyapi_viewer_role.yaml new file mode 100644 index 00000000..56bc7181 --- /dev/null +++ b/config/rbac/cloudkittyapi_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view cloudkittyapis. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: cloudkittyapi-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: telemetry-operator + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + name: cloudkittyapi-viewer-role +rules: +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyapis + verbs: + - get + - list + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyapis/status + verbs: + - get diff --git a/config/rbac/cloudkittyproc_editor_role.yaml b/config/rbac/cloudkittyproc_editor_role.yaml new file mode 100644 index 00000000..dcb4eac3 --- /dev/null +++ b/config/rbac/cloudkittyproc_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit cloudkittyprocs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: cloudkittyproc-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: telemetry-operator + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + name: cloudkittyproc-editor-role +rules: +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyprocs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyprocs/status + verbs: + - get diff --git a/config/rbac/cloudkittyproc_viewer_role.yaml b/config/rbac/cloudkittyproc_viewer_role.yaml new file mode 100644 index 00000000..a852a6f4 --- /dev/null +++ b/config/rbac/cloudkittyproc_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view cloudkittyprocs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: cloudkittyproc-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: telemetry-operator + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + name: cloudkittyproc-viewer-role +rules: +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyprocs + verbs: + - get + - list + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyprocs/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6e34456b..6fabb4df 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -4,6 +4,18 @@ kind: ClusterRole metadata: name: manager-role rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: @@ -46,6 +58,33 @@ rules: - patch - update - watch +- apiGroups: + - cloudkitty.openstack.org + resources: + - cloudkittyprocs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - cloudkitty.openstack.org + resources: + - cloudkittyprocs/finalizers + verbs: + - patch + - update +- apiGroups: + - cloudkitty.openstack.org + resources: + - cloudkittyprocs/status + verbs: + - get + - patch + - update - apiGroups: - "" resources: @@ -330,6 +369,15 @@ rules: - securitycontextconstraints verbs: - use +- apiGroups: + - security.openshift.io + resourceNames: + - anyuid + - privileged + resources: + - securitycontextconstraints + verbs: + - use - apiGroups: - telemetry.openstack.org resources: @@ -386,6 +434,88 @@ rules: - get - patch - update +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkitties + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkitties/finalizers + verbs: + - delete + - patch + - update +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkitties/status + verbs: + - get + - patch + - update +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyapis + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyapis/finalizers + verbs: + - patch + - update +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyapis/status + verbs: + - get + - patch + - update +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyprocs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyprocs/finalizers + verbs: + - patch + - update +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyprocs/status + verbs: + - get + - patch + - update - apiGroups: - telemetry.openstack.org resources: diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index 7e00a20f..6e689eee 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -5,4 +5,7 @@ resources: - telemetry_v1beta1_autoscaling.yaml - telemetry_v1beta1_logging.yaml - telemetry_v1beta1_metricstorage.yaml +- telemetry_v1beta1_cloudkittyapi.yaml +- telemetry_v1beta1_cloudkittyproc.yaml +- telemetry_v1beta1_cloudkitty.yaml #+kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/telemetry_v1beta1_cloudkitty.yaml b/config/samples/telemetry_v1beta1_cloudkitty.yaml new file mode 100644 index 00000000..0f649ad3 --- /dev/null +++ b/config/samples/telemetry_v1beta1_cloudkitty.yaml @@ -0,0 +1,12 @@ +apiVersion: telemetry.openstack.org/v1beta1 +kind: CloudKitty +metadata: + labels: + app.kubernetes.io/name: cloudkitty + app.kubernetes.io/instance: cloudkitty-sample + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: telemetry-operator + name: cloudkitty-sample +spec: + # TODO(user): Add fields here diff --git a/config/samples/telemetry_v1beta1_cloudkittyapi.yaml b/config/samples/telemetry_v1beta1_cloudkittyapi.yaml new file mode 100644 index 00000000..47725bf7 --- /dev/null +++ b/config/samples/telemetry_v1beta1_cloudkittyapi.yaml @@ -0,0 +1,12 @@ +apiVersion: telemetry.openstack.org/v1beta1 +kind: CloudKittyApi +metadata: + labels: + app.kubernetes.io/name: cloudkittyapi + app.kubernetes.io/instance: cloudkittyapi-sample + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: telemetry-operator + name: cloudkittyapi-sample +spec: + # TODO(user): Add fields here diff --git a/config/samples/telemetry_v1beta1_cloudkittyproc.yaml b/config/samples/telemetry_v1beta1_cloudkittyproc.yaml new file mode 100644 index 00000000..94e90340 --- /dev/null +++ b/config/samples/telemetry_v1beta1_cloudkittyproc.yaml @@ -0,0 +1,12 @@ +apiVersion: telemetry.openstack.org/v1beta1 +kind: CloudKittyProc +metadata: + labels: + app.kubernetes.io/name: cloudkittyproc + app.kubernetes.io/instance: cloudkittyproc-sample + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: telemetry-operator + name: cloudkittyproc-sample +spec: + # TODO(user): Add fields here diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 288d1e6f..c6b4c02c 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -44,6 +44,26 @@ webhooks: resources: - ceilometers sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-telemetry-openstack-org-v1beta1-cloudkitty + failurePolicy: Fail + name: mcloudkitty.kb.io + rules: + - apiGroups: + - telemetry.openstack.org + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - cloudkitties + sideEffects: None - admissionReviewVersions: - v1 clientConfig: @@ -130,6 +150,26 @@ webhooks: resources: - ceilometers sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-telemetry-openstack-org-v1beta1-cloudkitty + failurePolicy: Fail + name: vcloudkitty.kb.io + rules: + - apiGroups: + - telemetry.openstack.org + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - cloudkitties + sideEffects: None - admissionReviewVersions: - v1 clientConfig: diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go new file mode 100644 index 00000000..4dadd651 --- /dev/null +++ b/controllers/cloudkitty_controller.go @@ -0,0 +1,1085 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/go-logr/logr" + networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" + rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" + keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/job" + "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" + common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" + "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" + mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// GetClient - +func (r *CloudKittyReconciler) GetClient() client.Client { + return r.Client +} + +// GetKClient - +func (r *CloudKittyReconciler) GetKClient() kubernetes.Interface { + return r.Kclient +} + +// GetScheme - +func (r *CloudKittyReconciler) GetScheme() *runtime.Scheme { + return r.Scheme +} + +// CloudKittyReconciler reconciles a CloudKitty object +type CloudKittyReconciler struct { + client.Client + Kclient kubernetes.Interface + Scheme *runtime.Scheme +} + +// GetLogger returns a logger object with a logging prefix of "controller.name" and additional controller context fields +func (r *CloudKittyReconciler) GetLogger(ctx context.Context) logr.Logger { + return log.FromContext(ctx).WithName("Controllers").WithName("CloudKitty") +} + +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkitties,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkitties/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkitties/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyapis,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyapis/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyapis/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyprocs,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyprocs/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyprocs/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbdatabases,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbaccounts,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbaccounts/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=memcached.openstack.org,resources=memcacheds,verbs=get;list;watch; +// +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneapis,verbs=get;list;watch +// +kubebuilder:rbac:groups=rabbitmq.openstack.org,resources=transporturls,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch + +// service account, role, rolebinding +// +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;update;patch +// service account permissions that are needed to grant permission to the above +// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=anyuid;privileged,resources=securitycontextconstraints,verbs=use +// +kubebuilder:rbac:groups="",resources=pods,verbs=create;delete;get;list;patch;update;watch + +// Reconcile - +func (r *CloudKittyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { + Log := r.GetLogger(ctx) + + // Fetch the CloudKitty instance + instance := &telemetryv1.CloudKitty{} + err := r.Client.Get(ctx, req.NamespacedName, instance) + if err != nil { + if k8s_errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. + // For additional cleanup logic use finalizers. Return and don't requeue. + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + Log.Error(err, fmt.Sprintf("could not fetch CloudKitty instance %s", instance.Name)) + return ctrl.Result{}, err + } + + helper, err := helper.NewHelper( + instance, + r.Client, + r.Kclient, + r.Scheme, + Log, + ) + if err != nil { + Log.Error(err, fmt.Sprintf("could not instantiate helper for instance %s", instance.Name)) + return ctrl.Result{}, err + } + + // + // initialize status + // + isNewInstance := instance.Status.Conditions == nil + if isNewInstance { + instance.Status.Conditions = condition.Conditions{} + } + + // Save a copy of the condtions so that we can restore the LastTransitionTime + // when a condition's state doesn't change. + savedConditions := instance.Status.Conditions.DeepCopy() + + // Always patch the instance status when exiting this function so we can persist any changes. + defer func() { + // Don't update the status, if reconciler Panics + if r := recover(); r != nil { + Log.Info(fmt.Sprintf("panic during reconcile %v\n", r)) + panic(r) + } + condition.RestoreLastTransitionTimes(&instance.Status.Conditions, savedConditions) + if instance.Status.Conditions.IsUnknown(condition.ReadyCondition) { + instance.Status.Conditions.Set( + instance.Status.Conditions.Mirror(condition.ReadyCondition)) + } + err := helper.PatchInstance(ctx, instance) + if err != nil { + _err = err + return + } + }() + + // Always initialize conditions used later as Status=Unknown + cl := condition.CreateList( + condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), + condition.UnknownCondition(condition.DBReadyCondition, condition.InitReason, condition.DBReadyInitMessage), + condition.UnknownCondition(condition.DBSyncReadyCondition, condition.InitReason, condition.DBSyncReadyInitMessage), + condition.UnknownCondition(condition.RabbitMqTransportURLReadyCondition, condition.InitReason, condition.RabbitMqTransportURLReadyInitMessage), + condition.UnknownCondition(condition.MemcachedReadyCondition, condition.InitReason, condition.MemcachedReadyInitMessage), + condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), + condition.UnknownCondition(telemetryv1.CloudKittyAPIReadyCondition, condition.InitReason, telemetryv1.CloudKittyAPIReadyInitMessage), + condition.UnknownCondition(telemetryv1.CloudKittyProcReadyCondition, condition.InitReason, telemetryv1.CloudKittyProcReadyInitMessage), + condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), + // service account, role, rolebinding conditions + condition.UnknownCondition(condition.ServiceAccountReadyCondition, condition.InitReason, condition.ServiceAccountReadyInitMessage), + condition.UnknownCondition(condition.RoleReadyCondition, condition.InitReason, condition.RoleReadyInitMessage), + condition.UnknownCondition(condition.RoleBindingReadyCondition, condition.InitReason, condition.RoleBindingReadyInitMessage), + ) + instance.Status.Conditions.Init(&cl) + // Always mark the Generation as observed early on + instance.Status.ObservedGeneration = instance.Generation + + // If we're not deleting this and the service object doesn't have our finalizer, add it. + if (instance.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(instance, helper.GetFinalizer())) || isNewInstance { + // Register overall status immediately to have an early feedback e.g. in the cli + return ctrl.Result{}, nil + } + + if instance.Status.Hash == nil { + instance.Status.Hash = map[string]string{} + } + if instance.Status.APIEndpoints == nil { + instance.Status.APIEndpoints = map[string]map[string]string{} + } + + // Handle service delete + if !instance.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, instance, helper) + } + + // Handle non-deleted clusters + return r.reconcileNormal(ctx, instance, helper) +} + +// fields to index to reconcile when change +const ( + cloudKittyPasswordSecretField = ".spec.secret" + cloudKittyCaBundleSecretNameField = ".spec.tls.caBundleSecretName" + cloudKittyTlsAPIInternalField = ".spec.tls.api.internal.secretName" + cloudKittyTlsAPIPublicField = ".spec.tls.api.public.secretName" + cloudKittyTopologyField = ".spec.topologyRef.Name" +) + +var ( + commonWatchFields = []string{ + cloudKittyPasswordSecretField, + cloudKittyCaBundleSecretNameField, + cloudKittyTopologyField, + } + cloudKittyAPIWatchFields = []string{ + cloudKittyPasswordSecretField, + cloudKittyCaBundleSecretNameField, + cloudKittyTlsAPIInternalField, + cloudKittyTlsAPIPublicField, + cloudKittyTopologyField, + } +) + +// SetupWithManager sets up the controller with the Manager. +func (r *CloudKittyReconciler) SetupWithManager(mgr ctrl.Manager) error { + // transportURLSecretFn - Watch for changes made to the secret associated with the RabbitMQ + // TransportURL created and used by CloudKitty CRs. Watch functions return a list of namespace-scoped + // CRs that then get fed to the reconciler. Hence, in this case, we need to know the name of the + // CloudKitty CR associated with the secret we are examining in the function. We could parse the name + // out of the "%s-cloudkitty-transport" secret label, which would be faster than getting the list of + // the CloudKitty CRs and trying to match on each one. The downside there, however, is that technically + // someone could randomly label a secret "something-cloudkitty-transport" where "something" actually + // matches the name of an existing CloudKitty CR. In that case changes to that secret would trigger + // reconciliation for a CloudKitty CR that does not need it. + // + // TODO: We also need a watch func to monitor for changes to the secret referenced by CloudKitty.Spec.Secret + transportURLSecretFn := func(ctx context.Context, o client.Object) []reconcile.Request { + result := []reconcile.Request{} + + Log := r.GetLogger(ctx) + + // get all CloudKitty CRs + cloudkitties := &telemetryv1.CloudKittyList{} + listOpts := []client.ListOption{ + client.InNamespace(o.GetNamespace()), + } + if err := r.Client.List(ctx, cloudkitties, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve CloudKitty CRs %v") + return nil + } + + for _, ownerRef := range o.GetOwnerReferences() { + if ownerRef.Kind == "TransportURL" { + for _, cr := range cloudkitties.Items { + if ownerRef.Name == fmt.Sprintf("%s-cloudkitty-transport", cr.Name) { + // return namespace and Name of CR + name := client.ObjectKey{ + Namespace: o.GetNamespace(), + Name: cr.Name, + } + Log.Info(fmt.Sprintf("TransportURL Secret %s belongs to TransportURL belonging to CloudKitty CR %s", o.GetName(), cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + } + } + if len(result) > 0 { + return result + } + return nil + } + + memcachedFn := func(ctx context.Context, o client.Object) []reconcile.Request { + Log := r.GetLogger(ctx) + + result := []reconcile.Request{} + + // get all CloudKitty CRs + cloudkitties := &telemetryv1.CloudKittyList{} + listOpts := []client.ListOption{ + client.InNamespace(o.GetNamespace()), + } + if err := r.Client.List(ctx, cloudkitties, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve CloudKitty CRs %w") + return nil + } + + for _, cr := range cloudkitties.Items { + if o.GetName() == cr.Spec.MemcachedInstance { + name := client.ObjectKey{ + Namespace: o.GetNamespace(), + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Memcached %s is used by CloudKitty CR %s", o.GetName(), cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + if len(result) > 0 { + return result + } + return nil + } + + return ctrl.NewControllerManagedBy(mgr). + For(&telemetryv1.CloudKitty{}). + Owns(&mariadbv1.MariaDBDatabase{}). + Owns(&mariadbv1.MariaDBAccount{}). + Owns(&telemetryv1.CloudKittyAPI{}). + Owns(&telemetryv1.CloudKittyProc{}). + Owns(&rabbitmqv1.TransportURL{}). + Owns(&batchv1.Job{}). + Owns(&corev1.Secret{}). + Owns(&corev1.ServiceAccount{}). + Owns(&rbacv1.Role{}). + Owns(&rbacv1.RoleBinding{}). + // Watch for TransportURL Secrets which belong to any TransportURLs created by CloudKitty CRs + Watches(&corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(transportURLSecretFn)). + Watches(&memcachedv1.Memcached{}, + handler.EnqueueRequestsFromMapFunc(memcachedFn)). + Watches(&keystonev1.KeystoneAPI{}, + handler.EnqueueRequestsFromMapFunc(r.findObjectForSrc), + builder.WithPredicates(keystonev1.KeystoneAPIStatusChangedPredicate)). + Complete(r) +} + +func (r *CloudKittyReconciler) findObjectForSrc(ctx context.Context, src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + l := log.FromContext(ctx).WithName("Controllers").WithName("CloudKitty") + + crList := &telemetryv1.CloudKittyList{} + listOps := &client.ListOptions{ + Namespace: src.GetNamespace(), + } + err := r.Client.List(ctx, crList, listOps) + if err != nil { + l.Error(err, fmt.Sprintf("listing %s for namespace: %s", crList.GroupVersionKind().Kind, src.GetNamespace())) + return requests + } + + for _, item := range crList.Items { + l.Info(fmt.Sprintf("input source %s changed, reconcile: %s - %s", src.GetName(), item.GetName(), item.GetNamespace())) + + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + + return requests +} + +func (r *CloudKittyReconciler) reconcileDelete(ctx context.Context, instance *telemetryv1.CloudKitty, helper *helper.Helper) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s' delete", instance.Name)) + + // remove db finalizer first + db, err := mariadbv1.GetDatabaseByNameAndAccount(ctx, helper, cloudkitty.DatabaseName, instance.Spec.DatabaseAccount, instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + + if !k8s_errors.IsNotFound(err) { + if err := db.DeleteFinalizer(ctx, helper); err != nil { + return ctrl.Result{}, err + } + } + + // TODO: We might need to control how the sub-services (API and Proc) are + // deleted (when their parent CloudKitty CR is deleted) once we further develop their functionality + + // Service is deleted so remove the finalizer. + controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) + Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", instance.Name)) + + return ctrl.Result{}, nil +} + +func (r *CloudKittyReconciler) reconcileInit( + ctx context.Context, + instance *telemetryv1.CloudKitty, + helper *helper.Helper, + serviceLabels map[string]string, + serviceAnnotations map[string]string, +) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s' init", instance.Name)) + + // + // run CloudKitty db sync + // + dbSyncHash := instance.Status.Hash[telemetryv1.CKDbSyncHash] + jobDef := cloudkitty.DbSyncJob(instance, serviceLabels) + + dbSyncjob := job.NewJob( + jobDef, + telemetryv1.CKDbSyncHash, + instance.Spec.PreserveJobs, + cloudkitty.ShortDuration, + dbSyncHash, + ) + ctrlResult, err := dbSyncjob.DoJob( + ctx, + helper, + ) + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBSyncReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBSyncReadyRunningMessage)) + return ctrlResult, nil + } + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBSyncReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBSyncReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if dbSyncjob.HasChanged() { + instance.Status.Hash[telemetryv1.CKDbSyncHash] = dbSyncjob.GetHash() + Log.Info(fmt.Sprintf("Service '%s' - Job %s hash added - %s", instance.Name, jobDef.Name, instance.Status.Hash[telemetryv1.CKDbSyncHash])) + } + instance.Status.Conditions.MarkTrue(condition.DBSyncReadyCondition, condition.DBSyncReadyMessage) + + // run CloudKitty db sync - end + + Log.Info(fmt.Sprintf("Reconciled Service '%s' init successfully", instance.Name)) + return ctrl.Result{}, nil +} + +func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *telemetryv1.CloudKitty, helper *helper.Helper) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s'", instance.Name)) + + // Service account, role, binding + rbacRules := []rbacv1.PolicyRule{ + { + APIGroups: []string{"security.openshift.io"}, + ResourceNames: []string{"anyuid"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"create", "get", "list", "watch", "update", "patch", "delete"}, + }, + } + rbacResult, err := common_rbac.ReconcileRbac(ctx, helper, instance, rbacRules) + if err != nil { + return rbacResult, err + } else if (rbacResult != ctrl.Result{}) { + return rbacResult, nil + } + + serviceLabels := map[string]string{ + common.AppSelector: cloudkitty.ServiceName, + } + + configVars := make(map[string]env.Setter) + + // + // create RabbitMQ transportURL CR and get the actual URL from the associated secret that is created + // + + transportURL, op, err := r.transportURLCreateOrUpdate(ctx, instance, serviceLabels) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.RabbitMqTransportURLReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.RabbitMqTransportURLReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if op != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf("TransportURL %s successfully reconciled - operation: %s", transportURL.Name, string(op))) + } + + instance.Status.TransportURLSecret = transportURL.Status.SecretName + + if instance.Status.TransportURLSecret == "" { + Log.Info(fmt.Sprintf("Waiting for TransportURL %s secret to be created", transportURL.Name)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.RabbitMqTransportURLReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.RabbitMqTransportURLReadyRunningMessage)) + return cloudkitty.ResultRequeue, nil + } + + instance.Status.Conditions.MarkTrue(condition.RabbitMqTransportURLReadyCondition, condition.RabbitMqTransportURLReadyMessage) + + // end transportURL + + // + // Check for required memcached used for caching + // + memcached, err := memcachedv1.GetMemcachedByName(ctx, helper, instance.Spec.MemcachedInstance, instance.Namespace) + if err != nil { + Log.Info(fmt.Sprintf("%s... requeueing", condition.MemcachedReadyWaitingMessage)) + if k8s_errors.IsNotFound(err) { + Log.Info(fmt.Sprintf("memcached %s not found", instance.Spec.MemcachedInstance)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.MemcachedReadyWaitingMessage)) + return cloudkitty.ResultRequeue, nil + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.MemcachedReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if !memcached.IsReady() { + Log.Info(fmt.Sprintf("%s... requeueing", condition.MemcachedReadyWaitingMessage)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.MemcachedReadyWaitingMessage)) + return cloudkitty.ResultRequeue, nil + } + // Mark the Memcached Service as Ready if we get to this point with no errors + instance.Status.Conditions.MarkTrue( + condition.MemcachedReadyCondition, condition.MemcachedReadyMessage) + // run check memcached - end + + // + // check for required OpenStack secret holding passwords for service/admin user and add hash to the vars map + // + + result, err := cloudkitty.VerifyServiceSecret( + ctx, + types.NamespacedName{Namespace: instance.Namespace, Name: instance.Spec.Secret}, + []string{ + instance.Spec.PasswordSelectors.CloudKittyService, + }, + helper.GetClient(), + &instance.Status.Conditions, + cloudkitty.NormalDuration, + &configVars, + ) + if err != nil { + return result, err + } else if (result != ctrl.Result{}) { + return result, nil + } + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + // run check OpenStack secret - end + + db, result, err := r.ensureDB(ctx, helper, instance) + if err != nil { + return ctrl.Result{}, err + } else if (result != ctrl.Result{}) { + return result, nil + } + + // + // Create Secrets required as input for the Service and calculate an overall hash of hashes + // + err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels, memcached, db) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + // + // create hash over all the different input resources to identify if any those changed + // and a restart/recreate is required. + // + _, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configVars) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } else if hashChanged { + Log.Info(fmt.Sprintf("%s... requeueing", condition.ServiceConfigReadyInitMessage)) + instance.Status.Conditions.MarkFalse( + condition.ServiceConfigReadyCondition, + condition.InitReason, + condition.SeverityInfo, + condition.ServiceConfigReadyInitMessage) + // Hash changed and instance status should be updated (which will be done by main defer func), + // so we need to return and reconcile again + return ctrl.Result{}, nil + } + + instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) + + // + // TODO check when/if Init, Update, or Upgrade should/could be skipped + // + + // Check networks that the DBSync job will use in reconcileInit. The ones from the API service are always enough, + // it doesn't need the storage specific ones that volume or backup may have. + nadList := []networkv1.NetworkAttachmentDefinition{} + for _, netAtt := range instance.Spec.CloudKittyAPI.NetworkAttachments { + nad, err := nad.GetNADWithName(ctx, helper, netAtt, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + Log.Info(fmt.Sprintf("network-attachment-definition %s not found", netAtt)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.NetworkAttachmentsReadyWaitingMessage, + netAtt)) + return cloudkitty.ResultRequeue, fmt.Errorf(condition.NetworkAttachmentsReadyWaitingMessage, netAtt) + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if nad != nil { + nadList = append(nadList, *nad) + } + } + + instance.Status.Conditions.MarkTrue(condition.NetworkAttachmentsReadyCondition, condition.NetworkAttachmentsReadyMessage) + + serviceAnnotations, err := nad.EnsureNetworksAnnotation(nadList) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed create network annotation from %s: %w", + instance.Spec.CloudKittyAPI.NetworkAttachments, err) + } + + // Handle service init + ctrlResult, err := r.reconcileInit(ctx, instance, helper, serviceLabels, serviceAnnotations) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + // + // normal reconcile tasks + // + + // deploy cloudkitty-api + cloudKittyAPI, op, err := r.apiDeploymentCreateOrUpdate(ctx, instance) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyAPIReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + telemetryv1.CloudKittyAPIReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if op != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf("API CR for %s successfully %s", instance.Name, string(op))) + } + + // Mirror values when the data in the StatefulSet is for the current generation + if cloudKittyAPI.Generation == cloudKittyAPI.Status.ObservedGeneration { + // Mirror CloudKittyAPI status' APIEndpoints and ReadyCount to this parent CR + instance.Status.APIEndpoints = cloudKittyAPI.Status.APIEndpoints + instance.Status.ServiceIDs = cloudKittyAPI.Status.ServiceIDs + instance.Status.CloudKittyAPIReadyCount = cloudKittyAPI.Status.ReadyCount + + // Mirror CloudKittyAPI's condition status + c := cloudKittyAPI.Status.Conditions.Mirror(telemetryv1.CloudKittyAPIReadyCondition) + if c != nil { + instance.Status.Conditions.Set(c) + } + } + + // deploy CloudKitty Processor + cloudKittyProc, op, err := r.procDeploymentCreateOrUpdate(ctx, instance) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyProcReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + telemetryv1.CloudKittyProcReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if op != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf("Scheduler CR for %s successfully %s", instance.Name, string(op))) + } + + // Mirror values when the data in the StatefulSet is for the current generation + if cloudKittyProc.Generation == cloudKittyProc.Status.ObservedGeneration { + // Mirror CloudKitty Processor status' ReadyCount to this parent CR + instance.Status.CloudKittyProcReadyCount = cloudKittyProc.Status.ReadyCount + + // Mirror CloudKittyProc's condition status + c := cloudKittyProc.Status.Conditions.Mirror(telemetryv1.CloudKittyProcReadyCondition) + if c != nil { + instance.Status.Conditions.Set(c) + } + } + + err = mariadbv1.DeleteUnusedMariaDBAccountFinalizers(ctx, helper, cloudkitty.DatabaseName, instance.Spec.DatabaseAccount, instance.Namespace) + if err != nil { + return ctrl.Result{}, err + } + + Log.Info(fmt.Sprintf("Reconciled Service '%s' successfully", instance.Name)) + // update the overall status condition if service is ready + if instance.IsReady() { + instance.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) + } + return ctrl.Result{}, nil +} + +// generateServiceConfigs - create Secret which hold scripts and service configuration +func (r *CloudKittyReconciler) generateServiceConfigs( + ctx context.Context, + h *helper.Helper, + instance *telemetryv1.CloudKitty, + envVars *map[string]env.Setter, + serviceLabels map[string]string, + memcached *memcachedv1.Memcached, + db *mariadbv1.Database, +) error { + // + // create Secret required for cloudkitty input + // - %-scripts holds scripts to e.g. bootstrap the service + // - %-config holds minimal cloudkitty config required to get the service up + // + + labels := labels.GetLabels(instance, labels.GetGroupLabel(cloudkitty.ServiceName), serviceLabels) + + var tlsCfg *tls.Service + if instance.Spec.CloudKittyAPI.TLS.Ca.CaBundleSecretName != "" { + tlsCfg = &tls.Service{} + } + + // customData hold any customization for all cloudkitty services. + customData := map[string]string{ + cloudkitty.CustomConfigFileName: instance.Spec.CustomServiceConfig, + cloudkitty.MyCnfFileName: db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf + } + + keystoneAPI, err := keystonev1.GetKeystoneAPI(ctx, h, instance.Namespace, map[string]string{}) + if err != nil { + return err + } + keystoneInternalURL, err := keystoneAPI.GetEndpoint(endpoint.EndpointInternal) + if err != nil { + return err + } + keystonePublicURL, err := keystoneAPI.GetEndpoint(endpoint.EndpointPublic) + if err != nil { + return err + } + + ospSecret, _, err := secret.GetSecret(ctx, h, instance.Spec.Secret, instance.Namespace) + if err != nil { + return err + } + + transportURLSecret, _, err := secret.GetSecret(ctx, h, instance.Status.TransportURLSecret, instance.Namespace) + if err != nil { + return err + } + + databaseAccount := db.GetAccount() + dbSecret := db.GetSecret() + + templateParameters := make(map[string]interface{}) + templateParameters["ServiceUser"] = instance.Spec.ServiceUser + templateParameters["ServicePassword"] = string(ospSecret.Data[instance.Spec.PasswordSelectors.CloudKittyService]) + templateParameters["KeystoneInternalURL"] = keystoneInternalURL + templateParameters["KeystonePublicURL"] = keystonePublicURL + templateParameters["TransportURL"] = string(transportURLSecret.Data["transport_url"]) + templateParameters["DatabaseConnection"] = fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf", + databaseAccount.Spec.UserName, + string(dbSecret.Data[mariadbv1.DatabasePasswordSelector]), + instance.Status.DatabaseHostname, + cloudkitty.DatabaseName) + templateParameters["MemcachedServersWithInet"] = memcached.GetMemcachedServerListWithInetString() + templateParameters["TimeOut"] = instance.Spec.APITimeout + + // create httpd vhost template parameters + httpdVhostConfig := map[string]interface{}{} + for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} { + endptConfig := map[string]interface{}{} + endptConfig["ServerName"] = fmt.Sprintf("%s-%s.%s.svc", cloudkitty.ServiceName, endpt.String(), instance.Namespace) + endptConfig["TLS"] = false // default TLS to false, and set it bellow to true if enabled + if instance.Spec.CloudKittyAPI.TLS.API.Enabled(endpt) { + endptConfig["TLS"] = true + endptConfig["SSLCertificateFile"] = fmt.Sprintf("/etc/pki/tls/certs/%s.crt", endpt.String()) + endptConfig["SSLCertificateKeyFile"] = fmt.Sprintf("/etc/pki/tls/private/%s.key", endpt.String()) + } + httpdVhostConfig[endpt.String()] = endptConfig + } + templateParameters["VHosts"] = httpdVhostConfig + + configTemplates := []util.Template{ + { + Name: fmt.Sprintf("%s-scripts", instance.Name), + Namespace: instance.Namespace, + Type: util.TemplateTypeScripts, + InstanceType: instance.Kind, + Labels: labels, + }, + { + Name: fmt.Sprintf("%s-config-data", instance.Name), + Namespace: instance.Namespace, + Type: util.TemplateTypeConfig, + InstanceType: instance.Kind, + CustomData: customData, + ConfigOptions: templateParameters, + Labels: labels, + }, + } + + return secret.EnsureSecrets(ctx, h, instance, configTemplates, envVars) +} + +// createHashOfInputHashes - creates a hash of hashes which gets added to the resources which requires a restart +// if any of the input resources change, like configs, passwords, ... +// +// returns the hash, whether the hash changed (as a bool) and any error +func (r *CloudKittyReconciler) createHashOfInputHashes( + ctx context.Context, + instance *telemetryv1.CloudKitty, + envVars map[string]env.Setter, +) (string, bool, error) { + Log := r.GetLogger(ctx) + + var hashMap map[string]string + changed := false + mergedMapVars := env.MergeEnvs([]corev1.EnvVar{}, envVars) + hash, err := util.ObjectHash(mergedMapVars) + if err != nil { + return hash, changed, err + } + if hashMap, changed = util.SetHash(instance.Status.Hash, common.InputHashName, hash); changed { + instance.Status.Hash = hashMap + Log.Info(fmt.Sprintf("Input maps hash %s - %s", common.InputHashName, hash)) + } + return hash, changed, nil +} + +func (r *CloudKittyReconciler) transportURLCreateOrUpdate( + ctx context.Context, + instance *telemetryv1.CloudKitty, + serviceLabels map[string]string, +) (*rabbitmqv1.TransportURL, controllerutil.OperationResult, error) { + transportURL := &rabbitmqv1.TransportURL{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-cloudkitty-transport", instance.Name), + Namespace: instance.Namespace, + Labels: serviceLabels, + }, + } + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, transportURL, func() error { + transportURL.Spec.RabbitmqClusterName = instance.Spec.RabbitMqClusterName + + err := controllerutil.SetControllerReference(instance, transportURL, r.Scheme) + return err + }) + + return transportURL, op, err +} + +func (r *CloudKittyReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, instance *telemetryv1.CloudKitty) (*telemetryv1.CloudKittyAPI, controllerutil.OperationResult, error) { + cloudkittyAPISpec := telemetryv1.CloudKittyAPISpec{ + CloudKittyTemplate: instance.Spec.CloudKittyTemplate, + CloudKittyAPITemplate: instance.Spec.CloudKittyAPI, + DatabaseHostname: instance.Status.DatabaseHostname, + TransportURLSecret: instance.Status.TransportURLSecret, + ServiceAccount: instance.RbacResourceName(), + } + + if cloudkittyAPISpec.NodeSelector == nil { + cloudkittyAPISpec.NodeSelector = instance.Spec.NodeSelector + } + + // If topology is not present in the underlying CloudKittyAPI Spec, + // inherit from the top-level CR + if cloudkittyAPISpec.TopologyRef == nil { + cloudkittyAPISpec.TopologyRef = instance.Spec.TopologyRef + } + + deployment := &telemetryv1.CloudKittyAPI{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-api", instance.Name), + Namespace: instance.Namespace, + }, + } + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { + deployment.Spec = cloudkittyAPISpec + + err := controllerutil.SetControllerReference(instance, deployment, r.Scheme) + if err != nil { + return err + } + + return nil + }) + + return deployment, op, err +} + +func (r *CloudKittyReconciler) procDeploymentCreateOrUpdate(ctx context.Context, instance *telemetryv1.CloudKitty) (*telemetryv1.CloudKittyProc, controllerutil.OperationResult, error) { + cloudKittyProcSpec := telemetryv1.CloudKittyProcSpec{ + CloudKittyTemplate: instance.Spec.CloudKittyTemplate, + CloudKittyProcTemplate: instance.Spec.CloudKittyProc, + DatabaseHostname: instance.Status.DatabaseHostname, + TransportURLSecret: instance.Status.TransportURLSecret, + ServiceAccount: instance.RbacResourceName(), + TLS: instance.Spec.CloudKittyAPI.TLS.Ca, + } + + if cloudKittyProcSpec.NodeSelector == nil { + cloudKittyProcSpec.NodeSelector = instance.Spec.NodeSelector + } + + // If topology is not present in the underlying Scheduler Spec + // inherit from the top-level CR + if cloudKittyProcSpec.TopologyRef == nil { + cloudKittyProcSpec.TopologyRef = instance.Spec.TopologyRef + } + + deployment := &telemetryv1.CloudKittyProc{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-proc", instance.Name), + Namespace: instance.Namespace, + }, + } + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { + deployment.Spec = cloudKittyProcSpec + + err := controllerutil.SetControllerReference(instance, deployment, r.Scheme) + if err != nil { + return err + } + + return nil + }) + + return deployment, op, err +} + +func (r *CloudKittyReconciler) ensureDB( + ctx context.Context, + h *helper.Helper, + instance *telemetryv1.CloudKitty, +) (*mariadbv1.Database, ctrl.Result, error) { + Log := r.GetLogger(ctx) + + // ensure MariaDBAccount exists. This account record may be created by + // openstack-operator or the cloud operator up front without a specific + // MariaDBDatabase configured yet. Otherwise, a MariaDBAccount CR is + // created here with a generated username as well as a secret with + // generated password. The MariaDBAccount is created without being + // yet associated with any MariaDBDatabase. + _, _, err := mariadbv1.EnsureMariaDBAccount( + ctx, h, instance.Spec.DatabaseAccount, + instance.Namespace, false, "cloudkitty", + ) + + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + mariadbv1.MariaDBAccountReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + mariadbv1.MariaDBAccountNotReadyMessage, + err.Error())) + + return nil, ctrl.Result{}, err + } + instance.Status.Conditions.MarkTrue( + mariadbv1.MariaDBAccountReadyCondition, + mariadbv1.MariaDBAccountReadyMessage, + ) + + db := mariadbv1.NewDatabaseForAccount( + instance.Spec.DatabaseInstance, // mariadb/galera service to target + cloudkitty.DatabaseName, // name used in CREATE DATABASE in mariadb + cloudkitty.DatabaseName, // CR name for MariaDBDatabase + instance.Spec.DatabaseAccount, // CR name for MariaDBAccount + instance.Namespace, // namespace + ) + + // create or patch the DB + ctrlResult, err := db.CreateOrPatchAll(ctx, h) + + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBReadyErrorMessage, + err.Error())) + return db, ctrl.Result{}, err + } + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBReadyRunningMessage)) + return db, ctrlResult, nil + } + // wait for the DB to be setup + // (ksambor) should we use WaitForDBCreatedWithTimeout instead? + ctrlResult, err = db.WaitForDBCreated(ctx, h) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBReadyErrorMessage, + err.Error())) + return db, ctrlResult, err + } + if (ctrlResult != ctrl.Result{}) { + Log.Info(fmt.Sprintf("%s... requeueing", condition.DBReadyRunningMessage)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBReadyRunningMessage)) + return db, ctrlResult, nil + } + + // update Status.DatabaseHostname, used to config the service + instance.Status.DatabaseHostname = db.GetDatabaseHostname() + instance.Status.Conditions.MarkTrue(condition.DBReadyCondition, condition.DBReadyMessage) + return db, ctrlResult, nil +} diff --git a/controllers/cloudkittyapi_controller.go b/controllers/cloudkittyapi_controller.go new file mode 100644 index 00000000..082f6104 --- /dev/null +++ b/controllers/cloudkittyapi_controller.go @@ -0,0 +1,1119 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkittyapi" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/go-logr/logr" + networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" + "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" +) + +// GetClient - +func (r *CloudKittyAPIReconciler) GetClient() client.Client { + return r.Client +} + +// GetKClient - +func (r *CloudKittyAPIReconciler) GetKClient() kubernetes.Interface { + return r.Kclient +} + +// GetScheme - +func (r *CloudKittyAPIReconciler) GetScheme() *runtime.Scheme { + return r.Scheme +} + +// CloudKittyAPIReconciler reconciles a CloudKittyAPI object +type CloudKittyAPIReconciler struct { + client.Client + Kclient kubernetes.Interface + Scheme *runtime.Scheme +} + +// GetLogger returns a logger object with a logging prefix of "controller.name" and additional controller context fields +func (r *CloudKittyAPIReconciler) GetLogger(ctx context.Context) logr.Logger { + return log.FromContext(ctx).WithName("Controllers").WithName("CloudKittyAPI") +} + +var keystoneServices = []map[string]string{ + { + "type": cloudkitty.ServiceType, + "name": cloudkitty.ServiceName, + "desc": "CloudKitty V2 Service", + }, +} + +//+kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyapis,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyapis/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyapis/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list; +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneservices,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneendpoints,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch +// +kubebuilder:rbac:groups=topology.openstack.org,resources=topologies,verbs=get;list;watch;update + +// Reconcile - +func (r *CloudKittyAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { + Log := r.GetLogger(ctx) + + // Fetch the CloudKittyAPI instance + instance := &telemetryv1.CloudKittyAPI{} + err := r.Client.Get(ctx, req.NamespacedName, instance) + if err != nil { + if k8s_errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. + // For additional cleanup logic use finalizers. Return and don't requeue. + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + return ctrl.Result{}, err + } + + helper, err := helper.NewHelper( + instance, + r.Client, + r.Kclient, + r.Scheme, + Log, + ) + if err != nil { + return ctrl.Result{}, err + } + + // + // initialize status + // + isNewInstance := instance.Status.Conditions == nil + if isNewInstance { + instance.Status.Conditions = condition.Conditions{} + } + + // Save a copy of the condtions so that we can restore the LastTransitionTime + // when a condition's state doesn't change. + savedConditions := instance.Status.Conditions.DeepCopy() + + // Always patch the instance status when exiting this function so we can persist any changes. + defer func() { + // Don't update the status, if reconciler Panics + if r := recover(); r != nil { + Log.Info(fmt.Sprintf("panic during reconcile %v\n", r)) + panic(r) + } + condition.RestoreLastTransitionTimes(&instance.Status.Conditions, savedConditions) + if instance.Status.Conditions.IsUnknown(condition.ReadyCondition) { + instance.Status.Conditions.Set( + instance.Status.Conditions.Mirror(condition.ReadyCondition)) + } + err := helper.PatchInstance(ctx, instance) + if err != nil { + _err = err + return + } + }() + + // Always initialize conditions used later as Status=Unknown + cl := condition.CreateList( + condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), + condition.UnknownCondition(condition.CreateServiceReadyCondition, condition.InitReason, condition.CreateServiceReadyInitMessage), + condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), + condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), + // right now we have no dedicated KeystoneServiceReadyInitMessage and KeystoneEndpointReadyInitMessage + condition.UnknownCondition(condition.KeystoneServiceReadyCondition, condition.InitReason, ""), + condition.UnknownCondition(condition.KeystoneEndpointReadyCondition, condition.InitReason, ""), + condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), + condition.UnknownCondition(condition.TLSInputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + ) + instance.Status.Conditions.Init(&cl) + // Always mark the Generation as observed early on + instance.Status.ObservedGeneration = instance.Generation + + // If we're not deleting this and the service object doesn't have our finalizer, add it. + if (instance.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(instance, helper.GetFinalizer())) || isNewInstance { + // Register overall status immediately to have an early feedback e.g. in the cli + return ctrl.Result{}, nil + } + + if instance.Status.Hash == nil { + instance.Status.Hash = map[string]string{} + } + if instance.Status.APIEndpoints == nil { + instance.Status.APIEndpoints = map[string]map[string]string{} + } + if instance.Status.ServiceIDs == nil { + instance.Status.ServiceIDs = map[string]string{} + } + if instance.Status.NetworkAttachments == nil { + instance.Status.NetworkAttachments = map[string][]string{} + } + + // Handle service delete + if !instance.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, instance, helper) + } + + // Init Topology condition if there's a reference + if instance.Spec.TopologyRef != nil { + c := condition.UnknownCondition(condition.TopologyReadyCondition, condition.InitReason, condition.TopologyReadyInitMessage) + cl.Set(c) + } + + // Handle non-deleted clusters + return r.reconcileNormal(ctx, instance, helper) +} + +// SetupWithManager sets up the controller with the Manager. +func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { + Log := r.GetLogger(ctx) + + // Watch for changes to secrets we don't own. Global secrets + // (e.g. TransportURLSecret) are handled by the main cloudkitty controller. + secretFn := func(_ context.Context, o client.Object) []reconcile.Request { + var namespace string = o.GetNamespace() + var secretName string = o.GetName() + result := []reconcile.Request{} + + // get all API CRs + apis := &telemetryv1.CloudKittyAPIList{} + listOpts := []client.ListOption{ + client.InNamespace(namespace), + } + if err := r.Client.List(context.Background(), apis, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve API CRs %v") + return nil + } + + // Watch for changes to secrets where the owner label AND the + // CR.Spec.ManagingCrName label matches + label := o.GetLabels() + if l, ok := label[labels.GetOwnerNameLabelSelector(labels.GetGroupLabel(cloudkitty.ServiceName))]; ok { + for _, cr := range apis.Items { + // return reconcile event for the CR where the owner label AND the parentCloudKittyName matches + if l == cloudkitty.GetOwningCloudKittyName(&cr) { + // return namespace and Name of CR + name := client.ObjectKey{ + Namespace: o.GetNamespace(), + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Secret %s and CR %s marked with label: %s", o.GetName(), cr.Name, l)) + + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + } + + // Watch for changes to any CustomServiceConfigSecrets + for _, cr := range apis.Items { + for _, v := range cr.Spec.CustomServiceConfigSecrets { + if v == secretName { + name := client.ObjectKey{ + Namespace: namespace, + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Secret %s is used by CloudKitty CR %s", secretName, cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + } + if len(result) > 0 { + return result + } + return nil + } + + // index passwordSecretField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyPasswordSecretField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyAPI) + if cr.Spec.Secret == "" { + return nil + } + return []string{cr.Spec.Secret} + }); err != nil { + return err + } + + // index caBundleSecretNameField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyCaBundleSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyAPI) + if cr.Spec.TLS.CaBundleSecretName == "" { + return nil + } + return []string{cr.Spec.TLS.CaBundleSecretName} + }); err != nil { + return err + } + + // index tlsAPIInternalField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyTlsAPIInternalField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyAPI) + if cr.Spec.TLS.API.Internal.SecretName == nil { + return nil + } + return []string{*cr.Spec.TLS.API.Internal.SecretName} + }); err != nil { + return err + } + + // index tlsAPIPublicField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyTlsAPIPublicField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyAPI) + if cr.Spec.TLS.API.Public.SecretName == nil { + return nil + } + return []string{*cr.Spec.TLS.API.Public.SecretName} + }); err != nil { + return err + } + + // index topologyField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyTopologyField, func(rawObj client.Object) []string { + // Extract the topology name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyAPI) + if cr.Spec.TopologyRef == nil { + return nil + } + return []string{cr.Spec.TopologyRef.Name} + }); err != nil { + return err + } + + return ctrl.NewControllerManagedBy(mgr). + For(&telemetryv1.CloudKittyAPI{}). + Owns(&keystonev1.KeystoneService{}). + Owns(&keystonev1.KeystoneEndpoint{}). + Owns(&appsv1.StatefulSet{}). + Owns(&corev1.Service{}). + // watch the secrets we don't own + Watches(&corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(secretFn)). + Watches( + &corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). + Watches(&topologyv1.Topology{}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.GenerationChangedPredicate{})). + Complete(r) +} + +func (r *CloudKittyAPIReconciler) findObjectsForSrc(ctx context.Context, src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + l := log.FromContext(ctx).WithName("Controllers").WithName("CloudKittyAPI") + + for _, field := range cloudKittyAPIWatchFields { + crList := &telemetryv1.CloudKittyAPIList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.List(ctx, crList, listOps) + if err != nil { + l.Error(err, fmt.Sprintf("listing %s for field: %s - %s", crList.GroupVersionKind().Kind, field, src.GetNamespace())) + return requests + } + + for _, item := range crList.Items { + l.Info(fmt.Sprintf("input source %s changed, reconcile: %s - %s", src.GetName(), item.GetName(), item.GetNamespace())) + + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} + +func (r *CloudKittyAPIReconciler) reconcileDelete(ctx context.Context, instance *telemetryv1.CloudKittyAPI, helper *helper.Helper) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s' delete", instance.Name)) + + // It's possible to get here before the endpoints have been set in the status, so check for this + if instance.Status.APIEndpoints != nil { + for _, ksSvc := range keystoneServices { + + // Remove the finalizer from our KeystoneEndpoint CR + keystoneEndpoint, err := keystonev1.GetKeystoneEndpointWithName(ctx, helper, ksSvc["name"], instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + + if err == nil { + controllerutil.RemoveFinalizer(keystoneEndpoint, helper.GetFinalizer()) + if err = helper.GetClient().Update(ctx, keystoneEndpoint); err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + util.LogForObject(helper, "Removed finalizer from our KeystoneEndpoint", instance) + } + + // Remove the finalizer from our KeystoneService CR + keystoneService, err := keystonev1.GetKeystoneServiceWithName(ctx, helper, ksSvc["name"], instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + + if err == nil { + controllerutil.RemoveFinalizer(keystoneService, helper.GetFinalizer()) + if err = helper.GetClient().Update(ctx, keystoneService); err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + util.LogForObject(helper, "Removed finalizer from our KeystoneService", instance) + } + } + } + + // Remove finalizer on the Topology CR + if ctrlResult, err := topologyv1.EnsureDeletedTopologyRef( + ctx, + helper, + instance.Status.LastAppliedTopology, + instance.Name, + ); err != nil { + return ctrlResult, err + } + + // Service is deleted so remove the finalizer. + controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) + Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", instance.Name)) + + return ctrl.Result{}, nil +} + +func (r *CloudKittyAPIReconciler) reconcileInit( + ctx context.Context, + instance *telemetryv1.CloudKittyAPI, + helper *helper.Helper, + serviceLabels map[string]string, +) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s' init", instance.Name)) + + // + // expose the service (create service and return the created endpoint URLs) + // + + // V2 + publicEndpointData := endpoint.Data{ + Port: cloudkitty.CloudKittyPublicPort, + Path: "/v2", + } + internalEndpointData := endpoint.Data{ + Port: cloudkitty.CloudKittyInternalPort, + Path: "/v2", + } + cloudkittyEndpoints := map[service.Endpoint]endpoint.Data{ + service.EndpointPublic: publicEndpointData, + service.EndpointInternal: internalEndpointData, + } + + apiEndpointsV3 := make(map[string]string) + + for endpointType, data := range cloudkittyEndpoints { + endpointTypeStr := string(endpointType) + endpointName := cloudkitty.ServiceName + "-" + endpointTypeStr + svcOverride := instance.Spec.Override.Service[endpointType] + if svcOverride.EmbeddedLabelsAnnotations == nil { + svcOverride.EmbeddedLabelsAnnotations = &service.EmbeddedLabelsAnnotations{} + } + + exportLabels := util.MergeStringMaps( + serviceLabels, + map[string]string{ + service.AnnotationEndpointKey: endpointTypeStr, + }, + ) + + // Create the service + svc, err := service.NewService( + service.GenericService(&service.GenericServiceDetails{ + Name: endpointName, + Namespace: instance.Namespace, + Labels: exportLabels, + Selector: serviceLabels, + Port: service.GenericServicePort{ + Name: endpointName, + Port: data.Port, + Protocol: corev1.ProtocolTCP, + }, + }), + 5, + &svcOverride.OverrideSpec, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.CreateServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.CreateServiceReadyErrorMessage, + err.Error())) + + return ctrl.Result{}, err + } + + svc.AddAnnotation(map[string]string{ + service.AnnotationEndpointKey: endpointTypeStr, + }) + + // add Annotation to whether creating an ingress is required or not + if endpointType == service.EndpointPublic && svc.GetServiceType() == corev1.ServiceTypeClusterIP { + svc.AddAnnotation(map[string]string{ + service.AnnotationIngressCreateKey: "true", + }) + } else { + svc.AddAnnotation(map[string]string{ + service.AnnotationIngressCreateKey: "false", + }) + if svc.GetServiceType() == corev1.ServiceTypeLoadBalancer { + svc.AddAnnotation(map[string]string{ + service.AnnotationHostnameKey: svc.GetServiceHostname(), // add annotation to register service name in dnsmasq + }) + } + } + + ctrlResult, err := svc.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.CreateServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.CreateServiceReadyErrorMessage, + err.Error())) + + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.CreateServiceReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.CreateServiceReadyRunningMessage)) + return ctrlResult, nil + } + // create service - end + + // if TLS is enabled + if instance.Spec.TLS.API.Enabled(endpointType) { + // set endpoint protocol to https + data.Protocol = ptr.To(service.ProtocolHTTPS) + } + + apiEndpointsV3[string(endpointType)], err = svc.GetAPIEndpoint( + svcOverride.EndpointURL, data.Protocol, data.Path) + if err != nil { + instance.Status.Conditions.MarkFalse( + condition.CreateServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.CreateServiceReadyErrorMessage, + err.Error()) + return ctrl.Result{}, err + } + } + instance.Status.Conditions.MarkTrue(condition.CreateServiceReadyCondition, condition.CreateServiceReadyMessage) + + // + // Update instance status with service endpoint url from route host information + // + if instance.Status.APIEndpoints == nil { + instance.Status.APIEndpoints = map[string]map[string]string{} + } + instance.Status.APIEndpoints[cloudkitty.ServiceName] = apiEndpointsV3 + // V2 - end + + // expose service - end + + // + // create service and user in keystone - - https://docs.openstack.org/CloudKitty/latest/install/install-rdo.html#configure-user-and-endpoints + // TODO: rework this + // + if instance.Status.ServiceIDs == nil { + instance.Status.ServiceIDs = map[string]string{} + } + + for _, ksSvc := range keystoneServices { + ksSvcSpec := keystonev1.KeystoneServiceSpec{ + ServiceType: ksSvc["type"], + ServiceName: ksSvc["name"], + ServiceDescription: ksSvc["desc"], + Enabled: true, + ServiceUser: instance.Spec.ServiceUser, + Secret: instance.Spec.Secret, + PasswordSelector: instance.Spec.PasswordSelectors.CloudKittyService, + } + + ksSvcObj := keystonev1.NewKeystoneService(ksSvcSpec, instance.Namespace, serviceLabels, cloudkitty.NormalDuration) + ctrlResult, err := ksSvcObj.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.MarkFalse( + condition.KeystoneServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + "Creating KeyStoneService CR %s", + err.Error()) + return ctrlResult, err + } + + // mirror the Status, Reason, Severity and Message of the latest keystoneservice condition + // into a local condition with the type condition.KeystoneServiceReadyCondition + c := ksSvcObj.GetConditions().Mirror(condition.KeystoneServiceReadyCondition) + if c != nil { + instance.Status.Conditions.Set(c) + } + + if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + instance.Status.ServiceIDs[ksSvc["name"]] = ksSvcObj.GetServiceID() + + ksEndptSpec := keystonev1.KeystoneEndpointSpec{ + ServiceName: ksSvc["name"], + Endpoints: instance.Status.APIEndpoints[ksSvc["name"]], + } + + ksEndptObj := keystonev1.NewKeystoneEndpoint( + ksSvc["name"], + instance.Namespace, + ksEndptSpec, + serviceLabels, + cloudkitty.NormalDuration) + ctrlResult, err = ksEndptObj.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.MarkFalse( + condition.KeystoneEndpointReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + "Creating KeyStoneEndpoint CR %s", + err.Error()) + return ctrlResult, err + } + + // mirror the Status, Reason, Severity and Message of the latest keystoneendpoint condition + // into a local condition with the type condition.KeystoneEndpointReadyCondition + c = ksEndptObj.GetConditions().Mirror(condition.KeystoneEndpointReadyCondition) + if c != nil { + instance.Status.Conditions.Set(c) + } + + if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + } + + Log.Info(fmt.Sprintf("Reconciled Service '%s' init successfully", instance.Name)) + return ctrl.Result{}, nil +} + +func (r *CloudKittyAPIReconciler) reconcileNormal(ctx context.Context, instance *telemetryv1.CloudKittyAPI, helper *helper.Helper) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s'", instance.Name)) + + configVars := make(map[string]env.Setter) + + // + // check for required OpenStack secret holding passwords for service/admin user and add hash to the vars map + // + + ctrlResult, err := cloudkitty.VerifyServiceSecret( + ctx, + types.NamespacedName{Namespace: instance.Namespace, Name: instance.Spec.Secret}, + []string{ + instance.Spec.PasswordSelectors.CloudKittyService, + }, + helper.GetClient(), + &instance.Status.Conditions, + cloudkitty.NormalDuration, + &configVars, + ) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + // + // check for required Transport URL and config secrets + // + + parentCloudKittyName := cloudkitty.GetOwningCloudKittyName(instance) + secretNames := []string{ + instance.Spec.TransportURLSecret, // TransportURLSecret + fmt.Sprintf("%s-scripts", parentCloudKittyName), // ScriptsSecret + fmt.Sprintf("%s-config-data", parentCloudKittyName), // ConfigSecret + } + // Append CustomServiceConfigSecrets that should be checked + secretNames = append(secretNames, instance.Spec.CustomServiceConfigSecrets...) + + ctrlResult, err = cloudkitty.VerifyConfigSecrets( + ctx, + helper, + &instance.Status.Conditions, + secretNames, + instance.Namespace, + &configVars, + ) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + + // + // TLS input validation + // + // Validate the CA cert secret if provided + if instance.Spec.TLS.CaBundleSecretName != "" { + hash, err := tls.ValidateCACertSecret( + ctx, + helper.GetClient(), + types.NamespacedName{ + Name: instance.Spec.TLS.CaBundleSecretName, + Namespace: instance.Namespace, + }, + ) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + fmt.Sprintf(condition.TLSInputReadyWaitingMessage, instance.Spec.TLS.CaBundleSecretName))) + return ctrl.Result{}, nil + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if hash != "" { + configVars[tls.CABundleKey] = env.SetValue(hash) + } + } + + // Validate API service certs secrets + certsHash, err := instance.Spec.TLS.API.ValidateCertSecrets(ctx, helper, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + fmt.Sprintf(condition.TLSInputReadyWaitingMessage, err.Error()))) + return ctrl.Result{}, nil + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + configVars[tls.TLSHashName] = env.SetValue(certsHash) + + // all cert input checks out so report InputReady + instance.Status.Conditions.MarkTrue(condition.TLSInputReadyCondition, condition.InputReadyMessage) + + // + // Create secrets required as input for the Service and calculate an overall hash of hashes + // + serviceLabels := map[string]string{ + common.AppSelector: cloudkitty.ServiceName, + common.ComponentSelector: cloudkittyapi.ComponentName, + } + + // + // create custom config for this cloudkitty service + // + err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) + + // + // TODO check when/if Init, Update, or Upgrade should/could be skipped + // + + // networks to attach to + nadList := []networkv1.NetworkAttachmentDefinition{} + for _, netAtt := range instance.Spec.NetworkAttachments { + nad, err := nad.GetNADWithName(ctx, helper, netAtt, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + Log.Info(fmt.Sprintf("network-attachment-definition %s not found", netAtt)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.NetworkAttachmentsReadyWaitingMessage, + netAtt)) + return cloudkitty.ResultRequeue, fmt.Errorf("network-attachment-definition %s not found", netAtt) + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if nad != nil { + nadList = append(nadList, *nad) + } + } + + serviceAnnotations, err := nad.EnsureNetworksAnnotation(nadList) + if err != nil { + err = fmt.Errorf("failed create network annotation from %s: %w", instance.Spec.NetworkAttachments, err) + instance.Status.Conditions.MarkFalse( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error()) + return ctrl.Result{}, err + } + + // Handle service init + ctrlResult, err = r.reconcileInit(ctx, instance, helper, serviceLabels) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + // + // Handle Topology + // + topology, err := ensureTopology( + ctx, + helper, + instance, // topologyHandler + instance.Name, // finalizer + &instance.Status.Conditions, + labels.GetLabelSelector(serviceLabels), + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TopologyReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TopologyReadyErrorMessage, + err.Error())) + return ctrl.Result{}, fmt.Errorf("waiting for Topology requirements: %w", err) + } + + // + // normal reconcile tasks + // + + // + // create hash over all the different input resources to identify if any those changed + // and a restart/recreate is required. + // + inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configVars) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } else if hashChanged { + Log.Info(fmt.Sprintf("%s... requeueing", condition.ServiceConfigReadyInitMessage)) + instance.Status.Conditions.MarkFalse( + condition.ServiceConfigReadyCondition, + condition.InitReason, + condition.SeverityInfo, + condition.ServiceConfigReadyInitMessage) + // Hash changed and instance status should be updated (which will be done by main defer func), + // so we need to return and reconcile again + return ctrl.Result{}, nil + } + + // Deploy a statefulset + ssDef, err := cloudkittyapi.StatefulSet(instance, inputHash, serviceLabels, serviceAnnotations, topology) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + ss := statefulset.NewStatefulSet(ssDef, cloudkitty.ShortDuration) + + var ssData appsv1.StatefulSet + ctrlResult, err = ss.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrlResult, err + + } else if (ctrlResult == ctrl.Result{}) { + // Wait until the data in the StatefulSet is for the current generation + ssData = ss.GetStatefulSet() + if ssData.Generation != ssData.Status.ObservedGeneration { + ctrlResult = cloudkitty.ResultRequeue + err = fmt.Errorf("waiting for Statefulset %s to start reconciling", ssData.Name) + } + } + + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + // If the deployment is not ready, then neither are the NADs + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.NetworkAttachmentsReadyInitMessage)) + return ctrlResult, err + } + + instance.Status.ReadyCount = ssData.Status.ReadyReplicas + + // verify if network attachment matches expectations + networkReady := false + networkAttachmentStatus := map[string][]string{} + if *instance.Spec.Replicas > 0 { + networkReady, networkAttachmentStatus, err = nad.VerifyNetworkStatusFromAnnotation( + ctx, + helper, + instance.Spec.NetworkAttachments, + serviceLabels, + instance.Status.ReadyCount, + ) + if err != nil { + err = fmt.Errorf("verifying API NetworkAttachments (%s) %w", instance.Spec.NetworkAttachments, err) + instance.Status.Conditions.MarkFalse( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error()) + return ctrl.Result{}, err + } + } else { + networkReady = true + } + + instance.Status.NetworkAttachments = networkAttachmentStatus + if networkReady { + instance.Status.Conditions.MarkTrue(condition.NetworkAttachmentsReadyCondition, condition.NetworkAttachmentsReadyMessage) + } else { + err := fmt.Errorf("not all pods have interfaces with ips as configured in NetworkAttachments: %s", instance.Spec.NetworkAttachments) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + + return ctrl.Result{}, err + } + + if instance.Status.ReadyCount > 0 { + instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) + + } else if *instance.Spec.Replicas > 0 { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + + } else { + instance.Status.Conditions.MarkFalse( + condition.DeploymentReadyCondition, + condition.NotRequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyInitMessage) + } + // create StatefulSet - end + + Log.Info(fmt.Sprintf("Reconciled Service '%s' successfully", instance.Name)) + // update the overall status condition if service is ready + if instance.IsReady() { + instance.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) + } + // For non ready we'll let the main defer func handle the status update using the Mirror function + return ctrl.Result{}, nil +} + +// generateServiceConfigs - create Secret which holds the service configuration +func (r *CloudKittyAPIReconciler) generateServiceConfigs( + ctx context.Context, + h *helper.Helper, + instance *telemetryv1.CloudKittyAPI, + envVars *map[string]env.Setter, + serviceLabels map[string]string, +) error { + // + // create custom Secret for cloudkitty service-specific config input + // - %-config-data holds custom config for the service + // + + labels := labels.GetLabels(instance, labels.GetGroupLabel(cloudkitty.ServiceName), serviceLabels) + + // customData hold any customization for the service. + customData := map[string]string{cloudkitty.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig} + + // Fetch the two service config snippets (DefaultsConfigFileName and + // CustomConfigFileName) from the Secret generated by the top level + // cloudkitty controller, and add them to this service specific Secret. + cloudkittySecretName := cloudkitty.GetOwningCloudKittyName(instance) + "-config-data" + cloudkittySecret, _, err := secret.GetSecret(ctx, h, cloudkittySecretName, instance.Namespace) + if err != nil { + return err + } + customData[cloudkitty.DefaultsConfigFileName] = string(cloudkittySecret.Data[cloudkitty.DefaultsConfigFileName]) + customData[cloudkitty.CustomConfigFileName] = string(cloudkittySecret.Data[cloudkitty.CustomConfigFileName]) + + customSecrets := "" + for _, secretName := range instance.Spec.CustomServiceConfigSecrets { + secret, _, err := secret.GetSecret(ctx, h, secretName, instance.Namespace) + if err != nil { + return err + } + for _, data := range secret.Data { + customSecrets += string(data) + "\n" + } + } + customData[cloudkitty.CustomServiceConfigSecretsFileName] = customSecrets + + templateParameters := map[string]interface{}{ + "LogFile": cloudkittyapi.LogFile, + } + + configTemplates := []util.Template{ + { + Name: fmt.Sprintf("%s-config-data", instance.Name), + Namespace: instance.Namespace, + Type: util.TemplateTypeConfig, + InstanceType: instance.Kind, + CustomData: customData, + ConfigOptions: templateParameters, + Labels: labels, + }, + } + + return secret.EnsureSecrets(ctx, h, instance, configTemplates, envVars) +} + +// createHashOfInputHashes - creates a hash of hashes which gets added to the resources which requires a restart +// if any of the input resources change, like configs, passwords, ... +// +// returns the hash, whether the hash changed (as a bool) and any error +func (r *CloudKittyAPIReconciler) createHashOfInputHashes( + ctx context.Context, + instance *telemetryv1.CloudKittyAPI, + envVars map[string]env.Setter, +) (string, bool, error) { + Log := r.GetLogger(ctx) + + var hashMap map[string]string + changed := false + mergedMapVars := env.MergeEnvs([]corev1.EnvVar{}, envVars) + hash, err := util.ObjectHash(mergedMapVars) + if err != nil { + return hash, changed, err + } + if hashMap, changed = util.SetHash(instance.Status.Hash, common.InputHashName, hash); changed { + instance.Status.Hash = hashMap + Log.Info(fmt.Sprintf("Input maps hash %s - %s", common.InputHashName, hash)) + } + return hash, changed, nil +} diff --git a/controllers/cloudkittyproc_controller.go b/controllers/cloudkittyproc_controller.go new file mode 100644 index 00000000..fbccef63 --- /dev/null +++ b/controllers/cloudkittyproc_controller.go @@ -0,0 +1,759 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkittyproc" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/go-logr/logr" + networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" + "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" +) + +// GetClient - +func (r *CloudKittyProcReconciler) GetClient() client.Client { + return r.Client +} + +// GetKClient - +func (r *CloudKittyProcReconciler) GetKClient() kubernetes.Interface { + return r.Kclient +} + +// GetScheme - +func (r *CloudKittyProcReconciler) GetScheme() *runtime.Scheme { + return r.Scheme +} + +// CloudKittyProcReconciler reconciles a CloudKittyProc object +type CloudKittyProcReconciler struct { + client.Client + Kclient kubernetes.Interface + Scheme *runtime.Scheme +} + +// GetLogger returns a logger object with a logging prefix of "controller.name" and additional controller context fields +func (r *CloudKittyProcReconciler) GetLogger(ctx context.Context) logr.Logger { + return log.FromContext(ctx).WithName("Controllers").WithName("CloudKittyProc") +} + +//+kubebuilder:rbac:groups=cloudkitty.openstack.org,resources=cloudkittyprocs,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=cloudkitty.openstack.org,resources=cloudkittyprocs/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=cloudkitty.openstack.org,resources=cloudkittyprocs/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list; +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch +// +kubebuilder:rbac:groups=topology.openstack.org,resources=topologies,verbs=get;list;watch;update + +// Reconcile - +func (r *CloudKittyProcReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { + Log := r.GetLogger(ctx) + + // Fetch the CloudKittyProc instance + instance := &telemetryv1.CloudKittyProc{} + err := r.Client.Get(ctx, req.NamespacedName, instance) + if err != nil { + if k8s_errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. + // For additional cleanup logic use finalizers. Return and don't requeue. + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + return ctrl.Result{}, err + } + + helper, err := helper.NewHelper( + instance, + r.Client, + r.Kclient, + r.Scheme, + Log, + ) + if err != nil { + return ctrl.Result{}, err + } + + // + // initialize status + // + isNewInstance := instance.Status.Conditions == nil + if isNewInstance { + instance.Status.Conditions = condition.Conditions{} + } + + // Save a copy of the condtions so that we can restore the LastTransitionTime + // when a condition's state doesn't change. + savedConditions := instance.Status.Conditions.DeepCopy() + + // Always patch the instance status when exiting this function so we can persist any changes. + defer func() { + // Don't update the status, if reconciler Panics + if r := recover(); r != nil { + Log.Info(fmt.Sprintf("panic during reconcile %v\n", r)) + panic(r) + } + condition.RestoreLastTransitionTimes(&instance.Status.Conditions, savedConditions) + if instance.Status.Conditions.IsUnknown(condition.ReadyCondition) { + instance.Status.Conditions.Set( + instance.Status.Conditions.Mirror(condition.ReadyCondition)) + } + err := helper.PatchInstance(ctx, instance) + if err != nil { + _err = err + return + } + }() + + // Always initialize conditions used later as Status=Unknown + cl := condition.CreateList( + condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), + condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), + condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), + condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), + condition.UnknownCondition(condition.TLSInputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + ) + instance.Status.Conditions.Init(&cl) + // Always mark the Generation as observed early on + instance.Status.ObservedGeneration = instance.Generation + + // If we're not deleting this and the service object doesn't have our finalizer, add it. + if (instance.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(instance, helper.GetFinalizer())) || isNewInstance { + // Register overall status immediately to have an early feedback e.g. in the cli + return ctrl.Result{}, nil + } + + if instance.Status.Hash == nil { + instance.Status.Hash = map[string]string{} + } + if instance.Status.NetworkAttachments == nil { + instance.Status.NetworkAttachments = map[string][]string{} + } + + // Handle service delete + if !instance.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, instance, helper) + } + + // Init Topology condition if there's a reference + if instance.Spec.TopologyRef != nil { + c := condition.UnknownCondition(condition.TopologyReadyCondition, condition.InitReason, condition.TopologyReadyInitMessage) + cl.Set(c) + } + + // Handle non-deleted clusters + return r.reconcileNormal(ctx, instance, helper) +} + +// SetupWithManager sets up the controller with the Manager. +func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { + Log := r.GetLogger(ctx) + + // Watch for changes to secrets we don't own. Global secrets + // (e.g. TransportURLSecret) are handled by the main cloudkitty controller. + secretFn := func(_ context.Context, o client.Object) []reconcile.Request { + var namespace string = o.GetNamespace() + var secretName string = o.GetName() + result := []reconcile.Request{} + + // get all scheduler CRs + schedulers := &telemetryv1.CloudKittyProcList{} + listOpts := []client.ListOption{ + client.InNamespace(namespace), + } + if err := r.Client.List(context.Background(), schedulers, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve scheduler CRs %v") + return nil + } + + // Watch for changes to secrets where the owner label AND the + // CR.Spec.ManagingCrName label matches + label := o.GetLabels() + if l, ok := label[labels.GetOwnerNameLabelSelector(labels.GetGroupLabel(cloudkitty.ServiceName))]; ok { + for _, cr := range schedulers.Items { + // return reconcile event for the CR where the owner label AND the parentCloudKittyName matches + if l == cloudkitty.GetOwningCloudKittyName(&cr) { + // return namespace and Name of CR + name := client.ObjectKey{ + Namespace: o.GetNamespace(), + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Secret %s and CR %s marked with label: %s", o.GetName(), cr.Name, l)) + + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + } + + // Watch for changes to any CustomServiceConfigSecrets + for _, cr := range schedulers.Items { + for _, v := range cr.Spec.CustomServiceConfigSecrets { + if v == secretName { + name := client.ObjectKey{ + Namespace: namespace, + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Secret %s is used by CloudKitty CR %s", secretName, cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + } + if len(result) > 0 { + return result + } + return nil + } + + // index passwordSecretField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyProc{}, cloudKittyPasswordSecretField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyProc) + if cr.Spec.Secret == "" { + return nil + } + return []string{cr.Spec.Secret} + }); err != nil { + return err + } + + // index caBundleSecretNameField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyProc{}, cloudKittyCaBundleSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyProc) + if cr.Spec.TLS.CaBundleSecretName == "" { + return nil + } + return []string{cr.Spec.TLS.CaBundleSecretName} + }); err != nil { + return err + } + + // index topologyField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyProc{}, cloudKittyTopologyField, func(rawObj client.Object) []string { + // Extract the topology name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyProc) + if cr.Spec.TopologyRef == nil { + return nil + } + return []string{cr.Spec.TopologyRef.Name} + }); err != nil { + return err + } + + return ctrl.NewControllerManagedBy(mgr). + For(&telemetryv1.CloudKittyProc{}). + Owns(&appsv1.StatefulSet{}). + // watch the secrets we don't own + Watches(&corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(secretFn)). + Watches( + &corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). + Watches(&topologyv1.Topology{}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.GenerationChangedPredicate{})). + Complete(r) +} + +func (r *CloudKittyProcReconciler) findObjectsForSrc(ctx context.Context, src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + l := log.FromContext(ctx).WithName("Controllers").WithName("CloudKittyProc") + + for _, field := range commonWatchFields { + crList := &telemetryv1.CloudKittyProcList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.List(ctx, crList, listOps) + if err != nil { + l.Error(err, fmt.Sprintf("listing %s for field: %s - %s", crList.GroupVersionKind().Kind, field, src.GetNamespace())) + return requests + } + + for _, item := range crList.Items { + l.Info(fmt.Sprintf("input source %s changed, reconcile: %s - %s", src.GetName(), item.GetName(), item.GetNamespace())) + + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} + +func (r *CloudKittyProcReconciler) reconcileDelete(ctx context.Context, instance *telemetryv1.CloudKittyProc, helper *helper.Helper) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s' delete", instance.Name)) + + // Service is deleted so remove the finalizer. + controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) + Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", instance.Name)) + + // Remove finalizer on the Topology CR + if ctrlResult, err := topologyv1.EnsureDeletedTopologyRef( + ctx, + helper, + instance.Status.LastAppliedTopology, + instance.Name, + ); err != nil { + return ctrlResult, err + } + return ctrl.Result{}, nil +} + +func (r *CloudKittyProcReconciler) reconcileNormal(ctx context.Context, instance *telemetryv1.CloudKittyProc, helper *helper.Helper) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s'", instance.Name)) + + configVars := make(map[string]env.Setter) + + // + // check for required OpenStack secret holding passwords for service/admin user and add hash to the vars map + // + + ctrlResult, err := cloudkitty.VerifyServiceSecret( + ctx, + types.NamespacedName{Namespace: instance.Namespace, Name: instance.Spec.Secret}, + []string{ + instance.Spec.PasswordSelectors.CloudKittyService, + }, + helper.GetClient(), + &instance.Status.Conditions, + cloudkitty.NormalDuration, + &configVars, + ) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + // + // check for required Transport URL and config secrets + // + + parentCloudKittyName := cloudkitty.GetOwningCloudKittyName(instance) + secretNames := []string{ + instance.Spec.TransportURLSecret, // TransportURLSecret + fmt.Sprintf("%s-scripts", parentCloudKittyName), // ScriptsSecret + fmt.Sprintf("%s-config-data", parentCloudKittyName), // ConfigSecret + } + // Append CustomServiceConfigSecrets that should be checked + secretNames = append(secretNames, instance.Spec.CustomServiceConfigSecrets...) + + ctrlResult, err = cloudkitty.VerifyConfigSecrets( + ctx, + helper, + &instance.Status.Conditions, + secretNames, + instance.Namespace, + &configVars, + ) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + + // + // TLS input validation + // + // Validate the CA cert secret if provided + if instance.Spec.TLS.CaBundleSecretName != "" { + hash, err := tls.ValidateCACertSecret( + ctx, + helper.GetClient(), + types.NamespacedName{ + Name: instance.Spec.TLS.CaBundleSecretName, + Namespace: instance.Namespace, + }, + ) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + fmt.Sprintf(condition.TLSInputReadyWaitingMessage, instance.Spec.TLS.CaBundleSecretName))) + return ctrl.Result{}, nil + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if hash != "" { + configVars[tls.CABundleKey] = env.SetValue(hash) + } + } + // all cert input checks out so report InputReady + instance.Status.Conditions.MarkTrue(condition.TLSInputReadyCondition, condition.InputReadyMessage) + + // + // Create ConfigMaps required as input for the Service and calculate an overall hash of hashes + // + serviceLabels := map[string]string{ + common.AppSelector: cloudkitty.ServiceName, + common.ComponentSelector: cloudkittyproc.ComponentName, + } + + // + // create custom config for this cloudkitty service + // + err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + // + // create hash over all the different input resources to identify if any those changed + // and a restart/recreate is required. + // + inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configVars) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } else if hashChanged { + Log.Info(fmt.Sprintf("%s... requeueing", condition.ServiceConfigReadyInitMessage)) + instance.Status.Conditions.MarkFalse( + condition.ServiceConfigReadyCondition, + condition.InitReason, + condition.SeverityInfo, + condition.ServiceConfigReadyInitMessage) + // Hash changed and instance status should be updated (which will be done by main defer func), + // so we need to return and reconcile again + return ctrl.Result{}, nil + } + instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) + + // + // TODO check when/if Init, Update, or Upgrade should/could be skipped + // + + // networks to attach to + nadList := []networkv1.NetworkAttachmentDefinition{} + for _, netAtt := range instance.Spec.NetworkAttachments { + nad, err := nad.GetNADWithName(ctx, helper, netAtt, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + Log.Info(fmt.Sprintf("network-attachment-definition %s not found", netAtt)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.NetworkAttachmentsReadyWaitingMessage, + netAtt)) + return cloudkitty.ResultRequeue, fmt.Errorf("network-attachment-definition %s not found", netAtt) + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if nad != nil { + nadList = append(nadList, *nad) + } + } + + serviceAnnotations, err := nad.EnsureNetworksAnnotation(nadList) + if err != nil { + err = fmt.Errorf("failed create network annotation from %s: %w", instance.Spec.NetworkAttachments, err) + instance.Status.Conditions.MarkFalse( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error()) + return ctrl.Result{}, err + } + + // + // normal reconcile tasks + // + + // + // Handle Topology + // + topology, err := ensureTopology( + ctx, + helper, + instance, // topologyHandler + instance.Name, // finalizer + &instance.Status.Conditions, + labels.GetLabelSelector(serviceLabels), + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TopologyReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TopologyReadyErrorMessage, + err.Error())) + return ctrl.Result{}, fmt.Errorf("waiting for Topology requirements: %w", err) + } + + // Deploy a statefulset + ssDef := cloudkittyproc.StatefulSet(instance, inputHash, serviceLabels, serviceAnnotations, topology) + ss := statefulset.NewStatefulSet(ssDef, cloudkitty.ShortDuration) + + var ssData appsv1.StatefulSet + ctrlResult, err = ss.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrlResult, err + + } else if (ctrlResult == ctrl.Result{}) { + // Wait until the data in the StatefulSet is for the current generation + ssData = ss.GetStatefulSet() + if ssData.Generation != ssData.Status.ObservedGeneration { + ctrlResult = cloudkitty.ResultRequeue + err = fmt.Errorf("waiting for Statefulset %s to start reconciling", ssData.Name) + } + } + + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + // If the deployment is not ready, then neither are the NADs + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.NetworkAttachmentsReadyInitMessage)) + return ctrlResult, err + } + + instance.Status.ReadyCount = ssData.Status.ReadyReplicas + + // verify if network attachment matches expectations + networkReady := false + networkAttachmentStatus := map[string][]string{} + if *instance.Spec.Replicas > 0 { + networkReady, networkAttachmentStatus, err = nad.VerifyNetworkStatusFromAnnotation( + ctx, + helper, + instance.Spec.NetworkAttachments, + serviceLabels, + instance.Status.ReadyCount, + ) + if err != nil { + err = fmt.Errorf("verifying API NetworkAttachments (%s) %w", instance.Spec.NetworkAttachments, err) + instance.Status.Conditions.MarkFalse( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error()) + return ctrl.Result{}, err + } + } else { + networkReady = true + } + + instance.Status.NetworkAttachments = networkAttachmentStatus + if networkReady { + instance.Status.Conditions.MarkTrue(condition.NetworkAttachmentsReadyCondition, condition.NetworkAttachmentsReadyMessage) + } else { + err := fmt.Errorf("not all pods have interfaces with ips as configured in NetworkAttachments: %s", instance.Spec.NetworkAttachments) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + + return ctrl.Result{}, err + } + + if instance.Status.ReadyCount > 0 { + instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) + } else if *instance.Spec.Replicas > 0 { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + + } else { + instance.Status.Conditions.MarkFalse( + condition.DeploymentReadyCondition, + condition.NotRequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyInitMessage) + } + // create StatefulSet - end + + Log.Info(fmt.Sprintf("Reconciled Service '%s' successfully", instance.Name)) + // update the overall status condition if service is ready + if instance.IsReady() { + instance.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) + } + // For non ready we'll let the main defer func handle the status update using the Mirror function + return ctrl.Result{}, nil +} + +// generateServiceConfigs - create Secret which holds the service configuration +func (r *CloudKittyProcReconciler) generateServiceConfigs( + ctx context.Context, + h *helper.Helper, + instance *telemetryv1.CloudKittyProc, + envVars *map[string]env.Setter, + serviceLabels map[string]string, +) error { + // + // create custom Secret for cloudkitty service-specific config input + // - %-config-data holds custom config for the service + // + + labels := labels.GetLabels(instance, labels.GetGroupLabel(cloudkitty.ServiceName), serviceLabels) + + // customData hold any customization for the service. + customData := map[string]string{cloudkitty.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig} + + // Fetch the two service config snippets (DefaultsConfigFileName and + // CustomConfigFileName) from the Secret generated by the top level + // cloudkitty controller, and add them to this service specific Secret. + cloudkittySecretName := cloudkitty.GetOwningCloudKittyName(instance) + "-config-data" + cloudkittySecret, _, err := secret.GetSecret(ctx, h, cloudkittySecretName, instance.Namespace) + if err != nil { + return err + } + customData[cloudkitty.DefaultsConfigFileName] = string(cloudkittySecret.Data[cloudkitty.DefaultsConfigFileName]) + customData[cloudkitty.CustomConfigFileName] = string(cloudkittySecret.Data[cloudkitty.CustomConfigFileName]) + + customSecrets := "" + for _, secretName := range instance.Spec.CustomServiceConfigSecrets { + secret, _, err := secret.GetSecret(ctx, h, secretName, instance.Namespace) + if err != nil { + return err + } + for _, data := range secret.Data { + customSecrets += string(data) + "\n" + } + } + customData[cloudkitty.CustomServiceConfigSecretsFileName] = customSecrets + + configTemplates := []util.Template{ + { + Name: fmt.Sprintf("%s-config-data", instance.Name), + Namespace: instance.Namespace, + Type: util.TemplateTypeConfig, + InstanceType: instance.Kind, + CustomData: customData, + Labels: labels, + }, + } + + return secret.EnsureSecrets(ctx, h, instance, configTemplates, envVars) +} + +// createHashOfInputHashes - creates a hash of hashes which gets added to the resources which requires a restart +// if any of the input resources change, like configs, passwords, ... +// +// returns the hash, whether the hash changed (as a bool) and any error +func (r *CloudKittyProcReconciler) createHashOfInputHashes( + ctx context.Context, + instance *telemetryv1.CloudKittyProc, + envVars map[string]env.Setter, +) (string, bool, error) { + Log := r.GetLogger(ctx) + var hashMap map[string]string + changed := false + mergedMapVars := env.MergeEnvs([]corev1.EnvVar{}, envVars) + hash, err := util.ObjectHash(mergedMapVars) + if err != nil { + return hash, changed, err + } + if hashMap, changed = util.SetHash(instance.Status.Hash, common.InputHashName, hash); changed { + instance.Status.Hash = hashMap + Log.Info(fmt.Sprintf("Input maps hash %s - %s", common.InputHashName, hash)) + } + return hash, changed, nil +} diff --git a/controllers/telemetry_controller.go b/controllers/telemetry_controller.go index 470d98f1..ed784736 100644 --- a/controllers/telemetry_controller.go +++ b/controllers/telemetry_controller.go @@ -66,6 +66,9 @@ func (r *TelemetryReconciler) GetLogger(ctx context.Context) logr.Logger { // +kubebuilder:rbac:groups=telemetry.openstack.org,resources=loggings,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=telemetry.openstack.org,resources=loggings/status,verbs=get;update;patch // +kubebuilder:rbac:groups=telemetry.openstack.org,resources=loggings/finalizers,verbs=update;delete;patch +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkitties,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkitties/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkitties/finalizers,verbs=update;delete;patch // +kubebuilder:rbac:groups=rabbitmq.openstack.org,resources=transporturls,verbs=get;list;watch;create;update;patch;delete // Reconcile reconciles a Telemetry @@ -136,6 +139,7 @@ func (r *TelemetryReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( condition.UnknownCondition(telemetryv1.AutoscalingReadyCondition, condition.InitReason, telemetryv1.AutoscalingReadyInitMessage), condition.UnknownCondition(telemetryv1.MetricStorageReadyCondition, condition.InitReason, telemetryv1.MetricStorageReadyInitMessage), condition.UnknownCondition(telemetryv1.LoggingReadyCondition, condition.InitReason, telemetryv1.LoggingReadyInitMessage), + condition.UnknownCondition(telemetryv1.CloudKittyReadyCondition, condition.InitReason, telemetryv1.CloudKittyReadyInitMessage), ) instance.Status.Conditions.Init(&cl) @@ -220,6 +224,13 @@ func (r *TelemetryReconciler) reconcileNormal(ctx context.Context, instance *tel return ctrlResult, nil } + ctrlResult, err = r.reconcileCloudKitty(ctx, instance, helper) + if err != nil { + return ctrl.Result{}, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + // We reached the end of the Reconcile, update the Ready condition based on // the sub conditions if instance.Status.Conditions.AllSubConditionIsTrue() { @@ -548,6 +559,89 @@ func (r TelemetryReconciler) reconcileLogging(ctx context.Context, instance *tel return ctrl.Result{}, nil } +// reconcileAutoscaling ... +func (r TelemetryReconciler) reconcileCloudKitty(ctx context.Context, instance *telemetryv1.Telemetry, helper *helper.Helper) (ctrl.Result, error) { + const ( + cloudKittyNamespaceLabel = "CloudKitty.Namespace" + cloudKittyNameLabel = "CloudKitty.Name" + cloudKittyName = "cloudkitty" + ) + cloudKittyInstance := &telemetryv1.CloudKitty{ + ObjectMeta: metav1.ObjectMeta{ + Name: cloudKittyName, + Namespace: instance.Namespace, + }, + } + + if instance.Spec.CloudKitty.Enabled == nil || !*instance.Spec.CloudKitty.Enabled { + if res, err := utils.EnsureDeleted(ctx, helper, cloudKittyInstance); err != nil { + return res, err + } + instance.Status.Conditions.Remove(telemetryv1.CloudKittyReadyCondition) + return ctrl.Result{}, nil + } + + if instance.Spec.CloudKitty.NodeSelector == nil { + instance.Spec.CloudKitty.NodeSelector = instance.Spec.NodeSelector + } + + if instance.Spec.CloudKitty.TopologyRef == nil { + instance.Spec.CloudKitty.TopologyRef = instance.Spec.TopologyRef + } + + helper.GetLogger().Info("Reconciling CloudKitty", cloudKittyNamespaceLabel, instance.Namespace, cloudKittyNameLabel, cloudKittyName) + op, err := controllerutil.CreateOrPatch(ctx, helper.GetClient(), cloudKittyInstance, func() error { + instance.Spec.CloudKitty.CloudKittySpec.DeepCopyInto(&cloudKittyInstance.Spec) + + err := controllerutil.SetControllerReference(helper.GetBeforeObject(), cloudKittyInstance, helper.GetScheme()) + if err != nil { + return err + } + return nil + }) + + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + telemetryv1.CloudKittyReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + // Check the observed Generation and mirror the condition from the + // underlying resource reconciliation + autoObsGen, err := r.checkCloudKittyGeneration(instance) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + telemetryv1.CloudKittyReadyErrorMessage, + err.Error())) + return ctrl.Result{}, nil + } + if !autoObsGen { + instance.Status.Conditions.Set(condition.UnknownCondition( + telemetryv1.CloudKittyReadyCondition, + condition.InitReason, + telemetryv1.AutoscalingReadyRunningMessage, + )) + } else { + // Mirror Autoscaling condition status + c := cloudKittyInstance.Status.Conditions.Mirror(telemetryv1.CloudKittyReadyCondition) + if c != nil { + instance.Status.Conditions.Set(c) + } + } + if op != controllerutil.OperationResultNone && autoObsGen { + helper.GetLogger().Info(fmt.Sprintf("%s %s - %s", cloudKittyName, cloudKittyInstance.Name, op)) + } + + return ctrl.Result{}, nil +} + // SetupWithManager sets up the controller with the Manager. func (r *TelemetryReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). @@ -556,6 +650,7 @@ func (r *TelemetryReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&telemetryv1.Autoscaling{}). Owns(&telemetryv1.MetricStorage{}). Owns(&telemetryv1.Logging{}). + Owns(&telemetryv1.CloudKitty{}). Complete(r) } @@ -642,3 +737,24 @@ func (r *TelemetryReconciler) checkLoggingGeneration( } return true, nil } + +// checkCloudKittyGeneration - +func (r *TelemetryReconciler) checkCloudKittyGeneration( + instance *telemetryv1.Telemetry, +) (bool, error) { + Log := r.GetLogger(context.Background()) + clm := &telemetryv1.CloudKittyList{} + listOpts := []client.ListOption{ + client.InNamespace(instance.Namespace), + } + if err := r.Client.List(context.Background(), clm, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve CloudKitty CR %w") + return false, err + } + for _, item := range clm.Items { + if item.Generation != item.Status.ObservedGeneration { + return false, nil + } + } + return true, nil +} diff --git a/main.go b/main.go index 1c98beba..c5d48662 100644 --- a/main.go +++ b/main.go @@ -195,10 +195,38 @@ func main() { os.Exit(1) } + if err = (&controllers.CloudKittyReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create CloudKitty controller") + os.Exit(1) + } + + if err = (&controllers.CloudKittyAPIReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + }).SetupWithManager(context.Background(), mgr); err != nil { + setupLog.Error(err, "unable to create CloudKitty API controller") + os.Exit(1) + } + + if err = (&controllers.CloudKittyProcReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + }).SetupWithManager(context.Background(), mgr); err != nil { + setupLog.Error(err, "unable to create CloudKitty Processor controller") + os.Exit(1) + } + // Acquire environmental defaults and initialize defaults with them telemetryv1beta1.SetupDefaultsTelemetry() telemetryv1beta1.SetupDefaultsCeilometer() telemetryv1beta1.SetupDefaultsAutoscaling() + telemetryv1beta1.SetupDefaultsCloudKitty() // Setup webhooks if requested checker := healthz.Ping @@ -220,6 +248,10 @@ func main() { setupLog.Error(err, "unable to create webhook", "webhook", "MetricStorage") os.Exit(1) } + if err = (&telemetryv1beta1.CloudKitty{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "CloudKitty") + os.Exit(1) + } checker = mgr.GetWebhookServer().StartedChecker() } diff --git a/pkg/cloudkitty/common.go b/pkg/cloudkitty/common.go new file mode 100644 index 00000000..9c78b72b --- /dev/null +++ b/pkg/cloudkitty/common.go @@ -0,0 +1,161 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkitty + +import ( + "context" + "fmt" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "k8s.io/apimachinery/pkg/types" + "time" + + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +type conditionUpdater interface { + Set(c *condition.Condition) + MarkTrue(t condition.Type, messageFormat string, messageArgs ...interface{}) +} + +type topologyHandler interface { + GetSpecTopologyRef() *topologyv1.TopoRef + GetLastAppliedTopology() *topologyv1.TopoRef + SetLastAppliedTopology(t *topologyv1.TopoRef) +} + +// EnsureTopology - +func EnsureTopology( + ctx context.Context, + helper *helper.Helper, + instance topologyHandler, + finalizer string, + conditionUpdater conditionUpdater, + defaultLabelSelector metav1.LabelSelector, +) (*topologyv1.Topology, error) { + + topology, err := topologyv1.EnsureServiceTopology( + ctx, + helper, + instance.GetSpecTopologyRef(), + instance.GetLastAppliedTopology(), + finalizer, + defaultLabelSelector, + ) + if err != nil { + conditionUpdater.Set(condition.FalseCondition( + condition.TopologyReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TopologyReadyErrorMessage, + err.Error())) + return nil, fmt.Errorf("waiting for Topology requirements: %w", err) + } + // update the Status with the last retrieved Topology (or set it to nil) + instance.SetLastAppliedTopology(instance.GetSpecTopologyRef()) + // update the Topology condition only when a Topology is referenced and has + // been retrieved (err == nil) + if tr := instance.GetSpecTopologyRef(); tr != nil { + // update the TopologyRef associated condition + conditionUpdater.MarkTrue( + condition.TopologyReadyCondition, + condition.TopologyReadyMessage, + ) + } + return topology, nil +} + +// VerifyServiceSecret - ensures that the Secret object exists and the expected +// fields are in the Secret. It also sets a hash of the values of the expected +// fields passed as input. +func VerifyServiceSecret( + ctx context.Context, + secretName types.NamespacedName, + expectedFields []string, + reader client.Reader, + conditionUpdater conditionUpdater, + requeueTimeout time.Duration, + envVars *map[string]env.Setter, +) (ctrl.Result, error) { + + hash, res, err := secret.VerifySecret(ctx, secretName, expectedFields, reader, requeueTimeout) + if err != nil { + conditionUpdater.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyErrorMessage, + err.Error())) + return res, err + } else if (res != ctrl.Result{}) { + log.FromContext(ctx).Info(fmt.Sprintf("OpenStack secret %s not found", secretName)) + conditionUpdater.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.InputReadyWaitingMessage)) + return res, nil + } + (*envVars)[secretName.Name] = env.SetValue(hash) + return ctrl.Result{}, nil +} + +// VerifyConfigSecrets - It iterates over the secretNames passed as input and +// sets the hash of values in the envVars map. +func VerifyConfigSecrets( + ctx context.Context, + h *helper.Helper, + conditionUpdater conditionUpdater, + secretNames []string, + namespace string, + envVars *map[string]env.Setter, +) (ctrl.Result, error) { + var hash string + var err error + for _, secretName := range secretNames { + _, hash, err = secret.GetSecret(ctx, h, secretName, namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + log.FromContext(ctx).Info(fmt.Sprintf("Secret %s not found", secretName)) + conditionUpdater.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.InputReadyWaitingMessage)) + return ResultRequeue, nil + } + conditionUpdater.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + // Add a prefix to the var name to avoid accidental collision with other non-secret + // vars. The secret names themselves will be unique. + (*envVars)["secret-"+secretName] = env.SetValue(hash) + } + + return ctrl.Result{}, nil +} diff --git a/pkg/cloudkitty/const.go b/pkg/cloudkitty/const.go new file mode 100644 index 00000000..5fb5269a --- /dev/null +++ b/pkg/cloudkitty/const.go @@ -0,0 +1,54 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkitty + +import ( + "time" + + ctrl "sigs.k8s.io/controller-runtime" +) + +const ( + // ServiceName - + ServiceName = "cloudkitty" + // ServiceType - + ServiceType = "rating" + // DatabaseName - + DatabaseName = "cloudkitty" + + // DefaultsConfigFileName - + DefaultsConfigFileName = "cloudkitty.conf" + // ServiceConfigFileName - + ServiceConfigFileName = "01-service-defaults.conf" + // CustomConfigFileName - + CustomConfigFileName = "02-global-custom.conf" + // CustomServiceConfigFileName - + CustomServiceConfigFileName = "03-service-custom.conf" + // CustomServiceConfigSecretsFileName - + CustomServiceConfigSecretsFileName = "04-service-custom-secrets.conf" + // MyCnfFileName - + MyCnfFileName = "my.cnf" + + // CloudKittyPublicPort - + CloudKittyPublicPort int32 = 8889 + // CloudKittyInternalPort - + CloudKittyInternalPort int32 = 8889 + + ShortDuration = time.Duration(5) * time.Second + NormalDuration = time.Duration(10) * time.Second +) + +var ResultRequeue = ctrl.Result{RequeueAfter: NormalDuration} diff --git a/pkg/cloudkitty/dbsync.go b/pkg/cloudkitty/dbsync.go new file mode 100644 index 00000000..75a675e4 --- /dev/null +++ b/pkg/cloudkitty/dbsync.go @@ -0,0 +1,107 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cloudkitty + +import ( + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // DBSyncCommand - + // TODO: Once we work on update/upgrades revisit the command in the + // the cloudkitty-dbsync-config.json file. + // If we stop all services during the update/upgrade then we can keep + // the --bump-versions flag. + // If we are doing rolling upgrades we'll need to use the flag + // conditionally (only for adoption) and do the restart cycle of + // services as described in the upstream rolling upgrades process. + dbSyncCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start" +) + +// DbSyncJob func +func DbSyncJob(instance *telemetryv1.CloudKitty, labels map[string]string) *batchv1.Job { + args := []string{"-c"} + args = append(args, dbSyncCommand) + + // create Volume and VolumeMounts + volumes := GetVolumes("cloudkitty") + volumeMounts := GetVolumeMounts("cloudkitty-dbsync") + // add CA cert if defined + if instance.Spec.CloudKittyAPI.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.CloudKittyAPI.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.CloudKittyAPI.TLS.CreateVolumeMounts(nil)...) + } + + runAsUser := int64(0) + envVars := map[string]env.Setter{} + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["KOLLA_BOOTSTRAP"] = env.SetValue("TRUE") + cloudKittyPassword := []corev1.EnvVar{ + { + Name: "AodhPassword", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: instance.Spec.Secret, + }, + Key: instance.Spec.PasswordSelectors.CloudKittyService, + }, + }, + }, + } + + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: ServiceName + "-db-sync", + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + ServiceAccountName: instance.RbacResourceName(), + Containers: []corev1.Container{ + { + Name: ServiceName + "-db-sync", + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: instance.Spec.CloudKittyAPI.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &runAsUser, + }, + Env: env.MergeEnvs(cloudKittyPassword, envVars), + VolumeMounts: volumeMounts, + }, + }, + Volumes: volumes, + }, + }, + }, + } + + if instance.Spec.NodeSelector != nil { + job.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector + } + + return job +} diff --git a/pkg/cloudkitty/funcs.go b/pkg/cloudkitty/funcs.go new file mode 100644 index 00000000..43210dad --- /dev/null +++ b/pkg/cloudkitty/funcs.go @@ -0,0 +1,49 @@ +package cloudkitty + +import ( + common "github.com/openstack-k8s-operators/lib-common/modules/common" + "github.com/openstack-k8s-operators/lib-common/modules/common/affinity" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// GetOwningCloudKittyName - Given a CloudKittyAPI or CloudKittyProc +// object, returning the parent CloudKitty object that created it (if any) +func GetOwningCloudKittyName(instance client.Object) string { + for _, ownerRef := range instance.GetOwnerReferences() { + if ownerRef.Kind == "CloudKitty" { + return ownerRef.Name + } + } + + return "" +} + +// GetNetworkAttachmentAddrs - Returns a list of IP addresses for all network attachments. +func GetNetworkAttachmentAddrs(namespace string, networkAttachments []string, networkAttachmentStatus map[string][]string) []string { + networkAttachmentAddrs := []string{} + + for _, network := range networkAttachments { + networkName := namespace + "/" + network + if networkAddrs, ok := networkAttachmentStatus[networkName]; ok { + networkAttachmentAddrs = append(networkAttachmentAddrs, networkAddrs...) + } + } + + return networkAttachmentAddrs +} + +// GetPodAffinity - Returns a corev1.Affinity reference for the specified component. +func GetPodAffinity(componentName string) *corev1.Affinity { + // If possible two pods of the same component (e.g cloudkitty-api) should not + // run on the same worker node. If this is not possible they get still + // created on the same worker node. + return affinity.DistributePods( + common.ComponentSelector, + []string{ + componentName, + }, + corev1.LabelHostname, + ) +} diff --git a/pkg/cloudkitty/volumes.go b/pkg/cloudkitty/volumes.go new file mode 100644 index 00000000..97ed4ab5 --- /dev/null +++ b/pkg/cloudkitty/volumes.go @@ -0,0 +1,57 @@ +package cloudkitty + +import ( + corev1 "k8s.io/api/core/v1" +) + +var ( + // scriptMode is the default permissions mode for Scripts volume + scriptMode int32 = 0740 + // configMode is the 640 permissions mode + configMode int32 = 0640 +) + +// GetVolumes - service volumes +func GetVolumes(name string) []corev1.Volume { + return []corev1.Volume{ + { + Name: "scripts", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &scriptMode, + SecretName: name + "-scripts", + }, + }, + }, { + Name: "config-data", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &configMode, + SecretName: name + "-config-data", + }, + }, + }, + } +} + +// GetVolumeMounts - general VolumeMounts +func GetVolumeMounts(serviceName string) []corev1.VolumeMount { + return []corev1.VolumeMount{ + { + Name: "scripts", + MountPath: "/var/lib/openstack/bin", + ReadOnly: true, + }, + { + Name: "config-data", + MountPath: "/var/lib/openstack/config", + ReadOnly: true, + }, + { + Name: "config-data", + MountPath: "/var/lib/kolla/config_files/config.json", + SubPath: serviceName + "-config.json", + ReadOnly: true, + }, + } +} diff --git a/pkg/cloudkittyapi/const.go b/pkg/cloudkittyapi/const.go new file mode 100644 index 00000000..7d9a378b --- /dev/null +++ b/pkg/cloudkittyapi/const.go @@ -0,0 +1,24 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkittyapi + +const ( + // ComponentName - + ComponentName = "cloudkitty-api" + + //LogFile - + LogFile = "/var/log/cloudkitty/cloudkitty-api.log" +) diff --git a/pkg/cloudkittyapi/statefulset.go b/pkg/cloudkittyapi/statefulset.go new file mode 100644 index 00000000..94d4346c --- /dev/null +++ b/pkg/cloudkittyapi/statefulset.go @@ -0,0 +1,188 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkittyapi + +import ( + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + // ServiceCommand - + ServiceCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start" +) + +// StatefulSet func +func StatefulSet( + instance *telemetryv1.CloudKittyAPI, + configHash string, + labels map[string]string, + annotations map[string]string, + topology *topologyv1.Topology, +) (*appsv1.StatefulSet, error) { + runAsUser := int64(0) + //cloudKittyUser := int64(telemetryv1.CloudKittyUserID) + + livenessProbe := &corev1.Probe{ + // TODO might need tuning + TimeoutSeconds: 5, + PeriodSeconds: 3, + InitialDelaySeconds: 5, + } + readinessProbe := &corev1.Probe{ + // TODO might need tuning + TimeoutSeconds: 5, + PeriodSeconds: 5, + InitialDelaySeconds: 5, + } + + args := []string{"-c", ServiceCommand} + // + // https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ + // + livenessProbe.HTTPGet = &corev1.HTTPGetAction{ + Path: "/healthcheck", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: int32(cloudkitty.CloudKittyPublicPort)}, + } + readinessProbe.HTTPGet = livenessProbe.HTTPGet + + if instance.Spec.TLS.API.Enabled(service.EndpointPublic) { + livenessProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS + readinessProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS + } + + // create Volume and VolumeMounts + volumes := GetVolumes(cloudkitty.GetOwningCloudKittyName(instance), instance.Name) + volumeMounts := GetVolumeMounts(instance.Name) + + // add CA cert if defined + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + + for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} { + if instance.Spec.TLS.API.Enabled(endpt) { + var tlsEndptCfg tls.GenericService + switch endpt { + case service.EndpointPublic: + tlsEndptCfg = instance.Spec.TLS.API.Public + case service.EndpointInternal: + tlsEndptCfg = instance.Spec.TLS.API.Internal + } + + svc, err := tlsEndptCfg.ToService() + if err != nil { + return nil, err + } + volumes = append(volumes, svc.CreateVolume(endpt.String())) + volumeMounts = append(volumeMounts, svc.CreateVolumeMounts(endpt.String())...) + } + } + + envVars := map[string]env.Setter{} + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["CONFIG_HASH"] = env.SetValue(configHash) + + statefulset := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Name, + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: appsv1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + PodManagementPolicy: appsv1.ParallelPodManagement, + Replicas: instance.Spec.Replicas, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + Labels: labels, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: instance.Spec.ServiceAccount, + Containers: []corev1.Container{ + // the first container in a pod is the default selected + // by oc log so define the log stream container first. + { + Name: instance.Name + "-log", + Command: []string{ + "/usr/bin/dumb-init", + }, + Args: []string{ + "--single-child", + "--", + "/bin/sh", + "-c", + "/usr/bin/tail -n+1 -F " + LogFile + " 2>/dev/null", + }, + Image: instance.Spec.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &runAsUser, + }, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: []corev1.VolumeMount{GetLogVolumeMount()}, + Resources: instance.Spec.Resources, + }, + { + Name: ComponentName, + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: instance.Spec.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &runAsUser, + }, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + Resources: instance.Spec.Resources, + ReadinessProbe: readinessProbe, + LivenessProbe: livenessProbe, + }, + }, + Volumes: volumes, + }, + }, + }, + } + + if instance.Spec.NodeSelector != nil { + statefulset.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector + } + + if topology != nil { + topology.ApplyTo(&statefulset.Spec.Template) + } else { + // If possible two pods of the same service should not + // run on the same worker node. If this is not possible + // the get still created on the same worker node. + statefulset.Spec.Template.Spec.Affinity = cloudkitty.GetPodAffinity(ComponentName) + } + + return statefulset, nil +} diff --git a/pkg/cloudkittyapi/volumes.go b/pkg/cloudkittyapi/volumes.go new file mode 100644 index 00000000..6d9f36ab --- /dev/null +++ b/pkg/cloudkittyapi/volumes.go @@ -0,0 +1,54 @@ +package cloudkittyapi + +import ( + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + corev1 "k8s.io/api/core/v1" +) + +// GetVolumes - +func GetVolumes(parentName string, name string) []corev1.Volume { + var config0644AccessMode int32 = 0644 + + volumes := []corev1.Volume{ + { + Name: "config-data-custom", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &config0644AccessMode, + SecretName: name + "-config-data", + }, + }, + }, + { + Name: "logs", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{Medium: ""}, + }, + }, + } + + return append(cloudkitty.GetVolumes(parentName), volumes...) +} + +// GetVolumeMounts - CloudKitty API VolumeMounts +func GetVolumeMounts(parentName string) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{ + { + Name: "config-data-custom", + MountPath: "/etc/cloudkitty/cloudkitty.conf.d", + ReadOnly: true, + }, + GetLogVolumeMount(), + } + + return append(cloudkitty.GetVolumeMounts(parentName), volumeMounts...) +} + +// GetLogVolumeMount - CloudKitty API LogVolumeMount +func GetLogVolumeMount() corev1.VolumeMount { + return corev1.VolumeMount{ + Name: "logs", + MountPath: "/var/log/cloudkitty", + ReadOnly: false, + } +} diff --git a/pkg/cloudkittyproc/const.go b/pkg/cloudkittyproc/const.go new file mode 100644 index 00000000..8bda7978 --- /dev/null +++ b/pkg/cloudkittyproc/const.go @@ -0,0 +1,21 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkittyproc + +const ( + // ComponentName - + ComponentName = "cloudkitty-proc" +) diff --git a/pkg/cloudkittyproc/statefulset.go b/pkg/cloudkittyproc/statefulset.go new file mode 100644 index 00000000..3071b60e --- /dev/null +++ b/pkg/cloudkittyproc/statefulset.go @@ -0,0 +1,153 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkittyproc + +import ( + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + // ServiceCommand - + ServiceCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start" +) + +// StatefulSet func +func StatefulSet( + instance *telemetryv1.CloudKittyProc, + configHash string, + labels map[string]string, + annotations map[string]string, + topology *topologyv1.Topology, +) *appsv1.StatefulSet { + cloudKittyUser := int64(0) + // cloudKittyUser := int64(telemetryv1.CloudKittyUserID) + // cloudKittyGroup := int64(telemetryv1.CloudKittyGroupID) + + // TODO until we determine how to properly query for these + livenessProbe := &corev1.Probe{ + // TODO might need tuning + TimeoutSeconds: 5, + PeriodSeconds: 3, + InitialDelaySeconds: 3, + } + + startupProbe := &corev1.Probe{ + TimeoutSeconds: 5, + FailureThreshold: 12, + PeriodSeconds: 5, + InitialDelaySeconds: 5, + } + + args := []string{"-c", ServiceCommand} + var probeCommand []string + livenessProbe.HTTPGet = &corev1.HTTPGetAction{ + Port: intstr.FromInt(8080), + } + startupProbe.HTTPGet = livenessProbe.HTTPGet + probeCommand = []string{ + "/usr/local/bin/container-scripts/healthcheck.py", + "processor", + "/etc/cloudkitty/cloudkitty.conf.d", + } + + envVars := map[string]env.Setter{} + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["CONFIG_HASH"] = env.SetValue(configHash) + + volumes := GetVolumes(cloudkitty.GetOwningCloudKittyName(instance), instance.Name) + volumeMounts := GetVolumeMounts(instance.Name) + + // Add the CA bundle + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + + statefulset := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Name, + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: appsv1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Replicas: instance.Spec.Replicas, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + Labels: labels, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: instance.Spec.ServiceAccount, + Containers: []corev1.Container{ + { + Name: ComponentName, + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: instance.Spec.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &cloudKittyUser, + }, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + Resources: instance.Spec.Resources, + LivenessProbe: livenessProbe, + StartupProbe: startupProbe, + }, + { + Name: "probe", + Command: probeCommand, + Image: instance.Spec.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &cloudKittyUser, + //RunAsGroup: &cloudKittyGroup, + }, + VolumeMounts: volumeMounts, + }, + }, + Volumes: volumes, + }, + }, + }, + } + + if instance.Spec.NodeSelector != nil { + statefulset.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector + } + + if topology != nil { + topology.ApplyTo(&statefulset.Spec.Template) + } else { + // If possible two pods of the same service should not + // run on the same worker node. If this is not possible + // the get still created on the same worker node. + statefulset.Spec.Template.Spec.Affinity = cloudkitty.GetPodAffinity(ComponentName) + } + + return statefulset +} diff --git a/pkg/cloudkittyproc/volumes.go b/pkg/cloudkittyproc/volumes.go new file mode 100644 index 00000000..51ff46a7 --- /dev/null +++ b/pkg/cloudkittyproc/volumes.go @@ -0,0 +1,38 @@ +package cloudkittyproc + +import ( + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + corev1 "k8s.io/api/core/v1" +) + +// GetVolumes - +func GetVolumes(parentName string, name string) []corev1.Volume { + var config0644AccessMode int32 = 0644 + + volumes := []corev1.Volume{ + { + Name: "config-data-custom", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &config0644AccessMode, + SecretName: name + "-config-data", + }, + }, + }, + } + + return append(cloudkitty.GetVolumes(parentName), volumes...) +} + +// GetVolumeMounts - CloudKitty API VolumeMounts +func GetVolumeMounts(parentName string) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{ + { + Name: "config-data-custom", + MountPath: "/etc/cloudkitty/cloudkitty.conf.d", + ReadOnly: true, + }, + } + + return append(cloudkitty.GetVolumeMounts(parentName), volumeMounts...) +} diff --git a/templates/cloudkitty/bin/healthcheck.py b/templates/cloudkitty/bin/healthcheck.py new file mode 100755 index 00000000..4bce1182 --- /dev/null +++ b/templates/cloudkitty/bin/healthcheck.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# +# Copyright 2022 Red Hat Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Trivial HTTP server to check health of scheduler, backup and volume services. +# Cinder-API hast its own health check endpoint and does not need this. +# +# The only check this server currently does is using the heartbeat in the +# database service table, accessing the DB directly here using cinder's +# configuration options. +# +# The benefit of accessing the DB directly is that it doesn't depend on the +# Cinder-API service being up and we can also differentiate between the +# container not having a connection to the DB and the cinder service not doing +# the heartbeats. +# +# For volume services all enabled backends must be up to return 200, so it is +# recommended to use a different pod for each backend to avoid one backend +# affecting others. +# +# Requires the name of the service as the first argument (volume, backup, +# scheduler) and optionally a second argument with the location of the +# configuration directory (defaults to /etc/cinder/cinder.conf.d) + +from http import server +import signal +import socket +import sys +import time +import threading + +from oslo_config import cfg + +SERVER_PORT = 8080 +CONF = cfg.CONF + +class HTTPServerV6(server.HTTPServer): + address_family = socket.AF_INET6 + +class HeartbeatServer(server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + self.wfile.write('OK'.encode('utf-8')) + + +def get_stopper(server): + def stopper(signal_number=None, frame=None): + print("Stopping server.") + server.shutdown() + server.server_close() + print("Server stopped.") + sys.exit(0) + return stopper + + +if __name__ == "__main__": + hostname = socket.gethostname() + ipv6_address = socket.getaddrinfo(hostname, None, socket.AF_INET6) + if ipv6_address: + webServer = HTTPServerV6(("::",SERVER_PORT), HeartbeatServer) + else: + webServer = server.HTTPServer(("0.0.0.0", SERVER_PORT), HeartbeatServer) + stop = get_stopper(webServer) + + # Need to run the server on a different thread because its shutdown method + # will block if called from the same thread, and the signal handler must be + # on the main thread in Python. + thread = threading.Thread(target=webServer.serve_forever) + thread.daemon = True + thread.start() + print(f"CloudKitty Healthcheck Server started http://{hostname}:{SERVER_PORT}") + signal.signal(signal.SIGTERM, stop) + + try: + while True: + time.sleep(60) + except KeyboardInterrupt: + pass + finally: + stop() diff --git a/templates/cloudkitty/bin/run-on-host b/templates/cloudkitty/bin/run-on-host new file mode 100755 index 00000000..e7840ace --- /dev/null +++ b/templates/cloudkitty/bin/run-on-host @@ -0,0 +1,2 @@ +#!/bin/sh +exec nsenter -a -t 1 -- `realpath -s $0` "$@" diff --git a/templates/cloudkitty/config/10-cloudkitty_wsgi.conf b/templates/cloudkitty/config/10-cloudkitty_wsgi.conf new file mode 100644 index 00000000..1fc084ae --- /dev/null +++ b/templates/cloudkitty/config/10-cloudkitty_wsgi.conf @@ -0,0 +1,40 @@ +{{ range $endpt, $vhost := .VHosts }} +# {{ $endpt }} vhost {{ $vhost.ServerName }} configuration + + ServerName {{ $vhost.ServerName }} + + ## Vhost docroot + DocumentRoot "/var/www/cgi-bin/cloudkitty" + + ## Directories, there should at least be a declaration for /var/www/cgi-bin/cloudkitty + + + Options -Indexes +FollowSymLinks +MultiViews + AllowOverride None + Require all granted + + + Timeout {{ $.TimeOut }} + + ## Logging + ErrorLog /dev/stdout + ServerSignature Off + CustomLog /dev/stdout combined + +{{- if $vhost.TLS }} + SetEnvIf X-Forwarded-Proto https HTTPS=1 + + ## SSL directives + SSLEngine on + SSLCertificateFile "{{ $vhost.SSLCertificateFile }}" + SSLCertificateKeyFile "{{ $vhost.SSLCertificateKeyFile }}" +{{- end }} + + ## WSGI configuration + WSGIApplicationGroup %{GLOBAL} + WSGIDaemonProcess {{ $endpt }} display-name={{ $endpt }} group=cloudkitty processes=4 threads=1 user=cloudkitty + WSGIProcessGroup {{ $endpt }} + WSGIScriptAlias / "/var/www/cgi-bin/cloudkitty/cloudkitty-wsgi" + WSGIPassAuthorization On + +{{ end }} diff --git a/templates/cloudkitty/config/cloudkitty-api-config-httpd.json b/templates/cloudkitty/config/cloudkitty-api-config-httpd.json new file mode 100644 index 00000000..174349bf --- /dev/null +++ b/templates/cloudkitty/config/cloudkitty-api-config-httpd.json @@ -0,0 +1,51 @@ +{ + "command": "/usr/sbin/httpd -DFOREGROUND", + "config_files": [ + { + "source": "/var/lib/openstack/config/httpd.conf", + "dest": "/etc/httpd/conf/httpd.conf", + "owner": "root", + "perm": "0644" + }, + { + "source": "/var/lib/openstack/config/10-cloudkitty_wsgi.conf", + "dest": "/etc/httpd/conf.d/10-cloudkitty_wsgi.conf", + "owner": "root", + "perm": "0644" + }, + { + "source": "/var/lib/openstack/config/ssl.conf", + "dest": "/etc/httpd/conf.d/ssl.conf", + "owner": "cloudkitty", + "perm": "0644" + }, + { + "source": "/var/lib/config-data/tls/certs/*", + "dest": "/etc/pki/tls/certs/", + "owner": "cloudkitty", + "perm": "0640", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/config-data/tls/private/*", + "dest": "/etc/pki/tls/private/", + "owner": "cloudkitty", + "perm": "0600", + "optional": true, + "merge": true + } + ], + "permissions": [ + { + "path": "/var/log/cloudkitty", + "owner": "cloudkitty:apache", + "recurse": true + }, + { + "path": "/etc/httpd/run", + "owner": "cloudkitty:apache", + "recurse": true + } + ] +} diff --git a/templates/cloudkitty/config/cloudkitty-api-config.json b/templates/cloudkitty/config/cloudkitty-api-config.json new file mode 100644 index 00000000..11a6f291 --- /dev/null +++ b/templates/cloudkitty/config/cloudkitty-api-config.json @@ -0,0 +1,11 @@ +{ + "command": "/usr/bin/uwsgi --ini /etc/cloudkitty/cloudkitty-api-uwsgi.ini", + "config_files": [ + { + "source": "/var/lib/openstack/config/cloudkitty-api-uwsgi.ini", + "dest": "/etc/cloudkitty/cloudkitty-api-uwsgi.ini", + "owner": "cloudkitty", + "perm": "0600" + } + ] +} diff --git a/templates/cloudkitty/config/cloudkitty-api-uwsgi.ini b/templates/cloudkitty/config/cloudkitty-api-uwsgi.ini new file mode 100644 index 00000000..2cd8f602 --- /dev/null +++ b/templates/cloudkitty/config/cloudkitty-api-uwsgi.ini @@ -0,0 +1,17 @@ +[uwsgi] +chmod-socket = 666 +socket = /var/run/uwsgi/cloudkitty.socket +start-time = %t +lazy-apps = true +add-header = Connection: close +buffer-size = 65535 +hook-master-start = unix_signal:15 gracefully_kill_them_all +thunder-lock = true +plugins = http,python3 +enable-threads = true +worker-reload-mercy = 80 +exit-on-reload = false +die-on-term = true +master = true +processes = 2 +wsgi-file = /opt/stack/data/venv/bin/cloudkitty-api \ No newline at end of file diff --git a/templates/cloudkitty/config/cloudkitty-dbsync-config.json b/templates/cloudkitty/config/cloudkitty-dbsync-config.json new file mode 100644 index 00000000..1eb4f0a6 --- /dev/null +++ b/templates/cloudkitty/config/cloudkitty-dbsync-config.json @@ -0,0 +1,11 @@ +{ + "command": "/usr/bin/cloudkitty-dbsync upgrade", + "config_files": [ + { + "source": "/var/lib/openstack/config/cloudkitty.conf", + "dest": "/etc/cloudkitty/cloudkitty.conf", + "owner": "cloudkitty", + "perm": "0600" + } + ] +} diff --git a/templates/cloudkitty/config/cloudkitty-proc-config.json b/templates/cloudkitty/config/cloudkitty-proc-config.json new file mode 100644 index 00000000..410ed9d3 --- /dev/null +++ b/templates/cloudkitty/config/cloudkitty-proc-config.json @@ -0,0 +1,17 @@ +{ + "command": "/usr/bin/cloudkitty-processor --logfile /dev/stdout", + "config_files": [ + { + "source": "/var/lib/openstack/config/cloudkitty.conf", + "dest": "/etc/cloudkitty/cloudkitty.conf", + "owner": "cloudkitty", + "perm": "0600" + }, + { + "source": "/var/lib/openstack/config/metrics.yaml", + "dest": "/etc/cloudkitty/metrics.yaml", + "owner": "cloudkitty", + "perm": "0600" + } + ] +} diff --git a/templates/cloudkitty/config/cloudkitty.conf b/templates/cloudkitty/config/cloudkitty.conf new file mode 100644 index 00000000..e9896eb2 --- /dev/null +++ b/templates/cloudkitty/config/cloudkitty.conf @@ -0,0 +1,78 @@ +[oslo_policy] +policy_file = policy.yaml + +[DEFAULT] +auth_strategy = keystone +debug = True +notification_topics = notifications +transport_url = {{ .TransportURL }} + +[authinfos] +debug = True +project_domain_name = default +user_domain_name = default +region_name = RegionOne +tenant_name = service +project_name = service +password = {{ .ServicePassword }} +username = {{ .ServiceUser }} +identity_uri = {{ .KeystoneInternalURL }} +auth_url = {{ .KeystoneInternalURL }} +auth_protocol = http +auth_type = v3password +{{- if .TLS }} +cafile = {{ .CAFile }} +{{- end }} + +[fetcher] +backend = keystone + +[fetcher_keystone] +auth_section = authinfos + +[storage_influxdb] +port = 8086 +host = localhost +database = cloudkitty +password = cloudkitty +user = cloudkitty + +[collect] +period = 300 +wait_periods = 0 +metrics_conf = /etc/cloudkitty/metrics.yaml +collector = prometheus +scope_key = project + +[collector_prometheus] +prometheus_url = https://metric-storage-prometheus.openstack.svc:9090 +insecure = true + +[output] +pipeline = osrf +basepath = /opt/stack/data/cloudkitty/reports +backend = cloudkitty.backend.file.FileBackend + +[storage] +version = 2 +backend = influxdb + +[database] +connection = {{ .DatabaseConnection }} + +[keystone_authtoken] +memcached_servers = {{ .MemcachedServersWithInet }} +# memcache_pool_dead_retry = 10 +# memcache_pool_conn_get_timeout = 2 +project_domain_name = Default +project_name = service +user_domain_name = Default +password = {{ .ServicePassword }} +username = {{ .ServiceUser }} +auth_url = {{ .KeystoneInternalURL }} +interface = internal +auth_type = password +{{- if .TLS }} +cafile = {{ .CAFile }} +{{- end }} +# service_token_roles_required = true diff --git a/templates/cloudkitty/config/httpd.conf b/templates/cloudkitty/config/httpd.conf new file mode 100644 index 00000000..b6c862a5 --- /dev/null +++ b/templates/cloudkitty/config/httpd.conf @@ -0,0 +1,27 @@ +ServerTokens Prod +ServerSignature Off +TraceEnable Off +ServerRoot "/etc/httpd" +ServerName "cloudkitty.openstack.svc" + +User apache +Group apache + +Listen 8889 + +TypesConfig /etc/mime.types + +Include conf.modules.d/*.conf + +LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined +LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" proxy + +SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded +CustomLog /dev/stdout combined env=!forwarded +CustomLog /dev/stdout proxy env=forwarded +ErrorLog /dev/stdout + +# XXX: To disable SSL +#Include conf.d/*.conf +# If above include is commented include at least the cloudkitty wsgi file +Include conf.d/10-cloudkitty_wsgi.conf diff --git a/templates/cloudkitty/config/metrics.yaml b/templates/cloudkitty/config/metrics.yaml new file mode 100644 index 00000000..c42701ed --- /dev/null +++ b/templates/cloudkitty/config/metrics.yaml @@ -0,0 +1,77 @@ +metrics: + ceilometer_cpu: + unit: instance + alt_name: instance + groupby: + - id + - user_id + - project + metadata: + - flavor_name + - flavor_id + - vcpus + mutate: NUMBOOL + extra_args: + aggregation_method: max + + ceilometer_image_size: + unit: MiB + factor: 1/1048576 + groupby: + - id + - user_id + - project + metadata: + - container_format + - disk_format + extra_args: + aggregation_method: max + + ceilometer_volume_size: + unit: GiB + groupby: + - id + - user_id + - project + metadata: + - volume_type + extra_args: + aggregation_method: max + + ceilometer_network_outgoing_bytes_rate: + unit: MB + groupby: + - id + - project + - user_id + # Converting B/s to MB/h + factor: 3600/1000000 + metadata: + - instance_id + extra_args: + aggregation_method: max + + ceilometer_network_incoming_bytes_rate: + unit: MB + groupby: + - id + - project + - user_id + # Converting B/s to MB/h + factor: 3600/1000000 + metadata: + - instance_id + extra_args: + aggregation_method: max + + ceilometer_ip_floating: + unit: ip + groupby: + - id + - user_id + - project + metadata: + - state + mutate: NUMBOOL + extra_args: + aggregation_method: max \ No newline at end of file diff --git a/templates/cloudkitty/config/ssl.conf b/templates/cloudkitty/config/ssl.conf new file mode 100644 index 00000000..e3da4ecb --- /dev/null +++ b/templates/cloudkitty/config/ssl.conf @@ -0,0 +1,21 @@ + + SSLRandomSeed startup builtin + SSLRandomSeed startup file:/dev/urandom 512 + SSLRandomSeed connect builtin + SSLRandomSeed connect file:/dev/urandom 512 + + AddType application/x-x509-ca-cert .crt + AddType application/x-pkcs7-crl .crl + + SSLPassPhraseDialog builtin + SSLSessionCache "shmcb:/var/cache/mod_ssl/scache(512000)" + SSLSessionCacheTimeout 300 + Mutex default + SSLCryptoDevice builtin + SSLHonorCipherOrder On + SSLUseStapling Off + SSLStaplingCache "shmcb:/run/httpd/ssl_stapling(32768)" + SSLCipherSuite HIGH:MEDIUM:!aNULL:!MD5:!RC4:!3DES + SSLProtocol all -SSLv2 -SSLv3 -TLSv1 + SSLOptions StdEnvVars + From 294361aa7ced46ced7ebc66704f740443e07bdb3 Mon Sep 17 00:00:00 2001 From: jlarriba Date: Mon, 30 Jun 2025 15:17:26 +0200 Subject: [PATCH 02/12] CloudKitty TLS --- .../telemetry.openstack.org_cloudkitties.yaml | 11 ++ ...lemetry.openstack.org_cloudkittyprocs.yaml | 3 + .../telemetry.openstack.org_telemetries.yaml | 12 ++ api/v1beta1/cloudkitty_types.go | 6 +- api/v1beta1/cloudkittyproc_types.go | 10 +- api/v1beta1/conditions.go | 18 +++ api/v1beta1/zz_generated.deepcopy.go | 2 +- .../telemetry.openstack.org_cloudkitties.yaml | 11 ++ ...lemetry.openstack.org_cloudkittyprocs.yaml | 3 + .../telemetry.openstack.org_telemetries.yaml | 12 ++ controllers/cloudkitty_controller.go | 57 +++++++++- pkg/cloudkitty/dbsync.go | 2 +- pkg/cloudkitty/storageinit.go | 106 ++++++++++++++++++ pkg/cloudkittyproc/statefulset.go | 23 ++-- .../config/cloudkitty-api-config-httpd.json | 51 --------- .../config/cloudkitty-api-config.json | 69 +++++++++++- .../config/cloudkitty-storageinit-config.json | 11 ++ templates/cloudkitty/config/cloudkitty.conf | 10 +- templates/cloudkitty/config/httpd.conf | 2 +- ...udkitty_wsgi.conf => wsgi-cloudkitty.conf} | 2 +- 20 files changed, 335 insertions(+), 86 deletions(-) create mode 100644 pkg/cloudkitty/storageinit.go delete mode 100644 templates/cloudkitty/config/cloudkitty-api-config-httpd.json create mode 100644 templates/cloudkitty/config/cloudkitty-storageinit-config.json rename templates/cloudkitty/config/{10-cloudkitty_wsgi.conf => wsgi-cloudkitty.conf} (94%) diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml index bc5b7567..387bf62d 100644 --- a/api/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -450,6 +450,17 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in + a pre-created bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string + type: object topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml index 9cd9b6ef..6123f53c 100644 --- a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml +++ b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -196,6 +196,9 @@ spec: description: CaBundleSecretName - holding the CA certs in a pre-created bundle file type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string type: object topologyRef: description: |- diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index 0d8bf4d3..f69ad6d9 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -1008,6 +1008,18 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs + in a pre-created bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index 08181d6c..1d2d1a7e 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -32,11 +32,13 @@ const ( CloudKittyGroupID = 42408 // CloudKittyAPIContainerImage - default fall-back image for CloudKitty API - CloudKittyAPIContainerImage = "quay.io/podified-master-centos9/cloudkitty-api:current-podified" + CloudKittyAPIContainerImage = "quay.io/podified-master-centos9/openstack-cloudkitty-api:current-podified" // CloudKittyProcContainerImage - default fall-back image for CloudKitty Processor - CloudKittyProcContainerImage = "quay.io/podified-master-centos9/cloudkitty-processor:current-podified" + CloudKittyProcContainerImage = "quay.io/podified-master-centos9/openstack-cloudkitty-processor:current-podified" // CloudKittyDbSyncHash hash CKDbSyncHash = "ckdbsync" + // CKStorageInitHash hash + CKStorageInitHash = "ckstorageinit" // CloudKittyReplicas - The number of replicas per each service deployed CloudKittyReplicas = 1 ) diff --git a/api/v1beta1/cloudkittyproc_types.go b/api/v1beta1/cloudkittyproc_types.go index d2459afd..d7b80e19 100644 --- a/api/v1beta1/cloudkittyproc_types.go +++ b/api/v1beta1/cloudkittyproc_types.go @@ -33,6 +33,11 @@ type CloudKittyProcTemplateCore struct { // +kubebuilder:validation:Minimum=0 // Replicas - CloudKitty API Replicas Replicas *int32 `json:"replicas"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.SimpleService `json:"tls,omitempty"` } // CloudKittyProcTemplate defines the input parameters for the CloudKitty Processor service @@ -63,11 +68,6 @@ type CloudKittyProcSpec struct { // +kubebuilder:validation:Required // ServiceAccount - service account name used internally to provide CloudKitty services the default SA name ServiceAccount string `json:"serviceAccount"` - - // +kubebuilder:validation:Optional - // +operator-sdk:csv:customresourcedefinitions:type=spec - // TLS - Parameters related to the TLS - TLS tls.Ca `json:"tls,omitempty"` } // CloudKittyProcStatus defines the observed state of CloudKitty Processor diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index f8e0b4b7..709bd1cd 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -54,6 +54,9 @@ const ( // CloudKittyProcReadyCondition Status=True condition which indicates if the CloudKitty Processor is configured and operational CloudKittyProcReadyCondition condition.Type = "CloudKittyProcReady" + // CloudKittyStorageInitReadyCondition Status=True condition which indicates if the CloudKitty Storage Init process has ran + CloudKittyStorageInitReadyCondition condition.Type = "CloudKittyStorageInitReady" + // LoggingCLONamespaceReadyCondition Status=True condition which indicates if the cluster-logging-operator namespace is created LoggingCLONamespaceReadyCondition condition.Type = "LoggingCLONamespaceReady" @@ -223,6 +226,21 @@ const ( // CloudKittyReadyErrorMessage CloudKittyReadyErrorMessage = "CloudKitty error occured %s" + // + // CloudKittyStorageInit condition messages + // + // CloudKittyStorageInitReadyInitMessage + CloudKittyStorageInitReadyInitMessage = "CloudKittyStorageInit not started" + + // CloudKittyStorageInitReadyMessage + CloudKittyStorageInitReadyMessage = "CloudKittyStorageInit completed" + + // CloudKittyStorageInitReadyRunning + CloudKittyStorageInitReadyRunningMessage = "CloudKittyStorageInit job still running" + + // CloudKittyStorageInitReadyErrorMessage + CloudKittyStorageInitReadyErrorMessage = "CloudKittyStorageInit job error occurred %s" + // // CloudKittyAPIReady condition messages // diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 1c5006d6..c9d622b6 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -906,7 +906,6 @@ func (in *CloudKittyProcSpec) DeepCopyInto(out *CloudKittyProcSpec) { *out = *in out.CloudKittyTemplate = in.CloudKittyTemplate in.CloudKittyProcTemplate.DeepCopyInto(&out.CloudKittyProcTemplate) - out.TLS = in.TLS } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProcSpec. @@ -994,6 +993,7 @@ func (in *CloudKittyProcTemplateCore) DeepCopyInto(out *CloudKittyProcTemplateCo *out = new(int32) **out = **in } + in.TLS.DeepCopyInto(&out.TLS) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProcTemplateCore. diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml index bc5b7567..387bf62d 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -450,6 +450,17 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in + a pre-created bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string + type: object topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml index 9cd9b6ef..6123f53c 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -196,6 +196,9 @@ spec: description: CaBundleSecretName - holding the CA certs in a pre-created bundle file type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string type: object topologyRef: description: |- diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index 0d8bf4d3..f69ad6d9 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -1008,6 +1008,18 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs + in a pre-created bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index 4dadd651..00cc5996 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -181,6 +181,7 @@ func (r *CloudKittyReconciler) Reconcile(ctx context.Context, req ctrl.Request) condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), condition.UnknownCondition(condition.DBReadyCondition, condition.InitReason, condition.DBReadyInitMessage), condition.UnknownCondition(condition.DBSyncReadyCondition, condition.InitReason, condition.DBSyncReadyInitMessage), + condition.UnknownCondition(telemetryv1.CloudKittyStorageInitReadyCondition, condition.InitReason, telemetryv1.CloudKittyStorageInitReadyInitMessage), condition.UnknownCondition(condition.RabbitMqTransportURLReadyCondition, condition.InitReason, condition.RabbitMqTransportURLReadyInitMessage), condition.UnknownCondition(condition.MemcachedReadyCondition, condition.InitReason, condition.MemcachedReadyInitMessage), condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), @@ -419,10 +420,10 @@ func (r *CloudKittyReconciler) reconcileInit( // run CloudKitty db sync // dbSyncHash := instance.Status.Hash[telemetryv1.CKDbSyncHash] - jobDef := cloudkitty.DbSyncJob(instance, serviceLabels) + jobDbSyncDef := cloudkitty.DbSyncJob(instance, serviceLabels) dbSyncjob := job.NewJob( - jobDef, + jobDbSyncDef, telemetryv1.CKDbSyncHash, instance.Spec.PreserveJobs, cloudkitty.ShortDuration, @@ -451,12 +452,54 @@ func (r *CloudKittyReconciler) reconcileInit( } if dbSyncjob.HasChanged() { instance.Status.Hash[telemetryv1.CKDbSyncHash] = dbSyncjob.GetHash() - Log.Info(fmt.Sprintf("Service '%s' - Job %s hash added - %s", instance.Name, jobDef.Name, instance.Status.Hash[telemetryv1.CKDbSyncHash])) + Log.Info(fmt.Sprintf("Service '%s' - Job %s hash added - %s", instance.Name, jobDbSyncDef.Name, instance.Status.Hash[telemetryv1.CKDbSyncHash])) } instance.Status.Conditions.MarkTrue(condition.DBSyncReadyCondition, condition.DBSyncReadyMessage) // run CloudKitty db sync - end + // + // run CloudKitty Storage Init + // + ckStorageInitHash := instance.Status.Hash[telemetryv1.CKStorageInitHash] + jobStorageInitDef := cloudkitty.StorageInitJob(instance, serviceLabels) + + storageInitjob := job.NewJob( + jobStorageInitDef, + telemetryv1.CKStorageInitHash, + instance.Spec.PreserveJobs, + cloudkitty.ShortDuration, + ckStorageInitHash, + ) + ctrlResult, err = storageInitjob.DoJob( + ctx, + helper, + ) + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyStorageInitReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + telemetryv1.CloudKittyStorageInitReadyRunningMessage)) + return ctrlResult, nil + } + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyStorageInitReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + telemetryv1.CloudKittyStorageInitReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if storageInitjob.HasChanged() { + instance.Status.Hash[telemetryv1.CKStorageInitHash] = storageInitjob.GetHash() + Log.Info(fmt.Sprintf("Service '%s' - Job %s hash added - %s", instance.Name, jobStorageInitDef.Name, instance.Status.Hash[telemetryv1.CKStorageInitHash])) + } + instance.Status.Conditions.MarkTrue(telemetryv1.CloudKittyStorageInitReadyCondition, telemetryv1.CloudKittyStorageInitReadyMessage) + + // run CloudKitty Storage Init - end + Log.Info(fmt.Sprintf("Reconciled Service '%s' init successfully", instance.Name)) return ctrl.Result{}, nil } @@ -829,6 +872,12 @@ func (r *CloudKittyReconciler) generateServiceConfigs( templateParameters["MemcachedServersWithInet"] = memcached.GetMemcachedServerListWithInetString() templateParameters["TimeOut"] = instance.Spec.APITimeout + templateParameters["TLS"] = false + if instance.Spec.CloudKittyProc.TLS.Enabled() { + templateParameters["TLS"] = true + templateParameters["CAFile"] = tls.DownstreamTLSCABundlePath + } + // create httpd vhost template parameters httpdVhostConfig := map[string]interface{}{} for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} { @@ -961,7 +1010,7 @@ func (r *CloudKittyReconciler) procDeploymentCreateOrUpdate(ctx context.Context, DatabaseHostname: instance.Status.DatabaseHostname, TransportURLSecret: instance.Status.TransportURLSecret, ServiceAccount: instance.RbacResourceName(), - TLS: instance.Spec.CloudKittyAPI.TLS.Ca, + //TLS: instance.Spec.CloudKittyProc.TLS.Ca, } if cloudKittyProcSpec.NodeSelector == nil { diff --git a/pkg/cloudkitty/dbsync.go b/pkg/cloudkitty/dbsync.go index 75a675e4..3cd213e5 100644 --- a/pkg/cloudkitty/dbsync.go +++ b/pkg/cloudkitty/dbsync.go @@ -55,7 +55,7 @@ func DbSyncJob(instance *telemetryv1.CloudKitty, labels map[string]string) *batc envVars["KOLLA_BOOTSTRAP"] = env.SetValue("TRUE") cloudKittyPassword := []corev1.EnvVar{ { - Name: "AodhPassword", + Name: "CloudKittyPassword", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ diff --git a/pkg/cloudkitty/storageinit.go b/pkg/cloudkitty/storageinit.go new file mode 100644 index 00000000..283c938d --- /dev/null +++ b/pkg/cloudkitty/storageinit.go @@ -0,0 +1,106 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cloudkitty + +import ( + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // storageInitCommand - + // TODO: Once we work on update/upgrades revisit the command in the + // the cloudkitty-storageinit-config.json file. + // If we stop all services during the update/upgrade then we can keep + // the --bump-versions flag. + // If we are doing rolling upgrades we'll need to use the flag + // conditionally (only for adoption) and do the restart cycle of + // services as described in the upstream rolling upgrades process. + storageInitCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start" +) + +// StorageInitJob func +func StorageInitJob(instance *telemetryv1.CloudKitty, labels map[string]string) *batchv1.Job { + args := []string{"-c", storageInitCommand} + + // create Volume and VolumeMounts + volumes := GetVolumes("cloudkitty") + volumeMounts := GetVolumeMounts("cloudkitty-storageinit") + // add CA cert if defined + if instance.Spec.CloudKittyAPI.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.CloudKittyAPI.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.CloudKittyAPI.TLS.CreateVolumeMounts(nil)...) + } + + runAsUser := int64(0) + envVars := map[string]env.Setter{} + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["KOLLA_BOOTSTRAP"] = env.SetValue("TRUE") + cloudKittyPassword := []corev1.EnvVar{ + { + Name: "CloudKittyPassword", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: instance.Spec.Secret, + }, + Key: instance.Spec.PasswordSelectors.CloudKittyService, + }, + }, + }, + } + + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: ServiceName + "-storageinit", + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + ServiceAccountName: instance.RbacResourceName(), + Containers: []corev1.Container{ + { + Name: ServiceName + "-storageinit", + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: instance.Spec.CloudKittyAPI.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &runAsUser, + }, + Env: env.MergeEnvs(cloudKittyPassword, envVars), + VolumeMounts: volumeMounts, + }, + }, + Volumes: volumes, + }, + }, + }, + } + + if instance.Spec.NodeSelector != nil { + job.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector + } + + return job +} diff --git a/pkg/cloudkittyproc/statefulset.go b/pkg/cloudkittyproc/statefulset.go index 3071b60e..c5959e00 100644 --- a/pkg/cloudkittyproc/statefulset.go +++ b/pkg/cloudkittyproc/statefulset.go @@ -24,7 +24,6 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" ) const ( @@ -45,7 +44,7 @@ func StatefulSet( // cloudKittyGroup := int64(telemetryv1.CloudKittyGroupID) // TODO until we determine how to properly query for these - livenessProbe := &corev1.Probe{ + /*livenessProbe := &corev1.Probe{ // TODO might need tuning TimeoutSeconds: 5, PeriodSeconds: 3, @@ -57,10 +56,10 @@ func StatefulSet( FailureThreshold: 12, PeriodSeconds: 5, InitialDelaySeconds: 5, - } + }*/ args := []string{"-c", ServiceCommand} - var probeCommand []string + /*var probeCommand []string livenessProbe.HTTPGet = &corev1.HTTPGetAction{ Port: intstr.FromInt(8080), } @@ -69,7 +68,7 @@ func StatefulSet( "/usr/local/bin/container-scripts/healthcheck.py", "processor", "/etc/cloudkitty/cloudkitty.conf.d", - } + }*/ envVars := map[string]env.Setter{} envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") @@ -113,13 +112,13 @@ func StatefulSet( SecurityContext: &corev1.SecurityContext{ RunAsUser: &cloudKittyUser, }, - Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: volumeMounts, - Resources: instance.Spec.Resources, - LivenessProbe: livenessProbe, - StartupProbe: startupProbe, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + Resources: instance.Spec.Resources, + //LivenessProbe: livenessProbe, + //StartupProbe: startupProbe, }, - { + /*{ Name: "probe", Command: probeCommand, Image: instance.Spec.ContainerImage, @@ -128,7 +127,7 @@ func StatefulSet( //RunAsGroup: &cloudKittyGroup, }, VolumeMounts: volumeMounts, - }, + },*/ }, Volumes: volumes, }, diff --git a/templates/cloudkitty/config/cloudkitty-api-config-httpd.json b/templates/cloudkitty/config/cloudkitty-api-config-httpd.json deleted file mode 100644 index 174349bf..00000000 --- a/templates/cloudkitty/config/cloudkitty-api-config-httpd.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "command": "/usr/sbin/httpd -DFOREGROUND", - "config_files": [ - { - "source": "/var/lib/openstack/config/httpd.conf", - "dest": "/etc/httpd/conf/httpd.conf", - "owner": "root", - "perm": "0644" - }, - { - "source": "/var/lib/openstack/config/10-cloudkitty_wsgi.conf", - "dest": "/etc/httpd/conf.d/10-cloudkitty_wsgi.conf", - "owner": "root", - "perm": "0644" - }, - { - "source": "/var/lib/openstack/config/ssl.conf", - "dest": "/etc/httpd/conf.d/ssl.conf", - "owner": "cloudkitty", - "perm": "0644" - }, - { - "source": "/var/lib/config-data/tls/certs/*", - "dest": "/etc/pki/tls/certs/", - "owner": "cloudkitty", - "perm": "0640", - "optional": true, - "merge": true - }, - { - "source": "/var/lib/config-data/tls/private/*", - "dest": "/etc/pki/tls/private/", - "owner": "cloudkitty", - "perm": "0600", - "optional": true, - "merge": true - } - ], - "permissions": [ - { - "path": "/var/log/cloudkitty", - "owner": "cloudkitty:apache", - "recurse": true - }, - { - "path": "/etc/httpd/run", - "owner": "cloudkitty:apache", - "recurse": true - } - ] -} diff --git a/templates/cloudkitty/config/cloudkitty-api-config.json b/templates/cloudkitty/config/cloudkitty-api-config.json index 11a6f291..380bb3a0 100644 --- a/templates/cloudkitty/config/cloudkitty-api-config.json +++ b/templates/cloudkitty/config/cloudkitty-api-config.json @@ -1,11 +1,74 @@ { - "command": "/usr/bin/uwsgi --ini /etc/cloudkitty/cloudkitty-api-uwsgi.ini", + "command": "/usr/sbin/httpd -DFOREGROUND -E /dev/stdout", "config_files": [ { - "source": "/var/lib/openstack/config/cloudkitty-api-uwsgi.ini", - "dest": "/etc/cloudkitty/cloudkitty-api-uwsgi.ini", + "source": "/var/lib/openstack/config/cloudkitty.conf", + "dest": "/etc/cloudkitty/cloudkitty.conf", "owner": "cloudkitty", "perm": "0600" + }, + { + "source": "/var/lib/openstack/config/custom.conf", + "dest": "/etc/aodh/cloudkitty.conf.d/01-cloudkitty-custom.conf", + "owner": "cloudkitty", + "perm": "0600", + "optional": true + }, + { + "source": "/var/lib/openstack/config/wsgi-cloudkitty.conf", + "dest": "/etc/httpd/conf.d/00wsgi-cloudkitty.conf", + "owner": "cloudkitty", + "perm": "0644" + }, + { + "source": "/var/lib/openstack/config/httpd.conf", + "dest": "/etc/httpd/conf/httpd.conf", + "owner": "cloudkitty", + "perm": "0644" + }, + { + "source": "/var/lib/openstack/config/ssl.conf", + "dest": "/etc/httpd/conf.d/ssl.conf", + "owner": "cloudkitty", + "perm": "0644" + }, + { + "source": "/var/lib/config-data/tls/certs/*", + "dest": "/etc/pki/tls/certs/", + "owner": "cloudkitty", + "perm": "0440", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/config-data/tls/private/*", + "dest": "/etc/pki/tls/private/", + "owner": "cloudkitty", + "perm": "0400", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/openstack/config/my.cnf", + "dest": "/etc/my.cnf", + "owner": "cloudkitty", + "perm": "0644" + }, + { + "source": "/var/lib/config-data/mtls/certs/*", + "dest": "/etc/pki/tls/certs/", + "owner": "cloudkitty:cloudkitty", + "perm": "0640", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/config-data/mtls/private/*", + "dest": "/etc/pki/tls/private/", + "owner": "cloudkitty:cloudkitty", + "perm": "0640", + "optional": true, + "merge": true } ] } diff --git a/templates/cloudkitty/config/cloudkitty-storageinit-config.json b/templates/cloudkitty/config/cloudkitty-storageinit-config.json new file mode 100644 index 00000000..3d9561c5 --- /dev/null +++ b/templates/cloudkitty/config/cloudkitty-storageinit-config.json @@ -0,0 +1,11 @@ +{ + "command": "/usr/bin/cloudkitty-storage-init", + "config_files": [ + { + "source": "/var/lib/openstack/config/cloudkitty.conf", + "dest": "/etc/cloudkitty/cloudkitty.conf", + "owner": "cloudkitty", + "perm": "0600" + } + ] +} diff --git a/templates/cloudkitty/config/cloudkitty.conf b/templates/cloudkitty/config/cloudkitty.conf index e9896eb2..58531272 100644 --- a/templates/cloudkitty/config/cloudkitty.conf +++ b/templates/cloudkitty/config/cloudkitty.conf @@ -31,11 +31,11 @@ backend = keystone auth_section = authinfos [storage_influxdb] -port = 8086 -host = localhost -database = cloudkitty -password = cloudkitty -user = cloudkitty +version = 2 +token = cloudkitty +org = openstack +bucket = cloudkitty +url = http://influxdb:8086 [collect] period = 300 diff --git a/templates/cloudkitty/config/httpd.conf b/templates/cloudkitty/config/httpd.conf index b6c862a5..c742ea69 100644 --- a/templates/cloudkitty/config/httpd.conf +++ b/templates/cloudkitty/config/httpd.conf @@ -24,4 +24,4 @@ ErrorLog /dev/stdout # XXX: To disable SSL #Include conf.d/*.conf # If above include is commented include at least the cloudkitty wsgi file -Include conf.d/10-cloudkitty_wsgi.conf +Include conf.d/00wsgi-cloudkitty.conf diff --git a/templates/cloudkitty/config/10-cloudkitty_wsgi.conf b/templates/cloudkitty/config/wsgi-cloudkitty.conf similarity index 94% rename from templates/cloudkitty/config/10-cloudkitty_wsgi.conf rename to templates/cloudkitty/config/wsgi-cloudkitty.conf index 1fc084ae..b7407f3d 100644 --- a/templates/cloudkitty/config/10-cloudkitty_wsgi.conf +++ b/templates/cloudkitty/config/wsgi-cloudkitty.conf @@ -34,7 +34,7 @@ WSGIApplicationGroup %{GLOBAL} WSGIDaemonProcess {{ $endpt }} display-name={{ $endpt }} group=cloudkitty processes=4 threads=1 user=cloudkitty WSGIProcessGroup {{ $endpt }} - WSGIScriptAlias / "/var/www/cgi-bin/cloudkitty/cloudkitty-wsgi" + WSGIScriptAlias / "/var/www/cgi-bin/cloudkitty/cloudkitty-api" WSGIPassAuthorization On {{ end }} From d803db4b45a07284952381961e1d87ddfae6cdce Mon Sep 17 00:00:00 2001 From: mgirgisf Date: Thu, 17 Jul 2025 13:13:41 +0200 Subject: [PATCH 03/12] update prometheus_collector --- api/v1beta1/cloudkitty_types.go | 24 +++++++++++ controllers/cloudkitty_controller.go | 46 +++++++++++++++++++++ pkg/cloudkitty/const.go | 3 ++ templates/cloudkitty/config/cloudkitty.conf | 8 +++- 4 files changed, 80 insertions(+), 1 deletion(-) diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index 1d2d1a7e..9559ffbd 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -90,6 +90,21 @@ type CloudKittySpecBase struct { // TopologyRef to apply the Topology defined by the associated CR referenced // by name TopologyRef *topologyv1.TopoRef `json:"topologyRef,omitempty"` + + // Host of user deployed prometheus + // +kubebuilder:validation:Optional + PrometheusHost string `json:"prometheusHost,omitempty"` + + // Port of user deployed prometheus + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + // +kubebuilder:validation:Optional + PrometheusPort int32 `json:"prometheusPort,omitempty"` + + // If defined, specifies which CA certificate to use for user deployed prometheus + // +kubebuilder:validation:Optional + // +nullable + PrometheusTLSCaCertSecret *corev1.SecretKeySelector `json:"prometheusTLSCaCertSecret,omitempty"` } // CloudKittySpecCore the same as CloudKittySpec without ContainerImage references @@ -211,6 +226,15 @@ type CloudKittyStatus struct { // controller has not started processing the latest changes, and the status // and its conditions are likely stale. ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // PrometheusHost - Hostname for prometheus used for autoscaling + PrometheusHost string `json:"prometheusHostname,omitempty"` + + // PrometheusPort - Port for prometheus used for autoscaling + PrometheusPort int32 `json:"prometheusPort,omitempty"` + + // PrometheusTLS - Determines if TLS should be used for accessing prometheus + PrometheusTLS bool `json:"prometheusTLS,omitempty"` } //+kubebuilder:object:root=true diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index 00cc5996..e032e60a 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -19,7 +19,10 @@ package controllers import ( "context" "fmt" + "strconv" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/metricstorage" k8s_errors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -813,6 +816,7 @@ func (r *CloudKittyReconciler) generateServiceConfigs( memcached *memcachedv1.Memcached, db *mariadbv1.Database, ) error { + Log := r.GetLogger(ctx) // // create Secret required for cloudkitty input // - %-scripts holds scripts to e.g. bootstrap the service @@ -855,6 +859,46 @@ func (r *CloudKittyReconciler) generateServiceConfigs( return err } + if instance.Spec.PrometheusHost == "" { + // We're using MetricStorage for Prometheus. + prometheusEndpointSecret := &corev1.Secret{} + err = r.Client.Get(ctx, client.ObjectKey{ + Name: cloudkitty.PrometheusEndpointSecret, + Namespace: instance.Namespace, + }, prometheusEndpointSecret) + if err != nil { + Log.Info("Prometheus Endpoint Secret not found") + } + if prometheusEndpointSecret.Data != nil { + instance.Status.PrometheusHost = string(prometheusEndpointSecret.Data[metricstorage.PrometheusHost]) + port, err := strconv.Atoi(string(prometheusEndpointSecret.Data[metricstorage.PrometheusPort])) + if err != nil { + return err + } + instance.Status.PrometheusPort = int32(port) + + metricStorage := &telemetryv1.MetricStorage{} + err = r.Client.Get(ctx, client.ObjectKey{ + Namespace: instance.Namespace, + Name: telemetryv1.DefaultServiceName, + }, metricStorage) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + } + instance.Status.PrometheusTLS = metricStorage.Spec.PrometheusTLS.Enabled() + } + } else { + // We're using user-deployed Prometheus. + instance.Status.PrometheusHost = instance.Spec.PrometheusHost + instance.Status.PrometheusPort = instance.Spec.PrometheusPort + instance.Status.PrometheusTLS = instance.Spec.PrometheusTLSCaCertSecret != nil + } + databaseAccount := db.GetAccount() dbSecret := db.GetSecret() @@ -864,6 +908,8 @@ func (r *CloudKittyReconciler) generateServiceConfigs( templateParameters["KeystoneInternalURL"] = keystoneInternalURL templateParameters["KeystonePublicURL"] = keystonePublicURL templateParameters["TransportURL"] = string(transportURLSecret.Data["transport_url"]) + templateParameters["PrometheusHost"] = instance.Status.PrometheusHost + templateParameters["PrometheusPort"] = instance.Status.PrometheusPort templateParameters["DatabaseConnection"] = fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf", databaseAccount.Spec.UserName, string(dbSecret.Data[mariadbv1.DatabasePasswordSelector]), diff --git a/pkg/cloudkitty/const.go b/pkg/cloudkitty/const.go index 5fb5269a..8b372471 100644 --- a/pkg/cloudkitty/const.go +++ b/pkg/cloudkitty/const.go @@ -49,6 +49,9 @@ const ( ShortDuration = time.Duration(5) * time.Second NormalDuration = time.Duration(10) * time.Second + + // PrometheusEndpointSecret - The name of the secret that contains the Prometheus endpoint configuration. + PrometheusEndpointSecret = "metric-storage-prometheus-endpoint" ) var ResultRequeue = ctrl.Result{RequeueAfter: NormalDuration} diff --git a/templates/cloudkitty/config/cloudkitty.conf b/templates/cloudkitty/config/cloudkitty.conf index 58531272..1aa611bf 100644 --- a/templates/cloudkitty/config/cloudkitty.conf +++ b/templates/cloudkitty/config/cloudkitty.conf @@ -45,8 +45,14 @@ collector = prometheus scope_key = project [collector_prometheus] -prometheus_url = https://metric-storage-prometheus.openstack.svc:9090 +{{- if .TLS }} +prometheus_url = https://{{ .PrometheusHost }}:{{ .PrometheusPort }} +cafile = {{ .CAFile }} +insecure = false +{{- else }} +prometheus_url = http://{{ .PrometheusHost }}:{{ .PrometheusPort }} insecure = true +{{- end }} [output] pipeline = osrf From 6081ab16ab5f77cbb20355a4f40e56fc54d3c57c Mon Sep 17 00:00:00 2001 From: jlarriba Date: Thu, 24 Jul 2025 09:36:11 +0200 Subject: [PATCH 04/12] Make it compatible with previous oscp --- .../telemetry.openstack.org_cloudkitties.yaml | 44 ++++++++++++++++++- .../telemetry.openstack.org_telemetries.yaml | 36 ++++++++++++++- api/v1beta1/cloudkitty_types.go | 2 +- api/v1beta1/telemetry_types.go | 3 +- api/v1beta1/zz_generated.deepcopy.go | 5 +++ .../telemetry.openstack.org_cloudkitties.yaml | 44 ++++++++++++++++++- .../telemetry.openstack.org_telemetries.yaml | 36 ++++++++++++++- controllers/cloudkitty_controller.go | 2 +- .../config/cloudkitty-api-config.json | 22 ++++++---- templates/cloudkitty/config/cloudkitty.conf | 5 ++- 10 files changed, 180 insertions(+), 19 deletions(-) diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml index 387bf62d..07bd45f4 100644 --- a/api/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -42,7 +42,6 @@ spec: apiTimeout: default: 60 description: APITimeout for HAProxy, Apache, and rpc_response_timeout - minimum: 10 type: integer cloudKittyAPI: description: CloudKittyAPI - Spec definition for the API service of @@ -494,6 +493,7 @@ spec: DB, defaults to cloudkitty type: string databaseInstance: + default: openstack description: |- MariaDB instance name Right now required by the maridb-operator to get the credentials from the instance to create the DB @@ -538,6 +538,37 @@ spec: description: PreserveJobs - do not delete jobs after they finished e.g. to check logs type: boolean + prometheusHost: + description: Host of user deployed prometheus + type: string + prometheusPort: + description: Port of user deployed prometheus + format: int32 + maximum: 65535 + minimum: 1 + type: integer + prometheusTLSCaCertSecret: + description: If defined, specifies which CA certificate to use for + user deployed prometheus + nullable: true + properties: + key: + description: The key of the secret to select from. Must be a + valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic rabbitMqClusterName: default: rabbitmq description: |- @@ -657,6 +688,17 @@ spec: and its conditions are likely stale. format: int64 type: integer + prometheusHostname: + description: PrometheusHost - Hostname for prometheus used for autoscaling + type: string + prometheusPort: + description: PrometheusPort - Port for prometheus used for autoscaling + format: int32 + type: integer + prometheusTLS: + description: PrometheusTLS - Determines if TLS should be used for + accessing prometheus + type: boolean serviceIDs: additionalProperties: type: string diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index f69ad6d9..9798050d 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -600,7 +600,6 @@ spec: apiTimeout: default: 60 description: APITimeout for HAProxy, Apache, and rpc_response_timeout - minimum: 10 type: integer cloudKittyAPI: description: CloudKittyAPI - Spec definition for the API service @@ -1053,13 +1052,14 @@ spec: cloudkitty DB, defaults to cloudkitty type: string databaseInstance: + default: openstack description: |- MariaDB instance name Right now required by the maridb-operator to get the credentials from the instance to create the DB Might not be required in future type: string enabled: - default: true + default: false description: Enabled - Whether OpenStack CloudKitty service should be deployed and managed type: boolean @@ -1102,6 +1102,38 @@ spec: description: PreserveJobs - do not delete jobs after they finished e.g. to check logs type: boolean + prometheusHost: + description: Host of user deployed prometheus + type: string + prometheusPort: + description: Port of user deployed prometheus + format: int32 + maximum: 65535 + minimum: 1 + type: integer + prometheusTLSCaCertSecret: + description: If defined, specifies which CA certificate to use + for user deployed prometheus + nullable: true + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic rabbitMqClusterName: default: rabbitmq description: |- diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index 9559ffbd..e4e29684 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -47,6 +47,7 @@ type CloudKittySpecBase struct { CloudKittyTemplate `json:",inline"` // +kubebuilder:validation:Required + // +kubebuilder:default=openstack // MariaDB instance name // Right now required by the maridb-operator to get the credentials from the instance to create the DB // Might not be required in future @@ -82,7 +83,6 @@ type CloudKittySpecBase struct { // +kubebuilder:validation:Optional // +kubebuilder:default=60 - // +kubebuilder:validation:Minimum=10 // APITimeout for HAProxy, Apache, and rpc_response_timeout APITimeout int `json:"apiTimeout"` diff --git a/api/v1beta1/telemetry_types.go b/api/v1beta1/telemetry_types.go index 02b6ac64..46373586 100644 --- a/api/v1beta1/telemetry_types.go +++ b/api/v1beta1/telemetry_types.go @@ -187,14 +187,13 @@ type LoggingSection struct { // CloudKittySpec defines the desired state of the cloudkitty service type CloudKittySection struct { // +kubebuilder:validation:Optional - // +kubebuilder:default=true + // +kubebuilder:default=false // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:booleanSwitch"} // Enabled - Whether OpenStack CloudKitty service should be deployed and managed Enabled *bool `json:"enabled"` // +kubebuilder:validation:Optional //+operator-sdk:csv:customresourcedefinitions:type=spec - // +kubebuilder:default={apiTimeout: 60} // Template - Overrides to use when creating the OpenStack CloudKitty service CloudKittySpec `json:",inline"` } diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index c9d622b6..f9a4c470 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1107,6 +1107,11 @@ func (in *CloudKittySpecBase) DeepCopyInto(out *CloudKittySpecBase) { *out = new(topologyv1beta1.TopoRef) **out = **in } + if in.PrometheusTLSCaCertSecret != nil { + in, out := &in.PrometheusTLSCaCertSecret, &out.PrometheusTLSCaCertSecret + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittySpecBase. diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml index 387bf62d..07bd45f4 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -42,7 +42,6 @@ spec: apiTimeout: default: 60 description: APITimeout for HAProxy, Apache, and rpc_response_timeout - minimum: 10 type: integer cloudKittyAPI: description: CloudKittyAPI - Spec definition for the API service of @@ -494,6 +493,7 @@ spec: DB, defaults to cloudkitty type: string databaseInstance: + default: openstack description: |- MariaDB instance name Right now required by the maridb-operator to get the credentials from the instance to create the DB @@ -538,6 +538,37 @@ spec: description: PreserveJobs - do not delete jobs after they finished e.g. to check logs type: boolean + prometheusHost: + description: Host of user deployed prometheus + type: string + prometheusPort: + description: Port of user deployed prometheus + format: int32 + maximum: 65535 + minimum: 1 + type: integer + prometheusTLSCaCertSecret: + description: If defined, specifies which CA certificate to use for + user deployed prometheus + nullable: true + properties: + key: + description: The key of the secret to select from. Must be a + valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic rabbitMqClusterName: default: rabbitmq description: |- @@ -657,6 +688,17 @@ spec: and its conditions are likely stale. format: int64 type: integer + prometheusHostname: + description: PrometheusHost - Hostname for prometheus used for autoscaling + type: string + prometheusPort: + description: PrometheusPort - Port for prometheus used for autoscaling + format: int32 + type: integer + prometheusTLS: + description: PrometheusTLS - Determines if TLS should be used for + accessing prometheus + type: boolean serviceIDs: additionalProperties: type: string diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index f69ad6d9..9798050d 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -600,7 +600,6 @@ spec: apiTimeout: default: 60 description: APITimeout for HAProxy, Apache, and rpc_response_timeout - minimum: 10 type: integer cloudKittyAPI: description: CloudKittyAPI - Spec definition for the API service @@ -1053,13 +1052,14 @@ spec: cloudkitty DB, defaults to cloudkitty type: string databaseInstance: + default: openstack description: |- MariaDB instance name Right now required by the maridb-operator to get the credentials from the instance to create the DB Might not be required in future type: string enabled: - default: true + default: false description: Enabled - Whether OpenStack CloudKitty service should be deployed and managed type: boolean @@ -1102,6 +1102,38 @@ spec: description: PreserveJobs - do not delete jobs after they finished e.g. to check logs type: boolean + prometheusHost: + description: Host of user deployed prometheus + type: string + prometheusPort: + description: Port of user deployed prometheus + format: int32 + maximum: 65535 + minimum: 1 + type: integer + prometheusTLSCaCertSecret: + description: If defined, specifies which CA certificate to use + for user deployed prometheus + nullable: true + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic rabbitMqClusterName: default: rabbitmq description: |- diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index e032e60a..5a98a71d 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -993,7 +993,7 @@ func (r *CloudKittyReconciler) transportURLCreateOrUpdate( ) (*rabbitmqv1.TransportURL, controllerutil.OperationResult, error) { transportURL := &rabbitmqv1.TransportURL{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-cloudkitty-transport", instance.Name), + Name: fmt.Sprintf("%s-transport", instance.Name), Namespace: instance.Namespace, Labels: serviceLabels, }, diff --git a/templates/cloudkitty/config/cloudkitty-api-config.json b/templates/cloudkitty/config/cloudkitty-api-config.json index 380bb3a0..107cfa1a 100644 --- a/templates/cloudkitty/config/cloudkitty-api-config.json +++ b/templates/cloudkitty/config/cloudkitty-api-config.json @@ -36,7 +36,7 @@ "source": "/var/lib/config-data/tls/certs/*", "dest": "/etc/pki/tls/certs/", "owner": "cloudkitty", - "perm": "0440", + "perm": "0640", "optional": true, "merge": true }, @@ -44,16 +44,10 @@ "source": "/var/lib/config-data/tls/private/*", "dest": "/etc/pki/tls/private/", "owner": "cloudkitty", - "perm": "0400", + "perm": "0600", "optional": true, "merge": true }, - { - "source": "/var/lib/openstack/config/my.cnf", - "dest": "/etc/my.cnf", - "owner": "cloudkitty", - "perm": "0644" - }, { "source": "/var/lib/config-data/mtls/certs/*", "dest": "/etc/pki/tls/certs/", @@ -70,5 +64,17 @@ "optional": true, "merge": true } + ], + "permissions": [ + { + "path": "/var/log/cinder", + "owner": "cloudkitty:apache", + "recurse": true + }, + { + "path": "/etc/httpd/run", + "owner": "cloudkitty:apache", + "recurse": true + } ] } diff --git a/templates/cloudkitty/config/cloudkitty.conf b/templates/cloudkitty/config/cloudkitty.conf index 1aa611bf..318ed398 100644 --- a/templates/cloudkitty/config/cloudkitty.conf +++ b/templates/cloudkitty/config/cloudkitty.conf @@ -61,7 +61,10 @@ backend = cloudkitty.backend.file.FileBackend [storage] version = 2 -backend = influxdb +backend = loki + +[storage_loki] +url = http://loki:3100/loki/api/v1 [database] connection = {{ .DatabaseConnection }} From 1f8910664963e580c99b5c32d0c616227d406cf3 Mon Sep 17 00:00:00 2001 From: jlarriba Date: Tue, 26 Aug 2025 10:11:25 +0200 Subject: [PATCH 05/12] From mgirgis: Add Cloudkitty healthcheck.py --- pkg/cloudkittyproc/statefulset.go | 30 +++---- templates/cloudkitty/bin/healthcheck.py | 106 +++++++++++++++++++----- 2 files changed, 98 insertions(+), 38 deletions(-) diff --git a/pkg/cloudkittyproc/statefulset.go b/pkg/cloudkittyproc/statefulset.go index c5959e00..0c3f8b70 100644 --- a/pkg/cloudkittyproc/statefulset.go +++ b/pkg/cloudkittyproc/statefulset.go @@ -24,6 +24,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) const ( @@ -44,7 +45,7 @@ func StatefulSet( // cloudKittyGroup := int64(telemetryv1.CloudKittyGroupID) // TODO until we determine how to properly query for these - /*livenessProbe := &corev1.Probe{ + livenessProbe := &corev1.Probe{ // TODO might need tuning TimeoutSeconds: 5, PeriodSeconds: 3, @@ -56,19 +57,18 @@ func StatefulSet( FailureThreshold: 12, PeriodSeconds: 5, InitialDelaySeconds: 5, - }*/ + } args := []string{"-c", ServiceCommand} - /*var probeCommand []string + var probeCommand []string livenessProbe.HTTPGet = &corev1.HTTPGetAction{ Port: intstr.FromInt(8080), } startupProbe.HTTPGet = livenessProbe.HTTPGet probeCommand = []string{ - "/usr/local/bin/container-scripts/healthcheck.py", - "processor", - "/etc/cloudkitty/cloudkitty.conf.d", - }*/ + "/var/lib/openstack/bin/healthcheck.py", + "/etc/cloudkitty/cloudkitty.conf.d/cloudkitty.conf", + } envVars := map[string]env.Setter{} envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") @@ -112,13 +112,13 @@ func StatefulSet( SecurityContext: &corev1.SecurityContext{ RunAsUser: &cloudKittyUser, }, - Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: volumeMounts, - Resources: instance.Spec.Resources, - //LivenessProbe: livenessProbe, - //StartupProbe: startupProbe, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + Resources: instance.Spec.Resources, + LivenessProbe: livenessProbe, + StartupProbe: startupProbe, }, - /*{ + { Name: "probe", Command: probeCommand, Image: instance.Spec.ContainerImage, @@ -127,7 +127,7 @@ func StatefulSet( //RunAsGroup: &cloudKittyGroup, }, VolumeMounts: volumeMounts, - },*/ + }, }, Volumes: volumes, }, @@ -149,4 +149,4 @@ func StatefulSet( } return statefulset -} +} \ No newline at end of file diff --git a/templates/cloudkitty/bin/healthcheck.py b/templates/cloudkitty/bin/healthcheck.py index 4bce1182..59639ccd 100755 --- a/templates/cloudkitty/bin/healthcheck.py +++ b/templates/cloudkitty/bin/healthcheck.py @@ -14,25 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -# Trivial HTTP server to check health of scheduler, backup and volume services. -# Cinder-API hast its own health check endpoint and does not need this. -# -# The only check this server currently does is using the heartbeat in the -# database service table, accessing the DB directly here using cinder's -# configuration options. -# -# The benefit of accessing the DB directly is that it doesn't depend on the -# Cinder-API service being up and we can also differentiate between the -# container not having a connection to the DB and the cinder service not doing -# the heartbeats. -# -# For volume services all enabled backends must be up to return 200, so it is -# recommended to use a different pod for each backend to avoid one backend -# affecting others. -# -# Requires the name of the service as the first argument (volume, backup, -# scheduler) and optionally a second argument with the location of the -# configuration directory (defaults to /etc/cinder/cinder.conf.d) from http import server import signal @@ -40,17 +21,67 @@ import sys import time import threading +import requests from oslo_config import cfg + SERVER_PORT = 8080 CONF = cfg.CONF + class HTTPServerV6(server.HTTPServer): - address_family = socket.AF_INET6 + address_family = socket.AF_INET6 + class HeartbeatServer(server.BaseHTTPRequestHandler): + + @staticmethod + def check_services(): + print("Starting health checks") + results = {} + + # Todo Database Endpoint Reachability + # Keystone Endpoint Reachability + try: + keystone_uri = CONF.keystone_authtoken.auth_url + response = requests.get(keystone_uri, timeout=5) + response.raise_for_status() + server_header = response.headers.get('Server', '').lower() + if 'keystone' in server_header: + results['keystone_endpoint'] = 'OK' + print("Keystone endpoint reachable and responsive.") + else: + results['keystone_endpoint'] = 'WARN' + print(f"Keystone endpoint reachable, but not a valid Keystone service: {keystone_uri}") + except requests.exceptions.RequestException as e: + results['keystone_endpoint'] = 'FAIL' + print(f"ERROR: Keystone endpoint check failed: {e}") + raise Exception('ERROR: Keystone check failed', e) + + # Prometheus Collector Endpoint Reachability + try: + prometheus_url = CONF.collector_prometheus.prometheus_url + insecure = CONF.collector_prometheus.insecure + cafile = CONF.collector_prometheus.cafile + verify_ssl = cafile if cafile and not insecure else not insecure + + response = requests.get(prometheus_url, timeout=5, verify=verify_ssl) + response.raise_for_status() + results['collector_endpoint'] = 'OK' + print("Prometheus collector endpoint reachable.") + except requests.exceptions.RequestException as e: + results['collector_endpoint'] = 'FAIL' + print(f"ERROR: Prometheus collector check failed: {e}") + raise Exception('ERROR: Prometheus collector check failed', e) + def do_GET(self): + try: + self.check_services() + except Exception as exc: + self.send_error(500, exc.args[0], exc.args[1]) + return + self.send_response(200) self.send_header("Content-type", "text/html") self.end_headers() @@ -68,12 +99,41 @@ def stopper(signal_number=None, frame=None): if __name__ == "__main__": + # Register config options + cfg.CONF.register_group(cfg.OptGroup(name='database', title='Database connection options')) + cfg.CONF.register_opt(cfg.StrOpt('connection', default=None), group='database') + + cfg.CONF.register_group(cfg.OptGroup(name='keystone_authtoken', title='Keystone Auth Token Options')) + cfg.CONF.register_opt(cfg.StrOpt('auth_url', + default='https://keystone-internal.openstack.svc:5000'), + group='keystone_authtoken') + + cfg.CONF.register_group(cfg.OptGroup(name='collector_prometheus', title='Prometheus Collector Options')) + cfg.CONF.register_opt(cfg.StrOpt('prometheus_url', + default='http://metric-storage-prometheus.openstack.svc:9090'), + group='collector_prometheus') + cfg.CONF.register_opt(cfg.BoolOpt('insecure', default=False), group='collector_prometheus') + cfg.CONF.register_opt(cfg.StrOpt('cafile', default=None), group='collector_prometheus') + + # Load configuration from file + try: + cfg.CONF(sys.argv[1:], default_config_files=['/etc/cloudkitty/cloudkitty.conf.d/cloudkitty.conf']) + except cfg.ConfigFilesNotFoundError as e: + print(f"Health check failed: {e}", file=sys.stderr) + sys.exit(1) + + # Detect IPv6 support for binding hostname = socket.gethostname() - ipv6_address = socket.getaddrinfo(hostname, None, socket.AF_INET6) + try: + ipv6_address = socket.getaddrinfo(hostname, None, socket.AF_INET6) + except socket.gaierror: + ipv6_address = None + if ipv6_address: - webServer = HTTPServerV6(("::",SERVER_PORT), HeartbeatServer) + webServer = HTTPServerV6(("::", SERVER_PORT), HeartbeatServer) else: webServer = server.HTTPServer(("0.0.0.0", SERVER_PORT), HeartbeatServer) + stop = get_stopper(webServer) # Need to run the server on a different thread because its shutdown method @@ -91,4 +151,4 @@ def stopper(signal_number=None, frame=None): except KeyboardInterrupt: pass finally: - stop() + stop() \ No newline at end of file From 13ef450362e8f4fe8d70631c1f94b326117f2339 Mon Sep 17 00:00:00 2001 From: jlarriba Date: Tue, 26 Aug 2025 10:39:10 +0200 Subject: [PATCH 06/12] Remove /v2 from path in cloudkitty, as the CLI automatically adds it --- controllers/cloudkittyapi_controller.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/controllers/cloudkittyapi_controller.go b/controllers/cloudkittyapi_controller.go index 082f6104..835b1395 100644 --- a/controllers/cloudkittyapi_controller.go +++ b/controllers/cloudkittyapi_controller.go @@ -460,21 +460,20 @@ func (r *CloudKittyAPIReconciler) reconcileInit( // expose the service (create service and return the created endpoint URLs) // - // V2 publicEndpointData := endpoint.Data{ Port: cloudkitty.CloudKittyPublicPort, - Path: "/v2", + Path: "", } internalEndpointData := endpoint.Data{ Port: cloudkitty.CloudKittyInternalPort, - Path: "/v2", + Path: "", } cloudkittyEndpoints := map[service.Endpoint]endpoint.Data{ service.EndpointPublic: publicEndpointData, service.EndpointInternal: internalEndpointData, } - apiEndpointsV3 := make(map[string]string) + apiEndpoints := make(map[string]string) for endpointType, data := range cloudkittyEndpoints { endpointTypeStr := string(endpointType) @@ -564,7 +563,7 @@ func (r *CloudKittyAPIReconciler) reconcileInit( data.Protocol = ptr.To(service.ProtocolHTTPS) } - apiEndpointsV3[string(endpointType)], err = svc.GetAPIEndpoint( + apiEndpoints[string(endpointType)], err = svc.GetAPIEndpoint( svcOverride.EndpointURL, data.Protocol, data.Path) if err != nil { instance.Status.Conditions.MarkFalse( From 73afff22ea17c4c3defd91165b52cb27a56f2672 Mon Sep 17 00:00:00 2001 From: jlarriba Date: Tue, 26 Aug 2025 11:08:12 +0200 Subject: [PATCH 07/12] Fix pre-commit --- .../telemetry.openstack.org_cloudkitties.yaml | 15 ++++----------- ...telemetry.openstack.org_cloudkittyapis.yaml | 8 ++------ ...elemetry.openstack.org_cloudkittyprocs.yaml | 8 ++------ .../telemetry.openstack.org_telemetries.yaml | 15 ++++----------- api/v1beta1/cloudkitty_types.go | 18 ++++++++++-------- api/v1beta1/cloudkittyapi_types.go | 8 ++++---- api/v1beta1/cloudkittyproc_types.go | 8 ++++---- .../telemetry.openstack.org_cloudkitties.yaml | 15 ++++----------- ...telemetry.openstack.org_cloudkittyapis.yaml | 8 ++------ ...elemetry.openstack.org_cloudkittyprocs.yaml | 8 ++------ .../telemetry.openstack.org_telemetries.yaml | 15 ++++----------- controllers/cloudkitty_controller.go | 12 ++++++------ controllers/cloudkittyapi_controller.go | 6 +++--- pkg/cloudkitty/dbsync.go | 5 ++++- pkg/cloudkitty/storageinit.go | 5 ++++- pkg/cloudkittyproc/statefulset.go | 2 +- templates/cloudkitty/bin/healthcheck.py | 2 +- .../cloudkitty/config/cloudkitty-api-uwsgi.ini | 17 ----------------- templates/cloudkitty/config/metrics.yaml | 2 +- 19 files changed, 62 insertions(+), 115 deletions(-) delete mode 100644 templates/cloudkitty/config/cloudkitty-api-uwsgi.ini diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml index 07bd45f4..6b58ee4d 100644 --- a/api/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -65,12 +65,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -348,8 +350,6 @@ spec: current project type: string type: object - required: - - containerImage type: object cloudKittyProc: description: CloudKittyProc - Spec definition for the Scheduler service @@ -373,12 +373,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -478,8 +480,6 @@ spec: current project type: string type: object - required: - - containerImage type: object customServiceConfig: description: |- @@ -600,13 +600,6 @@ spec: current project type: string type: object - required: - - cloudKittyAPI - - cloudKittyProc - - databaseInstance - - memcachedInstance - - rabbitMqClusterName - - secret type: object status: description: CloudKittyStatus defines the observed state of CloudKitty diff --git a/api/bases/telemetry.openstack.org_cloudkittyapis.yaml b/api/bases/telemetry.openstack.org_cloudkittyapis.yaml index d33b713c..bdd3309d 100644 --- a/api/bases/telemetry.openstack.org_cloudkittyapis.yaml +++ b/api/bases/telemetry.openstack.org_cloudkittyapis.yaml @@ -57,6 +57,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic databaseAccount: default: cloudkitty description: DatabaseAccount - optional MariaDBAccount used for cloudkitty @@ -71,6 +72,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -384,12 +386,6 @@ spec: transportURLSecret: description: Secret containing RabbitMq transport URL type: string - required: - - containerImage - - databaseHostname - - secret - - serviceAccount - - transportURLSecret type: object status: description: CloudKittyAPIStatus defines the observed state of CloudKittyAPI diff --git a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml index 6123f53c..178ceb15 100644 --- a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml +++ b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -71,6 +71,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic databaseAccount: default: cloudkitty description: DatabaseAccount - optional MariaDBAccount used for cloudkitty @@ -85,6 +86,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -220,12 +222,6 @@ spec: transportURLSecret: description: Secret containing RabbitMq transport URL type: string - required: - - containerImage - - databaseHostname - - secret - - serviceAccount - - transportURLSecret type: object status: description: CloudKittyProcStatus defines the observed state of CloudKitty diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index 9798050d..b3120d77 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -623,12 +623,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -906,8 +908,6 @@ spec: current project type: string type: object - required: - - containerImage type: object cloudKittyProc: description: CloudKittyProc - Spec definition for the Scheduler @@ -931,12 +931,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -1037,8 +1039,6 @@ spec: current project type: string type: object - required: - - containerImage type: object customServiceConfig: description: |- @@ -1166,13 +1166,6 @@ spec: current project type: string type: object - required: - - cloudKittyAPI - - cloudKittyProc - - databaseInstance - - memcachedInstance - - rabbitMqClusterName - - secret type: object logging: description: Logging - Parameters related to the logging diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index e4e29684..5a8cc27b 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -46,20 +46,20 @@ const ( type CloudKittySpecBase struct { CloudKittyTemplate `json:",inline"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // +kubebuilder:default=openstack // MariaDB instance name // Right now required by the maridb-operator to get the credentials from the instance to create the DB // Might not be required in future DatabaseInstance string `json:"databaseInstance"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // +kubebuilder:default=rabbitmq // RabbitMQ instance name // Needed to request a transportURL that is created and used in CloudKitty RabbitMqClusterName string `json:"rabbitMqClusterName"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // +kubebuilder:default=memcached // Memcached instance name. MemcachedInstance string `json:"memcachedInstance"` @@ -111,11 +111,11 @@ type CloudKittySpecBase struct { type CloudKittySpecCore struct { CloudKittySpecBase `json:",inline"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // CloudKittyAPI - Spec definition for the API service of this CloudKitty deployment CloudKittyAPI CloudKittyAPITemplateCore `json:"cloudKittyAPI"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // CloudKittyProc - Spec definition for the Scheduler service of this CloudKitty deployment CloudKittyProc CloudKittyProcTemplateCore `json:"cloudKittyProc"` } @@ -124,11 +124,11 @@ type CloudKittySpecCore struct { type CloudKittySpec struct { CloudKittySpecBase `json:",inline"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // CloudKittyAPI - Spec definition for the API service of this CloudKitty deployment CloudKittyAPI CloudKittyAPITemplate `json:"cloudKittyAPI"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // CloudKittyProc - Spec definition for the Scheduler service of this CloudKitty deployment CloudKittyProc CloudKittyProcTemplate `json:"cloudKittyProc"` } @@ -145,7 +145,7 @@ type CloudKittyTemplate struct { // DatabaseAccount - optional MariaDBAccount used for cloudkitty DB, defaults to cloudkitty DatabaseAccount string `json:"databaseAccount"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // Secret containing OpenStack password information Secret string `json:"secret"` @@ -171,6 +171,7 @@ type CloudKittyServiceTemplate struct { CustomServiceConfig string `json:"customServiceConfig,omitempty"` // +kubebuilder:validation:Optional + // +listType=atomic // CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets // that contain sensitive service config data. The content of each Secret gets added to the // /etc//.conf.d directory as a custom config file. @@ -182,6 +183,7 @@ type CloudKittyServiceTemplate struct { Resources corev1.ResourceRequirements `json:"resources,omitempty"` // +kubebuilder:validation:Optional + // +listType=atomic // NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network NetworkAttachments []string `json:"networkAttachments,omitempty"` diff --git a/api/v1beta1/cloudkittyapi_types.go b/api/v1beta1/cloudkittyapi_types.go index 42660c7e..3a71afb5 100644 --- a/api/v1beta1/cloudkittyapi_types.go +++ b/api/v1beta1/cloudkittyapi_types.go @@ -46,7 +46,7 @@ type CloudKittyAPITemplateCore struct { // CloudKittyAPITemplate defines the input parameters for the CloudKitty API service type CloudKittyAPITemplate struct { - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // ContainerImage - CloudKitty Container Image URL (will be set to environmental default if empty) ContainerImage string `json:"containerImage"` @@ -61,15 +61,15 @@ type CloudKittyAPISpec struct { // Input parameters for the CloudKitty API service CloudKittyAPITemplate `json:",inline"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // DatabaseHostname - CloudKitty Database Hostname DatabaseHostname string `json:"databaseHostname"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // Secret containing RabbitMq transport URL TransportURLSecret string `json:"transportURLSecret"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // ServiceAccount - service account name used internally to provide CloudKitty services the default SA name ServiceAccount string `json:"serviceAccount"` } diff --git a/api/v1beta1/cloudkittyproc_types.go b/api/v1beta1/cloudkittyproc_types.go index d7b80e19..83de10d1 100644 --- a/api/v1beta1/cloudkittyproc_types.go +++ b/api/v1beta1/cloudkittyproc_types.go @@ -42,7 +42,7 @@ type CloudKittyProcTemplateCore struct { // CloudKittyProcTemplate defines the input parameters for the CloudKitty Processor service type CloudKittyProcTemplate struct { - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // ContainerImage - CloudKitty Container Image URL (will be set to environmental default if empty) ContainerImage string `json:"containerImage"` @@ -57,15 +57,15 @@ type CloudKittyProcSpec struct { // Input parameters for the CloudKitty Processor service CloudKittyProcTemplate `json:",inline"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // DatabaseHostname - CloudKitty Database Hostname DatabaseHostname string `json:"databaseHostname"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // Secret containing RabbitMq transport URL TransportURLSecret string `json:"transportURLSecret"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // ServiceAccount - service account name used internally to provide CloudKitty services the default SA name ServiceAccount string `json:"serviceAccount"` } diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml index 07bd45f4..6b58ee4d 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -65,12 +65,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -348,8 +350,6 @@ spec: current project type: string type: object - required: - - containerImage type: object cloudKittyProc: description: CloudKittyProc - Spec definition for the Scheduler service @@ -373,12 +373,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -478,8 +480,6 @@ spec: current project type: string type: object - required: - - containerImage type: object customServiceConfig: description: |- @@ -600,13 +600,6 @@ spec: current project type: string type: object - required: - - cloudKittyAPI - - cloudKittyProc - - databaseInstance - - memcachedInstance - - rabbitMqClusterName - - secret type: object status: description: CloudKittyStatus defines the observed state of CloudKitty diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml index d33b713c..bdd3309d 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml @@ -57,6 +57,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic databaseAccount: default: cloudkitty description: DatabaseAccount - optional MariaDBAccount used for cloudkitty @@ -71,6 +72,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -384,12 +386,6 @@ spec: transportURLSecret: description: Secret containing RabbitMq transport URL type: string - required: - - containerImage - - databaseHostname - - secret - - serviceAccount - - transportURLSecret type: object status: description: CloudKittyAPIStatus defines the observed state of CloudKittyAPI diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml index 6123f53c..178ceb15 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -71,6 +71,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic databaseAccount: default: cloudkitty description: DatabaseAccount - optional MariaDBAccount used for cloudkitty @@ -85,6 +86,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -220,12 +222,6 @@ spec: transportURLSecret: description: Secret containing RabbitMq transport URL type: string - required: - - containerImage - - databaseHostname - - secret - - serviceAccount - - transportURLSecret type: object status: description: CloudKittyProcStatus defines the observed state of CloudKitty diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index 9798050d..b3120d77 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -623,12 +623,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -906,8 +908,6 @@ spec: current project type: string type: object - required: - - containerImage type: object cloudKittyProc: description: CloudKittyProc - Spec definition for the Scheduler @@ -931,12 +931,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -1037,8 +1039,6 @@ spec: current project type: string type: object - required: - - containerImage type: object customServiceConfig: description: |- @@ -1166,13 +1166,6 @@ spec: current project type: string type: object - required: - - cloudKittyAPI - - cloudKittyProc - - databaseInstance - - memcachedInstance - - rabbitMqClusterName - - secret type: object logging: description: Logging - Parameters related to the logging diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index 5a98a71d..3020961a 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -227,8 +227,8 @@ func (r *CloudKittyReconciler) Reconcile(ctx context.Context, req ctrl.Request) const ( cloudKittyPasswordSecretField = ".spec.secret" cloudKittyCaBundleSecretNameField = ".spec.tls.caBundleSecretName" - cloudKittyTlsAPIInternalField = ".spec.tls.api.internal.secretName" - cloudKittyTlsAPIPublicField = ".spec.tls.api.public.secretName" + cloudKittyTLSAPIInternalField = ".spec.tls.api.internal.secretName" + cloudKittyTLSAPIPublicField = ".spec.tls.api.public.secretName" cloudKittyTopologyField = ".spec.topologyRef.Name" ) @@ -241,8 +241,8 @@ var ( cloudKittyAPIWatchFields = []string{ cloudKittyPasswordSecretField, cloudKittyCaBundleSecretNameField, - cloudKittyTlsAPIInternalField, - cloudKittyTlsAPIPublicField, + cloudKittyTLSAPIInternalField, + cloudKittyTLSAPIPublicField, cloudKittyTopologyField, } ) @@ -423,7 +423,7 @@ func (r *CloudKittyReconciler) reconcileInit( // run CloudKitty db sync // dbSyncHash := instance.Status.Hash[telemetryv1.CKDbSyncHash] - jobDbSyncDef := cloudkitty.DbSyncJob(instance, serviceLabels) + jobDbSyncDef := cloudkitty.DbSyncJob(instance, serviceLabels, serviceAnnotations) dbSyncjob := job.NewJob( jobDbSyncDef, @@ -465,7 +465,7 @@ func (r *CloudKittyReconciler) reconcileInit( // run CloudKitty Storage Init // ckStorageInitHash := instance.Status.Hash[telemetryv1.CKStorageInitHash] - jobStorageInitDef := cloudkitty.StorageInitJob(instance, serviceLabels) + jobStorageInitDef := cloudkitty.StorageInitJob(instance, serviceLabels, serviceAnnotations) storageInitjob := job.NewJob( jobStorageInitDef, diff --git a/controllers/cloudkittyapi_controller.go b/controllers/cloudkittyapi_controller.go index 835b1395..de2cb3ff 100644 --- a/controllers/cloudkittyapi_controller.go +++ b/controllers/cloudkittyapi_controller.go @@ -301,7 +301,7 @@ func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl } // index tlsAPIInternalField - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyTlsAPIInternalField, func(rawObj client.Object) []string { + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyTLSAPIInternalField, func(rawObj client.Object) []string { // Extract the secret name from the spec, if one is provided cr := rawObj.(*telemetryv1.CloudKittyAPI) if cr.Spec.TLS.API.Internal.SecretName == nil { @@ -313,7 +313,7 @@ func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl } // index tlsAPIPublicField - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyTlsAPIPublicField, func(rawObj client.Object) []string { + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyTLSAPIPublicField, func(rawObj client.Object) []string { // Extract the secret name from the spec, if one is provided cr := rawObj.(*telemetryv1.CloudKittyAPI) if cr.Spec.TLS.API.Public.SecretName == nil { @@ -583,7 +583,7 @@ func (r *CloudKittyAPIReconciler) reconcileInit( if instance.Status.APIEndpoints == nil { instance.Status.APIEndpoints = map[string]map[string]string{} } - instance.Status.APIEndpoints[cloudkitty.ServiceName] = apiEndpointsV3 + instance.Status.APIEndpoints[cloudkitty.ServiceName] = apiEndpoints // V2 - end // expose service - end diff --git a/pkg/cloudkitty/dbsync.go b/pkg/cloudkitty/dbsync.go index 3cd213e5..5a84523a 100644 --- a/pkg/cloudkitty/dbsync.go +++ b/pkg/cloudkitty/dbsync.go @@ -36,7 +36,7 @@ const ( ) // DbSyncJob func -func DbSyncJob(instance *telemetryv1.CloudKitty, labels map[string]string) *batchv1.Job { +func DbSyncJob(instance *telemetryv1.CloudKitty, labels map[string]string, annotations map[string]string) *batchv1.Job { args := []string{"-c"} args = append(args, dbSyncCommand) @@ -75,6 +75,9 @@ func DbSyncJob(instance *telemetryv1.CloudKitty, labels map[string]string) *batc }, Spec: batchv1.JobSpec{ Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + }, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyOnFailure, ServiceAccountName: instance.RbacResourceName(), diff --git a/pkg/cloudkitty/storageinit.go b/pkg/cloudkitty/storageinit.go index 283c938d..73b05cd8 100644 --- a/pkg/cloudkitty/storageinit.go +++ b/pkg/cloudkitty/storageinit.go @@ -36,7 +36,7 @@ const ( ) // StorageInitJob func -func StorageInitJob(instance *telemetryv1.CloudKitty, labels map[string]string) *batchv1.Job { +func StorageInitJob(instance *telemetryv1.CloudKitty, labels map[string]string, annotations map[string]string) *batchv1.Job { args := []string{"-c", storageInitCommand} // create Volume and VolumeMounts @@ -74,6 +74,9 @@ func StorageInitJob(instance *telemetryv1.CloudKitty, labels map[string]string) }, Spec: batchv1.JobSpec{ Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + }, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyOnFailure, ServiceAccountName: instance.RbacResourceName(), diff --git a/pkg/cloudkittyproc/statefulset.go b/pkg/cloudkittyproc/statefulset.go index 0c3f8b70..a568be4d 100644 --- a/pkg/cloudkittyproc/statefulset.go +++ b/pkg/cloudkittyproc/statefulset.go @@ -149,4 +149,4 @@ func StatefulSet( } return statefulset -} \ No newline at end of file +} diff --git a/templates/cloudkitty/bin/healthcheck.py b/templates/cloudkitty/bin/healthcheck.py index 59639ccd..906f688d 100755 --- a/templates/cloudkitty/bin/healthcheck.py +++ b/templates/cloudkitty/bin/healthcheck.py @@ -151,4 +151,4 @@ def stopper(signal_number=None, frame=None): except KeyboardInterrupt: pass finally: - stop() \ No newline at end of file + stop() diff --git a/templates/cloudkitty/config/cloudkitty-api-uwsgi.ini b/templates/cloudkitty/config/cloudkitty-api-uwsgi.ini deleted file mode 100644 index 2cd8f602..00000000 --- a/templates/cloudkitty/config/cloudkitty-api-uwsgi.ini +++ /dev/null @@ -1,17 +0,0 @@ -[uwsgi] -chmod-socket = 666 -socket = /var/run/uwsgi/cloudkitty.socket -start-time = %t -lazy-apps = true -add-header = Connection: close -buffer-size = 65535 -hook-master-start = unix_signal:15 gracefully_kill_them_all -thunder-lock = true -plugins = http,python3 -enable-threads = true -worker-reload-mercy = 80 -exit-on-reload = false -die-on-term = true -master = true -processes = 2 -wsgi-file = /opt/stack/data/venv/bin/cloudkitty-api \ No newline at end of file diff --git a/templates/cloudkitty/config/metrics.yaml b/templates/cloudkitty/config/metrics.yaml index c42701ed..6bb078af 100644 --- a/templates/cloudkitty/config/metrics.yaml +++ b/templates/cloudkitty/config/metrics.yaml @@ -74,4 +74,4 @@ metrics: - state mutate: NUMBOOL extra_args: - aggregation_method: max \ No newline at end of file + aggregation_method: max From 2ada72edbb1411cb9caeaa1f96604ddfc74c2f34 Mon Sep 17 00:00:00 2001 From: Jaromir Wysoglad Date: Mon, 15 Sep 2025 10:47:38 -0400 Subject: [PATCH 08/12] [DNM] Create LokiStack for CloudKitty --- .../telemetry.openstack.org_cloudkitties.yaml | 92 ++++++++ .../telemetry.openstack.org_telemetries.yaml | 92 ++++++++ api/go.mod | 11 +- api/go.sum | 22 +- api/v1beta1/cloudkitty_types.go | 9 + api/v1beta1/conditions.go | 38 +++ api/v1beta1/zz_generated.deepcopy.go | 1 + .../telemetry.openstack.org_cloudkitties.yaml | 92 ++++++++ .../telemetry.openstack.org_telemetries.yaml | 92 ++++++++ controllers/cloudkitty_controller.go | 223 +++++++++++++++++- controllers/cloudkittyapi_controller.go | 32 +++ controllers/cloudkittyproc_controller.go | 31 +++ controllers/metricstorage_controller.go | 90 +++---- go.mod | 14 +- go.sum | 28 ++- main.go | 12 +- pkg/cloudkitty/cert.go | 67 ++++++ pkg/cloudkitty/const.go | 5 + pkg/cloudkitty/lokistack.go | 118 +++++++++ pkg/cloudkitty/volumes.go | 13 + pkg/utils/utils.go | 59 +++++ templates/cloudkitty/config/cloudkitty.conf | 3 +- 22 files changed, 1042 insertions(+), 102 deletions(-) create mode 100644 pkg/cloudkitty/cert.go create mode 100644 pkg/cloudkitty/lokistack.go diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml index 6b58ee4d..15cfcf46 100644 --- a/api/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -575,6 +575,95 @@ spec: RabbitMQ instance name Needed to request a transportURL that is created and used in CloudKitty type: string + s3StorageConfig: + description: S3 related configuration passed to Loki + properties: + schemas: + default: + - effectiveDate: "2020-10-11" + version: v11 + description: Schemas for reading and writing logs. + items: + description: ObjectStorageSchema defines a schema version and + the date when it will become effective. + properties: + effectiveDate: + description: |- + EffectiveDate contains a date in YYYY-MM-DD format which is interpreted in the UTC time zone. + + + The configuration always needs at least one schema that is currently valid. This means that when creating a new + LokiStack it is recommended to add a schema with the latest available version and an effective date of "yesterday". + New schema versions added to the configuration always needs to be placed "in the future", so that Loki can start + using it once the day rolls over. + pattern: ^([0-9]{4,})([-]([0-9]{2})){2}$ + type: string + version: + description: Version for writing and reading logs. + enum: + - v11 + - v12 + - v13 + type: string + required: + - effectiveDate + - version + type: object + minItems: 1 + type: array + secret: + description: |- + Secret for object storage authentication. + Name of a secret in the same namespace as the LokiStack custom resource. + properties: + credentialMode: + description: |- + CredentialMode can be used to set the desired credential mode for authenticating with the object storage. + If this is not set, then the operator tries to infer the credential mode from the provided secret and its + own configuration. + enum: + - static + - token + - token-cco + type: string + name: + description: Name of a secret in the namespace configured + for object storage secrets. + type: string + type: + description: Type of object storage that should be used + enum: + - azure + - gcs + - s3 + - swift + - alibabacloud + type: string + required: + - name + - type + type: object + tls: + description: TLS configuration for reaching the object storage + endpoint. + properties: + caKey: + description: |- + Key is the data key of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + If empty, it defaults to "service-ca.crt". + type: string + caName: + description: |- + CA is the name of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + type: string + required: + - caName + type: object + required: + - secret + type: object secret: description: Secret containing OpenStack password information type: string @@ -583,6 +672,9 @@ spec: description: ServiceUser - optional username used for this service to register in cloudkitty type: string + storageClass: + description: Storage class used for Loki + type: string topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index b3120d77..72556e5d 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -1140,6 +1140,95 @@ spec: RabbitMQ instance name Needed to request a transportURL that is created and used in CloudKitty type: string + s3StorageConfig: + description: S3 related configuration passed to Loki + properties: + schemas: + default: + - effectiveDate: "2020-10-11" + version: v11 + description: Schemas for reading and writing logs. + items: + description: ObjectStorageSchema defines a schema version + and the date when it will become effective. + properties: + effectiveDate: + description: |- + EffectiveDate contains a date in YYYY-MM-DD format which is interpreted in the UTC time zone. + + + The configuration always needs at least one schema that is currently valid. This means that when creating a new + LokiStack it is recommended to add a schema with the latest available version and an effective date of "yesterday". + New schema versions added to the configuration always needs to be placed "in the future", so that Loki can start + using it once the day rolls over. + pattern: ^([0-9]{4,})([-]([0-9]{2})){2}$ + type: string + version: + description: Version for writing and reading logs. + enum: + - v11 + - v12 + - v13 + type: string + required: + - effectiveDate + - version + type: object + minItems: 1 + type: array + secret: + description: |- + Secret for object storage authentication. + Name of a secret in the same namespace as the LokiStack custom resource. + properties: + credentialMode: + description: |- + CredentialMode can be used to set the desired credential mode for authenticating with the object storage. + If this is not set, then the operator tries to infer the credential mode from the provided secret and its + own configuration. + enum: + - static + - token + - token-cco + type: string + name: + description: Name of a secret in the namespace configured + for object storage secrets. + type: string + type: + description: Type of object storage that should be used + enum: + - azure + - gcs + - s3 + - swift + - alibabacloud + type: string + required: + - name + - type + type: object + tls: + description: TLS configuration for reaching the object storage + endpoint. + properties: + caKey: + description: |- + Key is the data key of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + If empty, it defaults to "service-ca.crt". + type: string + caName: + description: |- + CA is the name of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + type: string + required: + - caName + type: object + required: + - secret + type: object secret: description: Secret containing OpenStack password information type: string @@ -1148,6 +1237,9 @@ spec: description: ServiceUser - optional username used for this service to register in cloudkitty type: string + storageClass: + description: Storage class used for Loki + type: string topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/api/go.mod b/api/go.mod index 671c2b3a..fb2d24a3 100644 --- a/api/go.mod +++ b/api/go.mod @@ -3,6 +3,7 @@ module github.com/openstack-k8s-operators/telemetry-operator/api go 1.21 require ( + github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba github.com/onsi/ginkgo/v2 v2.20.1 github.com/onsi/gomega v1.34.1 github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20250513115636-b549982a5d8f @@ -53,11 +54,11 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/net v0.28.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.18.0 // indirect - golang.org/x/sys v0.23.0 // indirect - golang.org/x/term v0.23.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.24.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect @@ -70,7 +71,7 @@ require ( k8s.io/component-base v0.29.15 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240322212309-b815d8309940 // indirect - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect + k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/api/go.sum b/api/go.sum index 5580678d..d969c0f2 100644 --- a/api/go.sum +++ b/api/go.sum @@ -47,6 +47,8 @@ github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQu github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba h1:P5Wgp2HfGfNPLCPpS+YqquKdrrl4tW0El7VX23D6vtg= +github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba/go.mod h1:OBAgJh0mLYRvziBzBKr4/anrPHqGY9qEfuNXCpnUNi0= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -125,8 +127,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -140,18 +142,18 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -197,8 +199,8 @@ k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240322212309-b815d8309940 h1:qVoMaQV5t62UUvHe16Q3eb2c5HPzLHYzsi0Tu/xLndo= k8s.io/kube-openapi v0.0.0-20240322212309-b815d8309940/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.17.6 h1:12IXsozEsIXWAMRpgRlYS1jjAHQXHtWEOMdULh3DbEw= sigs.k8s.io/controller-runtime v0.17.6/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index 5a8cc27b..b22592dd 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -22,6 +22,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/util" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + lokistackv1 "github.com/grafana/loki/operator/api/loki/v1" ) const ( @@ -105,6 +106,14 @@ type CloudKittySpecBase struct { // +kubebuilder:validation:Optional // +nullable PrometheusTLSCaCertSecret *corev1.SecretKeySelector `json:"prometheusTLSCaCertSecret,omitempty"` + + // S3 related configuration passed to Loki + // +kubebuilder:validation:Optional + S3StorageConfig lokistackv1.ObjectStorageSpec `json:"s3StorageConfig"` + + // Storage class used for Loki + // +kubebuilder:validation:Optional + StorageClass string `json:"storageClass,omitempty"` } // CloudKittySpecCore the same as CloudKittySpec without ContainerImage references diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index 709bd1cd..d2ccd731 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -57,6 +57,12 @@ const ( // CloudKittyStorageInitReadyCondition Status=True condition which indicates if the CloudKitty Storage Init process has ran CloudKittyStorageInitReadyCondition condition.Type = "CloudKittyStorageInitReady" + // CloudKittyClientCertReadyCondition Status=True condition which indicates if the CloudKitty client certificate is ready for use + CloudKittyClientCertReadyCondition condition.Type = "CloudKittyClientCertReady" + + // CloudKittyLokiStackReadyCondition Status=True condition which indicates if the CloudKitty LokiStack is ready + CloudKittyLokiStackReadyCondition condition.Type = "CloudKittyLokiStackReady" + // LoggingCLONamespaceReadyCondition Status=True condition which indicates if the cluster-logging-operator namespace is created LoggingCLONamespaceReadyCondition condition.Type = "LoggingCLONamespaceReady" @@ -271,6 +277,38 @@ const ( // CloudKittyProcReadyRunningMessage CloudKittyProcReadyRunningMessage = "CloudKittyProc in progress" + // + // CloudKittyClientCertReady condition messages + // + // CloudKittyClientCertReadyInitMessage + CloudKittyClientCertReadyInitMessage = "CloudKittyClientCert not created" + + // CloudKittyClientCertReadyMessage + CloudKittyClientCertReadyMessage = "CloudKittyClientCert ready for use" + + // CloudKittyClientCertReadyErrorMessage + CloudKittyClientCertReadyErrorMessage = "CloudKittyClientCert error occured %s" + + // CloudKittyClientCertReadyRunningMessage + CloudKittyClientCertReadyRunningMessage = "CloudKittyClientCert in progress" + + // + // CloudKittyLokiStackReady condition messages + // + // CloudKittyLokiStackReadyInitMessage + CloudKittyLokiStackReadyInitMessage = "CloudKittyLokiStack not created" + + // CloudKittyLokiStackReadyMessage + CloudKittyLokiStackReadyMessage = "CloudKittyLokiStack ready for use" + + // CloudKittyLokiStackReadyErrorMessage + CloudKittyLokiStackReadyErrorMessage = "CloudKittyLokiStack error occured %s" + + // CloudKittyLokiStackReadyRunningMessage + CloudKittyLokiStackReadyRunningMessage = "CloudKittyLokiStack in progress" + // CloudKittyLokiStackReadyRunningMessage + CloudKittyLokiStackUnableToOwnMessage = "Error occured when trying to own %s" + DashboardsNotEnabledMessage = "Dashboarding was not enabled, so no actions required" DashboardPrometheusRuleReadyInitMessage = "Dashboard PrometheusRule not started" diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index f9a4c470..e9f328c9 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1112,6 +1112,7 @@ func (in *CloudKittySpecBase) DeepCopyInto(out *CloudKittySpecBase) { *out = new(v1.SecretKeySelector) (*in).DeepCopyInto(*out) } + in.S3StorageConfig.DeepCopyInto(&out.S3StorageConfig) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittySpecBase. diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml index 6b58ee4d..15cfcf46 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -575,6 +575,95 @@ spec: RabbitMQ instance name Needed to request a transportURL that is created and used in CloudKitty type: string + s3StorageConfig: + description: S3 related configuration passed to Loki + properties: + schemas: + default: + - effectiveDate: "2020-10-11" + version: v11 + description: Schemas for reading and writing logs. + items: + description: ObjectStorageSchema defines a schema version and + the date when it will become effective. + properties: + effectiveDate: + description: |- + EffectiveDate contains a date in YYYY-MM-DD format which is interpreted in the UTC time zone. + + + The configuration always needs at least one schema that is currently valid. This means that when creating a new + LokiStack it is recommended to add a schema with the latest available version and an effective date of "yesterday". + New schema versions added to the configuration always needs to be placed "in the future", so that Loki can start + using it once the day rolls over. + pattern: ^([0-9]{4,})([-]([0-9]{2})){2}$ + type: string + version: + description: Version for writing and reading logs. + enum: + - v11 + - v12 + - v13 + type: string + required: + - effectiveDate + - version + type: object + minItems: 1 + type: array + secret: + description: |- + Secret for object storage authentication. + Name of a secret in the same namespace as the LokiStack custom resource. + properties: + credentialMode: + description: |- + CredentialMode can be used to set the desired credential mode for authenticating with the object storage. + If this is not set, then the operator tries to infer the credential mode from the provided secret and its + own configuration. + enum: + - static + - token + - token-cco + type: string + name: + description: Name of a secret in the namespace configured + for object storage secrets. + type: string + type: + description: Type of object storage that should be used + enum: + - azure + - gcs + - s3 + - swift + - alibabacloud + type: string + required: + - name + - type + type: object + tls: + description: TLS configuration for reaching the object storage + endpoint. + properties: + caKey: + description: |- + Key is the data key of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + If empty, it defaults to "service-ca.crt". + type: string + caName: + description: |- + CA is the name of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + type: string + required: + - caName + type: object + required: + - secret + type: object secret: description: Secret containing OpenStack password information type: string @@ -583,6 +672,9 @@ spec: description: ServiceUser - optional username used for this service to register in cloudkitty type: string + storageClass: + description: Storage class used for Loki + type: string topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index b3120d77..72556e5d 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -1140,6 +1140,95 @@ spec: RabbitMQ instance name Needed to request a transportURL that is created and used in CloudKitty type: string + s3StorageConfig: + description: S3 related configuration passed to Loki + properties: + schemas: + default: + - effectiveDate: "2020-10-11" + version: v11 + description: Schemas for reading and writing logs. + items: + description: ObjectStorageSchema defines a schema version + and the date when it will become effective. + properties: + effectiveDate: + description: |- + EffectiveDate contains a date in YYYY-MM-DD format which is interpreted in the UTC time zone. + + + The configuration always needs at least one schema that is currently valid. This means that when creating a new + LokiStack it is recommended to add a schema with the latest available version and an effective date of "yesterday". + New schema versions added to the configuration always needs to be placed "in the future", so that Loki can start + using it once the day rolls over. + pattern: ^([0-9]{4,})([-]([0-9]{2})){2}$ + type: string + version: + description: Version for writing and reading logs. + enum: + - v11 + - v12 + - v13 + type: string + required: + - effectiveDate + - version + type: object + minItems: 1 + type: array + secret: + description: |- + Secret for object storage authentication. + Name of a secret in the same namespace as the LokiStack custom resource. + properties: + credentialMode: + description: |- + CredentialMode can be used to set the desired credential mode for authenticating with the object storage. + If this is not set, then the operator tries to infer the credential mode from the provided secret and its + own configuration. + enum: + - static + - token + - token-cco + type: string + name: + description: Name of a secret in the namespace configured + for object storage secrets. + type: string + type: + description: Type of object storage that should be used + enum: + - azure + - gcs + - s3 + - swift + - alibabacloud + type: string + required: + - name + - type + type: object + tls: + description: TLS configuration for reaching the object storage + endpoint. + properties: + caKey: + description: |- + Key is the data key of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + If empty, it defaults to "service-ca.crt". + type: string + caName: + description: |- + CA is the name of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + type: string + required: + - caName + type: object + required: + - secret + type: object secret: description: Secret containing OpenStack password information type: string @@ -1148,6 +1237,9 @@ spec: description: ServiceUser - optional username used for this service to register in cloudkitty type: string + storageClass: + description: Storage class used for Loki + type: string topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index 3020961a..9891a4bc 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -19,10 +19,15 @@ package controllers import ( "context" "fmt" + "slices" "strconv" + "time" + + lokistackv1 "github.com/grafana/loki/operator/api/loki/v1" "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" "github.com/openstack-k8s-operators/telemetry-operator/pkg/metricstorage" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/utils" k8s_errors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -36,13 +41,16 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" + certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" "github.com/go-logr/logr" networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/certmanager" "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/configmap" "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" "github.com/openstack-k8s-operators/lib-common/modules/common/env" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" @@ -79,11 +87,7 @@ func (r *CloudKittyReconciler) GetScheme() *runtime.Scheme { } // CloudKittyReconciler reconciles a CloudKitty object -type CloudKittyReconciler struct { - client.Client - Kclient kubernetes.Interface - Scheme *runtime.Scheme -} +type CloudKittyReconciler utils.ConditionalWatchingReconciler // GetLogger returns a logger object with a logging prefix of "controller.name" and additional controller context fields func (r *CloudKittyReconciler) GetLogger(ctx context.Context) logr.Logger { @@ -191,6 +195,8 @@ func (r *CloudKittyReconciler) Reconcile(ctx context.Context, req ctrl.Request) condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), condition.UnknownCondition(telemetryv1.CloudKittyAPIReadyCondition, condition.InitReason, telemetryv1.CloudKittyAPIReadyInitMessage), condition.UnknownCondition(telemetryv1.CloudKittyProcReadyCondition, condition.InitReason, telemetryv1.CloudKittyProcReadyInitMessage), + condition.UnknownCondition(telemetryv1.CloudKittyClientCertReadyCondition, condition.InitReason, telemetryv1.CloudKittyClientCertReadyInitMessage), + condition.UnknownCondition(telemetryv1.CloudKittyLokiStackReadyCondition, condition.InitReason, telemetryv1.CloudKittyLokiStackReadyInitMessage), condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), // service account, role, rolebinding conditions condition.UnknownCondition(condition.ServiceAccountReadyCondition, condition.InitReason, condition.ServiceAccountReadyInitMessage), @@ -327,7 +333,7 @@ func (r *CloudKittyReconciler) SetupWithManager(mgr ctrl.Manager) error { return nil } - return ctrl.NewControllerManagedBy(mgr). + control, err := ctrl.NewControllerManagedBy(mgr). For(&telemetryv1.CloudKitty{}). Owns(&mariadbv1.MariaDBDatabase{}). Owns(&mariadbv1.MariaDBAccount{}). @@ -336,9 +342,11 @@ func (r *CloudKittyReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&rabbitmqv1.TransportURL{}). Owns(&batchv1.Job{}). Owns(&corev1.Secret{}). + Owns(&corev1.ConfigMap{}). Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.Role{}). Owns(&rbacv1.RoleBinding{}). + Owns(&certmgrv1.Certificate{}). // Watch for TransportURL Secrets which belong to any TransportURLs created by CloudKitty CRs Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(transportURLSecretFn)). @@ -347,7 +355,10 @@ func (r *CloudKittyReconciler) SetupWithManager(mgr ctrl.Manager) error { Watches(&keystonev1.KeystoneAPI{}, handler.EnqueueRequestsFromMapFunc(r.findObjectForSrc), builder.WithPredicates(keystonev1.KeystoneAPIStatusChangedPredicate)). - Complete(r) + // LokiStack watch added dynamically inside the controller code. + Build(r) + r.Controller = control + return err } func (r *CloudKittyReconciler) findObjectForSrc(ctx context.Context, src client.Object) []reconcile.Request { @@ -507,11 +518,203 @@ func (r *CloudKittyReconciler) reconcileInit( return ctrl.Result{}, nil } +// Original source: +// https://github.com/openstack-k8s-operators/openstack-operator/blob/cf133b39e91c05f53c57725d7c6f5a627d98dccd/pkg/openstack/ca.go#L687 +func getCAFromSecret( + ctx context.Context, + instance *telemetryv1.CloudKitty, + helper *helper.Helper, + secretName string, +) (string, ctrl.Result, error) { + caSecret, ctrlResult, err := secret.GetDataFromSecret(ctx, helper, secretName, time.Duration(5), "ca.crt") + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyLokiStackReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + telemetryv1.CloudKittyLokiStackReadyErrorMessage, + err.Error())) + + return "", ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyLokiStackReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + telemetryv1.CloudKittyLokiStackReadyRunningMessage)) + + return "", ctrlResult, nil + } + + return caSecret, ctrl.Result{}, nil +} + func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *telemetryv1.CloudKitty, helper *helper.Helper) (ctrl.Result, error) { Log := r.GetLogger(ctx) Log.Info(fmt.Sprintf("Reconciling Service '%s'", instance.Name)) + // Create cloudkitty client cert / key + certIssuer, err := certmanager.GetIssuerByLabels( + ctx, helper, instance.Namespace, + map[string]string{certmanager.RootCAIssuerInternalLabel: ""}, + ) + if err != nil { + Log.Error(err, "Failed to determine certificate issuer") + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyClientCertReadyCondition, + condition.ErrorReason, + condition.SeverityError, + telemetryv1.CloudKittyClientCertReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + certDefinition := cloudkitty.Certificate( + instance, serviceLabels, certIssuer, + ) + cert := certmanager.NewCertificate(certDefinition, 5) + ctrlResult, _, err := cert.CreateOrPatch(ctx, helper, nil) + + if err != nil { + Log.Error(err, "Failed to create or patch cloudkitty client certificate") + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyClientCertReadyCondition, + condition.ErrorReason, + condition.SeverityError, + telemetryv1.CloudKittyClientCertReadyErrorMessage, + err.Error())) + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + Log.Info("CloudKitty client certificate is being created") + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyClientCertReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + telemetryv1.CloudKittyClientCertReadyRunningMessage)) + return ctrlResult, nil + } + + caData, ctrlResult, err := getCAFromSecret( + ctx, instance, helper, certDefinition.Spec.SecretName, + ) + if err != nil { + Log.Error(err, "Failed to get cloudkitty client certificate CA data") + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyClientCertReadyCondition, + condition.ErrorReason, + condition.SeverityError, + telemetryv1.CloudKittyClientCertReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + cms := []util.Template{ + { + Name: cloudkitty.CaConfigmapName, + Namespace: instance.Namespace, + Type: util.TemplateTypeNone, + InstanceType: "cloudkitty", + CustomData: map[string]string{ + cloudkitty.CaConfigmapKey: caData, + }, + }, + } + + err = configmap.EnsureConfigMaps( + ctx, helper, instance, cms, nil, + ) + if err != nil { + Log.Error(err, "Failed to create CA configmap for cloudkitty client cert verification") + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyClientCertReadyCondition, + condition.ErrorReason, + condition.SeverityError, + telemetryv1.CloudKittyClientCertReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + instance.Status.Conditions.MarkTrue(telemetryv1.CloudKittyClientCertReadyCondition, telemetryv1.CloudKittyClientCertReadyMessage) + + // Deploy Loki + var eventHandler handler.EventHandler = handler.EnqueueRequestForOwner( + r.Scheme, + r.RESTMapper, + &telemetryv1.CloudKitty{}, + handler.OnlyControllerOwner(), + ) + + err = utils.EnsureWatches( + (*utils.ConditionalWatchingReconciler)(r), ctx, + "lokistacks.loki.grafana.com", + &lokistackv1.LokiStack{}, eventHandler, helper, + ) + if err != nil { + instance.Status.Conditions.MarkFalse(telemetryv1.CloudKittyLokiStackReadyCondition, + condition.Reason("Can't own LokiStack resource. The loki-operator probably isn't installed"), + condition.SeverityError, + telemetryv1.CloudKittyLokiStackUnableToOwnMessage, err) + Log.Info("Can't own LokiStack resource. The loki-operator probably isn't installed") + return ctrl.Result{RequeueAfter: telemetryv1.PauseBetweenWatchAttempts}, nil + } + + lokiStack := &lokistackv1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-lokistack", instance.Name), + Namespace: instance.Namespace, + }, + } + op, err := controllerutil.CreateOrPatch(ctx, r.Client, lokiStack, func() error { + desiredLokiStack := cloudkitty.LokiStack(instance, serviceLabels) + desiredLokiStack.Spec.DeepCopyInto(&lokiStack.Spec) + lokiStack.ObjectMeta.Labels = serviceLabels + err = controllerutil.SetControllerReference(instance, lokiStack, r.Scheme) + return err + }) + if err != nil { + Log.Error(err, "Failed to create or patch LokiStack") + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyLokiStackReadyCondition, + condition.ErrorReason, + condition.SeverityError, + telemetryv1.CloudKittyLokiStackReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if op != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf("LokiStack %s successfully changed - operation: %s", lokiStack.Name, string(op))) + } + + // Mirror LokiStacks's condition here. LokiStack uses conditions + // a little differently than o-k-o. Whats more, it can have + // multiple 'active' conditions, while we have only one 'master' + // condition LokiStack here. So we mirror hopefully the one most + // relevant active condition in this order of + // priority: Ready > Failed > Degraded > Pending > Warning. + + order := []string{"Ready", "Failed", "Degraded", "Pending", "Warning"} + index := len(order) + reason := condition.InitReason + message := telemetryv1.CloudKittyLokiStackReadyInitMessage + for _, c := range lokiStack.Status.Conditions { + conditionIndex := slices.Index(order, c.Type) + if c.Status == "True" && conditionIndex < index { + index = conditionIndex + reason = c.Reason + message = c.Message + } + } + if index < len(order) && order[index] == "Ready" { + instance.Status.Conditions.MarkTrue(telemetryv1.CloudKittyLokiStackReadyCondition, telemetryv1.CloudKittyLokiStackReadyMessage) + } else { + Log.Info("LokiStack not ready") + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyLokiStackReadyCondition, + condition.Reason(reason), + condition.SeverityWarning, + fmt.Sprintf("LokiStack issue: %s", message))) + } + // Service account, role, binding rbacRules := []rbacv1.PolicyRule{ { @@ -726,7 +929,7 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te } // Handle service init - ctrlResult, err := r.reconcileInit(ctx, instance, helper, serviceLabels, serviceAnnotations) + ctrlResult, err = r.reconcileInit(ctx, instance, helper, serviceLabels, serviceAnnotations) if err != nil { return ctrlResult, err } else if (ctrlResult != ctrl.Result{}) { @@ -902,6 +1105,8 @@ func (r *CloudKittyReconciler) generateServiceConfigs( databaseAccount := db.GetAccount() dbSecret := db.GetSecret() + lokiHost := fmt.Sprintf("%s-lokistack-gateway-http.%s.svc", instance.Name, instance.Namespace) + templateParameters := make(map[string]interface{}) templateParameters["ServiceUser"] = instance.Spec.ServiceUser templateParameters["ServicePassword"] = string(ospSecret.Data[instance.Spec.PasswordSelectors.CloudKittyService]) @@ -910,6 +1115,8 @@ func (r *CloudKittyReconciler) generateServiceConfigs( templateParameters["TransportURL"] = string(transportURLSecret.Data["transport_url"]) templateParameters["PrometheusHost"] = instance.Status.PrometheusHost templateParameters["PrometheusPort"] = instance.Status.PrometheusPort + templateParameters["LokiHost"] = lokiHost + templateParameters["LokiPort"] = 8080 templateParameters["DatabaseConnection"] = fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf", databaseAccount.Spec.UserName, string(dbSecret.Data[mariadbv1.DatabasePasswordSelector]), diff --git a/controllers/cloudkittyapi_controller.go b/controllers/cloudkittyapi_controller.go index de2cb3ff..6df6937c 100644 --- a/controllers/cloudkittyapi_controller.go +++ b/controllers/cloudkittyapi_controller.go @@ -270,6 +270,18 @@ func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl } } } + + // Watch for changes to the client cert secret + if secretName == cloudkitty.ClientCertSecretName { + for _, cr := range apis.Items { + name := client.ObjectKey{ + Namespace: namespace, + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Secret %s is used by CloudKittyAPI CR %s", secretName, cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } if len(result) > 0 { return result } @@ -889,6 +901,26 @@ func (r *CloudKittyAPIReconciler) reconcileNormal(ctx context.Context, instance // normal reconcile tasks // + // Add client cert secret hash to all the other config data, so that + // restart is triggered on certificate changes (e.g. when they + // rotate) + _, clientCertHash, err := secret.GetSecret( + ctx, + helper, + cloudkitty.ClientCertSecretName, + instance.Namespace, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + configVars["client-cert"] = env.SetValue(clientCertHash) + // // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. diff --git a/controllers/cloudkittyproc_controller.go b/controllers/cloudkittyproc_controller.go index fbccef63..e04d1f19 100644 --- a/controllers/cloudkittyproc_controller.go +++ b/controllers/cloudkittyproc_controller.go @@ -244,6 +244,17 @@ func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctr } } } + // Watch for changes to the client cert secret + if secretName == cloudkitty.ClientCertSecretName { + for _, cr := range schedulers.Items { + name := client.ObjectKey{ + Namespace: namespace, + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Secret %s is used by CloudKittyProc CR %s", secretName, cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } if len(result) > 0 { return result } @@ -475,6 +486,26 @@ func (r *CloudKittyProcReconciler) reconcileNormal(ctx context.Context, instance return ctrl.Result{}, err } + // Add client cert secret hash to all the other config data, so that + // restart is triggered on certificate changes (e.g. when they + // rotate) + _, clientCertHash, err := secret.GetSecret( + ctx, + helper, + cloudkitty.ClientCertSecretName, + instance.Namespace, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + configVars["client-cert"] = env.SetValue(clientCertHash) + // // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. diff --git a/controllers/metricstorage_controller.go b/controllers/metricstorage_controller.go index a05b4ba8..6db87531 100644 --- a/controllers/metricstorage_controller.go +++ b/controllers/metricstorage_controller.go @@ -31,25 +31,16 @@ import ( discoveryv1 "k8s.io/api/discovery/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" - "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - - "k8s.io/apimachinery/pkg/api/meta" logr "github.com/go-logr/logr" "github.com/openstack-k8s-operators/lib-common/modules/ansible" @@ -93,15 +84,7 @@ var ( ) // MetricStorageReconciler reconciles a MetricStorage object -type MetricStorageReconciler struct { - client.Client - Kclient kubernetes.Interface - Scheme *runtime.Scheme - Controller controller.Controller - Watching []string - RESTMapper meta.RESTMapper - Cache cache.Cache -} +type MetricStorageReconciler utils.ConditionalWatchingReconciler // ConnectionInfo holds information about connection to a compute node type ConnectionInfo struct { @@ -318,7 +301,11 @@ func (r *MetricStorageReconciler) reconcileNormal( // Deploy monitoring stack - err := r.ensureWatches(ctx, "monitoringstacks.monitoring.rhobs", &obov1.MonitoringStack{}, eventHandler) + err := utils.EnsureWatches( + (*utils.ConditionalWatchingReconciler)(r), ctx, + "monitoringstacks.monitoring.rhobs", + &obov1.MonitoringStack{}, eventHandler, helper, + ) if err != nil { instance.Status.Conditions.MarkFalse(telemetryv1.MonitoringStackReadyCondition, condition.Reason("Can't own MonitoringStack resource. The Cluster Observability Operator probably isn't installed"), @@ -366,7 +353,13 @@ func (r *MetricStorageReconciler) reconcileNormal( } return []reconcile.Request{{NamespacedName: name}} } - err = r.ensureWatches(ctx, "prometheuses.monitoring.rhobs", &monv1.Prometheus{}, handler.EnqueueRequestsFromMapFunc(prometheusWatchFn)) + err = utils.EnsureWatches( + (*utils.ConditionalWatchingReconciler)(r), + ctx, "prometheuses.monitoring.rhobs", + &monv1.Prometheus{}, + handler.EnqueueRequestsFromMapFunc(prometheusWatchFn), + helper, + ) if err != nil { instance.Status.Conditions.MarkFalse(telemetryv1.PrometheusReadyCondition, condition.Reason("Can't watch prometheus resource. The Cluster Observability Operator probably isn't installed"), @@ -575,7 +568,13 @@ func (r *MetricStorageReconciler) reconcileNormal( } return []reconcile.Request{{NamespacedName: name}} } - err = r.ensureWatches(ctx, "prometheuses.monitoring.rhobs", &monv1.Prometheus{}, handler.EnqueueRequestsFromMapFunc(prometheusWatchFn)) + err = utils.EnsureWatches( + (*utils.ConditionalWatchingReconciler)(r), + ctx, "prometheuses.monitoring.rhobs", + &monv1.Prometheus{}, + handler.EnqueueRequestsFromMapFunc(prometheusWatchFn), + helper, + ) if err != nil { instance.Status.Conditions.MarkFalse(telemetryv1.PrometheusReadyCondition, condition.Reason("Can't watch prometheus resource. The Cluster Observability Operator probably isn't installed"), @@ -710,7 +709,11 @@ func (r *MetricStorageReconciler) createScrapeConfigs( helper *helper.Helper, ) (ctrl.Result, error) { Log := r.GetLogger(ctx) - err := r.ensureWatches(ctx, "scrapeconfigs.monitoring.rhobs", &monv1alpha1.ScrapeConfig{}, eventHandler) + err := utils.EnsureWatches( + (*utils.ConditionalWatchingReconciler)(r), + ctx, "scrapeconfigs.monitoring.rhobs", + &monv1alpha1.ScrapeConfig{}, eventHandler, helper, + ) if err != nil { instance.Status.Conditions.MarkFalse(telemetryv1.ScrapeConfigReadyCondition, condition.Reason("Can't own ScrapeConfig resource. The Cluster Observability Operator probably isn't installed"), @@ -959,7 +962,11 @@ func (r *MetricStorageReconciler) createDashboardObjects(ctx context.Context, in } // Deploy PrometheusRule for dashboards - err = r.ensureWatches(ctx, "prometheusrules.monitoring.rhobs", &monv1.PrometheusRule{}, eventHandler) + err = utils.EnsureWatches( + (*utils.ConditionalWatchingReconciler)(r), + ctx, "prometheusrules.monitoring.rhobs", + &monv1.PrometheusRule{}, eventHandler, helper, + ) if err != nil { instance.Status.Conditions.MarkFalse(telemetryv1.DashboardPrometheusRuleReadyCondition, condition.Reason("Can't own PrometheusRule resource. The Cluster Observability Operator probably isn't installed"), @@ -1079,43 +1086,6 @@ func (r *MetricStorageReconciler) createDashboardObjects(ctx context.Context, in return ctrl.Result{}, err } -func (r *MetricStorageReconciler) ensureWatches( - ctx context.Context, - name string, - kind client.Object, - handler handler.EventHandler, -) error { - Log := r.GetLogger(ctx) - for _, item := range r.Watching { - if item == name { - // We are already watching the resource - return nil - } - } - u := &unstructured.Unstructured{} - u.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "apiextensions.k8s.io", - Kind: "CustomResourceDefinition", - Version: "v1", - }) - - err := r.Client.Get(context.Background(), client.ObjectKey{ - Name: name, - }, u) - if err != nil { - return err - } - - Log.Info(fmt.Sprintf("Starting to watch %s", name)) - err = r.Controller.Watch(source.Kind(r.Cache, kind), - handler, - ) - if err == nil { - r.Watching = append(r.Watching, name) - } - return err -} - func getComputeNodesConnectionInfo( instance *telemetryv1.MetricStorage, helper *helper.Helper, diff --git a/go.mod b/go.mod index 466c105c..36b2976c 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,9 @@ go 1.21 replace github.com/openstack-k8s-operators/telemetry-operator/api => ./api require ( + github.com/cert-manager/cert-manager v1.14.7 github.com/go-logr/logr v1.4.2 + github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.6 github.com/onsi/ginkgo/v2 v2.20.1 github.com/onsi/gomega v1.34.1 @@ -13,6 +15,7 @@ require ( github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20250513115636-b549982a5d8f github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20250604143452-2260c431b9f1 github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20250423055245-3cb2ae8df6f0 + github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.0 github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20250508141203-be026d3164f7 github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20250415060817-dc849adfa27e github.com/openstack-k8s-operators/telemetry-operator/api v0.3.1-0.20240529090522-c780bd90b147 @@ -22,7 +25,7 @@ require ( k8s.io/api v0.29.15 k8s.io/apimachinery v0.29.15 k8s.io/client-go v0.29.15 - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 + k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 sigs.k8s.io/controller-runtime v0.17.6 ) @@ -67,11 +70,11 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/net v0.28.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.18.0 // indirect - golang.org/x/sys v0.23.0 // indirect - golang.org/x/term v0.23.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.24.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect @@ -84,6 +87,7 @@ require ( k8s.io/component-base v0.29.15 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240322212309-b815d8309940 // indirect + sigs.k8s.io/gateway-api v1.0.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/go.sum b/go.sum index 6b71026f..a77e2c08 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cert-manager/cert-manager v1.14.7 h1:C2L59sMGMdSpd8SPx5qfPAL7ejZaNxJBRd24S7Ws5Ek= +github.com/cert-manager/cert-manager v1.14.7/go.mod h1:0QE/Hzfs2SxNrFFYgFh/d0c0cDfNv9qSrAev2LFt5nM= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -49,6 +51,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gophercloud/gophercloud v1.14.1 h1:DTCNaTVGl8/cFu58O1JwWgis9gtISAFONqpMKNg/Vpw= github.com/gophercloud/gophercloud v1.14.1/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= +github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba h1:P5Wgp2HfGfNPLCPpS+YqquKdrrl4tW0El7VX23D6vtg= +github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba/go.mod h1:OBAgJh0mLYRvziBzBKr4/anrPHqGY9qEfuNXCpnUNi0= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -86,6 +90,8 @@ github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20250604143452 github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20250604143452-2260c431b9f1/go.mod h1:dgYQJbbVaRuP98yZZB3K1rNpqnF54I1HM1ZTaOzPKBY= github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20250423055245-3cb2ae8df6f0 h1:QKmpBfn9zIYcmODrvhnnrOx2CV1Y2t6M8DwNgLbaRbI= github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20250423055245-3cb2ae8df6f0/go.mod h1:0bajRHochTUT6Ecfriw27l3vL0yezVrnUmt3bcIpu4w= +github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.0 h1:cFOyP37qQ9T1D6mVTCwuPGt86LB4sTErpHT+L1e+VKY= +github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.0/go.mod h1:jgfvFeljXxot0LODLYCmjESxoMXbClXcBcf0DaX4zA0= github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20250508141203-be026d3164f7 h1:c3h1q3fDoit3NmvNL89xUL9A12bJivaTF+IOPEOAwLc= github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20250508141203-be026d3164f7/go.mod h1:UwHXRIrMSPJD3lFqrA4oKmRXVLFQCRkLAj9x6KLEHiQ= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20250508141203-be026d3164f7 h1:IybBq3PrxwdvzAF19TjdMCqbEVkX2p3gIkme/Fju6do= @@ -149,8 +155,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -165,19 +171,19 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -223,10 +229,12 @@ k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240322212309-b815d8309940 h1:qVoMaQV5t62UUvHe16Q3eb2c5HPzLHYzsi0Tu/xLndo= k8s.io/kube-openapi v0.0.0-20240322212309-b815d8309940/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.17.6 h1:12IXsozEsIXWAMRpgRlYS1jjAHQXHtWEOMdULh3DbEw= sigs.k8s.io/controller-runtime v0.17.6/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY= +sigs.k8s.io/gateway-api v1.0.0 h1:iPTStSv41+d9p0xFydll6d7f7MOBGuqXM6p2/zVYMAs= +sigs.k8s.io/gateway-api v1.0.0/go.mod h1:4cUgr0Lnp5FZ0Cdq8FdRwCvpiWws7LVhLHGIudLlf4c= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= diff --git a/main.go b/main.go index 7905d299..80690fa9 100644 --- a/main.go +++ b/main.go @@ -28,6 +28,7 @@ import ( "k8s.io/client-go/kubernetes" _ "k8s.io/client-go/plugin/pkg/client/auth" + certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -39,6 +40,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + lokistackv1 "github.com/grafana/loki/operator/api/loki/v1" networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" heatv1 "github.com/openstack-k8s-operators/heat-operator/api/v1beta1" memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" @@ -81,6 +83,8 @@ func init() { utilruntime.Must(rabbitmqclusterv1.AddToScheme(scheme)) utilruntime.Must(networkv1.AddToScheme(scheme)) utilruntime.Must(topologyv1.AddToScheme(scheme)) + utilruntime.Must(lokistackv1.AddToScheme(scheme)) + utilruntime.Must(certmgrv1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } @@ -199,9 +203,11 @@ func main() { } if err = (&controllers.CloudKittyReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Kclient: kclient, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + RESTMapper: mgr.GetRESTMapper(), + Cache: mgr.GetCache(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create CloudKitty controller") os.Exit(1) diff --git a/pkg/cloudkitty/cert.go b/pkg/cloudkitty/cert.go new file mode 100644 index 00000000..4a72e4cb --- /dev/null +++ b/pkg/cloudkitty/cert.go @@ -0,0 +1,67 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkitty + +import ( + "fmt" + + certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + certmgrmetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + "github.com/openstack-k8s-operators/lib-common/modules/certmanager" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" +) + +// Certificate defines a client certificate for communication between cloudkitty and loki +func Certificate( + instance *telemetryv1.CloudKitty, + labels map[string]string, + issuer *certmgrv1.Issuer, +) *certmgrv1.Certificate { + cert := certmanager.Cert( + ClientCertSecretName, + instance.Namespace, + labels, + certmgrv1.CertificateSpec{ + CommonName: fmt.Sprintf("%s.%s.svc", instance.Name, instance.Namespace), + DNSNames: []string{ + fmt.Sprintf("%s.%s.svc", instance.Name, instance.Namespace), + fmt.Sprintf("%s.%s.svc.cluster.local", instance.Name, instance.Namespace), + }, + SecretName: ClientCertSecretName, + Subject: &certmgrv1.X509Subject{ + OrganizationalUnits: []string{ + instance.Name, + }, + }, + PrivateKey: &certmgrv1.CertificatePrivateKey{ + Algorithm: "RSA", + Size: 3072, + }, + Usages: []certmgrv1.KeyUsage{ + certmgrv1.UsageDigitalSignature, + certmgrv1.UsageKeyEncipherment, + certmgrv1.UsageClientAuth, + }, + IssuerRef: certmgrmetav1.ObjectReference{ + Name: issuer.Name, + Kind: issuer.Kind, + Group: issuer.GroupVersionKind().Group, + }, + }, + ) + return cert +} diff --git a/pkg/cloudkitty/const.go b/pkg/cloudkitty/const.go index 8b372471..dc6589ba 100644 --- a/pkg/cloudkitty/const.go +++ b/pkg/cloudkitty/const.go @@ -52,6 +52,11 @@ const ( // PrometheusEndpointSecret - The name of the secret that contains the Prometheus endpoint configuration. PrometheusEndpointSecret = "metric-storage-prometheus-endpoint" + + ClientCertSecretName = "cert-cloudkitty-client-internal" + + CaConfigmapName = "lokistack-ca" + CaConfigmapKey = "ca.crt" ) var ResultRequeue = ctrl.Result{RequeueAfter: NormalDuration} diff --git a/pkg/cloudkitty/lokistack.go b/pkg/cloudkitty/lokistack.go new file mode 100644 index 00000000..8f38b577 --- /dev/null +++ b/pkg/cloudkitty/lokistack.go @@ -0,0 +1,118 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkitty + +import ( + "fmt" + + lokistackv1 "github.com/grafana/loki/operator/api/loki/v1" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// LokiStack defines a lokistack for cloudkitty +func LokiStack( + instance *telemetryv1.CloudKitty, + labels map[string]string, +) *lokistackv1.LokiStack { + lokiStack := &lokistackv1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-lokistack", instance.Name), + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: lokistackv1.LokiStackSpec{ + // TODO: What size do we even want? I assume something + // smallish since only rating interact with this + Size: lokistackv1.LokiStackSizeType("1x.demo"), + Storage: instance.Spec.S3StorageConfig, + StorageClassName: instance.Spec.StorageClass, + Tenants: &lokistackv1.TenantsSpec{ + Mode: lokistackv1.Static, + Authentication: []lokistackv1.AuthenticationSpec{ + { + TenantName: instance.Name, + TenantID: instance.Name, + MTLS: &lokistackv1.MTLSSpec{ + CA: &lokistackv1.CASpec{ + CAKey: CaConfigmapKey, + CA: CaConfigmapName, + }, + }, + }, + }, + Authorization: &lokistackv1.AuthorizationSpec{ + // TODO: Determine what exactly this does and what's needed here + Roles: []lokistackv1.RoleSpec{ + { + Name: "cloudkitty-logs", + Resources: []string{ + "logs", + }, + Tenants: []string{ + "cloudkitty", + }, + Permissions: []lokistackv1.PermissionType{ + lokistackv1.Write, + lokistackv1.Read, + }, + }, + { + Name: "cluster-reader", + Resources: []string{ + "logs", + }, + Tenants: []string{ + "cloudkitty", + }, + Permissions: []lokistackv1.PermissionType{ + lokistackv1.Read, + }, + }, + }, + RoleBindings: []lokistackv1.RoleBindingsSpec{ + { + Name: "cloudkitty-logs", + Subjects: []lokistackv1.Subject{ + { + Name: "cloudkitty", + Kind: lokistackv1.Group, + }, + }, + Roles: []string{ + "cloudkitty-logs", + }, + }, + { + Name: "cluster-reader", + Subjects: []lokistackv1.Subject{ + { + Name: "cloudkitty-logs-admin", + Kind: lokistackv1.Group, + }, + }, + Roles: []string{ + "cluster-reader", + }, + }, + }, + }, + }, + }, + } + return lokiStack +} diff --git a/pkg/cloudkitty/volumes.go b/pkg/cloudkitty/volumes.go index 97ed4ab5..e370748a 100644 --- a/pkg/cloudkitty/volumes.go +++ b/pkg/cloudkitty/volumes.go @@ -30,6 +30,14 @@ func GetVolumes(name string) []corev1.Volume { SecretName: name + "-config-data", }, }, + }, { + Name: "client-cert", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &configMode, + SecretName: ClientCertSecretName, + }, + }, }, } } @@ -53,5 +61,10 @@ func GetVolumeMounts(serviceName string) []corev1.VolumeMount { SubPath: serviceName + "-config.json", ReadOnly: true, }, + { + Name: "client-cert", + MountPath: "/etc/cloudkitty/certs", + ReadOnly: true, + }, } } diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index e19c58ca..0cde4c6e 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -18,14 +18,34 @@ package utils import ( "context" + "fmt" k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/source" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" ) +type ConditionalWatchingReconciler struct { + client.Client + Kclient kubernetes.Interface + Scheme *runtime.Scheme + Controller controller.Controller + Watching []string + RESTMapper meta.RESTMapper + Cache cache.Cache +} + // EnsureDeleted - Delete the object which in turn will clean the sub resources func EnsureDeleted(ctx context.Context, helper *helper.Helper, obj client.Object) (ctrl.Result, error) { key := client.ObjectKeyFromObject(obj) @@ -44,3 +64,42 @@ func EnsureDeleted(ctx context.Context, helper *helper.Helper, obj client.Object return ctrl.Result{}, nil } + +func EnsureWatches( + r *ConditionalWatchingReconciler, + ctx context.Context, + name string, + kind client.Object, + handler handler.EventHandler, + helper *helper.Helper, +) error { + Log := helper.GetLogger() + for _, item := range r.Watching { + if item == name { + // We are already watching the resource + return nil + } + } + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "apiextensions.k8s.io", + Kind: "CustomResourceDefinition", + Version: "v1", + }) + + err := r.Client.Get(context.Background(), client.ObjectKey{ + Name: name, + }, u) + if err != nil { + return err + } + + Log.Info(fmt.Sprintf("Starting to watch %s", name)) + err = r.Controller.Watch(source.Kind(r.Cache, kind), + handler, + ) + if err == nil { + r.Watching = append(r.Watching, name) + } + return err +} diff --git a/templates/cloudkitty/config/cloudkitty.conf b/templates/cloudkitty/config/cloudkitty.conf index 318ed398..88662886 100644 --- a/templates/cloudkitty/config/cloudkitty.conf +++ b/templates/cloudkitty/config/cloudkitty.conf @@ -64,7 +64,8 @@ version = 2 backend = loki [storage_loki] -url = http://loki:3100/loki/api/v1 +url = https://{{ .LokiHost }}:{{ .LokiPort }}/api/logs/v1/cloudkitty/loki/api/v1 +# TODO: Use client cert from /etc/cloudkitty/certs [database] connection = {{ .DatabaseConnection }} From 2d8132ef3fa79bceec06b2b858d6be5a8989b943 Mon Sep 17 00:00:00 2001 From: Jaromir Wysoglad Date: Tue, 16 Sep 2025 13:33:35 -0400 Subject: [PATCH 09/12] Mount certificates correctly --- pkg/cloudkitty/volumes.go | 24 ++++-- pkg/cloudkittyproc/statefulset.go | 74 +++++++++---------- .../config/cloudkitty-api-config.json | 6 ++ templates/cloudkitty/config/cloudkitty.conf | 4 +- 4 files changed, 65 insertions(+), 43 deletions(-) diff --git a/pkg/cloudkitty/volumes.go b/pkg/cloudkitty/volumes.go index e370748a..400e7fcc 100644 --- a/pkg/cloudkitty/volumes.go +++ b/pkg/cloudkitty/volumes.go @@ -8,7 +8,7 @@ var ( // scriptMode is the default permissions mode for Scripts volume scriptMode int32 = 0740 // configMode is the 640 permissions mode - configMode int32 = 0640 + configMode int32 = 0644 ) // GetVolumes - service volumes @@ -31,11 +31,25 @@ func GetVolumes(name string) []corev1.Volume { }, }, }, { - Name: "client-cert", + Name: "certs", VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: ClientCertSecretName, + }, + }, + }, { + ConfigMap: &corev1.ConfigMapProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: name + "-lokistack-gateway-ca-bundle", + }, + }, + }, + }, DefaultMode: &configMode, - SecretName: ClientCertSecretName, }, }, }, @@ -62,7 +76,7 @@ func GetVolumeMounts(serviceName string) []corev1.VolumeMount { ReadOnly: true, }, { - Name: "client-cert", + Name: "certs", MountPath: "/etc/cloudkitty/certs", ReadOnly: true, }, diff --git a/pkg/cloudkittyproc/statefulset.go b/pkg/cloudkittyproc/statefulset.go index a568be4d..fc0c9949 100644 --- a/pkg/cloudkittyproc/statefulset.go +++ b/pkg/cloudkittyproc/statefulset.go @@ -24,7 +24,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" + // "k8s.io/apimachinery/pkg/util/intstr" ) const ( @@ -45,30 +45,30 @@ func StatefulSet( // cloudKittyGroup := int64(telemetryv1.CloudKittyGroupID) // TODO until we determine how to properly query for these - livenessProbe := &corev1.Probe{ - // TODO might need tuning - TimeoutSeconds: 5, - PeriodSeconds: 3, - InitialDelaySeconds: 3, - } - - startupProbe := &corev1.Probe{ - TimeoutSeconds: 5, - FailureThreshold: 12, - PeriodSeconds: 5, - InitialDelaySeconds: 5, - } + //livenessProbe := &corev1.Probe{ + // // TODO might need tuning + // TimeoutSeconds: 5, + // PeriodSeconds: 3, + // InitialDelaySeconds: 3, + //} + + //startupProbe := &corev1.Probe{ + // TimeoutSeconds: 5, + // FailureThreshold: 12, + // PeriodSeconds: 5, + // InitialDelaySeconds: 5, + //} args := []string{"-c", ServiceCommand} - var probeCommand []string - livenessProbe.HTTPGet = &corev1.HTTPGetAction{ - Port: intstr.FromInt(8080), - } - startupProbe.HTTPGet = livenessProbe.HTTPGet - probeCommand = []string{ - "/var/lib/openstack/bin/healthcheck.py", - "/etc/cloudkitty/cloudkitty.conf.d/cloudkitty.conf", - } + //var probeCommand []string + // livenessProbe.HTTPGet = &corev1.HTTPGetAction{ + // Port: intstr.FromInt(8080), + // } + // startupProbe.HTTPGet = livenessProbe.HTTPGet + //probeCommand = []string{ + // "/var/lib/openstack/bin/healthcheck.py", + // "/etc/cloudkitty/cloudkitty.conf.d/cloudkitty.conf", + // } envVars := map[string]env.Setter{} envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") @@ -112,22 +112,22 @@ func StatefulSet( SecurityContext: &corev1.SecurityContext{ RunAsUser: &cloudKittyUser, }, - Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: volumeMounts, - Resources: instance.Spec.Resources, - LivenessProbe: livenessProbe, - StartupProbe: startupProbe, - }, - { - Name: "probe", - Command: probeCommand, - Image: instance.Spec.ContainerImage, - SecurityContext: &corev1.SecurityContext{ - RunAsUser: &cloudKittyUser, - //RunAsGroup: &cloudKittyGroup, - }, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), VolumeMounts: volumeMounts, + Resources: instance.Spec.Resources, + // LivenessProbe: livenessProbe, + // StartupProbe: startupProbe, }, + // { + // Name: "probe", + // Command: probeCommand, + // Image: instance.Spec.ContainerImage, + // SecurityContext: &corev1.SecurityContext{ + // RunAsUser: &cloudKittyUser, + // //RunAsGroup: &cloudKittyGroup, + // }, + // VolumeMounts: volumeMounts, + // }, }, Volumes: volumes, }, diff --git a/templates/cloudkitty/config/cloudkitty-api-config.json b/templates/cloudkitty/config/cloudkitty-api-config.json index 107cfa1a..13ea54b4 100644 --- a/templates/cloudkitty/config/cloudkitty-api-config.json +++ b/templates/cloudkitty/config/cloudkitty-api-config.json @@ -7,6 +7,12 @@ "owner": "cloudkitty", "perm": "0600" }, + { + "source": "/var/lib/openstack/config/metrics.yaml", + "dest": "/etc/cloudkitty/metrics.yaml", + "owner": "cloudkitty", + "perm": "0600" + }, { "source": "/var/lib/openstack/config/custom.conf", "dest": "/etc/aodh/cloudkitty.conf.d/01-cloudkitty-custom.conf", diff --git a/templates/cloudkitty/config/cloudkitty.conf b/templates/cloudkitty/config/cloudkitty.conf index 88662886..5faf0864 100644 --- a/templates/cloudkitty/config/cloudkitty.conf +++ b/templates/cloudkitty/config/cloudkitty.conf @@ -65,7 +65,9 @@ backend = loki [storage_loki] url = https://{{ .LokiHost }}:{{ .LokiPort }}/api/logs/v1/cloudkitty/loki/api/v1 -# TODO: Use client cert from /etc/cloudkitty/certs +ca_file = /etc/cloudkitty/certs/service-ca.crt +cert_file = /etc/cloudkitty/certs/tls.crt +key_file = /etc/cloudkitty/certs/tls.key [database] connection = {{ .DatabaseConnection }} From 88199a1cd9a6e4386f4d00ef741d0921aafb609c Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Wed, 17 Sep 2025 15:34:10 -0400 Subject: [PATCH 10/12] [cloudkitty] Add a default value for cloudkitty.s3StorageConfig The s3StorageConfig.secret is passed to Lokistack, which requires some value be set. The default is needed, or else there will be an error when it is not configured. The default is set to a reasonable value for the secret name (cloudkitty-loki-s3) and the type (s3), which match the values we're planning on using for the CI environment. Lines added: 1 Lines generated: 16 --- api/bases/telemetry.openstack.org_cloudkitties.yaml | 4 ++++ api/bases/telemetry.openstack.org_telemetries.yaml | 4 ++++ api/v1beta1/cloudkitty_types.go | 1 + config/crd/bases/telemetry.openstack.org_cloudkitties.yaml | 4 ++++ config/crd/bases/telemetry.openstack.org_telemetries.yaml | 4 ++++ 5 files changed, 17 insertions(+) diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml index 15cfcf46..8911ed80 100644 --- a/api/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -576,6 +576,10 @@ spec: Needed to request a transportURL that is created and used in CloudKitty type: string s3StorageConfig: + default: + secret: + name: cloudkitty-loki-s3 + type: s3 description: S3 related configuration passed to Loki properties: schemas: diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index 72556e5d..f335891d 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -1141,6 +1141,10 @@ spec: Needed to request a transportURL that is created and used in CloudKitty type: string s3StorageConfig: + default: + secret: + name: cloudkitty-loki-s3 + type: s3 description: S3 related configuration passed to Loki properties: schemas: diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index b22592dd..baa82a7f 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -109,6 +109,7 @@ type CloudKittySpecBase struct { // S3 related configuration passed to Loki // +kubebuilder:validation:Optional + // +kubebuilder:default={secret: {name: "cloudkitty-loki-s3", type: "s3"}} S3StorageConfig lokistackv1.ObjectStorageSpec `json:"s3StorageConfig"` // Storage class used for Loki diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml index 15cfcf46..8911ed80 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -576,6 +576,10 @@ spec: Needed to request a transportURL that is created and used in CloudKitty type: string s3StorageConfig: + default: + secret: + name: cloudkitty-loki-s3 + type: s3 description: S3 related configuration passed to Loki properties: schemas: diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index 72556e5d..f335891d 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -1141,6 +1141,10 @@ spec: Needed to request a transportURL that is created and used in CloudKitty type: string s3StorageConfig: + default: + secret: + name: cloudkitty-loki-s3 + type: s3 description: S3 related configuration passed to Loki properties: schemas: From a93e0936f73a2b9883da91f22ce065a76bc84473 Mon Sep 17 00:00:00 2001 From: Jaromir Wysoglad Date: Fri, 19 Sep 2025 04:06:43 -0400 Subject: [PATCH 11/12] kuttl-tests and fixes --- Makefile | 6 ++ controllers/cloudkitty_controller.go | 2 +- controllers/cloudkittyapi_controller.go | 35 +++++++ controllers/cloudkittyproc_controller.go | 35 +++++++ pkg/cloudkitty/dbsync.go | 2 +- pkg/cloudkitty/lokistack.go | 2 +- pkg/cloudkitty/storageinit.go | 2 +- pkg/cloudkitty/volumes.go | 8 +- pkg/cloudkittyapi/statefulset.go | 2 +- pkg/cloudkittyapi/volumes.go | 4 +- pkg/cloudkittyproc/statefulset.go | 2 +- pkg/cloudkittyproc/volumes.go | 4 +- .../config/cloudkitty-api-config.json | 7 ++ .../config/cloudkitty-proc-config.json | 7 ++ tests/kuttl/suites/cloudkitty/config.yaml | 15 +++ .../deps/OpenStackControlPlane.yaml | 27 +++++ tests/kuttl/suites/cloudkitty/deps/infra.yaml | 42 ++++++++ .../suites/cloudkitty/deps/kustomization.yaml | 50 ++++++++++ .../suites/cloudkitty/deps/loki-operator.yaml | 27 +++++ .../cloudkitty/deps/loki-s3-secret.yaml | 10 ++ tests/kuttl/suites/cloudkitty/deps/minio.yaml | 99 +++++++++++++++++++ .../suites/cloudkitty/deps/namespace.yaml | 4 + .../suites/cloudkitty/deps/telemetry.yaml | 7 ++ tests/kuttl/suites/cloudkitty/output/.keep | 0 .../suites/cloudkitty/tests/00-deps.yaml | 9 ++ .../suites/cloudkitty/tests/01-assert.yaml | 81 +++++++++++++++ .../suites/cloudkitty/tests/01-deploy.yaml | 52 ++++++++++ 27 files changed, 528 insertions(+), 13 deletions(-) create mode 100644 tests/kuttl/suites/cloudkitty/config.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/OpenStackControlPlane.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/infra.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/kustomization.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/loki-operator.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/loki-s3-secret.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/minio.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/namespace.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/telemetry.yaml create mode 100644 tests/kuttl/suites/cloudkitty/output/.keep create mode 100644 tests/kuttl/suites/cloudkitty/tests/00-deps.yaml create mode 100644 tests/kuttl/suites/cloudkitty/tests/01-assert.yaml create mode 100644 tests/kuttl/suites/cloudkitty/tests/01-deploy.yaml diff --git a/Makefile b/Makefile index aa0d4c18..834a4473 100644 --- a/Makefile +++ b/Makefile @@ -393,6 +393,12 @@ kuttl-test-cleanup: if [ "$(KUTTL_SUITE)" == "ceilometer" ]; then \ oc delete --wait=true --all=true -n $(KUTTL_NAMESPACE) --timeout=120s Ceilometer; \ fi; \ + if [ "$(KUTTL_SUITE)" == "metric-storage" ]; then \ + oc delete --wait=true --all=true -n $(KUTTL_NAMESPACE) --timeout=120s MetricStorage; \ + fi; \ + if [ "$(KUTTL_SUITE)" == "cloudkitty" ]; then \ + oc delete --wait=true --all=true -n $(KUTTL_NAMESPACE) --timeout=120s CloudKitty; \ + fi; \ if [ "$(KUTTL_SUITE)" == "default" ]; then \ oc delete --wait=true --all=true -n $(KUTTL_NAMESPACE) --timeout=120s Telemetry; \ fi; \ diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index 9891a4bc..cb7a9ea8 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -610,7 +610,7 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te cms := []util.Template{ { - Name: cloudkitty.CaConfigmapName, + Name: fmt.Sprintf("%s-%s", instance.Name, cloudkitty.CaConfigmapName), Namespace: instance.Namespace, Type: util.TemplateTypeNone, InstanceType: "cloudkitty", diff --git a/controllers/cloudkittyapi_controller.go b/controllers/cloudkittyapi_controller.go index 6df6937c..a47014b1 100644 --- a/controllers/cloudkittyapi_controller.go +++ b/controllers/cloudkittyapi_controller.go @@ -288,6 +288,39 @@ func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl return nil } + // Watch for changes to configmaps we don't own. + configMapFn := func(_ context.Context, o client.Object) []reconcile.Request { + var namespace string = o.GetNamespace() + var configMapName string = o.GetName() + result := []reconcile.Request{} + + // get all API CRs + apis := &telemetryv1.CloudKittyAPIList{} + listOpts := []client.ListOption{ + client.InNamespace(namespace), + } + if err := r.Client.List(context.Background(), apis, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve API CRs %v") + return nil + } + + // Watch for changes to the ca cert config map + for _, cr := range apis.Items { + if configMapName == fmt.Sprintf("%s-lokistack-gateway-ca-bundle", cloudkitty.GetOwningCloudKittyName(&cr)) { + name := client.ObjectKey{ + Namespace: namespace, + Name: cr.Name, + } + Log.Info(fmt.Sprintf("ConfigMap %s is used by CloudKittyAPI CR %s", configMapName, cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + if len(result) > 0 { + return result + } + return nil + } + // index passwordSecretField if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyPasswordSecretField, func(rawObj client.Object) []string { // Extract the secret name from the spec, if one is provided @@ -362,6 +395,8 @@ func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), ). + Watches(&corev1.ConfigMap{}, + handler.EnqueueRequestsFromMapFunc(configMapFn)). Watches(&topologyv1.Topology{}, handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), builder.WithPredicates(predicate.GenerationChangedPredicate{})). diff --git a/controllers/cloudkittyproc_controller.go b/controllers/cloudkittyproc_controller.go index e04d1f19..495f3332 100644 --- a/controllers/cloudkittyproc_controller.go +++ b/controllers/cloudkittyproc_controller.go @@ -261,6 +261,39 @@ func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctr return nil } + // Watch for changes to configmaps we don't own. + configMapFn := func(_ context.Context, o client.Object) []reconcile.Request { + var namespace string = o.GetNamespace() + var configMapName string = o.GetName() + result := []reconcile.Request{} + + // get all Proc CRs + procs := &telemetryv1.CloudKittyProcList{} + listOpts := []client.ListOption{ + client.InNamespace(namespace), + } + if err := r.Client.List(context.Background(), procs, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve Proc CRs %v") + return nil + } + + // Watch for changes to the ca cert config map + for _, cr := range procs.Items { + if configMapName == fmt.Sprintf("%s-lokistack-gateway-ca-bundle", cloudkitty.GetOwningCloudKittyName(&cr)) { + name := client.ObjectKey{ + Namespace: namespace, + Name: cr.Name, + } + Log.Info(fmt.Sprintf("ConfigMap %s is used by CloudKittyProc CR %s", configMapName, cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + if len(result) > 0 { + return result + } + return nil + } + // index passwordSecretField if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyProc{}, cloudKittyPasswordSecretField, func(rawObj client.Object) []string { // Extract the secret name from the spec, if one is provided @@ -308,6 +341,8 @@ func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctr handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), ). + Watches(&corev1.ConfigMap{}, + handler.EnqueueRequestsFromMapFunc(configMapFn)). Watches(&topologyv1.Topology{}, handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), builder.WithPredicates(predicate.GenerationChangedPredicate{})). diff --git a/pkg/cloudkitty/dbsync.go b/pkg/cloudkitty/dbsync.go index 5a84523a..28fb3161 100644 --- a/pkg/cloudkitty/dbsync.go +++ b/pkg/cloudkitty/dbsync.go @@ -41,7 +41,7 @@ func DbSyncJob(instance *telemetryv1.CloudKitty, labels map[string]string, annot args = append(args, dbSyncCommand) // create Volume and VolumeMounts - volumes := GetVolumes("cloudkitty") + volumes := GetVolumes(instance.Name) volumeMounts := GetVolumeMounts("cloudkitty-dbsync") // add CA cert if defined if instance.Spec.CloudKittyAPI.TLS.CaBundleSecretName != "" { diff --git a/pkg/cloudkitty/lokistack.go b/pkg/cloudkitty/lokistack.go index 8f38b577..c7604dc1 100644 --- a/pkg/cloudkitty/lokistack.go +++ b/pkg/cloudkitty/lokistack.go @@ -50,7 +50,7 @@ func LokiStack( MTLS: &lokistackv1.MTLSSpec{ CA: &lokistackv1.CASpec{ CAKey: CaConfigmapKey, - CA: CaConfigmapName, + CA: fmt.Sprintf("%s-%s", instance.Name, CaConfigmapName), }, }, }, diff --git a/pkg/cloudkitty/storageinit.go b/pkg/cloudkitty/storageinit.go index 73b05cd8..05418f99 100644 --- a/pkg/cloudkitty/storageinit.go +++ b/pkg/cloudkitty/storageinit.go @@ -40,7 +40,7 @@ func StorageInitJob(instance *telemetryv1.CloudKitty, labels map[string]string, args := []string{"-c", storageInitCommand} // create Volume and VolumeMounts - volumes := GetVolumes("cloudkitty") + volumes := GetVolumes(instance.Name) volumeMounts := GetVolumeMounts("cloudkitty-storageinit") // add CA cert if defined if instance.Spec.CloudKittyAPI.TLS.CaBundleSecretName != "" { diff --git a/pkg/cloudkitty/volumes.go b/pkg/cloudkitty/volumes.go index 400e7fcc..82b3d738 100644 --- a/pkg/cloudkitty/volumes.go +++ b/pkg/cloudkitty/volumes.go @@ -8,7 +8,9 @@ var ( // scriptMode is the default permissions mode for Scripts volume scriptMode int32 = 0740 // configMode is the 640 permissions mode - configMode int32 = 0644 + configMode int32 = 0640 + // certMode is the 400 permissions mode + certMode int32 = 0400 ) // GetVolumes - service volumes @@ -49,7 +51,7 @@ func GetVolumes(name string) []corev1.Volume { }, }, }, - DefaultMode: &configMode, + DefaultMode: &certMode, }, }, }, @@ -77,7 +79,7 @@ func GetVolumeMounts(serviceName string) []corev1.VolumeMount { }, { Name: "certs", - MountPath: "/etc/cloudkitty/certs", + MountPath: "/var/lib/openstack/loki-certs", ReadOnly: true, }, } diff --git a/pkg/cloudkittyapi/statefulset.go b/pkg/cloudkittyapi/statefulset.go index 94d4346c..431a662d 100644 --- a/pkg/cloudkittyapi/statefulset.go +++ b/pkg/cloudkittyapi/statefulset.go @@ -75,7 +75,7 @@ func StatefulSet( // create Volume and VolumeMounts volumes := GetVolumes(cloudkitty.GetOwningCloudKittyName(instance), instance.Name) - volumeMounts := GetVolumeMounts(instance.Name) + volumeMounts := GetVolumeMounts() // add CA cert if defined if instance.Spec.TLS.CaBundleSecretName != "" { diff --git a/pkg/cloudkittyapi/volumes.go b/pkg/cloudkittyapi/volumes.go index 6d9f36ab..4108fc3d 100644 --- a/pkg/cloudkittyapi/volumes.go +++ b/pkg/cloudkittyapi/volumes.go @@ -31,7 +31,7 @@ func GetVolumes(parentName string, name string) []corev1.Volume { } // GetVolumeMounts - CloudKitty API VolumeMounts -func GetVolumeMounts(parentName string) []corev1.VolumeMount { +func GetVolumeMounts() []corev1.VolumeMount { volumeMounts := []corev1.VolumeMount{ { Name: "config-data-custom", @@ -41,7 +41,7 @@ func GetVolumeMounts(parentName string) []corev1.VolumeMount { GetLogVolumeMount(), } - return append(cloudkitty.GetVolumeMounts(parentName), volumeMounts...) + return append(cloudkitty.GetVolumeMounts(cloudkitty.ServiceName+"-api"), volumeMounts...) } // GetLogVolumeMount - CloudKitty API LogVolumeMount diff --git a/pkg/cloudkittyproc/statefulset.go b/pkg/cloudkittyproc/statefulset.go index fc0c9949..c49c036d 100644 --- a/pkg/cloudkittyproc/statefulset.go +++ b/pkg/cloudkittyproc/statefulset.go @@ -75,7 +75,7 @@ func StatefulSet( envVars["CONFIG_HASH"] = env.SetValue(configHash) volumes := GetVolumes(cloudkitty.GetOwningCloudKittyName(instance), instance.Name) - volumeMounts := GetVolumeMounts(instance.Name) + volumeMounts := GetVolumeMounts() // Add the CA bundle if instance.Spec.TLS.CaBundleSecretName != "" { diff --git a/pkg/cloudkittyproc/volumes.go b/pkg/cloudkittyproc/volumes.go index 51ff46a7..22fdc822 100644 --- a/pkg/cloudkittyproc/volumes.go +++ b/pkg/cloudkittyproc/volumes.go @@ -25,7 +25,7 @@ func GetVolumes(parentName string, name string) []corev1.Volume { } // GetVolumeMounts - CloudKitty API VolumeMounts -func GetVolumeMounts(parentName string) []corev1.VolumeMount { +func GetVolumeMounts() []corev1.VolumeMount { volumeMounts := []corev1.VolumeMount{ { Name: "config-data-custom", @@ -34,5 +34,5 @@ func GetVolumeMounts(parentName string) []corev1.VolumeMount { }, } - return append(cloudkitty.GetVolumeMounts(parentName), volumeMounts...) + return append(cloudkitty.GetVolumeMounts(cloudkitty.ServiceName+"-proc"), volumeMounts...) } diff --git a/templates/cloudkitty/config/cloudkitty-api-config.json b/templates/cloudkitty/config/cloudkitty-api-config.json index 13ea54b4..dbf4af6f 100644 --- a/templates/cloudkitty/config/cloudkitty-api-config.json +++ b/templates/cloudkitty/config/cloudkitty-api-config.json @@ -69,6 +69,13 @@ "perm": "0640", "optional": true, "merge": true + }, + { + "source": "/var/lib/openstack/loki-certs/*", + "dest": "/etc/cloudkitty/certs/", + "owner": "cloudkitty:cloudkitty", + "perm": "0400", + "merge": true } ], "permissions": [ diff --git a/templates/cloudkitty/config/cloudkitty-proc-config.json b/templates/cloudkitty/config/cloudkitty-proc-config.json index 410ed9d3..b6d83aef 100644 --- a/templates/cloudkitty/config/cloudkitty-proc-config.json +++ b/templates/cloudkitty/config/cloudkitty-proc-config.json @@ -12,6 +12,13 @@ "dest": "/etc/cloudkitty/metrics.yaml", "owner": "cloudkitty", "perm": "0600" + }, + { + "source": "/var/lib/openstack/loki-certs/*", + "dest": "/etc/cloudkitty/certs/", + "owner": "cloudkitty:cloudkitty", + "perm": "0400", + "merge": true } ] } diff --git a/tests/kuttl/suites/cloudkitty/config.yaml b/tests/kuttl/suites/cloudkitty/config.yaml new file mode 100644 index 00000000..20588e37 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/config.yaml @@ -0,0 +1,15 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestSuite +reportFormat: JSON +reportName: kuttl-cloudkitty-results +namespace: telemetry-kuttl-cloudkitty +# we could set this lower, but the initial image pull can take a while +# TODO 300 +timeout: 300 +parallel: 1 +skipDelete: true +testDirs: + - tests/kuttl/suites/cloudkitty/ +suppress: + - events +artifactsDir: tests/kuttl/suites/cloudkitty/output diff --git a/tests/kuttl/suites/cloudkitty/deps/OpenStackControlPlane.yaml b/tests/kuttl/suites/cloudkitty/deps/OpenStackControlPlane.yaml new file mode 100644 index 00000000..f1b9f207 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/OpenStackControlPlane.yaml @@ -0,0 +1,27 @@ +apiVersion: core.openstack.org/v1beta1 +kind: OpenStackControlPlane +metadata: + name: openstack +spec: + storageClass: "crc-csi-hostpath-provisioner" + keystone: + template: + databaseInstance: openstack + secret: osp-secret + ironic: + enabled: false + template: + ironicConductors: [] + manila: + enabled: false + template: + manilaShares: {} + horizon: + enabled: false + nova: + enabled: false + placement: + template: + databaseInstance: openstack + secret: osp-secret + dataplane: diff --git a/tests/kuttl/suites/cloudkitty/deps/infra.yaml b/tests/kuttl/suites/cloudkitty/deps/infra.yaml new file mode 100644 index 00000000..7c47e716 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/infra.yaml @@ -0,0 +1,42 @@ +apiVersion: core.openstack.org/v1beta1 +kind: OpenStackControlPlane +metadata: + name: openstack +spec: + mariadb: + enabled: false + templates: + openstack: + replicas: 0 + openstack-cell1: + replicas: 0 + galera: + enabled: true + templates: + openstack: + replicas: 1 + storageRequest: 500M + openstack-cell1: + replicas: 1 + storageRequest: 500M + secret: osp-secret + secret: osp-secret + rabbitmq: + templates: + rabbitmq: + replicas: 1 + image: quay.io/podified-antelope-centos9/openstack-rabbitmq@sha256:41c36935b8b8cd3c5e490d1c03549ba2c0e8ddff50238fb2400d74613aa2e087 + rabbitmq-cell1: + replicas: 1 + memcached: + templates: + memcached: + replicas: 1 + ovn: + enabled: false + template: + ovnController: + external-ids: + ovn-encap-type: geneve + ovs: + enabled: false diff --git a/tests/kuttl/suites/cloudkitty/deps/kustomization.yaml b/tests/kuttl/suites/cloudkitty/deps/kustomization.yaml new file mode 100644 index 00000000..b366cb73 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/kustomization.yaml @@ -0,0 +1,50 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: telemetry-kuttl-cloudkitty + +secretGenerator: +- literals: + - AdminPassword=password + - DbRootPassword=password + - DatabasePassword=password + - KeystoneDatabasePassword=password + - PlacementPassword=password + - PlacementDatabasePassword=password + - GlancePassword=password + - GlanceDatabasePassword=password + - NeutronPassword=password + - NeutronDatabasePassword=password + - NovaPassword=password + - NovaAPIDatabasePassword=password + - NovaCell0DatabasePassword=password + - NovaCell1DatabasePassword=password + - AodhPassword=password + - AodhDatabasePassword=password + - CeilometerPassword=password + - CeilometerDatabasePassword=password + - HeatPassword=password + - HeatDatabasePassword=password + - HeatAuthEncryptionKey=66699966699966600666999666999666 + - MetadataSecret=42 + - CloudKittyPassword=password + name: osp-secret +generatorOptions: + disableNameSuffixHash: true + labels: + type: osp-secret + +resources: +- namespace.yaml +- OpenStackControlPlane.yaml +- loki-s3-secret.yaml + +patches: +- patch: |- + apiVersion: core.openstack.org/v1beta1 + kind: OpenStackControlPlane + metadata: + name: openstack + spec: + secret: osp-secret +- path: infra.yaml +- path: telemetry.yaml diff --git a/tests/kuttl/suites/cloudkitty/deps/loki-operator.yaml b/tests/kuttl/suites/cloudkitty/deps/loki-operator.yaml new file mode 100644 index 00000000..5b3d937a --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/loki-operator.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-operators-redhat + labels: + name: openshift-operators-redhat +--- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: loki-operator + namespace: openshift-operators-redhat +spec: + upgradeStrategy: Default +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: loki-operator + namespace: openshift-operators-redhat +spec: + channel: stable-6.1 + installPlanApproval: Automatic + name: loki-operator + source: redhat-operators + sourceNamespace: openshift-marketplace diff --git a/tests/kuttl/suites/cloudkitty/deps/loki-s3-secret.yaml b/tests/kuttl/suites/cloudkitty/deps/loki-s3-secret.yaml new file mode 100644 index 00000000..b176d7f9 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/loki-s3-secret.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: logging-loki-s3 +stringData: + access_key_id: minio + access_key_secret: minio123 + bucketnames: loki + endpoint: http://minio.minio-dev.svc.cluster.local:9000 diff --git a/tests/kuttl/suites/cloudkitty/deps/minio.yaml b/tests/kuttl/suites/cloudkitty/deps/minio.yaml new file mode 100644 index 00000000..f1854ff4 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/minio.yaml @@ -0,0 +1,99 @@ +--- +# Deploys a new Namespace for the MinIO Pod +apiVersion: v1 +kind: Namespace +metadata: + name: minio-dev # Change this value if you want a different namespace name + labels: + name: minio-dev # Change this value to match metadata.name +--- +# Deploys a new MinIO Pod into the metadata.namespace Kubernetes namespace +# +apiVersion: v1 +kind: Pod +metadata: + labels: + app: minio + name: minio + namespace: minio-dev # Change this value to match the namespace metadata.name +spec: + containers: + - name: minio + image: quay.io/minio/minio:latest + command: + - /bin/bash + - -c + - | + mkdir -p /data/loki && \ + minio server /data + env: + - name: MINIO_ACCESS_KEY + value: minio + - name: MINIO_SECRET_KEY + value: minio123 + volumeMounts: + - mountPath: /data + name: storage # Corresponds to the `spec.volumes` Persistent Volume + volumes: + - name: storage + persistentVolumeClaim: + claimName: minio-pvc +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio-pvc + namespace: minio-dev +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: crc-csi-hostpath-provisioner +--- +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: minio-dev +spec: + selector: + app: minio + ports: + - name: api + protocol: TCP + port: 9000 + - name: console + protocol: TCP + port: 9090 +--- +kind: Route +apiVersion: route.openshift.io/v1 +metadata: + name: minio-console + namespace: minio-dev +spec: + host: console-minio-dev.apps-crc.testing + to: + kind: Service + name: minio + weight: 100 + port: + targetPort: console + wildcardPolicy: None +--- +kind: Route +apiVersion: route.openshift.io/v1 +metadata: + name: minio-api + namespace: minio-dev +spec: + host: api-minio-dev.apps-crc.testing + to: + kind: Service + name: minio + weight: 100 + port: + targetPort: api + wildcardPolicy: None diff --git a/tests/kuttl/suites/cloudkitty/deps/namespace.yaml b/tests/kuttl/suites/cloudkitty/deps/namespace.yaml new file mode 100644 index 00000000..63c85762 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: telemetry-kuttl diff --git a/tests/kuttl/suites/cloudkitty/deps/telemetry.yaml b/tests/kuttl/suites/cloudkitty/deps/telemetry.yaml new file mode 100644 index 00000000..5c2714a2 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/telemetry.yaml @@ -0,0 +1,7 @@ +apiVersion: core.openstack.org/v1beta1 +kind: OpenStackControlPlane +metadata: + name: openstack +spec: + telemetry: + enabled: false diff --git a/tests/kuttl/suites/cloudkitty/output/.keep b/tests/kuttl/suites/cloudkitty/output/.keep new file mode 100644 index 00000000..e69de29b diff --git a/tests/kuttl/suites/cloudkitty/tests/00-deps.yaml b/tests/kuttl/suites/cloudkitty/tests/00-deps.yaml new file mode 100644 index 00000000..f850b416 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/tests/00-deps.yaml @@ -0,0 +1,9 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + oc apply -f ../deps/loki-operator.yaml + until oc api-resources | grep -q grafana; do sleep 1; done + - script: | + oc apply -f ../deps/minio.yaml + oc wait --for='jsonpath={.status.conditions[?(@.type=="Ready")].status}=True' pod/minio -n minio-dev diff --git a/tests/kuttl/suites/cloudkitty/tests/01-assert.yaml b/tests/kuttl/suites/cloudkitty/tests/01-assert.yaml new file mode 100644 index 00000000..6ce586ee --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/tests/01-assert.yaml @@ -0,0 +1,81 @@ +apiVersion: loki.grafana.com/v1 +kind: LokiStack +metadata: + name: telemetry-kuttl-cloudkitty-lokistack + ownerReferences: + - kind: CloudKitty + name: telemetry-kuttl-cloudkitty +spec: + tenants: + authentication: + - tenantId: telemetry-kuttl-cloudkitty + tenantName: telemetry-kuttl-cloudkitty +# NOTE: It's hard to assert LokiStack condition with kuttl-tests, because +# their number can change even when LokiStack as whole is actually. +# And because kuttl-tests will compare the number of conditions defined +# here vs. the number of conditions inside the real CR, it could fail +# even when it shouldn't. +# But LokiStack condition is being copied into the master CloudKitty CR, +# so we can ensure that LokiStack is ready by checking that Cloudkitty +# is Ready +--- +apiVersion: telemetry.openstack.org/v1beta1 +kind: CloudKitty +metadata: + name: telemetry-kuttl-cloudkitty +status: + conditions: + - status: "True" + type: Ready + - status: "True" + type: CloudKittyAPIReady + - status: "True" + type: CloudKittyClientCertReady + - status: "True" + type: CloudKittyLokiStackReady + - status: "True" + type: CloudKittyProcReady + - status: "True" + type: CloudKittyStorageInitReady + - status: "True" + type: DBReady + - status: "True" + type: DBSyncReady + - status: "True" + type: InputReady + - status: "True" + type: MariaDBAccountReady + - status: "True" + type: MemcachedReady + - status: "True" + type: NetworkAttachmentsReady + - status: "True" + type: RabbitMqTransportURLReady + - status: "True" + type: RoleBindingReady + - status: "True" + type: RoleReady + - status: "True" + type: ServiceAccountReady + - status: "True" + type: ServiceConfigReady +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: cert-cloudkitty-client-internal + ownerReferences: + - kind: CloudKitty + name: telemetry-kuttl-cloudkitty +spec: + subject: + organizationalUnits: + - telemetry-kuttl-cloudkitty + usages: + - digital signature + - key encipherment + - client auth +status: + conditions: + - status: "True" + type: Ready diff --git a/tests/kuttl/suites/cloudkitty/tests/01-deploy.yaml b/tests/kuttl/suites/cloudkitty/tests/01-deploy.yaml new file mode 100644 index 00000000..2dff94fe --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/tests/01-deploy.yaml @@ -0,0 +1,52 @@ +apiVersion: telemetry.openstack.org/v1beta1 +kind: CloudKitty +metadata: + name: telemetry-kuttl-cloudkitty +spec: + apiTimeout: 0 + cloudKittyAPI: + containerImage: quay.io/jwysogla/cloudkitty-api:latest + override: + service: + internal: + metadata: + labels: + osctlplane: "" + osctlplane-service: telemetry + public: + metadata: + labels: + osctlplane: "" + osctlplane-service: telemetry + replicas: 1 + resources: {} + tls: + api: + internal: {} + public: {} + caBundleSecretName: combined-ca-bundle + cloudKittyProc: + containerImage: quay.io/jwysogla/cloudkitty-processor:latest + replicas: 1 + resources: {} + tls: + caBundleSecretName: combined-ca-bundle + databaseAccount: cloudkitty + databaseInstance: openstack + memcachedInstance: memcached + passwordSelector: + aodhService: AodhPassword + ceilometerService: CeilometerPassword + cloudKittyService: CloudKittyPassword + preserveJobs: false + rabbitMqClusterName: rabbitmq + s3StorageConfig: + schemas: + - effectiveDate: "2024-11-18" + version: v13 + secret: + name: logging-loki-s3 + type: s3 + secret: osp-secret + serviceUser: cloudkitty + storageClass: local-storage From 93afe864da689f7166f2718cb0765800266d5ac2 Mon Sep 17 00:00:00 2001 From: Jaromir Wysoglad Date: Thu, 18 Sep 2025 11:55:16 -0400 Subject: [PATCH 12/12] [cloudkitty] Add missing rbac annotations --- config/rbac/role.yaml | 32 ++++++++++++++++++++++++++++ controllers/cloudkitty_controller.go | 4 ++++ 2 files changed, 36 insertions(+) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6fabb4df..7bbc14ea 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -58,6 +58,26 @@ rules: - patch - update - watch +- apiGroups: + - cert-manager.io + resources: + - certificates + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - cert-manager.io + resources: + - issuers + verbs: + - get + - list + - watch - apiGroups: - cloudkitty.openstack.org resources: @@ -181,6 +201,18 @@ rules: - patch - update - watch +- apiGroups: + - loki.grafana.com + resources: + - lokistacks + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - mariadb.openstack.org resources: diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index cb7a9ea8..3d431be1 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -104,6 +104,7 @@ func (r *CloudKittyReconciler) GetLogger(ctx context.Context) logr.Logger { // +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyprocs/status,verbs=get;update;patch // +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyprocs/finalizers,verbs=update;patch // +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;create;update;patch;delete;watch // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;create;update;patch;delete;watch // +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbdatabases,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbaccounts,verbs=get;list;watch;create;update;patch;delete @@ -112,6 +113,9 @@ func (r *CloudKittyReconciler) GetLogger(ctx context.Context) logr.Logger { // +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneapis,verbs=get;list;watch // +kubebuilder:rbac:groups=rabbitmq.openstack.org,resources=transporturls,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch +// +kubebuilder:rbac:groups=cert-manager.io,resources=certificates,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=cert-manager.io,resources=issuers,verbs=get;list;watch +// +kubebuilder:rbac:groups=loki.grafana.com,resources=lokistacks,verbs=get;list;watch;create;update;patch;delete // service account, role, rolebinding // +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch