From a25975abf534c1b22334ca775a5f4a8721d95423 Mon Sep 17 00:00:00 2001 From: jlarriba Date: Fri, 20 Jun 2025 14:22:07 +0200 Subject: [PATCH 01/87] Cloudkitty deployment --- PROJECT | 31 +- .../telemetry.openstack.org_autoscalings.yaml | 5 + .../telemetry.openstack.org_ceilometers.yaml | 5 + .../telemetry.openstack.org_cloudkitties.yaml | 665 ++++++++++ ...elemetry.openstack.org_cloudkittyapis.yaml | 499 ++++++++ ...lemetry.openstack.org_cloudkittyprocs.yaml | 321 +++++ .../telemetry.openstack.org_telemetries.yaml | 546 ++++++++ api/v1beta1/autoscaling_types.go | 12 +- api/v1beta1/cloudkitty_types.go | 270 ++++ api/v1beta1/cloudkitty_webhook.go | 102 ++ api/v1beta1/cloudkittyapi_types.go | 155 +++ api/v1beta1/cloudkittyproc_types.go | 148 +++ api/v1beta1/conditions.go | 51 + api/v1beta1/telemetry_types.go | 33 +- api/v1beta1/zz_generated.deepcopy.go | 630 ++++++++++ .../telemetry.openstack.org_autoscalings.yaml | 5 + .../telemetry.openstack.org_ceilometers.yaml | 5 + .../telemetry.openstack.org_cloudkitties.yaml | 665 ++++++++++ ...elemetry.openstack.org_cloudkittyapis.yaml | 499 ++++++++ ...lemetry.openstack.org_cloudkittyprocs.yaml | 321 +++++ .../telemetry.openstack.org_telemetries.yaml | 546 ++++++++ config/crd/kustomization.yaml | 9 + .../patches/cainjection_in_cloudkitties.yaml | 7 + .../cainjection_in_cloudkittyapis.yaml | 7 + .../cainjection_in_cloudkittyprocs.yaml | 7 + .../crd/patches/webhook_in_cloudkitties.yaml | 16 + .../patches/webhook_in_cloudkittyapis.yaml | 16 + .../patches/webhook_in_cloudkittyprocs.yaml | 16 + config/rbac/cloudkitty_editor_role.yaml | 31 + config/rbac/cloudkitty_viewer_role.yaml | 27 + config/rbac/cloudkittyapi_editor_role.yaml | 31 + config/rbac/cloudkittyapi_viewer_role.yaml | 27 + config/rbac/cloudkittyproc_editor_role.yaml | 31 + config/rbac/cloudkittyproc_viewer_role.yaml | 27 + config/rbac/role.yaml | 130 ++ config/samples/kustomization.yaml | 3 + .../samples/telemetry_v1beta1_cloudkitty.yaml | 12 + .../telemetry_v1beta1_cloudkittyapi.yaml | 12 + .../telemetry_v1beta1_cloudkittyproc.yaml | 12 + config/webhook/manifests.yaml | 40 + controllers/cloudkitty_controller.go | 1085 ++++++++++++++++ controllers/cloudkittyapi_controller.go | 1119 +++++++++++++++++ controllers/cloudkittyproc_controller.go | 759 +++++++++++ controllers/telemetry_controller.go | 116 ++ main.go | 32 + pkg/cloudkitty/common.go | 161 +++ pkg/cloudkitty/const.go | 54 + pkg/cloudkitty/dbsync.go | 107 ++ pkg/cloudkitty/funcs.go | 49 + pkg/cloudkitty/volumes.go | 57 + pkg/cloudkittyapi/const.go | 24 + pkg/cloudkittyapi/statefulset.go | 188 +++ pkg/cloudkittyapi/volumes.go | 54 + pkg/cloudkittyproc/const.go | 21 + pkg/cloudkittyproc/statefulset.go | 153 +++ pkg/cloudkittyproc/volumes.go | 38 + templates/cloudkitty/bin/healthcheck.py | 94 ++ templates/cloudkitty/bin/run-on-host | 2 + .../cloudkitty/config/10-cloudkitty_wsgi.conf | 40 + .../config/cloudkitty-api-config-httpd.json | 51 + .../config/cloudkitty-api-config.json | 11 + .../config/cloudkitty-api-uwsgi.ini | 17 + .../config/cloudkitty-dbsync-config.json | 11 + .../config/cloudkitty-proc-config.json | 17 + templates/cloudkitty/config/cloudkitty.conf | 78 ++ templates/cloudkitty/config/httpd.conf | 27 + templates/cloudkitty/config/metrics.yaml | 77 ++ templates/cloudkitty/config/ssl.conf | 21 + 68 files changed, 10423 insertions(+), 15 deletions(-) create mode 100644 api/bases/telemetry.openstack.org_cloudkitties.yaml create mode 100644 api/bases/telemetry.openstack.org_cloudkittyapis.yaml create mode 100644 api/bases/telemetry.openstack.org_cloudkittyprocs.yaml create mode 100644 api/v1beta1/cloudkitty_types.go create mode 100644 api/v1beta1/cloudkitty_webhook.go create mode 100644 api/v1beta1/cloudkittyapi_types.go create mode 100644 api/v1beta1/cloudkittyproc_types.go create mode 100644 config/crd/bases/telemetry.openstack.org_cloudkitties.yaml create mode 100644 config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml create mode 100644 config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml create mode 100644 config/crd/patches/cainjection_in_cloudkitties.yaml create mode 100644 config/crd/patches/cainjection_in_cloudkittyapis.yaml create mode 100644 config/crd/patches/cainjection_in_cloudkittyprocs.yaml create mode 100644 config/crd/patches/webhook_in_cloudkitties.yaml create mode 100644 config/crd/patches/webhook_in_cloudkittyapis.yaml create mode 100644 config/crd/patches/webhook_in_cloudkittyprocs.yaml create mode 100644 config/rbac/cloudkitty_editor_role.yaml create mode 100644 config/rbac/cloudkitty_viewer_role.yaml create mode 100644 config/rbac/cloudkittyapi_editor_role.yaml create mode 100644 config/rbac/cloudkittyapi_viewer_role.yaml create mode 100644 config/rbac/cloudkittyproc_editor_role.yaml create mode 100644 config/rbac/cloudkittyproc_viewer_role.yaml create mode 100644 config/samples/telemetry_v1beta1_cloudkitty.yaml create mode 100644 config/samples/telemetry_v1beta1_cloudkittyapi.yaml create mode 100644 config/samples/telemetry_v1beta1_cloudkittyproc.yaml create mode 100644 controllers/cloudkitty_controller.go create mode 100644 controllers/cloudkittyapi_controller.go create mode 100644 controllers/cloudkittyproc_controller.go create mode 100644 pkg/cloudkitty/common.go create mode 100644 pkg/cloudkitty/const.go create mode 100644 pkg/cloudkitty/dbsync.go create mode 100644 pkg/cloudkitty/funcs.go create mode 100644 pkg/cloudkitty/volumes.go create mode 100644 pkg/cloudkittyapi/const.go create mode 100644 pkg/cloudkittyapi/statefulset.go create mode 100644 pkg/cloudkittyapi/volumes.go create mode 100644 pkg/cloudkittyproc/const.go create mode 100644 pkg/cloudkittyproc/statefulset.go create mode 100644 pkg/cloudkittyproc/volumes.go create mode 100755 templates/cloudkitty/bin/healthcheck.py create mode 100755 templates/cloudkitty/bin/run-on-host create mode 100644 templates/cloudkitty/config/10-cloudkitty_wsgi.conf create mode 100644 templates/cloudkitty/config/cloudkitty-api-config-httpd.json create mode 100644 templates/cloudkitty/config/cloudkitty-api-config.json create mode 100644 templates/cloudkitty/config/cloudkitty-api-uwsgi.ini create mode 100644 templates/cloudkitty/config/cloudkitty-dbsync-config.json create mode 100644 templates/cloudkitty/config/cloudkitty-proc-config.json create mode 100644 templates/cloudkitty/config/cloudkitty.conf create mode 100644 templates/cloudkitty/config/httpd.conf create mode 100644 templates/cloudkitty/config/metrics.yaml create mode 100644 templates/cloudkitty/config/ssl.conf diff --git a/PROJECT b/PROJECT index 5230867a..58516761 100644 --- a/PROJECT +++ b/PROJECT @@ -1,7 +1,3 @@ -# Code generated by tool. DO NOT EDIT. -# This file is used to track the info used to scaffold your project -# and allow the plugins properly work. -# More info: https://book.kubebuilder.io/reference/project-config.html domain: openstack.org layout: - go.kubebuilder.io/v3 @@ -69,4 +65,31 @@ resources: defaulting: true validation: true webhookVersion: v1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: openstack.org + group: telemetry + kind: CloudKittyApi + path: github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1 + version: v1beta1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: openstack.org + group: telemetry + kind: CloudKittyProc + path: github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1 + version: v1beta1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: openstack.org + group: telemetry + kind: CloudKitty + path: github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1 + version: v1beta1 version: "3" diff --git a/api/bases/telemetry.openstack.org_autoscalings.yaml b/api/bases/telemetry.openstack.org_autoscalings.yaml index 4cfabab5..a4ef6ccf 100644 --- a/api/bases/telemetry.openstack.org_autoscalings.yaml +++ b/api/bases/telemetry.openstack.org_autoscalings.yaml @@ -294,6 +294,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object preserveJobs: default: false diff --git a/api/bases/telemetry.openstack.org_ceilometers.yaml b/api/bases/telemetry.openstack.org_ceilometers.yaml index 824f5daa..3307bc27 100644 --- a/api/bases/telemetry.openstack.org_ceilometers.yaml +++ b/api/bases/telemetry.openstack.org_ceilometers.yaml @@ -210,6 +210,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object proxyImage: type: string diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml new file mode 100644 index 00000000..bc5b7567 --- /dev/null +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -0,0 +1,665 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: cloudkitties.telemetry.openstack.org +spec: + group: telemetry.openstack.org + names: + kind: CloudKitty + listKind: CloudKittyList + plural: cloudkitties + singular: cloudkitty + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: CloudKitty is the Schema for the cloudkitties API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CloudKittySpec defines the desired state of CloudKitty + properties: + apiTimeout: + default: 60 + description: APITimeout for HAProxy, Apache, and rpc_response_timeout + minimum: 10 + type: integer + cloudKittyAPI: + description: CloudKittyAPI - Spec definition for the API service of + this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the + configurations of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret + for the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object + public: + description: Public GenericService - holds the secret + for the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in + a pre-created bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + cloudKittyProc: + description: CloudKittyProc - Spec definition for the Scheduler service + of this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config for all CloudKitty services using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for cloudkitty + DB, defaults to cloudkitty + type: string + databaseInstance: + description: |- + MariaDB instance name + Right now required by the maridb-operator to get the credentials from the instance to create the DB + Might not be required in future + type: string + memcachedInstance: + default: memcached + description: Memcached instance name. + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting + NodeSelector here acts as a default value and can be overridden by service + specific NodeSelector Settings. + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service password + from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean + rabbitMqClusterName: + default: rabbitmq + description: |- + RabbitMQ instance name + Needed to request a transportURL that is created and used in CloudKitty + type: string + secret: + description: Secret containing OpenStack password information + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - cloudKittyAPI + - cloudKittyProc + - databaseInstance + - memcachedInstance + - rabbitMqClusterName + - secret + type: object + status: + description: CloudKittyStatus defines the observed state of CloudKitty + properties: + apiEndpoints: + additionalProperties: + additionalProperties: + type: string + type: object + description: API endpoints + type: object + cloudKittyAPIReadyCount: + default: 0 + description: ReadyCount of CloudKitty API instance + format: int32 + minimum: 0 + type: integer + cloudKittyProcReadyCounts: + default: 0 + description: ReadyCount of CloudKitty Processor instances + format: int32 + minimum: 0 + type: integer + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + databaseHostname: + description: CloudKitty Database Hostname + type: string + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object + observedGeneration: + description: |- + ObservedGeneration - the most recent generation observed for this service. + If the observed generation is different than the spec generation, then the + controller has not started processing the latest changes, and the status + and its conditions are likely stale. + format: int64 + type: integer + serviceIDs: + additionalProperties: + type: string + description: ServiceIDs + type: object + transportURLSecret: + description: TransportURLSecret - Secret containing RabbitMQ transportURL + type: string + required: + - cloudKittyAPIReadyCount + - cloudKittyProcReadyCounts + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/api/bases/telemetry.openstack.org_cloudkittyapis.yaml b/api/bases/telemetry.openstack.org_cloudkittyapis.yaml new file mode 100644 index 00000000..d33b713c --- /dev/null +++ b/api/bases/telemetry.openstack.org_cloudkittyapis.yaml @@ -0,0 +1,499 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: cloudkittyapis.telemetry.openstack.org +spec: + group: telemetry.openstack.org + names: + kind: CloudKittyAPI + listKind: CloudKittyAPIList + plural: cloudkittyapis + singular: cloudkittyapi + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: CloudKittyAPI is the Schema for the cloudkittyapis API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CloudKittyAPISpec defines the desired state of CloudKittyAPI + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for cloudkitty + DB, defaults to cloudkitty + type: string + databaseHostname: + description: DatabaseHostname - CloudKitty Database Hostname + type: string + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment resource + names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service password + from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + secret: + description: Secret containing OpenStack password information + type: string + serviceAccount: + description: ServiceAccount - service account name used internally + to provide CloudKitty services the default SA name + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret for + the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + public: + description: Public GenericService - holds the secret for + the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + transportURLSecret: + description: Secret containing RabbitMq transport URL + type: string + required: + - containerImage + - databaseHostname + - secret + - serviceAccount + - transportURLSecret + type: object + status: + description: CloudKittyAPIStatus defines the observed state of CloudKittyAPI + properties: + apiEndpoints: + additionalProperties: + additionalProperties: + type: string + type: object + description: API endpoints + type: object + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object + lastAppliedTopology: + description: LastAppliedTopology - the last applied Topology + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + networkAttachments: + additionalProperties: + items: + type: string + type: array + description: NetworkAttachments status of the deployment pods + type: object + observedGeneration: + description: |- + ObservedGeneration - the most recent generation observed for this service. + If the observed generation is different than the spec generation, then the + controller has not started processing the latest changes, and the status + and its conditions are likely stale. + format: int64 + type: integer + readyCount: + default: 0 + description: ReadyCount of CloudKitty API instances + format: int32 + minimum: 0 + type: integer + serviceIDs: + additionalProperties: + type: string + description: ServiceIDs + type: object + required: + - readyCount + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml new file mode 100644 index 00000000..9cd9b6ef --- /dev/null +++ b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -0,0 +1,321 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: cloudkittyprocs.telemetry.openstack.org +spec: + group: telemetry.openstack.org + names: + kind: CloudKittyProc + listKind: CloudKittyProcList + plural: cloudkittyprocs + singular: cloudkittyproc + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: NetworkAttachments + jsonPath: .status.networkAttachments + name: NetworkAttachments + type: string + - description: Status + jsonPath: .status.conditions[0].status + name: Status + type: string + - description: Message + jsonPath: .status.conditions[0].message + name: Message + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: CloudKittyProc is the Schema for the cloudkittprocs API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CloudKittyProcSpec defines the desired state of CloudKitty + Processor + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for cloudkitty + DB, defaults to cloudkitty + type: string + databaseHostname: + description: DatabaseHostname - CloudKitty Database Hostname + type: string + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment resource + names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service password + from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + secret: + description: Secret containing OpenStack password information + type: string + serviceAccount: + description: ServiceAccount - service account name used internally + to provide CloudKitty services the default SA name + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + transportURLSecret: + description: Secret containing RabbitMq transport URL + type: string + required: + - containerImage + - databaseHostname + - secret + - serviceAccount + - transportURLSecret + type: object + status: + description: CloudKittyProcStatus defines the observed state of CloudKitty + Processor + properties: + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object + lastAppliedTopology: + description: LastAppliedTopology - the last applied Topology + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + networkAttachments: + additionalProperties: + items: + type: string + type: array + description: NetworkAttachments status of the deployment pods + type: object + observedGeneration: + description: |- + ObservedGeneration - the most recent generation observed for this service. + If the observed generation is different than the spec generation, then the + controller has not started processing the latest changes, and the status + and its conditions are likely stale. + format: int64 + type: integer + readyCount: + default: 0 + description: ReadyCount of CloudKitty Processor instances + format: int32 + minimum: 0 + type: integer + required: + - readyCount + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index 6c18bdfc..0d8bf4d3 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -297,6 +297,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object preserveJobs: default: false @@ -525,6 +530,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object proxyImage: type: string @@ -584,6 +594,542 @@ spec: - secret - sgCoreImage type: object + cloudkitty: + description: CloudKitty - Parameters related to the cloudkitty service + properties: + apiTimeout: + default: 60 + description: APITimeout for HAProxy, Apache, and rpc_response_timeout + minimum: 10 + type: integer + cloudKittyAPI: + description: CloudKittyAPI - Spec definition for the API service + of this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL + (will be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + override: + description: Override, provides the ability to override the + generated manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains + the configurations of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret + for the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key + for the service + type: string + type: object + public: + description: Public GenericService - holds the secret + for the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key + for the service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs + in a pre-created bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + cloudKittyProc: + description: CloudKittyProc - Spec definition for the Scheduler + service of this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL + (will be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config for all CloudKitty services using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for + cloudkitty DB, defaults to cloudkitty + type: string + databaseInstance: + description: |- + MariaDB instance name + Right now required by the maridb-operator to get the credentials from the instance to create the DB + Might not be required in future + type: string + enabled: + default: true + description: Enabled - Whether OpenStack CloudKitty service should + be deployed and managed + type: boolean + memcachedInstance: + default: memcached + description: Memcached instance name. + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting + NodeSelector here acts as a default value and can be overridden by service + specific NodeSelector Settings. + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service + password from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean + rabbitMqClusterName: + default: rabbitmq + description: |- + RabbitMQ instance name + Needed to request a transportURL that is created and used in CloudKitty + type: string + secret: + description: Secret containing OpenStack password information + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - cloudKittyAPI + - cloudKittyProc + - databaseInstance + - memcachedInstance + - rabbitMqClusterName + - secret + type: object logging: description: Logging - Parameters related to the logging properties: diff --git a/api/v1beta1/autoscaling_types.go b/api/v1beta1/autoscaling_types.go index d6f3b99b..f541155e 100644 --- a/api/v1beta1/autoscaling_types.go +++ b/api/v1beta1/autoscaling_types.go @@ -17,13 +17,12 @@ limitations under the License. package v1beta1 import ( + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" - "github.com/openstack-k8s-operators/lib-common/modules/common/service" - "github.com/openstack-k8s-operators/lib-common/modules/common/util" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -139,13 +138,6 @@ type AodhCore struct { TopologyRef *topologyv1.TopoRef `json:"topologyRef,omitempty"` } -// APIOverrideSpec to override the generated manifest of several child resources. -type APIOverrideSpec struct { - // Override configuration for the Service created to serve traffic to the cluster. - // The key must be the endpoint type (public, internal) - Service map[service.Endpoint]service.RoutedOverrideSpec `json:"service,omitempty"` -} - // AutoscalingSpec defines the desired state of Autoscaling type AutoscalingSpec struct { AutoscalingSpecBase `json:",inline"` diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go new file mode 100644 index 00000000..08181d6c --- /dev/null +++ b/api/v1beta1/cloudkitty_types.go @@ -0,0 +1,270 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // CloudKittyUserID - Kolla's cloudkitty UID comes from the 'cloudkitty-user' in + // https://github.com/openstack/kolla/blob/master/kolla/common/users.py + CloudKittyUserID = 42408 + // CloudKittyGroupID - Kolla's cloudkitty GID + CloudKittyGroupID = 42408 + + // CloudKittyAPIContainerImage - default fall-back image for CloudKitty API + CloudKittyAPIContainerImage = "quay.io/podified-master-centos9/cloudkitty-api:current-podified" + // CloudKittyProcContainerImage - default fall-back image for CloudKitty Processor + CloudKittyProcContainerImage = "quay.io/podified-master-centos9/cloudkitty-processor:current-podified" + // CloudKittyDbSyncHash hash + CKDbSyncHash = "ckdbsync" + // CloudKittyReplicas - The number of replicas per each service deployed + CloudKittyReplicas = 1 +) + +type CloudKittySpecBase struct { + CloudKittyTemplate `json:",inline"` + + // +kubebuilder:validation:Required + // MariaDB instance name + // Right now required by the maridb-operator to get the credentials from the instance to create the DB + // Might not be required in future + DatabaseInstance string `json:"databaseInstance"` + + // +kubebuilder:validation:Required + // +kubebuilder:default=rabbitmq + // RabbitMQ instance name + // Needed to request a transportURL that is created and used in CloudKitty + RabbitMqClusterName string `json:"rabbitMqClusterName"` + + // +kubebuilder:validation:Required + // +kubebuilder:default=memcached + // Memcached instance name. + MemcachedInstance string `json:"memcachedInstance"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // PreserveJobs - do not delete jobs after they finished e.g. to check logs + PreserveJobs bool `json:"preserveJobs"` + + // +kubebuilder:validation:Optional + // CustomServiceConfig - customize the service config for all CloudKitty services using this parameter to change service defaults, + // or overwrite rendered information using raw OpenStack config format. The content gets added to + // to /etc//.conf.d directory as a custom config file. + CustomServiceConfig string `json:"customServiceConfig,omitempty"` + + // +kubebuilder:validation:Optional + // NodeSelector to target subset of worker nodes running this service. Setting + // NodeSelector here acts as a default value and can be overridden by service + // specific NodeSelector Settings. + NodeSelector *map[string]string `json:"nodeSelector,omitempty"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=60 + // +kubebuilder:validation:Minimum=10 + // APITimeout for HAProxy, Apache, and rpc_response_timeout + APITimeout int `json:"apiTimeout"` + + // +kubebuilder:validation:Optional + // TopologyRef to apply the Topology defined by the associated CR referenced + // by name + TopologyRef *topologyv1.TopoRef `json:"topologyRef,omitempty"` +} + +// CloudKittySpecCore the same as CloudKittySpec without ContainerImage references +type CloudKittySpecCore struct { + CloudKittySpecBase `json:",inline"` + + // +kubebuilder:validation:Required + // CloudKittyAPI - Spec definition for the API service of this CloudKitty deployment + CloudKittyAPI CloudKittyAPITemplateCore `json:"cloudKittyAPI"` + + // +kubebuilder:validation:Required + // CloudKittyProc - Spec definition for the Scheduler service of this CloudKitty deployment + CloudKittyProc CloudKittyProcTemplateCore `json:"cloudKittyProc"` +} + +// CloudKittySpec defines the desired state of CloudKitty +type CloudKittySpec struct { + CloudKittySpecBase `json:",inline"` + + // +kubebuilder:validation:Required + // CloudKittyAPI - Spec definition for the API service of this CloudKitty deployment + CloudKittyAPI CloudKittyAPITemplate `json:"cloudKittyAPI"` + + // +kubebuilder:validation:Required + // CloudKittyProc - Spec definition for the Scheduler service of this CloudKitty deployment + CloudKittyProc CloudKittyProcTemplate `json:"cloudKittyProc"` +} + +// CloudKittyTemplate defines common input parameters used by all CloudKitty services +type CloudKittyTemplate struct { + // +kubebuilder:validation:Optional + // +kubebuilder:default=cloudkitty + // ServiceUser - optional username used for this service to register in cloudkitty + ServiceUser string `json:"serviceUser"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=cloudkitty + // DatabaseAccount - optional MariaDBAccount used for cloudkitty DB, defaults to cloudkitty + DatabaseAccount string `json:"databaseAccount"` + + // +kubebuilder:validation:Required + // Secret containing OpenStack password information + Secret string `json:"secret"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default={cloudKittyService: CloudKittyPassword} + // PasswordsSelectors - Selectors to identify the ServiceUser password from the Secret + PasswordSelectors PasswordsSelector `json:"passwordSelector"` +} + +// CloudKittyServiceTemplate defines the input parameters that can be defined for a given +// CloudKitty service +type CloudKittyServiceTemplate struct { + + // +kubebuilder:validation:Optional + // NodeSelector to target subset of worker nodes running this service. Setting here overrides + // any global NodeSelector settings within the CloudKitty CR. + NodeSelector *map[string]string `json:"nodeSelector,omitempty"` + + // +kubebuilder:validation:Optional + // CustomServiceConfig - customize the service config using this parameter to change service defaults, + // or overwrite rendered information using raw OpenStack config format. The content gets added to + // to /etc//.conf.d directory as a custom config file. + CustomServiceConfig string `json:"customServiceConfig,omitempty"` + + // +kubebuilder:validation:Optional + // CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + // that contain sensitive service config data. The content of each Secret gets added to the + // /etc//.conf.d directory as a custom config file. + CustomServiceConfigSecrets []string `json:"customServiceConfigSecrets,omitempty"` + + // +kubebuilder:validation:Optional + // Resources - Compute Resources required by this service (Limits/Requests). + // https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + + // +kubebuilder:validation:Optional + // NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network + NetworkAttachments []string `json:"networkAttachments,omitempty"` + + // +kubebuilder:validation:Optional + // TopologyRef to apply the Topology defined by the associated CR referenced + // by name + TopologyRef *topologyv1.TopoRef `json:"topologyRef,omitempty"` +} + +// CloudKittyStatus defines the observed state of CloudKitty +type CloudKittyStatus struct { + // Map of hashes to track e.g. job status + Hash map[string]string `json:"hash,omitempty"` + + // Conditions + Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` + + // CloudKitty Database Hostname + DatabaseHostname string `json:"databaseHostname,omitempty"` + + // TransportURLSecret - Secret containing RabbitMQ transportURL + TransportURLSecret string `json:"transportURLSecret,omitempty"` + + // API endpoints + APIEndpoints map[string]map[string]string `json:"apiEndpoints,omitempty"` + + // ServiceIDs + ServiceIDs map[string]string `json:"serviceIDs,omitempty"` + + // ReadyCount of CloudKitty API instance + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:default=0 + CloudKittyAPIReadyCount int32 `json:"cloudKittyAPIReadyCount"` + + // ReadyCount of CloudKitty Processor instances + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:default=0 + CloudKittyProcReadyCount int32 `json:"cloudKittyProcReadyCounts"` + + // ObservedGeneration - the most recent generation observed for this service. + // If the observed generation is different than the spec generation, then the + // controller has not started processing the latest changes, and the status + // and its conditions are likely stale. + ObservedGeneration int64 `json:"observedGeneration,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// CloudKitty is the Schema for the cloudkitties API +type CloudKitty struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CloudKittySpec `json:"spec,omitempty"` + Status CloudKittyStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// CloudKittyList contains a list of CloudKitty +type CloudKittyList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CloudKitty `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CloudKitty{}, &CloudKittyList{}) +} + +// SetupDefaultsCloudKitty - initializes any CRD field defaults based on environment variables (the defaulting mechanism itself is implemented via webhooks) +func SetupDefaultsCloudKitty() { + // Acquire environmental defaults and initialize Telemetry defaults with them + cloudKittyDefaults := CloudKittyDefaults{ + APIContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CLOUDKITTY_API_IMAGE_URL_DEFAULT", CloudKittyAPIContainerImage), + ProcContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CLOUDKITTY_PROCESSOR_IMAGE_URL_DEFAULT", CloudKittyProcContainerImage), + } + + SetupCloudKittyDefaults(cloudKittyDefaults) +} + +// IsReady - returns true if all subresources Ready condition is true +func (instance CloudKitty) IsReady() bool { + return instance.Generation == instance.Status.ObservedGeneration && + instance.Status.Conditions.IsTrue(CloudKittyAPIReadyCondition) && + instance.Status.Conditions.IsTrue(CloudKittyProcReadyCondition) +} + +// RbacConditionsSet - set the conditions for the rbac object +func (instance CloudKitty) RbacConditionsSet(c *condition.Condition) { + instance.Status.Conditions.Set(c) +} + +// RbacNamespace - return the namespace +func (instance CloudKitty) RbacNamespace() string { + return instance.Namespace +} + +// RbacResourceName - return the name to be used for rbac objects (serviceaccount, role, rolebinding) +func (instance CloudKitty) RbacResourceName() string { + return "cloudkitty-" + instance.Name +} diff --git a/api/v1beta1/cloudkitty_webhook.go b/api/v1beta1/cloudkitty_webhook.go new file mode 100644 index 00000000..5ebd644a --- /dev/null +++ b/api/v1beta1/cloudkitty_webhook.go @@ -0,0 +1,102 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// CloudKittyDefaults - +type CloudKittyDefaults struct { + APIContainerImageURL string + ProcContainerImageURL string +} + +var cloudKittyDefaults CloudKittyDefaults + +// log is for logging in this package. +var cloudKittyLog = logf.Log.WithName("cloudkitty-resource") + +// SetupCloudKittyDefaults - initialize CloudKitty spec defaults for use with either internal or external webhooks +func SetupCloudKittyDefaults(defaults CloudKittyDefaults) { + cloudKittyDefaults = defaults + cloudKittyLog.Info("CloudKitty defaults initialized", "defaults", defaults) +} + +// SetupWebhookWithManager - setups webhook with the adequate manager +func (r *CloudKitty) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +// TODO(user): EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! + +//+kubebuilder:webhook:path=/mutate-telemetry-openstack-org-v1beta1-cloudkitty,mutating=true,failurePolicy=fail,sideEffects=None,groups=telemetry.openstack.org,resources=cloudkitties,verbs=create;update,versions=v1beta1,name=mcloudkitty.kb.io,admissionReviewVersions=v1 + +var _ webhook.Defaulter = &CloudKitty{} + +// Default implements webhook.Defaulter so a webhook will be registered for the type +func (r *CloudKitty) Default() { + cloudKittyLog.Info("default", "name", r.Name) + + r.Spec.Default() +} + +// Default - set defaults for this CloudKitty spec +func (spec *CloudKittySpec) Default() { + if spec.CloudKittyAPI.ContainerImage == "" { + spec.CloudKittyAPI.ContainerImage = cloudKittyDefaults.APIContainerImageURL + } + if spec.CloudKittyProc.ContainerImage == "" { + spec.CloudKittyProc.ContainerImage = cloudKittyDefaults.ProcContainerImageURL + } + +} + +// TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation. +//+kubebuilder:webhook:path=/validate-telemetry-openstack-org-v1beta1-cloudkitty,mutating=false,failurePolicy=fail,sideEffects=None,groups=telemetry.openstack.org,resources=cloudkitties,verbs=create;update,versions=v1beta1,name=vcloudkitty.kb.io,admissionReviewVersions=v1 + +var _ webhook.Validator = &CloudKitty{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type +func (r *CloudKitty) ValidateCreate() (admission.Warnings, error) { + cloudKittyLog.Info("validate create", "name", r.Name) + + // TODO(user): fill in your validation logic upon object creation. + return nil, nil +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type +func (r *CloudKitty) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { + cloudKittyLog.Info("validate update", "name", r.Name) + + // TODO(user): fill in your validation logic upon object update. + return nil, nil +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type +func (r *CloudKitty) ValidateDelete() (admission.Warnings, error) { + cloudKittyLog.Info("validate delete", "name", r.Name) + + // TODO(user): fill in your validation logic upon object deletion. + return nil, nil +} diff --git a/api/v1beta1/cloudkittyapi_types.go b/api/v1beta1/cloudkittyapi_types.go new file mode 100644 index 00000000..42660c7e --- /dev/null +++ b/api/v1beta1/cloudkittyapi_types.go @@ -0,0 +1,155 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// CloudKittyAPITemplateCore defines the input parameters for the CloudKitty API service +type CloudKittyAPITemplateCore struct { + // Common input parameters for the CloudKitty API service + CloudKittyServiceTemplate `json:",inline"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=1 + // +kubebuilder:validation:Minimum=0 + // Replicas - CloudKitty API Replicas + Replicas *int32 `json:"replicas"` + + // +kubebuilder:validation:Optional + // Override, provides the ability to override the generated manifest of several child resources. + Override APIOverrideSpec `json:"override,omitempty"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.API `json:"tls,omitempty"` +} + +// CloudKittyAPITemplate defines the input parameters for the CloudKitty API service +type CloudKittyAPITemplate struct { + // +kubebuilder:validation:Required + // ContainerImage - CloudKitty Container Image URL (will be set to environmental default if empty) + ContainerImage string `json:"containerImage"` + + CloudKittyAPITemplateCore `json:",inline"` +} + +// CloudKittyAPISpec defines the desired state of CloudKittyAPI +type CloudKittyAPISpec struct { + // Common input parameters for all CloudKitty services + CloudKittyTemplate `json:",inline"` + + // Input parameters for the CloudKitty API service + CloudKittyAPITemplate `json:",inline"` + + // +kubebuilder:validation:Required + // DatabaseHostname - CloudKitty Database Hostname + DatabaseHostname string `json:"databaseHostname"` + + // +kubebuilder:validation:Required + // Secret containing RabbitMq transport URL + TransportURLSecret string `json:"transportURLSecret"` + + // +kubebuilder:validation:Required + // ServiceAccount - service account name used internally to provide CloudKitty services the default SA name + ServiceAccount string `json:"serviceAccount"` +} + +// CloudKittyAPIStatus defines the observed state of CloudKittyAPI +type CloudKittyAPIStatus struct { + // Map of hashes to track e.g. job status + Hash map[string]string `json:"hash,omitempty"` + + // API endpoints + APIEndpoints map[string]map[string]string `json:"apiEndpoints,omitempty"` + + // Conditions + Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` + + // ReadyCount of CloudKitty API instances + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:default=0 + ReadyCount int32 `json:"readyCount"` + + // ServiceIDs + ServiceIDs map[string]string `json:"serviceIDs,omitempty"` + + // NetworkAttachments status of the deployment pods + NetworkAttachments map[string][]string `json:"networkAttachments,omitempty"` + + // ObservedGeneration - the most recent generation observed for this service. + // If the observed generation is different than the spec generation, then the + // controller has not started processing the latest changes, and the status + // and its conditions are likely stale. + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // LastAppliedTopology - the last applied Topology + LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// CloudKittyAPI is the Schema for the cloudkittyapis API +type CloudKittyAPI struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CloudKittyAPISpec `json:"spec,omitempty"` + Status CloudKittyAPIStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// CloudKittyAPIList contains a list of CloudKittyAPI +type CloudKittyAPIList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CloudKittyAPI `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CloudKittyAPI{}, &CloudKittyAPIList{}) +} + +// IsReady - returns true if service is ready to serve requests +func (instance CloudKittyAPI) IsReady() bool { + return instance.Generation == instance.Status.ObservedGeneration && + instance.Status.ReadyCount == *instance.Spec.Replicas && + (instance.Status.Conditions.IsTrue(condition.DeploymentReadyCondition) || + (instance.Status.Conditions.IsFalse(condition.DeploymentReadyCondition) && *instance.Spec.Replicas == 0)) +} + +// GetSpecTopologyRef - Returns the LastAppliedTopology Set in the Status +func (instance *CloudKittyAPI) GetSpecTopologyRef() *topologyv1.TopoRef { + return instance.Spec.TopologyRef +} + +// GetLastAppliedTopology - Returns the LastAppliedTopology Set in the Status +func (instance *CloudKittyAPI) GetLastAppliedTopology() *topologyv1.TopoRef { + return instance.Status.LastAppliedTopology +} + +// SetLastAppliedTopology - Sets the LastAppliedTopology value in the Status +func (instance *CloudKittyAPI) SetLastAppliedTopology(topologyRef *topologyv1.TopoRef) { + instance.Status.LastAppliedTopology = topologyRef +} diff --git a/api/v1beta1/cloudkittyproc_types.go b/api/v1beta1/cloudkittyproc_types.go new file mode 100644 index 00000000..d2459afd --- /dev/null +++ b/api/v1beta1/cloudkittyproc_types.go @@ -0,0 +1,148 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// CloudKittyProcTemplateCore defines the input parameters for the CloudKitty Processor service +type CloudKittyProcTemplateCore struct { + // Common input parameters for the CloudKitty Processor service + CloudKittyServiceTemplate `json:",inline"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=1 + // +kubebuilder:validation:Minimum=0 + // Replicas - CloudKitty API Replicas + Replicas *int32 `json:"replicas"` +} + +// CloudKittyProcTemplate defines the input parameters for the CloudKitty Processor service +type CloudKittyProcTemplate struct { + // +kubebuilder:validation:Required + // ContainerImage - CloudKitty Container Image URL (will be set to environmental default if empty) + ContainerImage string `json:"containerImage"` + + CloudKittyProcTemplateCore `json:",inline"` +} + +// CloudKittyProcSpec defines the desired state of CloudKitty Processor +type CloudKittyProcSpec struct { + // Common input parameters for all CloudKitty services + CloudKittyTemplate `json:",inline"` + + // Input parameters for the CloudKitty Processor service + CloudKittyProcTemplate `json:",inline"` + + // +kubebuilder:validation:Required + // DatabaseHostname - CloudKitty Database Hostname + DatabaseHostname string `json:"databaseHostname"` + + // +kubebuilder:validation:Required + // Secret containing RabbitMq transport URL + TransportURLSecret string `json:"transportURLSecret"` + + // +kubebuilder:validation:Required + // ServiceAccount - service account name used internally to provide CloudKitty services the default SA name + ServiceAccount string `json:"serviceAccount"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.Ca `json:"tls,omitempty"` +} + +// CloudKittyProcStatus defines the observed state of CloudKitty Processor +type CloudKittyProcStatus struct { + // Map of hashes to track e.g. job status + Hash map[string]string `json:"hash,omitempty"` + + // Conditions + Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` + + // ReadyCount of CloudKitty Processor instances + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:default=0 + ReadyCount int32 `json:"readyCount"` + + // NetworkAttachments status of the deployment pods + NetworkAttachments map[string][]string `json:"networkAttachments,omitempty"` + + // ObservedGeneration - the most recent generation observed for this service. + // If the observed generation is different than the spec generation, then the + // controller has not started processing the latest changes, and the status + // and its conditions are likely stale. + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // LastAppliedTopology - the last applied Topology + LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:printcolumn:name="NetworkAttachments",type="string",JSONPath=".status.networkAttachments",description="NetworkAttachments" +//+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[0].status",description="Status" +//+kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.conditions[0].message",description="Message" + +// CloudKittyProc is the Schema for the cloudkittprocs API +type CloudKittyProc struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CloudKittyProcSpec `json:"spec,omitempty"` + Status CloudKittyProcStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// CloudKittyProcList contains a list of CloudKittyProc +type CloudKittyProcList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CloudKittyProc `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CloudKittyProc{}, &CloudKittyProcList{}) +} + +// IsReady - returns true if service is ready to serve requests +func (instance CloudKittyProc) IsReady() bool { + return instance.Generation == instance.Status.ObservedGeneration && + instance.Status.ReadyCount == *instance.Spec.Replicas && + (instance.Status.Conditions.IsTrue(condition.DeploymentReadyCondition) || + (instance.Status.Conditions.IsFalse(condition.DeploymentReadyCondition) && *instance.Spec.Replicas == 0)) +} + +// GetSpecTopologyRef - Returns the LastAppliedTopology Set in the Status +func (instance *CloudKittyProc) GetSpecTopologyRef() *topologyv1.TopoRef { + return instance.Spec.TopologyRef +} + +// GetLastAppliedTopology - Returns the LastAppliedTopology Set in the Status +func (instance *CloudKittyProc) GetLastAppliedTopology() *topologyv1.TopoRef { + return instance.Status.LastAppliedTopology +} + +// SetLastAppliedTopology - Sets the LastAppliedTopology value in the Status +func (instance *CloudKittyProc) SetLastAppliedTopology(topologyRef *topologyv1.TopoRef) { + instance.Status.LastAppliedTopology = topologyRef +} diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index f9db6290..f8e0b4b7 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -45,6 +45,15 @@ const ( // LoggingReadyCondition Status=True condition which indicates if the Logging is configured and operational LoggingReadyCondition condition.Type = "LoggingReady" + // CloudKittyReadyCondition Status=True condition which indicates if the CloudKitty is configured and operational + CloudKittyReadyCondition condition.Type = "CloudKittyReady" + + // CloudKittyAPIReadyCondition Status=True condition which indicates if the CloudKitty API is configured and operational + CloudKittyAPIReadyCondition condition.Type = "CloudKittyAPIReady" + + // CloudKittyProcReadyCondition Status=True condition which indicates if the CloudKitty Processor is configured and operational + CloudKittyProcReadyCondition condition.Type = "CloudKittyProcReady" + // LoggingCLONamespaceReadyCondition Status=True condition which indicates if the cluster-logging-operator namespace is created LoggingCLONamespaceReadyCondition condition.Type = "LoggingCLONamespaceReady" @@ -202,6 +211,48 @@ const ( // LoggingCLONamespaceFailedMessage LoggingCLONamespaceFailedMessage = "CLO Namespace %s does not exist" + // + // CloudKittyReady condition messages + // + // CloudKittyReadyInitMessage + CloudKittyReadyInitMessage = "CloudKitty not started" + + // CloudKittyReadyMessage + CloudKittyReadyMessage = "CloudKitty completed" + + // CloudKittyReadyErrorMessage + CloudKittyReadyErrorMessage = "CloudKitty error occured %s" + + // + // CloudKittyAPIReady condition messages + // + // CloudKittyAPIReadyInitMessage + CloudKittyAPIReadyInitMessage = "CloudKittyAPI not started" + + // CloudKittyAPIReadyMessage + CloudKittyAPIReadyMessage = "CloudKittyAPI completed" + + // CloudKittyAPIReadyErrorMessage + CloudKittyAPIReadyErrorMessage = "CloudKittyAPI error occured %s" + + // CloudKittyAPIReadyRunningMessage + CloudKittyAPIReadyRunningMessage = "CloudKittyAPI in progress" + + // + // CloudKittyProcReady condition messages + // + // CloudKittyProcReadyInitMessage + CloudKittyProcReadyInitMessage = "CloudKittyProc not started" + + // CloudKittyProcReadyMessage + CloudKittyProcReadyMessage = "CloudKittyProc completed" + + // CloudKittyProcReadyErrorMessage + CloudKittyProcReadyErrorMessage = "CloudKittyProc error occured %s" + + // CloudKittyProcReadyRunningMessage + CloudKittyProcReadyRunningMessage = "CloudKittyProc in progress" + DashboardsNotEnabledMessage = "Dashboarding was not enabled, so no actions required" DashboardPrometheusRuleReadyInitMessage = "Dashboard PrometheusRule not started" diff --git a/api/v1beta1/telemetry_types.go b/api/v1beta1/telemetry_types.go index 7fb6a930..07621b5e 100644 --- a/api/v1beta1/telemetry_types.go +++ b/api/v1beta1/telemetry_types.go @@ -17,13 +17,21 @@ limitations under the License. package v1beta1 import ( + "github.com/openstack-k8s-operators/lib-common/modules/common/service" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/util" - topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" ) +// APIOverrideSpec to override the generated manifest of several child resources. +type APIOverrideSpec struct { + // Override configuration for the Service created to serve traffic to the cluster. + // The key must be the endpoint type (public, internal) + Service map[service.Endpoint]service.RoutedOverrideSpec `json:"service,omitempty"` +} + // PasswordsSelector to identify the Service password from the Secret type PasswordsSelector struct { // CeilometerService - Selector to get the ceilometer service password from the Secret @@ -35,6 +43,11 @@ type PasswordsSelector struct { // +kubebuilder:validation:Optional // +kubebuilder:default:=AodhPassword AodhService string `json:"aodhService"` + + // CloudKittyService - Selector to get the CloudKitty service password from the Secret + // +kubebuilder:validation:Optional + // +kubebuilder:default:=CloudKittyPassword + CloudKittyService string `json:"cloudKittyService"` } // TelemetrySpec defines the desired state of Telemetry @@ -73,6 +86,10 @@ type TelemetrySpecBase struct { // Logging - Parameters related to the logging Logging LoggingSection `json:"logging,omitempty"` + // +kubebuilder:validation:Optional + // CloudKitty - Parameters related to the cloudkitty service + CloudKitty CloudKittySection `json:"cloudkitty,omitempty"` + // +kubebuilder:validation:Optional // NodeSelector to target subset of worker nodes running this service NodeSelector *map[string]string `json:"nodeSelector,omitempty"` @@ -167,6 +184,20 @@ type LoggingSection struct { LoggingSpec `json:",inline"` } +// CloudKittySpec defines the desired state of the cloudkitty service +type CloudKittySection struct { + // +kubebuilder:validation:Optional + // +kubebuilder:default=true + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:booleanSwitch"} + // Enabled - Whether OpenStack CloudKitty service should be deployed and managed + Enabled *bool `json:"enabled"` + + // +kubebuilder:validation:Optional + //+operator-sdk:csv:customresourcedefinitions:type=spec + // Template - Overrides to use when creating the OpenStack CloudKitty service + CloudKittySpec `json:",inline"` +} + // TelemetryStatus defines the observed state of Telemetry type TelemetryStatus struct { // Map of hashes to track e.g. job status diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index da8d5649..1c5006d6 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -578,6 +578,635 @@ func (in *CeilometerStatus) DeepCopy() *CeilometerStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKitty) DeepCopyInto(out *CloudKitty) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKitty. +func (in *CloudKitty) DeepCopy() *CloudKitty { + if in == nil { + return nil + } + out := new(CloudKitty) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CloudKitty) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyAPI) DeepCopyInto(out *CloudKittyAPI) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyAPI. +func (in *CloudKittyAPI) DeepCopy() *CloudKittyAPI { + if in == nil { + return nil + } + out := new(CloudKittyAPI) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CloudKittyAPI) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyAPIList) DeepCopyInto(out *CloudKittyAPIList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CloudKittyAPI, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyAPIList. +func (in *CloudKittyAPIList) DeepCopy() *CloudKittyAPIList { + if in == nil { + return nil + } + out := new(CloudKittyAPIList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CloudKittyAPIList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyAPISpec) DeepCopyInto(out *CloudKittyAPISpec) { + *out = *in + out.CloudKittyTemplate = in.CloudKittyTemplate + in.CloudKittyAPITemplate.DeepCopyInto(&out.CloudKittyAPITemplate) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyAPISpec. +func (in *CloudKittyAPISpec) DeepCopy() *CloudKittyAPISpec { + if in == nil { + return nil + } + out := new(CloudKittyAPISpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyAPIStatus) DeepCopyInto(out *CloudKittyAPIStatus) { + *out = *in + if in.Hash != nil { + in, out := &in.Hash, &out.Hash + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.APIEndpoints != nil { + in, out := &in.APIEndpoints, &out.APIEndpoints + *out = make(map[string]map[string]string, len(*in)) + for key, val := range *in { + var outVal map[string]string + if val == nil { + (*out)[key] = nil + } else { + inVal := (*in)[key] + in, out := &inVal, &outVal + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + (*out)[key] = outVal + } + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(condition.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ServiceIDs != nil { + in, out := &in.ServiceIDs, &out.ServiceIDs + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.NetworkAttachments != nil { + in, out := &in.NetworkAttachments, &out.NetworkAttachments + *out = make(map[string][]string, len(*in)) + for key, val := range *in { + var outVal []string + if val == nil { + (*out)[key] = nil + } else { + inVal := (*in)[key] + in, out := &inVal, &outVal + *out = make([]string, len(*in)) + copy(*out, *in) + } + (*out)[key] = outVal + } + } + if in.LastAppliedTopology != nil { + in, out := &in.LastAppliedTopology, &out.LastAppliedTopology + *out = new(topologyv1beta1.TopoRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyAPIStatus. +func (in *CloudKittyAPIStatus) DeepCopy() *CloudKittyAPIStatus { + if in == nil { + return nil + } + out := new(CloudKittyAPIStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyAPITemplate) DeepCopyInto(out *CloudKittyAPITemplate) { + *out = *in + in.CloudKittyAPITemplateCore.DeepCopyInto(&out.CloudKittyAPITemplateCore) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyAPITemplate. +func (in *CloudKittyAPITemplate) DeepCopy() *CloudKittyAPITemplate { + if in == nil { + return nil + } + out := new(CloudKittyAPITemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyAPITemplateCore) DeepCopyInto(out *CloudKittyAPITemplateCore) { + *out = *in + in.CloudKittyServiceTemplate.DeepCopyInto(&out.CloudKittyServiceTemplate) + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + in.Override.DeepCopyInto(&out.Override) + in.TLS.DeepCopyInto(&out.TLS) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyAPITemplateCore. +func (in *CloudKittyAPITemplateCore) DeepCopy() *CloudKittyAPITemplateCore { + if in == nil { + return nil + } + out := new(CloudKittyAPITemplateCore) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyDefaults) DeepCopyInto(out *CloudKittyDefaults) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyDefaults. +func (in *CloudKittyDefaults) DeepCopy() *CloudKittyDefaults { + if in == nil { + return nil + } + out := new(CloudKittyDefaults) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyList) DeepCopyInto(out *CloudKittyList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CloudKitty, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyList. +func (in *CloudKittyList) DeepCopy() *CloudKittyList { + if in == nil { + return nil + } + out := new(CloudKittyList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CloudKittyList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyProc) DeepCopyInto(out *CloudKittyProc) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProc. +func (in *CloudKittyProc) DeepCopy() *CloudKittyProc { + if in == nil { + return nil + } + out := new(CloudKittyProc) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CloudKittyProc) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyProcList) DeepCopyInto(out *CloudKittyProcList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CloudKittyProc, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProcList. +func (in *CloudKittyProcList) DeepCopy() *CloudKittyProcList { + if in == nil { + return nil + } + out := new(CloudKittyProcList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CloudKittyProcList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyProcSpec) DeepCopyInto(out *CloudKittyProcSpec) { + *out = *in + out.CloudKittyTemplate = in.CloudKittyTemplate + in.CloudKittyProcTemplate.DeepCopyInto(&out.CloudKittyProcTemplate) + out.TLS = in.TLS +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProcSpec. +func (in *CloudKittyProcSpec) DeepCopy() *CloudKittyProcSpec { + if in == nil { + return nil + } + out := new(CloudKittyProcSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyProcStatus) DeepCopyInto(out *CloudKittyProcStatus) { + *out = *in + if in.Hash != nil { + in, out := &in.Hash, &out.Hash + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(condition.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.NetworkAttachments != nil { + in, out := &in.NetworkAttachments, &out.NetworkAttachments + *out = make(map[string][]string, len(*in)) + for key, val := range *in { + var outVal []string + if val == nil { + (*out)[key] = nil + } else { + inVal := (*in)[key] + in, out := &inVal, &outVal + *out = make([]string, len(*in)) + copy(*out, *in) + } + (*out)[key] = outVal + } + } + if in.LastAppliedTopology != nil { + in, out := &in.LastAppliedTopology, &out.LastAppliedTopology + *out = new(topologyv1beta1.TopoRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProcStatus. +func (in *CloudKittyProcStatus) DeepCopy() *CloudKittyProcStatus { + if in == nil { + return nil + } + out := new(CloudKittyProcStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyProcTemplate) DeepCopyInto(out *CloudKittyProcTemplate) { + *out = *in + in.CloudKittyProcTemplateCore.DeepCopyInto(&out.CloudKittyProcTemplateCore) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProcTemplate. +func (in *CloudKittyProcTemplate) DeepCopy() *CloudKittyProcTemplate { + if in == nil { + return nil + } + out := new(CloudKittyProcTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyProcTemplateCore) DeepCopyInto(out *CloudKittyProcTemplateCore) { + *out = *in + in.CloudKittyServiceTemplate.DeepCopyInto(&out.CloudKittyServiceTemplate) + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProcTemplateCore. +func (in *CloudKittyProcTemplateCore) DeepCopy() *CloudKittyProcTemplateCore { + if in == nil { + return nil + } + out := new(CloudKittyProcTemplateCore) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittySection) DeepCopyInto(out *CloudKittySection) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + in.CloudKittySpec.DeepCopyInto(&out.CloudKittySpec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittySection. +func (in *CloudKittySection) DeepCopy() *CloudKittySection { + if in == nil { + return nil + } + out := new(CloudKittySection) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyServiceTemplate) DeepCopyInto(out *CloudKittyServiceTemplate) { + *out = *in + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = new(map[string]string) + if **in != nil { + in, out := *in, *out + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + } + if in.CustomServiceConfigSecrets != nil { + in, out := &in.CustomServiceConfigSecrets, &out.CustomServiceConfigSecrets + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.Resources.DeepCopyInto(&out.Resources) + if in.NetworkAttachments != nil { + in, out := &in.NetworkAttachments, &out.NetworkAttachments + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.TopologyRef != nil { + in, out := &in.TopologyRef, &out.TopologyRef + *out = new(topologyv1beta1.TopoRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyServiceTemplate. +func (in *CloudKittyServiceTemplate) DeepCopy() *CloudKittyServiceTemplate { + if in == nil { + return nil + } + out := new(CloudKittyServiceTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittySpec) DeepCopyInto(out *CloudKittySpec) { + *out = *in + in.CloudKittySpecBase.DeepCopyInto(&out.CloudKittySpecBase) + in.CloudKittyAPI.DeepCopyInto(&out.CloudKittyAPI) + in.CloudKittyProc.DeepCopyInto(&out.CloudKittyProc) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittySpec. +func (in *CloudKittySpec) DeepCopy() *CloudKittySpec { + if in == nil { + return nil + } + out := new(CloudKittySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittySpecBase) DeepCopyInto(out *CloudKittySpecBase) { + *out = *in + out.CloudKittyTemplate = in.CloudKittyTemplate + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = new(map[string]string) + if **in != nil { + in, out := *in, *out + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + } + if in.TopologyRef != nil { + in, out := &in.TopologyRef, &out.TopologyRef + *out = new(topologyv1beta1.TopoRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittySpecBase. +func (in *CloudKittySpecBase) DeepCopy() *CloudKittySpecBase { + if in == nil { + return nil + } + out := new(CloudKittySpecBase) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittySpecCore) DeepCopyInto(out *CloudKittySpecCore) { + *out = *in + in.CloudKittySpecBase.DeepCopyInto(&out.CloudKittySpecBase) + in.CloudKittyAPI.DeepCopyInto(&out.CloudKittyAPI) + in.CloudKittyProc.DeepCopyInto(&out.CloudKittyProc) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittySpecCore. +func (in *CloudKittySpecCore) DeepCopy() *CloudKittySpecCore { + if in == nil { + return nil + } + out := new(CloudKittySpecCore) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyStatus) DeepCopyInto(out *CloudKittyStatus) { + *out = *in + if in.Hash != nil { + in, out := &in.Hash, &out.Hash + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(condition.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.APIEndpoints != nil { + in, out := &in.APIEndpoints, &out.APIEndpoints + *out = make(map[string]map[string]string, len(*in)) + for key, val := range *in { + var outVal map[string]string + if val == nil { + (*out)[key] = nil + } else { + inVal := (*in)[key] + in, out := &inVal, &outVal + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + (*out)[key] = outVal + } + } + if in.ServiceIDs != nil { + in, out := &in.ServiceIDs, &out.ServiceIDs + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyStatus. +func (in *CloudKittyStatus) DeepCopy() *CloudKittyStatus { + if in == nil { + return nil + } + out := new(CloudKittyStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyTemplate) DeepCopyInto(out *CloudKittyTemplate) { + *out = *in + out.PasswordSelectors = in.PasswordSelectors +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyTemplate. +func (in *CloudKittyTemplate) DeepCopy() *CloudKittyTemplate { + if in == nil { + return nil + } + out := new(CloudKittyTemplate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KSMStatus) DeepCopyInto(out *KSMStatus) { *out = *in @@ -1047,6 +1676,7 @@ func (in *TelemetrySpecBase) DeepCopyInto(out *TelemetrySpecBase) { *out = *in in.MetricStorage.DeepCopyInto(&out.MetricStorage) in.Logging.DeepCopyInto(&out.Logging) + in.CloudKitty.DeepCopyInto(&out.CloudKitty) if in.NodeSelector != nil { in, out := &in.NodeSelector, &out.NodeSelector *out = new(map[string]string) diff --git a/config/crd/bases/telemetry.openstack.org_autoscalings.yaml b/config/crd/bases/telemetry.openstack.org_autoscalings.yaml index 4cfabab5..a4ef6ccf 100644 --- a/config/crd/bases/telemetry.openstack.org_autoscalings.yaml +++ b/config/crd/bases/telemetry.openstack.org_autoscalings.yaml @@ -294,6 +294,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object preserveJobs: default: false diff --git a/config/crd/bases/telemetry.openstack.org_ceilometers.yaml b/config/crd/bases/telemetry.openstack.org_ceilometers.yaml index 824f5daa..3307bc27 100644 --- a/config/crd/bases/telemetry.openstack.org_ceilometers.yaml +++ b/config/crd/bases/telemetry.openstack.org_ceilometers.yaml @@ -210,6 +210,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object proxyImage: type: string diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml new file mode 100644 index 00000000..bc5b7567 --- /dev/null +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -0,0 +1,665 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: cloudkitties.telemetry.openstack.org +spec: + group: telemetry.openstack.org + names: + kind: CloudKitty + listKind: CloudKittyList + plural: cloudkitties + singular: cloudkitty + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: CloudKitty is the Schema for the cloudkitties API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CloudKittySpec defines the desired state of CloudKitty + properties: + apiTimeout: + default: 60 + description: APITimeout for HAProxy, Apache, and rpc_response_timeout + minimum: 10 + type: integer + cloudKittyAPI: + description: CloudKittyAPI - Spec definition for the API service of + this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the + configurations of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret + for the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object + public: + description: Public GenericService - holds the secret + for the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in + a pre-created bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + cloudKittyProc: + description: CloudKittyProc - Spec definition for the Scheduler service + of this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config for all CloudKitty services using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for cloudkitty + DB, defaults to cloudkitty + type: string + databaseInstance: + description: |- + MariaDB instance name + Right now required by the maridb-operator to get the credentials from the instance to create the DB + Might not be required in future + type: string + memcachedInstance: + default: memcached + description: Memcached instance name. + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting + NodeSelector here acts as a default value and can be overridden by service + specific NodeSelector Settings. + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service password + from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean + rabbitMqClusterName: + default: rabbitmq + description: |- + RabbitMQ instance name + Needed to request a transportURL that is created and used in CloudKitty + type: string + secret: + description: Secret containing OpenStack password information + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - cloudKittyAPI + - cloudKittyProc + - databaseInstance + - memcachedInstance + - rabbitMqClusterName + - secret + type: object + status: + description: CloudKittyStatus defines the observed state of CloudKitty + properties: + apiEndpoints: + additionalProperties: + additionalProperties: + type: string + type: object + description: API endpoints + type: object + cloudKittyAPIReadyCount: + default: 0 + description: ReadyCount of CloudKitty API instance + format: int32 + minimum: 0 + type: integer + cloudKittyProcReadyCounts: + default: 0 + description: ReadyCount of CloudKitty Processor instances + format: int32 + minimum: 0 + type: integer + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + databaseHostname: + description: CloudKitty Database Hostname + type: string + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object + observedGeneration: + description: |- + ObservedGeneration - the most recent generation observed for this service. + If the observed generation is different than the spec generation, then the + controller has not started processing the latest changes, and the status + and its conditions are likely stale. + format: int64 + type: integer + serviceIDs: + additionalProperties: + type: string + description: ServiceIDs + type: object + transportURLSecret: + description: TransportURLSecret - Secret containing RabbitMQ transportURL + type: string + required: + - cloudKittyAPIReadyCount + - cloudKittyProcReadyCounts + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml new file mode 100644 index 00000000..d33b713c --- /dev/null +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml @@ -0,0 +1,499 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: cloudkittyapis.telemetry.openstack.org +spec: + group: telemetry.openstack.org + names: + kind: CloudKittyAPI + listKind: CloudKittyAPIList + plural: cloudkittyapis + singular: cloudkittyapi + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: CloudKittyAPI is the Schema for the cloudkittyapis API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CloudKittyAPISpec defines the desired state of CloudKittyAPI + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for cloudkitty + DB, defaults to cloudkitty + type: string + databaseHostname: + description: DatabaseHostname - CloudKitty Database Hostname + type: string + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment resource + names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service password + from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + secret: + description: Secret containing OpenStack password information + type: string + serviceAccount: + description: ServiceAccount - service account name used internally + to provide CloudKitty services the default SA name + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret for + the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + public: + description: Public GenericService - holds the secret for + the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + transportURLSecret: + description: Secret containing RabbitMq transport URL + type: string + required: + - containerImage + - databaseHostname + - secret + - serviceAccount + - transportURLSecret + type: object + status: + description: CloudKittyAPIStatus defines the observed state of CloudKittyAPI + properties: + apiEndpoints: + additionalProperties: + additionalProperties: + type: string + type: object + description: API endpoints + type: object + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object + lastAppliedTopology: + description: LastAppliedTopology - the last applied Topology + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + networkAttachments: + additionalProperties: + items: + type: string + type: array + description: NetworkAttachments status of the deployment pods + type: object + observedGeneration: + description: |- + ObservedGeneration - the most recent generation observed for this service. + If the observed generation is different than the spec generation, then the + controller has not started processing the latest changes, and the status + and its conditions are likely stale. + format: int64 + type: integer + readyCount: + default: 0 + description: ReadyCount of CloudKitty API instances + format: int32 + minimum: 0 + type: integer + serviceIDs: + additionalProperties: + type: string + description: ServiceIDs + type: object + required: + - readyCount + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml new file mode 100644 index 00000000..9cd9b6ef --- /dev/null +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -0,0 +1,321 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: cloudkittyprocs.telemetry.openstack.org +spec: + group: telemetry.openstack.org + names: + kind: CloudKittyProc + listKind: CloudKittyProcList + plural: cloudkittyprocs + singular: cloudkittyproc + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: NetworkAttachments + jsonPath: .status.networkAttachments + name: NetworkAttachments + type: string + - description: Status + jsonPath: .status.conditions[0].status + name: Status + type: string + - description: Message + jsonPath: .status.conditions[0].message + name: Message + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: CloudKittyProc is the Schema for the cloudkittprocs API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CloudKittyProcSpec defines the desired state of CloudKitty + Processor + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for cloudkitty + DB, defaults to cloudkitty + type: string + databaseHostname: + description: DatabaseHostname - CloudKitty Database Hostname + type: string + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment resource + names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service password + from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + secret: + description: Secret containing OpenStack password information + type: string + serviceAccount: + description: ServiceAccount - service account name used internally + to provide CloudKitty services the default SA name + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + transportURLSecret: + description: Secret containing RabbitMq transport URL + type: string + required: + - containerImage + - databaseHostname + - secret + - serviceAccount + - transportURLSecret + type: object + status: + description: CloudKittyProcStatus defines the observed state of CloudKitty + Processor + properties: + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object + lastAppliedTopology: + description: LastAppliedTopology - the last applied Topology + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + networkAttachments: + additionalProperties: + items: + type: string + type: array + description: NetworkAttachments status of the deployment pods + type: object + observedGeneration: + description: |- + ObservedGeneration - the most recent generation observed for this service. + If the observed generation is different than the spec generation, then the + controller has not started processing the latest changes, and the status + and its conditions are likely stale. + format: int64 + type: integer + readyCount: + default: 0 + description: ReadyCount of CloudKitty Processor instances + format: int32 + minimum: 0 + type: integer + required: + - readyCount + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index 6c18bdfc..0d8bf4d3 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -297,6 +297,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object preserveJobs: default: false @@ -525,6 +530,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object proxyImage: type: string @@ -584,6 +594,542 @@ spec: - secret - sgCoreImage type: object + cloudkitty: + description: CloudKitty - Parameters related to the cloudkitty service + properties: + apiTimeout: + default: 60 + description: APITimeout for HAProxy, Apache, and rpc_response_timeout + minimum: 10 + type: integer + cloudKittyAPI: + description: CloudKittyAPI - Spec definition for the API service + of this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL + (will be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + override: + description: Override, provides the ability to override the + generated manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains + the configurations of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret + for the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key + for the service + type: string + type: object + public: + description: Public GenericService - holds the secret + for the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key + for the service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs + in a pre-created bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + cloudKittyProc: + description: CloudKittyProc - Spec definition for the Scheduler + service of this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL + (will be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config for all CloudKitty services using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for + cloudkitty DB, defaults to cloudkitty + type: string + databaseInstance: + description: |- + MariaDB instance name + Right now required by the maridb-operator to get the credentials from the instance to create the DB + Might not be required in future + type: string + enabled: + default: true + description: Enabled - Whether OpenStack CloudKitty service should + be deployed and managed + type: boolean + memcachedInstance: + default: memcached + description: Memcached instance name. + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting + NodeSelector here acts as a default value and can be overridden by service + specific NodeSelector Settings. + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service + password from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean + rabbitMqClusterName: + default: rabbitmq + description: |- + RabbitMQ instance name + Needed to request a transportURL that is created and used in CloudKitty + type: string + secret: + description: Secret containing OpenStack password information + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - cloudKittyAPI + - cloudKittyProc + - databaseInstance + - memcachedInstance + - rabbitMqClusterName + - secret + type: object logging: description: Logging - Parameters related to the logging properties: diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 703cccdd..e6a95c46 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -7,6 +7,9 @@ resources: - bases/telemetry.openstack.org_ceilometers.yaml - bases/telemetry.openstack.org_loggings.yaml - bases/telemetry.openstack.org_metricstorages.yaml +- bases/telemetry.openstack.org_cloudkittyapis.yaml +- bases/telemetry.openstack.org_cloudkittyprocs.yaml +- bases/telemetry.openstack.org_cloudkitties.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -17,6 +20,9 @@ patchesStrategicMerge: #- patches/webhook_in_autoscalings.yaml #- patches/webhook_in_loggings.yaml #- patches/webhook_in_metricstorages.yaml +#- patches/webhook_in_cloudkittyapis.yaml +#- patches/webhook_in_cloudkittyprocs.yaml +#- patches/webhook_in_cloudkitties.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. @@ -26,6 +32,9 @@ patchesStrategicMerge: #- patches/cainjection_in_autoscalings.yaml #- patches/cainjection_in_loggings.yaml #- patches/cainjection_in_metricstorages.yaml +#- patches/cainjection_in_cloudkittyapis.yaml +#- patches/cainjection_in_cloudkittyprocs.yaml +#- patches/cainjection_in_cloudkitties.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/crd/patches/cainjection_in_cloudkitties.yaml b/config/crd/patches/cainjection_in_cloudkitties.yaml new file mode 100644 index 00000000..b0360335 --- /dev/null +++ b/config/crd/patches/cainjection_in_cloudkitties.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: cloudkitties.telemetry.openstack.org diff --git a/config/crd/patches/cainjection_in_cloudkittyapis.yaml b/config/crd/patches/cainjection_in_cloudkittyapis.yaml new file mode 100644 index 00000000..35526ad6 --- /dev/null +++ b/config/crd/patches/cainjection_in_cloudkittyapis.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: cloudkittyapis.telemetry.openstack.org diff --git a/config/crd/patches/cainjection_in_cloudkittyprocs.yaml b/config/crd/patches/cainjection_in_cloudkittyprocs.yaml new file mode 100644 index 00000000..bfc383ec --- /dev/null +++ b/config/crd/patches/cainjection_in_cloudkittyprocs.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: cloudkittyprocs.telemetry.openstack.org diff --git a/config/crd/patches/webhook_in_cloudkitties.yaml b/config/crd/patches/webhook_in_cloudkitties.yaml new file mode 100644 index 00000000..18a2c841 --- /dev/null +++ b/config/crd/patches/webhook_in_cloudkitties.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cloudkitties.telemetry.openstack.org +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/crd/patches/webhook_in_cloudkittyapis.yaml b/config/crd/patches/webhook_in_cloudkittyapis.yaml new file mode 100644 index 00000000..cad85de9 --- /dev/null +++ b/config/crd/patches/webhook_in_cloudkittyapis.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cloudkittyapis.telemetry.openstack.org +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/crd/patches/webhook_in_cloudkittyprocs.yaml b/config/crd/patches/webhook_in_cloudkittyprocs.yaml new file mode 100644 index 00000000..90c0d18e --- /dev/null +++ b/config/crd/patches/webhook_in_cloudkittyprocs.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cloudkittyprocs.telemetry.openstack.org +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/rbac/cloudkitty_editor_role.yaml b/config/rbac/cloudkitty_editor_role.yaml new file mode 100644 index 00000000..940f3121 --- /dev/null +++ b/config/rbac/cloudkitty_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit cloudkitties. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: cloudkitty-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: telemetry-operator + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + name: cloudkitty-editor-role +rules: +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkitties + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkitties/status + verbs: + - get diff --git a/config/rbac/cloudkitty_viewer_role.yaml b/config/rbac/cloudkitty_viewer_role.yaml new file mode 100644 index 00000000..253ecd75 --- /dev/null +++ b/config/rbac/cloudkitty_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view cloudkitties. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: cloudkitty-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: telemetry-operator + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + name: cloudkitty-viewer-role +rules: +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkitties + verbs: + - get + - list + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkitties/status + verbs: + - get diff --git a/config/rbac/cloudkittyapi_editor_role.yaml b/config/rbac/cloudkittyapi_editor_role.yaml new file mode 100644 index 00000000..2a0e0027 --- /dev/null +++ b/config/rbac/cloudkittyapi_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit cloudkittyapis. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: cloudkittyapi-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: telemetry-operator + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + name: cloudkittyapi-editor-role +rules: +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyapis + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyapis/status + verbs: + - get diff --git a/config/rbac/cloudkittyapi_viewer_role.yaml b/config/rbac/cloudkittyapi_viewer_role.yaml new file mode 100644 index 00000000..56bc7181 --- /dev/null +++ b/config/rbac/cloudkittyapi_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view cloudkittyapis. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: cloudkittyapi-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: telemetry-operator + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + name: cloudkittyapi-viewer-role +rules: +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyapis + verbs: + - get + - list + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyapis/status + verbs: + - get diff --git a/config/rbac/cloudkittyproc_editor_role.yaml b/config/rbac/cloudkittyproc_editor_role.yaml new file mode 100644 index 00000000..dcb4eac3 --- /dev/null +++ b/config/rbac/cloudkittyproc_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit cloudkittyprocs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: cloudkittyproc-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: telemetry-operator + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + name: cloudkittyproc-editor-role +rules: +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyprocs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyprocs/status + verbs: + - get diff --git a/config/rbac/cloudkittyproc_viewer_role.yaml b/config/rbac/cloudkittyproc_viewer_role.yaml new file mode 100644 index 00000000..a852a6f4 --- /dev/null +++ b/config/rbac/cloudkittyproc_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view cloudkittyprocs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: cloudkittyproc-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: telemetry-operator + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + name: cloudkittyproc-viewer-role +rules: +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyprocs + verbs: + - get + - list + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyprocs/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6e34456b..6fabb4df 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -4,6 +4,18 @@ kind: ClusterRole metadata: name: manager-role rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: @@ -46,6 +58,33 @@ rules: - patch - update - watch +- apiGroups: + - cloudkitty.openstack.org + resources: + - cloudkittyprocs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - cloudkitty.openstack.org + resources: + - cloudkittyprocs/finalizers + verbs: + - patch + - update +- apiGroups: + - cloudkitty.openstack.org + resources: + - cloudkittyprocs/status + verbs: + - get + - patch + - update - apiGroups: - "" resources: @@ -330,6 +369,15 @@ rules: - securitycontextconstraints verbs: - use +- apiGroups: + - security.openshift.io + resourceNames: + - anyuid + - privileged + resources: + - securitycontextconstraints + verbs: + - use - apiGroups: - telemetry.openstack.org resources: @@ -386,6 +434,88 @@ rules: - get - patch - update +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkitties + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkitties/finalizers + verbs: + - delete + - patch + - update +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkitties/status + verbs: + - get + - patch + - update +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyapis + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyapis/finalizers + verbs: + - patch + - update +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyapis/status + verbs: + - get + - patch + - update +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyprocs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyprocs/finalizers + verbs: + - patch + - update +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyprocs/status + verbs: + - get + - patch + - update - apiGroups: - telemetry.openstack.org resources: diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index 7e00a20f..6e689eee 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -5,4 +5,7 @@ resources: - telemetry_v1beta1_autoscaling.yaml - telemetry_v1beta1_logging.yaml - telemetry_v1beta1_metricstorage.yaml +- telemetry_v1beta1_cloudkittyapi.yaml +- telemetry_v1beta1_cloudkittyproc.yaml +- telemetry_v1beta1_cloudkitty.yaml #+kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/telemetry_v1beta1_cloudkitty.yaml b/config/samples/telemetry_v1beta1_cloudkitty.yaml new file mode 100644 index 00000000..0f649ad3 --- /dev/null +++ b/config/samples/telemetry_v1beta1_cloudkitty.yaml @@ -0,0 +1,12 @@ +apiVersion: telemetry.openstack.org/v1beta1 +kind: CloudKitty +metadata: + labels: + app.kubernetes.io/name: cloudkitty + app.kubernetes.io/instance: cloudkitty-sample + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: telemetry-operator + name: cloudkitty-sample +spec: + # TODO(user): Add fields here diff --git a/config/samples/telemetry_v1beta1_cloudkittyapi.yaml b/config/samples/telemetry_v1beta1_cloudkittyapi.yaml new file mode 100644 index 00000000..47725bf7 --- /dev/null +++ b/config/samples/telemetry_v1beta1_cloudkittyapi.yaml @@ -0,0 +1,12 @@ +apiVersion: telemetry.openstack.org/v1beta1 +kind: CloudKittyApi +metadata: + labels: + app.kubernetes.io/name: cloudkittyapi + app.kubernetes.io/instance: cloudkittyapi-sample + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: telemetry-operator + name: cloudkittyapi-sample +spec: + # TODO(user): Add fields here diff --git a/config/samples/telemetry_v1beta1_cloudkittyproc.yaml b/config/samples/telemetry_v1beta1_cloudkittyproc.yaml new file mode 100644 index 00000000..94e90340 --- /dev/null +++ b/config/samples/telemetry_v1beta1_cloudkittyproc.yaml @@ -0,0 +1,12 @@ +apiVersion: telemetry.openstack.org/v1beta1 +kind: CloudKittyProc +metadata: + labels: + app.kubernetes.io/name: cloudkittyproc + app.kubernetes.io/instance: cloudkittyproc-sample + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: telemetry-operator + name: cloudkittyproc-sample +spec: + # TODO(user): Add fields here diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 288d1e6f..c6b4c02c 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -44,6 +44,26 @@ webhooks: resources: - ceilometers sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-telemetry-openstack-org-v1beta1-cloudkitty + failurePolicy: Fail + name: mcloudkitty.kb.io + rules: + - apiGroups: + - telemetry.openstack.org + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - cloudkitties + sideEffects: None - admissionReviewVersions: - v1 clientConfig: @@ -130,6 +150,26 @@ webhooks: resources: - ceilometers sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-telemetry-openstack-org-v1beta1-cloudkitty + failurePolicy: Fail + name: vcloudkitty.kb.io + rules: + - apiGroups: + - telemetry.openstack.org + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - cloudkitties + sideEffects: None - admissionReviewVersions: - v1 clientConfig: diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go new file mode 100644 index 00000000..4dadd651 --- /dev/null +++ b/controllers/cloudkitty_controller.go @@ -0,0 +1,1085 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/go-logr/logr" + networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" + rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" + keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/job" + "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" + common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" + "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" + mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// GetClient - +func (r *CloudKittyReconciler) GetClient() client.Client { + return r.Client +} + +// GetKClient - +func (r *CloudKittyReconciler) GetKClient() kubernetes.Interface { + return r.Kclient +} + +// GetScheme - +func (r *CloudKittyReconciler) GetScheme() *runtime.Scheme { + return r.Scheme +} + +// CloudKittyReconciler reconciles a CloudKitty object +type CloudKittyReconciler struct { + client.Client + Kclient kubernetes.Interface + Scheme *runtime.Scheme +} + +// GetLogger returns a logger object with a logging prefix of "controller.name" and additional controller context fields +func (r *CloudKittyReconciler) GetLogger(ctx context.Context) logr.Logger { + return log.FromContext(ctx).WithName("Controllers").WithName("CloudKitty") +} + +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkitties,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkitties/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkitties/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyapis,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyapis/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyapis/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyprocs,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyprocs/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyprocs/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbdatabases,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbaccounts,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbaccounts/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=memcached.openstack.org,resources=memcacheds,verbs=get;list;watch; +// +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneapis,verbs=get;list;watch +// +kubebuilder:rbac:groups=rabbitmq.openstack.org,resources=transporturls,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch + +// service account, role, rolebinding +// +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;update;patch +// service account permissions that are needed to grant permission to the above +// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=anyuid;privileged,resources=securitycontextconstraints,verbs=use +// +kubebuilder:rbac:groups="",resources=pods,verbs=create;delete;get;list;patch;update;watch + +// Reconcile - +func (r *CloudKittyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { + Log := r.GetLogger(ctx) + + // Fetch the CloudKitty instance + instance := &telemetryv1.CloudKitty{} + err := r.Client.Get(ctx, req.NamespacedName, instance) + if err != nil { + if k8s_errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. + // For additional cleanup logic use finalizers. Return and don't requeue. + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + Log.Error(err, fmt.Sprintf("could not fetch CloudKitty instance %s", instance.Name)) + return ctrl.Result{}, err + } + + helper, err := helper.NewHelper( + instance, + r.Client, + r.Kclient, + r.Scheme, + Log, + ) + if err != nil { + Log.Error(err, fmt.Sprintf("could not instantiate helper for instance %s", instance.Name)) + return ctrl.Result{}, err + } + + // + // initialize status + // + isNewInstance := instance.Status.Conditions == nil + if isNewInstance { + instance.Status.Conditions = condition.Conditions{} + } + + // Save a copy of the condtions so that we can restore the LastTransitionTime + // when a condition's state doesn't change. + savedConditions := instance.Status.Conditions.DeepCopy() + + // Always patch the instance status when exiting this function so we can persist any changes. + defer func() { + // Don't update the status, if reconciler Panics + if r := recover(); r != nil { + Log.Info(fmt.Sprintf("panic during reconcile %v\n", r)) + panic(r) + } + condition.RestoreLastTransitionTimes(&instance.Status.Conditions, savedConditions) + if instance.Status.Conditions.IsUnknown(condition.ReadyCondition) { + instance.Status.Conditions.Set( + instance.Status.Conditions.Mirror(condition.ReadyCondition)) + } + err := helper.PatchInstance(ctx, instance) + if err != nil { + _err = err + return + } + }() + + // Always initialize conditions used later as Status=Unknown + cl := condition.CreateList( + condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), + condition.UnknownCondition(condition.DBReadyCondition, condition.InitReason, condition.DBReadyInitMessage), + condition.UnknownCondition(condition.DBSyncReadyCondition, condition.InitReason, condition.DBSyncReadyInitMessage), + condition.UnknownCondition(condition.RabbitMqTransportURLReadyCondition, condition.InitReason, condition.RabbitMqTransportURLReadyInitMessage), + condition.UnknownCondition(condition.MemcachedReadyCondition, condition.InitReason, condition.MemcachedReadyInitMessage), + condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), + condition.UnknownCondition(telemetryv1.CloudKittyAPIReadyCondition, condition.InitReason, telemetryv1.CloudKittyAPIReadyInitMessage), + condition.UnknownCondition(telemetryv1.CloudKittyProcReadyCondition, condition.InitReason, telemetryv1.CloudKittyProcReadyInitMessage), + condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), + // service account, role, rolebinding conditions + condition.UnknownCondition(condition.ServiceAccountReadyCondition, condition.InitReason, condition.ServiceAccountReadyInitMessage), + condition.UnknownCondition(condition.RoleReadyCondition, condition.InitReason, condition.RoleReadyInitMessage), + condition.UnknownCondition(condition.RoleBindingReadyCondition, condition.InitReason, condition.RoleBindingReadyInitMessage), + ) + instance.Status.Conditions.Init(&cl) + // Always mark the Generation as observed early on + instance.Status.ObservedGeneration = instance.Generation + + // If we're not deleting this and the service object doesn't have our finalizer, add it. + if (instance.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(instance, helper.GetFinalizer())) || isNewInstance { + // Register overall status immediately to have an early feedback e.g. in the cli + return ctrl.Result{}, nil + } + + if instance.Status.Hash == nil { + instance.Status.Hash = map[string]string{} + } + if instance.Status.APIEndpoints == nil { + instance.Status.APIEndpoints = map[string]map[string]string{} + } + + // Handle service delete + if !instance.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, instance, helper) + } + + // Handle non-deleted clusters + return r.reconcileNormal(ctx, instance, helper) +} + +// fields to index to reconcile when change +const ( + cloudKittyPasswordSecretField = ".spec.secret" + cloudKittyCaBundleSecretNameField = ".spec.tls.caBundleSecretName" + cloudKittyTlsAPIInternalField = ".spec.tls.api.internal.secretName" + cloudKittyTlsAPIPublicField = ".spec.tls.api.public.secretName" + cloudKittyTopologyField = ".spec.topologyRef.Name" +) + +var ( + commonWatchFields = []string{ + cloudKittyPasswordSecretField, + cloudKittyCaBundleSecretNameField, + cloudKittyTopologyField, + } + cloudKittyAPIWatchFields = []string{ + cloudKittyPasswordSecretField, + cloudKittyCaBundleSecretNameField, + cloudKittyTlsAPIInternalField, + cloudKittyTlsAPIPublicField, + cloudKittyTopologyField, + } +) + +// SetupWithManager sets up the controller with the Manager. +func (r *CloudKittyReconciler) SetupWithManager(mgr ctrl.Manager) error { + // transportURLSecretFn - Watch for changes made to the secret associated with the RabbitMQ + // TransportURL created and used by CloudKitty CRs. Watch functions return a list of namespace-scoped + // CRs that then get fed to the reconciler. Hence, in this case, we need to know the name of the + // CloudKitty CR associated with the secret we are examining in the function. We could parse the name + // out of the "%s-cloudkitty-transport" secret label, which would be faster than getting the list of + // the CloudKitty CRs and trying to match on each one. The downside there, however, is that technically + // someone could randomly label a secret "something-cloudkitty-transport" where "something" actually + // matches the name of an existing CloudKitty CR. In that case changes to that secret would trigger + // reconciliation for a CloudKitty CR that does not need it. + // + // TODO: We also need a watch func to monitor for changes to the secret referenced by CloudKitty.Spec.Secret + transportURLSecretFn := func(ctx context.Context, o client.Object) []reconcile.Request { + result := []reconcile.Request{} + + Log := r.GetLogger(ctx) + + // get all CloudKitty CRs + cloudkitties := &telemetryv1.CloudKittyList{} + listOpts := []client.ListOption{ + client.InNamespace(o.GetNamespace()), + } + if err := r.Client.List(ctx, cloudkitties, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve CloudKitty CRs %v") + return nil + } + + for _, ownerRef := range o.GetOwnerReferences() { + if ownerRef.Kind == "TransportURL" { + for _, cr := range cloudkitties.Items { + if ownerRef.Name == fmt.Sprintf("%s-cloudkitty-transport", cr.Name) { + // return namespace and Name of CR + name := client.ObjectKey{ + Namespace: o.GetNamespace(), + Name: cr.Name, + } + Log.Info(fmt.Sprintf("TransportURL Secret %s belongs to TransportURL belonging to CloudKitty CR %s", o.GetName(), cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + } + } + if len(result) > 0 { + return result + } + return nil + } + + memcachedFn := func(ctx context.Context, o client.Object) []reconcile.Request { + Log := r.GetLogger(ctx) + + result := []reconcile.Request{} + + // get all CloudKitty CRs + cloudkitties := &telemetryv1.CloudKittyList{} + listOpts := []client.ListOption{ + client.InNamespace(o.GetNamespace()), + } + if err := r.Client.List(ctx, cloudkitties, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve CloudKitty CRs %w") + return nil + } + + for _, cr := range cloudkitties.Items { + if o.GetName() == cr.Spec.MemcachedInstance { + name := client.ObjectKey{ + Namespace: o.GetNamespace(), + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Memcached %s is used by CloudKitty CR %s", o.GetName(), cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + if len(result) > 0 { + return result + } + return nil + } + + return ctrl.NewControllerManagedBy(mgr). + For(&telemetryv1.CloudKitty{}). + Owns(&mariadbv1.MariaDBDatabase{}). + Owns(&mariadbv1.MariaDBAccount{}). + Owns(&telemetryv1.CloudKittyAPI{}). + Owns(&telemetryv1.CloudKittyProc{}). + Owns(&rabbitmqv1.TransportURL{}). + Owns(&batchv1.Job{}). + Owns(&corev1.Secret{}). + Owns(&corev1.ServiceAccount{}). + Owns(&rbacv1.Role{}). + Owns(&rbacv1.RoleBinding{}). + // Watch for TransportURL Secrets which belong to any TransportURLs created by CloudKitty CRs + Watches(&corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(transportURLSecretFn)). + Watches(&memcachedv1.Memcached{}, + handler.EnqueueRequestsFromMapFunc(memcachedFn)). + Watches(&keystonev1.KeystoneAPI{}, + handler.EnqueueRequestsFromMapFunc(r.findObjectForSrc), + builder.WithPredicates(keystonev1.KeystoneAPIStatusChangedPredicate)). + Complete(r) +} + +func (r *CloudKittyReconciler) findObjectForSrc(ctx context.Context, src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + l := log.FromContext(ctx).WithName("Controllers").WithName("CloudKitty") + + crList := &telemetryv1.CloudKittyList{} + listOps := &client.ListOptions{ + Namespace: src.GetNamespace(), + } + err := r.Client.List(ctx, crList, listOps) + if err != nil { + l.Error(err, fmt.Sprintf("listing %s for namespace: %s", crList.GroupVersionKind().Kind, src.GetNamespace())) + return requests + } + + for _, item := range crList.Items { + l.Info(fmt.Sprintf("input source %s changed, reconcile: %s - %s", src.GetName(), item.GetName(), item.GetNamespace())) + + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + + return requests +} + +func (r *CloudKittyReconciler) reconcileDelete(ctx context.Context, instance *telemetryv1.CloudKitty, helper *helper.Helper) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s' delete", instance.Name)) + + // remove db finalizer first + db, err := mariadbv1.GetDatabaseByNameAndAccount(ctx, helper, cloudkitty.DatabaseName, instance.Spec.DatabaseAccount, instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + + if !k8s_errors.IsNotFound(err) { + if err := db.DeleteFinalizer(ctx, helper); err != nil { + return ctrl.Result{}, err + } + } + + // TODO: We might need to control how the sub-services (API and Proc) are + // deleted (when their parent CloudKitty CR is deleted) once we further develop their functionality + + // Service is deleted so remove the finalizer. + controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) + Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", instance.Name)) + + return ctrl.Result{}, nil +} + +func (r *CloudKittyReconciler) reconcileInit( + ctx context.Context, + instance *telemetryv1.CloudKitty, + helper *helper.Helper, + serviceLabels map[string]string, + serviceAnnotations map[string]string, +) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s' init", instance.Name)) + + // + // run CloudKitty db sync + // + dbSyncHash := instance.Status.Hash[telemetryv1.CKDbSyncHash] + jobDef := cloudkitty.DbSyncJob(instance, serviceLabels) + + dbSyncjob := job.NewJob( + jobDef, + telemetryv1.CKDbSyncHash, + instance.Spec.PreserveJobs, + cloudkitty.ShortDuration, + dbSyncHash, + ) + ctrlResult, err := dbSyncjob.DoJob( + ctx, + helper, + ) + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBSyncReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBSyncReadyRunningMessage)) + return ctrlResult, nil + } + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBSyncReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBSyncReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if dbSyncjob.HasChanged() { + instance.Status.Hash[telemetryv1.CKDbSyncHash] = dbSyncjob.GetHash() + Log.Info(fmt.Sprintf("Service '%s' - Job %s hash added - %s", instance.Name, jobDef.Name, instance.Status.Hash[telemetryv1.CKDbSyncHash])) + } + instance.Status.Conditions.MarkTrue(condition.DBSyncReadyCondition, condition.DBSyncReadyMessage) + + // run CloudKitty db sync - end + + Log.Info(fmt.Sprintf("Reconciled Service '%s' init successfully", instance.Name)) + return ctrl.Result{}, nil +} + +func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *telemetryv1.CloudKitty, helper *helper.Helper) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s'", instance.Name)) + + // Service account, role, binding + rbacRules := []rbacv1.PolicyRule{ + { + APIGroups: []string{"security.openshift.io"}, + ResourceNames: []string{"anyuid"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"create", "get", "list", "watch", "update", "patch", "delete"}, + }, + } + rbacResult, err := common_rbac.ReconcileRbac(ctx, helper, instance, rbacRules) + if err != nil { + return rbacResult, err + } else if (rbacResult != ctrl.Result{}) { + return rbacResult, nil + } + + serviceLabels := map[string]string{ + common.AppSelector: cloudkitty.ServiceName, + } + + configVars := make(map[string]env.Setter) + + // + // create RabbitMQ transportURL CR and get the actual URL from the associated secret that is created + // + + transportURL, op, err := r.transportURLCreateOrUpdate(ctx, instance, serviceLabels) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.RabbitMqTransportURLReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.RabbitMqTransportURLReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if op != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf("TransportURL %s successfully reconciled - operation: %s", transportURL.Name, string(op))) + } + + instance.Status.TransportURLSecret = transportURL.Status.SecretName + + if instance.Status.TransportURLSecret == "" { + Log.Info(fmt.Sprintf("Waiting for TransportURL %s secret to be created", transportURL.Name)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.RabbitMqTransportURLReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.RabbitMqTransportURLReadyRunningMessage)) + return cloudkitty.ResultRequeue, nil + } + + instance.Status.Conditions.MarkTrue(condition.RabbitMqTransportURLReadyCondition, condition.RabbitMqTransportURLReadyMessage) + + // end transportURL + + // + // Check for required memcached used for caching + // + memcached, err := memcachedv1.GetMemcachedByName(ctx, helper, instance.Spec.MemcachedInstance, instance.Namespace) + if err != nil { + Log.Info(fmt.Sprintf("%s... requeueing", condition.MemcachedReadyWaitingMessage)) + if k8s_errors.IsNotFound(err) { + Log.Info(fmt.Sprintf("memcached %s not found", instance.Spec.MemcachedInstance)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.MemcachedReadyWaitingMessage)) + return cloudkitty.ResultRequeue, nil + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.MemcachedReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if !memcached.IsReady() { + Log.Info(fmt.Sprintf("%s... requeueing", condition.MemcachedReadyWaitingMessage)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.MemcachedReadyWaitingMessage)) + return cloudkitty.ResultRequeue, nil + } + // Mark the Memcached Service as Ready if we get to this point with no errors + instance.Status.Conditions.MarkTrue( + condition.MemcachedReadyCondition, condition.MemcachedReadyMessage) + // run check memcached - end + + // + // check for required OpenStack secret holding passwords for service/admin user and add hash to the vars map + // + + result, err := cloudkitty.VerifyServiceSecret( + ctx, + types.NamespacedName{Namespace: instance.Namespace, Name: instance.Spec.Secret}, + []string{ + instance.Spec.PasswordSelectors.CloudKittyService, + }, + helper.GetClient(), + &instance.Status.Conditions, + cloudkitty.NormalDuration, + &configVars, + ) + if err != nil { + return result, err + } else if (result != ctrl.Result{}) { + return result, nil + } + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + // run check OpenStack secret - end + + db, result, err := r.ensureDB(ctx, helper, instance) + if err != nil { + return ctrl.Result{}, err + } else if (result != ctrl.Result{}) { + return result, nil + } + + // + // Create Secrets required as input for the Service and calculate an overall hash of hashes + // + err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels, memcached, db) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + // + // create hash over all the different input resources to identify if any those changed + // and a restart/recreate is required. + // + _, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configVars) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } else if hashChanged { + Log.Info(fmt.Sprintf("%s... requeueing", condition.ServiceConfigReadyInitMessage)) + instance.Status.Conditions.MarkFalse( + condition.ServiceConfigReadyCondition, + condition.InitReason, + condition.SeverityInfo, + condition.ServiceConfigReadyInitMessage) + // Hash changed and instance status should be updated (which will be done by main defer func), + // so we need to return and reconcile again + return ctrl.Result{}, nil + } + + instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) + + // + // TODO check when/if Init, Update, or Upgrade should/could be skipped + // + + // Check networks that the DBSync job will use in reconcileInit. The ones from the API service are always enough, + // it doesn't need the storage specific ones that volume or backup may have. + nadList := []networkv1.NetworkAttachmentDefinition{} + for _, netAtt := range instance.Spec.CloudKittyAPI.NetworkAttachments { + nad, err := nad.GetNADWithName(ctx, helper, netAtt, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + Log.Info(fmt.Sprintf("network-attachment-definition %s not found", netAtt)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.NetworkAttachmentsReadyWaitingMessage, + netAtt)) + return cloudkitty.ResultRequeue, fmt.Errorf(condition.NetworkAttachmentsReadyWaitingMessage, netAtt) + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if nad != nil { + nadList = append(nadList, *nad) + } + } + + instance.Status.Conditions.MarkTrue(condition.NetworkAttachmentsReadyCondition, condition.NetworkAttachmentsReadyMessage) + + serviceAnnotations, err := nad.EnsureNetworksAnnotation(nadList) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed create network annotation from %s: %w", + instance.Spec.CloudKittyAPI.NetworkAttachments, err) + } + + // Handle service init + ctrlResult, err := r.reconcileInit(ctx, instance, helper, serviceLabels, serviceAnnotations) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + // + // normal reconcile tasks + // + + // deploy cloudkitty-api + cloudKittyAPI, op, err := r.apiDeploymentCreateOrUpdate(ctx, instance) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyAPIReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + telemetryv1.CloudKittyAPIReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if op != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf("API CR for %s successfully %s", instance.Name, string(op))) + } + + // Mirror values when the data in the StatefulSet is for the current generation + if cloudKittyAPI.Generation == cloudKittyAPI.Status.ObservedGeneration { + // Mirror CloudKittyAPI status' APIEndpoints and ReadyCount to this parent CR + instance.Status.APIEndpoints = cloudKittyAPI.Status.APIEndpoints + instance.Status.ServiceIDs = cloudKittyAPI.Status.ServiceIDs + instance.Status.CloudKittyAPIReadyCount = cloudKittyAPI.Status.ReadyCount + + // Mirror CloudKittyAPI's condition status + c := cloudKittyAPI.Status.Conditions.Mirror(telemetryv1.CloudKittyAPIReadyCondition) + if c != nil { + instance.Status.Conditions.Set(c) + } + } + + // deploy CloudKitty Processor + cloudKittyProc, op, err := r.procDeploymentCreateOrUpdate(ctx, instance) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyProcReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + telemetryv1.CloudKittyProcReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if op != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf("Scheduler CR for %s successfully %s", instance.Name, string(op))) + } + + // Mirror values when the data in the StatefulSet is for the current generation + if cloudKittyProc.Generation == cloudKittyProc.Status.ObservedGeneration { + // Mirror CloudKitty Processor status' ReadyCount to this parent CR + instance.Status.CloudKittyProcReadyCount = cloudKittyProc.Status.ReadyCount + + // Mirror CloudKittyProc's condition status + c := cloudKittyProc.Status.Conditions.Mirror(telemetryv1.CloudKittyProcReadyCondition) + if c != nil { + instance.Status.Conditions.Set(c) + } + } + + err = mariadbv1.DeleteUnusedMariaDBAccountFinalizers(ctx, helper, cloudkitty.DatabaseName, instance.Spec.DatabaseAccount, instance.Namespace) + if err != nil { + return ctrl.Result{}, err + } + + Log.Info(fmt.Sprintf("Reconciled Service '%s' successfully", instance.Name)) + // update the overall status condition if service is ready + if instance.IsReady() { + instance.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) + } + return ctrl.Result{}, nil +} + +// generateServiceConfigs - create Secret which hold scripts and service configuration +func (r *CloudKittyReconciler) generateServiceConfigs( + ctx context.Context, + h *helper.Helper, + instance *telemetryv1.CloudKitty, + envVars *map[string]env.Setter, + serviceLabels map[string]string, + memcached *memcachedv1.Memcached, + db *mariadbv1.Database, +) error { + // + // create Secret required for cloudkitty input + // - %-scripts holds scripts to e.g. bootstrap the service + // - %-config holds minimal cloudkitty config required to get the service up + // + + labels := labels.GetLabels(instance, labels.GetGroupLabel(cloudkitty.ServiceName), serviceLabels) + + var tlsCfg *tls.Service + if instance.Spec.CloudKittyAPI.TLS.Ca.CaBundleSecretName != "" { + tlsCfg = &tls.Service{} + } + + // customData hold any customization for all cloudkitty services. + customData := map[string]string{ + cloudkitty.CustomConfigFileName: instance.Spec.CustomServiceConfig, + cloudkitty.MyCnfFileName: db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf + } + + keystoneAPI, err := keystonev1.GetKeystoneAPI(ctx, h, instance.Namespace, map[string]string{}) + if err != nil { + return err + } + keystoneInternalURL, err := keystoneAPI.GetEndpoint(endpoint.EndpointInternal) + if err != nil { + return err + } + keystonePublicURL, err := keystoneAPI.GetEndpoint(endpoint.EndpointPublic) + if err != nil { + return err + } + + ospSecret, _, err := secret.GetSecret(ctx, h, instance.Spec.Secret, instance.Namespace) + if err != nil { + return err + } + + transportURLSecret, _, err := secret.GetSecret(ctx, h, instance.Status.TransportURLSecret, instance.Namespace) + if err != nil { + return err + } + + databaseAccount := db.GetAccount() + dbSecret := db.GetSecret() + + templateParameters := make(map[string]interface{}) + templateParameters["ServiceUser"] = instance.Spec.ServiceUser + templateParameters["ServicePassword"] = string(ospSecret.Data[instance.Spec.PasswordSelectors.CloudKittyService]) + templateParameters["KeystoneInternalURL"] = keystoneInternalURL + templateParameters["KeystonePublicURL"] = keystonePublicURL + templateParameters["TransportURL"] = string(transportURLSecret.Data["transport_url"]) + templateParameters["DatabaseConnection"] = fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf", + databaseAccount.Spec.UserName, + string(dbSecret.Data[mariadbv1.DatabasePasswordSelector]), + instance.Status.DatabaseHostname, + cloudkitty.DatabaseName) + templateParameters["MemcachedServersWithInet"] = memcached.GetMemcachedServerListWithInetString() + templateParameters["TimeOut"] = instance.Spec.APITimeout + + // create httpd vhost template parameters + httpdVhostConfig := map[string]interface{}{} + for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} { + endptConfig := map[string]interface{}{} + endptConfig["ServerName"] = fmt.Sprintf("%s-%s.%s.svc", cloudkitty.ServiceName, endpt.String(), instance.Namespace) + endptConfig["TLS"] = false // default TLS to false, and set it bellow to true if enabled + if instance.Spec.CloudKittyAPI.TLS.API.Enabled(endpt) { + endptConfig["TLS"] = true + endptConfig["SSLCertificateFile"] = fmt.Sprintf("/etc/pki/tls/certs/%s.crt", endpt.String()) + endptConfig["SSLCertificateKeyFile"] = fmt.Sprintf("/etc/pki/tls/private/%s.key", endpt.String()) + } + httpdVhostConfig[endpt.String()] = endptConfig + } + templateParameters["VHosts"] = httpdVhostConfig + + configTemplates := []util.Template{ + { + Name: fmt.Sprintf("%s-scripts", instance.Name), + Namespace: instance.Namespace, + Type: util.TemplateTypeScripts, + InstanceType: instance.Kind, + Labels: labels, + }, + { + Name: fmt.Sprintf("%s-config-data", instance.Name), + Namespace: instance.Namespace, + Type: util.TemplateTypeConfig, + InstanceType: instance.Kind, + CustomData: customData, + ConfigOptions: templateParameters, + Labels: labels, + }, + } + + return secret.EnsureSecrets(ctx, h, instance, configTemplates, envVars) +} + +// createHashOfInputHashes - creates a hash of hashes which gets added to the resources which requires a restart +// if any of the input resources change, like configs, passwords, ... +// +// returns the hash, whether the hash changed (as a bool) and any error +func (r *CloudKittyReconciler) createHashOfInputHashes( + ctx context.Context, + instance *telemetryv1.CloudKitty, + envVars map[string]env.Setter, +) (string, bool, error) { + Log := r.GetLogger(ctx) + + var hashMap map[string]string + changed := false + mergedMapVars := env.MergeEnvs([]corev1.EnvVar{}, envVars) + hash, err := util.ObjectHash(mergedMapVars) + if err != nil { + return hash, changed, err + } + if hashMap, changed = util.SetHash(instance.Status.Hash, common.InputHashName, hash); changed { + instance.Status.Hash = hashMap + Log.Info(fmt.Sprintf("Input maps hash %s - %s", common.InputHashName, hash)) + } + return hash, changed, nil +} + +func (r *CloudKittyReconciler) transportURLCreateOrUpdate( + ctx context.Context, + instance *telemetryv1.CloudKitty, + serviceLabels map[string]string, +) (*rabbitmqv1.TransportURL, controllerutil.OperationResult, error) { + transportURL := &rabbitmqv1.TransportURL{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-cloudkitty-transport", instance.Name), + Namespace: instance.Namespace, + Labels: serviceLabels, + }, + } + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, transportURL, func() error { + transportURL.Spec.RabbitmqClusterName = instance.Spec.RabbitMqClusterName + + err := controllerutil.SetControllerReference(instance, transportURL, r.Scheme) + return err + }) + + return transportURL, op, err +} + +func (r *CloudKittyReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, instance *telemetryv1.CloudKitty) (*telemetryv1.CloudKittyAPI, controllerutil.OperationResult, error) { + cloudkittyAPISpec := telemetryv1.CloudKittyAPISpec{ + CloudKittyTemplate: instance.Spec.CloudKittyTemplate, + CloudKittyAPITemplate: instance.Spec.CloudKittyAPI, + DatabaseHostname: instance.Status.DatabaseHostname, + TransportURLSecret: instance.Status.TransportURLSecret, + ServiceAccount: instance.RbacResourceName(), + } + + if cloudkittyAPISpec.NodeSelector == nil { + cloudkittyAPISpec.NodeSelector = instance.Spec.NodeSelector + } + + // If topology is not present in the underlying CloudKittyAPI Spec, + // inherit from the top-level CR + if cloudkittyAPISpec.TopologyRef == nil { + cloudkittyAPISpec.TopologyRef = instance.Spec.TopologyRef + } + + deployment := &telemetryv1.CloudKittyAPI{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-api", instance.Name), + Namespace: instance.Namespace, + }, + } + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { + deployment.Spec = cloudkittyAPISpec + + err := controllerutil.SetControllerReference(instance, deployment, r.Scheme) + if err != nil { + return err + } + + return nil + }) + + return deployment, op, err +} + +func (r *CloudKittyReconciler) procDeploymentCreateOrUpdate(ctx context.Context, instance *telemetryv1.CloudKitty) (*telemetryv1.CloudKittyProc, controllerutil.OperationResult, error) { + cloudKittyProcSpec := telemetryv1.CloudKittyProcSpec{ + CloudKittyTemplate: instance.Spec.CloudKittyTemplate, + CloudKittyProcTemplate: instance.Spec.CloudKittyProc, + DatabaseHostname: instance.Status.DatabaseHostname, + TransportURLSecret: instance.Status.TransportURLSecret, + ServiceAccount: instance.RbacResourceName(), + TLS: instance.Spec.CloudKittyAPI.TLS.Ca, + } + + if cloudKittyProcSpec.NodeSelector == nil { + cloudKittyProcSpec.NodeSelector = instance.Spec.NodeSelector + } + + // If topology is not present in the underlying Scheduler Spec + // inherit from the top-level CR + if cloudKittyProcSpec.TopologyRef == nil { + cloudKittyProcSpec.TopologyRef = instance.Spec.TopologyRef + } + + deployment := &telemetryv1.CloudKittyProc{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-proc", instance.Name), + Namespace: instance.Namespace, + }, + } + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { + deployment.Spec = cloudKittyProcSpec + + err := controllerutil.SetControllerReference(instance, deployment, r.Scheme) + if err != nil { + return err + } + + return nil + }) + + return deployment, op, err +} + +func (r *CloudKittyReconciler) ensureDB( + ctx context.Context, + h *helper.Helper, + instance *telemetryv1.CloudKitty, +) (*mariadbv1.Database, ctrl.Result, error) { + Log := r.GetLogger(ctx) + + // ensure MariaDBAccount exists. This account record may be created by + // openstack-operator or the cloud operator up front without a specific + // MariaDBDatabase configured yet. Otherwise, a MariaDBAccount CR is + // created here with a generated username as well as a secret with + // generated password. The MariaDBAccount is created without being + // yet associated with any MariaDBDatabase. + _, _, err := mariadbv1.EnsureMariaDBAccount( + ctx, h, instance.Spec.DatabaseAccount, + instance.Namespace, false, "cloudkitty", + ) + + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + mariadbv1.MariaDBAccountReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + mariadbv1.MariaDBAccountNotReadyMessage, + err.Error())) + + return nil, ctrl.Result{}, err + } + instance.Status.Conditions.MarkTrue( + mariadbv1.MariaDBAccountReadyCondition, + mariadbv1.MariaDBAccountReadyMessage, + ) + + db := mariadbv1.NewDatabaseForAccount( + instance.Spec.DatabaseInstance, // mariadb/galera service to target + cloudkitty.DatabaseName, // name used in CREATE DATABASE in mariadb + cloudkitty.DatabaseName, // CR name for MariaDBDatabase + instance.Spec.DatabaseAccount, // CR name for MariaDBAccount + instance.Namespace, // namespace + ) + + // create or patch the DB + ctrlResult, err := db.CreateOrPatchAll(ctx, h) + + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBReadyErrorMessage, + err.Error())) + return db, ctrl.Result{}, err + } + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBReadyRunningMessage)) + return db, ctrlResult, nil + } + // wait for the DB to be setup + // (ksambor) should we use WaitForDBCreatedWithTimeout instead? + ctrlResult, err = db.WaitForDBCreated(ctx, h) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBReadyErrorMessage, + err.Error())) + return db, ctrlResult, err + } + if (ctrlResult != ctrl.Result{}) { + Log.Info(fmt.Sprintf("%s... requeueing", condition.DBReadyRunningMessage)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBReadyRunningMessage)) + return db, ctrlResult, nil + } + + // update Status.DatabaseHostname, used to config the service + instance.Status.DatabaseHostname = db.GetDatabaseHostname() + instance.Status.Conditions.MarkTrue(condition.DBReadyCondition, condition.DBReadyMessage) + return db, ctrlResult, nil +} diff --git a/controllers/cloudkittyapi_controller.go b/controllers/cloudkittyapi_controller.go new file mode 100644 index 00000000..082f6104 --- /dev/null +++ b/controllers/cloudkittyapi_controller.go @@ -0,0 +1,1119 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkittyapi" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/go-logr/logr" + networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" + "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" +) + +// GetClient - +func (r *CloudKittyAPIReconciler) GetClient() client.Client { + return r.Client +} + +// GetKClient - +func (r *CloudKittyAPIReconciler) GetKClient() kubernetes.Interface { + return r.Kclient +} + +// GetScheme - +func (r *CloudKittyAPIReconciler) GetScheme() *runtime.Scheme { + return r.Scheme +} + +// CloudKittyAPIReconciler reconciles a CloudKittyAPI object +type CloudKittyAPIReconciler struct { + client.Client + Kclient kubernetes.Interface + Scheme *runtime.Scheme +} + +// GetLogger returns a logger object with a logging prefix of "controller.name" and additional controller context fields +func (r *CloudKittyAPIReconciler) GetLogger(ctx context.Context) logr.Logger { + return log.FromContext(ctx).WithName("Controllers").WithName("CloudKittyAPI") +} + +var keystoneServices = []map[string]string{ + { + "type": cloudkitty.ServiceType, + "name": cloudkitty.ServiceName, + "desc": "CloudKitty V2 Service", + }, +} + +//+kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyapis,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyapis/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyapis/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list; +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneservices,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneendpoints,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch +// +kubebuilder:rbac:groups=topology.openstack.org,resources=topologies,verbs=get;list;watch;update + +// Reconcile - +func (r *CloudKittyAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { + Log := r.GetLogger(ctx) + + // Fetch the CloudKittyAPI instance + instance := &telemetryv1.CloudKittyAPI{} + err := r.Client.Get(ctx, req.NamespacedName, instance) + if err != nil { + if k8s_errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. + // For additional cleanup logic use finalizers. Return and don't requeue. + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + return ctrl.Result{}, err + } + + helper, err := helper.NewHelper( + instance, + r.Client, + r.Kclient, + r.Scheme, + Log, + ) + if err != nil { + return ctrl.Result{}, err + } + + // + // initialize status + // + isNewInstance := instance.Status.Conditions == nil + if isNewInstance { + instance.Status.Conditions = condition.Conditions{} + } + + // Save a copy of the condtions so that we can restore the LastTransitionTime + // when a condition's state doesn't change. + savedConditions := instance.Status.Conditions.DeepCopy() + + // Always patch the instance status when exiting this function so we can persist any changes. + defer func() { + // Don't update the status, if reconciler Panics + if r := recover(); r != nil { + Log.Info(fmt.Sprintf("panic during reconcile %v\n", r)) + panic(r) + } + condition.RestoreLastTransitionTimes(&instance.Status.Conditions, savedConditions) + if instance.Status.Conditions.IsUnknown(condition.ReadyCondition) { + instance.Status.Conditions.Set( + instance.Status.Conditions.Mirror(condition.ReadyCondition)) + } + err := helper.PatchInstance(ctx, instance) + if err != nil { + _err = err + return + } + }() + + // Always initialize conditions used later as Status=Unknown + cl := condition.CreateList( + condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), + condition.UnknownCondition(condition.CreateServiceReadyCondition, condition.InitReason, condition.CreateServiceReadyInitMessage), + condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), + condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), + // right now we have no dedicated KeystoneServiceReadyInitMessage and KeystoneEndpointReadyInitMessage + condition.UnknownCondition(condition.KeystoneServiceReadyCondition, condition.InitReason, ""), + condition.UnknownCondition(condition.KeystoneEndpointReadyCondition, condition.InitReason, ""), + condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), + condition.UnknownCondition(condition.TLSInputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + ) + instance.Status.Conditions.Init(&cl) + // Always mark the Generation as observed early on + instance.Status.ObservedGeneration = instance.Generation + + // If we're not deleting this and the service object doesn't have our finalizer, add it. + if (instance.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(instance, helper.GetFinalizer())) || isNewInstance { + // Register overall status immediately to have an early feedback e.g. in the cli + return ctrl.Result{}, nil + } + + if instance.Status.Hash == nil { + instance.Status.Hash = map[string]string{} + } + if instance.Status.APIEndpoints == nil { + instance.Status.APIEndpoints = map[string]map[string]string{} + } + if instance.Status.ServiceIDs == nil { + instance.Status.ServiceIDs = map[string]string{} + } + if instance.Status.NetworkAttachments == nil { + instance.Status.NetworkAttachments = map[string][]string{} + } + + // Handle service delete + if !instance.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, instance, helper) + } + + // Init Topology condition if there's a reference + if instance.Spec.TopologyRef != nil { + c := condition.UnknownCondition(condition.TopologyReadyCondition, condition.InitReason, condition.TopologyReadyInitMessage) + cl.Set(c) + } + + // Handle non-deleted clusters + return r.reconcileNormal(ctx, instance, helper) +} + +// SetupWithManager sets up the controller with the Manager. +func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { + Log := r.GetLogger(ctx) + + // Watch for changes to secrets we don't own. Global secrets + // (e.g. TransportURLSecret) are handled by the main cloudkitty controller. + secretFn := func(_ context.Context, o client.Object) []reconcile.Request { + var namespace string = o.GetNamespace() + var secretName string = o.GetName() + result := []reconcile.Request{} + + // get all API CRs + apis := &telemetryv1.CloudKittyAPIList{} + listOpts := []client.ListOption{ + client.InNamespace(namespace), + } + if err := r.Client.List(context.Background(), apis, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve API CRs %v") + return nil + } + + // Watch for changes to secrets where the owner label AND the + // CR.Spec.ManagingCrName label matches + label := o.GetLabels() + if l, ok := label[labels.GetOwnerNameLabelSelector(labels.GetGroupLabel(cloudkitty.ServiceName))]; ok { + for _, cr := range apis.Items { + // return reconcile event for the CR where the owner label AND the parentCloudKittyName matches + if l == cloudkitty.GetOwningCloudKittyName(&cr) { + // return namespace and Name of CR + name := client.ObjectKey{ + Namespace: o.GetNamespace(), + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Secret %s and CR %s marked with label: %s", o.GetName(), cr.Name, l)) + + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + } + + // Watch for changes to any CustomServiceConfigSecrets + for _, cr := range apis.Items { + for _, v := range cr.Spec.CustomServiceConfigSecrets { + if v == secretName { + name := client.ObjectKey{ + Namespace: namespace, + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Secret %s is used by CloudKitty CR %s", secretName, cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + } + if len(result) > 0 { + return result + } + return nil + } + + // index passwordSecretField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyPasswordSecretField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyAPI) + if cr.Spec.Secret == "" { + return nil + } + return []string{cr.Spec.Secret} + }); err != nil { + return err + } + + // index caBundleSecretNameField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyCaBundleSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyAPI) + if cr.Spec.TLS.CaBundleSecretName == "" { + return nil + } + return []string{cr.Spec.TLS.CaBundleSecretName} + }); err != nil { + return err + } + + // index tlsAPIInternalField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyTlsAPIInternalField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyAPI) + if cr.Spec.TLS.API.Internal.SecretName == nil { + return nil + } + return []string{*cr.Spec.TLS.API.Internal.SecretName} + }); err != nil { + return err + } + + // index tlsAPIPublicField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyTlsAPIPublicField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyAPI) + if cr.Spec.TLS.API.Public.SecretName == nil { + return nil + } + return []string{*cr.Spec.TLS.API.Public.SecretName} + }); err != nil { + return err + } + + // index topologyField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyTopologyField, func(rawObj client.Object) []string { + // Extract the topology name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyAPI) + if cr.Spec.TopologyRef == nil { + return nil + } + return []string{cr.Spec.TopologyRef.Name} + }); err != nil { + return err + } + + return ctrl.NewControllerManagedBy(mgr). + For(&telemetryv1.CloudKittyAPI{}). + Owns(&keystonev1.KeystoneService{}). + Owns(&keystonev1.KeystoneEndpoint{}). + Owns(&appsv1.StatefulSet{}). + Owns(&corev1.Service{}). + // watch the secrets we don't own + Watches(&corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(secretFn)). + Watches( + &corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). + Watches(&topologyv1.Topology{}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.GenerationChangedPredicate{})). + Complete(r) +} + +func (r *CloudKittyAPIReconciler) findObjectsForSrc(ctx context.Context, src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + l := log.FromContext(ctx).WithName("Controllers").WithName("CloudKittyAPI") + + for _, field := range cloudKittyAPIWatchFields { + crList := &telemetryv1.CloudKittyAPIList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.List(ctx, crList, listOps) + if err != nil { + l.Error(err, fmt.Sprintf("listing %s for field: %s - %s", crList.GroupVersionKind().Kind, field, src.GetNamespace())) + return requests + } + + for _, item := range crList.Items { + l.Info(fmt.Sprintf("input source %s changed, reconcile: %s - %s", src.GetName(), item.GetName(), item.GetNamespace())) + + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} + +func (r *CloudKittyAPIReconciler) reconcileDelete(ctx context.Context, instance *telemetryv1.CloudKittyAPI, helper *helper.Helper) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s' delete", instance.Name)) + + // It's possible to get here before the endpoints have been set in the status, so check for this + if instance.Status.APIEndpoints != nil { + for _, ksSvc := range keystoneServices { + + // Remove the finalizer from our KeystoneEndpoint CR + keystoneEndpoint, err := keystonev1.GetKeystoneEndpointWithName(ctx, helper, ksSvc["name"], instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + + if err == nil { + controllerutil.RemoveFinalizer(keystoneEndpoint, helper.GetFinalizer()) + if err = helper.GetClient().Update(ctx, keystoneEndpoint); err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + util.LogForObject(helper, "Removed finalizer from our KeystoneEndpoint", instance) + } + + // Remove the finalizer from our KeystoneService CR + keystoneService, err := keystonev1.GetKeystoneServiceWithName(ctx, helper, ksSvc["name"], instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + + if err == nil { + controllerutil.RemoveFinalizer(keystoneService, helper.GetFinalizer()) + if err = helper.GetClient().Update(ctx, keystoneService); err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + util.LogForObject(helper, "Removed finalizer from our KeystoneService", instance) + } + } + } + + // Remove finalizer on the Topology CR + if ctrlResult, err := topologyv1.EnsureDeletedTopologyRef( + ctx, + helper, + instance.Status.LastAppliedTopology, + instance.Name, + ); err != nil { + return ctrlResult, err + } + + // Service is deleted so remove the finalizer. + controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) + Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", instance.Name)) + + return ctrl.Result{}, nil +} + +func (r *CloudKittyAPIReconciler) reconcileInit( + ctx context.Context, + instance *telemetryv1.CloudKittyAPI, + helper *helper.Helper, + serviceLabels map[string]string, +) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s' init", instance.Name)) + + // + // expose the service (create service and return the created endpoint URLs) + // + + // V2 + publicEndpointData := endpoint.Data{ + Port: cloudkitty.CloudKittyPublicPort, + Path: "/v2", + } + internalEndpointData := endpoint.Data{ + Port: cloudkitty.CloudKittyInternalPort, + Path: "/v2", + } + cloudkittyEndpoints := map[service.Endpoint]endpoint.Data{ + service.EndpointPublic: publicEndpointData, + service.EndpointInternal: internalEndpointData, + } + + apiEndpointsV3 := make(map[string]string) + + for endpointType, data := range cloudkittyEndpoints { + endpointTypeStr := string(endpointType) + endpointName := cloudkitty.ServiceName + "-" + endpointTypeStr + svcOverride := instance.Spec.Override.Service[endpointType] + if svcOverride.EmbeddedLabelsAnnotations == nil { + svcOverride.EmbeddedLabelsAnnotations = &service.EmbeddedLabelsAnnotations{} + } + + exportLabels := util.MergeStringMaps( + serviceLabels, + map[string]string{ + service.AnnotationEndpointKey: endpointTypeStr, + }, + ) + + // Create the service + svc, err := service.NewService( + service.GenericService(&service.GenericServiceDetails{ + Name: endpointName, + Namespace: instance.Namespace, + Labels: exportLabels, + Selector: serviceLabels, + Port: service.GenericServicePort{ + Name: endpointName, + Port: data.Port, + Protocol: corev1.ProtocolTCP, + }, + }), + 5, + &svcOverride.OverrideSpec, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.CreateServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.CreateServiceReadyErrorMessage, + err.Error())) + + return ctrl.Result{}, err + } + + svc.AddAnnotation(map[string]string{ + service.AnnotationEndpointKey: endpointTypeStr, + }) + + // add Annotation to whether creating an ingress is required or not + if endpointType == service.EndpointPublic && svc.GetServiceType() == corev1.ServiceTypeClusterIP { + svc.AddAnnotation(map[string]string{ + service.AnnotationIngressCreateKey: "true", + }) + } else { + svc.AddAnnotation(map[string]string{ + service.AnnotationIngressCreateKey: "false", + }) + if svc.GetServiceType() == corev1.ServiceTypeLoadBalancer { + svc.AddAnnotation(map[string]string{ + service.AnnotationHostnameKey: svc.GetServiceHostname(), // add annotation to register service name in dnsmasq + }) + } + } + + ctrlResult, err := svc.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.CreateServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.CreateServiceReadyErrorMessage, + err.Error())) + + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.CreateServiceReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.CreateServiceReadyRunningMessage)) + return ctrlResult, nil + } + // create service - end + + // if TLS is enabled + if instance.Spec.TLS.API.Enabled(endpointType) { + // set endpoint protocol to https + data.Protocol = ptr.To(service.ProtocolHTTPS) + } + + apiEndpointsV3[string(endpointType)], err = svc.GetAPIEndpoint( + svcOverride.EndpointURL, data.Protocol, data.Path) + if err != nil { + instance.Status.Conditions.MarkFalse( + condition.CreateServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.CreateServiceReadyErrorMessage, + err.Error()) + return ctrl.Result{}, err + } + } + instance.Status.Conditions.MarkTrue(condition.CreateServiceReadyCondition, condition.CreateServiceReadyMessage) + + // + // Update instance status with service endpoint url from route host information + // + if instance.Status.APIEndpoints == nil { + instance.Status.APIEndpoints = map[string]map[string]string{} + } + instance.Status.APIEndpoints[cloudkitty.ServiceName] = apiEndpointsV3 + // V2 - end + + // expose service - end + + // + // create service and user in keystone - - https://docs.openstack.org/CloudKitty/latest/install/install-rdo.html#configure-user-and-endpoints + // TODO: rework this + // + if instance.Status.ServiceIDs == nil { + instance.Status.ServiceIDs = map[string]string{} + } + + for _, ksSvc := range keystoneServices { + ksSvcSpec := keystonev1.KeystoneServiceSpec{ + ServiceType: ksSvc["type"], + ServiceName: ksSvc["name"], + ServiceDescription: ksSvc["desc"], + Enabled: true, + ServiceUser: instance.Spec.ServiceUser, + Secret: instance.Spec.Secret, + PasswordSelector: instance.Spec.PasswordSelectors.CloudKittyService, + } + + ksSvcObj := keystonev1.NewKeystoneService(ksSvcSpec, instance.Namespace, serviceLabels, cloudkitty.NormalDuration) + ctrlResult, err := ksSvcObj.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.MarkFalse( + condition.KeystoneServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + "Creating KeyStoneService CR %s", + err.Error()) + return ctrlResult, err + } + + // mirror the Status, Reason, Severity and Message of the latest keystoneservice condition + // into a local condition with the type condition.KeystoneServiceReadyCondition + c := ksSvcObj.GetConditions().Mirror(condition.KeystoneServiceReadyCondition) + if c != nil { + instance.Status.Conditions.Set(c) + } + + if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + instance.Status.ServiceIDs[ksSvc["name"]] = ksSvcObj.GetServiceID() + + ksEndptSpec := keystonev1.KeystoneEndpointSpec{ + ServiceName: ksSvc["name"], + Endpoints: instance.Status.APIEndpoints[ksSvc["name"]], + } + + ksEndptObj := keystonev1.NewKeystoneEndpoint( + ksSvc["name"], + instance.Namespace, + ksEndptSpec, + serviceLabels, + cloudkitty.NormalDuration) + ctrlResult, err = ksEndptObj.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.MarkFalse( + condition.KeystoneEndpointReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + "Creating KeyStoneEndpoint CR %s", + err.Error()) + return ctrlResult, err + } + + // mirror the Status, Reason, Severity and Message of the latest keystoneendpoint condition + // into a local condition with the type condition.KeystoneEndpointReadyCondition + c = ksEndptObj.GetConditions().Mirror(condition.KeystoneEndpointReadyCondition) + if c != nil { + instance.Status.Conditions.Set(c) + } + + if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + } + + Log.Info(fmt.Sprintf("Reconciled Service '%s' init successfully", instance.Name)) + return ctrl.Result{}, nil +} + +func (r *CloudKittyAPIReconciler) reconcileNormal(ctx context.Context, instance *telemetryv1.CloudKittyAPI, helper *helper.Helper) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s'", instance.Name)) + + configVars := make(map[string]env.Setter) + + // + // check for required OpenStack secret holding passwords for service/admin user and add hash to the vars map + // + + ctrlResult, err := cloudkitty.VerifyServiceSecret( + ctx, + types.NamespacedName{Namespace: instance.Namespace, Name: instance.Spec.Secret}, + []string{ + instance.Spec.PasswordSelectors.CloudKittyService, + }, + helper.GetClient(), + &instance.Status.Conditions, + cloudkitty.NormalDuration, + &configVars, + ) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + // + // check for required Transport URL and config secrets + // + + parentCloudKittyName := cloudkitty.GetOwningCloudKittyName(instance) + secretNames := []string{ + instance.Spec.TransportURLSecret, // TransportURLSecret + fmt.Sprintf("%s-scripts", parentCloudKittyName), // ScriptsSecret + fmt.Sprintf("%s-config-data", parentCloudKittyName), // ConfigSecret + } + // Append CustomServiceConfigSecrets that should be checked + secretNames = append(secretNames, instance.Spec.CustomServiceConfigSecrets...) + + ctrlResult, err = cloudkitty.VerifyConfigSecrets( + ctx, + helper, + &instance.Status.Conditions, + secretNames, + instance.Namespace, + &configVars, + ) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + + // + // TLS input validation + // + // Validate the CA cert secret if provided + if instance.Spec.TLS.CaBundleSecretName != "" { + hash, err := tls.ValidateCACertSecret( + ctx, + helper.GetClient(), + types.NamespacedName{ + Name: instance.Spec.TLS.CaBundleSecretName, + Namespace: instance.Namespace, + }, + ) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + fmt.Sprintf(condition.TLSInputReadyWaitingMessage, instance.Spec.TLS.CaBundleSecretName))) + return ctrl.Result{}, nil + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if hash != "" { + configVars[tls.CABundleKey] = env.SetValue(hash) + } + } + + // Validate API service certs secrets + certsHash, err := instance.Spec.TLS.API.ValidateCertSecrets(ctx, helper, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + fmt.Sprintf(condition.TLSInputReadyWaitingMessage, err.Error()))) + return ctrl.Result{}, nil + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + configVars[tls.TLSHashName] = env.SetValue(certsHash) + + // all cert input checks out so report InputReady + instance.Status.Conditions.MarkTrue(condition.TLSInputReadyCondition, condition.InputReadyMessage) + + // + // Create secrets required as input for the Service and calculate an overall hash of hashes + // + serviceLabels := map[string]string{ + common.AppSelector: cloudkitty.ServiceName, + common.ComponentSelector: cloudkittyapi.ComponentName, + } + + // + // create custom config for this cloudkitty service + // + err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) + + // + // TODO check when/if Init, Update, or Upgrade should/could be skipped + // + + // networks to attach to + nadList := []networkv1.NetworkAttachmentDefinition{} + for _, netAtt := range instance.Spec.NetworkAttachments { + nad, err := nad.GetNADWithName(ctx, helper, netAtt, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + Log.Info(fmt.Sprintf("network-attachment-definition %s not found", netAtt)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.NetworkAttachmentsReadyWaitingMessage, + netAtt)) + return cloudkitty.ResultRequeue, fmt.Errorf("network-attachment-definition %s not found", netAtt) + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if nad != nil { + nadList = append(nadList, *nad) + } + } + + serviceAnnotations, err := nad.EnsureNetworksAnnotation(nadList) + if err != nil { + err = fmt.Errorf("failed create network annotation from %s: %w", instance.Spec.NetworkAttachments, err) + instance.Status.Conditions.MarkFalse( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error()) + return ctrl.Result{}, err + } + + // Handle service init + ctrlResult, err = r.reconcileInit(ctx, instance, helper, serviceLabels) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + // + // Handle Topology + // + topology, err := ensureTopology( + ctx, + helper, + instance, // topologyHandler + instance.Name, // finalizer + &instance.Status.Conditions, + labels.GetLabelSelector(serviceLabels), + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TopologyReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TopologyReadyErrorMessage, + err.Error())) + return ctrl.Result{}, fmt.Errorf("waiting for Topology requirements: %w", err) + } + + // + // normal reconcile tasks + // + + // + // create hash over all the different input resources to identify if any those changed + // and a restart/recreate is required. + // + inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configVars) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } else if hashChanged { + Log.Info(fmt.Sprintf("%s... requeueing", condition.ServiceConfigReadyInitMessage)) + instance.Status.Conditions.MarkFalse( + condition.ServiceConfigReadyCondition, + condition.InitReason, + condition.SeverityInfo, + condition.ServiceConfigReadyInitMessage) + // Hash changed and instance status should be updated (which will be done by main defer func), + // so we need to return and reconcile again + return ctrl.Result{}, nil + } + + // Deploy a statefulset + ssDef, err := cloudkittyapi.StatefulSet(instance, inputHash, serviceLabels, serviceAnnotations, topology) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + ss := statefulset.NewStatefulSet(ssDef, cloudkitty.ShortDuration) + + var ssData appsv1.StatefulSet + ctrlResult, err = ss.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrlResult, err + + } else if (ctrlResult == ctrl.Result{}) { + // Wait until the data in the StatefulSet is for the current generation + ssData = ss.GetStatefulSet() + if ssData.Generation != ssData.Status.ObservedGeneration { + ctrlResult = cloudkitty.ResultRequeue + err = fmt.Errorf("waiting for Statefulset %s to start reconciling", ssData.Name) + } + } + + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + // If the deployment is not ready, then neither are the NADs + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.NetworkAttachmentsReadyInitMessage)) + return ctrlResult, err + } + + instance.Status.ReadyCount = ssData.Status.ReadyReplicas + + // verify if network attachment matches expectations + networkReady := false + networkAttachmentStatus := map[string][]string{} + if *instance.Spec.Replicas > 0 { + networkReady, networkAttachmentStatus, err = nad.VerifyNetworkStatusFromAnnotation( + ctx, + helper, + instance.Spec.NetworkAttachments, + serviceLabels, + instance.Status.ReadyCount, + ) + if err != nil { + err = fmt.Errorf("verifying API NetworkAttachments (%s) %w", instance.Spec.NetworkAttachments, err) + instance.Status.Conditions.MarkFalse( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error()) + return ctrl.Result{}, err + } + } else { + networkReady = true + } + + instance.Status.NetworkAttachments = networkAttachmentStatus + if networkReady { + instance.Status.Conditions.MarkTrue(condition.NetworkAttachmentsReadyCondition, condition.NetworkAttachmentsReadyMessage) + } else { + err := fmt.Errorf("not all pods have interfaces with ips as configured in NetworkAttachments: %s", instance.Spec.NetworkAttachments) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + + return ctrl.Result{}, err + } + + if instance.Status.ReadyCount > 0 { + instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) + + } else if *instance.Spec.Replicas > 0 { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + + } else { + instance.Status.Conditions.MarkFalse( + condition.DeploymentReadyCondition, + condition.NotRequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyInitMessage) + } + // create StatefulSet - end + + Log.Info(fmt.Sprintf("Reconciled Service '%s' successfully", instance.Name)) + // update the overall status condition if service is ready + if instance.IsReady() { + instance.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) + } + // For non ready we'll let the main defer func handle the status update using the Mirror function + return ctrl.Result{}, nil +} + +// generateServiceConfigs - create Secret which holds the service configuration +func (r *CloudKittyAPIReconciler) generateServiceConfigs( + ctx context.Context, + h *helper.Helper, + instance *telemetryv1.CloudKittyAPI, + envVars *map[string]env.Setter, + serviceLabels map[string]string, +) error { + // + // create custom Secret for cloudkitty service-specific config input + // - %-config-data holds custom config for the service + // + + labels := labels.GetLabels(instance, labels.GetGroupLabel(cloudkitty.ServiceName), serviceLabels) + + // customData hold any customization for the service. + customData := map[string]string{cloudkitty.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig} + + // Fetch the two service config snippets (DefaultsConfigFileName and + // CustomConfigFileName) from the Secret generated by the top level + // cloudkitty controller, and add them to this service specific Secret. + cloudkittySecretName := cloudkitty.GetOwningCloudKittyName(instance) + "-config-data" + cloudkittySecret, _, err := secret.GetSecret(ctx, h, cloudkittySecretName, instance.Namespace) + if err != nil { + return err + } + customData[cloudkitty.DefaultsConfigFileName] = string(cloudkittySecret.Data[cloudkitty.DefaultsConfigFileName]) + customData[cloudkitty.CustomConfigFileName] = string(cloudkittySecret.Data[cloudkitty.CustomConfigFileName]) + + customSecrets := "" + for _, secretName := range instance.Spec.CustomServiceConfigSecrets { + secret, _, err := secret.GetSecret(ctx, h, secretName, instance.Namespace) + if err != nil { + return err + } + for _, data := range secret.Data { + customSecrets += string(data) + "\n" + } + } + customData[cloudkitty.CustomServiceConfigSecretsFileName] = customSecrets + + templateParameters := map[string]interface{}{ + "LogFile": cloudkittyapi.LogFile, + } + + configTemplates := []util.Template{ + { + Name: fmt.Sprintf("%s-config-data", instance.Name), + Namespace: instance.Namespace, + Type: util.TemplateTypeConfig, + InstanceType: instance.Kind, + CustomData: customData, + ConfigOptions: templateParameters, + Labels: labels, + }, + } + + return secret.EnsureSecrets(ctx, h, instance, configTemplates, envVars) +} + +// createHashOfInputHashes - creates a hash of hashes which gets added to the resources which requires a restart +// if any of the input resources change, like configs, passwords, ... +// +// returns the hash, whether the hash changed (as a bool) and any error +func (r *CloudKittyAPIReconciler) createHashOfInputHashes( + ctx context.Context, + instance *telemetryv1.CloudKittyAPI, + envVars map[string]env.Setter, +) (string, bool, error) { + Log := r.GetLogger(ctx) + + var hashMap map[string]string + changed := false + mergedMapVars := env.MergeEnvs([]corev1.EnvVar{}, envVars) + hash, err := util.ObjectHash(mergedMapVars) + if err != nil { + return hash, changed, err + } + if hashMap, changed = util.SetHash(instance.Status.Hash, common.InputHashName, hash); changed { + instance.Status.Hash = hashMap + Log.Info(fmt.Sprintf("Input maps hash %s - %s", common.InputHashName, hash)) + } + return hash, changed, nil +} diff --git a/controllers/cloudkittyproc_controller.go b/controllers/cloudkittyproc_controller.go new file mode 100644 index 00000000..fbccef63 --- /dev/null +++ b/controllers/cloudkittyproc_controller.go @@ -0,0 +1,759 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkittyproc" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/go-logr/logr" + networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" + "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" +) + +// GetClient - +func (r *CloudKittyProcReconciler) GetClient() client.Client { + return r.Client +} + +// GetKClient - +func (r *CloudKittyProcReconciler) GetKClient() kubernetes.Interface { + return r.Kclient +} + +// GetScheme - +func (r *CloudKittyProcReconciler) GetScheme() *runtime.Scheme { + return r.Scheme +} + +// CloudKittyProcReconciler reconciles a CloudKittyProc object +type CloudKittyProcReconciler struct { + client.Client + Kclient kubernetes.Interface + Scheme *runtime.Scheme +} + +// GetLogger returns a logger object with a logging prefix of "controller.name" and additional controller context fields +func (r *CloudKittyProcReconciler) GetLogger(ctx context.Context) logr.Logger { + return log.FromContext(ctx).WithName("Controllers").WithName("CloudKittyProc") +} + +//+kubebuilder:rbac:groups=cloudkitty.openstack.org,resources=cloudkittyprocs,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=cloudkitty.openstack.org,resources=cloudkittyprocs/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=cloudkitty.openstack.org,resources=cloudkittyprocs/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list; +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch +// +kubebuilder:rbac:groups=topology.openstack.org,resources=topologies,verbs=get;list;watch;update + +// Reconcile - +func (r *CloudKittyProcReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { + Log := r.GetLogger(ctx) + + // Fetch the CloudKittyProc instance + instance := &telemetryv1.CloudKittyProc{} + err := r.Client.Get(ctx, req.NamespacedName, instance) + if err != nil { + if k8s_errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. + // For additional cleanup logic use finalizers. Return and don't requeue. + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + return ctrl.Result{}, err + } + + helper, err := helper.NewHelper( + instance, + r.Client, + r.Kclient, + r.Scheme, + Log, + ) + if err != nil { + return ctrl.Result{}, err + } + + // + // initialize status + // + isNewInstance := instance.Status.Conditions == nil + if isNewInstance { + instance.Status.Conditions = condition.Conditions{} + } + + // Save a copy of the condtions so that we can restore the LastTransitionTime + // when a condition's state doesn't change. + savedConditions := instance.Status.Conditions.DeepCopy() + + // Always patch the instance status when exiting this function so we can persist any changes. + defer func() { + // Don't update the status, if reconciler Panics + if r := recover(); r != nil { + Log.Info(fmt.Sprintf("panic during reconcile %v\n", r)) + panic(r) + } + condition.RestoreLastTransitionTimes(&instance.Status.Conditions, savedConditions) + if instance.Status.Conditions.IsUnknown(condition.ReadyCondition) { + instance.Status.Conditions.Set( + instance.Status.Conditions.Mirror(condition.ReadyCondition)) + } + err := helper.PatchInstance(ctx, instance) + if err != nil { + _err = err + return + } + }() + + // Always initialize conditions used later as Status=Unknown + cl := condition.CreateList( + condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), + condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), + condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), + condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), + condition.UnknownCondition(condition.TLSInputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + ) + instance.Status.Conditions.Init(&cl) + // Always mark the Generation as observed early on + instance.Status.ObservedGeneration = instance.Generation + + // If we're not deleting this and the service object doesn't have our finalizer, add it. + if (instance.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(instance, helper.GetFinalizer())) || isNewInstance { + // Register overall status immediately to have an early feedback e.g. in the cli + return ctrl.Result{}, nil + } + + if instance.Status.Hash == nil { + instance.Status.Hash = map[string]string{} + } + if instance.Status.NetworkAttachments == nil { + instance.Status.NetworkAttachments = map[string][]string{} + } + + // Handle service delete + if !instance.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, instance, helper) + } + + // Init Topology condition if there's a reference + if instance.Spec.TopologyRef != nil { + c := condition.UnknownCondition(condition.TopologyReadyCondition, condition.InitReason, condition.TopologyReadyInitMessage) + cl.Set(c) + } + + // Handle non-deleted clusters + return r.reconcileNormal(ctx, instance, helper) +} + +// SetupWithManager sets up the controller with the Manager. +func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { + Log := r.GetLogger(ctx) + + // Watch for changes to secrets we don't own. Global secrets + // (e.g. TransportURLSecret) are handled by the main cloudkitty controller. + secretFn := func(_ context.Context, o client.Object) []reconcile.Request { + var namespace string = o.GetNamespace() + var secretName string = o.GetName() + result := []reconcile.Request{} + + // get all scheduler CRs + schedulers := &telemetryv1.CloudKittyProcList{} + listOpts := []client.ListOption{ + client.InNamespace(namespace), + } + if err := r.Client.List(context.Background(), schedulers, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve scheduler CRs %v") + return nil + } + + // Watch for changes to secrets where the owner label AND the + // CR.Spec.ManagingCrName label matches + label := o.GetLabels() + if l, ok := label[labels.GetOwnerNameLabelSelector(labels.GetGroupLabel(cloudkitty.ServiceName))]; ok { + for _, cr := range schedulers.Items { + // return reconcile event for the CR where the owner label AND the parentCloudKittyName matches + if l == cloudkitty.GetOwningCloudKittyName(&cr) { + // return namespace and Name of CR + name := client.ObjectKey{ + Namespace: o.GetNamespace(), + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Secret %s and CR %s marked with label: %s", o.GetName(), cr.Name, l)) + + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + } + + // Watch for changes to any CustomServiceConfigSecrets + for _, cr := range schedulers.Items { + for _, v := range cr.Spec.CustomServiceConfigSecrets { + if v == secretName { + name := client.ObjectKey{ + Namespace: namespace, + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Secret %s is used by CloudKitty CR %s", secretName, cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + } + if len(result) > 0 { + return result + } + return nil + } + + // index passwordSecretField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyProc{}, cloudKittyPasswordSecretField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyProc) + if cr.Spec.Secret == "" { + return nil + } + return []string{cr.Spec.Secret} + }); err != nil { + return err + } + + // index caBundleSecretNameField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyProc{}, cloudKittyCaBundleSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyProc) + if cr.Spec.TLS.CaBundleSecretName == "" { + return nil + } + return []string{cr.Spec.TLS.CaBundleSecretName} + }); err != nil { + return err + } + + // index topologyField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyProc{}, cloudKittyTopologyField, func(rawObj client.Object) []string { + // Extract the topology name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyProc) + if cr.Spec.TopologyRef == nil { + return nil + } + return []string{cr.Spec.TopologyRef.Name} + }); err != nil { + return err + } + + return ctrl.NewControllerManagedBy(mgr). + For(&telemetryv1.CloudKittyProc{}). + Owns(&appsv1.StatefulSet{}). + // watch the secrets we don't own + Watches(&corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(secretFn)). + Watches( + &corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). + Watches(&topologyv1.Topology{}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.GenerationChangedPredicate{})). + Complete(r) +} + +func (r *CloudKittyProcReconciler) findObjectsForSrc(ctx context.Context, src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + l := log.FromContext(ctx).WithName("Controllers").WithName("CloudKittyProc") + + for _, field := range commonWatchFields { + crList := &telemetryv1.CloudKittyProcList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.List(ctx, crList, listOps) + if err != nil { + l.Error(err, fmt.Sprintf("listing %s for field: %s - %s", crList.GroupVersionKind().Kind, field, src.GetNamespace())) + return requests + } + + for _, item := range crList.Items { + l.Info(fmt.Sprintf("input source %s changed, reconcile: %s - %s", src.GetName(), item.GetName(), item.GetNamespace())) + + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} + +func (r *CloudKittyProcReconciler) reconcileDelete(ctx context.Context, instance *telemetryv1.CloudKittyProc, helper *helper.Helper) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s' delete", instance.Name)) + + // Service is deleted so remove the finalizer. + controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) + Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", instance.Name)) + + // Remove finalizer on the Topology CR + if ctrlResult, err := topologyv1.EnsureDeletedTopologyRef( + ctx, + helper, + instance.Status.LastAppliedTopology, + instance.Name, + ); err != nil { + return ctrlResult, err + } + return ctrl.Result{}, nil +} + +func (r *CloudKittyProcReconciler) reconcileNormal(ctx context.Context, instance *telemetryv1.CloudKittyProc, helper *helper.Helper) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s'", instance.Name)) + + configVars := make(map[string]env.Setter) + + // + // check for required OpenStack secret holding passwords for service/admin user and add hash to the vars map + // + + ctrlResult, err := cloudkitty.VerifyServiceSecret( + ctx, + types.NamespacedName{Namespace: instance.Namespace, Name: instance.Spec.Secret}, + []string{ + instance.Spec.PasswordSelectors.CloudKittyService, + }, + helper.GetClient(), + &instance.Status.Conditions, + cloudkitty.NormalDuration, + &configVars, + ) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + // + // check for required Transport URL and config secrets + // + + parentCloudKittyName := cloudkitty.GetOwningCloudKittyName(instance) + secretNames := []string{ + instance.Spec.TransportURLSecret, // TransportURLSecret + fmt.Sprintf("%s-scripts", parentCloudKittyName), // ScriptsSecret + fmt.Sprintf("%s-config-data", parentCloudKittyName), // ConfigSecret + } + // Append CustomServiceConfigSecrets that should be checked + secretNames = append(secretNames, instance.Spec.CustomServiceConfigSecrets...) + + ctrlResult, err = cloudkitty.VerifyConfigSecrets( + ctx, + helper, + &instance.Status.Conditions, + secretNames, + instance.Namespace, + &configVars, + ) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + + // + // TLS input validation + // + // Validate the CA cert secret if provided + if instance.Spec.TLS.CaBundleSecretName != "" { + hash, err := tls.ValidateCACertSecret( + ctx, + helper.GetClient(), + types.NamespacedName{ + Name: instance.Spec.TLS.CaBundleSecretName, + Namespace: instance.Namespace, + }, + ) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + fmt.Sprintf(condition.TLSInputReadyWaitingMessage, instance.Spec.TLS.CaBundleSecretName))) + return ctrl.Result{}, nil + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if hash != "" { + configVars[tls.CABundleKey] = env.SetValue(hash) + } + } + // all cert input checks out so report InputReady + instance.Status.Conditions.MarkTrue(condition.TLSInputReadyCondition, condition.InputReadyMessage) + + // + // Create ConfigMaps required as input for the Service and calculate an overall hash of hashes + // + serviceLabels := map[string]string{ + common.AppSelector: cloudkitty.ServiceName, + common.ComponentSelector: cloudkittyproc.ComponentName, + } + + // + // create custom config for this cloudkitty service + // + err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + // + // create hash over all the different input resources to identify if any those changed + // and a restart/recreate is required. + // + inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configVars) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } else if hashChanged { + Log.Info(fmt.Sprintf("%s... requeueing", condition.ServiceConfigReadyInitMessage)) + instance.Status.Conditions.MarkFalse( + condition.ServiceConfigReadyCondition, + condition.InitReason, + condition.SeverityInfo, + condition.ServiceConfigReadyInitMessage) + // Hash changed and instance status should be updated (which will be done by main defer func), + // so we need to return and reconcile again + return ctrl.Result{}, nil + } + instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) + + // + // TODO check when/if Init, Update, or Upgrade should/could be skipped + // + + // networks to attach to + nadList := []networkv1.NetworkAttachmentDefinition{} + for _, netAtt := range instance.Spec.NetworkAttachments { + nad, err := nad.GetNADWithName(ctx, helper, netAtt, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + Log.Info(fmt.Sprintf("network-attachment-definition %s not found", netAtt)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.NetworkAttachmentsReadyWaitingMessage, + netAtt)) + return cloudkitty.ResultRequeue, fmt.Errorf("network-attachment-definition %s not found", netAtt) + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if nad != nil { + nadList = append(nadList, *nad) + } + } + + serviceAnnotations, err := nad.EnsureNetworksAnnotation(nadList) + if err != nil { + err = fmt.Errorf("failed create network annotation from %s: %w", instance.Spec.NetworkAttachments, err) + instance.Status.Conditions.MarkFalse( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error()) + return ctrl.Result{}, err + } + + // + // normal reconcile tasks + // + + // + // Handle Topology + // + topology, err := ensureTopology( + ctx, + helper, + instance, // topologyHandler + instance.Name, // finalizer + &instance.Status.Conditions, + labels.GetLabelSelector(serviceLabels), + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TopologyReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TopologyReadyErrorMessage, + err.Error())) + return ctrl.Result{}, fmt.Errorf("waiting for Topology requirements: %w", err) + } + + // Deploy a statefulset + ssDef := cloudkittyproc.StatefulSet(instance, inputHash, serviceLabels, serviceAnnotations, topology) + ss := statefulset.NewStatefulSet(ssDef, cloudkitty.ShortDuration) + + var ssData appsv1.StatefulSet + ctrlResult, err = ss.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrlResult, err + + } else if (ctrlResult == ctrl.Result{}) { + // Wait until the data in the StatefulSet is for the current generation + ssData = ss.GetStatefulSet() + if ssData.Generation != ssData.Status.ObservedGeneration { + ctrlResult = cloudkitty.ResultRequeue + err = fmt.Errorf("waiting for Statefulset %s to start reconciling", ssData.Name) + } + } + + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + // If the deployment is not ready, then neither are the NADs + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.NetworkAttachmentsReadyInitMessage)) + return ctrlResult, err + } + + instance.Status.ReadyCount = ssData.Status.ReadyReplicas + + // verify if network attachment matches expectations + networkReady := false + networkAttachmentStatus := map[string][]string{} + if *instance.Spec.Replicas > 0 { + networkReady, networkAttachmentStatus, err = nad.VerifyNetworkStatusFromAnnotation( + ctx, + helper, + instance.Spec.NetworkAttachments, + serviceLabels, + instance.Status.ReadyCount, + ) + if err != nil { + err = fmt.Errorf("verifying API NetworkAttachments (%s) %w", instance.Spec.NetworkAttachments, err) + instance.Status.Conditions.MarkFalse( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error()) + return ctrl.Result{}, err + } + } else { + networkReady = true + } + + instance.Status.NetworkAttachments = networkAttachmentStatus + if networkReady { + instance.Status.Conditions.MarkTrue(condition.NetworkAttachmentsReadyCondition, condition.NetworkAttachmentsReadyMessage) + } else { + err := fmt.Errorf("not all pods have interfaces with ips as configured in NetworkAttachments: %s", instance.Spec.NetworkAttachments) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + + return ctrl.Result{}, err + } + + if instance.Status.ReadyCount > 0 { + instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) + } else if *instance.Spec.Replicas > 0 { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + + } else { + instance.Status.Conditions.MarkFalse( + condition.DeploymentReadyCondition, + condition.NotRequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyInitMessage) + } + // create StatefulSet - end + + Log.Info(fmt.Sprintf("Reconciled Service '%s' successfully", instance.Name)) + // update the overall status condition if service is ready + if instance.IsReady() { + instance.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) + } + // For non ready we'll let the main defer func handle the status update using the Mirror function + return ctrl.Result{}, nil +} + +// generateServiceConfigs - create Secret which holds the service configuration +func (r *CloudKittyProcReconciler) generateServiceConfigs( + ctx context.Context, + h *helper.Helper, + instance *telemetryv1.CloudKittyProc, + envVars *map[string]env.Setter, + serviceLabels map[string]string, +) error { + // + // create custom Secret for cloudkitty service-specific config input + // - %-config-data holds custom config for the service + // + + labels := labels.GetLabels(instance, labels.GetGroupLabel(cloudkitty.ServiceName), serviceLabels) + + // customData hold any customization for the service. + customData := map[string]string{cloudkitty.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig} + + // Fetch the two service config snippets (DefaultsConfigFileName and + // CustomConfigFileName) from the Secret generated by the top level + // cloudkitty controller, and add them to this service specific Secret. + cloudkittySecretName := cloudkitty.GetOwningCloudKittyName(instance) + "-config-data" + cloudkittySecret, _, err := secret.GetSecret(ctx, h, cloudkittySecretName, instance.Namespace) + if err != nil { + return err + } + customData[cloudkitty.DefaultsConfigFileName] = string(cloudkittySecret.Data[cloudkitty.DefaultsConfigFileName]) + customData[cloudkitty.CustomConfigFileName] = string(cloudkittySecret.Data[cloudkitty.CustomConfigFileName]) + + customSecrets := "" + for _, secretName := range instance.Spec.CustomServiceConfigSecrets { + secret, _, err := secret.GetSecret(ctx, h, secretName, instance.Namespace) + if err != nil { + return err + } + for _, data := range secret.Data { + customSecrets += string(data) + "\n" + } + } + customData[cloudkitty.CustomServiceConfigSecretsFileName] = customSecrets + + configTemplates := []util.Template{ + { + Name: fmt.Sprintf("%s-config-data", instance.Name), + Namespace: instance.Namespace, + Type: util.TemplateTypeConfig, + InstanceType: instance.Kind, + CustomData: customData, + Labels: labels, + }, + } + + return secret.EnsureSecrets(ctx, h, instance, configTemplates, envVars) +} + +// createHashOfInputHashes - creates a hash of hashes which gets added to the resources which requires a restart +// if any of the input resources change, like configs, passwords, ... +// +// returns the hash, whether the hash changed (as a bool) and any error +func (r *CloudKittyProcReconciler) createHashOfInputHashes( + ctx context.Context, + instance *telemetryv1.CloudKittyProc, + envVars map[string]env.Setter, +) (string, bool, error) { + Log := r.GetLogger(ctx) + var hashMap map[string]string + changed := false + mergedMapVars := env.MergeEnvs([]corev1.EnvVar{}, envVars) + hash, err := util.ObjectHash(mergedMapVars) + if err != nil { + return hash, changed, err + } + if hashMap, changed = util.SetHash(instance.Status.Hash, common.InputHashName, hash); changed { + instance.Status.Hash = hashMap + Log.Info(fmt.Sprintf("Input maps hash %s - %s", common.InputHashName, hash)) + } + return hash, changed, nil +} diff --git a/controllers/telemetry_controller.go b/controllers/telemetry_controller.go index 470d98f1..ed784736 100644 --- a/controllers/telemetry_controller.go +++ b/controllers/telemetry_controller.go @@ -66,6 +66,9 @@ func (r *TelemetryReconciler) GetLogger(ctx context.Context) logr.Logger { // +kubebuilder:rbac:groups=telemetry.openstack.org,resources=loggings,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=telemetry.openstack.org,resources=loggings/status,verbs=get;update;patch // +kubebuilder:rbac:groups=telemetry.openstack.org,resources=loggings/finalizers,verbs=update;delete;patch +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkitties,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkitties/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkitties/finalizers,verbs=update;delete;patch // +kubebuilder:rbac:groups=rabbitmq.openstack.org,resources=transporturls,verbs=get;list;watch;create;update;patch;delete // Reconcile reconciles a Telemetry @@ -136,6 +139,7 @@ func (r *TelemetryReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( condition.UnknownCondition(telemetryv1.AutoscalingReadyCondition, condition.InitReason, telemetryv1.AutoscalingReadyInitMessage), condition.UnknownCondition(telemetryv1.MetricStorageReadyCondition, condition.InitReason, telemetryv1.MetricStorageReadyInitMessage), condition.UnknownCondition(telemetryv1.LoggingReadyCondition, condition.InitReason, telemetryv1.LoggingReadyInitMessage), + condition.UnknownCondition(telemetryv1.CloudKittyReadyCondition, condition.InitReason, telemetryv1.CloudKittyReadyInitMessage), ) instance.Status.Conditions.Init(&cl) @@ -220,6 +224,13 @@ func (r *TelemetryReconciler) reconcileNormal(ctx context.Context, instance *tel return ctrlResult, nil } + ctrlResult, err = r.reconcileCloudKitty(ctx, instance, helper) + if err != nil { + return ctrl.Result{}, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + // We reached the end of the Reconcile, update the Ready condition based on // the sub conditions if instance.Status.Conditions.AllSubConditionIsTrue() { @@ -548,6 +559,89 @@ func (r TelemetryReconciler) reconcileLogging(ctx context.Context, instance *tel return ctrl.Result{}, nil } +// reconcileAutoscaling ... +func (r TelemetryReconciler) reconcileCloudKitty(ctx context.Context, instance *telemetryv1.Telemetry, helper *helper.Helper) (ctrl.Result, error) { + const ( + cloudKittyNamespaceLabel = "CloudKitty.Namespace" + cloudKittyNameLabel = "CloudKitty.Name" + cloudKittyName = "cloudkitty" + ) + cloudKittyInstance := &telemetryv1.CloudKitty{ + ObjectMeta: metav1.ObjectMeta{ + Name: cloudKittyName, + Namespace: instance.Namespace, + }, + } + + if instance.Spec.CloudKitty.Enabled == nil || !*instance.Spec.CloudKitty.Enabled { + if res, err := utils.EnsureDeleted(ctx, helper, cloudKittyInstance); err != nil { + return res, err + } + instance.Status.Conditions.Remove(telemetryv1.CloudKittyReadyCondition) + return ctrl.Result{}, nil + } + + if instance.Spec.CloudKitty.NodeSelector == nil { + instance.Spec.CloudKitty.NodeSelector = instance.Spec.NodeSelector + } + + if instance.Spec.CloudKitty.TopologyRef == nil { + instance.Spec.CloudKitty.TopologyRef = instance.Spec.TopologyRef + } + + helper.GetLogger().Info("Reconciling CloudKitty", cloudKittyNamespaceLabel, instance.Namespace, cloudKittyNameLabel, cloudKittyName) + op, err := controllerutil.CreateOrPatch(ctx, helper.GetClient(), cloudKittyInstance, func() error { + instance.Spec.CloudKitty.CloudKittySpec.DeepCopyInto(&cloudKittyInstance.Spec) + + err := controllerutil.SetControllerReference(helper.GetBeforeObject(), cloudKittyInstance, helper.GetScheme()) + if err != nil { + return err + } + return nil + }) + + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + telemetryv1.CloudKittyReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + // Check the observed Generation and mirror the condition from the + // underlying resource reconciliation + autoObsGen, err := r.checkCloudKittyGeneration(instance) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + telemetryv1.CloudKittyReadyErrorMessage, + err.Error())) + return ctrl.Result{}, nil + } + if !autoObsGen { + instance.Status.Conditions.Set(condition.UnknownCondition( + telemetryv1.CloudKittyReadyCondition, + condition.InitReason, + telemetryv1.AutoscalingReadyRunningMessage, + )) + } else { + // Mirror Autoscaling condition status + c := cloudKittyInstance.Status.Conditions.Mirror(telemetryv1.CloudKittyReadyCondition) + if c != nil { + instance.Status.Conditions.Set(c) + } + } + if op != controllerutil.OperationResultNone && autoObsGen { + helper.GetLogger().Info(fmt.Sprintf("%s %s - %s", cloudKittyName, cloudKittyInstance.Name, op)) + } + + return ctrl.Result{}, nil +} + // SetupWithManager sets up the controller with the Manager. func (r *TelemetryReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). @@ -556,6 +650,7 @@ func (r *TelemetryReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&telemetryv1.Autoscaling{}). Owns(&telemetryv1.MetricStorage{}). Owns(&telemetryv1.Logging{}). + Owns(&telemetryv1.CloudKitty{}). Complete(r) } @@ -642,3 +737,24 @@ func (r *TelemetryReconciler) checkLoggingGeneration( } return true, nil } + +// checkCloudKittyGeneration - +func (r *TelemetryReconciler) checkCloudKittyGeneration( + instance *telemetryv1.Telemetry, +) (bool, error) { + Log := r.GetLogger(context.Background()) + clm := &telemetryv1.CloudKittyList{} + listOpts := []client.ListOption{ + client.InNamespace(instance.Namespace), + } + if err := r.Client.List(context.Background(), clm, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve CloudKitty CR %w") + return false, err + } + for _, item := range clm.Items { + if item.Generation != item.Status.ObservedGeneration { + return false, nil + } + } + return true, nil +} diff --git a/main.go b/main.go index 1c98beba..c5d48662 100644 --- a/main.go +++ b/main.go @@ -195,10 +195,38 @@ func main() { os.Exit(1) } + if err = (&controllers.CloudKittyReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create CloudKitty controller") + os.Exit(1) + } + + if err = (&controllers.CloudKittyAPIReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + }).SetupWithManager(context.Background(), mgr); err != nil { + setupLog.Error(err, "unable to create CloudKitty API controller") + os.Exit(1) + } + + if err = (&controllers.CloudKittyProcReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + }).SetupWithManager(context.Background(), mgr); err != nil { + setupLog.Error(err, "unable to create CloudKitty Processor controller") + os.Exit(1) + } + // Acquire environmental defaults and initialize defaults with them telemetryv1beta1.SetupDefaultsTelemetry() telemetryv1beta1.SetupDefaultsCeilometer() telemetryv1beta1.SetupDefaultsAutoscaling() + telemetryv1beta1.SetupDefaultsCloudKitty() // Setup webhooks if requested checker := healthz.Ping @@ -220,6 +248,10 @@ func main() { setupLog.Error(err, "unable to create webhook", "webhook", "MetricStorage") os.Exit(1) } + if err = (&telemetryv1beta1.CloudKitty{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "CloudKitty") + os.Exit(1) + } checker = mgr.GetWebhookServer().StartedChecker() } diff --git a/pkg/cloudkitty/common.go b/pkg/cloudkitty/common.go new file mode 100644 index 00000000..9c78b72b --- /dev/null +++ b/pkg/cloudkitty/common.go @@ -0,0 +1,161 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkitty + +import ( + "context" + "fmt" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "k8s.io/apimachinery/pkg/types" + "time" + + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +type conditionUpdater interface { + Set(c *condition.Condition) + MarkTrue(t condition.Type, messageFormat string, messageArgs ...interface{}) +} + +type topologyHandler interface { + GetSpecTopologyRef() *topologyv1.TopoRef + GetLastAppliedTopology() *topologyv1.TopoRef + SetLastAppliedTopology(t *topologyv1.TopoRef) +} + +// EnsureTopology - +func EnsureTopology( + ctx context.Context, + helper *helper.Helper, + instance topologyHandler, + finalizer string, + conditionUpdater conditionUpdater, + defaultLabelSelector metav1.LabelSelector, +) (*topologyv1.Topology, error) { + + topology, err := topologyv1.EnsureServiceTopology( + ctx, + helper, + instance.GetSpecTopologyRef(), + instance.GetLastAppliedTopology(), + finalizer, + defaultLabelSelector, + ) + if err != nil { + conditionUpdater.Set(condition.FalseCondition( + condition.TopologyReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TopologyReadyErrorMessage, + err.Error())) + return nil, fmt.Errorf("waiting for Topology requirements: %w", err) + } + // update the Status with the last retrieved Topology (or set it to nil) + instance.SetLastAppliedTopology(instance.GetSpecTopologyRef()) + // update the Topology condition only when a Topology is referenced and has + // been retrieved (err == nil) + if tr := instance.GetSpecTopologyRef(); tr != nil { + // update the TopologyRef associated condition + conditionUpdater.MarkTrue( + condition.TopologyReadyCondition, + condition.TopologyReadyMessage, + ) + } + return topology, nil +} + +// VerifyServiceSecret - ensures that the Secret object exists and the expected +// fields are in the Secret. It also sets a hash of the values of the expected +// fields passed as input. +func VerifyServiceSecret( + ctx context.Context, + secretName types.NamespacedName, + expectedFields []string, + reader client.Reader, + conditionUpdater conditionUpdater, + requeueTimeout time.Duration, + envVars *map[string]env.Setter, +) (ctrl.Result, error) { + + hash, res, err := secret.VerifySecret(ctx, secretName, expectedFields, reader, requeueTimeout) + if err != nil { + conditionUpdater.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyErrorMessage, + err.Error())) + return res, err + } else if (res != ctrl.Result{}) { + log.FromContext(ctx).Info(fmt.Sprintf("OpenStack secret %s not found", secretName)) + conditionUpdater.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.InputReadyWaitingMessage)) + return res, nil + } + (*envVars)[secretName.Name] = env.SetValue(hash) + return ctrl.Result{}, nil +} + +// VerifyConfigSecrets - It iterates over the secretNames passed as input and +// sets the hash of values in the envVars map. +func VerifyConfigSecrets( + ctx context.Context, + h *helper.Helper, + conditionUpdater conditionUpdater, + secretNames []string, + namespace string, + envVars *map[string]env.Setter, +) (ctrl.Result, error) { + var hash string + var err error + for _, secretName := range secretNames { + _, hash, err = secret.GetSecret(ctx, h, secretName, namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + log.FromContext(ctx).Info(fmt.Sprintf("Secret %s not found", secretName)) + conditionUpdater.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.InputReadyWaitingMessage)) + return ResultRequeue, nil + } + conditionUpdater.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + // Add a prefix to the var name to avoid accidental collision with other non-secret + // vars. The secret names themselves will be unique. + (*envVars)["secret-"+secretName] = env.SetValue(hash) + } + + return ctrl.Result{}, nil +} diff --git a/pkg/cloudkitty/const.go b/pkg/cloudkitty/const.go new file mode 100644 index 00000000..5fb5269a --- /dev/null +++ b/pkg/cloudkitty/const.go @@ -0,0 +1,54 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkitty + +import ( + "time" + + ctrl "sigs.k8s.io/controller-runtime" +) + +const ( + // ServiceName - + ServiceName = "cloudkitty" + // ServiceType - + ServiceType = "rating" + // DatabaseName - + DatabaseName = "cloudkitty" + + // DefaultsConfigFileName - + DefaultsConfigFileName = "cloudkitty.conf" + // ServiceConfigFileName - + ServiceConfigFileName = "01-service-defaults.conf" + // CustomConfigFileName - + CustomConfigFileName = "02-global-custom.conf" + // CustomServiceConfigFileName - + CustomServiceConfigFileName = "03-service-custom.conf" + // CustomServiceConfigSecretsFileName - + CustomServiceConfigSecretsFileName = "04-service-custom-secrets.conf" + // MyCnfFileName - + MyCnfFileName = "my.cnf" + + // CloudKittyPublicPort - + CloudKittyPublicPort int32 = 8889 + // CloudKittyInternalPort - + CloudKittyInternalPort int32 = 8889 + + ShortDuration = time.Duration(5) * time.Second + NormalDuration = time.Duration(10) * time.Second +) + +var ResultRequeue = ctrl.Result{RequeueAfter: NormalDuration} diff --git a/pkg/cloudkitty/dbsync.go b/pkg/cloudkitty/dbsync.go new file mode 100644 index 00000000..75a675e4 --- /dev/null +++ b/pkg/cloudkitty/dbsync.go @@ -0,0 +1,107 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cloudkitty + +import ( + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // DBSyncCommand - + // TODO: Once we work on update/upgrades revisit the command in the + // the cloudkitty-dbsync-config.json file. + // If we stop all services during the update/upgrade then we can keep + // the --bump-versions flag. + // If we are doing rolling upgrades we'll need to use the flag + // conditionally (only for adoption) and do the restart cycle of + // services as described in the upstream rolling upgrades process. + dbSyncCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start" +) + +// DbSyncJob func +func DbSyncJob(instance *telemetryv1.CloudKitty, labels map[string]string) *batchv1.Job { + args := []string{"-c"} + args = append(args, dbSyncCommand) + + // create Volume and VolumeMounts + volumes := GetVolumes("cloudkitty") + volumeMounts := GetVolumeMounts("cloudkitty-dbsync") + // add CA cert if defined + if instance.Spec.CloudKittyAPI.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.CloudKittyAPI.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.CloudKittyAPI.TLS.CreateVolumeMounts(nil)...) + } + + runAsUser := int64(0) + envVars := map[string]env.Setter{} + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["KOLLA_BOOTSTRAP"] = env.SetValue("TRUE") + cloudKittyPassword := []corev1.EnvVar{ + { + Name: "AodhPassword", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: instance.Spec.Secret, + }, + Key: instance.Spec.PasswordSelectors.CloudKittyService, + }, + }, + }, + } + + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: ServiceName + "-db-sync", + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + ServiceAccountName: instance.RbacResourceName(), + Containers: []corev1.Container{ + { + Name: ServiceName + "-db-sync", + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: instance.Spec.CloudKittyAPI.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &runAsUser, + }, + Env: env.MergeEnvs(cloudKittyPassword, envVars), + VolumeMounts: volumeMounts, + }, + }, + Volumes: volumes, + }, + }, + }, + } + + if instance.Spec.NodeSelector != nil { + job.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector + } + + return job +} diff --git a/pkg/cloudkitty/funcs.go b/pkg/cloudkitty/funcs.go new file mode 100644 index 00000000..43210dad --- /dev/null +++ b/pkg/cloudkitty/funcs.go @@ -0,0 +1,49 @@ +package cloudkitty + +import ( + common "github.com/openstack-k8s-operators/lib-common/modules/common" + "github.com/openstack-k8s-operators/lib-common/modules/common/affinity" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// GetOwningCloudKittyName - Given a CloudKittyAPI or CloudKittyProc +// object, returning the parent CloudKitty object that created it (if any) +func GetOwningCloudKittyName(instance client.Object) string { + for _, ownerRef := range instance.GetOwnerReferences() { + if ownerRef.Kind == "CloudKitty" { + return ownerRef.Name + } + } + + return "" +} + +// GetNetworkAttachmentAddrs - Returns a list of IP addresses for all network attachments. +func GetNetworkAttachmentAddrs(namespace string, networkAttachments []string, networkAttachmentStatus map[string][]string) []string { + networkAttachmentAddrs := []string{} + + for _, network := range networkAttachments { + networkName := namespace + "/" + network + if networkAddrs, ok := networkAttachmentStatus[networkName]; ok { + networkAttachmentAddrs = append(networkAttachmentAddrs, networkAddrs...) + } + } + + return networkAttachmentAddrs +} + +// GetPodAffinity - Returns a corev1.Affinity reference for the specified component. +func GetPodAffinity(componentName string) *corev1.Affinity { + // If possible two pods of the same component (e.g cloudkitty-api) should not + // run on the same worker node. If this is not possible they get still + // created on the same worker node. + return affinity.DistributePods( + common.ComponentSelector, + []string{ + componentName, + }, + corev1.LabelHostname, + ) +} diff --git a/pkg/cloudkitty/volumes.go b/pkg/cloudkitty/volumes.go new file mode 100644 index 00000000..97ed4ab5 --- /dev/null +++ b/pkg/cloudkitty/volumes.go @@ -0,0 +1,57 @@ +package cloudkitty + +import ( + corev1 "k8s.io/api/core/v1" +) + +var ( + // scriptMode is the default permissions mode for Scripts volume + scriptMode int32 = 0740 + // configMode is the 640 permissions mode + configMode int32 = 0640 +) + +// GetVolumes - service volumes +func GetVolumes(name string) []corev1.Volume { + return []corev1.Volume{ + { + Name: "scripts", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &scriptMode, + SecretName: name + "-scripts", + }, + }, + }, { + Name: "config-data", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &configMode, + SecretName: name + "-config-data", + }, + }, + }, + } +} + +// GetVolumeMounts - general VolumeMounts +func GetVolumeMounts(serviceName string) []corev1.VolumeMount { + return []corev1.VolumeMount{ + { + Name: "scripts", + MountPath: "/var/lib/openstack/bin", + ReadOnly: true, + }, + { + Name: "config-data", + MountPath: "/var/lib/openstack/config", + ReadOnly: true, + }, + { + Name: "config-data", + MountPath: "/var/lib/kolla/config_files/config.json", + SubPath: serviceName + "-config.json", + ReadOnly: true, + }, + } +} diff --git a/pkg/cloudkittyapi/const.go b/pkg/cloudkittyapi/const.go new file mode 100644 index 00000000..7d9a378b --- /dev/null +++ b/pkg/cloudkittyapi/const.go @@ -0,0 +1,24 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkittyapi + +const ( + // ComponentName - + ComponentName = "cloudkitty-api" + + //LogFile - + LogFile = "/var/log/cloudkitty/cloudkitty-api.log" +) diff --git a/pkg/cloudkittyapi/statefulset.go b/pkg/cloudkittyapi/statefulset.go new file mode 100644 index 00000000..94d4346c --- /dev/null +++ b/pkg/cloudkittyapi/statefulset.go @@ -0,0 +1,188 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkittyapi + +import ( + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + // ServiceCommand - + ServiceCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start" +) + +// StatefulSet func +func StatefulSet( + instance *telemetryv1.CloudKittyAPI, + configHash string, + labels map[string]string, + annotations map[string]string, + topology *topologyv1.Topology, +) (*appsv1.StatefulSet, error) { + runAsUser := int64(0) + //cloudKittyUser := int64(telemetryv1.CloudKittyUserID) + + livenessProbe := &corev1.Probe{ + // TODO might need tuning + TimeoutSeconds: 5, + PeriodSeconds: 3, + InitialDelaySeconds: 5, + } + readinessProbe := &corev1.Probe{ + // TODO might need tuning + TimeoutSeconds: 5, + PeriodSeconds: 5, + InitialDelaySeconds: 5, + } + + args := []string{"-c", ServiceCommand} + // + // https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ + // + livenessProbe.HTTPGet = &corev1.HTTPGetAction{ + Path: "/healthcheck", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: int32(cloudkitty.CloudKittyPublicPort)}, + } + readinessProbe.HTTPGet = livenessProbe.HTTPGet + + if instance.Spec.TLS.API.Enabled(service.EndpointPublic) { + livenessProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS + readinessProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS + } + + // create Volume and VolumeMounts + volumes := GetVolumes(cloudkitty.GetOwningCloudKittyName(instance), instance.Name) + volumeMounts := GetVolumeMounts(instance.Name) + + // add CA cert if defined + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + + for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} { + if instance.Spec.TLS.API.Enabled(endpt) { + var tlsEndptCfg tls.GenericService + switch endpt { + case service.EndpointPublic: + tlsEndptCfg = instance.Spec.TLS.API.Public + case service.EndpointInternal: + tlsEndptCfg = instance.Spec.TLS.API.Internal + } + + svc, err := tlsEndptCfg.ToService() + if err != nil { + return nil, err + } + volumes = append(volumes, svc.CreateVolume(endpt.String())) + volumeMounts = append(volumeMounts, svc.CreateVolumeMounts(endpt.String())...) + } + } + + envVars := map[string]env.Setter{} + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["CONFIG_HASH"] = env.SetValue(configHash) + + statefulset := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Name, + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: appsv1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + PodManagementPolicy: appsv1.ParallelPodManagement, + Replicas: instance.Spec.Replicas, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + Labels: labels, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: instance.Spec.ServiceAccount, + Containers: []corev1.Container{ + // the first container in a pod is the default selected + // by oc log so define the log stream container first. + { + Name: instance.Name + "-log", + Command: []string{ + "/usr/bin/dumb-init", + }, + Args: []string{ + "--single-child", + "--", + "/bin/sh", + "-c", + "/usr/bin/tail -n+1 -F " + LogFile + " 2>/dev/null", + }, + Image: instance.Spec.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &runAsUser, + }, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: []corev1.VolumeMount{GetLogVolumeMount()}, + Resources: instance.Spec.Resources, + }, + { + Name: ComponentName, + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: instance.Spec.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &runAsUser, + }, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + Resources: instance.Spec.Resources, + ReadinessProbe: readinessProbe, + LivenessProbe: livenessProbe, + }, + }, + Volumes: volumes, + }, + }, + }, + } + + if instance.Spec.NodeSelector != nil { + statefulset.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector + } + + if topology != nil { + topology.ApplyTo(&statefulset.Spec.Template) + } else { + // If possible two pods of the same service should not + // run on the same worker node. If this is not possible + // the get still created on the same worker node. + statefulset.Spec.Template.Spec.Affinity = cloudkitty.GetPodAffinity(ComponentName) + } + + return statefulset, nil +} diff --git a/pkg/cloudkittyapi/volumes.go b/pkg/cloudkittyapi/volumes.go new file mode 100644 index 00000000..6d9f36ab --- /dev/null +++ b/pkg/cloudkittyapi/volumes.go @@ -0,0 +1,54 @@ +package cloudkittyapi + +import ( + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + corev1 "k8s.io/api/core/v1" +) + +// GetVolumes - +func GetVolumes(parentName string, name string) []corev1.Volume { + var config0644AccessMode int32 = 0644 + + volumes := []corev1.Volume{ + { + Name: "config-data-custom", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &config0644AccessMode, + SecretName: name + "-config-data", + }, + }, + }, + { + Name: "logs", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{Medium: ""}, + }, + }, + } + + return append(cloudkitty.GetVolumes(parentName), volumes...) +} + +// GetVolumeMounts - CloudKitty API VolumeMounts +func GetVolumeMounts(parentName string) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{ + { + Name: "config-data-custom", + MountPath: "/etc/cloudkitty/cloudkitty.conf.d", + ReadOnly: true, + }, + GetLogVolumeMount(), + } + + return append(cloudkitty.GetVolumeMounts(parentName), volumeMounts...) +} + +// GetLogVolumeMount - CloudKitty API LogVolumeMount +func GetLogVolumeMount() corev1.VolumeMount { + return corev1.VolumeMount{ + Name: "logs", + MountPath: "/var/log/cloudkitty", + ReadOnly: false, + } +} diff --git a/pkg/cloudkittyproc/const.go b/pkg/cloudkittyproc/const.go new file mode 100644 index 00000000..8bda7978 --- /dev/null +++ b/pkg/cloudkittyproc/const.go @@ -0,0 +1,21 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkittyproc + +const ( + // ComponentName - + ComponentName = "cloudkitty-proc" +) diff --git a/pkg/cloudkittyproc/statefulset.go b/pkg/cloudkittyproc/statefulset.go new file mode 100644 index 00000000..3071b60e --- /dev/null +++ b/pkg/cloudkittyproc/statefulset.go @@ -0,0 +1,153 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkittyproc + +import ( + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + // ServiceCommand - + ServiceCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start" +) + +// StatefulSet func +func StatefulSet( + instance *telemetryv1.CloudKittyProc, + configHash string, + labels map[string]string, + annotations map[string]string, + topology *topologyv1.Topology, +) *appsv1.StatefulSet { + cloudKittyUser := int64(0) + // cloudKittyUser := int64(telemetryv1.CloudKittyUserID) + // cloudKittyGroup := int64(telemetryv1.CloudKittyGroupID) + + // TODO until we determine how to properly query for these + livenessProbe := &corev1.Probe{ + // TODO might need tuning + TimeoutSeconds: 5, + PeriodSeconds: 3, + InitialDelaySeconds: 3, + } + + startupProbe := &corev1.Probe{ + TimeoutSeconds: 5, + FailureThreshold: 12, + PeriodSeconds: 5, + InitialDelaySeconds: 5, + } + + args := []string{"-c", ServiceCommand} + var probeCommand []string + livenessProbe.HTTPGet = &corev1.HTTPGetAction{ + Port: intstr.FromInt(8080), + } + startupProbe.HTTPGet = livenessProbe.HTTPGet + probeCommand = []string{ + "/usr/local/bin/container-scripts/healthcheck.py", + "processor", + "/etc/cloudkitty/cloudkitty.conf.d", + } + + envVars := map[string]env.Setter{} + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["CONFIG_HASH"] = env.SetValue(configHash) + + volumes := GetVolumes(cloudkitty.GetOwningCloudKittyName(instance), instance.Name) + volumeMounts := GetVolumeMounts(instance.Name) + + // Add the CA bundle + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + + statefulset := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Name, + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: appsv1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Replicas: instance.Spec.Replicas, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + Labels: labels, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: instance.Spec.ServiceAccount, + Containers: []corev1.Container{ + { + Name: ComponentName, + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: instance.Spec.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &cloudKittyUser, + }, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + Resources: instance.Spec.Resources, + LivenessProbe: livenessProbe, + StartupProbe: startupProbe, + }, + { + Name: "probe", + Command: probeCommand, + Image: instance.Spec.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &cloudKittyUser, + //RunAsGroup: &cloudKittyGroup, + }, + VolumeMounts: volumeMounts, + }, + }, + Volumes: volumes, + }, + }, + }, + } + + if instance.Spec.NodeSelector != nil { + statefulset.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector + } + + if topology != nil { + topology.ApplyTo(&statefulset.Spec.Template) + } else { + // If possible two pods of the same service should not + // run on the same worker node. If this is not possible + // the get still created on the same worker node. + statefulset.Spec.Template.Spec.Affinity = cloudkitty.GetPodAffinity(ComponentName) + } + + return statefulset +} diff --git a/pkg/cloudkittyproc/volumes.go b/pkg/cloudkittyproc/volumes.go new file mode 100644 index 00000000..51ff46a7 --- /dev/null +++ b/pkg/cloudkittyproc/volumes.go @@ -0,0 +1,38 @@ +package cloudkittyproc + +import ( + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + corev1 "k8s.io/api/core/v1" +) + +// GetVolumes - +func GetVolumes(parentName string, name string) []corev1.Volume { + var config0644AccessMode int32 = 0644 + + volumes := []corev1.Volume{ + { + Name: "config-data-custom", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &config0644AccessMode, + SecretName: name + "-config-data", + }, + }, + }, + } + + return append(cloudkitty.GetVolumes(parentName), volumes...) +} + +// GetVolumeMounts - CloudKitty API VolumeMounts +func GetVolumeMounts(parentName string) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{ + { + Name: "config-data-custom", + MountPath: "/etc/cloudkitty/cloudkitty.conf.d", + ReadOnly: true, + }, + } + + return append(cloudkitty.GetVolumeMounts(parentName), volumeMounts...) +} diff --git a/templates/cloudkitty/bin/healthcheck.py b/templates/cloudkitty/bin/healthcheck.py new file mode 100755 index 00000000..4bce1182 --- /dev/null +++ b/templates/cloudkitty/bin/healthcheck.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# +# Copyright 2022 Red Hat Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Trivial HTTP server to check health of scheduler, backup and volume services. +# Cinder-API hast its own health check endpoint and does not need this. +# +# The only check this server currently does is using the heartbeat in the +# database service table, accessing the DB directly here using cinder's +# configuration options. +# +# The benefit of accessing the DB directly is that it doesn't depend on the +# Cinder-API service being up and we can also differentiate between the +# container not having a connection to the DB and the cinder service not doing +# the heartbeats. +# +# For volume services all enabled backends must be up to return 200, so it is +# recommended to use a different pod for each backend to avoid one backend +# affecting others. +# +# Requires the name of the service as the first argument (volume, backup, +# scheduler) and optionally a second argument with the location of the +# configuration directory (defaults to /etc/cinder/cinder.conf.d) + +from http import server +import signal +import socket +import sys +import time +import threading + +from oslo_config import cfg + +SERVER_PORT = 8080 +CONF = cfg.CONF + +class HTTPServerV6(server.HTTPServer): + address_family = socket.AF_INET6 + +class HeartbeatServer(server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + self.wfile.write('OK'.encode('utf-8')) + + +def get_stopper(server): + def stopper(signal_number=None, frame=None): + print("Stopping server.") + server.shutdown() + server.server_close() + print("Server stopped.") + sys.exit(0) + return stopper + + +if __name__ == "__main__": + hostname = socket.gethostname() + ipv6_address = socket.getaddrinfo(hostname, None, socket.AF_INET6) + if ipv6_address: + webServer = HTTPServerV6(("::",SERVER_PORT), HeartbeatServer) + else: + webServer = server.HTTPServer(("0.0.0.0", SERVER_PORT), HeartbeatServer) + stop = get_stopper(webServer) + + # Need to run the server on a different thread because its shutdown method + # will block if called from the same thread, and the signal handler must be + # on the main thread in Python. + thread = threading.Thread(target=webServer.serve_forever) + thread.daemon = True + thread.start() + print(f"CloudKitty Healthcheck Server started http://{hostname}:{SERVER_PORT}") + signal.signal(signal.SIGTERM, stop) + + try: + while True: + time.sleep(60) + except KeyboardInterrupt: + pass + finally: + stop() diff --git a/templates/cloudkitty/bin/run-on-host b/templates/cloudkitty/bin/run-on-host new file mode 100755 index 00000000..e7840ace --- /dev/null +++ b/templates/cloudkitty/bin/run-on-host @@ -0,0 +1,2 @@ +#!/bin/sh +exec nsenter -a -t 1 -- `realpath -s $0` "$@" diff --git a/templates/cloudkitty/config/10-cloudkitty_wsgi.conf b/templates/cloudkitty/config/10-cloudkitty_wsgi.conf new file mode 100644 index 00000000..1fc084ae --- /dev/null +++ b/templates/cloudkitty/config/10-cloudkitty_wsgi.conf @@ -0,0 +1,40 @@ +{{ range $endpt, $vhost := .VHosts }} +# {{ $endpt }} vhost {{ $vhost.ServerName }} configuration + + ServerName {{ $vhost.ServerName }} + + ## Vhost docroot + DocumentRoot "/var/www/cgi-bin/cloudkitty" + + ## Directories, there should at least be a declaration for /var/www/cgi-bin/cloudkitty + + + Options -Indexes +FollowSymLinks +MultiViews + AllowOverride None + Require all granted + + + Timeout {{ $.TimeOut }} + + ## Logging + ErrorLog /dev/stdout + ServerSignature Off + CustomLog /dev/stdout combined + +{{- if $vhost.TLS }} + SetEnvIf X-Forwarded-Proto https HTTPS=1 + + ## SSL directives + SSLEngine on + SSLCertificateFile "{{ $vhost.SSLCertificateFile }}" + SSLCertificateKeyFile "{{ $vhost.SSLCertificateKeyFile }}" +{{- end }} + + ## WSGI configuration + WSGIApplicationGroup %{GLOBAL} + WSGIDaemonProcess {{ $endpt }} display-name={{ $endpt }} group=cloudkitty processes=4 threads=1 user=cloudkitty + WSGIProcessGroup {{ $endpt }} + WSGIScriptAlias / "/var/www/cgi-bin/cloudkitty/cloudkitty-wsgi" + WSGIPassAuthorization On + +{{ end }} diff --git a/templates/cloudkitty/config/cloudkitty-api-config-httpd.json b/templates/cloudkitty/config/cloudkitty-api-config-httpd.json new file mode 100644 index 00000000..174349bf --- /dev/null +++ b/templates/cloudkitty/config/cloudkitty-api-config-httpd.json @@ -0,0 +1,51 @@ +{ + "command": "/usr/sbin/httpd -DFOREGROUND", + "config_files": [ + { + "source": "/var/lib/openstack/config/httpd.conf", + "dest": "/etc/httpd/conf/httpd.conf", + "owner": "root", + "perm": "0644" + }, + { + "source": "/var/lib/openstack/config/10-cloudkitty_wsgi.conf", + "dest": "/etc/httpd/conf.d/10-cloudkitty_wsgi.conf", + "owner": "root", + "perm": "0644" + }, + { + "source": "/var/lib/openstack/config/ssl.conf", + "dest": "/etc/httpd/conf.d/ssl.conf", + "owner": "cloudkitty", + "perm": "0644" + }, + { + "source": "/var/lib/config-data/tls/certs/*", + "dest": "/etc/pki/tls/certs/", + "owner": "cloudkitty", + "perm": "0640", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/config-data/tls/private/*", + "dest": "/etc/pki/tls/private/", + "owner": "cloudkitty", + "perm": "0600", + "optional": true, + "merge": true + } + ], + "permissions": [ + { + "path": "/var/log/cloudkitty", + "owner": "cloudkitty:apache", + "recurse": true + }, + { + "path": "/etc/httpd/run", + "owner": "cloudkitty:apache", + "recurse": true + } + ] +} diff --git a/templates/cloudkitty/config/cloudkitty-api-config.json b/templates/cloudkitty/config/cloudkitty-api-config.json new file mode 100644 index 00000000..11a6f291 --- /dev/null +++ b/templates/cloudkitty/config/cloudkitty-api-config.json @@ -0,0 +1,11 @@ +{ + "command": "/usr/bin/uwsgi --ini /etc/cloudkitty/cloudkitty-api-uwsgi.ini", + "config_files": [ + { + "source": "/var/lib/openstack/config/cloudkitty-api-uwsgi.ini", + "dest": "/etc/cloudkitty/cloudkitty-api-uwsgi.ini", + "owner": "cloudkitty", + "perm": "0600" + } + ] +} diff --git a/templates/cloudkitty/config/cloudkitty-api-uwsgi.ini b/templates/cloudkitty/config/cloudkitty-api-uwsgi.ini new file mode 100644 index 00000000..2cd8f602 --- /dev/null +++ b/templates/cloudkitty/config/cloudkitty-api-uwsgi.ini @@ -0,0 +1,17 @@ +[uwsgi] +chmod-socket = 666 +socket = /var/run/uwsgi/cloudkitty.socket +start-time = %t +lazy-apps = true +add-header = Connection: close +buffer-size = 65535 +hook-master-start = unix_signal:15 gracefully_kill_them_all +thunder-lock = true +plugins = http,python3 +enable-threads = true +worker-reload-mercy = 80 +exit-on-reload = false +die-on-term = true +master = true +processes = 2 +wsgi-file = /opt/stack/data/venv/bin/cloudkitty-api \ No newline at end of file diff --git a/templates/cloudkitty/config/cloudkitty-dbsync-config.json b/templates/cloudkitty/config/cloudkitty-dbsync-config.json new file mode 100644 index 00000000..1eb4f0a6 --- /dev/null +++ b/templates/cloudkitty/config/cloudkitty-dbsync-config.json @@ -0,0 +1,11 @@ +{ + "command": "/usr/bin/cloudkitty-dbsync upgrade", + "config_files": [ + { + "source": "/var/lib/openstack/config/cloudkitty.conf", + "dest": "/etc/cloudkitty/cloudkitty.conf", + "owner": "cloudkitty", + "perm": "0600" + } + ] +} diff --git a/templates/cloudkitty/config/cloudkitty-proc-config.json b/templates/cloudkitty/config/cloudkitty-proc-config.json new file mode 100644 index 00000000..410ed9d3 --- /dev/null +++ b/templates/cloudkitty/config/cloudkitty-proc-config.json @@ -0,0 +1,17 @@ +{ + "command": "/usr/bin/cloudkitty-processor --logfile /dev/stdout", + "config_files": [ + { + "source": "/var/lib/openstack/config/cloudkitty.conf", + "dest": "/etc/cloudkitty/cloudkitty.conf", + "owner": "cloudkitty", + "perm": "0600" + }, + { + "source": "/var/lib/openstack/config/metrics.yaml", + "dest": "/etc/cloudkitty/metrics.yaml", + "owner": "cloudkitty", + "perm": "0600" + } + ] +} diff --git a/templates/cloudkitty/config/cloudkitty.conf b/templates/cloudkitty/config/cloudkitty.conf new file mode 100644 index 00000000..e9896eb2 --- /dev/null +++ b/templates/cloudkitty/config/cloudkitty.conf @@ -0,0 +1,78 @@ +[oslo_policy] +policy_file = policy.yaml + +[DEFAULT] +auth_strategy = keystone +debug = True +notification_topics = notifications +transport_url = {{ .TransportURL }} + +[authinfos] +debug = True +project_domain_name = default +user_domain_name = default +region_name = RegionOne +tenant_name = service +project_name = service +password = {{ .ServicePassword }} +username = {{ .ServiceUser }} +identity_uri = {{ .KeystoneInternalURL }} +auth_url = {{ .KeystoneInternalURL }} +auth_protocol = http +auth_type = v3password +{{- if .TLS }} +cafile = {{ .CAFile }} +{{- end }} + +[fetcher] +backend = keystone + +[fetcher_keystone] +auth_section = authinfos + +[storage_influxdb] +port = 8086 +host = localhost +database = cloudkitty +password = cloudkitty +user = cloudkitty + +[collect] +period = 300 +wait_periods = 0 +metrics_conf = /etc/cloudkitty/metrics.yaml +collector = prometheus +scope_key = project + +[collector_prometheus] +prometheus_url = https://metric-storage-prometheus.openstack.svc:9090 +insecure = true + +[output] +pipeline = osrf +basepath = /opt/stack/data/cloudkitty/reports +backend = cloudkitty.backend.file.FileBackend + +[storage] +version = 2 +backend = influxdb + +[database] +connection = {{ .DatabaseConnection }} + +[keystone_authtoken] +memcached_servers = {{ .MemcachedServersWithInet }} +# memcache_pool_dead_retry = 10 +# memcache_pool_conn_get_timeout = 2 +project_domain_name = Default +project_name = service +user_domain_name = Default +password = {{ .ServicePassword }} +username = {{ .ServiceUser }} +auth_url = {{ .KeystoneInternalURL }} +interface = internal +auth_type = password +{{- if .TLS }} +cafile = {{ .CAFile }} +{{- end }} +# service_token_roles_required = true diff --git a/templates/cloudkitty/config/httpd.conf b/templates/cloudkitty/config/httpd.conf new file mode 100644 index 00000000..b6c862a5 --- /dev/null +++ b/templates/cloudkitty/config/httpd.conf @@ -0,0 +1,27 @@ +ServerTokens Prod +ServerSignature Off +TraceEnable Off +ServerRoot "/etc/httpd" +ServerName "cloudkitty.openstack.svc" + +User apache +Group apache + +Listen 8889 + +TypesConfig /etc/mime.types + +Include conf.modules.d/*.conf + +LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined +LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" proxy + +SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded +CustomLog /dev/stdout combined env=!forwarded +CustomLog /dev/stdout proxy env=forwarded +ErrorLog /dev/stdout + +# XXX: To disable SSL +#Include conf.d/*.conf +# If above include is commented include at least the cloudkitty wsgi file +Include conf.d/10-cloudkitty_wsgi.conf diff --git a/templates/cloudkitty/config/metrics.yaml b/templates/cloudkitty/config/metrics.yaml new file mode 100644 index 00000000..c42701ed --- /dev/null +++ b/templates/cloudkitty/config/metrics.yaml @@ -0,0 +1,77 @@ +metrics: + ceilometer_cpu: + unit: instance + alt_name: instance + groupby: + - id + - user_id + - project + metadata: + - flavor_name + - flavor_id + - vcpus + mutate: NUMBOOL + extra_args: + aggregation_method: max + + ceilometer_image_size: + unit: MiB + factor: 1/1048576 + groupby: + - id + - user_id + - project + metadata: + - container_format + - disk_format + extra_args: + aggregation_method: max + + ceilometer_volume_size: + unit: GiB + groupby: + - id + - user_id + - project + metadata: + - volume_type + extra_args: + aggregation_method: max + + ceilometer_network_outgoing_bytes_rate: + unit: MB + groupby: + - id + - project + - user_id + # Converting B/s to MB/h + factor: 3600/1000000 + metadata: + - instance_id + extra_args: + aggregation_method: max + + ceilometer_network_incoming_bytes_rate: + unit: MB + groupby: + - id + - project + - user_id + # Converting B/s to MB/h + factor: 3600/1000000 + metadata: + - instance_id + extra_args: + aggregation_method: max + + ceilometer_ip_floating: + unit: ip + groupby: + - id + - user_id + - project + metadata: + - state + mutate: NUMBOOL + extra_args: + aggregation_method: max \ No newline at end of file diff --git a/templates/cloudkitty/config/ssl.conf b/templates/cloudkitty/config/ssl.conf new file mode 100644 index 00000000..e3da4ecb --- /dev/null +++ b/templates/cloudkitty/config/ssl.conf @@ -0,0 +1,21 @@ + + SSLRandomSeed startup builtin + SSLRandomSeed startup file:/dev/urandom 512 + SSLRandomSeed connect builtin + SSLRandomSeed connect file:/dev/urandom 512 + + AddType application/x-x509-ca-cert .crt + AddType application/x-pkcs7-crl .crl + + SSLPassPhraseDialog builtin + SSLSessionCache "shmcb:/var/cache/mod_ssl/scache(512000)" + SSLSessionCacheTimeout 300 + Mutex default + SSLCryptoDevice builtin + SSLHonorCipherOrder On + SSLUseStapling Off + SSLStaplingCache "shmcb:/run/httpd/ssl_stapling(32768)" + SSLCipherSuite HIGH:MEDIUM:!aNULL:!MD5:!RC4:!3DES + SSLProtocol all -SSLv2 -SSLv3 -TLSv1 + SSLOptions StdEnvVars + From 294361aa7ced46ced7ebc66704f740443e07bdb3 Mon Sep 17 00:00:00 2001 From: jlarriba Date: Mon, 30 Jun 2025 15:17:26 +0200 Subject: [PATCH 02/87] CloudKitty TLS --- .../telemetry.openstack.org_cloudkitties.yaml | 11 ++ ...lemetry.openstack.org_cloudkittyprocs.yaml | 3 + .../telemetry.openstack.org_telemetries.yaml | 12 ++ api/v1beta1/cloudkitty_types.go | 6 +- api/v1beta1/cloudkittyproc_types.go | 10 +- api/v1beta1/conditions.go | 18 +++ api/v1beta1/zz_generated.deepcopy.go | 2 +- .../telemetry.openstack.org_cloudkitties.yaml | 11 ++ ...lemetry.openstack.org_cloudkittyprocs.yaml | 3 + .../telemetry.openstack.org_telemetries.yaml | 12 ++ controllers/cloudkitty_controller.go | 57 +++++++++- pkg/cloudkitty/dbsync.go | 2 +- pkg/cloudkitty/storageinit.go | 106 ++++++++++++++++++ pkg/cloudkittyproc/statefulset.go | 23 ++-- .../config/cloudkitty-api-config-httpd.json | 51 --------- .../config/cloudkitty-api-config.json | 69 +++++++++++- .../config/cloudkitty-storageinit-config.json | 11 ++ templates/cloudkitty/config/cloudkitty.conf | 10 +- templates/cloudkitty/config/httpd.conf | 2 +- ...udkitty_wsgi.conf => wsgi-cloudkitty.conf} | 2 +- 20 files changed, 335 insertions(+), 86 deletions(-) create mode 100644 pkg/cloudkitty/storageinit.go delete mode 100644 templates/cloudkitty/config/cloudkitty-api-config-httpd.json create mode 100644 templates/cloudkitty/config/cloudkitty-storageinit-config.json rename templates/cloudkitty/config/{10-cloudkitty_wsgi.conf => wsgi-cloudkitty.conf} (94%) diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml index bc5b7567..387bf62d 100644 --- a/api/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -450,6 +450,17 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in + a pre-created bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string + type: object topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml index 9cd9b6ef..6123f53c 100644 --- a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml +++ b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -196,6 +196,9 @@ spec: description: CaBundleSecretName - holding the CA certs in a pre-created bundle file type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string type: object topologyRef: description: |- diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index 0d8bf4d3..f69ad6d9 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -1008,6 +1008,18 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs + in a pre-created bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index 08181d6c..1d2d1a7e 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -32,11 +32,13 @@ const ( CloudKittyGroupID = 42408 // CloudKittyAPIContainerImage - default fall-back image for CloudKitty API - CloudKittyAPIContainerImage = "quay.io/podified-master-centos9/cloudkitty-api:current-podified" + CloudKittyAPIContainerImage = "quay.io/podified-master-centos9/openstack-cloudkitty-api:current-podified" // CloudKittyProcContainerImage - default fall-back image for CloudKitty Processor - CloudKittyProcContainerImage = "quay.io/podified-master-centos9/cloudkitty-processor:current-podified" + CloudKittyProcContainerImage = "quay.io/podified-master-centos9/openstack-cloudkitty-processor:current-podified" // CloudKittyDbSyncHash hash CKDbSyncHash = "ckdbsync" + // CKStorageInitHash hash + CKStorageInitHash = "ckstorageinit" // CloudKittyReplicas - The number of replicas per each service deployed CloudKittyReplicas = 1 ) diff --git a/api/v1beta1/cloudkittyproc_types.go b/api/v1beta1/cloudkittyproc_types.go index d2459afd..d7b80e19 100644 --- a/api/v1beta1/cloudkittyproc_types.go +++ b/api/v1beta1/cloudkittyproc_types.go @@ -33,6 +33,11 @@ type CloudKittyProcTemplateCore struct { // +kubebuilder:validation:Minimum=0 // Replicas - CloudKitty API Replicas Replicas *int32 `json:"replicas"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.SimpleService `json:"tls,omitempty"` } // CloudKittyProcTemplate defines the input parameters for the CloudKitty Processor service @@ -63,11 +68,6 @@ type CloudKittyProcSpec struct { // +kubebuilder:validation:Required // ServiceAccount - service account name used internally to provide CloudKitty services the default SA name ServiceAccount string `json:"serviceAccount"` - - // +kubebuilder:validation:Optional - // +operator-sdk:csv:customresourcedefinitions:type=spec - // TLS - Parameters related to the TLS - TLS tls.Ca `json:"tls,omitempty"` } // CloudKittyProcStatus defines the observed state of CloudKitty Processor diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index f8e0b4b7..709bd1cd 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -54,6 +54,9 @@ const ( // CloudKittyProcReadyCondition Status=True condition which indicates if the CloudKitty Processor is configured and operational CloudKittyProcReadyCondition condition.Type = "CloudKittyProcReady" + // CloudKittyStorageInitReadyCondition Status=True condition which indicates if the CloudKitty Storage Init process has ran + CloudKittyStorageInitReadyCondition condition.Type = "CloudKittyStorageInitReady" + // LoggingCLONamespaceReadyCondition Status=True condition which indicates if the cluster-logging-operator namespace is created LoggingCLONamespaceReadyCondition condition.Type = "LoggingCLONamespaceReady" @@ -223,6 +226,21 @@ const ( // CloudKittyReadyErrorMessage CloudKittyReadyErrorMessage = "CloudKitty error occured %s" + // + // CloudKittyStorageInit condition messages + // + // CloudKittyStorageInitReadyInitMessage + CloudKittyStorageInitReadyInitMessage = "CloudKittyStorageInit not started" + + // CloudKittyStorageInitReadyMessage + CloudKittyStorageInitReadyMessage = "CloudKittyStorageInit completed" + + // CloudKittyStorageInitReadyRunning + CloudKittyStorageInitReadyRunningMessage = "CloudKittyStorageInit job still running" + + // CloudKittyStorageInitReadyErrorMessage + CloudKittyStorageInitReadyErrorMessage = "CloudKittyStorageInit job error occurred %s" + // // CloudKittyAPIReady condition messages // diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 1c5006d6..c9d622b6 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -906,7 +906,6 @@ func (in *CloudKittyProcSpec) DeepCopyInto(out *CloudKittyProcSpec) { *out = *in out.CloudKittyTemplate = in.CloudKittyTemplate in.CloudKittyProcTemplate.DeepCopyInto(&out.CloudKittyProcTemplate) - out.TLS = in.TLS } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProcSpec. @@ -994,6 +993,7 @@ func (in *CloudKittyProcTemplateCore) DeepCopyInto(out *CloudKittyProcTemplateCo *out = new(int32) **out = **in } + in.TLS.DeepCopyInto(&out.TLS) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProcTemplateCore. diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml index bc5b7567..387bf62d 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -450,6 +450,17 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in + a pre-created bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string + type: object topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml index 9cd9b6ef..6123f53c 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -196,6 +196,9 @@ spec: description: CaBundleSecretName - holding the CA certs in a pre-created bundle file type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string type: object topologyRef: description: |- diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index 0d8bf4d3..f69ad6d9 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -1008,6 +1008,18 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs + in a pre-created bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index 4dadd651..00cc5996 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -181,6 +181,7 @@ func (r *CloudKittyReconciler) Reconcile(ctx context.Context, req ctrl.Request) condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), condition.UnknownCondition(condition.DBReadyCondition, condition.InitReason, condition.DBReadyInitMessage), condition.UnknownCondition(condition.DBSyncReadyCondition, condition.InitReason, condition.DBSyncReadyInitMessage), + condition.UnknownCondition(telemetryv1.CloudKittyStorageInitReadyCondition, condition.InitReason, telemetryv1.CloudKittyStorageInitReadyInitMessage), condition.UnknownCondition(condition.RabbitMqTransportURLReadyCondition, condition.InitReason, condition.RabbitMqTransportURLReadyInitMessage), condition.UnknownCondition(condition.MemcachedReadyCondition, condition.InitReason, condition.MemcachedReadyInitMessage), condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), @@ -419,10 +420,10 @@ func (r *CloudKittyReconciler) reconcileInit( // run CloudKitty db sync // dbSyncHash := instance.Status.Hash[telemetryv1.CKDbSyncHash] - jobDef := cloudkitty.DbSyncJob(instance, serviceLabels) + jobDbSyncDef := cloudkitty.DbSyncJob(instance, serviceLabels) dbSyncjob := job.NewJob( - jobDef, + jobDbSyncDef, telemetryv1.CKDbSyncHash, instance.Spec.PreserveJobs, cloudkitty.ShortDuration, @@ -451,12 +452,54 @@ func (r *CloudKittyReconciler) reconcileInit( } if dbSyncjob.HasChanged() { instance.Status.Hash[telemetryv1.CKDbSyncHash] = dbSyncjob.GetHash() - Log.Info(fmt.Sprintf("Service '%s' - Job %s hash added - %s", instance.Name, jobDef.Name, instance.Status.Hash[telemetryv1.CKDbSyncHash])) + Log.Info(fmt.Sprintf("Service '%s' - Job %s hash added - %s", instance.Name, jobDbSyncDef.Name, instance.Status.Hash[telemetryv1.CKDbSyncHash])) } instance.Status.Conditions.MarkTrue(condition.DBSyncReadyCondition, condition.DBSyncReadyMessage) // run CloudKitty db sync - end + // + // run CloudKitty Storage Init + // + ckStorageInitHash := instance.Status.Hash[telemetryv1.CKStorageInitHash] + jobStorageInitDef := cloudkitty.StorageInitJob(instance, serviceLabels) + + storageInitjob := job.NewJob( + jobStorageInitDef, + telemetryv1.CKStorageInitHash, + instance.Spec.PreserveJobs, + cloudkitty.ShortDuration, + ckStorageInitHash, + ) + ctrlResult, err = storageInitjob.DoJob( + ctx, + helper, + ) + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyStorageInitReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + telemetryv1.CloudKittyStorageInitReadyRunningMessage)) + return ctrlResult, nil + } + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyStorageInitReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + telemetryv1.CloudKittyStorageInitReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if storageInitjob.HasChanged() { + instance.Status.Hash[telemetryv1.CKStorageInitHash] = storageInitjob.GetHash() + Log.Info(fmt.Sprintf("Service '%s' - Job %s hash added - %s", instance.Name, jobStorageInitDef.Name, instance.Status.Hash[telemetryv1.CKStorageInitHash])) + } + instance.Status.Conditions.MarkTrue(telemetryv1.CloudKittyStorageInitReadyCondition, telemetryv1.CloudKittyStorageInitReadyMessage) + + // run CloudKitty Storage Init - end + Log.Info(fmt.Sprintf("Reconciled Service '%s' init successfully", instance.Name)) return ctrl.Result{}, nil } @@ -829,6 +872,12 @@ func (r *CloudKittyReconciler) generateServiceConfigs( templateParameters["MemcachedServersWithInet"] = memcached.GetMemcachedServerListWithInetString() templateParameters["TimeOut"] = instance.Spec.APITimeout + templateParameters["TLS"] = false + if instance.Spec.CloudKittyProc.TLS.Enabled() { + templateParameters["TLS"] = true + templateParameters["CAFile"] = tls.DownstreamTLSCABundlePath + } + // create httpd vhost template parameters httpdVhostConfig := map[string]interface{}{} for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} { @@ -961,7 +1010,7 @@ func (r *CloudKittyReconciler) procDeploymentCreateOrUpdate(ctx context.Context, DatabaseHostname: instance.Status.DatabaseHostname, TransportURLSecret: instance.Status.TransportURLSecret, ServiceAccount: instance.RbacResourceName(), - TLS: instance.Spec.CloudKittyAPI.TLS.Ca, + //TLS: instance.Spec.CloudKittyProc.TLS.Ca, } if cloudKittyProcSpec.NodeSelector == nil { diff --git a/pkg/cloudkitty/dbsync.go b/pkg/cloudkitty/dbsync.go index 75a675e4..3cd213e5 100644 --- a/pkg/cloudkitty/dbsync.go +++ b/pkg/cloudkitty/dbsync.go @@ -55,7 +55,7 @@ func DbSyncJob(instance *telemetryv1.CloudKitty, labels map[string]string) *batc envVars["KOLLA_BOOTSTRAP"] = env.SetValue("TRUE") cloudKittyPassword := []corev1.EnvVar{ { - Name: "AodhPassword", + Name: "CloudKittyPassword", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ diff --git a/pkg/cloudkitty/storageinit.go b/pkg/cloudkitty/storageinit.go new file mode 100644 index 00000000..283c938d --- /dev/null +++ b/pkg/cloudkitty/storageinit.go @@ -0,0 +1,106 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cloudkitty + +import ( + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // storageInitCommand - + // TODO: Once we work on update/upgrades revisit the command in the + // the cloudkitty-storageinit-config.json file. + // If we stop all services during the update/upgrade then we can keep + // the --bump-versions flag. + // If we are doing rolling upgrades we'll need to use the flag + // conditionally (only for adoption) and do the restart cycle of + // services as described in the upstream rolling upgrades process. + storageInitCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start" +) + +// StorageInitJob func +func StorageInitJob(instance *telemetryv1.CloudKitty, labels map[string]string) *batchv1.Job { + args := []string{"-c", storageInitCommand} + + // create Volume and VolumeMounts + volumes := GetVolumes("cloudkitty") + volumeMounts := GetVolumeMounts("cloudkitty-storageinit") + // add CA cert if defined + if instance.Spec.CloudKittyAPI.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.CloudKittyAPI.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.CloudKittyAPI.TLS.CreateVolumeMounts(nil)...) + } + + runAsUser := int64(0) + envVars := map[string]env.Setter{} + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["KOLLA_BOOTSTRAP"] = env.SetValue("TRUE") + cloudKittyPassword := []corev1.EnvVar{ + { + Name: "CloudKittyPassword", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: instance.Spec.Secret, + }, + Key: instance.Spec.PasswordSelectors.CloudKittyService, + }, + }, + }, + } + + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: ServiceName + "-storageinit", + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + ServiceAccountName: instance.RbacResourceName(), + Containers: []corev1.Container{ + { + Name: ServiceName + "-storageinit", + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: instance.Spec.CloudKittyAPI.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &runAsUser, + }, + Env: env.MergeEnvs(cloudKittyPassword, envVars), + VolumeMounts: volumeMounts, + }, + }, + Volumes: volumes, + }, + }, + }, + } + + if instance.Spec.NodeSelector != nil { + job.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector + } + + return job +} diff --git a/pkg/cloudkittyproc/statefulset.go b/pkg/cloudkittyproc/statefulset.go index 3071b60e..c5959e00 100644 --- a/pkg/cloudkittyproc/statefulset.go +++ b/pkg/cloudkittyproc/statefulset.go @@ -24,7 +24,6 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" ) const ( @@ -45,7 +44,7 @@ func StatefulSet( // cloudKittyGroup := int64(telemetryv1.CloudKittyGroupID) // TODO until we determine how to properly query for these - livenessProbe := &corev1.Probe{ + /*livenessProbe := &corev1.Probe{ // TODO might need tuning TimeoutSeconds: 5, PeriodSeconds: 3, @@ -57,10 +56,10 @@ func StatefulSet( FailureThreshold: 12, PeriodSeconds: 5, InitialDelaySeconds: 5, - } + }*/ args := []string{"-c", ServiceCommand} - var probeCommand []string + /*var probeCommand []string livenessProbe.HTTPGet = &corev1.HTTPGetAction{ Port: intstr.FromInt(8080), } @@ -69,7 +68,7 @@ func StatefulSet( "/usr/local/bin/container-scripts/healthcheck.py", "processor", "/etc/cloudkitty/cloudkitty.conf.d", - } + }*/ envVars := map[string]env.Setter{} envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") @@ -113,13 +112,13 @@ func StatefulSet( SecurityContext: &corev1.SecurityContext{ RunAsUser: &cloudKittyUser, }, - Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: volumeMounts, - Resources: instance.Spec.Resources, - LivenessProbe: livenessProbe, - StartupProbe: startupProbe, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + Resources: instance.Spec.Resources, + //LivenessProbe: livenessProbe, + //StartupProbe: startupProbe, }, - { + /*{ Name: "probe", Command: probeCommand, Image: instance.Spec.ContainerImage, @@ -128,7 +127,7 @@ func StatefulSet( //RunAsGroup: &cloudKittyGroup, }, VolumeMounts: volumeMounts, - }, + },*/ }, Volumes: volumes, }, diff --git a/templates/cloudkitty/config/cloudkitty-api-config-httpd.json b/templates/cloudkitty/config/cloudkitty-api-config-httpd.json deleted file mode 100644 index 174349bf..00000000 --- a/templates/cloudkitty/config/cloudkitty-api-config-httpd.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "command": "/usr/sbin/httpd -DFOREGROUND", - "config_files": [ - { - "source": "/var/lib/openstack/config/httpd.conf", - "dest": "/etc/httpd/conf/httpd.conf", - "owner": "root", - "perm": "0644" - }, - { - "source": "/var/lib/openstack/config/10-cloudkitty_wsgi.conf", - "dest": "/etc/httpd/conf.d/10-cloudkitty_wsgi.conf", - "owner": "root", - "perm": "0644" - }, - { - "source": "/var/lib/openstack/config/ssl.conf", - "dest": "/etc/httpd/conf.d/ssl.conf", - "owner": "cloudkitty", - "perm": "0644" - }, - { - "source": "/var/lib/config-data/tls/certs/*", - "dest": "/etc/pki/tls/certs/", - "owner": "cloudkitty", - "perm": "0640", - "optional": true, - "merge": true - }, - { - "source": "/var/lib/config-data/tls/private/*", - "dest": "/etc/pki/tls/private/", - "owner": "cloudkitty", - "perm": "0600", - "optional": true, - "merge": true - } - ], - "permissions": [ - { - "path": "/var/log/cloudkitty", - "owner": "cloudkitty:apache", - "recurse": true - }, - { - "path": "/etc/httpd/run", - "owner": "cloudkitty:apache", - "recurse": true - } - ] -} diff --git a/templates/cloudkitty/config/cloudkitty-api-config.json b/templates/cloudkitty/config/cloudkitty-api-config.json index 11a6f291..380bb3a0 100644 --- a/templates/cloudkitty/config/cloudkitty-api-config.json +++ b/templates/cloudkitty/config/cloudkitty-api-config.json @@ -1,11 +1,74 @@ { - "command": "/usr/bin/uwsgi --ini /etc/cloudkitty/cloudkitty-api-uwsgi.ini", + "command": "/usr/sbin/httpd -DFOREGROUND -E /dev/stdout", "config_files": [ { - "source": "/var/lib/openstack/config/cloudkitty-api-uwsgi.ini", - "dest": "/etc/cloudkitty/cloudkitty-api-uwsgi.ini", + "source": "/var/lib/openstack/config/cloudkitty.conf", + "dest": "/etc/cloudkitty/cloudkitty.conf", "owner": "cloudkitty", "perm": "0600" + }, + { + "source": "/var/lib/openstack/config/custom.conf", + "dest": "/etc/aodh/cloudkitty.conf.d/01-cloudkitty-custom.conf", + "owner": "cloudkitty", + "perm": "0600", + "optional": true + }, + { + "source": "/var/lib/openstack/config/wsgi-cloudkitty.conf", + "dest": "/etc/httpd/conf.d/00wsgi-cloudkitty.conf", + "owner": "cloudkitty", + "perm": "0644" + }, + { + "source": "/var/lib/openstack/config/httpd.conf", + "dest": "/etc/httpd/conf/httpd.conf", + "owner": "cloudkitty", + "perm": "0644" + }, + { + "source": "/var/lib/openstack/config/ssl.conf", + "dest": "/etc/httpd/conf.d/ssl.conf", + "owner": "cloudkitty", + "perm": "0644" + }, + { + "source": "/var/lib/config-data/tls/certs/*", + "dest": "/etc/pki/tls/certs/", + "owner": "cloudkitty", + "perm": "0440", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/config-data/tls/private/*", + "dest": "/etc/pki/tls/private/", + "owner": "cloudkitty", + "perm": "0400", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/openstack/config/my.cnf", + "dest": "/etc/my.cnf", + "owner": "cloudkitty", + "perm": "0644" + }, + { + "source": "/var/lib/config-data/mtls/certs/*", + "dest": "/etc/pki/tls/certs/", + "owner": "cloudkitty:cloudkitty", + "perm": "0640", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/config-data/mtls/private/*", + "dest": "/etc/pki/tls/private/", + "owner": "cloudkitty:cloudkitty", + "perm": "0640", + "optional": true, + "merge": true } ] } diff --git a/templates/cloudkitty/config/cloudkitty-storageinit-config.json b/templates/cloudkitty/config/cloudkitty-storageinit-config.json new file mode 100644 index 00000000..3d9561c5 --- /dev/null +++ b/templates/cloudkitty/config/cloudkitty-storageinit-config.json @@ -0,0 +1,11 @@ +{ + "command": "/usr/bin/cloudkitty-storage-init", + "config_files": [ + { + "source": "/var/lib/openstack/config/cloudkitty.conf", + "dest": "/etc/cloudkitty/cloudkitty.conf", + "owner": "cloudkitty", + "perm": "0600" + } + ] +} diff --git a/templates/cloudkitty/config/cloudkitty.conf b/templates/cloudkitty/config/cloudkitty.conf index e9896eb2..58531272 100644 --- a/templates/cloudkitty/config/cloudkitty.conf +++ b/templates/cloudkitty/config/cloudkitty.conf @@ -31,11 +31,11 @@ backend = keystone auth_section = authinfos [storage_influxdb] -port = 8086 -host = localhost -database = cloudkitty -password = cloudkitty -user = cloudkitty +version = 2 +token = cloudkitty +org = openstack +bucket = cloudkitty +url = http://influxdb:8086 [collect] period = 300 diff --git a/templates/cloudkitty/config/httpd.conf b/templates/cloudkitty/config/httpd.conf index b6c862a5..c742ea69 100644 --- a/templates/cloudkitty/config/httpd.conf +++ b/templates/cloudkitty/config/httpd.conf @@ -24,4 +24,4 @@ ErrorLog /dev/stdout # XXX: To disable SSL #Include conf.d/*.conf # If above include is commented include at least the cloudkitty wsgi file -Include conf.d/10-cloudkitty_wsgi.conf +Include conf.d/00wsgi-cloudkitty.conf diff --git a/templates/cloudkitty/config/10-cloudkitty_wsgi.conf b/templates/cloudkitty/config/wsgi-cloudkitty.conf similarity index 94% rename from templates/cloudkitty/config/10-cloudkitty_wsgi.conf rename to templates/cloudkitty/config/wsgi-cloudkitty.conf index 1fc084ae..b7407f3d 100644 --- a/templates/cloudkitty/config/10-cloudkitty_wsgi.conf +++ b/templates/cloudkitty/config/wsgi-cloudkitty.conf @@ -34,7 +34,7 @@ WSGIApplicationGroup %{GLOBAL} WSGIDaemonProcess {{ $endpt }} display-name={{ $endpt }} group=cloudkitty processes=4 threads=1 user=cloudkitty WSGIProcessGroup {{ $endpt }} - WSGIScriptAlias / "/var/www/cgi-bin/cloudkitty/cloudkitty-wsgi" + WSGIScriptAlias / "/var/www/cgi-bin/cloudkitty/cloudkitty-api" WSGIPassAuthorization On {{ end }} From d803db4b45a07284952381961e1d87ddfae6cdce Mon Sep 17 00:00:00 2001 From: mgirgisf Date: Thu, 17 Jul 2025 13:13:41 +0200 Subject: [PATCH 03/87] update prometheus_collector --- api/v1beta1/cloudkitty_types.go | 24 +++++++++++ controllers/cloudkitty_controller.go | 46 +++++++++++++++++++++ pkg/cloudkitty/const.go | 3 ++ templates/cloudkitty/config/cloudkitty.conf | 8 +++- 4 files changed, 80 insertions(+), 1 deletion(-) diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index 1d2d1a7e..9559ffbd 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -90,6 +90,21 @@ type CloudKittySpecBase struct { // TopologyRef to apply the Topology defined by the associated CR referenced // by name TopologyRef *topologyv1.TopoRef `json:"topologyRef,omitempty"` + + // Host of user deployed prometheus + // +kubebuilder:validation:Optional + PrometheusHost string `json:"prometheusHost,omitempty"` + + // Port of user deployed prometheus + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + // +kubebuilder:validation:Optional + PrometheusPort int32 `json:"prometheusPort,omitempty"` + + // If defined, specifies which CA certificate to use for user deployed prometheus + // +kubebuilder:validation:Optional + // +nullable + PrometheusTLSCaCertSecret *corev1.SecretKeySelector `json:"prometheusTLSCaCertSecret,omitempty"` } // CloudKittySpecCore the same as CloudKittySpec without ContainerImage references @@ -211,6 +226,15 @@ type CloudKittyStatus struct { // controller has not started processing the latest changes, and the status // and its conditions are likely stale. ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // PrometheusHost - Hostname for prometheus used for autoscaling + PrometheusHost string `json:"prometheusHostname,omitempty"` + + // PrometheusPort - Port for prometheus used for autoscaling + PrometheusPort int32 `json:"prometheusPort,omitempty"` + + // PrometheusTLS - Determines if TLS should be used for accessing prometheus + PrometheusTLS bool `json:"prometheusTLS,omitempty"` } //+kubebuilder:object:root=true diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index 00cc5996..e032e60a 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -19,7 +19,10 @@ package controllers import ( "context" "fmt" + "strconv" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/metricstorage" k8s_errors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -813,6 +816,7 @@ func (r *CloudKittyReconciler) generateServiceConfigs( memcached *memcachedv1.Memcached, db *mariadbv1.Database, ) error { + Log := r.GetLogger(ctx) // // create Secret required for cloudkitty input // - %-scripts holds scripts to e.g. bootstrap the service @@ -855,6 +859,46 @@ func (r *CloudKittyReconciler) generateServiceConfigs( return err } + if instance.Spec.PrometheusHost == "" { + // We're using MetricStorage for Prometheus. + prometheusEndpointSecret := &corev1.Secret{} + err = r.Client.Get(ctx, client.ObjectKey{ + Name: cloudkitty.PrometheusEndpointSecret, + Namespace: instance.Namespace, + }, prometheusEndpointSecret) + if err != nil { + Log.Info("Prometheus Endpoint Secret not found") + } + if prometheusEndpointSecret.Data != nil { + instance.Status.PrometheusHost = string(prometheusEndpointSecret.Data[metricstorage.PrometheusHost]) + port, err := strconv.Atoi(string(prometheusEndpointSecret.Data[metricstorage.PrometheusPort])) + if err != nil { + return err + } + instance.Status.PrometheusPort = int32(port) + + metricStorage := &telemetryv1.MetricStorage{} + err = r.Client.Get(ctx, client.ObjectKey{ + Namespace: instance.Namespace, + Name: telemetryv1.DefaultServiceName, + }, metricStorage) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + } + instance.Status.PrometheusTLS = metricStorage.Spec.PrometheusTLS.Enabled() + } + } else { + // We're using user-deployed Prometheus. + instance.Status.PrometheusHost = instance.Spec.PrometheusHost + instance.Status.PrometheusPort = instance.Spec.PrometheusPort + instance.Status.PrometheusTLS = instance.Spec.PrometheusTLSCaCertSecret != nil + } + databaseAccount := db.GetAccount() dbSecret := db.GetSecret() @@ -864,6 +908,8 @@ func (r *CloudKittyReconciler) generateServiceConfigs( templateParameters["KeystoneInternalURL"] = keystoneInternalURL templateParameters["KeystonePublicURL"] = keystonePublicURL templateParameters["TransportURL"] = string(transportURLSecret.Data["transport_url"]) + templateParameters["PrometheusHost"] = instance.Status.PrometheusHost + templateParameters["PrometheusPort"] = instance.Status.PrometheusPort templateParameters["DatabaseConnection"] = fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf", databaseAccount.Spec.UserName, string(dbSecret.Data[mariadbv1.DatabasePasswordSelector]), diff --git a/pkg/cloudkitty/const.go b/pkg/cloudkitty/const.go index 5fb5269a..8b372471 100644 --- a/pkg/cloudkitty/const.go +++ b/pkg/cloudkitty/const.go @@ -49,6 +49,9 @@ const ( ShortDuration = time.Duration(5) * time.Second NormalDuration = time.Duration(10) * time.Second + + // PrometheusEndpointSecret - The name of the secret that contains the Prometheus endpoint configuration. + PrometheusEndpointSecret = "metric-storage-prometheus-endpoint" ) var ResultRequeue = ctrl.Result{RequeueAfter: NormalDuration} diff --git a/templates/cloudkitty/config/cloudkitty.conf b/templates/cloudkitty/config/cloudkitty.conf index 58531272..1aa611bf 100644 --- a/templates/cloudkitty/config/cloudkitty.conf +++ b/templates/cloudkitty/config/cloudkitty.conf @@ -45,8 +45,14 @@ collector = prometheus scope_key = project [collector_prometheus] -prometheus_url = https://metric-storage-prometheus.openstack.svc:9090 +{{- if .TLS }} +prometheus_url = https://{{ .PrometheusHost }}:{{ .PrometheusPort }} +cafile = {{ .CAFile }} +insecure = false +{{- else }} +prometheus_url = http://{{ .PrometheusHost }}:{{ .PrometheusPort }} insecure = true +{{- end }} [output] pipeline = osrf From 6081ab16ab5f77cbb20355a4f40e56fc54d3c57c Mon Sep 17 00:00:00 2001 From: jlarriba Date: Thu, 24 Jul 2025 09:36:11 +0200 Subject: [PATCH 04/87] Make it compatible with previous oscp --- .../telemetry.openstack.org_cloudkitties.yaml | 44 ++++++++++++++++++- .../telemetry.openstack.org_telemetries.yaml | 36 ++++++++++++++- api/v1beta1/cloudkitty_types.go | 2 +- api/v1beta1/telemetry_types.go | 3 +- api/v1beta1/zz_generated.deepcopy.go | 5 +++ .../telemetry.openstack.org_cloudkitties.yaml | 44 ++++++++++++++++++- .../telemetry.openstack.org_telemetries.yaml | 36 ++++++++++++++- controllers/cloudkitty_controller.go | 2 +- .../config/cloudkitty-api-config.json | 22 ++++++---- templates/cloudkitty/config/cloudkitty.conf | 5 ++- 10 files changed, 180 insertions(+), 19 deletions(-) diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml index 387bf62d..07bd45f4 100644 --- a/api/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -42,7 +42,6 @@ spec: apiTimeout: default: 60 description: APITimeout for HAProxy, Apache, and rpc_response_timeout - minimum: 10 type: integer cloudKittyAPI: description: CloudKittyAPI - Spec definition for the API service of @@ -494,6 +493,7 @@ spec: DB, defaults to cloudkitty type: string databaseInstance: + default: openstack description: |- MariaDB instance name Right now required by the maridb-operator to get the credentials from the instance to create the DB @@ -538,6 +538,37 @@ spec: description: PreserveJobs - do not delete jobs after they finished e.g. to check logs type: boolean + prometheusHost: + description: Host of user deployed prometheus + type: string + prometheusPort: + description: Port of user deployed prometheus + format: int32 + maximum: 65535 + minimum: 1 + type: integer + prometheusTLSCaCertSecret: + description: If defined, specifies which CA certificate to use for + user deployed prometheus + nullable: true + properties: + key: + description: The key of the secret to select from. Must be a + valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic rabbitMqClusterName: default: rabbitmq description: |- @@ -657,6 +688,17 @@ spec: and its conditions are likely stale. format: int64 type: integer + prometheusHostname: + description: PrometheusHost - Hostname for prometheus used for autoscaling + type: string + prometheusPort: + description: PrometheusPort - Port for prometheus used for autoscaling + format: int32 + type: integer + prometheusTLS: + description: PrometheusTLS - Determines if TLS should be used for + accessing prometheus + type: boolean serviceIDs: additionalProperties: type: string diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index f69ad6d9..9798050d 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -600,7 +600,6 @@ spec: apiTimeout: default: 60 description: APITimeout for HAProxy, Apache, and rpc_response_timeout - minimum: 10 type: integer cloudKittyAPI: description: CloudKittyAPI - Spec definition for the API service @@ -1053,13 +1052,14 @@ spec: cloudkitty DB, defaults to cloudkitty type: string databaseInstance: + default: openstack description: |- MariaDB instance name Right now required by the maridb-operator to get the credentials from the instance to create the DB Might not be required in future type: string enabled: - default: true + default: false description: Enabled - Whether OpenStack CloudKitty service should be deployed and managed type: boolean @@ -1102,6 +1102,38 @@ spec: description: PreserveJobs - do not delete jobs after they finished e.g. to check logs type: boolean + prometheusHost: + description: Host of user deployed prometheus + type: string + prometheusPort: + description: Port of user deployed prometheus + format: int32 + maximum: 65535 + minimum: 1 + type: integer + prometheusTLSCaCertSecret: + description: If defined, specifies which CA certificate to use + for user deployed prometheus + nullable: true + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic rabbitMqClusterName: default: rabbitmq description: |- diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index 9559ffbd..e4e29684 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -47,6 +47,7 @@ type CloudKittySpecBase struct { CloudKittyTemplate `json:",inline"` // +kubebuilder:validation:Required + // +kubebuilder:default=openstack // MariaDB instance name // Right now required by the maridb-operator to get the credentials from the instance to create the DB // Might not be required in future @@ -82,7 +83,6 @@ type CloudKittySpecBase struct { // +kubebuilder:validation:Optional // +kubebuilder:default=60 - // +kubebuilder:validation:Minimum=10 // APITimeout for HAProxy, Apache, and rpc_response_timeout APITimeout int `json:"apiTimeout"` diff --git a/api/v1beta1/telemetry_types.go b/api/v1beta1/telemetry_types.go index 02b6ac64..46373586 100644 --- a/api/v1beta1/telemetry_types.go +++ b/api/v1beta1/telemetry_types.go @@ -187,14 +187,13 @@ type LoggingSection struct { // CloudKittySpec defines the desired state of the cloudkitty service type CloudKittySection struct { // +kubebuilder:validation:Optional - // +kubebuilder:default=true + // +kubebuilder:default=false // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:booleanSwitch"} // Enabled - Whether OpenStack CloudKitty service should be deployed and managed Enabled *bool `json:"enabled"` // +kubebuilder:validation:Optional //+operator-sdk:csv:customresourcedefinitions:type=spec - // +kubebuilder:default={apiTimeout: 60} // Template - Overrides to use when creating the OpenStack CloudKitty service CloudKittySpec `json:",inline"` } diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index c9d622b6..f9a4c470 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1107,6 +1107,11 @@ func (in *CloudKittySpecBase) DeepCopyInto(out *CloudKittySpecBase) { *out = new(topologyv1beta1.TopoRef) **out = **in } + if in.PrometheusTLSCaCertSecret != nil { + in, out := &in.PrometheusTLSCaCertSecret, &out.PrometheusTLSCaCertSecret + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittySpecBase. diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml index 387bf62d..07bd45f4 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -42,7 +42,6 @@ spec: apiTimeout: default: 60 description: APITimeout for HAProxy, Apache, and rpc_response_timeout - minimum: 10 type: integer cloudKittyAPI: description: CloudKittyAPI - Spec definition for the API service of @@ -494,6 +493,7 @@ spec: DB, defaults to cloudkitty type: string databaseInstance: + default: openstack description: |- MariaDB instance name Right now required by the maridb-operator to get the credentials from the instance to create the DB @@ -538,6 +538,37 @@ spec: description: PreserveJobs - do not delete jobs after they finished e.g. to check logs type: boolean + prometheusHost: + description: Host of user deployed prometheus + type: string + prometheusPort: + description: Port of user deployed prometheus + format: int32 + maximum: 65535 + minimum: 1 + type: integer + prometheusTLSCaCertSecret: + description: If defined, specifies which CA certificate to use for + user deployed prometheus + nullable: true + properties: + key: + description: The key of the secret to select from. Must be a + valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic rabbitMqClusterName: default: rabbitmq description: |- @@ -657,6 +688,17 @@ spec: and its conditions are likely stale. format: int64 type: integer + prometheusHostname: + description: PrometheusHost - Hostname for prometheus used for autoscaling + type: string + prometheusPort: + description: PrometheusPort - Port for prometheus used for autoscaling + format: int32 + type: integer + prometheusTLS: + description: PrometheusTLS - Determines if TLS should be used for + accessing prometheus + type: boolean serviceIDs: additionalProperties: type: string diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index f69ad6d9..9798050d 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -600,7 +600,6 @@ spec: apiTimeout: default: 60 description: APITimeout for HAProxy, Apache, and rpc_response_timeout - minimum: 10 type: integer cloudKittyAPI: description: CloudKittyAPI - Spec definition for the API service @@ -1053,13 +1052,14 @@ spec: cloudkitty DB, defaults to cloudkitty type: string databaseInstance: + default: openstack description: |- MariaDB instance name Right now required by the maridb-operator to get the credentials from the instance to create the DB Might not be required in future type: string enabled: - default: true + default: false description: Enabled - Whether OpenStack CloudKitty service should be deployed and managed type: boolean @@ -1102,6 +1102,38 @@ spec: description: PreserveJobs - do not delete jobs after they finished e.g. to check logs type: boolean + prometheusHost: + description: Host of user deployed prometheus + type: string + prometheusPort: + description: Port of user deployed prometheus + format: int32 + maximum: 65535 + minimum: 1 + type: integer + prometheusTLSCaCertSecret: + description: If defined, specifies which CA certificate to use + for user deployed prometheus + nullable: true + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic rabbitMqClusterName: default: rabbitmq description: |- diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index e032e60a..5a98a71d 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -993,7 +993,7 @@ func (r *CloudKittyReconciler) transportURLCreateOrUpdate( ) (*rabbitmqv1.TransportURL, controllerutil.OperationResult, error) { transportURL := &rabbitmqv1.TransportURL{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-cloudkitty-transport", instance.Name), + Name: fmt.Sprintf("%s-transport", instance.Name), Namespace: instance.Namespace, Labels: serviceLabels, }, diff --git a/templates/cloudkitty/config/cloudkitty-api-config.json b/templates/cloudkitty/config/cloudkitty-api-config.json index 380bb3a0..107cfa1a 100644 --- a/templates/cloudkitty/config/cloudkitty-api-config.json +++ b/templates/cloudkitty/config/cloudkitty-api-config.json @@ -36,7 +36,7 @@ "source": "/var/lib/config-data/tls/certs/*", "dest": "/etc/pki/tls/certs/", "owner": "cloudkitty", - "perm": "0440", + "perm": "0640", "optional": true, "merge": true }, @@ -44,16 +44,10 @@ "source": "/var/lib/config-data/tls/private/*", "dest": "/etc/pki/tls/private/", "owner": "cloudkitty", - "perm": "0400", + "perm": "0600", "optional": true, "merge": true }, - { - "source": "/var/lib/openstack/config/my.cnf", - "dest": "/etc/my.cnf", - "owner": "cloudkitty", - "perm": "0644" - }, { "source": "/var/lib/config-data/mtls/certs/*", "dest": "/etc/pki/tls/certs/", @@ -70,5 +64,17 @@ "optional": true, "merge": true } + ], + "permissions": [ + { + "path": "/var/log/cinder", + "owner": "cloudkitty:apache", + "recurse": true + }, + { + "path": "/etc/httpd/run", + "owner": "cloudkitty:apache", + "recurse": true + } ] } diff --git a/templates/cloudkitty/config/cloudkitty.conf b/templates/cloudkitty/config/cloudkitty.conf index 1aa611bf..318ed398 100644 --- a/templates/cloudkitty/config/cloudkitty.conf +++ b/templates/cloudkitty/config/cloudkitty.conf @@ -61,7 +61,10 @@ backend = cloudkitty.backend.file.FileBackend [storage] version = 2 -backend = influxdb +backend = loki + +[storage_loki] +url = http://loki:3100/loki/api/v1 [database] connection = {{ .DatabaseConnection }} From 1f8910664963e580c99b5c32d0c616227d406cf3 Mon Sep 17 00:00:00 2001 From: jlarriba Date: Tue, 26 Aug 2025 10:11:25 +0200 Subject: [PATCH 05/87] From mgirgis: Add Cloudkitty healthcheck.py --- pkg/cloudkittyproc/statefulset.go | 30 +++---- templates/cloudkitty/bin/healthcheck.py | 106 +++++++++++++++++++----- 2 files changed, 98 insertions(+), 38 deletions(-) diff --git a/pkg/cloudkittyproc/statefulset.go b/pkg/cloudkittyproc/statefulset.go index c5959e00..0c3f8b70 100644 --- a/pkg/cloudkittyproc/statefulset.go +++ b/pkg/cloudkittyproc/statefulset.go @@ -24,6 +24,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) const ( @@ -44,7 +45,7 @@ func StatefulSet( // cloudKittyGroup := int64(telemetryv1.CloudKittyGroupID) // TODO until we determine how to properly query for these - /*livenessProbe := &corev1.Probe{ + livenessProbe := &corev1.Probe{ // TODO might need tuning TimeoutSeconds: 5, PeriodSeconds: 3, @@ -56,19 +57,18 @@ func StatefulSet( FailureThreshold: 12, PeriodSeconds: 5, InitialDelaySeconds: 5, - }*/ + } args := []string{"-c", ServiceCommand} - /*var probeCommand []string + var probeCommand []string livenessProbe.HTTPGet = &corev1.HTTPGetAction{ Port: intstr.FromInt(8080), } startupProbe.HTTPGet = livenessProbe.HTTPGet probeCommand = []string{ - "/usr/local/bin/container-scripts/healthcheck.py", - "processor", - "/etc/cloudkitty/cloudkitty.conf.d", - }*/ + "/var/lib/openstack/bin/healthcheck.py", + "/etc/cloudkitty/cloudkitty.conf.d/cloudkitty.conf", + } envVars := map[string]env.Setter{} envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") @@ -112,13 +112,13 @@ func StatefulSet( SecurityContext: &corev1.SecurityContext{ RunAsUser: &cloudKittyUser, }, - Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: volumeMounts, - Resources: instance.Spec.Resources, - //LivenessProbe: livenessProbe, - //StartupProbe: startupProbe, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + Resources: instance.Spec.Resources, + LivenessProbe: livenessProbe, + StartupProbe: startupProbe, }, - /*{ + { Name: "probe", Command: probeCommand, Image: instance.Spec.ContainerImage, @@ -127,7 +127,7 @@ func StatefulSet( //RunAsGroup: &cloudKittyGroup, }, VolumeMounts: volumeMounts, - },*/ + }, }, Volumes: volumes, }, @@ -149,4 +149,4 @@ func StatefulSet( } return statefulset -} +} \ No newline at end of file diff --git a/templates/cloudkitty/bin/healthcheck.py b/templates/cloudkitty/bin/healthcheck.py index 4bce1182..59639ccd 100755 --- a/templates/cloudkitty/bin/healthcheck.py +++ b/templates/cloudkitty/bin/healthcheck.py @@ -14,25 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -# Trivial HTTP server to check health of scheduler, backup and volume services. -# Cinder-API hast its own health check endpoint and does not need this. -# -# The only check this server currently does is using the heartbeat in the -# database service table, accessing the DB directly here using cinder's -# configuration options. -# -# The benefit of accessing the DB directly is that it doesn't depend on the -# Cinder-API service being up and we can also differentiate between the -# container not having a connection to the DB and the cinder service not doing -# the heartbeats. -# -# For volume services all enabled backends must be up to return 200, so it is -# recommended to use a different pod for each backend to avoid one backend -# affecting others. -# -# Requires the name of the service as the first argument (volume, backup, -# scheduler) and optionally a second argument with the location of the -# configuration directory (defaults to /etc/cinder/cinder.conf.d) from http import server import signal @@ -40,17 +21,67 @@ import sys import time import threading +import requests from oslo_config import cfg + SERVER_PORT = 8080 CONF = cfg.CONF + class HTTPServerV6(server.HTTPServer): - address_family = socket.AF_INET6 + address_family = socket.AF_INET6 + class HeartbeatServer(server.BaseHTTPRequestHandler): + + @staticmethod + def check_services(): + print("Starting health checks") + results = {} + + # Todo Database Endpoint Reachability + # Keystone Endpoint Reachability + try: + keystone_uri = CONF.keystone_authtoken.auth_url + response = requests.get(keystone_uri, timeout=5) + response.raise_for_status() + server_header = response.headers.get('Server', '').lower() + if 'keystone' in server_header: + results['keystone_endpoint'] = 'OK' + print("Keystone endpoint reachable and responsive.") + else: + results['keystone_endpoint'] = 'WARN' + print(f"Keystone endpoint reachable, but not a valid Keystone service: {keystone_uri}") + except requests.exceptions.RequestException as e: + results['keystone_endpoint'] = 'FAIL' + print(f"ERROR: Keystone endpoint check failed: {e}") + raise Exception('ERROR: Keystone check failed', e) + + # Prometheus Collector Endpoint Reachability + try: + prometheus_url = CONF.collector_prometheus.prometheus_url + insecure = CONF.collector_prometheus.insecure + cafile = CONF.collector_prometheus.cafile + verify_ssl = cafile if cafile and not insecure else not insecure + + response = requests.get(prometheus_url, timeout=5, verify=verify_ssl) + response.raise_for_status() + results['collector_endpoint'] = 'OK' + print("Prometheus collector endpoint reachable.") + except requests.exceptions.RequestException as e: + results['collector_endpoint'] = 'FAIL' + print(f"ERROR: Prometheus collector check failed: {e}") + raise Exception('ERROR: Prometheus collector check failed', e) + def do_GET(self): + try: + self.check_services() + except Exception as exc: + self.send_error(500, exc.args[0], exc.args[1]) + return + self.send_response(200) self.send_header("Content-type", "text/html") self.end_headers() @@ -68,12 +99,41 @@ def stopper(signal_number=None, frame=None): if __name__ == "__main__": + # Register config options + cfg.CONF.register_group(cfg.OptGroup(name='database', title='Database connection options')) + cfg.CONF.register_opt(cfg.StrOpt('connection', default=None), group='database') + + cfg.CONF.register_group(cfg.OptGroup(name='keystone_authtoken', title='Keystone Auth Token Options')) + cfg.CONF.register_opt(cfg.StrOpt('auth_url', + default='https://keystone-internal.openstack.svc:5000'), + group='keystone_authtoken') + + cfg.CONF.register_group(cfg.OptGroup(name='collector_prometheus', title='Prometheus Collector Options')) + cfg.CONF.register_opt(cfg.StrOpt('prometheus_url', + default='http://metric-storage-prometheus.openstack.svc:9090'), + group='collector_prometheus') + cfg.CONF.register_opt(cfg.BoolOpt('insecure', default=False), group='collector_prometheus') + cfg.CONF.register_opt(cfg.StrOpt('cafile', default=None), group='collector_prometheus') + + # Load configuration from file + try: + cfg.CONF(sys.argv[1:], default_config_files=['/etc/cloudkitty/cloudkitty.conf.d/cloudkitty.conf']) + except cfg.ConfigFilesNotFoundError as e: + print(f"Health check failed: {e}", file=sys.stderr) + sys.exit(1) + + # Detect IPv6 support for binding hostname = socket.gethostname() - ipv6_address = socket.getaddrinfo(hostname, None, socket.AF_INET6) + try: + ipv6_address = socket.getaddrinfo(hostname, None, socket.AF_INET6) + except socket.gaierror: + ipv6_address = None + if ipv6_address: - webServer = HTTPServerV6(("::",SERVER_PORT), HeartbeatServer) + webServer = HTTPServerV6(("::", SERVER_PORT), HeartbeatServer) else: webServer = server.HTTPServer(("0.0.0.0", SERVER_PORT), HeartbeatServer) + stop = get_stopper(webServer) # Need to run the server on a different thread because its shutdown method @@ -91,4 +151,4 @@ def stopper(signal_number=None, frame=None): except KeyboardInterrupt: pass finally: - stop() + stop() \ No newline at end of file From 13ef450362e8f4fe8d70631c1f94b326117f2339 Mon Sep 17 00:00:00 2001 From: jlarriba Date: Tue, 26 Aug 2025 10:39:10 +0200 Subject: [PATCH 06/87] Remove /v2 from path in cloudkitty, as the CLI automatically adds it --- controllers/cloudkittyapi_controller.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/controllers/cloudkittyapi_controller.go b/controllers/cloudkittyapi_controller.go index 082f6104..835b1395 100644 --- a/controllers/cloudkittyapi_controller.go +++ b/controllers/cloudkittyapi_controller.go @@ -460,21 +460,20 @@ func (r *CloudKittyAPIReconciler) reconcileInit( // expose the service (create service and return the created endpoint URLs) // - // V2 publicEndpointData := endpoint.Data{ Port: cloudkitty.CloudKittyPublicPort, - Path: "/v2", + Path: "", } internalEndpointData := endpoint.Data{ Port: cloudkitty.CloudKittyInternalPort, - Path: "/v2", + Path: "", } cloudkittyEndpoints := map[service.Endpoint]endpoint.Data{ service.EndpointPublic: publicEndpointData, service.EndpointInternal: internalEndpointData, } - apiEndpointsV3 := make(map[string]string) + apiEndpoints := make(map[string]string) for endpointType, data := range cloudkittyEndpoints { endpointTypeStr := string(endpointType) @@ -564,7 +563,7 @@ func (r *CloudKittyAPIReconciler) reconcileInit( data.Protocol = ptr.To(service.ProtocolHTTPS) } - apiEndpointsV3[string(endpointType)], err = svc.GetAPIEndpoint( + apiEndpoints[string(endpointType)], err = svc.GetAPIEndpoint( svcOverride.EndpointURL, data.Protocol, data.Path) if err != nil { instance.Status.Conditions.MarkFalse( From 73afff22ea17c4c3defd91165b52cb27a56f2672 Mon Sep 17 00:00:00 2001 From: jlarriba Date: Tue, 26 Aug 2025 11:08:12 +0200 Subject: [PATCH 07/87] Fix pre-commit --- .../telemetry.openstack.org_cloudkitties.yaml | 15 ++++----------- ...telemetry.openstack.org_cloudkittyapis.yaml | 8 ++------ ...elemetry.openstack.org_cloudkittyprocs.yaml | 8 ++------ .../telemetry.openstack.org_telemetries.yaml | 15 ++++----------- api/v1beta1/cloudkitty_types.go | 18 ++++++++++-------- api/v1beta1/cloudkittyapi_types.go | 8 ++++---- api/v1beta1/cloudkittyproc_types.go | 8 ++++---- .../telemetry.openstack.org_cloudkitties.yaml | 15 ++++----------- ...telemetry.openstack.org_cloudkittyapis.yaml | 8 ++------ ...elemetry.openstack.org_cloudkittyprocs.yaml | 8 ++------ .../telemetry.openstack.org_telemetries.yaml | 15 ++++----------- controllers/cloudkitty_controller.go | 12 ++++++------ controllers/cloudkittyapi_controller.go | 6 +++--- pkg/cloudkitty/dbsync.go | 5 ++++- pkg/cloudkitty/storageinit.go | 5 ++++- pkg/cloudkittyproc/statefulset.go | 2 +- templates/cloudkitty/bin/healthcheck.py | 2 +- .../cloudkitty/config/cloudkitty-api-uwsgi.ini | 17 ----------------- templates/cloudkitty/config/metrics.yaml | 2 +- 19 files changed, 62 insertions(+), 115 deletions(-) delete mode 100644 templates/cloudkitty/config/cloudkitty-api-uwsgi.ini diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml index 07bd45f4..6b58ee4d 100644 --- a/api/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -65,12 +65,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -348,8 +350,6 @@ spec: current project type: string type: object - required: - - containerImage type: object cloudKittyProc: description: CloudKittyProc - Spec definition for the Scheduler service @@ -373,12 +373,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -478,8 +480,6 @@ spec: current project type: string type: object - required: - - containerImage type: object customServiceConfig: description: |- @@ -600,13 +600,6 @@ spec: current project type: string type: object - required: - - cloudKittyAPI - - cloudKittyProc - - databaseInstance - - memcachedInstance - - rabbitMqClusterName - - secret type: object status: description: CloudKittyStatus defines the observed state of CloudKitty diff --git a/api/bases/telemetry.openstack.org_cloudkittyapis.yaml b/api/bases/telemetry.openstack.org_cloudkittyapis.yaml index d33b713c..bdd3309d 100644 --- a/api/bases/telemetry.openstack.org_cloudkittyapis.yaml +++ b/api/bases/telemetry.openstack.org_cloudkittyapis.yaml @@ -57,6 +57,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic databaseAccount: default: cloudkitty description: DatabaseAccount - optional MariaDBAccount used for cloudkitty @@ -71,6 +72,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -384,12 +386,6 @@ spec: transportURLSecret: description: Secret containing RabbitMq transport URL type: string - required: - - containerImage - - databaseHostname - - secret - - serviceAccount - - transportURLSecret type: object status: description: CloudKittyAPIStatus defines the observed state of CloudKittyAPI diff --git a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml index 6123f53c..178ceb15 100644 --- a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml +++ b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -71,6 +71,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic databaseAccount: default: cloudkitty description: DatabaseAccount - optional MariaDBAccount used for cloudkitty @@ -85,6 +86,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -220,12 +222,6 @@ spec: transportURLSecret: description: Secret containing RabbitMq transport URL type: string - required: - - containerImage - - databaseHostname - - secret - - serviceAccount - - transportURLSecret type: object status: description: CloudKittyProcStatus defines the observed state of CloudKitty diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index 9798050d..b3120d77 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -623,12 +623,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -906,8 +908,6 @@ spec: current project type: string type: object - required: - - containerImage type: object cloudKittyProc: description: CloudKittyProc - Spec definition for the Scheduler @@ -931,12 +931,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -1037,8 +1039,6 @@ spec: current project type: string type: object - required: - - containerImage type: object customServiceConfig: description: |- @@ -1166,13 +1166,6 @@ spec: current project type: string type: object - required: - - cloudKittyAPI - - cloudKittyProc - - databaseInstance - - memcachedInstance - - rabbitMqClusterName - - secret type: object logging: description: Logging - Parameters related to the logging diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index e4e29684..5a8cc27b 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -46,20 +46,20 @@ const ( type CloudKittySpecBase struct { CloudKittyTemplate `json:",inline"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // +kubebuilder:default=openstack // MariaDB instance name // Right now required by the maridb-operator to get the credentials from the instance to create the DB // Might not be required in future DatabaseInstance string `json:"databaseInstance"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // +kubebuilder:default=rabbitmq // RabbitMQ instance name // Needed to request a transportURL that is created and used in CloudKitty RabbitMqClusterName string `json:"rabbitMqClusterName"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // +kubebuilder:default=memcached // Memcached instance name. MemcachedInstance string `json:"memcachedInstance"` @@ -111,11 +111,11 @@ type CloudKittySpecBase struct { type CloudKittySpecCore struct { CloudKittySpecBase `json:",inline"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // CloudKittyAPI - Spec definition for the API service of this CloudKitty deployment CloudKittyAPI CloudKittyAPITemplateCore `json:"cloudKittyAPI"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // CloudKittyProc - Spec definition for the Scheduler service of this CloudKitty deployment CloudKittyProc CloudKittyProcTemplateCore `json:"cloudKittyProc"` } @@ -124,11 +124,11 @@ type CloudKittySpecCore struct { type CloudKittySpec struct { CloudKittySpecBase `json:",inline"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // CloudKittyAPI - Spec definition for the API service of this CloudKitty deployment CloudKittyAPI CloudKittyAPITemplate `json:"cloudKittyAPI"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // CloudKittyProc - Spec definition for the Scheduler service of this CloudKitty deployment CloudKittyProc CloudKittyProcTemplate `json:"cloudKittyProc"` } @@ -145,7 +145,7 @@ type CloudKittyTemplate struct { // DatabaseAccount - optional MariaDBAccount used for cloudkitty DB, defaults to cloudkitty DatabaseAccount string `json:"databaseAccount"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // Secret containing OpenStack password information Secret string `json:"secret"` @@ -171,6 +171,7 @@ type CloudKittyServiceTemplate struct { CustomServiceConfig string `json:"customServiceConfig,omitempty"` // +kubebuilder:validation:Optional + // +listType=atomic // CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets // that contain sensitive service config data. The content of each Secret gets added to the // /etc//.conf.d directory as a custom config file. @@ -182,6 +183,7 @@ type CloudKittyServiceTemplate struct { Resources corev1.ResourceRequirements `json:"resources,omitempty"` // +kubebuilder:validation:Optional + // +listType=atomic // NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network NetworkAttachments []string `json:"networkAttachments,omitempty"` diff --git a/api/v1beta1/cloudkittyapi_types.go b/api/v1beta1/cloudkittyapi_types.go index 42660c7e..3a71afb5 100644 --- a/api/v1beta1/cloudkittyapi_types.go +++ b/api/v1beta1/cloudkittyapi_types.go @@ -46,7 +46,7 @@ type CloudKittyAPITemplateCore struct { // CloudKittyAPITemplate defines the input parameters for the CloudKitty API service type CloudKittyAPITemplate struct { - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // ContainerImage - CloudKitty Container Image URL (will be set to environmental default if empty) ContainerImage string `json:"containerImage"` @@ -61,15 +61,15 @@ type CloudKittyAPISpec struct { // Input parameters for the CloudKitty API service CloudKittyAPITemplate `json:",inline"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // DatabaseHostname - CloudKitty Database Hostname DatabaseHostname string `json:"databaseHostname"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // Secret containing RabbitMq transport URL TransportURLSecret string `json:"transportURLSecret"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // ServiceAccount - service account name used internally to provide CloudKitty services the default SA name ServiceAccount string `json:"serviceAccount"` } diff --git a/api/v1beta1/cloudkittyproc_types.go b/api/v1beta1/cloudkittyproc_types.go index d7b80e19..83de10d1 100644 --- a/api/v1beta1/cloudkittyproc_types.go +++ b/api/v1beta1/cloudkittyproc_types.go @@ -42,7 +42,7 @@ type CloudKittyProcTemplateCore struct { // CloudKittyProcTemplate defines the input parameters for the CloudKitty Processor service type CloudKittyProcTemplate struct { - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // ContainerImage - CloudKitty Container Image URL (will be set to environmental default if empty) ContainerImage string `json:"containerImage"` @@ -57,15 +57,15 @@ type CloudKittyProcSpec struct { // Input parameters for the CloudKitty Processor service CloudKittyProcTemplate `json:",inline"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // DatabaseHostname - CloudKitty Database Hostname DatabaseHostname string `json:"databaseHostname"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // Secret containing RabbitMq transport URL TransportURLSecret string `json:"transportURLSecret"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // ServiceAccount - service account name used internally to provide CloudKitty services the default SA name ServiceAccount string `json:"serviceAccount"` } diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml index 07bd45f4..6b58ee4d 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -65,12 +65,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -348,8 +350,6 @@ spec: current project type: string type: object - required: - - containerImage type: object cloudKittyProc: description: CloudKittyProc - Spec definition for the Scheduler service @@ -373,12 +373,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -478,8 +480,6 @@ spec: current project type: string type: object - required: - - containerImage type: object customServiceConfig: description: |- @@ -600,13 +600,6 @@ spec: current project type: string type: object - required: - - cloudKittyAPI - - cloudKittyProc - - databaseInstance - - memcachedInstance - - rabbitMqClusterName - - secret type: object status: description: CloudKittyStatus defines the observed state of CloudKitty diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml index d33b713c..bdd3309d 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml @@ -57,6 +57,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic databaseAccount: default: cloudkitty description: DatabaseAccount - optional MariaDBAccount used for cloudkitty @@ -71,6 +72,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -384,12 +386,6 @@ spec: transportURLSecret: description: Secret containing RabbitMq transport URL type: string - required: - - containerImage - - databaseHostname - - secret - - serviceAccount - - transportURLSecret type: object status: description: CloudKittyAPIStatus defines the observed state of CloudKittyAPI diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml index 6123f53c..178ceb15 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -71,6 +71,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic databaseAccount: default: cloudkitty description: DatabaseAccount - optional MariaDBAccount used for cloudkitty @@ -85,6 +86,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -220,12 +222,6 @@ spec: transportURLSecret: description: Secret containing RabbitMq transport URL type: string - required: - - containerImage - - databaseHostname - - secret - - serviceAccount - - transportURLSecret type: object status: description: CloudKittyProcStatus defines the observed state of CloudKitty diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index 9798050d..b3120d77 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -623,12 +623,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -906,8 +908,6 @@ spec: current project type: string type: object - required: - - containerImage type: object cloudKittyProc: description: CloudKittyProc - Spec definition for the Scheduler @@ -931,12 +931,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -1037,8 +1039,6 @@ spec: current project type: string type: object - required: - - containerImage type: object customServiceConfig: description: |- @@ -1166,13 +1166,6 @@ spec: current project type: string type: object - required: - - cloudKittyAPI - - cloudKittyProc - - databaseInstance - - memcachedInstance - - rabbitMqClusterName - - secret type: object logging: description: Logging - Parameters related to the logging diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index 5a98a71d..3020961a 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -227,8 +227,8 @@ func (r *CloudKittyReconciler) Reconcile(ctx context.Context, req ctrl.Request) const ( cloudKittyPasswordSecretField = ".spec.secret" cloudKittyCaBundleSecretNameField = ".spec.tls.caBundleSecretName" - cloudKittyTlsAPIInternalField = ".spec.tls.api.internal.secretName" - cloudKittyTlsAPIPublicField = ".spec.tls.api.public.secretName" + cloudKittyTLSAPIInternalField = ".spec.tls.api.internal.secretName" + cloudKittyTLSAPIPublicField = ".spec.tls.api.public.secretName" cloudKittyTopologyField = ".spec.topologyRef.Name" ) @@ -241,8 +241,8 @@ var ( cloudKittyAPIWatchFields = []string{ cloudKittyPasswordSecretField, cloudKittyCaBundleSecretNameField, - cloudKittyTlsAPIInternalField, - cloudKittyTlsAPIPublicField, + cloudKittyTLSAPIInternalField, + cloudKittyTLSAPIPublicField, cloudKittyTopologyField, } ) @@ -423,7 +423,7 @@ func (r *CloudKittyReconciler) reconcileInit( // run CloudKitty db sync // dbSyncHash := instance.Status.Hash[telemetryv1.CKDbSyncHash] - jobDbSyncDef := cloudkitty.DbSyncJob(instance, serviceLabels) + jobDbSyncDef := cloudkitty.DbSyncJob(instance, serviceLabels, serviceAnnotations) dbSyncjob := job.NewJob( jobDbSyncDef, @@ -465,7 +465,7 @@ func (r *CloudKittyReconciler) reconcileInit( // run CloudKitty Storage Init // ckStorageInitHash := instance.Status.Hash[telemetryv1.CKStorageInitHash] - jobStorageInitDef := cloudkitty.StorageInitJob(instance, serviceLabels) + jobStorageInitDef := cloudkitty.StorageInitJob(instance, serviceLabels, serviceAnnotations) storageInitjob := job.NewJob( jobStorageInitDef, diff --git a/controllers/cloudkittyapi_controller.go b/controllers/cloudkittyapi_controller.go index 835b1395..de2cb3ff 100644 --- a/controllers/cloudkittyapi_controller.go +++ b/controllers/cloudkittyapi_controller.go @@ -301,7 +301,7 @@ func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl } // index tlsAPIInternalField - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyTlsAPIInternalField, func(rawObj client.Object) []string { + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyTLSAPIInternalField, func(rawObj client.Object) []string { // Extract the secret name from the spec, if one is provided cr := rawObj.(*telemetryv1.CloudKittyAPI) if cr.Spec.TLS.API.Internal.SecretName == nil { @@ -313,7 +313,7 @@ func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl } // index tlsAPIPublicField - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyTlsAPIPublicField, func(rawObj client.Object) []string { + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyTLSAPIPublicField, func(rawObj client.Object) []string { // Extract the secret name from the spec, if one is provided cr := rawObj.(*telemetryv1.CloudKittyAPI) if cr.Spec.TLS.API.Public.SecretName == nil { @@ -583,7 +583,7 @@ func (r *CloudKittyAPIReconciler) reconcileInit( if instance.Status.APIEndpoints == nil { instance.Status.APIEndpoints = map[string]map[string]string{} } - instance.Status.APIEndpoints[cloudkitty.ServiceName] = apiEndpointsV3 + instance.Status.APIEndpoints[cloudkitty.ServiceName] = apiEndpoints // V2 - end // expose service - end diff --git a/pkg/cloudkitty/dbsync.go b/pkg/cloudkitty/dbsync.go index 3cd213e5..5a84523a 100644 --- a/pkg/cloudkitty/dbsync.go +++ b/pkg/cloudkitty/dbsync.go @@ -36,7 +36,7 @@ const ( ) // DbSyncJob func -func DbSyncJob(instance *telemetryv1.CloudKitty, labels map[string]string) *batchv1.Job { +func DbSyncJob(instance *telemetryv1.CloudKitty, labels map[string]string, annotations map[string]string) *batchv1.Job { args := []string{"-c"} args = append(args, dbSyncCommand) @@ -75,6 +75,9 @@ func DbSyncJob(instance *telemetryv1.CloudKitty, labels map[string]string) *batc }, Spec: batchv1.JobSpec{ Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + }, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyOnFailure, ServiceAccountName: instance.RbacResourceName(), diff --git a/pkg/cloudkitty/storageinit.go b/pkg/cloudkitty/storageinit.go index 283c938d..73b05cd8 100644 --- a/pkg/cloudkitty/storageinit.go +++ b/pkg/cloudkitty/storageinit.go @@ -36,7 +36,7 @@ const ( ) // StorageInitJob func -func StorageInitJob(instance *telemetryv1.CloudKitty, labels map[string]string) *batchv1.Job { +func StorageInitJob(instance *telemetryv1.CloudKitty, labels map[string]string, annotations map[string]string) *batchv1.Job { args := []string{"-c", storageInitCommand} // create Volume and VolumeMounts @@ -74,6 +74,9 @@ func StorageInitJob(instance *telemetryv1.CloudKitty, labels map[string]string) }, Spec: batchv1.JobSpec{ Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + }, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyOnFailure, ServiceAccountName: instance.RbacResourceName(), diff --git a/pkg/cloudkittyproc/statefulset.go b/pkg/cloudkittyproc/statefulset.go index 0c3f8b70..a568be4d 100644 --- a/pkg/cloudkittyproc/statefulset.go +++ b/pkg/cloudkittyproc/statefulset.go @@ -149,4 +149,4 @@ func StatefulSet( } return statefulset -} \ No newline at end of file +} diff --git a/templates/cloudkitty/bin/healthcheck.py b/templates/cloudkitty/bin/healthcheck.py index 59639ccd..906f688d 100755 --- a/templates/cloudkitty/bin/healthcheck.py +++ b/templates/cloudkitty/bin/healthcheck.py @@ -151,4 +151,4 @@ def stopper(signal_number=None, frame=None): except KeyboardInterrupt: pass finally: - stop() \ No newline at end of file + stop() diff --git a/templates/cloudkitty/config/cloudkitty-api-uwsgi.ini b/templates/cloudkitty/config/cloudkitty-api-uwsgi.ini deleted file mode 100644 index 2cd8f602..00000000 --- a/templates/cloudkitty/config/cloudkitty-api-uwsgi.ini +++ /dev/null @@ -1,17 +0,0 @@ -[uwsgi] -chmod-socket = 666 -socket = /var/run/uwsgi/cloudkitty.socket -start-time = %t -lazy-apps = true -add-header = Connection: close -buffer-size = 65535 -hook-master-start = unix_signal:15 gracefully_kill_them_all -thunder-lock = true -plugins = http,python3 -enable-threads = true -worker-reload-mercy = 80 -exit-on-reload = false -die-on-term = true -master = true -processes = 2 -wsgi-file = /opt/stack/data/venv/bin/cloudkitty-api \ No newline at end of file diff --git a/templates/cloudkitty/config/metrics.yaml b/templates/cloudkitty/config/metrics.yaml index c42701ed..6bb078af 100644 --- a/templates/cloudkitty/config/metrics.yaml +++ b/templates/cloudkitty/config/metrics.yaml @@ -74,4 +74,4 @@ metrics: - state mutate: NUMBOOL extra_args: - aggregation_method: max \ No newline at end of file + aggregation_method: max From 76faf41de12250d5b1f3e9508913cb6b9ab78c7b Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 29 Aug 2025 16:18:34 +0100 Subject: [PATCH 08/87] [CloudKitty] Add osp-secret as the default for secret When the secret is unset, cloudkitty will not start, since there was no default set. The osp-secret is used as the default secret for other services. Thes default value for the cloudkitty secret is updated to match the other service defaults, since the required information (CK password) is expected to be in the same secret as the other services. --- api/bases/telemetry.openstack.org_cloudkitties.yaml | 1 + api/bases/telemetry.openstack.org_cloudkittyapis.yaml | 1 + api/bases/telemetry.openstack.org_cloudkittyprocs.yaml | 1 + api/bases/telemetry.openstack.org_telemetries.yaml | 1 + api/v1beta1/cloudkitty_types.go | 1 + config/crd/bases/telemetry.openstack.org_cloudkitties.yaml | 1 + config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml | 1 + config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml | 1 + config/crd/bases/telemetry.openstack.org_telemetries.yaml | 1 + 9 files changed, 9 insertions(+) diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml index 6b58ee4d..07ee7e0f 100644 --- a/api/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -576,6 +576,7 @@ spec: Needed to request a transportURL that is created and used in CloudKitty type: string secret: + default: osp-secret description: Secret containing OpenStack password information type: string serviceUser: diff --git a/api/bases/telemetry.openstack.org_cloudkittyapis.yaml b/api/bases/telemetry.openstack.org_cloudkittyapis.yaml index bdd3309d..2c475c7c 100644 --- a/api/bases/telemetry.openstack.org_cloudkittyapis.yaml +++ b/api/bases/telemetry.openstack.org_cloudkittyapis.yaml @@ -325,6 +325,7 @@ spec: type: object type: object secret: + default: osp-secret description: Secret containing OpenStack password information type: string serviceAccount: diff --git a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml index 178ceb15..2024e7ca 100644 --- a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml +++ b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -180,6 +180,7 @@ spec: type: object type: object secret: + default: osp-secret description: Secret containing OpenStack password information type: string serviceAccount: diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index b3120d77..ec1d2893 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -1141,6 +1141,7 @@ spec: Needed to request a transportURL that is created and used in CloudKitty type: string secret: + default: osp-secret description: Secret containing OpenStack password information type: string serviceUser: diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index 5a8cc27b..b24d4ccc 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -146,6 +146,7 @@ type CloudKittyTemplate struct { DatabaseAccount string `json:"databaseAccount"` // +kubebuilder:validation:Optional + // +kubebuilder:default=osp-secret // Secret containing OpenStack password information Secret string `json:"secret"` diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml index 6b58ee4d..07ee7e0f 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -576,6 +576,7 @@ spec: Needed to request a transportURL that is created and used in CloudKitty type: string secret: + default: osp-secret description: Secret containing OpenStack password information type: string serviceUser: diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml index bdd3309d..2c475c7c 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml @@ -325,6 +325,7 @@ spec: type: object type: object secret: + default: osp-secret description: Secret containing OpenStack password information type: string serviceAccount: diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml index 178ceb15..2024e7ca 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -180,6 +180,7 @@ spec: type: object type: object secret: + default: osp-secret description: Secret containing OpenStack password information type: string serviceAccount: diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index b3120d77..ec1d2893 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -1141,6 +1141,7 @@ spec: Needed to request a transportURL that is created and used in CloudKitty type: string secret: + default: osp-secret description: Secret containing OpenStack password information type: string serviceUser: From 63e39e1812fca69364acc1177767e2a13d4e78a9 Mon Sep 17 00:00:00 2001 From: Jaromir Wysoglad Date: Fri, 19 Sep 2025 05:13:28 -0400 Subject: [PATCH 09/87] [cloudkitty] Add LokiStack deployment Also add mTLS certificates deployment needed for communication between Loki and CloudKitty. All is tested with kuttl-tests, some minor fixes across CloudKitty related code included (other needed fixes were discussed and will follow in the future). EnsureWatches was moved from metricstorage_controller into pkg/utils. All calls to this function throughout the whole metric storage controller needed to be modified to point to the new location, but other than that there aren't any changes to the metric storage controller. --- Makefile | 6 + .../telemetry.openstack.org_cloudkitties.yaml | 92 ++++++++ .../telemetry.openstack.org_telemetries.yaml | 92 ++++++++ api/go.mod | 11 +- api/go.sum | 22 +- api/v1beta1/cloudkitty_types.go | 9 + api/v1beta1/conditions.go | 38 +++ api/v1beta1/zz_generated.deepcopy.go | 1 + .../telemetry.openstack.org_cloudkitties.yaml | 92 ++++++++ .../telemetry.openstack.org_telemetries.yaml | 92 ++++++++ controllers/cloudkitty_controller.go | 223 +++++++++++++++++- controllers/cloudkittyapi_controller.go | 67 ++++++ controllers/cloudkittyproc_controller.go | 66 ++++++ controllers/metricstorage_controller.go | 90 +++---- go.mod | 14 +- go.sum | 28 ++- main.go | 12 +- pkg/cloudkitty/cert.go | 67 ++++++ pkg/cloudkitty/const.go | 5 + pkg/cloudkitty/dbsync.go | 2 +- pkg/cloudkitty/lokistack.go | 118 +++++++++ pkg/cloudkitty/storageinit.go | 2 +- pkg/cloudkitty/volumes.go | 29 +++ pkg/cloudkittyapi/statefulset.go | 2 +- pkg/cloudkittyapi/volumes.go | 4 +- pkg/cloudkittyproc/statefulset.go | 2 +- pkg/cloudkittyproc/volumes.go | 4 +- pkg/utils/utils.go | 59 +++++ .../config/cloudkitty-api-config.json | 7 + .../config/cloudkitty-proc-config.json | 7 + templates/cloudkitty/config/cloudkitty.conf | 5 +- tests/kuttl/suites/cloudkitty/config.yaml | 14 ++ .../deps/OpenStackControlPlane.yaml | 27 +++ tests/kuttl/suites/cloudkitty/deps/infra.yaml | 42 ++++ .../suites/cloudkitty/deps/kustomization.yaml | 50 ++++ .../suites/cloudkitty/deps/loki-operator.yaml | 27 +++ .../cloudkitty/deps/loki-s3-secret.yaml | 10 + tests/kuttl/suites/cloudkitty/deps/minio.yaml | 99 ++++++++ .../suites/cloudkitty/deps/namespace.yaml | 4 + .../suites/cloudkitty/deps/telemetry.yaml | 7 + tests/kuttl/suites/cloudkitty/output/.keep | 0 .../suites/cloudkitty/tests/00-deps.yaml | 9 + .../suites/cloudkitty/tests/01-assert.yaml | 81 +++++++ .../suites/cloudkitty/tests/01-deploy.yaml | 54 +++++ 44 files changed, 1582 insertions(+), 110 deletions(-) create mode 100644 pkg/cloudkitty/cert.go create mode 100644 pkg/cloudkitty/lokistack.go create mode 100644 tests/kuttl/suites/cloudkitty/config.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/OpenStackControlPlane.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/infra.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/kustomization.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/loki-operator.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/loki-s3-secret.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/minio.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/namespace.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/telemetry.yaml create mode 100644 tests/kuttl/suites/cloudkitty/output/.keep create mode 100644 tests/kuttl/suites/cloudkitty/tests/00-deps.yaml create mode 100644 tests/kuttl/suites/cloudkitty/tests/01-assert.yaml create mode 100644 tests/kuttl/suites/cloudkitty/tests/01-deploy.yaml diff --git a/Makefile b/Makefile index aa0d4c18..834a4473 100644 --- a/Makefile +++ b/Makefile @@ -393,6 +393,12 @@ kuttl-test-cleanup: if [ "$(KUTTL_SUITE)" == "ceilometer" ]; then \ oc delete --wait=true --all=true -n $(KUTTL_NAMESPACE) --timeout=120s Ceilometer; \ fi; \ + if [ "$(KUTTL_SUITE)" == "metric-storage" ]; then \ + oc delete --wait=true --all=true -n $(KUTTL_NAMESPACE) --timeout=120s MetricStorage; \ + fi; \ + if [ "$(KUTTL_SUITE)" == "cloudkitty" ]; then \ + oc delete --wait=true --all=true -n $(KUTTL_NAMESPACE) --timeout=120s CloudKitty; \ + fi; \ if [ "$(KUTTL_SUITE)" == "default" ]; then \ oc delete --wait=true --all=true -n $(KUTTL_NAMESPACE) --timeout=120s Telemetry; \ fi; \ diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml index 6b58ee4d..15cfcf46 100644 --- a/api/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -575,6 +575,95 @@ spec: RabbitMQ instance name Needed to request a transportURL that is created and used in CloudKitty type: string + s3StorageConfig: + description: S3 related configuration passed to Loki + properties: + schemas: + default: + - effectiveDate: "2020-10-11" + version: v11 + description: Schemas for reading and writing logs. + items: + description: ObjectStorageSchema defines a schema version and + the date when it will become effective. + properties: + effectiveDate: + description: |- + EffectiveDate contains a date in YYYY-MM-DD format which is interpreted in the UTC time zone. + + + The configuration always needs at least one schema that is currently valid. This means that when creating a new + LokiStack it is recommended to add a schema with the latest available version and an effective date of "yesterday". + New schema versions added to the configuration always needs to be placed "in the future", so that Loki can start + using it once the day rolls over. + pattern: ^([0-9]{4,})([-]([0-9]{2})){2}$ + type: string + version: + description: Version for writing and reading logs. + enum: + - v11 + - v12 + - v13 + type: string + required: + - effectiveDate + - version + type: object + minItems: 1 + type: array + secret: + description: |- + Secret for object storage authentication. + Name of a secret in the same namespace as the LokiStack custom resource. + properties: + credentialMode: + description: |- + CredentialMode can be used to set the desired credential mode for authenticating with the object storage. + If this is not set, then the operator tries to infer the credential mode from the provided secret and its + own configuration. + enum: + - static + - token + - token-cco + type: string + name: + description: Name of a secret in the namespace configured + for object storage secrets. + type: string + type: + description: Type of object storage that should be used + enum: + - azure + - gcs + - s3 + - swift + - alibabacloud + type: string + required: + - name + - type + type: object + tls: + description: TLS configuration for reaching the object storage + endpoint. + properties: + caKey: + description: |- + Key is the data key of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + If empty, it defaults to "service-ca.crt". + type: string + caName: + description: |- + CA is the name of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + type: string + required: + - caName + type: object + required: + - secret + type: object secret: description: Secret containing OpenStack password information type: string @@ -583,6 +672,9 @@ spec: description: ServiceUser - optional username used for this service to register in cloudkitty type: string + storageClass: + description: Storage class used for Loki + type: string topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index b3120d77..72556e5d 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -1140,6 +1140,95 @@ spec: RabbitMQ instance name Needed to request a transportURL that is created and used in CloudKitty type: string + s3StorageConfig: + description: S3 related configuration passed to Loki + properties: + schemas: + default: + - effectiveDate: "2020-10-11" + version: v11 + description: Schemas for reading and writing logs. + items: + description: ObjectStorageSchema defines a schema version + and the date when it will become effective. + properties: + effectiveDate: + description: |- + EffectiveDate contains a date in YYYY-MM-DD format which is interpreted in the UTC time zone. + + + The configuration always needs at least one schema that is currently valid. This means that when creating a new + LokiStack it is recommended to add a schema with the latest available version and an effective date of "yesterday". + New schema versions added to the configuration always needs to be placed "in the future", so that Loki can start + using it once the day rolls over. + pattern: ^([0-9]{4,})([-]([0-9]{2})){2}$ + type: string + version: + description: Version for writing and reading logs. + enum: + - v11 + - v12 + - v13 + type: string + required: + - effectiveDate + - version + type: object + minItems: 1 + type: array + secret: + description: |- + Secret for object storage authentication. + Name of a secret in the same namespace as the LokiStack custom resource. + properties: + credentialMode: + description: |- + CredentialMode can be used to set the desired credential mode for authenticating with the object storage. + If this is not set, then the operator tries to infer the credential mode from the provided secret and its + own configuration. + enum: + - static + - token + - token-cco + type: string + name: + description: Name of a secret in the namespace configured + for object storage secrets. + type: string + type: + description: Type of object storage that should be used + enum: + - azure + - gcs + - s3 + - swift + - alibabacloud + type: string + required: + - name + - type + type: object + tls: + description: TLS configuration for reaching the object storage + endpoint. + properties: + caKey: + description: |- + Key is the data key of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + If empty, it defaults to "service-ca.crt". + type: string + caName: + description: |- + CA is the name of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + type: string + required: + - caName + type: object + required: + - secret + type: object secret: description: Secret containing OpenStack password information type: string @@ -1148,6 +1237,9 @@ spec: description: ServiceUser - optional username used for this service to register in cloudkitty type: string + storageClass: + description: Storage class used for Loki + type: string topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/api/go.mod b/api/go.mod index 671c2b3a..fb2d24a3 100644 --- a/api/go.mod +++ b/api/go.mod @@ -3,6 +3,7 @@ module github.com/openstack-k8s-operators/telemetry-operator/api go 1.21 require ( + github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba github.com/onsi/ginkgo/v2 v2.20.1 github.com/onsi/gomega v1.34.1 github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20250513115636-b549982a5d8f @@ -53,11 +54,11 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/net v0.28.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.18.0 // indirect - golang.org/x/sys v0.23.0 // indirect - golang.org/x/term v0.23.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.24.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect @@ -70,7 +71,7 @@ require ( k8s.io/component-base v0.29.15 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240322212309-b815d8309940 // indirect - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect + k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/api/go.sum b/api/go.sum index 5580678d..d969c0f2 100644 --- a/api/go.sum +++ b/api/go.sum @@ -47,6 +47,8 @@ github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQu github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba h1:P5Wgp2HfGfNPLCPpS+YqquKdrrl4tW0El7VX23D6vtg= +github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba/go.mod h1:OBAgJh0mLYRvziBzBKr4/anrPHqGY9qEfuNXCpnUNi0= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -125,8 +127,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -140,18 +142,18 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -197,8 +199,8 @@ k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240322212309-b815d8309940 h1:qVoMaQV5t62UUvHe16Q3eb2c5HPzLHYzsi0Tu/xLndo= k8s.io/kube-openapi v0.0.0-20240322212309-b815d8309940/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.17.6 h1:12IXsozEsIXWAMRpgRlYS1jjAHQXHtWEOMdULh3DbEw= sigs.k8s.io/controller-runtime v0.17.6/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index 5a8cc27b..b22592dd 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -22,6 +22,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/util" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + lokistackv1 "github.com/grafana/loki/operator/api/loki/v1" ) const ( @@ -105,6 +106,14 @@ type CloudKittySpecBase struct { // +kubebuilder:validation:Optional // +nullable PrometheusTLSCaCertSecret *corev1.SecretKeySelector `json:"prometheusTLSCaCertSecret,omitempty"` + + // S3 related configuration passed to Loki + // +kubebuilder:validation:Optional + S3StorageConfig lokistackv1.ObjectStorageSpec `json:"s3StorageConfig"` + + // Storage class used for Loki + // +kubebuilder:validation:Optional + StorageClass string `json:"storageClass,omitempty"` } // CloudKittySpecCore the same as CloudKittySpec without ContainerImage references diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index 709bd1cd..d2ccd731 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -57,6 +57,12 @@ const ( // CloudKittyStorageInitReadyCondition Status=True condition which indicates if the CloudKitty Storage Init process has ran CloudKittyStorageInitReadyCondition condition.Type = "CloudKittyStorageInitReady" + // CloudKittyClientCertReadyCondition Status=True condition which indicates if the CloudKitty client certificate is ready for use + CloudKittyClientCertReadyCondition condition.Type = "CloudKittyClientCertReady" + + // CloudKittyLokiStackReadyCondition Status=True condition which indicates if the CloudKitty LokiStack is ready + CloudKittyLokiStackReadyCondition condition.Type = "CloudKittyLokiStackReady" + // LoggingCLONamespaceReadyCondition Status=True condition which indicates if the cluster-logging-operator namespace is created LoggingCLONamespaceReadyCondition condition.Type = "LoggingCLONamespaceReady" @@ -271,6 +277,38 @@ const ( // CloudKittyProcReadyRunningMessage CloudKittyProcReadyRunningMessage = "CloudKittyProc in progress" + // + // CloudKittyClientCertReady condition messages + // + // CloudKittyClientCertReadyInitMessage + CloudKittyClientCertReadyInitMessage = "CloudKittyClientCert not created" + + // CloudKittyClientCertReadyMessage + CloudKittyClientCertReadyMessage = "CloudKittyClientCert ready for use" + + // CloudKittyClientCertReadyErrorMessage + CloudKittyClientCertReadyErrorMessage = "CloudKittyClientCert error occured %s" + + // CloudKittyClientCertReadyRunningMessage + CloudKittyClientCertReadyRunningMessage = "CloudKittyClientCert in progress" + + // + // CloudKittyLokiStackReady condition messages + // + // CloudKittyLokiStackReadyInitMessage + CloudKittyLokiStackReadyInitMessage = "CloudKittyLokiStack not created" + + // CloudKittyLokiStackReadyMessage + CloudKittyLokiStackReadyMessage = "CloudKittyLokiStack ready for use" + + // CloudKittyLokiStackReadyErrorMessage + CloudKittyLokiStackReadyErrorMessage = "CloudKittyLokiStack error occured %s" + + // CloudKittyLokiStackReadyRunningMessage + CloudKittyLokiStackReadyRunningMessage = "CloudKittyLokiStack in progress" + // CloudKittyLokiStackReadyRunningMessage + CloudKittyLokiStackUnableToOwnMessage = "Error occured when trying to own %s" + DashboardsNotEnabledMessage = "Dashboarding was not enabled, so no actions required" DashboardPrometheusRuleReadyInitMessage = "Dashboard PrometheusRule not started" diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index f9a4c470..e9f328c9 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1112,6 +1112,7 @@ func (in *CloudKittySpecBase) DeepCopyInto(out *CloudKittySpecBase) { *out = new(v1.SecretKeySelector) (*in).DeepCopyInto(*out) } + in.S3StorageConfig.DeepCopyInto(&out.S3StorageConfig) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittySpecBase. diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml index 6b58ee4d..15cfcf46 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -575,6 +575,95 @@ spec: RabbitMQ instance name Needed to request a transportURL that is created and used in CloudKitty type: string + s3StorageConfig: + description: S3 related configuration passed to Loki + properties: + schemas: + default: + - effectiveDate: "2020-10-11" + version: v11 + description: Schemas for reading and writing logs. + items: + description: ObjectStorageSchema defines a schema version and + the date when it will become effective. + properties: + effectiveDate: + description: |- + EffectiveDate contains a date in YYYY-MM-DD format which is interpreted in the UTC time zone. + + + The configuration always needs at least one schema that is currently valid. This means that when creating a new + LokiStack it is recommended to add a schema with the latest available version and an effective date of "yesterday". + New schema versions added to the configuration always needs to be placed "in the future", so that Loki can start + using it once the day rolls over. + pattern: ^([0-9]{4,})([-]([0-9]{2})){2}$ + type: string + version: + description: Version for writing and reading logs. + enum: + - v11 + - v12 + - v13 + type: string + required: + - effectiveDate + - version + type: object + minItems: 1 + type: array + secret: + description: |- + Secret for object storage authentication. + Name of a secret in the same namespace as the LokiStack custom resource. + properties: + credentialMode: + description: |- + CredentialMode can be used to set the desired credential mode for authenticating with the object storage. + If this is not set, then the operator tries to infer the credential mode from the provided secret and its + own configuration. + enum: + - static + - token + - token-cco + type: string + name: + description: Name of a secret in the namespace configured + for object storage secrets. + type: string + type: + description: Type of object storage that should be used + enum: + - azure + - gcs + - s3 + - swift + - alibabacloud + type: string + required: + - name + - type + type: object + tls: + description: TLS configuration for reaching the object storage + endpoint. + properties: + caKey: + description: |- + Key is the data key of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + If empty, it defaults to "service-ca.crt". + type: string + caName: + description: |- + CA is the name of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + type: string + required: + - caName + type: object + required: + - secret + type: object secret: description: Secret containing OpenStack password information type: string @@ -583,6 +672,9 @@ spec: description: ServiceUser - optional username used for this service to register in cloudkitty type: string + storageClass: + description: Storage class used for Loki + type: string topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index b3120d77..72556e5d 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -1140,6 +1140,95 @@ spec: RabbitMQ instance name Needed to request a transportURL that is created and used in CloudKitty type: string + s3StorageConfig: + description: S3 related configuration passed to Loki + properties: + schemas: + default: + - effectiveDate: "2020-10-11" + version: v11 + description: Schemas for reading and writing logs. + items: + description: ObjectStorageSchema defines a schema version + and the date when it will become effective. + properties: + effectiveDate: + description: |- + EffectiveDate contains a date in YYYY-MM-DD format which is interpreted in the UTC time zone. + + + The configuration always needs at least one schema that is currently valid. This means that when creating a new + LokiStack it is recommended to add a schema with the latest available version and an effective date of "yesterday". + New schema versions added to the configuration always needs to be placed "in the future", so that Loki can start + using it once the day rolls over. + pattern: ^([0-9]{4,})([-]([0-9]{2})){2}$ + type: string + version: + description: Version for writing and reading logs. + enum: + - v11 + - v12 + - v13 + type: string + required: + - effectiveDate + - version + type: object + minItems: 1 + type: array + secret: + description: |- + Secret for object storage authentication. + Name of a secret in the same namespace as the LokiStack custom resource. + properties: + credentialMode: + description: |- + CredentialMode can be used to set the desired credential mode for authenticating with the object storage. + If this is not set, then the operator tries to infer the credential mode from the provided secret and its + own configuration. + enum: + - static + - token + - token-cco + type: string + name: + description: Name of a secret in the namespace configured + for object storage secrets. + type: string + type: + description: Type of object storage that should be used + enum: + - azure + - gcs + - s3 + - swift + - alibabacloud + type: string + required: + - name + - type + type: object + tls: + description: TLS configuration for reaching the object storage + endpoint. + properties: + caKey: + description: |- + Key is the data key of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + If empty, it defaults to "service-ca.crt". + type: string + caName: + description: |- + CA is the name of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + type: string + required: + - caName + type: object + required: + - secret + type: object secret: description: Secret containing OpenStack password information type: string @@ -1148,6 +1237,9 @@ spec: description: ServiceUser - optional username used for this service to register in cloudkitty type: string + storageClass: + description: Storage class used for Loki + type: string topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index 3020961a..cb7a9ea8 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -19,10 +19,15 @@ package controllers import ( "context" "fmt" + "slices" "strconv" + "time" + + lokistackv1 "github.com/grafana/loki/operator/api/loki/v1" "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" "github.com/openstack-k8s-operators/telemetry-operator/pkg/metricstorage" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/utils" k8s_errors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -36,13 +41,16 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" + certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" "github.com/go-logr/logr" networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/certmanager" "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/configmap" "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" "github.com/openstack-k8s-operators/lib-common/modules/common/env" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" @@ -79,11 +87,7 @@ func (r *CloudKittyReconciler) GetScheme() *runtime.Scheme { } // CloudKittyReconciler reconciles a CloudKitty object -type CloudKittyReconciler struct { - client.Client - Kclient kubernetes.Interface - Scheme *runtime.Scheme -} +type CloudKittyReconciler utils.ConditionalWatchingReconciler // GetLogger returns a logger object with a logging prefix of "controller.name" and additional controller context fields func (r *CloudKittyReconciler) GetLogger(ctx context.Context) logr.Logger { @@ -191,6 +195,8 @@ func (r *CloudKittyReconciler) Reconcile(ctx context.Context, req ctrl.Request) condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), condition.UnknownCondition(telemetryv1.CloudKittyAPIReadyCondition, condition.InitReason, telemetryv1.CloudKittyAPIReadyInitMessage), condition.UnknownCondition(telemetryv1.CloudKittyProcReadyCondition, condition.InitReason, telemetryv1.CloudKittyProcReadyInitMessage), + condition.UnknownCondition(telemetryv1.CloudKittyClientCertReadyCondition, condition.InitReason, telemetryv1.CloudKittyClientCertReadyInitMessage), + condition.UnknownCondition(telemetryv1.CloudKittyLokiStackReadyCondition, condition.InitReason, telemetryv1.CloudKittyLokiStackReadyInitMessage), condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), // service account, role, rolebinding conditions condition.UnknownCondition(condition.ServiceAccountReadyCondition, condition.InitReason, condition.ServiceAccountReadyInitMessage), @@ -327,7 +333,7 @@ func (r *CloudKittyReconciler) SetupWithManager(mgr ctrl.Manager) error { return nil } - return ctrl.NewControllerManagedBy(mgr). + control, err := ctrl.NewControllerManagedBy(mgr). For(&telemetryv1.CloudKitty{}). Owns(&mariadbv1.MariaDBDatabase{}). Owns(&mariadbv1.MariaDBAccount{}). @@ -336,9 +342,11 @@ func (r *CloudKittyReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&rabbitmqv1.TransportURL{}). Owns(&batchv1.Job{}). Owns(&corev1.Secret{}). + Owns(&corev1.ConfigMap{}). Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.Role{}). Owns(&rbacv1.RoleBinding{}). + Owns(&certmgrv1.Certificate{}). // Watch for TransportURL Secrets which belong to any TransportURLs created by CloudKitty CRs Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(transportURLSecretFn)). @@ -347,7 +355,10 @@ func (r *CloudKittyReconciler) SetupWithManager(mgr ctrl.Manager) error { Watches(&keystonev1.KeystoneAPI{}, handler.EnqueueRequestsFromMapFunc(r.findObjectForSrc), builder.WithPredicates(keystonev1.KeystoneAPIStatusChangedPredicate)). - Complete(r) + // LokiStack watch added dynamically inside the controller code. + Build(r) + r.Controller = control + return err } func (r *CloudKittyReconciler) findObjectForSrc(ctx context.Context, src client.Object) []reconcile.Request { @@ -507,11 +518,203 @@ func (r *CloudKittyReconciler) reconcileInit( return ctrl.Result{}, nil } +// Original source: +// https://github.com/openstack-k8s-operators/openstack-operator/blob/cf133b39e91c05f53c57725d7c6f5a627d98dccd/pkg/openstack/ca.go#L687 +func getCAFromSecret( + ctx context.Context, + instance *telemetryv1.CloudKitty, + helper *helper.Helper, + secretName string, +) (string, ctrl.Result, error) { + caSecret, ctrlResult, err := secret.GetDataFromSecret(ctx, helper, secretName, time.Duration(5), "ca.crt") + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyLokiStackReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + telemetryv1.CloudKittyLokiStackReadyErrorMessage, + err.Error())) + + return "", ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyLokiStackReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + telemetryv1.CloudKittyLokiStackReadyRunningMessage)) + + return "", ctrlResult, nil + } + + return caSecret, ctrl.Result{}, nil +} + func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *telemetryv1.CloudKitty, helper *helper.Helper) (ctrl.Result, error) { Log := r.GetLogger(ctx) Log.Info(fmt.Sprintf("Reconciling Service '%s'", instance.Name)) + // Create cloudkitty client cert / key + certIssuer, err := certmanager.GetIssuerByLabels( + ctx, helper, instance.Namespace, + map[string]string{certmanager.RootCAIssuerInternalLabel: ""}, + ) + if err != nil { + Log.Error(err, "Failed to determine certificate issuer") + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyClientCertReadyCondition, + condition.ErrorReason, + condition.SeverityError, + telemetryv1.CloudKittyClientCertReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + certDefinition := cloudkitty.Certificate( + instance, serviceLabels, certIssuer, + ) + cert := certmanager.NewCertificate(certDefinition, 5) + ctrlResult, _, err := cert.CreateOrPatch(ctx, helper, nil) + + if err != nil { + Log.Error(err, "Failed to create or patch cloudkitty client certificate") + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyClientCertReadyCondition, + condition.ErrorReason, + condition.SeverityError, + telemetryv1.CloudKittyClientCertReadyErrorMessage, + err.Error())) + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + Log.Info("CloudKitty client certificate is being created") + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyClientCertReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + telemetryv1.CloudKittyClientCertReadyRunningMessage)) + return ctrlResult, nil + } + + caData, ctrlResult, err := getCAFromSecret( + ctx, instance, helper, certDefinition.Spec.SecretName, + ) + if err != nil { + Log.Error(err, "Failed to get cloudkitty client certificate CA data") + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyClientCertReadyCondition, + condition.ErrorReason, + condition.SeverityError, + telemetryv1.CloudKittyClientCertReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + cms := []util.Template{ + { + Name: fmt.Sprintf("%s-%s", instance.Name, cloudkitty.CaConfigmapName), + Namespace: instance.Namespace, + Type: util.TemplateTypeNone, + InstanceType: "cloudkitty", + CustomData: map[string]string{ + cloudkitty.CaConfigmapKey: caData, + }, + }, + } + + err = configmap.EnsureConfigMaps( + ctx, helper, instance, cms, nil, + ) + if err != nil { + Log.Error(err, "Failed to create CA configmap for cloudkitty client cert verification") + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyClientCertReadyCondition, + condition.ErrorReason, + condition.SeverityError, + telemetryv1.CloudKittyClientCertReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + instance.Status.Conditions.MarkTrue(telemetryv1.CloudKittyClientCertReadyCondition, telemetryv1.CloudKittyClientCertReadyMessage) + + // Deploy Loki + var eventHandler handler.EventHandler = handler.EnqueueRequestForOwner( + r.Scheme, + r.RESTMapper, + &telemetryv1.CloudKitty{}, + handler.OnlyControllerOwner(), + ) + + err = utils.EnsureWatches( + (*utils.ConditionalWatchingReconciler)(r), ctx, + "lokistacks.loki.grafana.com", + &lokistackv1.LokiStack{}, eventHandler, helper, + ) + if err != nil { + instance.Status.Conditions.MarkFalse(telemetryv1.CloudKittyLokiStackReadyCondition, + condition.Reason("Can't own LokiStack resource. The loki-operator probably isn't installed"), + condition.SeverityError, + telemetryv1.CloudKittyLokiStackUnableToOwnMessage, err) + Log.Info("Can't own LokiStack resource. The loki-operator probably isn't installed") + return ctrl.Result{RequeueAfter: telemetryv1.PauseBetweenWatchAttempts}, nil + } + + lokiStack := &lokistackv1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-lokistack", instance.Name), + Namespace: instance.Namespace, + }, + } + op, err := controllerutil.CreateOrPatch(ctx, r.Client, lokiStack, func() error { + desiredLokiStack := cloudkitty.LokiStack(instance, serviceLabels) + desiredLokiStack.Spec.DeepCopyInto(&lokiStack.Spec) + lokiStack.ObjectMeta.Labels = serviceLabels + err = controllerutil.SetControllerReference(instance, lokiStack, r.Scheme) + return err + }) + if err != nil { + Log.Error(err, "Failed to create or patch LokiStack") + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyLokiStackReadyCondition, + condition.ErrorReason, + condition.SeverityError, + telemetryv1.CloudKittyLokiStackReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if op != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf("LokiStack %s successfully changed - operation: %s", lokiStack.Name, string(op))) + } + + // Mirror LokiStacks's condition here. LokiStack uses conditions + // a little differently than o-k-o. Whats more, it can have + // multiple 'active' conditions, while we have only one 'master' + // condition LokiStack here. So we mirror hopefully the one most + // relevant active condition in this order of + // priority: Ready > Failed > Degraded > Pending > Warning. + + order := []string{"Ready", "Failed", "Degraded", "Pending", "Warning"} + index := len(order) + reason := condition.InitReason + message := telemetryv1.CloudKittyLokiStackReadyInitMessage + for _, c := range lokiStack.Status.Conditions { + conditionIndex := slices.Index(order, c.Type) + if c.Status == "True" && conditionIndex < index { + index = conditionIndex + reason = c.Reason + message = c.Message + } + } + if index < len(order) && order[index] == "Ready" { + instance.Status.Conditions.MarkTrue(telemetryv1.CloudKittyLokiStackReadyCondition, telemetryv1.CloudKittyLokiStackReadyMessage) + } else { + Log.Info("LokiStack not ready") + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyLokiStackReadyCondition, + condition.Reason(reason), + condition.SeverityWarning, + fmt.Sprintf("LokiStack issue: %s", message))) + } + // Service account, role, binding rbacRules := []rbacv1.PolicyRule{ { @@ -726,7 +929,7 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te } // Handle service init - ctrlResult, err := r.reconcileInit(ctx, instance, helper, serviceLabels, serviceAnnotations) + ctrlResult, err = r.reconcileInit(ctx, instance, helper, serviceLabels, serviceAnnotations) if err != nil { return ctrlResult, err } else if (ctrlResult != ctrl.Result{}) { @@ -902,6 +1105,8 @@ func (r *CloudKittyReconciler) generateServiceConfigs( databaseAccount := db.GetAccount() dbSecret := db.GetSecret() + lokiHost := fmt.Sprintf("%s-lokistack-gateway-http.%s.svc", instance.Name, instance.Namespace) + templateParameters := make(map[string]interface{}) templateParameters["ServiceUser"] = instance.Spec.ServiceUser templateParameters["ServicePassword"] = string(ospSecret.Data[instance.Spec.PasswordSelectors.CloudKittyService]) @@ -910,6 +1115,8 @@ func (r *CloudKittyReconciler) generateServiceConfigs( templateParameters["TransportURL"] = string(transportURLSecret.Data["transport_url"]) templateParameters["PrometheusHost"] = instance.Status.PrometheusHost templateParameters["PrometheusPort"] = instance.Status.PrometheusPort + templateParameters["LokiHost"] = lokiHost + templateParameters["LokiPort"] = 8080 templateParameters["DatabaseConnection"] = fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf", databaseAccount.Spec.UserName, string(dbSecret.Data[mariadbv1.DatabasePasswordSelector]), diff --git a/controllers/cloudkittyapi_controller.go b/controllers/cloudkittyapi_controller.go index de2cb3ff..a47014b1 100644 --- a/controllers/cloudkittyapi_controller.go +++ b/controllers/cloudkittyapi_controller.go @@ -270,6 +270,51 @@ func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl } } } + + // Watch for changes to the client cert secret + if secretName == cloudkitty.ClientCertSecretName { + for _, cr := range apis.Items { + name := client.ObjectKey{ + Namespace: namespace, + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Secret %s is used by CloudKittyAPI CR %s", secretName, cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + if len(result) > 0 { + return result + } + return nil + } + + // Watch for changes to configmaps we don't own. + configMapFn := func(_ context.Context, o client.Object) []reconcile.Request { + var namespace string = o.GetNamespace() + var configMapName string = o.GetName() + result := []reconcile.Request{} + + // get all API CRs + apis := &telemetryv1.CloudKittyAPIList{} + listOpts := []client.ListOption{ + client.InNamespace(namespace), + } + if err := r.Client.List(context.Background(), apis, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve API CRs %v") + return nil + } + + // Watch for changes to the ca cert config map + for _, cr := range apis.Items { + if configMapName == fmt.Sprintf("%s-lokistack-gateway-ca-bundle", cloudkitty.GetOwningCloudKittyName(&cr)) { + name := client.ObjectKey{ + Namespace: namespace, + Name: cr.Name, + } + Log.Info(fmt.Sprintf("ConfigMap %s is used by CloudKittyAPI CR %s", configMapName, cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } if len(result) > 0 { return result } @@ -350,6 +395,8 @@ func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), ). + Watches(&corev1.ConfigMap{}, + handler.EnqueueRequestsFromMapFunc(configMapFn)). Watches(&topologyv1.Topology{}, handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), builder.WithPredicates(predicate.GenerationChangedPredicate{})). @@ -889,6 +936,26 @@ func (r *CloudKittyAPIReconciler) reconcileNormal(ctx context.Context, instance // normal reconcile tasks // + // Add client cert secret hash to all the other config data, so that + // restart is triggered on certificate changes (e.g. when they + // rotate) + _, clientCertHash, err := secret.GetSecret( + ctx, + helper, + cloudkitty.ClientCertSecretName, + instance.Namespace, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + configVars["client-cert"] = env.SetValue(clientCertHash) + // // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. diff --git a/controllers/cloudkittyproc_controller.go b/controllers/cloudkittyproc_controller.go index fbccef63..495f3332 100644 --- a/controllers/cloudkittyproc_controller.go +++ b/controllers/cloudkittyproc_controller.go @@ -244,6 +244,50 @@ func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctr } } } + // Watch for changes to the client cert secret + if secretName == cloudkitty.ClientCertSecretName { + for _, cr := range schedulers.Items { + name := client.ObjectKey{ + Namespace: namespace, + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Secret %s is used by CloudKittyProc CR %s", secretName, cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + if len(result) > 0 { + return result + } + return nil + } + + // Watch for changes to configmaps we don't own. + configMapFn := func(_ context.Context, o client.Object) []reconcile.Request { + var namespace string = o.GetNamespace() + var configMapName string = o.GetName() + result := []reconcile.Request{} + + // get all Proc CRs + procs := &telemetryv1.CloudKittyProcList{} + listOpts := []client.ListOption{ + client.InNamespace(namespace), + } + if err := r.Client.List(context.Background(), procs, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve Proc CRs %v") + return nil + } + + // Watch for changes to the ca cert config map + for _, cr := range procs.Items { + if configMapName == fmt.Sprintf("%s-lokistack-gateway-ca-bundle", cloudkitty.GetOwningCloudKittyName(&cr)) { + name := client.ObjectKey{ + Namespace: namespace, + Name: cr.Name, + } + Log.Info(fmt.Sprintf("ConfigMap %s is used by CloudKittyProc CR %s", configMapName, cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } if len(result) > 0 { return result } @@ -297,6 +341,8 @@ func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctr handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), ). + Watches(&corev1.ConfigMap{}, + handler.EnqueueRequestsFromMapFunc(configMapFn)). Watches(&topologyv1.Topology{}, handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), builder.WithPredicates(predicate.GenerationChangedPredicate{})). @@ -475,6 +521,26 @@ func (r *CloudKittyProcReconciler) reconcileNormal(ctx context.Context, instance return ctrl.Result{}, err } + // Add client cert secret hash to all the other config data, so that + // restart is triggered on certificate changes (e.g. when they + // rotate) + _, clientCertHash, err := secret.GetSecret( + ctx, + helper, + cloudkitty.ClientCertSecretName, + instance.Namespace, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + configVars["client-cert"] = env.SetValue(clientCertHash) + // // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. diff --git a/controllers/metricstorage_controller.go b/controllers/metricstorage_controller.go index a05b4ba8..6db87531 100644 --- a/controllers/metricstorage_controller.go +++ b/controllers/metricstorage_controller.go @@ -31,25 +31,16 @@ import ( discoveryv1 "k8s.io/api/discovery/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" - "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - - "k8s.io/apimachinery/pkg/api/meta" logr "github.com/go-logr/logr" "github.com/openstack-k8s-operators/lib-common/modules/ansible" @@ -93,15 +84,7 @@ var ( ) // MetricStorageReconciler reconciles a MetricStorage object -type MetricStorageReconciler struct { - client.Client - Kclient kubernetes.Interface - Scheme *runtime.Scheme - Controller controller.Controller - Watching []string - RESTMapper meta.RESTMapper - Cache cache.Cache -} +type MetricStorageReconciler utils.ConditionalWatchingReconciler // ConnectionInfo holds information about connection to a compute node type ConnectionInfo struct { @@ -318,7 +301,11 @@ func (r *MetricStorageReconciler) reconcileNormal( // Deploy monitoring stack - err := r.ensureWatches(ctx, "monitoringstacks.monitoring.rhobs", &obov1.MonitoringStack{}, eventHandler) + err := utils.EnsureWatches( + (*utils.ConditionalWatchingReconciler)(r), ctx, + "monitoringstacks.monitoring.rhobs", + &obov1.MonitoringStack{}, eventHandler, helper, + ) if err != nil { instance.Status.Conditions.MarkFalse(telemetryv1.MonitoringStackReadyCondition, condition.Reason("Can't own MonitoringStack resource. The Cluster Observability Operator probably isn't installed"), @@ -366,7 +353,13 @@ func (r *MetricStorageReconciler) reconcileNormal( } return []reconcile.Request{{NamespacedName: name}} } - err = r.ensureWatches(ctx, "prometheuses.monitoring.rhobs", &monv1.Prometheus{}, handler.EnqueueRequestsFromMapFunc(prometheusWatchFn)) + err = utils.EnsureWatches( + (*utils.ConditionalWatchingReconciler)(r), + ctx, "prometheuses.monitoring.rhobs", + &monv1.Prometheus{}, + handler.EnqueueRequestsFromMapFunc(prometheusWatchFn), + helper, + ) if err != nil { instance.Status.Conditions.MarkFalse(telemetryv1.PrometheusReadyCondition, condition.Reason("Can't watch prometheus resource. The Cluster Observability Operator probably isn't installed"), @@ -575,7 +568,13 @@ func (r *MetricStorageReconciler) reconcileNormal( } return []reconcile.Request{{NamespacedName: name}} } - err = r.ensureWatches(ctx, "prometheuses.monitoring.rhobs", &monv1.Prometheus{}, handler.EnqueueRequestsFromMapFunc(prometheusWatchFn)) + err = utils.EnsureWatches( + (*utils.ConditionalWatchingReconciler)(r), + ctx, "prometheuses.monitoring.rhobs", + &monv1.Prometheus{}, + handler.EnqueueRequestsFromMapFunc(prometheusWatchFn), + helper, + ) if err != nil { instance.Status.Conditions.MarkFalse(telemetryv1.PrometheusReadyCondition, condition.Reason("Can't watch prometheus resource. The Cluster Observability Operator probably isn't installed"), @@ -710,7 +709,11 @@ func (r *MetricStorageReconciler) createScrapeConfigs( helper *helper.Helper, ) (ctrl.Result, error) { Log := r.GetLogger(ctx) - err := r.ensureWatches(ctx, "scrapeconfigs.monitoring.rhobs", &monv1alpha1.ScrapeConfig{}, eventHandler) + err := utils.EnsureWatches( + (*utils.ConditionalWatchingReconciler)(r), + ctx, "scrapeconfigs.monitoring.rhobs", + &monv1alpha1.ScrapeConfig{}, eventHandler, helper, + ) if err != nil { instance.Status.Conditions.MarkFalse(telemetryv1.ScrapeConfigReadyCondition, condition.Reason("Can't own ScrapeConfig resource. The Cluster Observability Operator probably isn't installed"), @@ -959,7 +962,11 @@ func (r *MetricStorageReconciler) createDashboardObjects(ctx context.Context, in } // Deploy PrometheusRule for dashboards - err = r.ensureWatches(ctx, "prometheusrules.monitoring.rhobs", &monv1.PrometheusRule{}, eventHandler) + err = utils.EnsureWatches( + (*utils.ConditionalWatchingReconciler)(r), + ctx, "prometheusrules.monitoring.rhobs", + &monv1.PrometheusRule{}, eventHandler, helper, + ) if err != nil { instance.Status.Conditions.MarkFalse(telemetryv1.DashboardPrometheusRuleReadyCondition, condition.Reason("Can't own PrometheusRule resource. The Cluster Observability Operator probably isn't installed"), @@ -1079,43 +1086,6 @@ func (r *MetricStorageReconciler) createDashboardObjects(ctx context.Context, in return ctrl.Result{}, err } -func (r *MetricStorageReconciler) ensureWatches( - ctx context.Context, - name string, - kind client.Object, - handler handler.EventHandler, -) error { - Log := r.GetLogger(ctx) - for _, item := range r.Watching { - if item == name { - // We are already watching the resource - return nil - } - } - u := &unstructured.Unstructured{} - u.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "apiextensions.k8s.io", - Kind: "CustomResourceDefinition", - Version: "v1", - }) - - err := r.Client.Get(context.Background(), client.ObjectKey{ - Name: name, - }, u) - if err != nil { - return err - } - - Log.Info(fmt.Sprintf("Starting to watch %s", name)) - err = r.Controller.Watch(source.Kind(r.Cache, kind), - handler, - ) - if err == nil { - r.Watching = append(r.Watching, name) - } - return err -} - func getComputeNodesConnectionInfo( instance *telemetryv1.MetricStorage, helper *helper.Helper, diff --git a/go.mod b/go.mod index 466c105c..36b2976c 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,9 @@ go 1.21 replace github.com/openstack-k8s-operators/telemetry-operator/api => ./api require ( + github.com/cert-manager/cert-manager v1.14.7 github.com/go-logr/logr v1.4.2 + github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.6 github.com/onsi/ginkgo/v2 v2.20.1 github.com/onsi/gomega v1.34.1 @@ -13,6 +15,7 @@ require ( github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20250513115636-b549982a5d8f github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20250604143452-2260c431b9f1 github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20250423055245-3cb2ae8df6f0 + github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.0 github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20250508141203-be026d3164f7 github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20250415060817-dc849adfa27e github.com/openstack-k8s-operators/telemetry-operator/api v0.3.1-0.20240529090522-c780bd90b147 @@ -22,7 +25,7 @@ require ( k8s.io/api v0.29.15 k8s.io/apimachinery v0.29.15 k8s.io/client-go v0.29.15 - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 + k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 sigs.k8s.io/controller-runtime v0.17.6 ) @@ -67,11 +70,11 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/net v0.28.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.18.0 // indirect - golang.org/x/sys v0.23.0 // indirect - golang.org/x/term v0.23.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.24.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect @@ -84,6 +87,7 @@ require ( k8s.io/component-base v0.29.15 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240322212309-b815d8309940 // indirect + sigs.k8s.io/gateway-api v1.0.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/go.sum b/go.sum index 6b71026f..a77e2c08 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cert-manager/cert-manager v1.14.7 h1:C2L59sMGMdSpd8SPx5qfPAL7ejZaNxJBRd24S7Ws5Ek= +github.com/cert-manager/cert-manager v1.14.7/go.mod h1:0QE/Hzfs2SxNrFFYgFh/d0c0cDfNv9qSrAev2LFt5nM= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -49,6 +51,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gophercloud/gophercloud v1.14.1 h1:DTCNaTVGl8/cFu58O1JwWgis9gtISAFONqpMKNg/Vpw= github.com/gophercloud/gophercloud v1.14.1/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= +github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba h1:P5Wgp2HfGfNPLCPpS+YqquKdrrl4tW0El7VX23D6vtg= +github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba/go.mod h1:OBAgJh0mLYRvziBzBKr4/anrPHqGY9qEfuNXCpnUNi0= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -86,6 +90,8 @@ github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20250604143452 github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20250604143452-2260c431b9f1/go.mod h1:dgYQJbbVaRuP98yZZB3K1rNpqnF54I1HM1ZTaOzPKBY= github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20250423055245-3cb2ae8df6f0 h1:QKmpBfn9zIYcmODrvhnnrOx2CV1Y2t6M8DwNgLbaRbI= github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20250423055245-3cb2ae8df6f0/go.mod h1:0bajRHochTUT6Ecfriw27l3vL0yezVrnUmt3bcIpu4w= +github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.0 h1:cFOyP37qQ9T1D6mVTCwuPGt86LB4sTErpHT+L1e+VKY= +github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.0/go.mod h1:jgfvFeljXxot0LODLYCmjESxoMXbClXcBcf0DaX4zA0= github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20250508141203-be026d3164f7 h1:c3h1q3fDoit3NmvNL89xUL9A12bJivaTF+IOPEOAwLc= github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20250508141203-be026d3164f7/go.mod h1:UwHXRIrMSPJD3lFqrA4oKmRXVLFQCRkLAj9x6KLEHiQ= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20250508141203-be026d3164f7 h1:IybBq3PrxwdvzAF19TjdMCqbEVkX2p3gIkme/Fju6do= @@ -149,8 +155,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -165,19 +171,19 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -223,10 +229,12 @@ k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240322212309-b815d8309940 h1:qVoMaQV5t62UUvHe16Q3eb2c5HPzLHYzsi0Tu/xLndo= k8s.io/kube-openapi v0.0.0-20240322212309-b815d8309940/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.17.6 h1:12IXsozEsIXWAMRpgRlYS1jjAHQXHtWEOMdULh3DbEw= sigs.k8s.io/controller-runtime v0.17.6/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY= +sigs.k8s.io/gateway-api v1.0.0 h1:iPTStSv41+d9p0xFydll6d7f7MOBGuqXM6p2/zVYMAs= +sigs.k8s.io/gateway-api v1.0.0/go.mod h1:4cUgr0Lnp5FZ0Cdq8FdRwCvpiWws7LVhLHGIudLlf4c= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= diff --git a/main.go b/main.go index 7905d299..80690fa9 100644 --- a/main.go +++ b/main.go @@ -28,6 +28,7 @@ import ( "k8s.io/client-go/kubernetes" _ "k8s.io/client-go/plugin/pkg/client/auth" + certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -39,6 +40,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + lokistackv1 "github.com/grafana/loki/operator/api/loki/v1" networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" heatv1 "github.com/openstack-k8s-operators/heat-operator/api/v1beta1" memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" @@ -81,6 +83,8 @@ func init() { utilruntime.Must(rabbitmqclusterv1.AddToScheme(scheme)) utilruntime.Must(networkv1.AddToScheme(scheme)) utilruntime.Must(topologyv1.AddToScheme(scheme)) + utilruntime.Must(lokistackv1.AddToScheme(scheme)) + utilruntime.Must(certmgrv1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } @@ -199,9 +203,11 @@ func main() { } if err = (&controllers.CloudKittyReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Kclient: kclient, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + RESTMapper: mgr.GetRESTMapper(), + Cache: mgr.GetCache(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create CloudKitty controller") os.Exit(1) diff --git a/pkg/cloudkitty/cert.go b/pkg/cloudkitty/cert.go new file mode 100644 index 00000000..4a72e4cb --- /dev/null +++ b/pkg/cloudkitty/cert.go @@ -0,0 +1,67 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkitty + +import ( + "fmt" + + certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + certmgrmetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + "github.com/openstack-k8s-operators/lib-common/modules/certmanager" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" +) + +// Certificate defines a client certificate for communication between cloudkitty and loki +func Certificate( + instance *telemetryv1.CloudKitty, + labels map[string]string, + issuer *certmgrv1.Issuer, +) *certmgrv1.Certificate { + cert := certmanager.Cert( + ClientCertSecretName, + instance.Namespace, + labels, + certmgrv1.CertificateSpec{ + CommonName: fmt.Sprintf("%s.%s.svc", instance.Name, instance.Namespace), + DNSNames: []string{ + fmt.Sprintf("%s.%s.svc", instance.Name, instance.Namespace), + fmt.Sprintf("%s.%s.svc.cluster.local", instance.Name, instance.Namespace), + }, + SecretName: ClientCertSecretName, + Subject: &certmgrv1.X509Subject{ + OrganizationalUnits: []string{ + instance.Name, + }, + }, + PrivateKey: &certmgrv1.CertificatePrivateKey{ + Algorithm: "RSA", + Size: 3072, + }, + Usages: []certmgrv1.KeyUsage{ + certmgrv1.UsageDigitalSignature, + certmgrv1.UsageKeyEncipherment, + certmgrv1.UsageClientAuth, + }, + IssuerRef: certmgrmetav1.ObjectReference{ + Name: issuer.Name, + Kind: issuer.Kind, + Group: issuer.GroupVersionKind().Group, + }, + }, + ) + return cert +} diff --git a/pkg/cloudkitty/const.go b/pkg/cloudkitty/const.go index 8b372471..dc6589ba 100644 --- a/pkg/cloudkitty/const.go +++ b/pkg/cloudkitty/const.go @@ -52,6 +52,11 @@ const ( // PrometheusEndpointSecret - The name of the secret that contains the Prometheus endpoint configuration. PrometheusEndpointSecret = "metric-storage-prometheus-endpoint" + + ClientCertSecretName = "cert-cloudkitty-client-internal" + + CaConfigmapName = "lokistack-ca" + CaConfigmapKey = "ca.crt" ) var ResultRequeue = ctrl.Result{RequeueAfter: NormalDuration} diff --git a/pkg/cloudkitty/dbsync.go b/pkg/cloudkitty/dbsync.go index 5a84523a..28fb3161 100644 --- a/pkg/cloudkitty/dbsync.go +++ b/pkg/cloudkitty/dbsync.go @@ -41,7 +41,7 @@ func DbSyncJob(instance *telemetryv1.CloudKitty, labels map[string]string, annot args = append(args, dbSyncCommand) // create Volume and VolumeMounts - volumes := GetVolumes("cloudkitty") + volumes := GetVolumes(instance.Name) volumeMounts := GetVolumeMounts("cloudkitty-dbsync") // add CA cert if defined if instance.Spec.CloudKittyAPI.TLS.CaBundleSecretName != "" { diff --git a/pkg/cloudkitty/lokistack.go b/pkg/cloudkitty/lokistack.go new file mode 100644 index 00000000..c7604dc1 --- /dev/null +++ b/pkg/cloudkitty/lokistack.go @@ -0,0 +1,118 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkitty + +import ( + "fmt" + + lokistackv1 "github.com/grafana/loki/operator/api/loki/v1" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// LokiStack defines a lokistack for cloudkitty +func LokiStack( + instance *telemetryv1.CloudKitty, + labels map[string]string, +) *lokistackv1.LokiStack { + lokiStack := &lokistackv1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-lokistack", instance.Name), + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: lokistackv1.LokiStackSpec{ + // TODO: What size do we even want? I assume something + // smallish since only rating interact with this + Size: lokistackv1.LokiStackSizeType("1x.demo"), + Storage: instance.Spec.S3StorageConfig, + StorageClassName: instance.Spec.StorageClass, + Tenants: &lokistackv1.TenantsSpec{ + Mode: lokistackv1.Static, + Authentication: []lokistackv1.AuthenticationSpec{ + { + TenantName: instance.Name, + TenantID: instance.Name, + MTLS: &lokistackv1.MTLSSpec{ + CA: &lokistackv1.CASpec{ + CAKey: CaConfigmapKey, + CA: fmt.Sprintf("%s-%s", instance.Name, CaConfigmapName), + }, + }, + }, + }, + Authorization: &lokistackv1.AuthorizationSpec{ + // TODO: Determine what exactly this does and what's needed here + Roles: []lokistackv1.RoleSpec{ + { + Name: "cloudkitty-logs", + Resources: []string{ + "logs", + }, + Tenants: []string{ + "cloudkitty", + }, + Permissions: []lokistackv1.PermissionType{ + lokistackv1.Write, + lokistackv1.Read, + }, + }, + { + Name: "cluster-reader", + Resources: []string{ + "logs", + }, + Tenants: []string{ + "cloudkitty", + }, + Permissions: []lokistackv1.PermissionType{ + lokistackv1.Read, + }, + }, + }, + RoleBindings: []lokistackv1.RoleBindingsSpec{ + { + Name: "cloudkitty-logs", + Subjects: []lokistackv1.Subject{ + { + Name: "cloudkitty", + Kind: lokistackv1.Group, + }, + }, + Roles: []string{ + "cloudkitty-logs", + }, + }, + { + Name: "cluster-reader", + Subjects: []lokistackv1.Subject{ + { + Name: "cloudkitty-logs-admin", + Kind: lokistackv1.Group, + }, + }, + Roles: []string{ + "cluster-reader", + }, + }, + }, + }, + }, + }, + } + return lokiStack +} diff --git a/pkg/cloudkitty/storageinit.go b/pkg/cloudkitty/storageinit.go index 73b05cd8..05418f99 100644 --- a/pkg/cloudkitty/storageinit.go +++ b/pkg/cloudkitty/storageinit.go @@ -40,7 +40,7 @@ func StorageInitJob(instance *telemetryv1.CloudKitty, labels map[string]string, args := []string{"-c", storageInitCommand} // create Volume and VolumeMounts - volumes := GetVolumes("cloudkitty") + volumes := GetVolumes(instance.Name) volumeMounts := GetVolumeMounts("cloudkitty-storageinit") // add CA cert if defined if instance.Spec.CloudKittyAPI.TLS.CaBundleSecretName != "" { diff --git a/pkg/cloudkitty/volumes.go b/pkg/cloudkitty/volumes.go index 97ed4ab5..82b3d738 100644 --- a/pkg/cloudkitty/volumes.go +++ b/pkg/cloudkitty/volumes.go @@ -9,6 +9,8 @@ var ( scriptMode int32 = 0740 // configMode is the 640 permissions mode configMode int32 = 0640 + // certMode is the 400 permissions mode + certMode int32 = 0400 ) // GetVolumes - service volumes @@ -30,6 +32,28 @@ func GetVolumes(name string) []corev1.Volume { SecretName: name + "-config-data", }, }, + }, { + Name: "certs", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: ClientCertSecretName, + }, + }, + }, { + ConfigMap: &corev1.ConfigMapProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: name + "-lokistack-gateway-ca-bundle", + }, + }, + }, + }, + DefaultMode: &certMode, + }, + }, }, } } @@ -53,5 +77,10 @@ func GetVolumeMounts(serviceName string) []corev1.VolumeMount { SubPath: serviceName + "-config.json", ReadOnly: true, }, + { + Name: "certs", + MountPath: "/var/lib/openstack/loki-certs", + ReadOnly: true, + }, } } diff --git a/pkg/cloudkittyapi/statefulset.go b/pkg/cloudkittyapi/statefulset.go index 94d4346c..431a662d 100644 --- a/pkg/cloudkittyapi/statefulset.go +++ b/pkg/cloudkittyapi/statefulset.go @@ -75,7 +75,7 @@ func StatefulSet( // create Volume and VolumeMounts volumes := GetVolumes(cloudkitty.GetOwningCloudKittyName(instance), instance.Name) - volumeMounts := GetVolumeMounts(instance.Name) + volumeMounts := GetVolumeMounts() // add CA cert if defined if instance.Spec.TLS.CaBundleSecretName != "" { diff --git a/pkg/cloudkittyapi/volumes.go b/pkg/cloudkittyapi/volumes.go index 6d9f36ab..4108fc3d 100644 --- a/pkg/cloudkittyapi/volumes.go +++ b/pkg/cloudkittyapi/volumes.go @@ -31,7 +31,7 @@ func GetVolumes(parentName string, name string) []corev1.Volume { } // GetVolumeMounts - CloudKitty API VolumeMounts -func GetVolumeMounts(parentName string) []corev1.VolumeMount { +func GetVolumeMounts() []corev1.VolumeMount { volumeMounts := []corev1.VolumeMount{ { Name: "config-data-custom", @@ -41,7 +41,7 @@ func GetVolumeMounts(parentName string) []corev1.VolumeMount { GetLogVolumeMount(), } - return append(cloudkitty.GetVolumeMounts(parentName), volumeMounts...) + return append(cloudkitty.GetVolumeMounts(cloudkitty.ServiceName+"-api"), volumeMounts...) } // GetLogVolumeMount - CloudKitty API LogVolumeMount diff --git a/pkg/cloudkittyproc/statefulset.go b/pkg/cloudkittyproc/statefulset.go index a568be4d..b04baaa0 100644 --- a/pkg/cloudkittyproc/statefulset.go +++ b/pkg/cloudkittyproc/statefulset.go @@ -75,7 +75,7 @@ func StatefulSet( envVars["CONFIG_HASH"] = env.SetValue(configHash) volumes := GetVolumes(cloudkitty.GetOwningCloudKittyName(instance), instance.Name) - volumeMounts := GetVolumeMounts(instance.Name) + volumeMounts := GetVolumeMounts() // Add the CA bundle if instance.Spec.TLS.CaBundleSecretName != "" { diff --git a/pkg/cloudkittyproc/volumes.go b/pkg/cloudkittyproc/volumes.go index 51ff46a7..22fdc822 100644 --- a/pkg/cloudkittyproc/volumes.go +++ b/pkg/cloudkittyproc/volumes.go @@ -25,7 +25,7 @@ func GetVolumes(parentName string, name string) []corev1.Volume { } // GetVolumeMounts - CloudKitty API VolumeMounts -func GetVolumeMounts(parentName string) []corev1.VolumeMount { +func GetVolumeMounts() []corev1.VolumeMount { volumeMounts := []corev1.VolumeMount{ { Name: "config-data-custom", @@ -34,5 +34,5 @@ func GetVolumeMounts(parentName string) []corev1.VolumeMount { }, } - return append(cloudkitty.GetVolumeMounts(parentName), volumeMounts...) + return append(cloudkitty.GetVolumeMounts(cloudkitty.ServiceName+"-proc"), volumeMounts...) } diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index e19c58ca..0cde4c6e 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -18,14 +18,34 @@ package utils import ( "context" + "fmt" k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/source" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" ) +type ConditionalWatchingReconciler struct { + client.Client + Kclient kubernetes.Interface + Scheme *runtime.Scheme + Controller controller.Controller + Watching []string + RESTMapper meta.RESTMapper + Cache cache.Cache +} + // EnsureDeleted - Delete the object which in turn will clean the sub resources func EnsureDeleted(ctx context.Context, helper *helper.Helper, obj client.Object) (ctrl.Result, error) { key := client.ObjectKeyFromObject(obj) @@ -44,3 +64,42 @@ func EnsureDeleted(ctx context.Context, helper *helper.Helper, obj client.Object return ctrl.Result{}, nil } + +func EnsureWatches( + r *ConditionalWatchingReconciler, + ctx context.Context, + name string, + kind client.Object, + handler handler.EventHandler, + helper *helper.Helper, +) error { + Log := helper.GetLogger() + for _, item := range r.Watching { + if item == name { + // We are already watching the resource + return nil + } + } + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "apiextensions.k8s.io", + Kind: "CustomResourceDefinition", + Version: "v1", + }) + + err := r.Client.Get(context.Background(), client.ObjectKey{ + Name: name, + }, u) + if err != nil { + return err + } + + Log.Info(fmt.Sprintf("Starting to watch %s", name)) + err = r.Controller.Watch(source.Kind(r.Cache, kind), + handler, + ) + if err == nil { + r.Watching = append(r.Watching, name) + } + return err +} diff --git a/templates/cloudkitty/config/cloudkitty-api-config.json b/templates/cloudkitty/config/cloudkitty-api-config.json index 107cfa1a..9ff96965 100644 --- a/templates/cloudkitty/config/cloudkitty-api-config.json +++ b/templates/cloudkitty/config/cloudkitty-api-config.json @@ -63,6 +63,13 @@ "perm": "0640", "optional": true, "merge": true + }, + { + "source": "/var/lib/openstack/loki-certs/*", + "dest": "/etc/cloudkitty/certs/", + "owner": "cloudkitty:cloudkitty", + "perm": "0400", + "merge": true } ], "permissions": [ diff --git a/templates/cloudkitty/config/cloudkitty-proc-config.json b/templates/cloudkitty/config/cloudkitty-proc-config.json index 410ed9d3..b6d83aef 100644 --- a/templates/cloudkitty/config/cloudkitty-proc-config.json +++ b/templates/cloudkitty/config/cloudkitty-proc-config.json @@ -12,6 +12,13 @@ "dest": "/etc/cloudkitty/metrics.yaml", "owner": "cloudkitty", "perm": "0600" + }, + { + "source": "/var/lib/openstack/loki-certs/*", + "dest": "/etc/cloudkitty/certs/", + "owner": "cloudkitty:cloudkitty", + "perm": "0400", + "merge": true } ] } diff --git a/templates/cloudkitty/config/cloudkitty.conf b/templates/cloudkitty/config/cloudkitty.conf index 318ed398..5faf0864 100644 --- a/templates/cloudkitty/config/cloudkitty.conf +++ b/templates/cloudkitty/config/cloudkitty.conf @@ -64,7 +64,10 @@ version = 2 backend = loki [storage_loki] -url = http://loki:3100/loki/api/v1 +url = https://{{ .LokiHost }}:{{ .LokiPort }}/api/logs/v1/cloudkitty/loki/api/v1 +ca_file = /etc/cloudkitty/certs/service-ca.crt +cert_file = /etc/cloudkitty/certs/tls.crt +key_file = /etc/cloudkitty/certs/tls.key [database] connection = {{ .DatabaseConnection }} diff --git a/tests/kuttl/suites/cloudkitty/config.yaml b/tests/kuttl/suites/cloudkitty/config.yaml new file mode 100644 index 00000000..bd1d84da --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/config.yaml @@ -0,0 +1,14 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestSuite +reportFormat: JSON +reportName: kuttl-cloudkitty-results +namespace: telemetry-kuttl-cloudkitty +# we could set this lower, but the initial image pull can take a while +timeout: 300 +parallel: 1 +skipDelete: true +testDirs: + - tests/kuttl/suites/cloudkitty/ +suppress: + - events +artifactsDir: tests/kuttl/suites/cloudkitty/output diff --git a/tests/kuttl/suites/cloudkitty/deps/OpenStackControlPlane.yaml b/tests/kuttl/suites/cloudkitty/deps/OpenStackControlPlane.yaml new file mode 100644 index 00000000..f1b9f207 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/OpenStackControlPlane.yaml @@ -0,0 +1,27 @@ +apiVersion: core.openstack.org/v1beta1 +kind: OpenStackControlPlane +metadata: + name: openstack +spec: + storageClass: "crc-csi-hostpath-provisioner" + keystone: + template: + databaseInstance: openstack + secret: osp-secret + ironic: + enabled: false + template: + ironicConductors: [] + manila: + enabled: false + template: + manilaShares: {} + horizon: + enabled: false + nova: + enabled: false + placement: + template: + databaseInstance: openstack + secret: osp-secret + dataplane: diff --git a/tests/kuttl/suites/cloudkitty/deps/infra.yaml b/tests/kuttl/suites/cloudkitty/deps/infra.yaml new file mode 100644 index 00000000..7c47e716 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/infra.yaml @@ -0,0 +1,42 @@ +apiVersion: core.openstack.org/v1beta1 +kind: OpenStackControlPlane +metadata: + name: openstack +spec: + mariadb: + enabled: false + templates: + openstack: + replicas: 0 + openstack-cell1: + replicas: 0 + galera: + enabled: true + templates: + openstack: + replicas: 1 + storageRequest: 500M + openstack-cell1: + replicas: 1 + storageRequest: 500M + secret: osp-secret + secret: osp-secret + rabbitmq: + templates: + rabbitmq: + replicas: 1 + image: quay.io/podified-antelope-centos9/openstack-rabbitmq@sha256:41c36935b8b8cd3c5e490d1c03549ba2c0e8ddff50238fb2400d74613aa2e087 + rabbitmq-cell1: + replicas: 1 + memcached: + templates: + memcached: + replicas: 1 + ovn: + enabled: false + template: + ovnController: + external-ids: + ovn-encap-type: geneve + ovs: + enabled: false diff --git a/tests/kuttl/suites/cloudkitty/deps/kustomization.yaml b/tests/kuttl/suites/cloudkitty/deps/kustomization.yaml new file mode 100644 index 00000000..b366cb73 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/kustomization.yaml @@ -0,0 +1,50 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: telemetry-kuttl-cloudkitty + +secretGenerator: +- literals: + - AdminPassword=password + - DbRootPassword=password + - DatabasePassword=password + - KeystoneDatabasePassword=password + - PlacementPassword=password + - PlacementDatabasePassword=password + - GlancePassword=password + - GlanceDatabasePassword=password + - NeutronPassword=password + - NeutronDatabasePassword=password + - NovaPassword=password + - NovaAPIDatabasePassword=password + - NovaCell0DatabasePassword=password + - NovaCell1DatabasePassword=password + - AodhPassword=password + - AodhDatabasePassword=password + - CeilometerPassword=password + - CeilometerDatabasePassword=password + - HeatPassword=password + - HeatDatabasePassword=password + - HeatAuthEncryptionKey=66699966699966600666999666999666 + - MetadataSecret=42 + - CloudKittyPassword=password + name: osp-secret +generatorOptions: + disableNameSuffixHash: true + labels: + type: osp-secret + +resources: +- namespace.yaml +- OpenStackControlPlane.yaml +- loki-s3-secret.yaml + +patches: +- patch: |- + apiVersion: core.openstack.org/v1beta1 + kind: OpenStackControlPlane + metadata: + name: openstack + spec: + secret: osp-secret +- path: infra.yaml +- path: telemetry.yaml diff --git a/tests/kuttl/suites/cloudkitty/deps/loki-operator.yaml b/tests/kuttl/suites/cloudkitty/deps/loki-operator.yaml new file mode 100644 index 00000000..5b3d937a --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/loki-operator.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-operators-redhat + labels: + name: openshift-operators-redhat +--- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: loki-operator + namespace: openshift-operators-redhat +spec: + upgradeStrategy: Default +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: loki-operator + namespace: openshift-operators-redhat +spec: + channel: stable-6.1 + installPlanApproval: Automatic + name: loki-operator + source: redhat-operators + sourceNamespace: openshift-marketplace diff --git a/tests/kuttl/suites/cloudkitty/deps/loki-s3-secret.yaml b/tests/kuttl/suites/cloudkitty/deps/loki-s3-secret.yaml new file mode 100644 index 00000000..b176d7f9 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/loki-s3-secret.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: logging-loki-s3 +stringData: + access_key_id: minio + access_key_secret: minio123 + bucketnames: loki + endpoint: http://minio.minio-dev.svc.cluster.local:9000 diff --git a/tests/kuttl/suites/cloudkitty/deps/minio.yaml b/tests/kuttl/suites/cloudkitty/deps/minio.yaml new file mode 100644 index 00000000..f1854ff4 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/minio.yaml @@ -0,0 +1,99 @@ +--- +# Deploys a new Namespace for the MinIO Pod +apiVersion: v1 +kind: Namespace +metadata: + name: minio-dev # Change this value if you want a different namespace name + labels: + name: minio-dev # Change this value to match metadata.name +--- +# Deploys a new MinIO Pod into the metadata.namespace Kubernetes namespace +# +apiVersion: v1 +kind: Pod +metadata: + labels: + app: minio + name: minio + namespace: minio-dev # Change this value to match the namespace metadata.name +spec: + containers: + - name: minio + image: quay.io/minio/minio:latest + command: + - /bin/bash + - -c + - | + mkdir -p /data/loki && \ + minio server /data + env: + - name: MINIO_ACCESS_KEY + value: minio + - name: MINIO_SECRET_KEY + value: minio123 + volumeMounts: + - mountPath: /data + name: storage # Corresponds to the `spec.volumes` Persistent Volume + volumes: + - name: storage + persistentVolumeClaim: + claimName: minio-pvc +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio-pvc + namespace: minio-dev +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: crc-csi-hostpath-provisioner +--- +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: minio-dev +spec: + selector: + app: minio + ports: + - name: api + protocol: TCP + port: 9000 + - name: console + protocol: TCP + port: 9090 +--- +kind: Route +apiVersion: route.openshift.io/v1 +metadata: + name: minio-console + namespace: minio-dev +spec: + host: console-minio-dev.apps-crc.testing + to: + kind: Service + name: minio + weight: 100 + port: + targetPort: console + wildcardPolicy: None +--- +kind: Route +apiVersion: route.openshift.io/v1 +metadata: + name: minio-api + namespace: minio-dev +spec: + host: api-minio-dev.apps-crc.testing + to: + kind: Service + name: minio + weight: 100 + port: + targetPort: api + wildcardPolicy: None diff --git a/tests/kuttl/suites/cloudkitty/deps/namespace.yaml b/tests/kuttl/suites/cloudkitty/deps/namespace.yaml new file mode 100644 index 00000000..63c85762 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: telemetry-kuttl diff --git a/tests/kuttl/suites/cloudkitty/deps/telemetry.yaml b/tests/kuttl/suites/cloudkitty/deps/telemetry.yaml new file mode 100644 index 00000000..5c2714a2 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/telemetry.yaml @@ -0,0 +1,7 @@ +apiVersion: core.openstack.org/v1beta1 +kind: OpenStackControlPlane +metadata: + name: openstack +spec: + telemetry: + enabled: false diff --git a/tests/kuttl/suites/cloudkitty/output/.keep b/tests/kuttl/suites/cloudkitty/output/.keep new file mode 100644 index 00000000..e69de29b diff --git a/tests/kuttl/suites/cloudkitty/tests/00-deps.yaml b/tests/kuttl/suites/cloudkitty/tests/00-deps.yaml new file mode 100644 index 00000000..f850b416 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/tests/00-deps.yaml @@ -0,0 +1,9 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + oc apply -f ../deps/loki-operator.yaml + until oc api-resources | grep -q grafana; do sleep 1; done + - script: | + oc apply -f ../deps/minio.yaml + oc wait --for='jsonpath={.status.conditions[?(@.type=="Ready")].status}=True' pod/minio -n minio-dev diff --git a/tests/kuttl/suites/cloudkitty/tests/01-assert.yaml b/tests/kuttl/suites/cloudkitty/tests/01-assert.yaml new file mode 100644 index 00000000..ad927419 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/tests/01-assert.yaml @@ -0,0 +1,81 @@ +apiVersion: loki.grafana.com/v1 +kind: LokiStack +metadata: + name: telemetry-kuttl-cloudkitty-lokistack + ownerReferences: + - kind: CloudKitty + name: telemetry-kuttl-cloudkitty +spec: + tenants: + authentication: + - tenantId: telemetry-kuttl-cloudkitty + tenantName: telemetry-kuttl-cloudkitty +# NOTE: It's hard to assert LokiStack condition with kuttl-tests, because +# their number can change even when LokiStack as whole is actually ready. +# And because kuttl-tests will compare the number of conditions defined +# here vs. the number of conditions inside the real CR, it could fail +# even when it shouldn't. +# But LokiStack condition is being copied into the master CloudKitty CR, +# so we can ensure that LokiStack is ready by checking that Cloudkitty +# is Ready +--- +apiVersion: telemetry.openstack.org/v1beta1 +kind: CloudKitty +metadata: + name: telemetry-kuttl-cloudkitty +status: + conditions: + - status: "True" + type: Ready + - status: "True" + type: CloudKittyAPIReady + - status: "True" + type: CloudKittyClientCertReady + - status: "True" + type: CloudKittyLokiStackReady + - status: "True" + type: CloudKittyProcReady + - status: "True" + type: CloudKittyStorageInitReady + - status: "True" + type: DBReady + - status: "True" + type: DBSyncReady + - status: "True" + type: InputReady + - status: "True" + type: MariaDBAccountReady + - status: "True" + type: MemcachedReady + - status: "True" + type: NetworkAttachmentsReady + - status: "True" + type: RabbitMqTransportURLReady + - status: "True" + type: RoleBindingReady + - status: "True" + type: RoleReady + - status: "True" + type: ServiceAccountReady + - status: "True" + type: ServiceConfigReady +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: cert-cloudkitty-client-internal + ownerReferences: + - kind: CloudKitty + name: telemetry-kuttl-cloudkitty +spec: + subject: + organizationalUnits: + - telemetry-kuttl-cloudkitty + usages: + - digital signature + - key encipherment + - client auth +status: + conditions: + - status: "True" + type: Ready diff --git a/tests/kuttl/suites/cloudkitty/tests/01-deploy.yaml b/tests/kuttl/suites/cloudkitty/tests/01-deploy.yaml new file mode 100644 index 00000000..135146e2 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/tests/01-deploy.yaml @@ -0,0 +1,54 @@ +apiVersion: telemetry.openstack.org/v1beta1 +kind: CloudKitty +metadata: + name: telemetry-kuttl-cloudkitty +spec: + apiTimeout: 0 + cloudKittyAPI: + # TODO: This should go away once CK images are taken from openstackversions + containerImage: quay.io/jwysogla/cloudkitty-api:latest + override: + service: + internal: + metadata: + labels: + osctlplane: "" + osctlplane-service: telemetry + public: + metadata: + labels: + osctlplane: "" + osctlplane-service: telemetry + replicas: 1 + resources: {} + tls: + api: + internal: {} + public: {} + caBundleSecretName: combined-ca-bundle + cloudKittyProc: + # TODO: This should go away once CK images are taken from openstackversions + containerImage: quay.io/jwysogla/cloudkitty-processor:latest + replicas: 1 + resources: {} + tls: + caBundleSecretName: combined-ca-bundle + databaseAccount: cloudkitty + databaseInstance: openstack + memcachedInstance: memcached + passwordSelector: + aodhService: AodhPassword + ceilometerService: CeilometerPassword + cloudKittyService: CloudKittyPassword + preserveJobs: false + rabbitMqClusterName: rabbitmq + s3StorageConfig: + schemas: + - effectiveDate: "2024-11-18" + version: v13 + secret: + name: logging-loki-s3 + type: s3 + secret: osp-secret + serviceUser: cloudkitty + storageClass: local-storage From 60038aaf69c88026cca8819a72d7a06cdbdd9272 Mon Sep 17 00:00:00 2001 From: Jaromir Wysoglad Date: Thu, 18 Sep 2025 11:55:16 -0400 Subject: [PATCH 10/87] [cloudkitty] Add missing rbac annotations --- config/rbac/role.yaml | 32 ++++++++++++++++++++++++++++ controllers/cloudkitty_controller.go | 4 ++++ 2 files changed, 36 insertions(+) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6fabb4df..7bbc14ea 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -58,6 +58,26 @@ rules: - patch - update - watch +- apiGroups: + - cert-manager.io + resources: + - certificates + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - cert-manager.io + resources: + - issuers + verbs: + - get + - list + - watch - apiGroups: - cloudkitty.openstack.org resources: @@ -181,6 +201,18 @@ rules: - patch - update - watch +- apiGroups: + - loki.grafana.com + resources: + - lokistacks + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - mariadb.openstack.org resources: diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index cb7a9ea8..3d431be1 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -104,6 +104,7 @@ func (r *CloudKittyReconciler) GetLogger(ctx context.Context) logr.Logger { // +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyprocs/status,verbs=get;update;patch // +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyprocs/finalizers,verbs=update;patch // +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;create;update;patch;delete;watch // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;create;update;patch;delete;watch // +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbdatabases,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbaccounts,verbs=get;list;watch;create;update;patch;delete @@ -112,6 +113,9 @@ func (r *CloudKittyReconciler) GetLogger(ctx context.Context) logr.Logger { // +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneapis,verbs=get;list;watch // +kubebuilder:rbac:groups=rabbitmq.openstack.org,resources=transporturls,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch +// +kubebuilder:rbac:groups=cert-manager.io,resources=certificates,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=cert-manager.io,resources=issuers,verbs=get;list;watch +// +kubebuilder:rbac:groups=loki.grafana.com,resources=lokistacks,verbs=get;list;watch;create;update;patch;delete // service account, role, rolebinding // +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch From 4674fc01ba6d1e0ac05517939af622bd24998299 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Wed, 17 Sep 2025 15:34:10 -0400 Subject: [PATCH 11/87] [cloudkitty] Add a default value for cloudkitty.s3StorageConfig The s3StorageConfig.secret is passed to Lokistack, which requires some value be set. The default is needed, or else there will be an error when it is not configured. The default is set to a reasonable value for the secret name (cloudkitty-loki-s3) and the type (s3), which match the values we're planning on using for the CI environment. Lines added: 1 Lines generated: 16 --- api/bases/telemetry.openstack.org_cloudkitties.yaml | 4 ++++ api/bases/telemetry.openstack.org_telemetries.yaml | 4 ++++ api/v1beta1/cloudkitty_types.go | 1 + config/crd/bases/telemetry.openstack.org_cloudkitties.yaml | 4 ++++ config/crd/bases/telemetry.openstack.org_telemetries.yaml | 4 ++++ 5 files changed, 17 insertions(+) diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml index 15cfcf46..8911ed80 100644 --- a/api/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -576,6 +576,10 @@ spec: Needed to request a transportURL that is created and used in CloudKitty type: string s3StorageConfig: + default: + secret: + name: cloudkitty-loki-s3 + type: s3 description: S3 related configuration passed to Loki properties: schemas: diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index 72556e5d..f335891d 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -1141,6 +1141,10 @@ spec: Needed to request a transportURL that is created and used in CloudKitty type: string s3StorageConfig: + default: + secret: + name: cloudkitty-loki-s3 + type: s3 description: S3 related configuration passed to Loki properties: schemas: diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index b22592dd..baa82a7f 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -109,6 +109,7 @@ type CloudKittySpecBase struct { // S3 related configuration passed to Loki // +kubebuilder:validation:Optional + // +kubebuilder:default={secret: {name: "cloudkitty-loki-s3", type: "s3"}} S3StorageConfig lokistackv1.ObjectStorageSpec `json:"s3StorageConfig"` // Storage class used for Loki diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml index 15cfcf46..8911ed80 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -576,6 +576,10 @@ spec: Needed to request a transportURL that is created and used in CloudKitty type: string s3StorageConfig: + default: + secret: + name: cloudkitty-loki-s3 + type: s3 description: S3 related configuration passed to Loki properties: schemas: diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index 72556e5d..f335891d 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -1141,6 +1141,10 @@ spec: Needed to request a transportURL that is created and used in CloudKitty type: string s3StorageConfig: + default: + secret: + name: cloudkitty-loki-s3 + type: s3 description: S3 related configuration passed to Loki properties: schemas: From d5203256414027143921489aecf001f12790a137 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Wed, 10 Sep 2025 10:30:42 -0400 Subject: [PATCH 12/87] [cloudkitty] Update probes and mounts for CloudKitty The healthcheck for cloudkitty-proc/probe was failing because the CA was not trusted. The TLS-related file were not available in the expected location in the container. The kolla_set_config command is used so that the mounted config files get copied into the expected location for the healthcheck. The command was updated to use 'bash -c' so that both the kolla_set_config and the healthcheck command would be run on startup. [API] Update script used in uwsgi template The script location changed between antelope and master containers. In both cases, it exists in the same alternative location Update initial delay on liveness and readiness probes This give a chance for the api to start up before the first healthcheck [API] Add metrics.yaml into the api pod --- pkg/cloudkittyapi/statefulset.go | 6 +-- pkg/cloudkittyapi/volumes.go | 2 +- pkg/cloudkittyproc/statefulset.go | 18 ++++---- pkg/cloudkittyproc/volumes.go | 4 +- templates/cloudkitty/bin/healthcheck.py | 2 +- .../config/cloudkitty-api-config.json | 22 +++++++--- .../config/cloudkitty-proc-config.json | 43 ++++++++++++++++++- .../cloudkitty/config/wsgi-cloudkitty.conf | 2 +- 8 files changed, 76 insertions(+), 23 deletions(-) diff --git a/pkg/cloudkittyapi/statefulset.go b/pkg/cloudkittyapi/statefulset.go index 94d4346c..5be9cd37 100644 --- a/pkg/cloudkittyapi/statefulset.go +++ b/pkg/cloudkittyapi/statefulset.go @@ -48,14 +48,14 @@ func StatefulSet( livenessProbe := &corev1.Probe{ // TODO might need tuning TimeoutSeconds: 5, - PeriodSeconds: 3, - InitialDelaySeconds: 5, + PeriodSeconds: 5, + InitialDelaySeconds: 30, } readinessProbe := &corev1.Probe{ // TODO might need tuning TimeoutSeconds: 5, PeriodSeconds: 5, - InitialDelaySeconds: 5, + InitialDelaySeconds: 30, } args := []string{"-c", ServiceCommand} diff --git a/pkg/cloudkittyapi/volumes.go b/pkg/cloudkittyapi/volumes.go index 6d9f36ab..eab449ed 100644 --- a/pkg/cloudkittyapi/volumes.go +++ b/pkg/cloudkittyapi/volumes.go @@ -35,7 +35,7 @@ func GetVolumeMounts(parentName string) []corev1.VolumeMount { volumeMounts := []corev1.VolumeMount{ { Name: "config-data-custom", - MountPath: "/etc/cloudkitty/cloudkitty.conf.d", + MountPath: "/var/lib/openstack/service-config/", ReadOnly: true, }, GetLogVolumeMount(), diff --git a/pkg/cloudkittyproc/statefulset.go b/pkg/cloudkittyproc/statefulset.go index a568be4d..c0e17521 100644 --- a/pkg/cloudkittyproc/statefulset.go +++ b/pkg/cloudkittyproc/statefulset.go @@ -60,15 +60,12 @@ func StatefulSet( } args := []string{"-c", ServiceCommand} - var probeCommand []string + var probeCommand string livenessProbe.HTTPGet = &corev1.HTTPGetAction{ Port: intstr.FromInt(8080), } startupProbe.HTTPGet = livenessProbe.HTTPGet - probeCommand = []string{ - "/var/lib/openstack/bin/healthcheck.py", - "/etc/cloudkitty/cloudkitty.conf.d/cloudkitty.conf", - } + probeCommand = "/usr/local/bin/kolla_set_configs && /var/lib/openstack/bin/healthcheck.py --config-dir /etc/cloudkitty/cloudkitty.conf.d/" envVars := map[string]env.Setter{} envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") @@ -119,14 +116,19 @@ func StatefulSet( StartupProbe: startupProbe, }, { - Name: "probe", - Command: probeCommand, - Image: instance.Spec.ContainerImage, + Name: "probe", + Command: []string{ + "/bin/bash", + }, + Args: []string{"-c", probeCommand}, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + Image: instance.Spec.ContainerImage, SecurityContext: &corev1.SecurityContext{ RunAsUser: &cloudKittyUser, //RunAsGroup: &cloudKittyGroup, }, VolumeMounts: volumeMounts, + Resources: instance.Spec.Resources, }, }, Volumes: volumes, diff --git a/pkg/cloudkittyproc/volumes.go b/pkg/cloudkittyproc/volumes.go index 51ff46a7..69edd93a 100644 --- a/pkg/cloudkittyproc/volumes.go +++ b/pkg/cloudkittyproc/volumes.go @@ -24,12 +24,12 @@ func GetVolumes(parentName string, name string) []corev1.Volume { return append(cloudkitty.GetVolumes(parentName), volumes...) } -// GetVolumeMounts - CloudKitty API VolumeMounts +// GetVolumeMounts - CloudKitty Processor VolumeMounts func GetVolumeMounts(parentName string) []corev1.VolumeMount { volumeMounts := []corev1.VolumeMount{ { Name: "config-data-custom", - MountPath: "/etc/cloudkitty/cloudkitty.conf.d", + MountPath: "/var/lib/openstack/service-config/", ReadOnly: true, }, } diff --git a/templates/cloudkitty/bin/healthcheck.py b/templates/cloudkitty/bin/healthcheck.py index 906f688d..3ab36078 100755 --- a/templates/cloudkitty/bin/healthcheck.py +++ b/templates/cloudkitty/bin/healthcheck.py @@ -117,7 +117,7 @@ def stopper(signal_number=None, frame=None): # Load configuration from file try: - cfg.CONF(sys.argv[1:], default_config_files=['/etc/cloudkitty/cloudkitty.conf.d/cloudkitty.conf']) + cfg.CONF(sys.argv[1:]) except cfg.ConfigFilesNotFoundError as e: print(f"Health check failed: {e}", file=sys.stderr) sys.exit(1) diff --git a/templates/cloudkitty/config/cloudkitty-api-config.json b/templates/cloudkitty/config/cloudkitty-api-config.json index 107cfa1a..ddf03b22 100644 --- a/templates/cloudkitty/config/cloudkitty-api-config.json +++ b/templates/cloudkitty/config/cloudkitty-api-config.json @@ -2,18 +2,30 @@ "command": "/usr/sbin/httpd -DFOREGROUND -E /dev/stdout", "config_files": [ { - "source": "/var/lib/openstack/config/cloudkitty.conf", - "dest": "/etc/cloudkitty/cloudkitty.conf", + "source": "/var/lib/openstack/service-config/cloudkitty.conf", + "dest": "/etc/cloudkitty/cloudkitty.conf.d/00-cloudkitty.conf", "owner": "cloudkitty", "perm": "0600" }, { - "source": "/var/lib/openstack/config/custom.conf", - "dest": "/etc/aodh/cloudkitty.conf.d/01-cloudkitty-custom.conf", + "source": "/var/lib/openstack/config/metrics.yaml", + "dest": "/etc/cloudkitty/metrics.yaml", + "owner": "cloudkitty", + "perm": "0600" + }, + { + "source": "/var/lib/openstack/service-config/0*.conf", + "dest": "/etc/cloudkitty/cloudkitty.conf.d/", "owner": "cloudkitty", "perm": "0600", "optional": true }, + { + "source": "/var/lib/openstack/config/metrics.yaml", + "dest": "/etc/cloudkitty/metrics.yaml", + "owner": "cloudkitty", + "perm": "0600" + }, { "source": "/var/lib/openstack/config/wsgi-cloudkitty.conf", "dest": "/etc/httpd/conf.d/00wsgi-cloudkitty.conf", @@ -67,7 +79,7 @@ ], "permissions": [ { - "path": "/var/log/cinder", + "path": "/var/log/cloudkitty", "owner": "cloudkitty:apache", "recurse": true }, diff --git a/templates/cloudkitty/config/cloudkitty-proc-config.json b/templates/cloudkitty/config/cloudkitty-proc-config.json index 410ed9d3..ed81471f 100644 --- a/templates/cloudkitty/config/cloudkitty-proc-config.json +++ b/templates/cloudkitty/config/cloudkitty-proc-config.json @@ -2,16 +2,55 @@ "command": "/usr/bin/cloudkitty-processor --logfile /dev/stdout", "config_files": [ { - "source": "/var/lib/openstack/config/cloudkitty.conf", - "dest": "/etc/cloudkitty/cloudkitty.conf", + "source": "/var/lib/openstack/service-config/cloudkitty.conf", + "dest": "/etc/cloudkitty/cloudkitty.conf.d/00-cloudkitty.conf", "owner": "cloudkitty", "perm": "0600" }, + { + "source": "/var/lib/openstack/service-config/0*.conf", + "dest": "/etc/cloudkitty/cloudkitty.conf.d/", + "owner": "cloudkitty", + "perm": "0600", + "optional": true + }, { "source": "/var/lib/openstack/config/metrics.yaml", "dest": "/etc/cloudkitty/metrics.yaml", "owner": "cloudkitty", "perm": "0600" + }, + { + "source": "/var/lib/config-data/tls/certs/*", + "dest": "/etc/pki/tls/certs/", + "owner": "cloudkitty", + "perm": "0640", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/config-data/tls/private/*", + "dest": "/etc/pki/tls/private/", + "owner": "cloudkitty", + "perm": "0600", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/config-data/mtls/certs/*", + "dest": "/etc/pki/tls/certs/", + "owner": "cloudkitty:cloudkitty", + "perm": "0640", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/config-data/mtls/private/*", + "dest": "/etc/pki/tls/private/", + "owner": "cloudkitty:cloudkitty", + "perm": "0640", + "optional": true, + "merge": true } ] } diff --git a/templates/cloudkitty/config/wsgi-cloudkitty.conf b/templates/cloudkitty/config/wsgi-cloudkitty.conf index b7407f3d..7559d46c 100644 --- a/templates/cloudkitty/config/wsgi-cloudkitty.conf +++ b/templates/cloudkitty/config/wsgi-cloudkitty.conf @@ -34,7 +34,7 @@ WSGIApplicationGroup %{GLOBAL} WSGIDaemonProcess {{ $endpt }} display-name={{ $endpt }} group=cloudkitty processes=4 threads=1 user=cloudkitty WSGIProcessGroup {{ $endpt }} - WSGIScriptAlias / "/var/www/cgi-bin/cloudkitty/cloudkitty-api" + WSGIScriptAlias / "/usr/bin/cloudkitty-api" WSGIPassAuthorization On {{ end }} From 64d9b00f8d6850baf4b29be9d7b03a5f4844ca2d Mon Sep 17 00:00:00 2001 From: jlarriba Date: Fri, 20 Jun 2025 14:22:07 +0200 Subject: [PATCH 13/87] Cloudkitty deployment --- PROJECT | 31 +- .../telemetry.openstack.org_autoscalings.yaml | 5 + .../telemetry.openstack.org_ceilometers.yaml | 5 + .../telemetry.openstack.org_cloudkitties.yaml | 665 ++++++++++ ...elemetry.openstack.org_cloudkittyapis.yaml | 499 ++++++++ ...lemetry.openstack.org_cloudkittyprocs.yaml | 321 +++++ .../telemetry.openstack.org_telemetries.yaml | 546 ++++++++ api/v1beta1/autoscaling_types.go | 12 +- api/v1beta1/cloudkitty_types.go | 270 ++++ api/v1beta1/cloudkitty_webhook.go | 102 ++ api/v1beta1/cloudkittyapi_types.go | 155 +++ api/v1beta1/cloudkittyproc_types.go | 148 +++ api/v1beta1/conditions.go | 51 + api/v1beta1/telemetry_types.go | 33 +- api/v1beta1/zz_generated.deepcopy.go | 630 ++++++++++ .../telemetry.openstack.org_autoscalings.yaml | 5 + .../telemetry.openstack.org_ceilometers.yaml | 5 + .../telemetry.openstack.org_cloudkitties.yaml | 665 ++++++++++ ...elemetry.openstack.org_cloudkittyapis.yaml | 499 ++++++++ ...lemetry.openstack.org_cloudkittyprocs.yaml | 321 +++++ .../telemetry.openstack.org_telemetries.yaml | 546 ++++++++ config/crd/kustomization.yaml | 9 + .../patches/cainjection_in_cloudkitties.yaml | 7 + .../cainjection_in_cloudkittyapis.yaml | 7 + .../cainjection_in_cloudkittyprocs.yaml | 7 + .../crd/patches/webhook_in_cloudkitties.yaml | 16 + .../patches/webhook_in_cloudkittyapis.yaml | 16 + .../patches/webhook_in_cloudkittyprocs.yaml | 16 + config/rbac/cloudkitty_editor_role.yaml | 31 + config/rbac/cloudkitty_viewer_role.yaml | 27 + config/rbac/cloudkittyapi_editor_role.yaml | 31 + config/rbac/cloudkittyapi_viewer_role.yaml | 27 + config/rbac/cloudkittyproc_editor_role.yaml | 31 + config/rbac/cloudkittyproc_viewer_role.yaml | 27 + config/rbac/role.yaml | 130 ++ config/samples/kustomization.yaml | 3 + .../samples/telemetry_v1beta1_cloudkitty.yaml | 12 + .../telemetry_v1beta1_cloudkittyapi.yaml | 12 + .../telemetry_v1beta1_cloudkittyproc.yaml | 12 + config/webhook/manifests.yaml | 40 + controllers/cloudkitty_controller.go | 1085 ++++++++++++++++ controllers/cloudkittyapi_controller.go | 1119 +++++++++++++++++ controllers/cloudkittyproc_controller.go | 759 +++++++++++ controllers/telemetry_controller.go | 116 ++ main.go | 32 + pkg/cloudkitty/common.go | 161 +++ pkg/cloudkitty/const.go | 54 + pkg/cloudkitty/dbsync.go | 107 ++ pkg/cloudkitty/funcs.go | 49 + pkg/cloudkitty/volumes.go | 57 + pkg/cloudkittyapi/const.go | 24 + pkg/cloudkittyapi/statefulset.go | 188 +++ pkg/cloudkittyapi/volumes.go | 54 + pkg/cloudkittyproc/const.go | 21 + pkg/cloudkittyproc/statefulset.go | 153 +++ pkg/cloudkittyproc/volumes.go | 38 + templates/cloudkitty/bin/healthcheck.py | 94 ++ templates/cloudkitty/bin/run-on-host | 2 + .../cloudkitty/config/10-cloudkitty_wsgi.conf | 40 + .../config/cloudkitty-api-config-httpd.json | 51 + .../config/cloudkitty-api-config.json | 11 + .../config/cloudkitty-api-uwsgi.ini | 17 + .../config/cloudkitty-dbsync-config.json | 11 + .../config/cloudkitty-proc-config.json | 17 + templates/cloudkitty/config/cloudkitty.conf | 78 ++ templates/cloudkitty/config/httpd.conf | 27 + templates/cloudkitty/config/metrics.yaml | 77 ++ templates/cloudkitty/config/ssl.conf | 21 + 68 files changed, 10423 insertions(+), 15 deletions(-) create mode 100644 api/bases/telemetry.openstack.org_cloudkitties.yaml create mode 100644 api/bases/telemetry.openstack.org_cloudkittyapis.yaml create mode 100644 api/bases/telemetry.openstack.org_cloudkittyprocs.yaml create mode 100644 api/v1beta1/cloudkitty_types.go create mode 100644 api/v1beta1/cloudkitty_webhook.go create mode 100644 api/v1beta1/cloudkittyapi_types.go create mode 100644 api/v1beta1/cloudkittyproc_types.go create mode 100644 config/crd/bases/telemetry.openstack.org_cloudkitties.yaml create mode 100644 config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml create mode 100644 config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml create mode 100644 config/crd/patches/cainjection_in_cloudkitties.yaml create mode 100644 config/crd/patches/cainjection_in_cloudkittyapis.yaml create mode 100644 config/crd/patches/cainjection_in_cloudkittyprocs.yaml create mode 100644 config/crd/patches/webhook_in_cloudkitties.yaml create mode 100644 config/crd/patches/webhook_in_cloudkittyapis.yaml create mode 100644 config/crd/patches/webhook_in_cloudkittyprocs.yaml create mode 100644 config/rbac/cloudkitty_editor_role.yaml create mode 100644 config/rbac/cloudkitty_viewer_role.yaml create mode 100644 config/rbac/cloudkittyapi_editor_role.yaml create mode 100644 config/rbac/cloudkittyapi_viewer_role.yaml create mode 100644 config/rbac/cloudkittyproc_editor_role.yaml create mode 100644 config/rbac/cloudkittyproc_viewer_role.yaml create mode 100644 config/samples/telemetry_v1beta1_cloudkitty.yaml create mode 100644 config/samples/telemetry_v1beta1_cloudkittyapi.yaml create mode 100644 config/samples/telemetry_v1beta1_cloudkittyproc.yaml create mode 100644 controllers/cloudkitty_controller.go create mode 100644 controllers/cloudkittyapi_controller.go create mode 100644 controllers/cloudkittyproc_controller.go create mode 100644 pkg/cloudkitty/common.go create mode 100644 pkg/cloudkitty/const.go create mode 100644 pkg/cloudkitty/dbsync.go create mode 100644 pkg/cloudkitty/funcs.go create mode 100644 pkg/cloudkitty/volumes.go create mode 100644 pkg/cloudkittyapi/const.go create mode 100644 pkg/cloudkittyapi/statefulset.go create mode 100644 pkg/cloudkittyapi/volumes.go create mode 100644 pkg/cloudkittyproc/const.go create mode 100644 pkg/cloudkittyproc/statefulset.go create mode 100644 pkg/cloudkittyproc/volumes.go create mode 100755 templates/cloudkitty/bin/healthcheck.py create mode 100755 templates/cloudkitty/bin/run-on-host create mode 100644 templates/cloudkitty/config/10-cloudkitty_wsgi.conf create mode 100644 templates/cloudkitty/config/cloudkitty-api-config-httpd.json create mode 100644 templates/cloudkitty/config/cloudkitty-api-config.json create mode 100644 templates/cloudkitty/config/cloudkitty-api-uwsgi.ini create mode 100644 templates/cloudkitty/config/cloudkitty-dbsync-config.json create mode 100644 templates/cloudkitty/config/cloudkitty-proc-config.json create mode 100644 templates/cloudkitty/config/cloudkitty.conf create mode 100644 templates/cloudkitty/config/httpd.conf create mode 100644 templates/cloudkitty/config/metrics.yaml create mode 100644 templates/cloudkitty/config/ssl.conf diff --git a/PROJECT b/PROJECT index 5230867a..58516761 100644 --- a/PROJECT +++ b/PROJECT @@ -1,7 +1,3 @@ -# Code generated by tool. DO NOT EDIT. -# This file is used to track the info used to scaffold your project -# and allow the plugins properly work. -# More info: https://book.kubebuilder.io/reference/project-config.html domain: openstack.org layout: - go.kubebuilder.io/v3 @@ -69,4 +65,31 @@ resources: defaulting: true validation: true webhookVersion: v1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: openstack.org + group: telemetry + kind: CloudKittyApi + path: github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1 + version: v1beta1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: openstack.org + group: telemetry + kind: CloudKittyProc + path: github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1 + version: v1beta1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: openstack.org + group: telemetry + kind: CloudKitty + path: github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1 + version: v1beta1 version: "3" diff --git a/api/bases/telemetry.openstack.org_autoscalings.yaml b/api/bases/telemetry.openstack.org_autoscalings.yaml index 4cfabab5..a4ef6ccf 100644 --- a/api/bases/telemetry.openstack.org_autoscalings.yaml +++ b/api/bases/telemetry.openstack.org_autoscalings.yaml @@ -294,6 +294,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object preserveJobs: default: false diff --git a/api/bases/telemetry.openstack.org_ceilometers.yaml b/api/bases/telemetry.openstack.org_ceilometers.yaml index 824f5daa..3307bc27 100644 --- a/api/bases/telemetry.openstack.org_ceilometers.yaml +++ b/api/bases/telemetry.openstack.org_ceilometers.yaml @@ -210,6 +210,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object proxyImage: type: string diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml new file mode 100644 index 00000000..bc5b7567 --- /dev/null +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -0,0 +1,665 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: cloudkitties.telemetry.openstack.org +spec: + group: telemetry.openstack.org + names: + kind: CloudKitty + listKind: CloudKittyList + plural: cloudkitties + singular: cloudkitty + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: CloudKitty is the Schema for the cloudkitties API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CloudKittySpec defines the desired state of CloudKitty + properties: + apiTimeout: + default: 60 + description: APITimeout for HAProxy, Apache, and rpc_response_timeout + minimum: 10 + type: integer + cloudKittyAPI: + description: CloudKittyAPI - Spec definition for the API service of + this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the + configurations of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret + for the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object + public: + description: Public GenericService - holds the secret + for the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in + a pre-created bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + cloudKittyProc: + description: CloudKittyProc - Spec definition for the Scheduler service + of this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config for all CloudKitty services using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for cloudkitty + DB, defaults to cloudkitty + type: string + databaseInstance: + description: |- + MariaDB instance name + Right now required by the maridb-operator to get the credentials from the instance to create the DB + Might not be required in future + type: string + memcachedInstance: + default: memcached + description: Memcached instance name. + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting + NodeSelector here acts as a default value and can be overridden by service + specific NodeSelector Settings. + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service password + from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean + rabbitMqClusterName: + default: rabbitmq + description: |- + RabbitMQ instance name + Needed to request a transportURL that is created and used in CloudKitty + type: string + secret: + description: Secret containing OpenStack password information + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - cloudKittyAPI + - cloudKittyProc + - databaseInstance + - memcachedInstance + - rabbitMqClusterName + - secret + type: object + status: + description: CloudKittyStatus defines the observed state of CloudKitty + properties: + apiEndpoints: + additionalProperties: + additionalProperties: + type: string + type: object + description: API endpoints + type: object + cloudKittyAPIReadyCount: + default: 0 + description: ReadyCount of CloudKitty API instance + format: int32 + minimum: 0 + type: integer + cloudKittyProcReadyCounts: + default: 0 + description: ReadyCount of CloudKitty Processor instances + format: int32 + minimum: 0 + type: integer + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + databaseHostname: + description: CloudKitty Database Hostname + type: string + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object + observedGeneration: + description: |- + ObservedGeneration - the most recent generation observed for this service. + If the observed generation is different than the spec generation, then the + controller has not started processing the latest changes, and the status + and its conditions are likely stale. + format: int64 + type: integer + serviceIDs: + additionalProperties: + type: string + description: ServiceIDs + type: object + transportURLSecret: + description: TransportURLSecret - Secret containing RabbitMQ transportURL + type: string + required: + - cloudKittyAPIReadyCount + - cloudKittyProcReadyCounts + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/api/bases/telemetry.openstack.org_cloudkittyapis.yaml b/api/bases/telemetry.openstack.org_cloudkittyapis.yaml new file mode 100644 index 00000000..d33b713c --- /dev/null +++ b/api/bases/telemetry.openstack.org_cloudkittyapis.yaml @@ -0,0 +1,499 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: cloudkittyapis.telemetry.openstack.org +spec: + group: telemetry.openstack.org + names: + kind: CloudKittyAPI + listKind: CloudKittyAPIList + plural: cloudkittyapis + singular: cloudkittyapi + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: CloudKittyAPI is the Schema for the cloudkittyapis API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CloudKittyAPISpec defines the desired state of CloudKittyAPI + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for cloudkitty + DB, defaults to cloudkitty + type: string + databaseHostname: + description: DatabaseHostname - CloudKitty Database Hostname + type: string + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment resource + names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service password + from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + secret: + description: Secret containing OpenStack password information + type: string + serviceAccount: + description: ServiceAccount - service account name used internally + to provide CloudKitty services the default SA name + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret for + the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + public: + description: Public GenericService - holds the secret for + the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + transportURLSecret: + description: Secret containing RabbitMq transport URL + type: string + required: + - containerImage + - databaseHostname + - secret + - serviceAccount + - transportURLSecret + type: object + status: + description: CloudKittyAPIStatus defines the observed state of CloudKittyAPI + properties: + apiEndpoints: + additionalProperties: + additionalProperties: + type: string + type: object + description: API endpoints + type: object + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object + lastAppliedTopology: + description: LastAppliedTopology - the last applied Topology + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + networkAttachments: + additionalProperties: + items: + type: string + type: array + description: NetworkAttachments status of the deployment pods + type: object + observedGeneration: + description: |- + ObservedGeneration - the most recent generation observed for this service. + If the observed generation is different than the spec generation, then the + controller has not started processing the latest changes, and the status + and its conditions are likely stale. + format: int64 + type: integer + readyCount: + default: 0 + description: ReadyCount of CloudKitty API instances + format: int32 + minimum: 0 + type: integer + serviceIDs: + additionalProperties: + type: string + description: ServiceIDs + type: object + required: + - readyCount + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml new file mode 100644 index 00000000..9cd9b6ef --- /dev/null +++ b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -0,0 +1,321 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: cloudkittyprocs.telemetry.openstack.org +spec: + group: telemetry.openstack.org + names: + kind: CloudKittyProc + listKind: CloudKittyProcList + plural: cloudkittyprocs + singular: cloudkittyproc + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: NetworkAttachments + jsonPath: .status.networkAttachments + name: NetworkAttachments + type: string + - description: Status + jsonPath: .status.conditions[0].status + name: Status + type: string + - description: Message + jsonPath: .status.conditions[0].message + name: Message + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: CloudKittyProc is the Schema for the cloudkittprocs API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CloudKittyProcSpec defines the desired state of CloudKitty + Processor + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for cloudkitty + DB, defaults to cloudkitty + type: string + databaseHostname: + description: DatabaseHostname - CloudKitty Database Hostname + type: string + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment resource + names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service password + from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + secret: + description: Secret containing OpenStack password information + type: string + serviceAccount: + description: ServiceAccount - service account name used internally + to provide CloudKitty services the default SA name + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + transportURLSecret: + description: Secret containing RabbitMq transport URL + type: string + required: + - containerImage + - databaseHostname + - secret + - serviceAccount + - transportURLSecret + type: object + status: + description: CloudKittyProcStatus defines the observed state of CloudKitty + Processor + properties: + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object + lastAppliedTopology: + description: LastAppliedTopology - the last applied Topology + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + networkAttachments: + additionalProperties: + items: + type: string + type: array + description: NetworkAttachments status of the deployment pods + type: object + observedGeneration: + description: |- + ObservedGeneration - the most recent generation observed for this service. + If the observed generation is different than the spec generation, then the + controller has not started processing the latest changes, and the status + and its conditions are likely stale. + format: int64 + type: integer + readyCount: + default: 0 + description: ReadyCount of CloudKitty Processor instances + format: int32 + minimum: 0 + type: integer + required: + - readyCount + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index 6c18bdfc..0d8bf4d3 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -297,6 +297,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object preserveJobs: default: false @@ -525,6 +530,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object proxyImage: type: string @@ -584,6 +594,542 @@ spec: - secret - sgCoreImage type: object + cloudkitty: + description: CloudKitty - Parameters related to the cloudkitty service + properties: + apiTimeout: + default: 60 + description: APITimeout for HAProxy, Apache, and rpc_response_timeout + minimum: 10 + type: integer + cloudKittyAPI: + description: CloudKittyAPI - Spec definition for the API service + of this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL + (will be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + override: + description: Override, provides the ability to override the + generated manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains + the configurations of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret + for the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key + for the service + type: string + type: object + public: + description: Public GenericService - holds the secret + for the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key + for the service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs + in a pre-created bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + cloudKittyProc: + description: CloudKittyProc - Spec definition for the Scheduler + service of this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL + (will be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config for all CloudKitty services using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for + cloudkitty DB, defaults to cloudkitty + type: string + databaseInstance: + description: |- + MariaDB instance name + Right now required by the maridb-operator to get the credentials from the instance to create the DB + Might not be required in future + type: string + enabled: + default: true + description: Enabled - Whether OpenStack CloudKitty service should + be deployed and managed + type: boolean + memcachedInstance: + default: memcached + description: Memcached instance name. + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting + NodeSelector here acts as a default value and can be overridden by service + specific NodeSelector Settings. + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service + password from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean + rabbitMqClusterName: + default: rabbitmq + description: |- + RabbitMQ instance name + Needed to request a transportURL that is created and used in CloudKitty + type: string + secret: + description: Secret containing OpenStack password information + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - cloudKittyAPI + - cloudKittyProc + - databaseInstance + - memcachedInstance + - rabbitMqClusterName + - secret + type: object logging: description: Logging - Parameters related to the logging properties: diff --git a/api/v1beta1/autoscaling_types.go b/api/v1beta1/autoscaling_types.go index d6f3b99b..f541155e 100644 --- a/api/v1beta1/autoscaling_types.go +++ b/api/v1beta1/autoscaling_types.go @@ -17,13 +17,12 @@ limitations under the License. package v1beta1 import ( + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" - "github.com/openstack-k8s-operators/lib-common/modules/common/service" - "github.com/openstack-k8s-operators/lib-common/modules/common/util" "k8s.io/apimachinery/pkg/util/validation/field" ) @@ -139,13 +138,6 @@ type AodhCore struct { TopologyRef *topologyv1.TopoRef `json:"topologyRef,omitempty"` } -// APIOverrideSpec to override the generated manifest of several child resources. -type APIOverrideSpec struct { - // Override configuration for the Service created to serve traffic to the cluster. - // The key must be the endpoint type (public, internal) - Service map[service.Endpoint]service.RoutedOverrideSpec `json:"service,omitempty"` -} - // AutoscalingSpec defines the desired state of Autoscaling type AutoscalingSpec struct { AutoscalingSpecBase `json:",inline"` diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go new file mode 100644 index 00000000..08181d6c --- /dev/null +++ b/api/v1beta1/cloudkitty_types.go @@ -0,0 +1,270 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // CloudKittyUserID - Kolla's cloudkitty UID comes from the 'cloudkitty-user' in + // https://github.com/openstack/kolla/blob/master/kolla/common/users.py + CloudKittyUserID = 42408 + // CloudKittyGroupID - Kolla's cloudkitty GID + CloudKittyGroupID = 42408 + + // CloudKittyAPIContainerImage - default fall-back image for CloudKitty API + CloudKittyAPIContainerImage = "quay.io/podified-master-centos9/cloudkitty-api:current-podified" + // CloudKittyProcContainerImage - default fall-back image for CloudKitty Processor + CloudKittyProcContainerImage = "quay.io/podified-master-centos9/cloudkitty-processor:current-podified" + // CloudKittyDbSyncHash hash + CKDbSyncHash = "ckdbsync" + // CloudKittyReplicas - The number of replicas per each service deployed + CloudKittyReplicas = 1 +) + +type CloudKittySpecBase struct { + CloudKittyTemplate `json:",inline"` + + // +kubebuilder:validation:Required + // MariaDB instance name + // Right now required by the maridb-operator to get the credentials from the instance to create the DB + // Might not be required in future + DatabaseInstance string `json:"databaseInstance"` + + // +kubebuilder:validation:Required + // +kubebuilder:default=rabbitmq + // RabbitMQ instance name + // Needed to request a transportURL that is created and used in CloudKitty + RabbitMqClusterName string `json:"rabbitMqClusterName"` + + // +kubebuilder:validation:Required + // +kubebuilder:default=memcached + // Memcached instance name. + MemcachedInstance string `json:"memcachedInstance"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // PreserveJobs - do not delete jobs after they finished e.g. to check logs + PreserveJobs bool `json:"preserveJobs"` + + // +kubebuilder:validation:Optional + // CustomServiceConfig - customize the service config for all CloudKitty services using this parameter to change service defaults, + // or overwrite rendered information using raw OpenStack config format. The content gets added to + // to /etc//.conf.d directory as a custom config file. + CustomServiceConfig string `json:"customServiceConfig,omitempty"` + + // +kubebuilder:validation:Optional + // NodeSelector to target subset of worker nodes running this service. Setting + // NodeSelector here acts as a default value and can be overridden by service + // specific NodeSelector Settings. + NodeSelector *map[string]string `json:"nodeSelector,omitempty"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=60 + // +kubebuilder:validation:Minimum=10 + // APITimeout for HAProxy, Apache, and rpc_response_timeout + APITimeout int `json:"apiTimeout"` + + // +kubebuilder:validation:Optional + // TopologyRef to apply the Topology defined by the associated CR referenced + // by name + TopologyRef *topologyv1.TopoRef `json:"topologyRef,omitempty"` +} + +// CloudKittySpecCore the same as CloudKittySpec without ContainerImage references +type CloudKittySpecCore struct { + CloudKittySpecBase `json:",inline"` + + // +kubebuilder:validation:Required + // CloudKittyAPI - Spec definition for the API service of this CloudKitty deployment + CloudKittyAPI CloudKittyAPITemplateCore `json:"cloudKittyAPI"` + + // +kubebuilder:validation:Required + // CloudKittyProc - Spec definition for the Scheduler service of this CloudKitty deployment + CloudKittyProc CloudKittyProcTemplateCore `json:"cloudKittyProc"` +} + +// CloudKittySpec defines the desired state of CloudKitty +type CloudKittySpec struct { + CloudKittySpecBase `json:",inline"` + + // +kubebuilder:validation:Required + // CloudKittyAPI - Spec definition for the API service of this CloudKitty deployment + CloudKittyAPI CloudKittyAPITemplate `json:"cloudKittyAPI"` + + // +kubebuilder:validation:Required + // CloudKittyProc - Spec definition for the Scheduler service of this CloudKitty deployment + CloudKittyProc CloudKittyProcTemplate `json:"cloudKittyProc"` +} + +// CloudKittyTemplate defines common input parameters used by all CloudKitty services +type CloudKittyTemplate struct { + // +kubebuilder:validation:Optional + // +kubebuilder:default=cloudkitty + // ServiceUser - optional username used for this service to register in cloudkitty + ServiceUser string `json:"serviceUser"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=cloudkitty + // DatabaseAccount - optional MariaDBAccount used for cloudkitty DB, defaults to cloudkitty + DatabaseAccount string `json:"databaseAccount"` + + // +kubebuilder:validation:Required + // Secret containing OpenStack password information + Secret string `json:"secret"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default={cloudKittyService: CloudKittyPassword} + // PasswordsSelectors - Selectors to identify the ServiceUser password from the Secret + PasswordSelectors PasswordsSelector `json:"passwordSelector"` +} + +// CloudKittyServiceTemplate defines the input parameters that can be defined for a given +// CloudKitty service +type CloudKittyServiceTemplate struct { + + // +kubebuilder:validation:Optional + // NodeSelector to target subset of worker nodes running this service. Setting here overrides + // any global NodeSelector settings within the CloudKitty CR. + NodeSelector *map[string]string `json:"nodeSelector,omitempty"` + + // +kubebuilder:validation:Optional + // CustomServiceConfig - customize the service config using this parameter to change service defaults, + // or overwrite rendered information using raw OpenStack config format. The content gets added to + // to /etc//.conf.d directory as a custom config file. + CustomServiceConfig string `json:"customServiceConfig,omitempty"` + + // +kubebuilder:validation:Optional + // CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + // that contain sensitive service config data. The content of each Secret gets added to the + // /etc//.conf.d directory as a custom config file. + CustomServiceConfigSecrets []string `json:"customServiceConfigSecrets,omitempty"` + + // +kubebuilder:validation:Optional + // Resources - Compute Resources required by this service (Limits/Requests). + // https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + + // +kubebuilder:validation:Optional + // NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network + NetworkAttachments []string `json:"networkAttachments,omitempty"` + + // +kubebuilder:validation:Optional + // TopologyRef to apply the Topology defined by the associated CR referenced + // by name + TopologyRef *topologyv1.TopoRef `json:"topologyRef,omitempty"` +} + +// CloudKittyStatus defines the observed state of CloudKitty +type CloudKittyStatus struct { + // Map of hashes to track e.g. job status + Hash map[string]string `json:"hash,omitempty"` + + // Conditions + Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` + + // CloudKitty Database Hostname + DatabaseHostname string `json:"databaseHostname,omitempty"` + + // TransportURLSecret - Secret containing RabbitMQ transportURL + TransportURLSecret string `json:"transportURLSecret,omitempty"` + + // API endpoints + APIEndpoints map[string]map[string]string `json:"apiEndpoints,omitempty"` + + // ServiceIDs + ServiceIDs map[string]string `json:"serviceIDs,omitempty"` + + // ReadyCount of CloudKitty API instance + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:default=0 + CloudKittyAPIReadyCount int32 `json:"cloudKittyAPIReadyCount"` + + // ReadyCount of CloudKitty Processor instances + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:default=0 + CloudKittyProcReadyCount int32 `json:"cloudKittyProcReadyCounts"` + + // ObservedGeneration - the most recent generation observed for this service. + // If the observed generation is different than the spec generation, then the + // controller has not started processing the latest changes, and the status + // and its conditions are likely stale. + ObservedGeneration int64 `json:"observedGeneration,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// CloudKitty is the Schema for the cloudkitties API +type CloudKitty struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CloudKittySpec `json:"spec,omitempty"` + Status CloudKittyStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// CloudKittyList contains a list of CloudKitty +type CloudKittyList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CloudKitty `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CloudKitty{}, &CloudKittyList{}) +} + +// SetupDefaultsCloudKitty - initializes any CRD field defaults based on environment variables (the defaulting mechanism itself is implemented via webhooks) +func SetupDefaultsCloudKitty() { + // Acquire environmental defaults and initialize Telemetry defaults with them + cloudKittyDefaults := CloudKittyDefaults{ + APIContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CLOUDKITTY_API_IMAGE_URL_DEFAULT", CloudKittyAPIContainerImage), + ProcContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CLOUDKITTY_PROCESSOR_IMAGE_URL_DEFAULT", CloudKittyProcContainerImage), + } + + SetupCloudKittyDefaults(cloudKittyDefaults) +} + +// IsReady - returns true if all subresources Ready condition is true +func (instance CloudKitty) IsReady() bool { + return instance.Generation == instance.Status.ObservedGeneration && + instance.Status.Conditions.IsTrue(CloudKittyAPIReadyCondition) && + instance.Status.Conditions.IsTrue(CloudKittyProcReadyCondition) +} + +// RbacConditionsSet - set the conditions for the rbac object +func (instance CloudKitty) RbacConditionsSet(c *condition.Condition) { + instance.Status.Conditions.Set(c) +} + +// RbacNamespace - return the namespace +func (instance CloudKitty) RbacNamespace() string { + return instance.Namespace +} + +// RbacResourceName - return the name to be used for rbac objects (serviceaccount, role, rolebinding) +func (instance CloudKitty) RbacResourceName() string { + return "cloudkitty-" + instance.Name +} diff --git a/api/v1beta1/cloudkitty_webhook.go b/api/v1beta1/cloudkitty_webhook.go new file mode 100644 index 00000000..5ebd644a --- /dev/null +++ b/api/v1beta1/cloudkitty_webhook.go @@ -0,0 +1,102 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// CloudKittyDefaults - +type CloudKittyDefaults struct { + APIContainerImageURL string + ProcContainerImageURL string +} + +var cloudKittyDefaults CloudKittyDefaults + +// log is for logging in this package. +var cloudKittyLog = logf.Log.WithName("cloudkitty-resource") + +// SetupCloudKittyDefaults - initialize CloudKitty spec defaults for use with either internal or external webhooks +func SetupCloudKittyDefaults(defaults CloudKittyDefaults) { + cloudKittyDefaults = defaults + cloudKittyLog.Info("CloudKitty defaults initialized", "defaults", defaults) +} + +// SetupWebhookWithManager - setups webhook with the adequate manager +func (r *CloudKitty) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(r). + Complete() +} + +// TODO(user): EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! + +//+kubebuilder:webhook:path=/mutate-telemetry-openstack-org-v1beta1-cloudkitty,mutating=true,failurePolicy=fail,sideEffects=None,groups=telemetry.openstack.org,resources=cloudkitties,verbs=create;update,versions=v1beta1,name=mcloudkitty.kb.io,admissionReviewVersions=v1 + +var _ webhook.Defaulter = &CloudKitty{} + +// Default implements webhook.Defaulter so a webhook will be registered for the type +func (r *CloudKitty) Default() { + cloudKittyLog.Info("default", "name", r.Name) + + r.Spec.Default() +} + +// Default - set defaults for this CloudKitty spec +func (spec *CloudKittySpec) Default() { + if spec.CloudKittyAPI.ContainerImage == "" { + spec.CloudKittyAPI.ContainerImage = cloudKittyDefaults.APIContainerImageURL + } + if spec.CloudKittyProc.ContainerImage == "" { + spec.CloudKittyProc.ContainerImage = cloudKittyDefaults.ProcContainerImageURL + } + +} + +// TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation. +//+kubebuilder:webhook:path=/validate-telemetry-openstack-org-v1beta1-cloudkitty,mutating=false,failurePolicy=fail,sideEffects=None,groups=telemetry.openstack.org,resources=cloudkitties,verbs=create;update,versions=v1beta1,name=vcloudkitty.kb.io,admissionReviewVersions=v1 + +var _ webhook.Validator = &CloudKitty{} + +// ValidateCreate implements webhook.Validator so a webhook will be registered for the type +func (r *CloudKitty) ValidateCreate() (admission.Warnings, error) { + cloudKittyLog.Info("validate create", "name", r.Name) + + // TODO(user): fill in your validation logic upon object creation. + return nil, nil +} + +// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type +func (r *CloudKitty) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { + cloudKittyLog.Info("validate update", "name", r.Name) + + // TODO(user): fill in your validation logic upon object update. + return nil, nil +} + +// ValidateDelete implements webhook.Validator so a webhook will be registered for the type +func (r *CloudKitty) ValidateDelete() (admission.Warnings, error) { + cloudKittyLog.Info("validate delete", "name", r.Name) + + // TODO(user): fill in your validation logic upon object deletion. + return nil, nil +} diff --git a/api/v1beta1/cloudkittyapi_types.go b/api/v1beta1/cloudkittyapi_types.go new file mode 100644 index 00000000..42660c7e --- /dev/null +++ b/api/v1beta1/cloudkittyapi_types.go @@ -0,0 +1,155 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// CloudKittyAPITemplateCore defines the input parameters for the CloudKitty API service +type CloudKittyAPITemplateCore struct { + // Common input parameters for the CloudKitty API service + CloudKittyServiceTemplate `json:",inline"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=1 + // +kubebuilder:validation:Minimum=0 + // Replicas - CloudKitty API Replicas + Replicas *int32 `json:"replicas"` + + // +kubebuilder:validation:Optional + // Override, provides the ability to override the generated manifest of several child resources. + Override APIOverrideSpec `json:"override,omitempty"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.API `json:"tls,omitempty"` +} + +// CloudKittyAPITemplate defines the input parameters for the CloudKitty API service +type CloudKittyAPITemplate struct { + // +kubebuilder:validation:Required + // ContainerImage - CloudKitty Container Image URL (will be set to environmental default if empty) + ContainerImage string `json:"containerImage"` + + CloudKittyAPITemplateCore `json:",inline"` +} + +// CloudKittyAPISpec defines the desired state of CloudKittyAPI +type CloudKittyAPISpec struct { + // Common input parameters for all CloudKitty services + CloudKittyTemplate `json:",inline"` + + // Input parameters for the CloudKitty API service + CloudKittyAPITemplate `json:",inline"` + + // +kubebuilder:validation:Required + // DatabaseHostname - CloudKitty Database Hostname + DatabaseHostname string `json:"databaseHostname"` + + // +kubebuilder:validation:Required + // Secret containing RabbitMq transport URL + TransportURLSecret string `json:"transportURLSecret"` + + // +kubebuilder:validation:Required + // ServiceAccount - service account name used internally to provide CloudKitty services the default SA name + ServiceAccount string `json:"serviceAccount"` +} + +// CloudKittyAPIStatus defines the observed state of CloudKittyAPI +type CloudKittyAPIStatus struct { + // Map of hashes to track e.g. job status + Hash map[string]string `json:"hash,omitempty"` + + // API endpoints + APIEndpoints map[string]map[string]string `json:"apiEndpoints,omitempty"` + + // Conditions + Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` + + // ReadyCount of CloudKitty API instances + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:default=0 + ReadyCount int32 `json:"readyCount"` + + // ServiceIDs + ServiceIDs map[string]string `json:"serviceIDs,omitempty"` + + // NetworkAttachments status of the deployment pods + NetworkAttachments map[string][]string `json:"networkAttachments,omitempty"` + + // ObservedGeneration - the most recent generation observed for this service. + // If the observed generation is different than the spec generation, then the + // controller has not started processing the latest changes, and the status + // and its conditions are likely stale. + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // LastAppliedTopology - the last applied Topology + LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// CloudKittyAPI is the Schema for the cloudkittyapis API +type CloudKittyAPI struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CloudKittyAPISpec `json:"spec,omitempty"` + Status CloudKittyAPIStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// CloudKittyAPIList contains a list of CloudKittyAPI +type CloudKittyAPIList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CloudKittyAPI `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CloudKittyAPI{}, &CloudKittyAPIList{}) +} + +// IsReady - returns true if service is ready to serve requests +func (instance CloudKittyAPI) IsReady() bool { + return instance.Generation == instance.Status.ObservedGeneration && + instance.Status.ReadyCount == *instance.Spec.Replicas && + (instance.Status.Conditions.IsTrue(condition.DeploymentReadyCondition) || + (instance.Status.Conditions.IsFalse(condition.DeploymentReadyCondition) && *instance.Spec.Replicas == 0)) +} + +// GetSpecTopologyRef - Returns the LastAppliedTopology Set in the Status +func (instance *CloudKittyAPI) GetSpecTopologyRef() *topologyv1.TopoRef { + return instance.Spec.TopologyRef +} + +// GetLastAppliedTopology - Returns the LastAppliedTopology Set in the Status +func (instance *CloudKittyAPI) GetLastAppliedTopology() *topologyv1.TopoRef { + return instance.Status.LastAppliedTopology +} + +// SetLastAppliedTopology - Sets the LastAppliedTopology value in the Status +func (instance *CloudKittyAPI) SetLastAppliedTopology(topologyRef *topologyv1.TopoRef) { + instance.Status.LastAppliedTopology = topologyRef +} diff --git a/api/v1beta1/cloudkittyproc_types.go b/api/v1beta1/cloudkittyproc_types.go new file mode 100644 index 00000000..d2459afd --- /dev/null +++ b/api/v1beta1/cloudkittyproc_types.go @@ -0,0 +1,148 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// CloudKittyProcTemplateCore defines the input parameters for the CloudKitty Processor service +type CloudKittyProcTemplateCore struct { + // Common input parameters for the CloudKitty Processor service + CloudKittyServiceTemplate `json:",inline"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=1 + // +kubebuilder:validation:Minimum=0 + // Replicas - CloudKitty API Replicas + Replicas *int32 `json:"replicas"` +} + +// CloudKittyProcTemplate defines the input parameters for the CloudKitty Processor service +type CloudKittyProcTemplate struct { + // +kubebuilder:validation:Required + // ContainerImage - CloudKitty Container Image URL (will be set to environmental default if empty) + ContainerImage string `json:"containerImage"` + + CloudKittyProcTemplateCore `json:",inline"` +} + +// CloudKittyProcSpec defines the desired state of CloudKitty Processor +type CloudKittyProcSpec struct { + // Common input parameters for all CloudKitty services + CloudKittyTemplate `json:",inline"` + + // Input parameters for the CloudKitty Processor service + CloudKittyProcTemplate `json:",inline"` + + // +kubebuilder:validation:Required + // DatabaseHostname - CloudKitty Database Hostname + DatabaseHostname string `json:"databaseHostname"` + + // +kubebuilder:validation:Required + // Secret containing RabbitMq transport URL + TransportURLSecret string `json:"transportURLSecret"` + + // +kubebuilder:validation:Required + // ServiceAccount - service account name used internally to provide CloudKitty services the default SA name + ServiceAccount string `json:"serviceAccount"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.Ca `json:"tls,omitempty"` +} + +// CloudKittyProcStatus defines the observed state of CloudKitty Processor +type CloudKittyProcStatus struct { + // Map of hashes to track e.g. job status + Hash map[string]string `json:"hash,omitempty"` + + // Conditions + Conditions condition.Conditions `json:"conditions,omitempty" optional:"true"` + + // ReadyCount of CloudKitty Processor instances + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:default=0 + ReadyCount int32 `json:"readyCount"` + + // NetworkAttachments status of the deployment pods + NetworkAttachments map[string][]string `json:"networkAttachments,omitempty"` + + // ObservedGeneration - the most recent generation observed for this service. + // If the observed generation is different than the spec generation, then the + // controller has not started processing the latest changes, and the status + // and its conditions are likely stale. + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // LastAppliedTopology - the last applied Topology + LastAppliedTopology *topologyv1.TopoRef `json:"lastAppliedTopology,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:printcolumn:name="NetworkAttachments",type="string",JSONPath=".status.networkAttachments",description="NetworkAttachments" +//+kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[0].status",description="Status" +//+kubebuilder:printcolumn:name="Message",type="string",JSONPath=".status.conditions[0].message",description="Message" + +// CloudKittyProc is the Schema for the cloudkittprocs API +type CloudKittyProc struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CloudKittyProcSpec `json:"spec,omitempty"` + Status CloudKittyProcStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// CloudKittyProcList contains a list of CloudKittyProc +type CloudKittyProcList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CloudKittyProc `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CloudKittyProc{}, &CloudKittyProcList{}) +} + +// IsReady - returns true if service is ready to serve requests +func (instance CloudKittyProc) IsReady() bool { + return instance.Generation == instance.Status.ObservedGeneration && + instance.Status.ReadyCount == *instance.Spec.Replicas && + (instance.Status.Conditions.IsTrue(condition.DeploymentReadyCondition) || + (instance.Status.Conditions.IsFalse(condition.DeploymentReadyCondition) && *instance.Spec.Replicas == 0)) +} + +// GetSpecTopologyRef - Returns the LastAppliedTopology Set in the Status +func (instance *CloudKittyProc) GetSpecTopologyRef() *topologyv1.TopoRef { + return instance.Spec.TopologyRef +} + +// GetLastAppliedTopology - Returns the LastAppliedTopology Set in the Status +func (instance *CloudKittyProc) GetLastAppliedTopology() *topologyv1.TopoRef { + return instance.Status.LastAppliedTopology +} + +// SetLastAppliedTopology - Sets the LastAppliedTopology value in the Status +func (instance *CloudKittyProc) SetLastAppliedTopology(topologyRef *topologyv1.TopoRef) { + instance.Status.LastAppliedTopology = topologyRef +} diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index f9db6290..f8e0b4b7 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -45,6 +45,15 @@ const ( // LoggingReadyCondition Status=True condition which indicates if the Logging is configured and operational LoggingReadyCondition condition.Type = "LoggingReady" + // CloudKittyReadyCondition Status=True condition which indicates if the CloudKitty is configured and operational + CloudKittyReadyCondition condition.Type = "CloudKittyReady" + + // CloudKittyAPIReadyCondition Status=True condition which indicates if the CloudKitty API is configured and operational + CloudKittyAPIReadyCondition condition.Type = "CloudKittyAPIReady" + + // CloudKittyProcReadyCondition Status=True condition which indicates if the CloudKitty Processor is configured and operational + CloudKittyProcReadyCondition condition.Type = "CloudKittyProcReady" + // LoggingCLONamespaceReadyCondition Status=True condition which indicates if the cluster-logging-operator namespace is created LoggingCLONamespaceReadyCondition condition.Type = "LoggingCLONamespaceReady" @@ -202,6 +211,48 @@ const ( // LoggingCLONamespaceFailedMessage LoggingCLONamespaceFailedMessage = "CLO Namespace %s does not exist" + // + // CloudKittyReady condition messages + // + // CloudKittyReadyInitMessage + CloudKittyReadyInitMessage = "CloudKitty not started" + + // CloudKittyReadyMessage + CloudKittyReadyMessage = "CloudKitty completed" + + // CloudKittyReadyErrorMessage + CloudKittyReadyErrorMessage = "CloudKitty error occured %s" + + // + // CloudKittyAPIReady condition messages + // + // CloudKittyAPIReadyInitMessage + CloudKittyAPIReadyInitMessage = "CloudKittyAPI not started" + + // CloudKittyAPIReadyMessage + CloudKittyAPIReadyMessage = "CloudKittyAPI completed" + + // CloudKittyAPIReadyErrorMessage + CloudKittyAPIReadyErrorMessage = "CloudKittyAPI error occured %s" + + // CloudKittyAPIReadyRunningMessage + CloudKittyAPIReadyRunningMessage = "CloudKittyAPI in progress" + + // + // CloudKittyProcReady condition messages + // + // CloudKittyProcReadyInitMessage + CloudKittyProcReadyInitMessage = "CloudKittyProc not started" + + // CloudKittyProcReadyMessage + CloudKittyProcReadyMessage = "CloudKittyProc completed" + + // CloudKittyProcReadyErrorMessage + CloudKittyProcReadyErrorMessage = "CloudKittyProc error occured %s" + + // CloudKittyProcReadyRunningMessage + CloudKittyProcReadyRunningMessage = "CloudKittyProc in progress" + DashboardsNotEnabledMessage = "Dashboarding was not enabled, so no actions required" DashboardPrometheusRuleReadyInitMessage = "Dashboard PrometheusRule not started" diff --git a/api/v1beta1/telemetry_types.go b/api/v1beta1/telemetry_types.go index 7fb6a930..07621b5e 100644 --- a/api/v1beta1/telemetry_types.go +++ b/api/v1beta1/telemetry_types.go @@ -17,13 +17,21 @@ limitations under the License. package v1beta1 import ( + "github.com/openstack-k8s-operators/lib-common/modules/common/service" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" "github.com/openstack-k8s-operators/lib-common/modules/common/util" - topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" ) +// APIOverrideSpec to override the generated manifest of several child resources. +type APIOverrideSpec struct { + // Override configuration for the Service created to serve traffic to the cluster. + // The key must be the endpoint type (public, internal) + Service map[service.Endpoint]service.RoutedOverrideSpec `json:"service,omitempty"` +} + // PasswordsSelector to identify the Service password from the Secret type PasswordsSelector struct { // CeilometerService - Selector to get the ceilometer service password from the Secret @@ -35,6 +43,11 @@ type PasswordsSelector struct { // +kubebuilder:validation:Optional // +kubebuilder:default:=AodhPassword AodhService string `json:"aodhService"` + + // CloudKittyService - Selector to get the CloudKitty service password from the Secret + // +kubebuilder:validation:Optional + // +kubebuilder:default:=CloudKittyPassword + CloudKittyService string `json:"cloudKittyService"` } // TelemetrySpec defines the desired state of Telemetry @@ -73,6 +86,10 @@ type TelemetrySpecBase struct { // Logging - Parameters related to the logging Logging LoggingSection `json:"logging,omitempty"` + // +kubebuilder:validation:Optional + // CloudKitty - Parameters related to the cloudkitty service + CloudKitty CloudKittySection `json:"cloudkitty,omitempty"` + // +kubebuilder:validation:Optional // NodeSelector to target subset of worker nodes running this service NodeSelector *map[string]string `json:"nodeSelector,omitempty"` @@ -167,6 +184,20 @@ type LoggingSection struct { LoggingSpec `json:",inline"` } +// CloudKittySpec defines the desired state of the cloudkitty service +type CloudKittySection struct { + // +kubebuilder:validation:Optional + // +kubebuilder:default=true + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:booleanSwitch"} + // Enabled - Whether OpenStack CloudKitty service should be deployed and managed + Enabled *bool `json:"enabled"` + + // +kubebuilder:validation:Optional + //+operator-sdk:csv:customresourcedefinitions:type=spec + // Template - Overrides to use when creating the OpenStack CloudKitty service + CloudKittySpec `json:",inline"` +} + // TelemetryStatus defines the observed state of Telemetry type TelemetryStatus struct { // Map of hashes to track e.g. job status diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index da8d5649..1c5006d6 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -578,6 +578,635 @@ func (in *CeilometerStatus) DeepCopy() *CeilometerStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKitty) DeepCopyInto(out *CloudKitty) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKitty. +func (in *CloudKitty) DeepCopy() *CloudKitty { + if in == nil { + return nil + } + out := new(CloudKitty) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CloudKitty) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyAPI) DeepCopyInto(out *CloudKittyAPI) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyAPI. +func (in *CloudKittyAPI) DeepCopy() *CloudKittyAPI { + if in == nil { + return nil + } + out := new(CloudKittyAPI) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CloudKittyAPI) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyAPIList) DeepCopyInto(out *CloudKittyAPIList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CloudKittyAPI, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyAPIList. +func (in *CloudKittyAPIList) DeepCopy() *CloudKittyAPIList { + if in == nil { + return nil + } + out := new(CloudKittyAPIList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CloudKittyAPIList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyAPISpec) DeepCopyInto(out *CloudKittyAPISpec) { + *out = *in + out.CloudKittyTemplate = in.CloudKittyTemplate + in.CloudKittyAPITemplate.DeepCopyInto(&out.CloudKittyAPITemplate) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyAPISpec. +func (in *CloudKittyAPISpec) DeepCopy() *CloudKittyAPISpec { + if in == nil { + return nil + } + out := new(CloudKittyAPISpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyAPIStatus) DeepCopyInto(out *CloudKittyAPIStatus) { + *out = *in + if in.Hash != nil { + in, out := &in.Hash, &out.Hash + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.APIEndpoints != nil { + in, out := &in.APIEndpoints, &out.APIEndpoints + *out = make(map[string]map[string]string, len(*in)) + for key, val := range *in { + var outVal map[string]string + if val == nil { + (*out)[key] = nil + } else { + inVal := (*in)[key] + in, out := &inVal, &outVal + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + (*out)[key] = outVal + } + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(condition.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ServiceIDs != nil { + in, out := &in.ServiceIDs, &out.ServiceIDs + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.NetworkAttachments != nil { + in, out := &in.NetworkAttachments, &out.NetworkAttachments + *out = make(map[string][]string, len(*in)) + for key, val := range *in { + var outVal []string + if val == nil { + (*out)[key] = nil + } else { + inVal := (*in)[key] + in, out := &inVal, &outVal + *out = make([]string, len(*in)) + copy(*out, *in) + } + (*out)[key] = outVal + } + } + if in.LastAppliedTopology != nil { + in, out := &in.LastAppliedTopology, &out.LastAppliedTopology + *out = new(topologyv1beta1.TopoRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyAPIStatus. +func (in *CloudKittyAPIStatus) DeepCopy() *CloudKittyAPIStatus { + if in == nil { + return nil + } + out := new(CloudKittyAPIStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyAPITemplate) DeepCopyInto(out *CloudKittyAPITemplate) { + *out = *in + in.CloudKittyAPITemplateCore.DeepCopyInto(&out.CloudKittyAPITemplateCore) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyAPITemplate. +func (in *CloudKittyAPITemplate) DeepCopy() *CloudKittyAPITemplate { + if in == nil { + return nil + } + out := new(CloudKittyAPITemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyAPITemplateCore) DeepCopyInto(out *CloudKittyAPITemplateCore) { + *out = *in + in.CloudKittyServiceTemplate.DeepCopyInto(&out.CloudKittyServiceTemplate) + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + in.Override.DeepCopyInto(&out.Override) + in.TLS.DeepCopyInto(&out.TLS) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyAPITemplateCore. +func (in *CloudKittyAPITemplateCore) DeepCopy() *CloudKittyAPITemplateCore { + if in == nil { + return nil + } + out := new(CloudKittyAPITemplateCore) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyDefaults) DeepCopyInto(out *CloudKittyDefaults) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyDefaults. +func (in *CloudKittyDefaults) DeepCopy() *CloudKittyDefaults { + if in == nil { + return nil + } + out := new(CloudKittyDefaults) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyList) DeepCopyInto(out *CloudKittyList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CloudKitty, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyList. +func (in *CloudKittyList) DeepCopy() *CloudKittyList { + if in == nil { + return nil + } + out := new(CloudKittyList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CloudKittyList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyProc) DeepCopyInto(out *CloudKittyProc) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProc. +func (in *CloudKittyProc) DeepCopy() *CloudKittyProc { + if in == nil { + return nil + } + out := new(CloudKittyProc) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CloudKittyProc) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyProcList) DeepCopyInto(out *CloudKittyProcList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CloudKittyProc, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProcList. +func (in *CloudKittyProcList) DeepCopy() *CloudKittyProcList { + if in == nil { + return nil + } + out := new(CloudKittyProcList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CloudKittyProcList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyProcSpec) DeepCopyInto(out *CloudKittyProcSpec) { + *out = *in + out.CloudKittyTemplate = in.CloudKittyTemplate + in.CloudKittyProcTemplate.DeepCopyInto(&out.CloudKittyProcTemplate) + out.TLS = in.TLS +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProcSpec. +func (in *CloudKittyProcSpec) DeepCopy() *CloudKittyProcSpec { + if in == nil { + return nil + } + out := new(CloudKittyProcSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyProcStatus) DeepCopyInto(out *CloudKittyProcStatus) { + *out = *in + if in.Hash != nil { + in, out := &in.Hash, &out.Hash + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(condition.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.NetworkAttachments != nil { + in, out := &in.NetworkAttachments, &out.NetworkAttachments + *out = make(map[string][]string, len(*in)) + for key, val := range *in { + var outVal []string + if val == nil { + (*out)[key] = nil + } else { + inVal := (*in)[key] + in, out := &inVal, &outVal + *out = make([]string, len(*in)) + copy(*out, *in) + } + (*out)[key] = outVal + } + } + if in.LastAppliedTopology != nil { + in, out := &in.LastAppliedTopology, &out.LastAppliedTopology + *out = new(topologyv1beta1.TopoRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProcStatus. +func (in *CloudKittyProcStatus) DeepCopy() *CloudKittyProcStatus { + if in == nil { + return nil + } + out := new(CloudKittyProcStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyProcTemplate) DeepCopyInto(out *CloudKittyProcTemplate) { + *out = *in + in.CloudKittyProcTemplateCore.DeepCopyInto(&out.CloudKittyProcTemplateCore) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProcTemplate. +func (in *CloudKittyProcTemplate) DeepCopy() *CloudKittyProcTemplate { + if in == nil { + return nil + } + out := new(CloudKittyProcTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyProcTemplateCore) DeepCopyInto(out *CloudKittyProcTemplateCore) { + *out = *in + in.CloudKittyServiceTemplate.DeepCopyInto(&out.CloudKittyServiceTemplate) + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProcTemplateCore. +func (in *CloudKittyProcTemplateCore) DeepCopy() *CloudKittyProcTemplateCore { + if in == nil { + return nil + } + out := new(CloudKittyProcTemplateCore) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittySection) DeepCopyInto(out *CloudKittySection) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + in.CloudKittySpec.DeepCopyInto(&out.CloudKittySpec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittySection. +func (in *CloudKittySection) DeepCopy() *CloudKittySection { + if in == nil { + return nil + } + out := new(CloudKittySection) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyServiceTemplate) DeepCopyInto(out *CloudKittyServiceTemplate) { + *out = *in + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = new(map[string]string) + if **in != nil { + in, out := *in, *out + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + } + if in.CustomServiceConfigSecrets != nil { + in, out := &in.CustomServiceConfigSecrets, &out.CustomServiceConfigSecrets + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.Resources.DeepCopyInto(&out.Resources) + if in.NetworkAttachments != nil { + in, out := &in.NetworkAttachments, &out.NetworkAttachments + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.TopologyRef != nil { + in, out := &in.TopologyRef, &out.TopologyRef + *out = new(topologyv1beta1.TopoRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyServiceTemplate. +func (in *CloudKittyServiceTemplate) DeepCopy() *CloudKittyServiceTemplate { + if in == nil { + return nil + } + out := new(CloudKittyServiceTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittySpec) DeepCopyInto(out *CloudKittySpec) { + *out = *in + in.CloudKittySpecBase.DeepCopyInto(&out.CloudKittySpecBase) + in.CloudKittyAPI.DeepCopyInto(&out.CloudKittyAPI) + in.CloudKittyProc.DeepCopyInto(&out.CloudKittyProc) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittySpec. +func (in *CloudKittySpec) DeepCopy() *CloudKittySpec { + if in == nil { + return nil + } + out := new(CloudKittySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittySpecBase) DeepCopyInto(out *CloudKittySpecBase) { + *out = *in + out.CloudKittyTemplate = in.CloudKittyTemplate + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = new(map[string]string) + if **in != nil { + in, out := *in, *out + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + } + if in.TopologyRef != nil { + in, out := &in.TopologyRef, &out.TopologyRef + *out = new(topologyv1beta1.TopoRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittySpecBase. +func (in *CloudKittySpecBase) DeepCopy() *CloudKittySpecBase { + if in == nil { + return nil + } + out := new(CloudKittySpecBase) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittySpecCore) DeepCopyInto(out *CloudKittySpecCore) { + *out = *in + in.CloudKittySpecBase.DeepCopyInto(&out.CloudKittySpecBase) + in.CloudKittyAPI.DeepCopyInto(&out.CloudKittyAPI) + in.CloudKittyProc.DeepCopyInto(&out.CloudKittyProc) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittySpecCore. +func (in *CloudKittySpecCore) DeepCopy() *CloudKittySpecCore { + if in == nil { + return nil + } + out := new(CloudKittySpecCore) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyStatus) DeepCopyInto(out *CloudKittyStatus) { + *out = *in + if in.Hash != nil { + in, out := &in.Hash, &out.Hash + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(condition.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.APIEndpoints != nil { + in, out := &in.APIEndpoints, &out.APIEndpoints + *out = make(map[string]map[string]string, len(*in)) + for key, val := range *in { + var outVal map[string]string + if val == nil { + (*out)[key] = nil + } else { + inVal := (*in)[key] + in, out := &inVal, &outVal + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + (*out)[key] = outVal + } + } + if in.ServiceIDs != nil { + in, out := &in.ServiceIDs, &out.ServiceIDs + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyStatus. +func (in *CloudKittyStatus) DeepCopy() *CloudKittyStatus { + if in == nil { + return nil + } + out := new(CloudKittyStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittyTemplate) DeepCopyInto(out *CloudKittyTemplate) { + *out = *in + out.PasswordSelectors = in.PasswordSelectors +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyTemplate. +func (in *CloudKittyTemplate) DeepCopy() *CloudKittyTemplate { + if in == nil { + return nil + } + out := new(CloudKittyTemplate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KSMStatus) DeepCopyInto(out *KSMStatus) { *out = *in @@ -1047,6 +1676,7 @@ func (in *TelemetrySpecBase) DeepCopyInto(out *TelemetrySpecBase) { *out = *in in.MetricStorage.DeepCopyInto(&out.MetricStorage) in.Logging.DeepCopyInto(&out.Logging) + in.CloudKitty.DeepCopyInto(&out.CloudKitty) if in.NodeSelector != nil { in, out := &in.NodeSelector, &out.NodeSelector *out = new(map[string]string) diff --git a/config/crd/bases/telemetry.openstack.org_autoscalings.yaml b/config/crd/bases/telemetry.openstack.org_autoscalings.yaml index 4cfabab5..a4ef6ccf 100644 --- a/config/crd/bases/telemetry.openstack.org_autoscalings.yaml +++ b/config/crd/bases/telemetry.openstack.org_autoscalings.yaml @@ -294,6 +294,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object preserveJobs: default: false diff --git a/config/crd/bases/telemetry.openstack.org_ceilometers.yaml b/config/crd/bases/telemetry.openstack.org_ceilometers.yaml index 824f5daa..3307bc27 100644 --- a/config/crd/bases/telemetry.openstack.org_ceilometers.yaml +++ b/config/crd/bases/telemetry.openstack.org_ceilometers.yaml @@ -210,6 +210,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object proxyImage: type: string diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml new file mode 100644 index 00000000..bc5b7567 --- /dev/null +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -0,0 +1,665 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: cloudkitties.telemetry.openstack.org +spec: + group: telemetry.openstack.org + names: + kind: CloudKitty + listKind: CloudKittyList + plural: cloudkitties + singular: cloudkitty + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: CloudKitty is the Schema for the cloudkitties API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CloudKittySpec defines the desired state of CloudKitty + properties: + apiTimeout: + default: 60 + description: APITimeout for HAProxy, Apache, and rpc_response_timeout + minimum: 10 + type: integer + cloudKittyAPI: + description: CloudKittyAPI - Spec definition for the API service of + this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the + configurations of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret + for the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object + public: + description: Public GenericService - holds the secret + for the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for + the service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in + a pre-created bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + cloudKittyProc: + description: CloudKittyProc - Spec definition for the Scheduler service + of this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config for all CloudKitty services using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for cloudkitty + DB, defaults to cloudkitty + type: string + databaseInstance: + description: |- + MariaDB instance name + Right now required by the maridb-operator to get the credentials from the instance to create the DB + Might not be required in future + type: string + memcachedInstance: + default: memcached + description: Memcached instance name. + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting + NodeSelector here acts as a default value and can be overridden by service + specific NodeSelector Settings. + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service password + from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean + rabbitMqClusterName: + default: rabbitmq + description: |- + RabbitMQ instance name + Needed to request a transportURL that is created and used in CloudKitty + type: string + secret: + description: Secret containing OpenStack password information + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - cloudKittyAPI + - cloudKittyProc + - databaseInstance + - memcachedInstance + - rabbitMqClusterName + - secret + type: object + status: + description: CloudKittyStatus defines the observed state of CloudKitty + properties: + apiEndpoints: + additionalProperties: + additionalProperties: + type: string + type: object + description: API endpoints + type: object + cloudKittyAPIReadyCount: + default: 0 + description: ReadyCount of CloudKitty API instance + format: int32 + minimum: 0 + type: integer + cloudKittyProcReadyCounts: + default: 0 + description: ReadyCount of CloudKitty Processor instances + format: int32 + minimum: 0 + type: integer + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + databaseHostname: + description: CloudKitty Database Hostname + type: string + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object + observedGeneration: + description: |- + ObservedGeneration - the most recent generation observed for this service. + If the observed generation is different than the spec generation, then the + controller has not started processing the latest changes, and the status + and its conditions are likely stale. + format: int64 + type: integer + serviceIDs: + additionalProperties: + type: string + description: ServiceIDs + type: object + transportURLSecret: + description: TransportURLSecret - Secret containing RabbitMQ transportURL + type: string + required: + - cloudKittyAPIReadyCount + - cloudKittyProcReadyCounts + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml new file mode 100644 index 00000000..d33b713c --- /dev/null +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml @@ -0,0 +1,499 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: cloudkittyapis.telemetry.openstack.org +spec: + group: telemetry.openstack.org + names: + kind: CloudKittyAPI + listKind: CloudKittyAPIList + plural: cloudkittyapis + singular: cloudkittyapi + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: CloudKittyAPI is the Schema for the cloudkittyapis API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CloudKittyAPISpec defines the desired state of CloudKittyAPI + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for cloudkitty + DB, defaults to cloudkitty + type: string + databaseHostname: + description: DatabaseHostname - CloudKitty Database Hostname + type: string + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment resource + names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + override: + description: Override, provides the ability to override the generated + manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service password + from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + secret: + description: Secret containing OpenStack password information + type: string + serviceAccount: + description: ServiceAccount - service account name used internally + to provide CloudKitty services the default SA name + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret for + the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + public: + description: Public GenericService - holds the secret for + the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + transportURLSecret: + description: Secret containing RabbitMq transport URL + type: string + required: + - containerImage + - databaseHostname + - secret + - serviceAccount + - transportURLSecret + type: object + status: + description: CloudKittyAPIStatus defines the observed state of CloudKittyAPI + properties: + apiEndpoints: + additionalProperties: + additionalProperties: + type: string + type: object + description: API endpoints + type: object + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object + lastAppliedTopology: + description: LastAppliedTopology - the last applied Topology + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + networkAttachments: + additionalProperties: + items: + type: string + type: array + description: NetworkAttachments status of the deployment pods + type: object + observedGeneration: + description: |- + ObservedGeneration - the most recent generation observed for this service. + If the observed generation is different than the spec generation, then the + controller has not started processing the latest changes, and the status + and its conditions are likely stale. + format: int64 + type: integer + readyCount: + default: 0 + description: ReadyCount of CloudKitty API instances + format: int32 + minimum: 0 + type: integer + serviceIDs: + additionalProperties: + type: string + description: ServiceIDs + type: object + required: + - readyCount + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml new file mode 100644 index 00000000..9cd9b6ef --- /dev/null +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -0,0 +1,321 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: cloudkittyprocs.telemetry.openstack.org +spec: + group: telemetry.openstack.org + names: + kind: CloudKittyProc + listKind: CloudKittyProcList + plural: cloudkittyprocs + singular: cloudkittyproc + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: NetworkAttachments + jsonPath: .status.networkAttachments + name: NetworkAttachments + type: string + - description: Status + jsonPath: .status.conditions[0].status + name: Status + type: string + - description: Message + jsonPath: .status.conditions[0].message + name: Message + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: CloudKittyProc is the Schema for the cloudkittprocs API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CloudKittyProcSpec defines the desired state of CloudKitty + Processor + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL (will + be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for cloudkitty + DB, defaults to cloudkitty + type: string + databaseHostname: + description: DatabaseHostname - CloudKitty Database Hostname + type: string + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment resource + names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service password + from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + secret: + description: Secret containing OpenStack password information + type: string + serviceAccount: + description: ServiceAccount - service account name used internally + to provide CloudKitty services the default SA name + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + transportURLSecret: + description: Secret containing RabbitMq transport URL + type: string + required: + - containerImage + - databaseHostname + - secret + - serviceAccount + - transportURLSecret + type: object + status: + description: CloudKittyProcStatus defines the observed state of CloudKitty + Processor + properties: + conditions: + description: Conditions + items: + description: Condition defines an observation of a API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition + in CamelCase. + type: string + severity: + description: |- + Severity provides a classification of Reason code, so the current situation is immediately + understandable and could act accordingly. + It is meant for situations where Status=False and it should be indicated if it is just + informational, warning (next reconciliation might fix it) or an error (e.g. DB create issue + and no actions to automatically resolve the issue can/should be done). + For conditions where Status=Unknown or Status=True the Severity should be SeverityNone. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition in CamelCase. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + hash: + additionalProperties: + type: string + description: Map of hashes to track e.g. job status + type: object + lastAppliedTopology: + description: LastAppliedTopology - the last applied Topology + properties: + name: + description: Name - The Topology CR name that the Service references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + networkAttachments: + additionalProperties: + items: + type: string + type: array + description: NetworkAttachments status of the deployment pods + type: object + observedGeneration: + description: |- + ObservedGeneration - the most recent generation observed for this service. + If the observed generation is different than the spec generation, then the + controller has not started processing the latest changes, and the status + and its conditions are likely stale. + format: int64 + type: integer + readyCount: + default: 0 + description: ReadyCount of CloudKitty Processor instances + format: int32 + minimum: 0 + type: integer + required: + - readyCount + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index 6c18bdfc..0d8bf4d3 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -297,6 +297,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object preserveJobs: default: false @@ -525,6 +530,11 @@ spec: description: CeilometerService - Selector to get the ceilometer service password from the Secret type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string type: object proxyImage: type: string @@ -584,6 +594,542 @@ spec: - secret - sgCoreImage type: object + cloudkitty: + description: CloudKitty - Parameters related to the cloudkitty service + properties: + apiTimeout: + default: 60 + description: APITimeout for HAProxy, Apache, and rpc_response_timeout + minimum: 10 + type: integer + cloudKittyAPI: + description: CloudKittyAPI - Spec definition for the API service + of this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL + (will be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + override: + description: Override, provides the ability to override the + generated manifest of several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + x-kubernetes-list-type: atomic + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains + the configurations of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret + for the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key + for the service + type: string + type: object + public: + description: Public GenericService - holds the secret + for the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key + for the service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs + in a pre-created bundle file + type: string + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + cloudKittyProc: + description: CloudKittyProc - Spec definition for the Scheduler + service of this CloudKitty deployment + properties: + containerImage: + description: ContainerImage - CloudKitty Container Image URL + (will be set to environmental default if empty) + type: string + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + customServiceConfigSecrets: + description: |- + CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets + that contain sensitive service config data. The content of each Secret gets added to the + /etc//.conf.d directory as a custom config file. + items: + type: string + type: array + networkAttachments: + description: NetworkAttachments is a list of NetworkAttachment + resource names to expose the services to the given network + items: + type: string + type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting here overrides + any global NodeSelector settings within the CloudKitty CR. + type: object + replicas: + default: 1 + description: Replicas - CloudKitty API Replicas + format: int32 + minimum: 0 + type: integer + resources: + description: |- + Resources - Compute Resources required by this service (Limits/Requests). + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - containerImage + type: object + customServiceConfig: + description: |- + CustomServiceConfig - customize the service config for all CloudKitty services using this parameter to change service defaults, + or overwrite rendered information using raw OpenStack config format. The content gets added to + to /etc//.conf.d directory as a custom config file. + type: string + databaseAccount: + default: cloudkitty + description: DatabaseAccount - optional MariaDBAccount used for + cloudkitty DB, defaults to cloudkitty + type: string + databaseInstance: + description: |- + MariaDB instance name + Right now required by the maridb-operator to get the credentials from the instance to create the DB + Might not be required in future + type: string + enabled: + default: true + description: Enabled - Whether OpenStack CloudKitty service should + be deployed and managed + type: boolean + memcachedInstance: + default: memcached + description: Memcached instance name. + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector to target subset of worker nodes running this service. Setting + NodeSelector here acts as a default value and can be overridden by service + specific NodeSelector Settings. + type: object + passwordSelector: + default: + cloudKittyService: CloudKittyPassword + description: PasswordsSelectors - Selectors to identify the ServiceUser + password from the Secret + properties: + aodhService: + default: AodhPassword + description: AodhService - Selector to get the aodh service + password from the Secret + type: string + ceilometerService: + default: CeilometerPassword + description: CeilometerService - Selector to get the ceilometer + service password from the Secret + type: string + cloudKittyService: + default: CloudKittyPassword + description: CloudKittyService - Selector to get the CloudKitty + service password from the Secret + type: string + type: object + preserveJobs: + default: false + description: PreserveJobs - do not delete jobs after they finished + e.g. to check logs + type: boolean + rabbitMqClusterName: + default: rabbitmq + description: |- + RabbitMQ instance name + Needed to request a transportURL that is created and used in CloudKitty + type: string + secret: + description: Secret containing OpenStack password information + type: string + serviceUser: + default: cloudkitty + description: ServiceUser - optional username used for this service + to register in cloudkitty + type: string + topologyRef: + description: |- + TopologyRef to apply the Topology defined by the associated CR referenced + by name + properties: + name: + description: Name - The Topology CR name that the Service + references + type: string + namespace: + description: |- + Namespace - The Namespace to fetch the Topology CR referenced + NOTE: Namespace currently points by default to the same namespace where + the Service is deployed. Customizing the namespace is not supported and + webhooks prevent editing this field to a value different from the + current project + type: string + type: object + required: + - cloudKittyAPI + - cloudKittyProc + - databaseInstance + - memcachedInstance + - rabbitMqClusterName + - secret + type: object logging: description: Logging - Parameters related to the logging properties: diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 703cccdd..e6a95c46 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -7,6 +7,9 @@ resources: - bases/telemetry.openstack.org_ceilometers.yaml - bases/telemetry.openstack.org_loggings.yaml - bases/telemetry.openstack.org_metricstorages.yaml +- bases/telemetry.openstack.org_cloudkittyapis.yaml +- bases/telemetry.openstack.org_cloudkittyprocs.yaml +- bases/telemetry.openstack.org_cloudkitties.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -17,6 +20,9 @@ patchesStrategicMerge: #- patches/webhook_in_autoscalings.yaml #- patches/webhook_in_loggings.yaml #- patches/webhook_in_metricstorages.yaml +#- patches/webhook_in_cloudkittyapis.yaml +#- patches/webhook_in_cloudkittyprocs.yaml +#- patches/webhook_in_cloudkitties.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. @@ -26,6 +32,9 @@ patchesStrategicMerge: #- patches/cainjection_in_autoscalings.yaml #- patches/cainjection_in_loggings.yaml #- patches/cainjection_in_metricstorages.yaml +#- patches/cainjection_in_cloudkittyapis.yaml +#- patches/cainjection_in_cloudkittyprocs.yaml +#- patches/cainjection_in_cloudkitties.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/crd/patches/cainjection_in_cloudkitties.yaml b/config/crd/patches/cainjection_in_cloudkitties.yaml new file mode 100644 index 00000000..b0360335 --- /dev/null +++ b/config/crd/patches/cainjection_in_cloudkitties.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: cloudkitties.telemetry.openstack.org diff --git a/config/crd/patches/cainjection_in_cloudkittyapis.yaml b/config/crd/patches/cainjection_in_cloudkittyapis.yaml new file mode 100644 index 00000000..35526ad6 --- /dev/null +++ b/config/crd/patches/cainjection_in_cloudkittyapis.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: cloudkittyapis.telemetry.openstack.org diff --git a/config/crd/patches/cainjection_in_cloudkittyprocs.yaml b/config/crd/patches/cainjection_in_cloudkittyprocs.yaml new file mode 100644 index 00000000..bfc383ec --- /dev/null +++ b/config/crd/patches/cainjection_in_cloudkittyprocs.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: cloudkittyprocs.telemetry.openstack.org diff --git a/config/crd/patches/webhook_in_cloudkitties.yaml b/config/crd/patches/webhook_in_cloudkitties.yaml new file mode 100644 index 00000000..18a2c841 --- /dev/null +++ b/config/crd/patches/webhook_in_cloudkitties.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cloudkitties.telemetry.openstack.org +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/crd/patches/webhook_in_cloudkittyapis.yaml b/config/crd/patches/webhook_in_cloudkittyapis.yaml new file mode 100644 index 00000000..cad85de9 --- /dev/null +++ b/config/crd/patches/webhook_in_cloudkittyapis.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cloudkittyapis.telemetry.openstack.org +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/crd/patches/webhook_in_cloudkittyprocs.yaml b/config/crd/patches/webhook_in_cloudkittyprocs.yaml new file mode 100644 index 00000000..90c0d18e --- /dev/null +++ b/config/crd/patches/webhook_in_cloudkittyprocs.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cloudkittyprocs.telemetry.openstack.org +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/rbac/cloudkitty_editor_role.yaml b/config/rbac/cloudkitty_editor_role.yaml new file mode 100644 index 00000000..940f3121 --- /dev/null +++ b/config/rbac/cloudkitty_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit cloudkitties. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: cloudkitty-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: telemetry-operator + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + name: cloudkitty-editor-role +rules: +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkitties + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkitties/status + verbs: + - get diff --git a/config/rbac/cloudkitty_viewer_role.yaml b/config/rbac/cloudkitty_viewer_role.yaml new file mode 100644 index 00000000..253ecd75 --- /dev/null +++ b/config/rbac/cloudkitty_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view cloudkitties. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: cloudkitty-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: telemetry-operator + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + name: cloudkitty-viewer-role +rules: +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkitties + verbs: + - get + - list + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkitties/status + verbs: + - get diff --git a/config/rbac/cloudkittyapi_editor_role.yaml b/config/rbac/cloudkittyapi_editor_role.yaml new file mode 100644 index 00000000..2a0e0027 --- /dev/null +++ b/config/rbac/cloudkittyapi_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit cloudkittyapis. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: cloudkittyapi-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: telemetry-operator + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + name: cloudkittyapi-editor-role +rules: +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyapis + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyapis/status + verbs: + - get diff --git a/config/rbac/cloudkittyapi_viewer_role.yaml b/config/rbac/cloudkittyapi_viewer_role.yaml new file mode 100644 index 00000000..56bc7181 --- /dev/null +++ b/config/rbac/cloudkittyapi_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view cloudkittyapis. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: cloudkittyapi-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: telemetry-operator + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + name: cloudkittyapi-viewer-role +rules: +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyapis + verbs: + - get + - list + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyapis/status + verbs: + - get diff --git a/config/rbac/cloudkittyproc_editor_role.yaml b/config/rbac/cloudkittyproc_editor_role.yaml new file mode 100644 index 00000000..dcb4eac3 --- /dev/null +++ b/config/rbac/cloudkittyproc_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit cloudkittyprocs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: cloudkittyproc-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: telemetry-operator + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + name: cloudkittyproc-editor-role +rules: +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyprocs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyprocs/status + verbs: + - get diff --git a/config/rbac/cloudkittyproc_viewer_role.yaml b/config/rbac/cloudkittyproc_viewer_role.yaml new file mode 100644 index 00000000..a852a6f4 --- /dev/null +++ b/config/rbac/cloudkittyproc_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view cloudkittyprocs. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: cloudkittyproc-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: telemetry-operator + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + name: cloudkittyproc-viewer-role +rules: +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyprocs + verbs: + - get + - list + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyprocs/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6e34456b..6fabb4df 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -4,6 +4,18 @@ kind: ClusterRole metadata: name: manager-role rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: @@ -46,6 +58,33 @@ rules: - patch - update - watch +- apiGroups: + - cloudkitty.openstack.org + resources: + - cloudkittyprocs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - cloudkitty.openstack.org + resources: + - cloudkittyprocs/finalizers + verbs: + - patch + - update +- apiGroups: + - cloudkitty.openstack.org + resources: + - cloudkittyprocs/status + verbs: + - get + - patch + - update - apiGroups: - "" resources: @@ -330,6 +369,15 @@ rules: - securitycontextconstraints verbs: - use +- apiGroups: + - security.openshift.io + resourceNames: + - anyuid + - privileged + resources: + - securitycontextconstraints + verbs: + - use - apiGroups: - telemetry.openstack.org resources: @@ -386,6 +434,88 @@ rules: - get - patch - update +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkitties + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkitties/finalizers + verbs: + - delete + - patch + - update +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkitties/status + verbs: + - get + - patch + - update +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyapis + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyapis/finalizers + verbs: + - patch + - update +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyapis/status + verbs: + - get + - patch + - update +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyprocs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyprocs/finalizers + verbs: + - patch + - update +- apiGroups: + - telemetry.openstack.org + resources: + - cloudkittyprocs/status + verbs: + - get + - patch + - update - apiGroups: - telemetry.openstack.org resources: diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index 7e00a20f..6e689eee 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -5,4 +5,7 @@ resources: - telemetry_v1beta1_autoscaling.yaml - telemetry_v1beta1_logging.yaml - telemetry_v1beta1_metricstorage.yaml +- telemetry_v1beta1_cloudkittyapi.yaml +- telemetry_v1beta1_cloudkittyproc.yaml +- telemetry_v1beta1_cloudkitty.yaml #+kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/telemetry_v1beta1_cloudkitty.yaml b/config/samples/telemetry_v1beta1_cloudkitty.yaml new file mode 100644 index 00000000..0f649ad3 --- /dev/null +++ b/config/samples/telemetry_v1beta1_cloudkitty.yaml @@ -0,0 +1,12 @@ +apiVersion: telemetry.openstack.org/v1beta1 +kind: CloudKitty +metadata: + labels: + app.kubernetes.io/name: cloudkitty + app.kubernetes.io/instance: cloudkitty-sample + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: telemetry-operator + name: cloudkitty-sample +spec: + # TODO(user): Add fields here diff --git a/config/samples/telemetry_v1beta1_cloudkittyapi.yaml b/config/samples/telemetry_v1beta1_cloudkittyapi.yaml new file mode 100644 index 00000000..47725bf7 --- /dev/null +++ b/config/samples/telemetry_v1beta1_cloudkittyapi.yaml @@ -0,0 +1,12 @@ +apiVersion: telemetry.openstack.org/v1beta1 +kind: CloudKittyApi +metadata: + labels: + app.kubernetes.io/name: cloudkittyapi + app.kubernetes.io/instance: cloudkittyapi-sample + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: telemetry-operator + name: cloudkittyapi-sample +spec: + # TODO(user): Add fields here diff --git a/config/samples/telemetry_v1beta1_cloudkittyproc.yaml b/config/samples/telemetry_v1beta1_cloudkittyproc.yaml new file mode 100644 index 00000000..94e90340 --- /dev/null +++ b/config/samples/telemetry_v1beta1_cloudkittyproc.yaml @@ -0,0 +1,12 @@ +apiVersion: telemetry.openstack.org/v1beta1 +kind: CloudKittyProc +metadata: + labels: + app.kubernetes.io/name: cloudkittyproc + app.kubernetes.io/instance: cloudkittyproc-sample + app.kubernetes.io/part-of: telemetry-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: telemetry-operator + name: cloudkittyproc-sample +spec: + # TODO(user): Add fields here diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index 288d1e6f..c6b4c02c 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -44,6 +44,26 @@ webhooks: resources: - ceilometers sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-telemetry-openstack-org-v1beta1-cloudkitty + failurePolicy: Fail + name: mcloudkitty.kb.io + rules: + - apiGroups: + - telemetry.openstack.org + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - cloudkitties + sideEffects: None - admissionReviewVersions: - v1 clientConfig: @@ -130,6 +150,26 @@ webhooks: resources: - ceilometers sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-telemetry-openstack-org-v1beta1-cloudkitty + failurePolicy: Fail + name: vcloudkitty.kb.io + rules: + - apiGroups: + - telemetry.openstack.org + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - cloudkitties + sideEffects: None - admissionReviewVersions: - v1 clientConfig: diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go new file mode 100644 index 00000000..4dadd651 --- /dev/null +++ b/controllers/cloudkitty_controller.go @@ -0,0 +1,1085 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/go-logr/logr" + networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" + rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" + keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/job" + "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" + common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" + "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" + mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// GetClient - +func (r *CloudKittyReconciler) GetClient() client.Client { + return r.Client +} + +// GetKClient - +func (r *CloudKittyReconciler) GetKClient() kubernetes.Interface { + return r.Kclient +} + +// GetScheme - +func (r *CloudKittyReconciler) GetScheme() *runtime.Scheme { + return r.Scheme +} + +// CloudKittyReconciler reconciles a CloudKitty object +type CloudKittyReconciler struct { + client.Client + Kclient kubernetes.Interface + Scheme *runtime.Scheme +} + +// GetLogger returns a logger object with a logging prefix of "controller.name" and additional controller context fields +func (r *CloudKittyReconciler) GetLogger(ctx context.Context) logr.Logger { + return log.FromContext(ctx).WithName("Controllers").WithName("CloudKitty") +} + +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkitties,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkitties/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkitties/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyapis,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyapis/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyapis/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyprocs,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyprocs/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyprocs/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbdatabases,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbaccounts,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbaccounts/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=memcached.openstack.org,resources=memcacheds,verbs=get;list;watch; +// +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneapis,verbs=get;list;watch +// +kubebuilder:rbac:groups=rabbitmq.openstack.org,resources=transporturls,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch + +// service account, role, rolebinding +// +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=get;list;watch;create;update;patch +// +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;update;patch +// service account permissions that are needed to grant permission to the above +// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=anyuid;privileged,resources=securitycontextconstraints,verbs=use +// +kubebuilder:rbac:groups="",resources=pods,verbs=create;delete;get;list;patch;update;watch + +// Reconcile - +func (r *CloudKittyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { + Log := r.GetLogger(ctx) + + // Fetch the CloudKitty instance + instance := &telemetryv1.CloudKitty{} + err := r.Client.Get(ctx, req.NamespacedName, instance) + if err != nil { + if k8s_errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. + // For additional cleanup logic use finalizers. Return and don't requeue. + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + Log.Error(err, fmt.Sprintf("could not fetch CloudKitty instance %s", instance.Name)) + return ctrl.Result{}, err + } + + helper, err := helper.NewHelper( + instance, + r.Client, + r.Kclient, + r.Scheme, + Log, + ) + if err != nil { + Log.Error(err, fmt.Sprintf("could not instantiate helper for instance %s", instance.Name)) + return ctrl.Result{}, err + } + + // + // initialize status + // + isNewInstance := instance.Status.Conditions == nil + if isNewInstance { + instance.Status.Conditions = condition.Conditions{} + } + + // Save a copy of the condtions so that we can restore the LastTransitionTime + // when a condition's state doesn't change. + savedConditions := instance.Status.Conditions.DeepCopy() + + // Always patch the instance status when exiting this function so we can persist any changes. + defer func() { + // Don't update the status, if reconciler Panics + if r := recover(); r != nil { + Log.Info(fmt.Sprintf("panic during reconcile %v\n", r)) + panic(r) + } + condition.RestoreLastTransitionTimes(&instance.Status.Conditions, savedConditions) + if instance.Status.Conditions.IsUnknown(condition.ReadyCondition) { + instance.Status.Conditions.Set( + instance.Status.Conditions.Mirror(condition.ReadyCondition)) + } + err := helper.PatchInstance(ctx, instance) + if err != nil { + _err = err + return + } + }() + + // Always initialize conditions used later as Status=Unknown + cl := condition.CreateList( + condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), + condition.UnknownCondition(condition.DBReadyCondition, condition.InitReason, condition.DBReadyInitMessage), + condition.UnknownCondition(condition.DBSyncReadyCondition, condition.InitReason, condition.DBSyncReadyInitMessage), + condition.UnknownCondition(condition.RabbitMqTransportURLReadyCondition, condition.InitReason, condition.RabbitMqTransportURLReadyInitMessage), + condition.UnknownCondition(condition.MemcachedReadyCondition, condition.InitReason, condition.MemcachedReadyInitMessage), + condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), + condition.UnknownCondition(telemetryv1.CloudKittyAPIReadyCondition, condition.InitReason, telemetryv1.CloudKittyAPIReadyInitMessage), + condition.UnknownCondition(telemetryv1.CloudKittyProcReadyCondition, condition.InitReason, telemetryv1.CloudKittyProcReadyInitMessage), + condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), + // service account, role, rolebinding conditions + condition.UnknownCondition(condition.ServiceAccountReadyCondition, condition.InitReason, condition.ServiceAccountReadyInitMessage), + condition.UnknownCondition(condition.RoleReadyCondition, condition.InitReason, condition.RoleReadyInitMessage), + condition.UnknownCondition(condition.RoleBindingReadyCondition, condition.InitReason, condition.RoleBindingReadyInitMessage), + ) + instance.Status.Conditions.Init(&cl) + // Always mark the Generation as observed early on + instance.Status.ObservedGeneration = instance.Generation + + // If we're not deleting this and the service object doesn't have our finalizer, add it. + if (instance.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(instance, helper.GetFinalizer())) || isNewInstance { + // Register overall status immediately to have an early feedback e.g. in the cli + return ctrl.Result{}, nil + } + + if instance.Status.Hash == nil { + instance.Status.Hash = map[string]string{} + } + if instance.Status.APIEndpoints == nil { + instance.Status.APIEndpoints = map[string]map[string]string{} + } + + // Handle service delete + if !instance.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, instance, helper) + } + + // Handle non-deleted clusters + return r.reconcileNormal(ctx, instance, helper) +} + +// fields to index to reconcile when change +const ( + cloudKittyPasswordSecretField = ".spec.secret" + cloudKittyCaBundleSecretNameField = ".spec.tls.caBundleSecretName" + cloudKittyTlsAPIInternalField = ".spec.tls.api.internal.secretName" + cloudKittyTlsAPIPublicField = ".spec.tls.api.public.secretName" + cloudKittyTopologyField = ".spec.topologyRef.Name" +) + +var ( + commonWatchFields = []string{ + cloudKittyPasswordSecretField, + cloudKittyCaBundleSecretNameField, + cloudKittyTopologyField, + } + cloudKittyAPIWatchFields = []string{ + cloudKittyPasswordSecretField, + cloudKittyCaBundleSecretNameField, + cloudKittyTlsAPIInternalField, + cloudKittyTlsAPIPublicField, + cloudKittyTopologyField, + } +) + +// SetupWithManager sets up the controller with the Manager. +func (r *CloudKittyReconciler) SetupWithManager(mgr ctrl.Manager) error { + // transportURLSecretFn - Watch for changes made to the secret associated with the RabbitMQ + // TransportURL created and used by CloudKitty CRs. Watch functions return a list of namespace-scoped + // CRs that then get fed to the reconciler. Hence, in this case, we need to know the name of the + // CloudKitty CR associated with the secret we are examining in the function. We could parse the name + // out of the "%s-cloudkitty-transport" secret label, which would be faster than getting the list of + // the CloudKitty CRs and trying to match on each one. The downside there, however, is that technically + // someone could randomly label a secret "something-cloudkitty-transport" where "something" actually + // matches the name of an existing CloudKitty CR. In that case changes to that secret would trigger + // reconciliation for a CloudKitty CR that does not need it. + // + // TODO: We also need a watch func to monitor for changes to the secret referenced by CloudKitty.Spec.Secret + transportURLSecretFn := func(ctx context.Context, o client.Object) []reconcile.Request { + result := []reconcile.Request{} + + Log := r.GetLogger(ctx) + + // get all CloudKitty CRs + cloudkitties := &telemetryv1.CloudKittyList{} + listOpts := []client.ListOption{ + client.InNamespace(o.GetNamespace()), + } + if err := r.Client.List(ctx, cloudkitties, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve CloudKitty CRs %v") + return nil + } + + for _, ownerRef := range o.GetOwnerReferences() { + if ownerRef.Kind == "TransportURL" { + for _, cr := range cloudkitties.Items { + if ownerRef.Name == fmt.Sprintf("%s-cloudkitty-transport", cr.Name) { + // return namespace and Name of CR + name := client.ObjectKey{ + Namespace: o.GetNamespace(), + Name: cr.Name, + } + Log.Info(fmt.Sprintf("TransportURL Secret %s belongs to TransportURL belonging to CloudKitty CR %s", o.GetName(), cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + } + } + if len(result) > 0 { + return result + } + return nil + } + + memcachedFn := func(ctx context.Context, o client.Object) []reconcile.Request { + Log := r.GetLogger(ctx) + + result := []reconcile.Request{} + + // get all CloudKitty CRs + cloudkitties := &telemetryv1.CloudKittyList{} + listOpts := []client.ListOption{ + client.InNamespace(o.GetNamespace()), + } + if err := r.Client.List(ctx, cloudkitties, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve CloudKitty CRs %w") + return nil + } + + for _, cr := range cloudkitties.Items { + if o.GetName() == cr.Spec.MemcachedInstance { + name := client.ObjectKey{ + Namespace: o.GetNamespace(), + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Memcached %s is used by CloudKitty CR %s", o.GetName(), cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + if len(result) > 0 { + return result + } + return nil + } + + return ctrl.NewControllerManagedBy(mgr). + For(&telemetryv1.CloudKitty{}). + Owns(&mariadbv1.MariaDBDatabase{}). + Owns(&mariadbv1.MariaDBAccount{}). + Owns(&telemetryv1.CloudKittyAPI{}). + Owns(&telemetryv1.CloudKittyProc{}). + Owns(&rabbitmqv1.TransportURL{}). + Owns(&batchv1.Job{}). + Owns(&corev1.Secret{}). + Owns(&corev1.ServiceAccount{}). + Owns(&rbacv1.Role{}). + Owns(&rbacv1.RoleBinding{}). + // Watch for TransportURL Secrets which belong to any TransportURLs created by CloudKitty CRs + Watches(&corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(transportURLSecretFn)). + Watches(&memcachedv1.Memcached{}, + handler.EnqueueRequestsFromMapFunc(memcachedFn)). + Watches(&keystonev1.KeystoneAPI{}, + handler.EnqueueRequestsFromMapFunc(r.findObjectForSrc), + builder.WithPredicates(keystonev1.KeystoneAPIStatusChangedPredicate)). + Complete(r) +} + +func (r *CloudKittyReconciler) findObjectForSrc(ctx context.Context, src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + l := log.FromContext(ctx).WithName("Controllers").WithName("CloudKitty") + + crList := &telemetryv1.CloudKittyList{} + listOps := &client.ListOptions{ + Namespace: src.GetNamespace(), + } + err := r.Client.List(ctx, crList, listOps) + if err != nil { + l.Error(err, fmt.Sprintf("listing %s for namespace: %s", crList.GroupVersionKind().Kind, src.GetNamespace())) + return requests + } + + for _, item := range crList.Items { + l.Info(fmt.Sprintf("input source %s changed, reconcile: %s - %s", src.GetName(), item.GetName(), item.GetNamespace())) + + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + + return requests +} + +func (r *CloudKittyReconciler) reconcileDelete(ctx context.Context, instance *telemetryv1.CloudKitty, helper *helper.Helper) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s' delete", instance.Name)) + + // remove db finalizer first + db, err := mariadbv1.GetDatabaseByNameAndAccount(ctx, helper, cloudkitty.DatabaseName, instance.Spec.DatabaseAccount, instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + + if !k8s_errors.IsNotFound(err) { + if err := db.DeleteFinalizer(ctx, helper); err != nil { + return ctrl.Result{}, err + } + } + + // TODO: We might need to control how the sub-services (API and Proc) are + // deleted (when their parent CloudKitty CR is deleted) once we further develop their functionality + + // Service is deleted so remove the finalizer. + controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) + Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", instance.Name)) + + return ctrl.Result{}, nil +} + +func (r *CloudKittyReconciler) reconcileInit( + ctx context.Context, + instance *telemetryv1.CloudKitty, + helper *helper.Helper, + serviceLabels map[string]string, + serviceAnnotations map[string]string, +) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s' init", instance.Name)) + + // + // run CloudKitty db sync + // + dbSyncHash := instance.Status.Hash[telemetryv1.CKDbSyncHash] + jobDef := cloudkitty.DbSyncJob(instance, serviceLabels) + + dbSyncjob := job.NewJob( + jobDef, + telemetryv1.CKDbSyncHash, + instance.Spec.PreserveJobs, + cloudkitty.ShortDuration, + dbSyncHash, + ) + ctrlResult, err := dbSyncjob.DoJob( + ctx, + helper, + ) + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBSyncReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBSyncReadyRunningMessage)) + return ctrlResult, nil + } + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBSyncReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBSyncReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if dbSyncjob.HasChanged() { + instance.Status.Hash[telemetryv1.CKDbSyncHash] = dbSyncjob.GetHash() + Log.Info(fmt.Sprintf("Service '%s' - Job %s hash added - %s", instance.Name, jobDef.Name, instance.Status.Hash[telemetryv1.CKDbSyncHash])) + } + instance.Status.Conditions.MarkTrue(condition.DBSyncReadyCondition, condition.DBSyncReadyMessage) + + // run CloudKitty db sync - end + + Log.Info(fmt.Sprintf("Reconciled Service '%s' init successfully", instance.Name)) + return ctrl.Result{}, nil +} + +func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *telemetryv1.CloudKitty, helper *helper.Helper) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s'", instance.Name)) + + // Service account, role, binding + rbacRules := []rbacv1.PolicyRule{ + { + APIGroups: []string{"security.openshift.io"}, + ResourceNames: []string{"anyuid"}, + Resources: []string{"securitycontextconstraints"}, + Verbs: []string{"use"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"create", "get", "list", "watch", "update", "patch", "delete"}, + }, + } + rbacResult, err := common_rbac.ReconcileRbac(ctx, helper, instance, rbacRules) + if err != nil { + return rbacResult, err + } else if (rbacResult != ctrl.Result{}) { + return rbacResult, nil + } + + serviceLabels := map[string]string{ + common.AppSelector: cloudkitty.ServiceName, + } + + configVars := make(map[string]env.Setter) + + // + // create RabbitMQ transportURL CR and get the actual URL from the associated secret that is created + // + + transportURL, op, err := r.transportURLCreateOrUpdate(ctx, instance, serviceLabels) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.RabbitMqTransportURLReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.RabbitMqTransportURLReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if op != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf("TransportURL %s successfully reconciled - operation: %s", transportURL.Name, string(op))) + } + + instance.Status.TransportURLSecret = transportURL.Status.SecretName + + if instance.Status.TransportURLSecret == "" { + Log.Info(fmt.Sprintf("Waiting for TransportURL %s secret to be created", transportURL.Name)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.RabbitMqTransportURLReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.RabbitMqTransportURLReadyRunningMessage)) + return cloudkitty.ResultRequeue, nil + } + + instance.Status.Conditions.MarkTrue(condition.RabbitMqTransportURLReadyCondition, condition.RabbitMqTransportURLReadyMessage) + + // end transportURL + + // + // Check for required memcached used for caching + // + memcached, err := memcachedv1.GetMemcachedByName(ctx, helper, instance.Spec.MemcachedInstance, instance.Namespace) + if err != nil { + Log.Info(fmt.Sprintf("%s... requeueing", condition.MemcachedReadyWaitingMessage)) + if k8s_errors.IsNotFound(err) { + Log.Info(fmt.Sprintf("memcached %s not found", instance.Spec.MemcachedInstance)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.MemcachedReadyWaitingMessage)) + return cloudkitty.ResultRequeue, nil + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.MemcachedReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if !memcached.IsReady() { + Log.Info(fmt.Sprintf("%s... requeueing", condition.MemcachedReadyWaitingMessage)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.MemcachedReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.MemcachedReadyWaitingMessage)) + return cloudkitty.ResultRequeue, nil + } + // Mark the Memcached Service as Ready if we get to this point with no errors + instance.Status.Conditions.MarkTrue( + condition.MemcachedReadyCondition, condition.MemcachedReadyMessage) + // run check memcached - end + + // + // check for required OpenStack secret holding passwords for service/admin user and add hash to the vars map + // + + result, err := cloudkitty.VerifyServiceSecret( + ctx, + types.NamespacedName{Namespace: instance.Namespace, Name: instance.Spec.Secret}, + []string{ + instance.Spec.PasswordSelectors.CloudKittyService, + }, + helper.GetClient(), + &instance.Status.Conditions, + cloudkitty.NormalDuration, + &configVars, + ) + if err != nil { + return result, err + } else if (result != ctrl.Result{}) { + return result, nil + } + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + // run check OpenStack secret - end + + db, result, err := r.ensureDB(ctx, helper, instance) + if err != nil { + return ctrl.Result{}, err + } else if (result != ctrl.Result{}) { + return result, nil + } + + // + // Create Secrets required as input for the Service and calculate an overall hash of hashes + // + err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels, memcached, db) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + // + // create hash over all the different input resources to identify if any those changed + // and a restart/recreate is required. + // + _, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configVars) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } else if hashChanged { + Log.Info(fmt.Sprintf("%s... requeueing", condition.ServiceConfigReadyInitMessage)) + instance.Status.Conditions.MarkFalse( + condition.ServiceConfigReadyCondition, + condition.InitReason, + condition.SeverityInfo, + condition.ServiceConfigReadyInitMessage) + // Hash changed and instance status should be updated (which will be done by main defer func), + // so we need to return and reconcile again + return ctrl.Result{}, nil + } + + instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) + + // + // TODO check when/if Init, Update, or Upgrade should/could be skipped + // + + // Check networks that the DBSync job will use in reconcileInit. The ones from the API service are always enough, + // it doesn't need the storage specific ones that volume or backup may have. + nadList := []networkv1.NetworkAttachmentDefinition{} + for _, netAtt := range instance.Spec.CloudKittyAPI.NetworkAttachments { + nad, err := nad.GetNADWithName(ctx, helper, netAtt, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + Log.Info(fmt.Sprintf("network-attachment-definition %s not found", netAtt)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.NetworkAttachmentsReadyWaitingMessage, + netAtt)) + return cloudkitty.ResultRequeue, fmt.Errorf(condition.NetworkAttachmentsReadyWaitingMessage, netAtt) + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if nad != nil { + nadList = append(nadList, *nad) + } + } + + instance.Status.Conditions.MarkTrue(condition.NetworkAttachmentsReadyCondition, condition.NetworkAttachmentsReadyMessage) + + serviceAnnotations, err := nad.EnsureNetworksAnnotation(nadList) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed create network annotation from %s: %w", + instance.Spec.CloudKittyAPI.NetworkAttachments, err) + } + + // Handle service init + ctrlResult, err := r.reconcileInit(ctx, instance, helper, serviceLabels, serviceAnnotations) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + // + // normal reconcile tasks + // + + // deploy cloudkitty-api + cloudKittyAPI, op, err := r.apiDeploymentCreateOrUpdate(ctx, instance) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyAPIReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + telemetryv1.CloudKittyAPIReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if op != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf("API CR for %s successfully %s", instance.Name, string(op))) + } + + // Mirror values when the data in the StatefulSet is for the current generation + if cloudKittyAPI.Generation == cloudKittyAPI.Status.ObservedGeneration { + // Mirror CloudKittyAPI status' APIEndpoints and ReadyCount to this parent CR + instance.Status.APIEndpoints = cloudKittyAPI.Status.APIEndpoints + instance.Status.ServiceIDs = cloudKittyAPI.Status.ServiceIDs + instance.Status.CloudKittyAPIReadyCount = cloudKittyAPI.Status.ReadyCount + + // Mirror CloudKittyAPI's condition status + c := cloudKittyAPI.Status.Conditions.Mirror(telemetryv1.CloudKittyAPIReadyCondition) + if c != nil { + instance.Status.Conditions.Set(c) + } + } + + // deploy CloudKitty Processor + cloudKittyProc, op, err := r.procDeploymentCreateOrUpdate(ctx, instance) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyProcReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + telemetryv1.CloudKittyProcReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if op != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf("Scheduler CR for %s successfully %s", instance.Name, string(op))) + } + + // Mirror values when the data in the StatefulSet is for the current generation + if cloudKittyProc.Generation == cloudKittyProc.Status.ObservedGeneration { + // Mirror CloudKitty Processor status' ReadyCount to this parent CR + instance.Status.CloudKittyProcReadyCount = cloudKittyProc.Status.ReadyCount + + // Mirror CloudKittyProc's condition status + c := cloudKittyProc.Status.Conditions.Mirror(telemetryv1.CloudKittyProcReadyCondition) + if c != nil { + instance.Status.Conditions.Set(c) + } + } + + err = mariadbv1.DeleteUnusedMariaDBAccountFinalizers(ctx, helper, cloudkitty.DatabaseName, instance.Spec.DatabaseAccount, instance.Namespace) + if err != nil { + return ctrl.Result{}, err + } + + Log.Info(fmt.Sprintf("Reconciled Service '%s' successfully", instance.Name)) + // update the overall status condition if service is ready + if instance.IsReady() { + instance.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) + } + return ctrl.Result{}, nil +} + +// generateServiceConfigs - create Secret which hold scripts and service configuration +func (r *CloudKittyReconciler) generateServiceConfigs( + ctx context.Context, + h *helper.Helper, + instance *telemetryv1.CloudKitty, + envVars *map[string]env.Setter, + serviceLabels map[string]string, + memcached *memcachedv1.Memcached, + db *mariadbv1.Database, +) error { + // + // create Secret required for cloudkitty input + // - %-scripts holds scripts to e.g. bootstrap the service + // - %-config holds minimal cloudkitty config required to get the service up + // + + labels := labels.GetLabels(instance, labels.GetGroupLabel(cloudkitty.ServiceName), serviceLabels) + + var tlsCfg *tls.Service + if instance.Spec.CloudKittyAPI.TLS.Ca.CaBundleSecretName != "" { + tlsCfg = &tls.Service{} + } + + // customData hold any customization for all cloudkitty services. + customData := map[string]string{ + cloudkitty.CustomConfigFileName: instance.Spec.CustomServiceConfig, + cloudkitty.MyCnfFileName: db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf + } + + keystoneAPI, err := keystonev1.GetKeystoneAPI(ctx, h, instance.Namespace, map[string]string{}) + if err != nil { + return err + } + keystoneInternalURL, err := keystoneAPI.GetEndpoint(endpoint.EndpointInternal) + if err != nil { + return err + } + keystonePublicURL, err := keystoneAPI.GetEndpoint(endpoint.EndpointPublic) + if err != nil { + return err + } + + ospSecret, _, err := secret.GetSecret(ctx, h, instance.Spec.Secret, instance.Namespace) + if err != nil { + return err + } + + transportURLSecret, _, err := secret.GetSecret(ctx, h, instance.Status.TransportURLSecret, instance.Namespace) + if err != nil { + return err + } + + databaseAccount := db.GetAccount() + dbSecret := db.GetSecret() + + templateParameters := make(map[string]interface{}) + templateParameters["ServiceUser"] = instance.Spec.ServiceUser + templateParameters["ServicePassword"] = string(ospSecret.Data[instance.Spec.PasswordSelectors.CloudKittyService]) + templateParameters["KeystoneInternalURL"] = keystoneInternalURL + templateParameters["KeystonePublicURL"] = keystonePublicURL + templateParameters["TransportURL"] = string(transportURLSecret.Data["transport_url"]) + templateParameters["DatabaseConnection"] = fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf", + databaseAccount.Spec.UserName, + string(dbSecret.Data[mariadbv1.DatabasePasswordSelector]), + instance.Status.DatabaseHostname, + cloudkitty.DatabaseName) + templateParameters["MemcachedServersWithInet"] = memcached.GetMemcachedServerListWithInetString() + templateParameters["TimeOut"] = instance.Spec.APITimeout + + // create httpd vhost template parameters + httpdVhostConfig := map[string]interface{}{} + for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} { + endptConfig := map[string]interface{}{} + endptConfig["ServerName"] = fmt.Sprintf("%s-%s.%s.svc", cloudkitty.ServiceName, endpt.String(), instance.Namespace) + endptConfig["TLS"] = false // default TLS to false, and set it bellow to true if enabled + if instance.Spec.CloudKittyAPI.TLS.API.Enabled(endpt) { + endptConfig["TLS"] = true + endptConfig["SSLCertificateFile"] = fmt.Sprintf("/etc/pki/tls/certs/%s.crt", endpt.String()) + endptConfig["SSLCertificateKeyFile"] = fmt.Sprintf("/etc/pki/tls/private/%s.key", endpt.String()) + } + httpdVhostConfig[endpt.String()] = endptConfig + } + templateParameters["VHosts"] = httpdVhostConfig + + configTemplates := []util.Template{ + { + Name: fmt.Sprintf("%s-scripts", instance.Name), + Namespace: instance.Namespace, + Type: util.TemplateTypeScripts, + InstanceType: instance.Kind, + Labels: labels, + }, + { + Name: fmt.Sprintf("%s-config-data", instance.Name), + Namespace: instance.Namespace, + Type: util.TemplateTypeConfig, + InstanceType: instance.Kind, + CustomData: customData, + ConfigOptions: templateParameters, + Labels: labels, + }, + } + + return secret.EnsureSecrets(ctx, h, instance, configTemplates, envVars) +} + +// createHashOfInputHashes - creates a hash of hashes which gets added to the resources which requires a restart +// if any of the input resources change, like configs, passwords, ... +// +// returns the hash, whether the hash changed (as a bool) and any error +func (r *CloudKittyReconciler) createHashOfInputHashes( + ctx context.Context, + instance *telemetryv1.CloudKitty, + envVars map[string]env.Setter, +) (string, bool, error) { + Log := r.GetLogger(ctx) + + var hashMap map[string]string + changed := false + mergedMapVars := env.MergeEnvs([]corev1.EnvVar{}, envVars) + hash, err := util.ObjectHash(mergedMapVars) + if err != nil { + return hash, changed, err + } + if hashMap, changed = util.SetHash(instance.Status.Hash, common.InputHashName, hash); changed { + instance.Status.Hash = hashMap + Log.Info(fmt.Sprintf("Input maps hash %s - %s", common.InputHashName, hash)) + } + return hash, changed, nil +} + +func (r *CloudKittyReconciler) transportURLCreateOrUpdate( + ctx context.Context, + instance *telemetryv1.CloudKitty, + serviceLabels map[string]string, +) (*rabbitmqv1.TransportURL, controllerutil.OperationResult, error) { + transportURL := &rabbitmqv1.TransportURL{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-cloudkitty-transport", instance.Name), + Namespace: instance.Namespace, + Labels: serviceLabels, + }, + } + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, transportURL, func() error { + transportURL.Spec.RabbitmqClusterName = instance.Spec.RabbitMqClusterName + + err := controllerutil.SetControllerReference(instance, transportURL, r.Scheme) + return err + }) + + return transportURL, op, err +} + +func (r *CloudKittyReconciler) apiDeploymentCreateOrUpdate(ctx context.Context, instance *telemetryv1.CloudKitty) (*telemetryv1.CloudKittyAPI, controllerutil.OperationResult, error) { + cloudkittyAPISpec := telemetryv1.CloudKittyAPISpec{ + CloudKittyTemplate: instance.Spec.CloudKittyTemplate, + CloudKittyAPITemplate: instance.Spec.CloudKittyAPI, + DatabaseHostname: instance.Status.DatabaseHostname, + TransportURLSecret: instance.Status.TransportURLSecret, + ServiceAccount: instance.RbacResourceName(), + } + + if cloudkittyAPISpec.NodeSelector == nil { + cloudkittyAPISpec.NodeSelector = instance.Spec.NodeSelector + } + + // If topology is not present in the underlying CloudKittyAPI Spec, + // inherit from the top-level CR + if cloudkittyAPISpec.TopologyRef == nil { + cloudkittyAPISpec.TopologyRef = instance.Spec.TopologyRef + } + + deployment := &telemetryv1.CloudKittyAPI{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-api", instance.Name), + Namespace: instance.Namespace, + }, + } + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { + deployment.Spec = cloudkittyAPISpec + + err := controllerutil.SetControllerReference(instance, deployment, r.Scheme) + if err != nil { + return err + } + + return nil + }) + + return deployment, op, err +} + +func (r *CloudKittyReconciler) procDeploymentCreateOrUpdate(ctx context.Context, instance *telemetryv1.CloudKitty) (*telemetryv1.CloudKittyProc, controllerutil.OperationResult, error) { + cloudKittyProcSpec := telemetryv1.CloudKittyProcSpec{ + CloudKittyTemplate: instance.Spec.CloudKittyTemplate, + CloudKittyProcTemplate: instance.Spec.CloudKittyProc, + DatabaseHostname: instance.Status.DatabaseHostname, + TransportURLSecret: instance.Status.TransportURLSecret, + ServiceAccount: instance.RbacResourceName(), + TLS: instance.Spec.CloudKittyAPI.TLS.Ca, + } + + if cloudKittyProcSpec.NodeSelector == nil { + cloudKittyProcSpec.NodeSelector = instance.Spec.NodeSelector + } + + // If topology is not present in the underlying Scheduler Spec + // inherit from the top-level CR + if cloudKittyProcSpec.TopologyRef == nil { + cloudKittyProcSpec.TopologyRef = instance.Spec.TopologyRef + } + + deployment := &telemetryv1.CloudKittyProc{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-proc", instance.Name), + Namespace: instance.Namespace, + }, + } + + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, deployment, func() error { + deployment.Spec = cloudKittyProcSpec + + err := controllerutil.SetControllerReference(instance, deployment, r.Scheme) + if err != nil { + return err + } + + return nil + }) + + return deployment, op, err +} + +func (r *CloudKittyReconciler) ensureDB( + ctx context.Context, + h *helper.Helper, + instance *telemetryv1.CloudKitty, +) (*mariadbv1.Database, ctrl.Result, error) { + Log := r.GetLogger(ctx) + + // ensure MariaDBAccount exists. This account record may be created by + // openstack-operator or the cloud operator up front without a specific + // MariaDBDatabase configured yet. Otherwise, a MariaDBAccount CR is + // created here with a generated username as well as a secret with + // generated password. The MariaDBAccount is created without being + // yet associated with any MariaDBDatabase. + _, _, err := mariadbv1.EnsureMariaDBAccount( + ctx, h, instance.Spec.DatabaseAccount, + instance.Namespace, false, "cloudkitty", + ) + + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + mariadbv1.MariaDBAccountReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + mariadbv1.MariaDBAccountNotReadyMessage, + err.Error())) + + return nil, ctrl.Result{}, err + } + instance.Status.Conditions.MarkTrue( + mariadbv1.MariaDBAccountReadyCondition, + mariadbv1.MariaDBAccountReadyMessage, + ) + + db := mariadbv1.NewDatabaseForAccount( + instance.Spec.DatabaseInstance, // mariadb/galera service to target + cloudkitty.DatabaseName, // name used in CREATE DATABASE in mariadb + cloudkitty.DatabaseName, // CR name for MariaDBDatabase + instance.Spec.DatabaseAccount, // CR name for MariaDBAccount + instance.Namespace, // namespace + ) + + // create or patch the DB + ctrlResult, err := db.CreateOrPatchAll(ctx, h) + + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBReadyErrorMessage, + err.Error())) + return db, ctrl.Result{}, err + } + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBReadyRunningMessage)) + return db, ctrlResult, nil + } + // wait for the DB to be setup + // (ksambor) should we use WaitForDBCreatedWithTimeout instead? + ctrlResult, err = db.WaitForDBCreated(ctx, h) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DBReadyErrorMessage, + err.Error())) + return db, ctrlResult, err + } + if (ctrlResult != ctrl.Result{}) { + Log.Info(fmt.Sprintf("%s... requeueing", condition.DBReadyRunningMessage)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DBReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DBReadyRunningMessage)) + return db, ctrlResult, nil + } + + // update Status.DatabaseHostname, used to config the service + instance.Status.DatabaseHostname = db.GetDatabaseHostname() + instance.Status.Conditions.MarkTrue(condition.DBReadyCondition, condition.DBReadyMessage) + return db, ctrlResult, nil +} diff --git a/controllers/cloudkittyapi_controller.go b/controllers/cloudkittyapi_controller.go new file mode 100644 index 00000000..082f6104 --- /dev/null +++ b/controllers/cloudkittyapi_controller.go @@ -0,0 +1,1119 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkittyapi" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/go-logr/logr" + networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" + "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" +) + +// GetClient - +func (r *CloudKittyAPIReconciler) GetClient() client.Client { + return r.Client +} + +// GetKClient - +func (r *CloudKittyAPIReconciler) GetKClient() kubernetes.Interface { + return r.Kclient +} + +// GetScheme - +func (r *CloudKittyAPIReconciler) GetScheme() *runtime.Scheme { + return r.Scheme +} + +// CloudKittyAPIReconciler reconciles a CloudKittyAPI object +type CloudKittyAPIReconciler struct { + client.Client + Kclient kubernetes.Interface + Scheme *runtime.Scheme +} + +// GetLogger returns a logger object with a logging prefix of "controller.name" and additional controller context fields +func (r *CloudKittyAPIReconciler) GetLogger(ctx context.Context) logr.Logger { + return log.FromContext(ctx).WithName("Controllers").WithName("CloudKittyAPI") +} + +var keystoneServices = []map[string]string{ + { + "type": cloudkitty.ServiceType, + "name": cloudkitty.ServiceName, + "desc": "CloudKitty V2 Service", + }, +} + +//+kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyapis,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyapis/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyapis/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list; +// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneservices,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneendpoints,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch +// +kubebuilder:rbac:groups=topology.openstack.org,resources=topologies,verbs=get;list;watch;update + +// Reconcile - +func (r *CloudKittyAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { + Log := r.GetLogger(ctx) + + // Fetch the CloudKittyAPI instance + instance := &telemetryv1.CloudKittyAPI{} + err := r.Client.Get(ctx, req.NamespacedName, instance) + if err != nil { + if k8s_errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. + // For additional cleanup logic use finalizers. Return and don't requeue. + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + return ctrl.Result{}, err + } + + helper, err := helper.NewHelper( + instance, + r.Client, + r.Kclient, + r.Scheme, + Log, + ) + if err != nil { + return ctrl.Result{}, err + } + + // + // initialize status + // + isNewInstance := instance.Status.Conditions == nil + if isNewInstance { + instance.Status.Conditions = condition.Conditions{} + } + + // Save a copy of the condtions so that we can restore the LastTransitionTime + // when a condition's state doesn't change. + savedConditions := instance.Status.Conditions.DeepCopy() + + // Always patch the instance status when exiting this function so we can persist any changes. + defer func() { + // Don't update the status, if reconciler Panics + if r := recover(); r != nil { + Log.Info(fmt.Sprintf("panic during reconcile %v\n", r)) + panic(r) + } + condition.RestoreLastTransitionTimes(&instance.Status.Conditions, savedConditions) + if instance.Status.Conditions.IsUnknown(condition.ReadyCondition) { + instance.Status.Conditions.Set( + instance.Status.Conditions.Mirror(condition.ReadyCondition)) + } + err := helper.PatchInstance(ctx, instance) + if err != nil { + _err = err + return + } + }() + + // Always initialize conditions used later as Status=Unknown + cl := condition.CreateList( + condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), + condition.UnknownCondition(condition.CreateServiceReadyCondition, condition.InitReason, condition.CreateServiceReadyInitMessage), + condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), + condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), + // right now we have no dedicated KeystoneServiceReadyInitMessage and KeystoneEndpointReadyInitMessage + condition.UnknownCondition(condition.KeystoneServiceReadyCondition, condition.InitReason, ""), + condition.UnknownCondition(condition.KeystoneEndpointReadyCondition, condition.InitReason, ""), + condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), + condition.UnknownCondition(condition.TLSInputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + ) + instance.Status.Conditions.Init(&cl) + // Always mark the Generation as observed early on + instance.Status.ObservedGeneration = instance.Generation + + // If we're not deleting this and the service object doesn't have our finalizer, add it. + if (instance.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(instance, helper.GetFinalizer())) || isNewInstance { + // Register overall status immediately to have an early feedback e.g. in the cli + return ctrl.Result{}, nil + } + + if instance.Status.Hash == nil { + instance.Status.Hash = map[string]string{} + } + if instance.Status.APIEndpoints == nil { + instance.Status.APIEndpoints = map[string]map[string]string{} + } + if instance.Status.ServiceIDs == nil { + instance.Status.ServiceIDs = map[string]string{} + } + if instance.Status.NetworkAttachments == nil { + instance.Status.NetworkAttachments = map[string][]string{} + } + + // Handle service delete + if !instance.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, instance, helper) + } + + // Init Topology condition if there's a reference + if instance.Spec.TopologyRef != nil { + c := condition.UnknownCondition(condition.TopologyReadyCondition, condition.InitReason, condition.TopologyReadyInitMessage) + cl.Set(c) + } + + // Handle non-deleted clusters + return r.reconcileNormal(ctx, instance, helper) +} + +// SetupWithManager sets up the controller with the Manager. +func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { + Log := r.GetLogger(ctx) + + // Watch for changes to secrets we don't own. Global secrets + // (e.g. TransportURLSecret) are handled by the main cloudkitty controller. + secretFn := func(_ context.Context, o client.Object) []reconcile.Request { + var namespace string = o.GetNamespace() + var secretName string = o.GetName() + result := []reconcile.Request{} + + // get all API CRs + apis := &telemetryv1.CloudKittyAPIList{} + listOpts := []client.ListOption{ + client.InNamespace(namespace), + } + if err := r.Client.List(context.Background(), apis, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve API CRs %v") + return nil + } + + // Watch for changes to secrets where the owner label AND the + // CR.Spec.ManagingCrName label matches + label := o.GetLabels() + if l, ok := label[labels.GetOwnerNameLabelSelector(labels.GetGroupLabel(cloudkitty.ServiceName))]; ok { + for _, cr := range apis.Items { + // return reconcile event for the CR where the owner label AND the parentCloudKittyName matches + if l == cloudkitty.GetOwningCloudKittyName(&cr) { + // return namespace and Name of CR + name := client.ObjectKey{ + Namespace: o.GetNamespace(), + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Secret %s and CR %s marked with label: %s", o.GetName(), cr.Name, l)) + + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + } + + // Watch for changes to any CustomServiceConfigSecrets + for _, cr := range apis.Items { + for _, v := range cr.Spec.CustomServiceConfigSecrets { + if v == secretName { + name := client.ObjectKey{ + Namespace: namespace, + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Secret %s is used by CloudKitty CR %s", secretName, cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + } + if len(result) > 0 { + return result + } + return nil + } + + // index passwordSecretField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyPasswordSecretField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyAPI) + if cr.Spec.Secret == "" { + return nil + } + return []string{cr.Spec.Secret} + }); err != nil { + return err + } + + // index caBundleSecretNameField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyCaBundleSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyAPI) + if cr.Spec.TLS.CaBundleSecretName == "" { + return nil + } + return []string{cr.Spec.TLS.CaBundleSecretName} + }); err != nil { + return err + } + + // index tlsAPIInternalField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyTlsAPIInternalField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyAPI) + if cr.Spec.TLS.API.Internal.SecretName == nil { + return nil + } + return []string{*cr.Spec.TLS.API.Internal.SecretName} + }); err != nil { + return err + } + + // index tlsAPIPublicField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyTlsAPIPublicField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyAPI) + if cr.Spec.TLS.API.Public.SecretName == nil { + return nil + } + return []string{*cr.Spec.TLS.API.Public.SecretName} + }); err != nil { + return err + } + + // index topologyField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyTopologyField, func(rawObj client.Object) []string { + // Extract the topology name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyAPI) + if cr.Spec.TopologyRef == nil { + return nil + } + return []string{cr.Spec.TopologyRef.Name} + }); err != nil { + return err + } + + return ctrl.NewControllerManagedBy(mgr). + For(&telemetryv1.CloudKittyAPI{}). + Owns(&keystonev1.KeystoneService{}). + Owns(&keystonev1.KeystoneEndpoint{}). + Owns(&appsv1.StatefulSet{}). + Owns(&corev1.Service{}). + // watch the secrets we don't own + Watches(&corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(secretFn)). + Watches( + &corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). + Watches(&topologyv1.Topology{}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.GenerationChangedPredicate{})). + Complete(r) +} + +func (r *CloudKittyAPIReconciler) findObjectsForSrc(ctx context.Context, src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + l := log.FromContext(ctx).WithName("Controllers").WithName("CloudKittyAPI") + + for _, field := range cloudKittyAPIWatchFields { + crList := &telemetryv1.CloudKittyAPIList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.List(ctx, crList, listOps) + if err != nil { + l.Error(err, fmt.Sprintf("listing %s for field: %s - %s", crList.GroupVersionKind().Kind, field, src.GetNamespace())) + return requests + } + + for _, item := range crList.Items { + l.Info(fmt.Sprintf("input source %s changed, reconcile: %s - %s", src.GetName(), item.GetName(), item.GetNamespace())) + + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} + +func (r *CloudKittyAPIReconciler) reconcileDelete(ctx context.Context, instance *telemetryv1.CloudKittyAPI, helper *helper.Helper) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s' delete", instance.Name)) + + // It's possible to get here before the endpoints have been set in the status, so check for this + if instance.Status.APIEndpoints != nil { + for _, ksSvc := range keystoneServices { + + // Remove the finalizer from our KeystoneEndpoint CR + keystoneEndpoint, err := keystonev1.GetKeystoneEndpointWithName(ctx, helper, ksSvc["name"], instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + + if err == nil { + controllerutil.RemoveFinalizer(keystoneEndpoint, helper.GetFinalizer()) + if err = helper.GetClient().Update(ctx, keystoneEndpoint); err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + util.LogForObject(helper, "Removed finalizer from our KeystoneEndpoint", instance) + } + + // Remove the finalizer from our KeystoneService CR + keystoneService, err := keystonev1.GetKeystoneServiceWithName(ctx, helper, ksSvc["name"], instance.Namespace) + if err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + + if err == nil { + controllerutil.RemoveFinalizer(keystoneService, helper.GetFinalizer()) + if err = helper.GetClient().Update(ctx, keystoneService); err != nil && !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + util.LogForObject(helper, "Removed finalizer from our KeystoneService", instance) + } + } + } + + // Remove finalizer on the Topology CR + if ctrlResult, err := topologyv1.EnsureDeletedTopologyRef( + ctx, + helper, + instance.Status.LastAppliedTopology, + instance.Name, + ); err != nil { + return ctrlResult, err + } + + // Service is deleted so remove the finalizer. + controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) + Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", instance.Name)) + + return ctrl.Result{}, nil +} + +func (r *CloudKittyAPIReconciler) reconcileInit( + ctx context.Context, + instance *telemetryv1.CloudKittyAPI, + helper *helper.Helper, + serviceLabels map[string]string, +) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s' init", instance.Name)) + + // + // expose the service (create service and return the created endpoint URLs) + // + + // V2 + publicEndpointData := endpoint.Data{ + Port: cloudkitty.CloudKittyPublicPort, + Path: "/v2", + } + internalEndpointData := endpoint.Data{ + Port: cloudkitty.CloudKittyInternalPort, + Path: "/v2", + } + cloudkittyEndpoints := map[service.Endpoint]endpoint.Data{ + service.EndpointPublic: publicEndpointData, + service.EndpointInternal: internalEndpointData, + } + + apiEndpointsV3 := make(map[string]string) + + for endpointType, data := range cloudkittyEndpoints { + endpointTypeStr := string(endpointType) + endpointName := cloudkitty.ServiceName + "-" + endpointTypeStr + svcOverride := instance.Spec.Override.Service[endpointType] + if svcOverride.EmbeddedLabelsAnnotations == nil { + svcOverride.EmbeddedLabelsAnnotations = &service.EmbeddedLabelsAnnotations{} + } + + exportLabels := util.MergeStringMaps( + serviceLabels, + map[string]string{ + service.AnnotationEndpointKey: endpointTypeStr, + }, + ) + + // Create the service + svc, err := service.NewService( + service.GenericService(&service.GenericServiceDetails{ + Name: endpointName, + Namespace: instance.Namespace, + Labels: exportLabels, + Selector: serviceLabels, + Port: service.GenericServicePort{ + Name: endpointName, + Port: data.Port, + Protocol: corev1.ProtocolTCP, + }, + }), + 5, + &svcOverride.OverrideSpec, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.CreateServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.CreateServiceReadyErrorMessage, + err.Error())) + + return ctrl.Result{}, err + } + + svc.AddAnnotation(map[string]string{ + service.AnnotationEndpointKey: endpointTypeStr, + }) + + // add Annotation to whether creating an ingress is required or not + if endpointType == service.EndpointPublic && svc.GetServiceType() == corev1.ServiceTypeClusterIP { + svc.AddAnnotation(map[string]string{ + service.AnnotationIngressCreateKey: "true", + }) + } else { + svc.AddAnnotation(map[string]string{ + service.AnnotationIngressCreateKey: "false", + }) + if svc.GetServiceType() == corev1.ServiceTypeLoadBalancer { + svc.AddAnnotation(map[string]string{ + service.AnnotationHostnameKey: svc.GetServiceHostname(), // add annotation to register service name in dnsmasq + }) + } + } + + ctrlResult, err := svc.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.CreateServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.CreateServiceReadyErrorMessage, + err.Error())) + + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.CreateServiceReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.CreateServiceReadyRunningMessage)) + return ctrlResult, nil + } + // create service - end + + // if TLS is enabled + if instance.Spec.TLS.API.Enabled(endpointType) { + // set endpoint protocol to https + data.Protocol = ptr.To(service.ProtocolHTTPS) + } + + apiEndpointsV3[string(endpointType)], err = svc.GetAPIEndpoint( + svcOverride.EndpointURL, data.Protocol, data.Path) + if err != nil { + instance.Status.Conditions.MarkFalse( + condition.CreateServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.CreateServiceReadyErrorMessage, + err.Error()) + return ctrl.Result{}, err + } + } + instance.Status.Conditions.MarkTrue(condition.CreateServiceReadyCondition, condition.CreateServiceReadyMessage) + + // + // Update instance status with service endpoint url from route host information + // + if instance.Status.APIEndpoints == nil { + instance.Status.APIEndpoints = map[string]map[string]string{} + } + instance.Status.APIEndpoints[cloudkitty.ServiceName] = apiEndpointsV3 + // V2 - end + + // expose service - end + + // + // create service and user in keystone - - https://docs.openstack.org/CloudKitty/latest/install/install-rdo.html#configure-user-and-endpoints + // TODO: rework this + // + if instance.Status.ServiceIDs == nil { + instance.Status.ServiceIDs = map[string]string{} + } + + for _, ksSvc := range keystoneServices { + ksSvcSpec := keystonev1.KeystoneServiceSpec{ + ServiceType: ksSvc["type"], + ServiceName: ksSvc["name"], + ServiceDescription: ksSvc["desc"], + Enabled: true, + ServiceUser: instance.Spec.ServiceUser, + Secret: instance.Spec.Secret, + PasswordSelector: instance.Spec.PasswordSelectors.CloudKittyService, + } + + ksSvcObj := keystonev1.NewKeystoneService(ksSvcSpec, instance.Namespace, serviceLabels, cloudkitty.NormalDuration) + ctrlResult, err := ksSvcObj.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.MarkFalse( + condition.KeystoneServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + "Creating KeyStoneService CR %s", + err.Error()) + return ctrlResult, err + } + + // mirror the Status, Reason, Severity and Message of the latest keystoneservice condition + // into a local condition with the type condition.KeystoneServiceReadyCondition + c := ksSvcObj.GetConditions().Mirror(condition.KeystoneServiceReadyCondition) + if c != nil { + instance.Status.Conditions.Set(c) + } + + if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + instance.Status.ServiceIDs[ksSvc["name"]] = ksSvcObj.GetServiceID() + + ksEndptSpec := keystonev1.KeystoneEndpointSpec{ + ServiceName: ksSvc["name"], + Endpoints: instance.Status.APIEndpoints[ksSvc["name"]], + } + + ksEndptObj := keystonev1.NewKeystoneEndpoint( + ksSvc["name"], + instance.Namespace, + ksEndptSpec, + serviceLabels, + cloudkitty.NormalDuration) + ctrlResult, err = ksEndptObj.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.MarkFalse( + condition.KeystoneEndpointReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + "Creating KeyStoneEndpoint CR %s", + err.Error()) + return ctrlResult, err + } + + // mirror the Status, Reason, Severity and Message of the latest keystoneendpoint condition + // into a local condition with the type condition.KeystoneEndpointReadyCondition + c = ksEndptObj.GetConditions().Mirror(condition.KeystoneEndpointReadyCondition) + if c != nil { + instance.Status.Conditions.Set(c) + } + + if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + } + + Log.Info(fmt.Sprintf("Reconciled Service '%s' init successfully", instance.Name)) + return ctrl.Result{}, nil +} + +func (r *CloudKittyAPIReconciler) reconcileNormal(ctx context.Context, instance *telemetryv1.CloudKittyAPI, helper *helper.Helper) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s'", instance.Name)) + + configVars := make(map[string]env.Setter) + + // + // check for required OpenStack secret holding passwords for service/admin user and add hash to the vars map + // + + ctrlResult, err := cloudkitty.VerifyServiceSecret( + ctx, + types.NamespacedName{Namespace: instance.Namespace, Name: instance.Spec.Secret}, + []string{ + instance.Spec.PasswordSelectors.CloudKittyService, + }, + helper.GetClient(), + &instance.Status.Conditions, + cloudkitty.NormalDuration, + &configVars, + ) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + // + // check for required Transport URL and config secrets + // + + parentCloudKittyName := cloudkitty.GetOwningCloudKittyName(instance) + secretNames := []string{ + instance.Spec.TransportURLSecret, // TransportURLSecret + fmt.Sprintf("%s-scripts", parentCloudKittyName), // ScriptsSecret + fmt.Sprintf("%s-config-data", parentCloudKittyName), // ConfigSecret + } + // Append CustomServiceConfigSecrets that should be checked + secretNames = append(secretNames, instance.Spec.CustomServiceConfigSecrets...) + + ctrlResult, err = cloudkitty.VerifyConfigSecrets( + ctx, + helper, + &instance.Status.Conditions, + secretNames, + instance.Namespace, + &configVars, + ) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + + // + // TLS input validation + // + // Validate the CA cert secret if provided + if instance.Spec.TLS.CaBundleSecretName != "" { + hash, err := tls.ValidateCACertSecret( + ctx, + helper.GetClient(), + types.NamespacedName{ + Name: instance.Spec.TLS.CaBundleSecretName, + Namespace: instance.Namespace, + }, + ) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + fmt.Sprintf(condition.TLSInputReadyWaitingMessage, instance.Spec.TLS.CaBundleSecretName))) + return ctrl.Result{}, nil + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if hash != "" { + configVars[tls.CABundleKey] = env.SetValue(hash) + } + } + + // Validate API service certs secrets + certsHash, err := instance.Spec.TLS.API.ValidateCertSecrets(ctx, helper, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + fmt.Sprintf(condition.TLSInputReadyWaitingMessage, err.Error()))) + return ctrl.Result{}, nil + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + configVars[tls.TLSHashName] = env.SetValue(certsHash) + + // all cert input checks out so report InputReady + instance.Status.Conditions.MarkTrue(condition.TLSInputReadyCondition, condition.InputReadyMessage) + + // + // Create secrets required as input for the Service and calculate an overall hash of hashes + // + serviceLabels := map[string]string{ + common.AppSelector: cloudkitty.ServiceName, + common.ComponentSelector: cloudkittyapi.ComponentName, + } + + // + // create custom config for this cloudkitty service + // + err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) + + // + // TODO check when/if Init, Update, or Upgrade should/could be skipped + // + + // networks to attach to + nadList := []networkv1.NetworkAttachmentDefinition{} + for _, netAtt := range instance.Spec.NetworkAttachments { + nad, err := nad.GetNADWithName(ctx, helper, netAtt, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + Log.Info(fmt.Sprintf("network-attachment-definition %s not found", netAtt)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.NetworkAttachmentsReadyWaitingMessage, + netAtt)) + return cloudkitty.ResultRequeue, fmt.Errorf("network-attachment-definition %s not found", netAtt) + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if nad != nil { + nadList = append(nadList, *nad) + } + } + + serviceAnnotations, err := nad.EnsureNetworksAnnotation(nadList) + if err != nil { + err = fmt.Errorf("failed create network annotation from %s: %w", instance.Spec.NetworkAttachments, err) + instance.Status.Conditions.MarkFalse( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error()) + return ctrl.Result{}, err + } + + // Handle service init + ctrlResult, err = r.reconcileInit(ctx, instance, helper, serviceLabels) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + // + // Handle Topology + // + topology, err := ensureTopology( + ctx, + helper, + instance, // topologyHandler + instance.Name, // finalizer + &instance.Status.Conditions, + labels.GetLabelSelector(serviceLabels), + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TopologyReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TopologyReadyErrorMessage, + err.Error())) + return ctrl.Result{}, fmt.Errorf("waiting for Topology requirements: %w", err) + } + + // + // normal reconcile tasks + // + + // + // create hash over all the different input resources to identify if any those changed + // and a restart/recreate is required. + // + inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configVars) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } else if hashChanged { + Log.Info(fmt.Sprintf("%s... requeueing", condition.ServiceConfigReadyInitMessage)) + instance.Status.Conditions.MarkFalse( + condition.ServiceConfigReadyCondition, + condition.InitReason, + condition.SeverityInfo, + condition.ServiceConfigReadyInitMessage) + // Hash changed and instance status should be updated (which will be done by main defer func), + // so we need to return and reconcile again + return ctrl.Result{}, nil + } + + // Deploy a statefulset + ssDef, err := cloudkittyapi.StatefulSet(instance, inputHash, serviceLabels, serviceAnnotations, topology) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + ss := statefulset.NewStatefulSet(ssDef, cloudkitty.ShortDuration) + + var ssData appsv1.StatefulSet + ctrlResult, err = ss.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrlResult, err + + } else if (ctrlResult == ctrl.Result{}) { + // Wait until the data in the StatefulSet is for the current generation + ssData = ss.GetStatefulSet() + if ssData.Generation != ssData.Status.ObservedGeneration { + ctrlResult = cloudkitty.ResultRequeue + err = fmt.Errorf("waiting for Statefulset %s to start reconciling", ssData.Name) + } + } + + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + // If the deployment is not ready, then neither are the NADs + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.NetworkAttachmentsReadyInitMessage)) + return ctrlResult, err + } + + instance.Status.ReadyCount = ssData.Status.ReadyReplicas + + // verify if network attachment matches expectations + networkReady := false + networkAttachmentStatus := map[string][]string{} + if *instance.Spec.Replicas > 0 { + networkReady, networkAttachmentStatus, err = nad.VerifyNetworkStatusFromAnnotation( + ctx, + helper, + instance.Spec.NetworkAttachments, + serviceLabels, + instance.Status.ReadyCount, + ) + if err != nil { + err = fmt.Errorf("verifying API NetworkAttachments (%s) %w", instance.Spec.NetworkAttachments, err) + instance.Status.Conditions.MarkFalse( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error()) + return ctrl.Result{}, err + } + } else { + networkReady = true + } + + instance.Status.NetworkAttachments = networkAttachmentStatus + if networkReady { + instance.Status.Conditions.MarkTrue(condition.NetworkAttachmentsReadyCondition, condition.NetworkAttachmentsReadyMessage) + } else { + err := fmt.Errorf("not all pods have interfaces with ips as configured in NetworkAttachments: %s", instance.Spec.NetworkAttachments) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + + return ctrl.Result{}, err + } + + if instance.Status.ReadyCount > 0 { + instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) + + } else if *instance.Spec.Replicas > 0 { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + + } else { + instance.Status.Conditions.MarkFalse( + condition.DeploymentReadyCondition, + condition.NotRequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyInitMessage) + } + // create StatefulSet - end + + Log.Info(fmt.Sprintf("Reconciled Service '%s' successfully", instance.Name)) + // update the overall status condition if service is ready + if instance.IsReady() { + instance.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) + } + // For non ready we'll let the main defer func handle the status update using the Mirror function + return ctrl.Result{}, nil +} + +// generateServiceConfigs - create Secret which holds the service configuration +func (r *CloudKittyAPIReconciler) generateServiceConfigs( + ctx context.Context, + h *helper.Helper, + instance *telemetryv1.CloudKittyAPI, + envVars *map[string]env.Setter, + serviceLabels map[string]string, +) error { + // + // create custom Secret for cloudkitty service-specific config input + // - %-config-data holds custom config for the service + // + + labels := labels.GetLabels(instance, labels.GetGroupLabel(cloudkitty.ServiceName), serviceLabels) + + // customData hold any customization for the service. + customData := map[string]string{cloudkitty.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig} + + // Fetch the two service config snippets (DefaultsConfigFileName and + // CustomConfigFileName) from the Secret generated by the top level + // cloudkitty controller, and add them to this service specific Secret. + cloudkittySecretName := cloudkitty.GetOwningCloudKittyName(instance) + "-config-data" + cloudkittySecret, _, err := secret.GetSecret(ctx, h, cloudkittySecretName, instance.Namespace) + if err != nil { + return err + } + customData[cloudkitty.DefaultsConfigFileName] = string(cloudkittySecret.Data[cloudkitty.DefaultsConfigFileName]) + customData[cloudkitty.CustomConfigFileName] = string(cloudkittySecret.Data[cloudkitty.CustomConfigFileName]) + + customSecrets := "" + for _, secretName := range instance.Spec.CustomServiceConfigSecrets { + secret, _, err := secret.GetSecret(ctx, h, secretName, instance.Namespace) + if err != nil { + return err + } + for _, data := range secret.Data { + customSecrets += string(data) + "\n" + } + } + customData[cloudkitty.CustomServiceConfigSecretsFileName] = customSecrets + + templateParameters := map[string]interface{}{ + "LogFile": cloudkittyapi.LogFile, + } + + configTemplates := []util.Template{ + { + Name: fmt.Sprintf("%s-config-data", instance.Name), + Namespace: instance.Namespace, + Type: util.TemplateTypeConfig, + InstanceType: instance.Kind, + CustomData: customData, + ConfigOptions: templateParameters, + Labels: labels, + }, + } + + return secret.EnsureSecrets(ctx, h, instance, configTemplates, envVars) +} + +// createHashOfInputHashes - creates a hash of hashes which gets added to the resources which requires a restart +// if any of the input resources change, like configs, passwords, ... +// +// returns the hash, whether the hash changed (as a bool) and any error +func (r *CloudKittyAPIReconciler) createHashOfInputHashes( + ctx context.Context, + instance *telemetryv1.CloudKittyAPI, + envVars map[string]env.Setter, +) (string, bool, error) { + Log := r.GetLogger(ctx) + + var hashMap map[string]string + changed := false + mergedMapVars := env.MergeEnvs([]corev1.EnvVar{}, envVars) + hash, err := util.ObjectHash(mergedMapVars) + if err != nil { + return hash, changed, err + } + if hashMap, changed = util.SetHash(instance.Status.Hash, common.InputHashName, hash); changed { + instance.Status.Hash = hashMap + Log.Info(fmt.Sprintf("Input maps hash %s - %s", common.InputHashName, hash)) + } + return hash, changed, nil +} diff --git a/controllers/cloudkittyproc_controller.go b/controllers/cloudkittyproc_controller.go new file mode 100644 index 00000000..fbccef63 --- /dev/null +++ b/controllers/cloudkittyproc_controller.go @@ -0,0 +1,759 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkittyproc" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/go-logr/logr" + networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/labels" + nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" + "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "github.com/openstack-k8s-operators/lib-common/modules/common/statefulset" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + "github.com/openstack-k8s-operators/lib-common/modules/common/util" +) + +// GetClient - +func (r *CloudKittyProcReconciler) GetClient() client.Client { + return r.Client +} + +// GetKClient - +func (r *CloudKittyProcReconciler) GetKClient() kubernetes.Interface { + return r.Kclient +} + +// GetScheme - +func (r *CloudKittyProcReconciler) GetScheme() *runtime.Scheme { + return r.Scheme +} + +// CloudKittyProcReconciler reconciles a CloudKittyProc object +type CloudKittyProcReconciler struct { + client.Client + Kclient kubernetes.Interface + Scheme *runtime.Scheme +} + +// GetLogger returns a logger object with a logging prefix of "controller.name" and additional controller context fields +func (r *CloudKittyProcReconciler) GetLogger(ctx context.Context) logr.Logger { + return log.FromContext(ctx).WithName("Controllers").WithName("CloudKittyProc") +} + +//+kubebuilder:rbac:groups=cloudkitty.openstack.org,resources=cloudkittyprocs,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=cloudkitty.openstack.org,resources=cloudkittyprocs/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=cloudkitty.openstack.org,resources=cloudkittyprocs/finalizers,verbs=update;patch +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list; +// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch +// +kubebuilder:rbac:groups=topology.openstack.org,resources=topologies,verbs=get;list;watch;update + +// Reconcile - +func (r *CloudKittyProcReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, _err error) { + Log := r.GetLogger(ctx) + + // Fetch the CloudKittyProc instance + instance := &telemetryv1.CloudKittyProc{} + err := r.Client.Get(ctx, req.NamespacedName, instance) + if err != nil { + if k8s_errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. + // For additional cleanup logic use finalizers. Return and don't requeue. + return ctrl.Result{}, nil + } + // Error reading the object - requeue the request. + return ctrl.Result{}, err + } + + helper, err := helper.NewHelper( + instance, + r.Client, + r.Kclient, + r.Scheme, + Log, + ) + if err != nil { + return ctrl.Result{}, err + } + + // + // initialize status + // + isNewInstance := instance.Status.Conditions == nil + if isNewInstance { + instance.Status.Conditions = condition.Conditions{} + } + + // Save a copy of the condtions so that we can restore the LastTransitionTime + // when a condition's state doesn't change. + savedConditions := instance.Status.Conditions.DeepCopy() + + // Always patch the instance status when exiting this function so we can persist any changes. + defer func() { + // Don't update the status, if reconciler Panics + if r := recover(); r != nil { + Log.Info(fmt.Sprintf("panic during reconcile %v\n", r)) + panic(r) + } + condition.RestoreLastTransitionTimes(&instance.Status.Conditions, savedConditions) + if instance.Status.Conditions.IsUnknown(condition.ReadyCondition) { + instance.Status.Conditions.Set( + instance.Status.Conditions.Mirror(condition.ReadyCondition)) + } + err := helper.PatchInstance(ctx, instance) + if err != nil { + _err = err + return + } + }() + + // Always initialize conditions used later as Status=Unknown + cl := condition.CreateList( + condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), + condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), + condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), + condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), + condition.UnknownCondition(condition.TLSInputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), + ) + instance.Status.Conditions.Init(&cl) + // Always mark the Generation as observed early on + instance.Status.ObservedGeneration = instance.Generation + + // If we're not deleting this and the service object doesn't have our finalizer, add it. + if (instance.DeletionTimestamp.IsZero() && controllerutil.AddFinalizer(instance, helper.GetFinalizer())) || isNewInstance { + // Register overall status immediately to have an early feedback e.g. in the cli + return ctrl.Result{}, nil + } + + if instance.Status.Hash == nil { + instance.Status.Hash = map[string]string{} + } + if instance.Status.NetworkAttachments == nil { + instance.Status.NetworkAttachments = map[string][]string{} + } + + // Handle service delete + if !instance.DeletionTimestamp.IsZero() { + return r.reconcileDelete(ctx, instance, helper) + } + + // Init Topology condition if there's a reference + if instance.Spec.TopologyRef != nil { + c := condition.UnknownCondition(condition.TopologyReadyCondition, condition.InitReason, condition.TopologyReadyInitMessage) + cl.Set(c) + } + + // Handle non-deleted clusters + return r.reconcileNormal(ctx, instance, helper) +} + +// SetupWithManager sets up the controller with the Manager. +func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { + Log := r.GetLogger(ctx) + + // Watch for changes to secrets we don't own. Global secrets + // (e.g. TransportURLSecret) are handled by the main cloudkitty controller. + secretFn := func(_ context.Context, o client.Object) []reconcile.Request { + var namespace string = o.GetNamespace() + var secretName string = o.GetName() + result := []reconcile.Request{} + + // get all scheduler CRs + schedulers := &telemetryv1.CloudKittyProcList{} + listOpts := []client.ListOption{ + client.InNamespace(namespace), + } + if err := r.Client.List(context.Background(), schedulers, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve scheduler CRs %v") + return nil + } + + // Watch for changes to secrets where the owner label AND the + // CR.Spec.ManagingCrName label matches + label := o.GetLabels() + if l, ok := label[labels.GetOwnerNameLabelSelector(labels.GetGroupLabel(cloudkitty.ServiceName))]; ok { + for _, cr := range schedulers.Items { + // return reconcile event for the CR where the owner label AND the parentCloudKittyName matches + if l == cloudkitty.GetOwningCloudKittyName(&cr) { + // return namespace and Name of CR + name := client.ObjectKey{ + Namespace: o.GetNamespace(), + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Secret %s and CR %s marked with label: %s", o.GetName(), cr.Name, l)) + + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + } + + // Watch for changes to any CustomServiceConfigSecrets + for _, cr := range schedulers.Items { + for _, v := range cr.Spec.CustomServiceConfigSecrets { + if v == secretName { + name := client.ObjectKey{ + Namespace: namespace, + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Secret %s is used by CloudKitty CR %s", secretName, cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + } + if len(result) > 0 { + return result + } + return nil + } + + // index passwordSecretField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyProc{}, cloudKittyPasswordSecretField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyProc) + if cr.Spec.Secret == "" { + return nil + } + return []string{cr.Spec.Secret} + }); err != nil { + return err + } + + // index caBundleSecretNameField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyProc{}, cloudKittyCaBundleSecretNameField, func(rawObj client.Object) []string { + // Extract the secret name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyProc) + if cr.Spec.TLS.CaBundleSecretName == "" { + return nil + } + return []string{cr.Spec.TLS.CaBundleSecretName} + }); err != nil { + return err + } + + // index topologyField + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyProc{}, cloudKittyTopologyField, func(rawObj client.Object) []string { + // Extract the topology name from the spec, if one is provided + cr := rawObj.(*telemetryv1.CloudKittyProc) + if cr.Spec.TopologyRef == nil { + return nil + } + return []string{cr.Spec.TopologyRef.Name} + }); err != nil { + return err + } + + return ctrl.NewControllerManagedBy(mgr). + For(&telemetryv1.CloudKittyProc{}). + Owns(&appsv1.StatefulSet{}). + // watch the secrets we don't own + Watches(&corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(secretFn)). + Watches( + &corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). + Watches(&topologyv1.Topology{}, + handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), + builder.WithPredicates(predicate.GenerationChangedPredicate{})). + Complete(r) +} + +func (r *CloudKittyProcReconciler) findObjectsForSrc(ctx context.Context, src client.Object) []reconcile.Request { + requests := []reconcile.Request{} + + l := log.FromContext(ctx).WithName("Controllers").WithName("CloudKittyProc") + + for _, field := range commonWatchFields { + crList := &telemetryv1.CloudKittyProcList{} + listOps := &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), + Namespace: src.GetNamespace(), + } + err := r.List(ctx, crList, listOps) + if err != nil { + l.Error(err, fmt.Sprintf("listing %s for field: %s - %s", crList.GroupVersionKind().Kind, field, src.GetNamespace())) + return requests + } + + for _, item := range crList.Items { + l.Info(fmt.Sprintf("input source %s changed, reconcile: %s - %s", src.GetName(), item.GetName(), item.GetNamespace())) + + requests = append(requests, + reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + }, + }, + ) + } + } + + return requests +} + +func (r *CloudKittyProcReconciler) reconcileDelete(ctx context.Context, instance *telemetryv1.CloudKittyProc, helper *helper.Helper) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s' delete", instance.Name)) + + // Service is deleted so remove the finalizer. + controllerutil.RemoveFinalizer(instance, helper.GetFinalizer()) + Log.Info(fmt.Sprintf("Reconciled Service '%s' delete successfully", instance.Name)) + + // Remove finalizer on the Topology CR + if ctrlResult, err := topologyv1.EnsureDeletedTopologyRef( + ctx, + helper, + instance.Status.LastAppliedTopology, + instance.Name, + ); err != nil { + return ctrlResult, err + } + return ctrl.Result{}, nil +} + +func (r *CloudKittyProcReconciler) reconcileNormal(ctx context.Context, instance *telemetryv1.CloudKittyProc, helper *helper.Helper) (ctrl.Result, error) { + Log := r.GetLogger(ctx) + + Log.Info(fmt.Sprintf("Reconciling Service '%s'", instance.Name)) + + configVars := make(map[string]env.Setter) + + // + // check for required OpenStack secret holding passwords for service/admin user and add hash to the vars map + // + + ctrlResult, err := cloudkitty.VerifyServiceSecret( + ctx, + types.NamespacedName{Namespace: instance.Namespace, Name: instance.Spec.Secret}, + []string{ + instance.Spec.PasswordSelectors.CloudKittyService, + }, + helper.GetClient(), + &instance.Status.Conditions, + cloudkitty.NormalDuration, + &configVars, + ) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + // + // check for required Transport URL and config secrets + // + + parentCloudKittyName := cloudkitty.GetOwningCloudKittyName(instance) + secretNames := []string{ + instance.Spec.TransportURLSecret, // TransportURLSecret + fmt.Sprintf("%s-scripts", parentCloudKittyName), // ScriptsSecret + fmt.Sprintf("%s-config-data", parentCloudKittyName), // ConfigSecret + } + // Append CustomServiceConfigSecrets that should be checked + secretNames = append(secretNames, instance.Spec.CustomServiceConfigSecrets...) + + ctrlResult, err = cloudkitty.VerifyConfigSecrets( + ctx, + helper, + &instance.Status.Conditions, + secretNames, + instance.Namespace, + &configVars, + ) + if err != nil { + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) + + // + // TLS input validation + // + // Validate the CA cert secret if provided + if instance.Spec.TLS.CaBundleSecretName != "" { + hash, err := tls.ValidateCACertSecret( + ctx, + helper.GetClient(), + types.NamespacedName{ + Name: instance.Spec.TLS.CaBundleSecretName, + Namespace: instance.Namespace, + }, + ) + if err != nil { + if k8s_errors.IsNotFound(err) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + fmt.Sprintf(condition.TLSInputReadyWaitingMessage, instance.Spec.TLS.CaBundleSecretName))) + return ctrl.Result{}, nil + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TLSInputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TLSInputErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if hash != "" { + configVars[tls.CABundleKey] = env.SetValue(hash) + } + } + // all cert input checks out so report InputReady + instance.Status.Conditions.MarkTrue(condition.TLSInputReadyCondition, condition.InputReadyMessage) + + // + // Create ConfigMaps required as input for the Service and calculate an overall hash of hashes + // + serviceLabels := map[string]string{ + common.AppSelector: cloudkitty.ServiceName, + common.ComponentSelector: cloudkittyproc.ComponentName, + } + + // + // create custom config for this cloudkitty service + // + err = r.generateServiceConfigs(ctx, helper, instance, &configVars, serviceLabels) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + // + // create hash over all the different input resources to identify if any those changed + // and a restart/recreate is required. + // + inputHash, hashChanged, err := r.createHashOfInputHashes(ctx, instance, configVars) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } else if hashChanged { + Log.Info(fmt.Sprintf("%s... requeueing", condition.ServiceConfigReadyInitMessage)) + instance.Status.Conditions.MarkFalse( + condition.ServiceConfigReadyCondition, + condition.InitReason, + condition.SeverityInfo, + condition.ServiceConfigReadyInitMessage) + // Hash changed and instance status should be updated (which will be done by main defer func), + // so we need to return and reconcile again + return ctrl.Result{}, nil + } + instance.Status.Conditions.MarkTrue(condition.ServiceConfigReadyCondition, condition.ServiceConfigReadyMessage) + + // + // TODO check when/if Init, Update, or Upgrade should/could be skipped + // + + // networks to attach to + nadList := []networkv1.NetworkAttachmentDefinition{} + for _, netAtt := range instance.Spec.NetworkAttachments { + nad, err := nad.GetNADWithName(ctx, helper, netAtt, instance.Namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + Log.Info(fmt.Sprintf("network-attachment-definition %s not found", netAtt)) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.NetworkAttachmentsReadyWaitingMessage, + netAtt)) + return cloudkitty.ResultRequeue, fmt.Errorf("network-attachment-definition %s not found", netAtt) + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + if nad != nil { + nadList = append(nadList, *nad) + } + } + + serviceAnnotations, err := nad.EnsureNetworksAnnotation(nadList) + if err != nil { + err = fmt.Errorf("failed create network annotation from %s: %w", instance.Spec.NetworkAttachments, err) + instance.Status.Conditions.MarkFalse( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error()) + return ctrl.Result{}, err + } + + // + // normal reconcile tasks + // + + // + // Handle Topology + // + topology, err := ensureTopology( + ctx, + helper, + instance, // topologyHandler + instance.Name, // finalizer + &instance.Status.Conditions, + labels.GetLabelSelector(serviceLabels), + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.TopologyReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TopologyReadyErrorMessage, + err.Error())) + return ctrl.Result{}, fmt.Errorf("waiting for Topology requirements: %w", err) + } + + // Deploy a statefulset + ssDef := cloudkittyproc.StatefulSet(instance, inputHash, serviceLabels, serviceAnnotations, topology) + ss := statefulset.NewStatefulSet(ssDef, cloudkitty.ShortDuration) + + var ssData appsv1.StatefulSet + ctrlResult, err = ss.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.DeploymentReadyErrorMessage, + err.Error())) + return ctrlResult, err + + } else if (ctrlResult == ctrl.Result{}) { + // Wait until the data in the StatefulSet is for the current generation + ssData = ss.GetStatefulSet() + if ssData.Generation != ssData.Status.ObservedGeneration { + ctrlResult = cloudkitty.ResultRequeue + err = fmt.Errorf("waiting for Statefulset %s to start reconciling", ssData.Name) + } + } + + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + // If the deployment is not ready, then neither are the NADs + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.NetworkAttachmentsReadyInitMessage)) + return ctrlResult, err + } + + instance.Status.ReadyCount = ssData.Status.ReadyReplicas + + // verify if network attachment matches expectations + networkReady := false + networkAttachmentStatus := map[string][]string{} + if *instance.Spec.Replicas > 0 { + networkReady, networkAttachmentStatus, err = nad.VerifyNetworkStatusFromAnnotation( + ctx, + helper, + instance.Spec.NetworkAttachments, + serviceLabels, + instance.Status.ReadyCount, + ) + if err != nil { + err = fmt.Errorf("verifying API NetworkAttachments (%s) %w", instance.Spec.NetworkAttachments, err) + instance.Status.Conditions.MarkFalse( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error()) + return ctrl.Result{}, err + } + } else { + networkReady = true + } + + instance.Status.NetworkAttachments = networkAttachmentStatus + if networkReady { + instance.Status.Conditions.MarkTrue(condition.NetworkAttachmentsReadyCondition, condition.NetworkAttachmentsReadyMessage) + } else { + err := fmt.Errorf("not all pods have interfaces with ips as configured in NetworkAttachments: %s", instance.Spec.NetworkAttachments) + instance.Status.Conditions.Set(condition.FalseCondition( + condition.NetworkAttachmentsReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.NetworkAttachmentsReadyErrorMessage, + err.Error())) + + return ctrl.Result{}, err + } + + if instance.Status.ReadyCount > 0 { + instance.Status.Conditions.MarkTrue(condition.DeploymentReadyCondition, condition.DeploymentReadyMessage) + } else if *instance.Spec.Replicas > 0 { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.DeploymentReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyRunningMessage)) + + } else { + instance.Status.Conditions.MarkFalse( + condition.DeploymentReadyCondition, + condition.NotRequestedReason, + condition.SeverityInfo, + condition.DeploymentReadyInitMessage) + } + // create StatefulSet - end + + Log.Info(fmt.Sprintf("Reconciled Service '%s' successfully", instance.Name)) + // update the overall status condition if service is ready + if instance.IsReady() { + instance.Status.Conditions.MarkTrue(condition.ReadyCondition, condition.ReadyMessage) + } + // For non ready we'll let the main defer func handle the status update using the Mirror function + return ctrl.Result{}, nil +} + +// generateServiceConfigs - create Secret which holds the service configuration +func (r *CloudKittyProcReconciler) generateServiceConfigs( + ctx context.Context, + h *helper.Helper, + instance *telemetryv1.CloudKittyProc, + envVars *map[string]env.Setter, + serviceLabels map[string]string, +) error { + // + // create custom Secret for cloudkitty service-specific config input + // - %-config-data holds custom config for the service + // + + labels := labels.GetLabels(instance, labels.GetGroupLabel(cloudkitty.ServiceName), serviceLabels) + + // customData hold any customization for the service. + customData := map[string]string{cloudkitty.CustomServiceConfigFileName: instance.Spec.CustomServiceConfig} + + // Fetch the two service config snippets (DefaultsConfigFileName and + // CustomConfigFileName) from the Secret generated by the top level + // cloudkitty controller, and add them to this service specific Secret. + cloudkittySecretName := cloudkitty.GetOwningCloudKittyName(instance) + "-config-data" + cloudkittySecret, _, err := secret.GetSecret(ctx, h, cloudkittySecretName, instance.Namespace) + if err != nil { + return err + } + customData[cloudkitty.DefaultsConfigFileName] = string(cloudkittySecret.Data[cloudkitty.DefaultsConfigFileName]) + customData[cloudkitty.CustomConfigFileName] = string(cloudkittySecret.Data[cloudkitty.CustomConfigFileName]) + + customSecrets := "" + for _, secretName := range instance.Spec.CustomServiceConfigSecrets { + secret, _, err := secret.GetSecret(ctx, h, secretName, instance.Namespace) + if err != nil { + return err + } + for _, data := range secret.Data { + customSecrets += string(data) + "\n" + } + } + customData[cloudkitty.CustomServiceConfigSecretsFileName] = customSecrets + + configTemplates := []util.Template{ + { + Name: fmt.Sprintf("%s-config-data", instance.Name), + Namespace: instance.Namespace, + Type: util.TemplateTypeConfig, + InstanceType: instance.Kind, + CustomData: customData, + Labels: labels, + }, + } + + return secret.EnsureSecrets(ctx, h, instance, configTemplates, envVars) +} + +// createHashOfInputHashes - creates a hash of hashes which gets added to the resources which requires a restart +// if any of the input resources change, like configs, passwords, ... +// +// returns the hash, whether the hash changed (as a bool) and any error +func (r *CloudKittyProcReconciler) createHashOfInputHashes( + ctx context.Context, + instance *telemetryv1.CloudKittyProc, + envVars map[string]env.Setter, +) (string, bool, error) { + Log := r.GetLogger(ctx) + var hashMap map[string]string + changed := false + mergedMapVars := env.MergeEnvs([]corev1.EnvVar{}, envVars) + hash, err := util.ObjectHash(mergedMapVars) + if err != nil { + return hash, changed, err + } + if hashMap, changed = util.SetHash(instance.Status.Hash, common.InputHashName, hash); changed { + instance.Status.Hash = hashMap + Log.Info(fmt.Sprintf("Input maps hash %s - %s", common.InputHashName, hash)) + } + return hash, changed, nil +} diff --git a/controllers/telemetry_controller.go b/controllers/telemetry_controller.go index e053c760..b3550324 100644 --- a/controllers/telemetry_controller.go +++ b/controllers/telemetry_controller.go @@ -66,6 +66,9 @@ func (r *TelemetryReconciler) GetLogger(ctx context.Context) logr.Logger { // +kubebuilder:rbac:groups=telemetry.openstack.org,resources=loggings,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=telemetry.openstack.org,resources=loggings/status,verbs=get;update;patch // +kubebuilder:rbac:groups=telemetry.openstack.org,resources=loggings/finalizers,verbs=update;delete;patch +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkitties,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkitties/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkitties/finalizers,verbs=update;delete;patch // +kubebuilder:rbac:groups=rabbitmq.openstack.org,resources=transporturls,verbs=get;list;watch;create;update;patch;delete // Reconcile reconciles a Telemetry @@ -141,6 +144,7 @@ func (r *TelemetryReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( condition.UnknownCondition(telemetryv1.AutoscalingReadyCondition, condition.InitReason, telemetryv1.AutoscalingReadyInitMessage), condition.UnknownCondition(telemetryv1.MetricStorageReadyCondition, condition.InitReason, telemetryv1.MetricStorageReadyInitMessage), condition.UnknownCondition(telemetryv1.LoggingReadyCondition, condition.InitReason, telemetryv1.LoggingReadyInitMessage), + condition.UnknownCondition(telemetryv1.CloudKittyReadyCondition, condition.InitReason, telemetryv1.CloudKittyReadyInitMessage), ) instance.Status.Conditions.Init(&cl) @@ -225,6 +229,13 @@ func (r *TelemetryReconciler) reconcileNormal(ctx context.Context, instance *tel return ctrlResult, nil } + ctrlResult, err = r.reconcileCloudKitty(ctx, instance, helper) + if err != nil { + return ctrl.Result{}, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil + } + // We reached the end of the Reconcile, update the Ready condition based on // the sub conditions if instance.Status.Conditions.AllSubConditionIsTrue() { @@ -553,6 +564,89 @@ func (r TelemetryReconciler) reconcileLogging(ctx context.Context, instance *tel return ctrl.Result{}, nil } +// reconcileAutoscaling ... +func (r TelemetryReconciler) reconcileCloudKitty(ctx context.Context, instance *telemetryv1.Telemetry, helper *helper.Helper) (ctrl.Result, error) { + const ( + cloudKittyNamespaceLabel = "CloudKitty.Namespace" + cloudKittyNameLabel = "CloudKitty.Name" + cloudKittyName = "cloudkitty" + ) + cloudKittyInstance := &telemetryv1.CloudKitty{ + ObjectMeta: metav1.ObjectMeta{ + Name: cloudKittyName, + Namespace: instance.Namespace, + }, + } + + if instance.Spec.CloudKitty.Enabled == nil || !*instance.Spec.CloudKitty.Enabled { + if res, err := utils.EnsureDeleted(ctx, helper, cloudKittyInstance); err != nil { + return res, err + } + instance.Status.Conditions.Remove(telemetryv1.CloudKittyReadyCondition) + return ctrl.Result{}, nil + } + + if instance.Spec.CloudKitty.NodeSelector == nil { + instance.Spec.CloudKitty.NodeSelector = instance.Spec.NodeSelector + } + + if instance.Spec.CloudKitty.TopologyRef == nil { + instance.Spec.CloudKitty.TopologyRef = instance.Spec.TopologyRef + } + + helper.GetLogger().Info("Reconciling CloudKitty", cloudKittyNamespaceLabel, instance.Namespace, cloudKittyNameLabel, cloudKittyName) + op, err := controllerutil.CreateOrPatch(ctx, helper.GetClient(), cloudKittyInstance, func() error { + instance.Spec.CloudKitty.CloudKittySpec.DeepCopyInto(&cloudKittyInstance.Spec) + + err := controllerutil.SetControllerReference(helper.GetBeforeObject(), cloudKittyInstance, helper.GetScheme()) + if err != nil { + return err + } + return nil + }) + + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + telemetryv1.CloudKittyReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + // Check the observed Generation and mirror the condition from the + // underlying resource reconciliation + autoObsGen, err := r.checkCloudKittyGeneration(instance) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + telemetryv1.CloudKittyReadyErrorMessage, + err.Error())) + return ctrl.Result{}, nil + } + if !autoObsGen { + instance.Status.Conditions.Set(condition.UnknownCondition( + telemetryv1.CloudKittyReadyCondition, + condition.InitReason, + telemetryv1.AutoscalingReadyRunningMessage, + )) + } else { + // Mirror Autoscaling condition status + c := cloudKittyInstance.Status.Conditions.Mirror(telemetryv1.CloudKittyReadyCondition) + if c != nil { + instance.Status.Conditions.Set(c) + } + } + if op != controllerutil.OperationResultNone && autoObsGen { + helper.GetLogger().Info(fmt.Sprintf("%s %s - %s", cloudKittyName, cloudKittyInstance.Name, op)) + } + + return ctrl.Result{}, nil +} + // SetupWithManager sets up the controller with the Manager. func (r *TelemetryReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). @@ -561,6 +655,7 @@ func (r *TelemetryReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&telemetryv1.Autoscaling{}). Owns(&telemetryv1.MetricStorage{}). Owns(&telemetryv1.Logging{}). + Owns(&telemetryv1.CloudKitty{}). Complete(r) } @@ -647,3 +742,24 @@ func (r *TelemetryReconciler) checkLoggingGeneration( } return true, nil } + +// checkCloudKittyGeneration - +func (r *TelemetryReconciler) checkCloudKittyGeneration( + instance *telemetryv1.Telemetry, +) (bool, error) { + Log := r.GetLogger(context.Background()) + clm := &telemetryv1.CloudKittyList{} + listOpts := []client.ListOption{ + client.InNamespace(instance.Namespace), + } + if err := r.Client.List(context.Background(), clm, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve CloudKitty CR %w") + return false, err + } + for _, item := range clm.Items { + if item.Generation != item.Status.ObservedGeneration { + return false, nil + } + } + return true, nil +} diff --git a/main.go b/main.go index 66e17ea5..7905d299 100644 --- a/main.go +++ b/main.go @@ -198,10 +198,38 @@ func main() { os.Exit(1) } + if err = (&controllers.CloudKittyReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create CloudKitty controller") + os.Exit(1) + } + + if err = (&controllers.CloudKittyAPIReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + }).SetupWithManager(context.Background(), mgr); err != nil { + setupLog.Error(err, "unable to create CloudKitty API controller") + os.Exit(1) + } + + if err = (&controllers.CloudKittyProcReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + }).SetupWithManager(context.Background(), mgr); err != nil { + setupLog.Error(err, "unable to create CloudKitty Processor controller") + os.Exit(1) + } + // Acquire environmental defaults and initialize defaults with them telemetryv1beta1.SetupDefaultsTelemetry() telemetryv1beta1.SetupDefaultsCeilometer() telemetryv1beta1.SetupDefaultsAutoscaling() + telemetryv1beta1.SetupDefaultsCloudKitty() // Setup webhooks if requested checker := healthz.Ping @@ -223,6 +251,10 @@ func main() { setupLog.Error(err, "unable to create webhook", "webhook", "MetricStorage") os.Exit(1) } + if err = (&telemetryv1beta1.CloudKitty{}).SetupWebhookWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "CloudKitty") + os.Exit(1) + } checker = mgr.GetWebhookServer().StartedChecker() } diff --git a/pkg/cloudkitty/common.go b/pkg/cloudkitty/common.go new file mode 100644 index 00000000..9c78b72b --- /dev/null +++ b/pkg/cloudkitty/common.go @@ -0,0 +1,161 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkitty + +import ( + "context" + "fmt" + "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "k8s.io/apimachinery/pkg/types" + "time" + + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + k8s_errors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +type conditionUpdater interface { + Set(c *condition.Condition) + MarkTrue(t condition.Type, messageFormat string, messageArgs ...interface{}) +} + +type topologyHandler interface { + GetSpecTopologyRef() *topologyv1.TopoRef + GetLastAppliedTopology() *topologyv1.TopoRef + SetLastAppliedTopology(t *topologyv1.TopoRef) +} + +// EnsureTopology - +func EnsureTopology( + ctx context.Context, + helper *helper.Helper, + instance topologyHandler, + finalizer string, + conditionUpdater conditionUpdater, + defaultLabelSelector metav1.LabelSelector, +) (*topologyv1.Topology, error) { + + topology, err := topologyv1.EnsureServiceTopology( + ctx, + helper, + instance.GetSpecTopologyRef(), + instance.GetLastAppliedTopology(), + finalizer, + defaultLabelSelector, + ) + if err != nil { + conditionUpdater.Set(condition.FalseCondition( + condition.TopologyReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.TopologyReadyErrorMessage, + err.Error())) + return nil, fmt.Errorf("waiting for Topology requirements: %w", err) + } + // update the Status with the last retrieved Topology (or set it to nil) + instance.SetLastAppliedTopology(instance.GetSpecTopologyRef()) + // update the Topology condition only when a Topology is referenced and has + // been retrieved (err == nil) + if tr := instance.GetSpecTopologyRef(); tr != nil { + // update the TopologyRef associated condition + conditionUpdater.MarkTrue( + condition.TopologyReadyCondition, + condition.TopologyReadyMessage, + ) + } + return topology, nil +} + +// VerifyServiceSecret - ensures that the Secret object exists and the expected +// fields are in the Secret. It also sets a hash of the values of the expected +// fields passed as input. +func VerifyServiceSecret( + ctx context.Context, + secretName types.NamespacedName, + expectedFields []string, + reader client.Reader, + conditionUpdater conditionUpdater, + requeueTimeout time.Duration, + envVars *map[string]env.Setter, +) (ctrl.Result, error) { + + hash, res, err := secret.VerifySecret(ctx, secretName, expectedFields, reader, requeueTimeout) + if err != nil { + conditionUpdater.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyErrorMessage, + err.Error())) + return res, err + } else if (res != ctrl.Result{}) { + log.FromContext(ctx).Info(fmt.Sprintf("OpenStack secret %s not found", secretName)) + conditionUpdater.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.InputReadyWaitingMessage)) + return res, nil + } + (*envVars)[secretName.Name] = env.SetValue(hash) + return ctrl.Result{}, nil +} + +// VerifyConfigSecrets - It iterates over the secretNames passed as input and +// sets the hash of values in the envVars map. +func VerifyConfigSecrets( + ctx context.Context, + h *helper.Helper, + conditionUpdater conditionUpdater, + secretNames []string, + namespace string, + envVars *map[string]env.Setter, +) (ctrl.Result, error) { + var hash string + var err error + for _, secretName := range secretNames { + _, hash, err = secret.GetSecret(ctx, h, secretName, namespace) + if err != nil { + if k8s_errors.IsNotFound(err) { + log.FromContext(ctx).Info(fmt.Sprintf("Secret %s not found", secretName)) + conditionUpdater.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.InputReadyWaitingMessage)) + return ResultRequeue, nil + } + conditionUpdater.Set(condition.FalseCondition( + condition.InputReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.InputReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + // Add a prefix to the var name to avoid accidental collision with other non-secret + // vars. The secret names themselves will be unique. + (*envVars)["secret-"+secretName] = env.SetValue(hash) + } + + return ctrl.Result{}, nil +} diff --git a/pkg/cloudkitty/const.go b/pkg/cloudkitty/const.go new file mode 100644 index 00000000..5fb5269a --- /dev/null +++ b/pkg/cloudkitty/const.go @@ -0,0 +1,54 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkitty + +import ( + "time" + + ctrl "sigs.k8s.io/controller-runtime" +) + +const ( + // ServiceName - + ServiceName = "cloudkitty" + // ServiceType - + ServiceType = "rating" + // DatabaseName - + DatabaseName = "cloudkitty" + + // DefaultsConfigFileName - + DefaultsConfigFileName = "cloudkitty.conf" + // ServiceConfigFileName - + ServiceConfigFileName = "01-service-defaults.conf" + // CustomConfigFileName - + CustomConfigFileName = "02-global-custom.conf" + // CustomServiceConfigFileName - + CustomServiceConfigFileName = "03-service-custom.conf" + // CustomServiceConfigSecretsFileName - + CustomServiceConfigSecretsFileName = "04-service-custom-secrets.conf" + // MyCnfFileName - + MyCnfFileName = "my.cnf" + + // CloudKittyPublicPort - + CloudKittyPublicPort int32 = 8889 + // CloudKittyInternalPort - + CloudKittyInternalPort int32 = 8889 + + ShortDuration = time.Duration(5) * time.Second + NormalDuration = time.Duration(10) * time.Second +) + +var ResultRequeue = ctrl.Result{RequeueAfter: NormalDuration} diff --git a/pkg/cloudkitty/dbsync.go b/pkg/cloudkitty/dbsync.go new file mode 100644 index 00000000..75a675e4 --- /dev/null +++ b/pkg/cloudkitty/dbsync.go @@ -0,0 +1,107 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cloudkitty + +import ( + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // DBSyncCommand - + // TODO: Once we work on update/upgrades revisit the command in the + // the cloudkitty-dbsync-config.json file. + // If we stop all services during the update/upgrade then we can keep + // the --bump-versions flag. + // If we are doing rolling upgrades we'll need to use the flag + // conditionally (only for adoption) and do the restart cycle of + // services as described in the upstream rolling upgrades process. + dbSyncCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start" +) + +// DbSyncJob func +func DbSyncJob(instance *telemetryv1.CloudKitty, labels map[string]string) *batchv1.Job { + args := []string{"-c"} + args = append(args, dbSyncCommand) + + // create Volume and VolumeMounts + volumes := GetVolumes("cloudkitty") + volumeMounts := GetVolumeMounts("cloudkitty-dbsync") + // add CA cert if defined + if instance.Spec.CloudKittyAPI.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.CloudKittyAPI.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.CloudKittyAPI.TLS.CreateVolumeMounts(nil)...) + } + + runAsUser := int64(0) + envVars := map[string]env.Setter{} + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["KOLLA_BOOTSTRAP"] = env.SetValue("TRUE") + cloudKittyPassword := []corev1.EnvVar{ + { + Name: "AodhPassword", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: instance.Spec.Secret, + }, + Key: instance.Spec.PasswordSelectors.CloudKittyService, + }, + }, + }, + } + + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: ServiceName + "-db-sync", + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + ServiceAccountName: instance.RbacResourceName(), + Containers: []corev1.Container{ + { + Name: ServiceName + "-db-sync", + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: instance.Spec.CloudKittyAPI.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &runAsUser, + }, + Env: env.MergeEnvs(cloudKittyPassword, envVars), + VolumeMounts: volumeMounts, + }, + }, + Volumes: volumes, + }, + }, + }, + } + + if instance.Spec.NodeSelector != nil { + job.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector + } + + return job +} diff --git a/pkg/cloudkitty/funcs.go b/pkg/cloudkitty/funcs.go new file mode 100644 index 00000000..43210dad --- /dev/null +++ b/pkg/cloudkitty/funcs.go @@ -0,0 +1,49 @@ +package cloudkitty + +import ( + common "github.com/openstack-k8s-operators/lib-common/modules/common" + "github.com/openstack-k8s-operators/lib-common/modules/common/affinity" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// GetOwningCloudKittyName - Given a CloudKittyAPI or CloudKittyProc +// object, returning the parent CloudKitty object that created it (if any) +func GetOwningCloudKittyName(instance client.Object) string { + for _, ownerRef := range instance.GetOwnerReferences() { + if ownerRef.Kind == "CloudKitty" { + return ownerRef.Name + } + } + + return "" +} + +// GetNetworkAttachmentAddrs - Returns a list of IP addresses for all network attachments. +func GetNetworkAttachmentAddrs(namespace string, networkAttachments []string, networkAttachmentStatus map[string][]string) []string { + networkAttachmentAddrs := []string{} + + for _, network := range networkAttachments { + networkName := namespace + "/" + network + if networkAddrs, ok := networkAttachmentStatus[networkName]; ok { + networkAttachmentAddrs = append(networkAttachmentAddrs, networkAddrs...) + } + } + + return networkAttachmentAddrs +} + +// GetPodAffinity - Returns a corev1.Affinity reference for the specified component. +func GetPodAffinity(componentName string) *corev1.Affinity { + // If possible two pods of the same component (e.g cloudkitty-api) should not + // run on the same worker node. If this is not possible they get still + // created on the same worker node. + return affinity.DistributePods( + common.ComponentSelector, + []string{ + componentName, + }, + corev1.LabelHostname, + ) +} diff --git a/pkg/cloudkitty/volumes.go b/pkg/cloudkitty/volumes.go new file mode 100644 index 00000000..97ed4ab5 --- /dev/null +++ b/pkg/cloudkitty/volumes.go @@ -0,0 +1,57 @@ +package cloudkitty + +import ( + corev1 "k8s.io/api/core/v1" +) + +var ( + // scriptMode is the default permissions mode for Scripts volume + scriptMode int32 = 0740 + // configMode is the 640 permissions mode + configMode int32 = 0640 +) + +// GetVolumes - service volumes +func GetVolumes(name string) []corev1.Volume { + return []corev1.Volume{ + { + Name: "scripts", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &scriptMode, + SecretName: name + "-scripts", + }, + }, + }, { + Name: "config-data", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &configMode, + SecretName: name + "-config-data", + }, + }, + }, + } +} + +// GetVolumeMounts - general VolumeMounts +func GetVolumeMounts(serviceName string) []corev1.VolumeMount { + return []corev1.VolumeMount{ + { + Name: "scripts", + MountPath: "/var/lib/openstack/bin", + ReadOnly: true, + }, + { + Name: "config-data", + MountPath: "/var/lib/openstack/config", + ReadOnly: true, + }, + { + Name: "config-data", + MountPath: "/var/lib/kolla/config_files/config.json", + SubPath: serviceName + "-config.json", + ReadOnly: true, + }, + } +} diff --git a/pkg/cloudkittyapi/const.go b/pkg/cloudkittyapi/const.go new file mode 100644 index 00000000..7d9a378b --- /dev/null +++ b/pkg/cloudkittyapi/const.go @@ -0,0 +1,24 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkittyapi + +const ( + // ComponentName - + ComponentName = "cloudkitty-api" + + //LogFile - + LogFile = "/var/log/cloudkitty/cloudkitty-api.log" +) diff --git a/pkg/cloudkittyapi/statefulset.go b/pkg/cloudkittyapi/statefulset.go new file mode 100644 index 00000000..94d4346c --- /dev/null +++ b/pkg/cloudkittyapi/statefulset.go @@ -0,0 +1,188 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkittyapi + +import ( + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + // ServiceCommand - + ServiceCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start" +) + +// StatefulSet func +func StatefulSet( + instance *telemetryv1.CloudKittyAPI, + configHash string, + labels map[string]string, + annotations map[string]string, + topology *topologyv1.Topology, +) (*appsv1.StatefulSet, error) { + runAsUser := int64(0) + //cloudKittyUser := int64(telemetryv1.CloudKittyUserID) + + livenessProbe := &corev1.Probe{ + // TODO might need tuning + TimeoutSeconds: 5, + PeriodSeconds: 3, + InitialDelaySeconds: 5, + } + readinessProbe := &corev1.Probe{ + // TODO might need tuning + TimeoutSeconds: 5, + PeriodSeconds: 5, + InitialDelaySeconds: 5, + } + + args := []string{"-c", ServiceCommand} + // + // https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ + // + livenessProbe.HTTPGet = &corev1.HTTPGetAction{ + Path: "/healthcheck", + Port: intstr.IntOrString{Type: intstr.Int, IntVal: int32(cloudkitty.CloudKittyPublicPort)}, + } + readinessProbe.HTTPGet = livenessProbe.HTTPGet + + if instance.Spec.TLS.API.Enabled(service.EndpointPublic) { + livenessProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS + readinessProbe.HTTPGet.Scheme = corev1.URISchemeHTTPS + } + + // create Volume and VolumeMounts + volumes := GetVolumes(cloudkitty.GetOwningCloudKittyName(instance), instance.Name) + volumeMounts := GetVolumeMounts(instance.Name) + + // add CA cert if defined + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + + for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} { + if instance.Spec.TLS.API.Enabled(endpt) { + var tlsEndptCfg tls.GenericService + switch endpt { + case service.EndpointPublic: + tlsEndptCfg = instance.Spec.TLS.API.Public + case service.EndpointInternal: + tlsEndptCfg = instance.Spec.TLS.API.Internal + } + + svc, err := tlsEndptCfg.ToService() + if err != nil { + return nil, err + } + volumes = append(volumes, svc.CreateVolume(endpt.String())) + volumeMounts = append(volumeMounts, svc.CreateVolumeMounts(endpt.String())...) + } + } + + envVars := map[string]env.Setter{} + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["CONFIG_HASH"] = env.SetValue(configHash) + + statefulset := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Name, + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: appsv1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + PodManagementPolicy: appsv1.ParallelPodManagement, + Replicas: instance.Spec.Replicas, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + Labels: labels, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: instance.Spec.ServiceAccount, + Containers: []corev1.Container{ + // the first container in a pod is the default selected + // by oc log so define the log stream container first. + { + Name: instance.Name + "-log", + Command: []string{ + "/usr/bin/dumb-init", + }, + Args: []string{ + "--single-child", + "--", + "/bin/sh", + "-c", + "/usr/bin/tail -n+1 -F " + LogFile + " 2>/dev/null", + }, + Image: instance.Spec.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &runAsUser, + }, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: []corev1.VolumeMount{GetLogVolumeMount()}, + Resources: instance.Spec.Resources, + }, + { + Name: ComponentName, + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: instance.Spec.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &runAsUser, + }, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + Resources: instance.Spec.Resources, + ReadinessProbe: readinessProbe, + LivenessProbe: livenessProbe, + }, + }, + Volumes: volumes, + }, + }, + }, + } + + if instance.Spec.NodeSelector != nil { + statefulset.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector + } + + if topology != nil { + topology.ApplyTo(&statefulset.Spec.Template) + } else { + // If possible two pods of the same service should not + // run on the same worker node. If this is not possible + // the get still created on the same worker node. + statefulset.Spec.Template.Spec.Affinity = cloudkitty.GetPodAffinity(ComponentName) + } + + return statefulset, nil +} diff --git a/pkg/cloudkittyapi/volumes.go b/pkg/cloudkittyapi/volumes.go new file mode 100644 index 00000000..6d9f36ab --- /dev/null +++ b/pkg/cloudkittyapi/volumes.go @@ -0,0 +1,54 @@ +package cloudkittyapi + +import ( + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + corev1 "k8s.io/api/core/v1" +) + +// GetVolumes - +func GetVolumes(parentName string, name string) []corev1.Volume { + var config0644AccessMode int32 = 0644 + + volumes := []corev1.Volume{ + { + Name: "config-data-custom", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &config0644AccessMode, + SecretName: name + "-config-data", + }, + }, + }, + { + Name: "logs", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{Medium: ""}, + }, + }, + } + + return append(cloudkitty.GetVolumes(parentName), volumes...) +} + +// GetVolumeMounts - CloudKitty API VolumeMounts +func GetVolumeMounts(parentName string) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{ + { + Name: "config-data-custom", + MountPath: "/etc/cloudkitty/cloudkitty.conf.d", + ReadOnly: true, + }, + GetLogVolumeMount(), + } + + return append(cloudkitty.GetVolumeMounts(parentName), volumeMounts...) +} + +// GetLogVolumeMount - CloudKitty API LogVolumeMount +func GetLogVolumeMount() corev1.VolumeMount { + return corev1.VolumeMount{ + Name: "logs", + MountPath: "/var/log/cloudkitty", + ReadOnly: false, + } +} diff --git a/pkg/cloudkittyproc/const.go b/pkg/cloudkittyproc/const.go new file mode 100644 index 00000000..8bda7978 --- /dev/null +++ b/pkg/cloudkittyproc/const.go @@ -0,0 +1,21 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkittyproc + +const ( + // ComponentName - + ComponentName = "cloudkitty-proc" +) diff --git a/pkg/cloudkittyproc/statefulset.go b/pkg/cloudkittyproc/statefulset.go new file mode 100644 index 00000000..3071b60e --- /dev/null +++ b/pkg/cloudkittyproc/statefulset.go @@ -0,0 +1,153 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkittyproc + +import ( + topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + // ServiceCommand - + ServiceCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start" +) + +// StatefulSet func +func StatefulSet( + instance *telemetryv1.CloudKittyProc, + configHash string, + labels map[string]string, + annotations map[string]string, + topology *topologyv1.Topology, +) *appsv1.StatefulSet { + cloudKittyUser := int64(0) + // cloudKittyUser := int64(telemetryv1.CloudKittyUserID) + // cloudKittyGroup := int64(telemetryv1.CloudKittyGroupID) + + // TODO until we determine how to properly query for these + livenessProbe := &corev1.Probe{ + // TODO might need tuning + TimeoutSeconds: 5, + PeriodSeconds: 3, + InitialDelaySeconds: 3, + } + + startupProbe := &corev1.Probe{ + TimeoutSeconds: 5, + FailureThreshold: 12, + PeriodSeconds: 5, + InitialDelaySeconds: 5, + } + + args := []string{"-c", ServiceCommand} + var probeCommand []string + livenessProbe.HTTPGet = &corev1.HTTPGetAction{ + Port: intstr.FromInt(8080), + } + startupProbe.HTTPGet = livenessProbe.HTTPGet + probeCommand = []string{ + "/usr/local/bin/container-scripts/healthcheck.py", + "processor", + "/etc/cloudkitty/cloudkitty.conf.d", + } + + envVars := map[string]env.Setter{} + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["CONFIG_HASH"] = env.SetValue(configHash) + + volumes := GetVolumes(cloudkitty.GetOwningCloudKittyName(instance), instance.Name) + volumeMounts := GetVolumeMounts(instance.Name) + + // Add the CA bundle + if instance.Spec.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + + statefulset := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Name, + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: appsv1.StatefulSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Replicas: instance.Spec.Replicas, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + Labels: labels, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: instance.Spec.ServiceAccount, + Containers: []corev1.Container{ + { + Name: ComponentName, + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: instance.Spec.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &cloudKittyUser, + }, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + Resources: instance.Spec.Resources, + LivenessProbe: livenessProbe, + StartupProbe: startupProbe, + }, + { + Name: "probe", + Command: probeCommand, + Image: instance.Spec.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &cloudKittyUser, + //RunAsGroup: &cloudKittyGroup, + }, + VolumeMounts: volumeMounts, + }, + }, + Volumes: volumes, + }, + }, + }, + } + + if instance.Spec.NodeSelector != nil { + statefulset.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector + } + + if topology != nil { + topology.ApplyTo(&statefulset.Spec.Template) + } else { + // If possible two pods of the same service should not + // run on the same worker node. If this is not possible + // the get still created on the same worker node. + statefulset.Spec.Template.Spec.Affinity = cloudkitty.GetPodAffinity(ComponentName) + } + + return statefulset +} diff --git a/pkg/cloudkittyproc/volumes.go b/pkg/cloudkittyproc/volumes.go new file mode 100644 index 00000000..51ff46a7 --- /dev/null +++ b/pkg/cloudkittyproc/volumes.go @@ -0,0 +1,38 @@ +package cloudkittyproc + +import ( + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + corev1 "k8s.io/api/core/v1" +) + +// GetVolumes - +func GetVolumes(parentName string, name string) []corev1.Volume { + var config0644AccessMode int32 = 0644 + + volumes := []corev1.Volume{ + { + Name: "config-data-custom", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &config0644AccessMode, + SecretName: name + "-config-data", + }, + }, + }, + } + + return append(cloudkitty.GetVolumes(parentName), volumes...) +} + +// GetVolumeMounts - CloudKitty API VolumeMounts +func GetVolumeMounts(parentName string) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{ + { + Name: "config-data-custom", + MountPath: "/etc/cloudkitty/cloudkitty.conf.d", + ReadOnly: true, + }, + } + + return append(cloudkitty.GetVolumeMounts(parentName), volumeMounts...) +} diff --git a/templates/cloudkitty/bin/healthcheck.py b/templates/cloudkitty/bin/healthcheck.py new file mode 100755 index 00000000..4bce1182 --- /dev/null +++ b/templates/cloudkitty/bin/healthcheck.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# +# Copyright 2022 Red Hat Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Trivial HTTP server to check health of scheduler, backup and volume services. +# Cinder-API hast its own health check endpoint and does not need this. +# +# The only check this server currently does is using the heartbeat in the +# database service table, accessing the DB directly here using cinder's +# configuration options. +# +# The benefit of accessing the DB directly is that it doesn't depend on the +# Cinder-API service being up and we can also differentiate between the +# container not having a connection to the DB and the cinder service not doing +# the heartbeats. +# +# For volume services all enabled backends must be up to return 200, so it is +# recommended to use a different pod for each backend to avoid one backend +# affecting others. +# +# Requires the name of the service as the first argument (volume, backup, +# scheduler) and optionally a second argument with the location of the +# configuration directory (defaults to /etc/cinder/cinder.conf.d) + +from http import server +import signal +import socket +import sys +import time +import threading + +from oslo_config import cfg + +SERVER_PORT = 8080 +CONF = cfg.CONF + +class HTTPServerV6(server.HTTPServer): + address_family = socket.AF_INET6 + +class HeartbeatServer(server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-type", "text/html") + self.end_headers() + self.wfile.write('OK'.encode('utf-8')) + + +def get_stopper(server): + def stopper(signal_number=None, frame=None): + print("Stopping server.") + server.shutdown() + server.server_close() + print("Server stopped.") + sys.exit(0) + return stopper + + +if __name__ == "__main__": + hostname = socket.gethostname() + ipv6_address = socket.getaddrinfo(hostname, None, socket.AF_INET6) + if ipv6_address: + webServer = HTTPServerV6(("::",SERVER_PORT), HeartbeatServer) + else: + webServer = server.HTTPServer(("0.0.0.0", SERVER_PORT), HeartbeatServer) + stop = get_stopper(webServer) + + # Need to run the server on a different thread because its shutdown method + # will block if called from the same thread, and the signal handler must be + # on the main thread in Python. + thread = threading.Thread(target=webServer.serve_forever) + thread.daemon = True + thread.start() + print(f"CloudKitty Healthcheck Server started http://{hostname}:{SERVER_PORT}") + signal.signal(signal.SIGTERM, stop) + + try: + while True: + time.sleep(60) + except KeyboardInterrupt: + pass + finally: + stop() diff --git a/templates/cloudkitty/bin/run-on-host b/templates/cloudkitty/bin/run-on-host new file mode 100755 index 00000000..e7840ace --- /dev/null +++ b/templates/cloudkitty/bin/run-on-host @@ -0,0 +1,2 @@ +#!/bin/sh +exec nsenter -a -t 1 -- `realpath -s $0` "$@" diff --git a/templates/cloudkitty/config/10-cloudkitty_wsgi.conf b/templates/cloudkitty/config/10-cloudkitty_wsgi.conf new file mode 100644 index 00000000..1fc084ae --- /dev/null +++ b/templates/cloudkitty/config/10-cloudkitty_wsgi.conf @@ -0,0 +1,40 @@ +{{ range $endpt, $vhost := .VHosts }} +# {{ $endpt }} vhost {{ $vhost.ServerName }} configuration + + ServerName {{ $vhost.ServerName }} + + ## Vhost docroot + DocumentRoot "/var/www/cgi-bin/cloudkitty" + + ## Directories, there should at least be a declaration for /var/www/cgi-bin/cloudkitty + + + Options -Indexes +FollowSymLinks +MultiViews + AllowOverride None + Require all granted + + + Timeout {{ $.TimeOut }} + + ## Logging + ErrorLog /dev/stdout + ServerSignature Off + CustomLog /dev/stdout combined + +{{- if $vhost.TLS }} + SetEnvIf X-Forwarded-Proto https HTTPS=1 + + ## SSL directives + SSLEngine on + SSLCertificateFile "{{ $vhost.SSLCertificateFile }}" + SSLCertificateKeyFile "{{ $vhost.SSLCertificateKeyFile }}" +{{- end }} + + ## WSGI configuration + WSGIApplicationGroup %{GLOBAL} + WSGIDaemonProcess {{ $endpt }} display-name={{ $endpt }} group=cloudkitty processes=4 threads=1 user=cloudkitty + WSGIProcessGroup {{ $endpt }} + WSGIScriptAlias / "/var/www/cgi-bin/cloudkitty/cloudkitty-wsgi" + WSGIPassAuthorization On + +{{ end }} diff --git a/templates/cloudkitty/config/cloudkitty-api-config-httpd.json b/templates/cloudkitty/config/cloudkitty-api-config-httpd.json new file mode 100644 index 00000000..174349bf --- /dev/null +++ b/templates/cloudkitty/config/cloudkitty-api-config-httpd.json @@ -0,0 +1,51 @@ +{ + "command": "/usr/sbin/httpd -DFOREGROUND", + "config_files": [ + { + "source": "/var/lib/openstack/config/httpd.conf", + "dest": "/etc/httpd/conf/httpd.conf", + "owner": "root", + "perm": "0644" + }, + { + "source": "/var/lib/openstack/config/10-cloudkitty_wsgi.conf", + "dest": "/etc/httpd/conf.d/10-cloudkitty_wsgi.conf", + "owner": "root", + "perm": "0644" + }, + { + "source": "/var/lib/openstack/config/ssl.conf", + "dest": "/etc/httpd/conf.d/ssl.conf", + "owner": "cloudkitty", + "perm": "0644" + }, + { + "source": "/var/lib/config-data/tls/certs/*", + "dest": "/etc/pki/tls/certs/", + "owner": "cloudkitty", + "perm": "0640", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/config-data/tls/private/*", + "dest": "/etc/pki/tls/private/", + "owner": "cloudkitty", + "perm": "0600", + "optional": true, + "merge": true + } + ], + "permissions": [ + { + "path": "/var/log/cloudkitty", + "owner": "cloudkitty:apache", + "recurse": true + }, + { + "path": "/etc/httpd/run", + "owner": "cloudkitty:apache", + "recurse": true + } + ] +} diff --git a/templates/cloudkitty/config/cloudkitty-api-config.json b/templates/cloudkitty/config/cloudkitty-api-config.json new file mode 100644 index 00000000..11a6f291 --- /dev/null +++ b/templates/cloudkitty/config/cloudkitty-api-config.json @@ -0,0 +1,11 @@ +{ + "command": "/usr/bin/uwsgi --ini /etc/cloudkitty/cloudkitty-api-uwsgi.ini", + "config_files": [ + { + "source": "/var/lib/openstack/config/cloudkitty-api-uwsgi.ini", + "dest": "/etc/cloudkitty/cloudkitty-api-uwsgi.ini", + "owner": "cloudkitty", + "perm": "0600" + } + ] +} diff --git a/templates/cloudkitty/config/cloudkitty-api-uwsgi.ini b/templates/cloudkitty/config/cloudkitty-api-uwsgi.ini new file mode 100644 index 00000000..2cd8f602 --- /dev/null +++ b/templates/cloudkitty/config/cloudkitty-api-uwsgi.ini @@ -0,0 +1,17 @@ +[uwsgi] +chmod-socket = 666 +socket = /var/run/uwsgi/cloudkitty.socket +start-time = %t +lazy-apps = true +add-header = Connection: close +buffer-size = 65535 +hook-master-start = unix_signal:15 gracefully_kill_them_all +thunder-lock = true +plugins = http,python3 +enable-threads = true +worker-reload-mercy = 80 +exit-on-reload = false +die-on-term = true +master = true +processes = 2 +wsgi-file = /opt/stack/data/venv/bin/cloudkitty-api \ No newline at end of file diff --git a/templates/cloudkitty/config/cloudkitty-dbsync-config.json b/templates/cloudkitty/config/cloudkitty-dbsync-config.json new file mode 100644 index 00000000..1eb4f0a6 --- /dev/null +++ b/templates/cloudkitty/config/cloudkitty-dbsync-config.json @@ -0,0 +1,11 @@ +{ + "command": "/usr/bin/cloudkitty-dbsync upgrade", + "config_files": [ + { + "source": "/var/lib/openstack/config/cloudkitty.conf", + "dest": "/etc/cloudkitty/cloudkitty.conf", + "owner": "cloudkitty", + "perm": "0600" + } + ] +} diff --git a/templates/cloudkitty/config/cloudkitty-proc-config.json b/templates/cloudkitty/config/cloudkitty-proc-config.json new file mode 100644 index 00000000..410ed9d3 --- /dev/null +++ b/templates/cloudkitty/config/cloudkitty-proc-config.json @@ -0,0 +1,17 @@ +{ + "command": "/usr/bin/cloudkitty-processor --logfile /dev/stdout", + "config_files": [ + { + "source": "/var/lib/openstack/config/cloudkitty.conf", + "dest": "/etc/cloudkitty/cloudkitty.conf", + "owner": "cloudkitty", + "perm": "0600" + }, + { + "source": "/var/lib/openstack/config/metrics.yaml", + "dest": "/etc/cloudkitty/metrics.yaml", + "owner": "cloudkitty", + "perm": "0600" + } + ] +} diff --git a/templates/cloudkitty/config/cloudkitty.conf b/templates/cloudkitty/config/cloudkitty.conf new file mode 100644 index 00000000..e9896eb2 --- /dev/null +++ b/templates/cloudkitty/config/cloudkitty.conf @@ -0,0 +1,78 @@ +[oslo_policy] +policy_file = policy.yaml + +[DEFAULT] +auth_strategy = keystone +debug = True +notification_topics = notifications +transport_url = {{ .TransportURL }} + +[authinfos] +debug = True +project_domain_name = default +user_domain_name = default +region_name = RegionOne +tenant_name = service +project_name = service +password = {{ .ServicePassword }} +username = {{ .ServiceUser }} +identity_uri = {{ .KeystoneInternalURL }} +auth_url = {{ .KeystoneInternalURL }} +auth_protocol = http +auth_type = v3password +{{- if .TLS }} +cafile = {{ .CAFile }} +{{- end }} + +[fetcher] +backend = keystone + +[fetcher_keystone] +auth_section = authinfos + +[storage_influxdb] +port = 8086 +host = localhost +database = cloudkitty +password = cloudkitty +user = cloudkitty + +[collect] +period = 300 +wait_periods = 0 +metrics_conf = /etc/cloudkitty/metrics.yaml +collector = prometheus +scope_key = project + +[collector_prometheus] +prometheus_url = https://metric-storage-prometheus.openstack.svc:9090 +insecure = true + +[output] +pipeline = osrf +basepath = /opt/stack/data/cloudkitty/reports +backend = cloudkitty.backend.file.FileBackend + +[storage] +version = 2 +backend = influxdb + +[database] +connection = {{ .DatabaseConnection }} + +[keystone_authtoken] +memcached_servers = {{ .MemcachedServersWithInet }} +# memcache_pool_dead_retry = 10 +# memcache_pool_conn_get_timeout = 2 +project_domain_name = Default +project_name = service +user_domain_name = Default +password = {{ .ServicePassword }} +username = {{ .ServiceUser }} +auth_url = {{ .KeystoneInternalURL }} +interface = internal +auth_type = password +{{- if .TLS }} +cafile = {{ .CAFile }} +{{- end }} +# service_token_roles_required = true diff --git a/templates/cloudkitty/config/httpd.conf b/templates/cloudkitty/config/httpd.conf new file mode 100644 index 00000000..b6c862a5 --- /dev/null +++ b/templates/cloudkitty/config/httpd.conf @@ -0,0 +1,27 @@ +ServerTokens Prod +ServerSignature Off +TraceEnable Off +ServerRoot "/etc/httpd" +ServerName "cloudkitty.openstack.svc" + +User apache +Group apache + +Listen 8889 + +TypesConfig /etc/mime.types + +Include conf.modules.d/*.conf + +LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined +LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" proxy + +SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded +CustomLog /dev/stdout combined env=!forwarded +CustomLog /dev/stdout proxy env=forwarded +ErrorLog /dev/stdout + +# XXX: To disable SSL +#Include conf.d/*.conf +# If above include is commented include at least the cloudkitty wsgi file +Include conf.d/10-cloudkitty_wsgi.conf diff --git a/templates/cloudkitty/config/metrics.yaml b/templates/cloudkitty/config/metrics.yaml new file mode 100644 index 00000000..c42701ed --- /dev/null +++ b/templates/cloudkitty/config/metrics.yaml @@ -0,0 +1,77 @@ +metrics: + ceilometer_cpu: + unit: instance + alt_name: instance + groupby: + - id + - user_id + - project + metadata: + - flavor_name + - flavor_id + - vcpus + mutate: NUMBOOL + extra_args: + aggregation_method: max + + ceilometer_image_size: + unit: MiB + factor: 1/1048576 + groupby: + - id + - user_id + - project + metadata: + - container_format + - disk_format + extra_args: + aggregation_method: max + + ceilometer_volume_size: + unit: GiB + groupby: + - id + - user_id + - project + metadata: + - volume_type + extra_args: + aggregation_method: max + + ceilometer_network_outgoing_bytes_rate: + unit: MB + groupby: + - id + - project + - user_id + # Converting B/s to MB/h + factor: 3600/1000000 + metadata: + - instance_id + extra_args: + aggregation_method: max + + ceilometer_network_incoming_bytes_rate: + unit: MB + groupby: + - id + - project + - user_id + # Converting B/s to MB/h + factor: 3600/1000000 + metadata: + - instance_id + extra_args: + aggregation_method: max + + ceilometer_ip_floating: + unit: ip + groupby: + - id + - user_id + - project + metadata: + - state + mutate: NUMBOOL + extra_args: + aggregation_method: max \ No newline at end of file diff --git a/templates/cloudkitty/config/ssl.conf b/templates/cloudkitty/config/ssl.conf new file mode 100644 index 00000000..e3da4ecb --- /dev/null +++ b/templates/cloudkitty/config/ssl.conf @@ -0,0 +1,21 @@ + + SSLRandomSeed startup builtin + SSLRandomSeed startup file:/dev/urandom 512 + SSLRandomSeed connect builtin + SSLRandomSeed connect file:/dev/urandom 512 + + AddType application/x-x509-ca-cert .crt + AddType application/x-pkcs7-crl .crl + + SSLPassPhraseDialog builtin + SSLSessionCache "shmcb:/var/cache/mod_ssl/scache(512000)" + SSLSessionCacheTimeout 300 + Mutex default + SSLCryptoDevice builtin + SSLHonorCipherOrder On + SSLUseStapling Off + SSLStaplingCache "shmcb:/run/httpd/ssl_stapling(32768)" + SSLCipherSuite HIGH:MEDIUM:!aNULL:!MD5:!RC4:!3DES + SSLProtocol all -SSLv2 -SSLv3 -TLSv1 + SSLOptions StdEnvVars + From 1acd39894348c0d50bdc5d8c8813820a08a2d024 Mon Sep 17 00:00:00 2001 From: jlarriba Date: Mon, 30 Jun 2025 15:17:26 +0200 Subject: [PATCH 14/87] CloudKitty TLS --- .../telemetry.openstack.org_cloudkitties.yaml | 11 ++ ...lemetry.openstack.org_cloudkittyprocs.yaml | 3 + .../telemetry.openstack.org_telemetries.yaml | 12 ++ api/v1beta1/cloudkitty_types.go | 6 +- api/v1beta1/cloudkittyproc_types.go | 10 +- api/v1beta1/conditions.go | 18 +++ api/v1beta1/zz_generated.deepcopy.go | 2 +- .../telemetry.openstack.org_cloudkitties.yaml | 11 ++ ...lemetry.openstack.org_cloudkittyprocs.yaml | 3 + .../telemetry.openstack.org_telemetries.yaml | 12 ++ controllers/cloudkitty_controller.go | 57 +++++++++- pkg/cloudkitty/dbsync.go | 2 +- pkg/cloudkitty/storageinit.go | 106 ++++++++++++++++++ pkg/cloudkittyproc/statefulset.go | 23 ++-- .../config/cloudkitty-api-config-httpd.json | 51 --------- .../config/cloudkitty-api-config.json | 69 +++++++++++- .../config/cloudkitty-storageinit-config.json | 11 ++ templates/cloudkitty/config/cloudkitty.conf | 10 +- templates/cloudkitty/config/httpd.conf | 2 +- ...udkitty_wsgi.conf => wsgi-cloudkitty.conf} | 2 +- 20 files changed, 335 insertions(+), 86 deletions(-) create mode 100644 pkg/cloudkitty/storageinit.go delete mode 100644 templates/cloudkitty/config/cloudkitty-api-config-httpd.json create mode 100644 templates/cloudkitty/config/cloudkitty-storageinit-config.json rename templates/cloudkitty/config/{10-cloudkitty_wsgi.conf => wsgi-cloudkitty.conf} (94%) diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml index bc5b7567..387bf62d 100644 --- a/api/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -450,6 +450,17 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in + a pre-created bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string + type: object topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml index 9cd9b6ef..6123f53c 100644 --- a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml +++ b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -196,6 +196,9 @@ spec: description: CaBundleSecretName - holding the CA certs in a pre-created bundle file type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string type: object topologyRef: description: |- diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index 0d8bf4d3..f69ad6d9 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -1008,6 +1008,18 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs + in a pre-created bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index 08181d6c..1d2d1a7e 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -32,11 +32,13 @@ const ( CloudKittyGroupID = 42408 // CloudKittyAPIContainerImage - default fall-back image for CloudKitty API - CloudKittyAPIContainerImage = "quay.io/podified-master-centos9/cloudkitty-api:current-podified" + CloudKittyAPIContainerImage = "quay.io/podified-master-centos9/openstack-cloudkitty-api:current-podified" // CloudKittyProcContainerImage - default fall-back image for CloudKitty Processor - CloudKittyProcContainerImage = "quay.io/podified-master-centos9/cloudkitty-processor:current-podified" + CloudKittyProcContainerImage = "quay.io/podified-master-centos9/openstack-cloudkitty-processor:current-podified" // CloudKittyDbSyncHash hash CKDbSyncHash = "ckdbsync" + // CKStorageInitHash hash + CKStorageInitHash = "ckstorageinit" // CloudKittyReplicas - The number of replicas per each service deployed CloudKittyReplicas = 1 ) diff --git a/api/v1beta1/cloudkittyproc_types.go b/api/v1beta1/cloudkittyproc_types.go index d2459afd..d7b80e19 100644 --- a/api/v1beta1/cloudkittyproc_types.go +++ b/api/v1beta1/cloudkittyproc_types.go @@ -33,6 +33,11 @@ type CloudKittyProcTemplateCore struct { // +kubebuilder:validation:Minimum=0 // Replicas - CloudKitty API Replicas Replicas *int32 `json:"replicas"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.SimpleService `json:"tls,omitempty"` } // CloudKittyProcTemplate defines the input parameters for the CloudKitty Processor service @@ -63,11 +68,6 @@ type CloudKittyProcSpec struct { // +kubebuilder:validation:Required // ServiceAccount - service account name used internally to provide CloudKitty services the default SA name ServiceAccount string `json:"serviceAccount"` - - // +kubebuilder:validation:Optional - // +operator-sdk:csv:customresourcedefinitions:type=spec - // TLS - Parameters related to the TLS - TLS tls.Ca `json:"tls,omitempty"` } // CloudKittyProcStatus defines the observed state of CloudKitty Processor diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index f8e0b4b7..709bd1cd 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -54,6 +54,9 @@ const ( // CloudKittyProcReadyCondition Status=True condition which indicates if the CloudKitty Processor is configured and operational CloudKittyProcReadyCondition condition.Type = "CloudKittyProcReady" + // CloudKittyStorageInitReadyCondition Status=True condition which indicates if the CloudKitty Storage Init process has ran + CloudKittyStorageInitReadyCondition condition.Type = "CloudKittyStorageInitReady" + // LoggingCLONamespaceReadyCondition Status=True condition which indicates if the cluster-logging-operator namespace is created LoggingCLONamespaceReadyCondition condition.Type = "LoggingCLONamespaceReady" @@ -223,6 +226,21 @@ const ( // CloudKittyReadyErrorMessage CloudKittyReadyErrorMessage = "CloudKitty error occured %s" + // + // CloudKittyStorageInit condition messages + // + // CloudKittyStorageInitReadyInitMessage + CloudKittyStorageInitReadyInitMessage = "CloudKittyStorageInit not started" + + // CloudKittyStorageInitReadyMessage + CloudKittyStorageInitReadyMessage = "CloudKittyStorageInit completed" + + // CloudKittyStorageInitReadyRunning + CloudKittyStorageInitReadyRunningMessage = "CloudKittyStorageInit job still running" + + // CloudKittyStorageInitReadyErrorMessage + CloudKittyStorageInitReadyErrorMessage = "CloudKittyStorageInit job error occurred %s" + // // CloudKittyAPIReady condition messages // diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 1c5006d6..c9d622b6 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -906,7 +906,6 @@ func (in *CloudKittyProcSpec) DeepCopyInto(out *CloudKittyProcSpec) { *out = *in out.CloudKittyTemplate = in.CloudKittyTemplate in.CloudKittyProcTemplate.DeepCopyInto(&out.CloudKittyProcTemplate) - out.TLS = in.TLS } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProcSpec. @@ -994,6 +993,7 @@ func (in *CloudKittyProcTemplateCore) DeepCopyInto(out *CloudKittyProcTemplateCo *out = new(int32) **out = **in } + in.TLS.DeepCopyInto(&out.TLS) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittyProcTemplateCore. diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml index bc5b7567..387bf62d 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -450,6 +450,17 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in + a pre-created bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string + type: object topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml index 9cd9b6ef..6123f53c 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -196,6 +196,9 @@ spec: description: CaBundleSecretName - holding the CA certs in a pre-created bundle file type: string + secretName: + description: SecretName - holding the cert, key for the service + type: string type: object topologyRef: description: |- diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index 0d8bf4d3..f69ad6d9 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -1008,6 +1008,18 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + tls: + description: TLS - Parameters related to the TLS + properties: + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs + in a pre-created bundle file + type: string + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index 4dadd651..00cc5996 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -181,6 +181,7 @@ func (r *CloudKittyReconciler) Reconcile(ctx context.Context, req ctrl.Request) condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), condition.UnknownCondition(condition.DBReadyCondition, condition.InitReason, condition.DBReadyInitMessage), condition.UnknownCondition(condition.DBSyncReadyCondition, condition.InitReason, condition.DBSyncReadyInitMessage), + condition.UnknownCondition(telemetryv1.CloudKittyStorageInitReadyCondition, condition.InitReason, telemetryv1.CloudKittyStorageInitReadyInitMessage), condition.UnknownCondition(condition.RabbitMqTransportURLReadyCondition, condition.InitReason, condition.RabbitMqTransportURLReadyInitMessage), condition.UnknownCondition(condition.MemcachedReadyCondition, condition.InitReason, condition.MemcachedReadyInitMessage), condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), @@ -419,10 +420,10 @@ func (r *CloudKittyReconciler) reconcileInit( // run CloudKitty db sync // dbSyncHash := instance.Status.Hash[telemetryv1.CKDbSyncHash] - jobDef := cloudkitty.DbSyncJob(instance, serviceLabels) + jobDbSyncDef := cloudkitty.DbSyncJob(instance, serviceLabels) dbSyncjob := job.NewJob( - jobDef, + jobDbSyncDef, telemetryv1.CKDbSyncHash, instance.Spec.PreserveJobs, cloudkitty.ShortDuration, @@ -451,12 +452,54 @@ func (r *CloudKittyReconciler) reconcileInit( } if dbSyncjob.HasChanged() { instance.Status.Hash[telemetryv1.CKDbSyncHash] = dbSyncjob.GetHash() - Log.Info(fmt.Sprintf("Service '%s' - Job %s hash added - %s", instance.Name, jobDef.Name, instance.Status.Hash[telemetryv1.CKDbSyncHash])) + Log.Info(fmt.Sprintf("Service '%s' - Job %s hash added - %s", instance.Name, jobDbSyncDef.Name, instance.Status.Hash[telemetryv1.CKDbSyncHash])) } instance.Status.Conditions.MarkTrue(condition.DBSyncReadyCondition, condition.DBSyncReadyMessage) // run CloudKitty db sync - end + // + // run CloudKitty Storage Init + // + ckStorageInitHash := instance.Status.Hash[telemetryv1.CKStorageInitHash] + jobStorageInitDef := cloudkitty.StorageInitJob(instance, serviceLabels) + + storageInitjob := job.NewJob( + jobStorageInitDef, + telemetryv1.CKStorageInitHash, + instance.Spec.PreserveJobs, + cloudkitty.ShortDuration, + ckStorageInitHash, + ) + ctrlResult, err = storageInitjob.DoJob( + ctx, + helper, + ) + if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyStorageInitReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + telemetryv1.CloudKittyStorageInitReadyRunningMessage)) + return ctrlResult, nil + } + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyStorageInitReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + telemetryv1.CloudKittyStorageInitReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if storageInitjob.HasChanged() { + instance.Status.Hash[telemetryv1.CKStorageInitHash] = storageInitjob.GetHash() + Log.Info(fmt.Sprintf("Service '%s' - Job %s hash added - %s", instance.Name, jobStorageInitDef.Name, instance.Status.Hash[telemetryv1.CKStorageInitHash])) + } + instance.Status.Conditions.MarkTrue(telemetryv1.CloudKittyStorageInitReadyCondition, telemetryv1.CloudKittyStorageInitReadyMessage) + + // run CloudKitty Storage Init - end + Log.Info(fmt.Sprintf("Reconciled Service '%s' init successfully", instance.Name)) return ctrl.Result{}, nil } @@ -829,6 +872,12 @@ func (r *CloudKittyReconciler) generateServiceConfigs( templateParameters["MemcachedServersWithInet"] = memcached.GetMemcachedServerListWithInetString() templateParameters["TimeOut"] = instance.Spec.APITimeout + templateParameters["TLS"] = false + if instance.Spec.CloudKittyProc.TLS.Enabled() { + templateParameters["TLS"] = true + templateParameters["CAFile"] = tls.DownstreamTLSCABundlePath + } + // create httpd vhost template parameters httpdVhostConfig := map[string]interface{}{} for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} { @@ -961,7 +1010,7 @@ func (r *CloudKittyReconciler) procDeploymentCreateOrUpdate(ctx context.Context, DatabaseHostname: instance.Status.DatabaseHostname, TransportURLSecret: instance.Status.TransportURLSecret, ServiceAccount: instance.RbacResourceName(), - TLS: instance.Spec.CloudKittyAPI.TLS.Ca, + //TLS: instance.Spec.CloudKittyProc.TLS.Ca, } if cloudKittyProcSpec.NodeSelector == nil { diff --git a/pkg/cloudkitty/dbsync.go b/pkg/cloudkitty/dbsync.go index 75a675e4..3cd213e5 100644 --- a/pkg/cloudkitty/dbsync.go +++ b/pkg/cloudkitty/dbsync.go @@ -55,7 +55,7 @@ func DbSyncJob(instance *telemetryv1.CloudKitty, labels map[string]string) *batc envVars["KOLLA_BOOTSTRAP"] = env.SetValue("TRUE") cloudKittyPassword := []corev1.EnvVar{ { - Name: "AodhPassword", + Name: "CloudKittyPassword", ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{ diff --git a/pkg/cloudkitty/storageinit.go b/pkg/cloudkitty/storageinit.go new file mode 100644 index 00000000..283c938d --- /dev/null +++ b/pkg/cloudkitty/storageinit.go @@ -0,0 +1,106 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cloudkitty + +import ( + "github.com/openstack-k8s-operators/lib-common/modules/common/env" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // storageInitCommand - + // TODO: Once we work on update/upgrades revisit the command in the + // the cloudkitty-storageinit-config.json file. + // If we stop all services during the update/upgrade then we can keep + // the --bump-versions flag. + // If we are doing rolling upgrades we'll need to use the flag + // conditionally (only for adoption) and do the restart cycle of + // services as described in the upstream rolling upgrades process. + storageInitCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start" +) + +// StorageInitJob func +func StorageInitJob(instance *telemetryv1.CloudKitty, labels map[string]string) *batchv1.Job { + args := []string{"-c", storageInitCommand} + + // create Volume and VolumeMounts + volumes := GetVolumes("cloudkitty") + volumeMounts := GetVolumeMounts("cloudkitty-storageinit") + // add CA cert if defined + if instance.Spec.CloudKittyAPI.TLS.CaBundleSecretName != "" { + volumes = append(volumes, instance.Spec.CloudKittyAPI.TLS.CreateVolume()) + volumeMounts = append(volumeMounts, instance.Spec.CloudKittyAPI.TLS.CreateVolumeMounts(nil)...) + } + + runAsUser := int64(0) + envVars := map[string]env.Setter{} + envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") + envVars["KOLLA_BOOTSTRAP"] = env.SetValue("TRUE") + cloudKittyPassword := []corev1.EnvVar{ + { + Name: "CloudKittyPassword", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: instance.Spec.Secret, + }, + Key: instance.Spec.PasswordSelectors.CloudKittyService, + }, + }, + }, + } + + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: ServiceName + "-storageinit", + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyOnFailure, + ServiceAccountName: instance.RbacResourceName(), + Containers: []corev1.Container{ + { + Name: ServiceName + "-storageinit", + Command: []string{ + "/bin/bash", + }, + Args: args, + Image: instance.Spec.CloudKittyAPI.ContainerImage, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &runAsUser, + }, + Env: env.MergeEnvs(cloudKittyPassword, envVars), + VolumeMounts: volumeMounts, + }, + }, + Volumes: volumes, + }, + }, + }, + } + + if instance.Spec.NodeSelector != nil { + job.Spec.Template.Spec.NodeSelector = *instance.Spec.NodeSelector + } + + return job +} diff --git a/pkg/cloudkittyproc/statefulset.go b/pkg/cloudkittyproc/statefulset.go index 3071b60e..c5959e00 100644 --- a/pkg/cloudkittyproc/statefulset.go +++ b/pkg/cloudkittyproc/statefulset.go @@ -24,7 +24,6 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" ) const ( @@ -45,7 +44,7 @@ func StatefulSet( // cloudKittyGroup := int64(telemetryv1.CloudKittyGroupID) // TODO until we determine how to properly query for these - livenessProbe := &corev1.Probe{ + /*livenessProbe := &corev1.Probe{ // TODO might need tuning TimeoutSeconds: 5, PeriodSeconds: 3, @@ -57,10 +56,10 @@ func StatefulSet( FailureThreshold: 12, PeriodSeconds: 5, InitialDelaySeconds: 5, - } + }*/ args := []string{"-c", ServiceCommand} - var probeCommand []string + /*var probeCommand []string livenessProbe.HTTPGet = &corev1.HTTPGetAction{ Port: intstr.FromInt(8080), } @@ -69,7 +68,7 @@ func StatefulSet( "/usr/local/bin/container-scripts/healthcheck.py", "processor", "/etc/cloudkitty/cloudkitty.conf.d", - } + }*/ envVars := map[string]env.Setter{} envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") @@ -113,13 +112,13 @@ func StatefulSet( SecurityContext: &corev1.SecurityContext{ RunAsUser: &cloudKittyUser, }, - Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: volumeMounts, - Resources: instance.Spec.Resources, - LivenessProbe: livenessProbe, - StartupProbe: startupProbe, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + Resources: instance.Spec.Resources, + //LivenessProbe: livenessProbe, + //StartupProbe: startupProbe, }, - { + /*{ Name: "probe", Command: probeCommand, Image: instance.Spec.ContainerImage, @@ -128,7 +127,7 @@ func StatefulSet( //RunAsGroup: &cloudKittyGroup, }, VolumeMounts: volumeMounts, - }, + },*/ }, Volumes: volumes, }, diff --git a/templates/cloudkitty/config/cloudkitty-api-config-httpd.json b/templates/cloudkitty/config/cloudkitty-api-config-httpd.json deleted file mode 100644 index 174349bf..00000000 --- a/templates/cloudkitty/config/cloudkitty-api-config-httpd.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "command": "/usr/sbin/httpd -DFOREGROUND", - "config_files": [ - { - "source": "/var/lib/openstack/config/httpd.conf", - "dest": "/etc/httpd/conf/httpd.conf", - "owner": "root", - "perm": "0644" - }, - { - "source": "/var/lib/openstack/config/10-cloudkitty_wsgi.conf", - "dest": "/etc/httpd/conf.d/10-cloudkitty_wsgi.conf", - "owner": "root", - "perm": "0644" - }, - { - "source": "/var/lib/openstack/config/ssl.conf", - "dest": "/etc/httpd/conf.d/ssl.conf", - "owner": "cloudkitty", - "perm": "0644" - }, - { - "source": "/var/lib/config-data/tls/certs/*", - "dest": "/etc/pki/tls/certs/", - "owner": "cloudkitty", - "perm": "0640", - "optional": true, - "merge": true - }, - { - "source": "/var/lib/config-data/tls/private/*", - "dest": "/etc/pki/tls/private/", - "owner": "cloudkitty", - "perm": "0600", - "optional": true, - "merge": true - } - ], - "permissions": [ - { - "path": "/var/log/cloudkitty", - "owner": "cloudkitty:apache", - "recurse": true - }, - { - "path": "/etc/httpd/run", - "owner": "cloudkitty:apache", - "recurse": true - } - ] -} diff --git a/templates/cloudkitty/config/cloudkitty-api-config.json b/templates/cloudkitty/config/cloudkitty-api-config.json index 11a6f291..380bb3a0 100644 --- a/templates/cloudkitty/config/cloudkitty-api-config.json +++ b/templates/cloudkitty/config/cloudkitty-api-config.json @@ -1,11 +1,74 @@ { - "command": "/usr/bin/uwsgi --ini /etc/cloudkitty/cloudkitty-api-uwsgi.ini", + "command": "/usr/sbin/httpd -DFOREGROUND -E /dev/stdout", "config_files": [ { - "source": "/var/lib/openstack/config/cloudkitty-api-uwsgi.ini", - "dest": "/etc/cloudkitty/cloudkitty-api-uwsgi.ini", + "source": "/var/lib/openstack/config/cloudkitty.conf", + "dest": "/etc/cloudkitty/cloudkitty.conf", "owner": "cloudkitty", "perm": "0600" + }, + { + "source": "/var/lib/openstack/config/custom.conf", + "dest": "/etc/aodh/cloudkitty.conf.d/01-cloudkitty-custom.conf", + "owner": "cloudkitty", + "perm": "0600", + "optional": true + }, + { + "source": "/var/lib/openstack/config/wsgi-cloudkitty.conf", + "dest": "/etc/httpd/conf.d/00wsgi-cloudkitty.conf", + "owner": "cloudkitty", + "perm": "0644" + }, + { + "source": "/var/lib/openstack/config/httpd.conf", + "dest": "/etc/httpd/conf/httpd.conf", + "owner": "cloudkitty", + "perm": "0644" + }, + { + "source": "/var/lib/openstack/config/ssl.conf", + "dest": "/etc/httpd/conf.d/ssl.conf", + "owner": "cloudkitty", + "perm": "0644" + }, + { + "source": "/var/lib/config-data/tls/certs/*", + "dest": "/etc/pki/tls/certs/", + "owner": "cloudkitty", + "perm": "0440", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/config-data/tls/private/*", + "dest": "/etc/pki/tls/private/", + "owner": "cloudkitty", + "perm": "0400", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/openstack/config/my.cnf", + "dest": "/etc/my.cnf", + "owner": "cloudkitty", + "perm": "0644" + }, + { + "source": "/var/lib/config-data/mtls/certs/*", + "dest": "/etc/pki/tls/certs/", + "owner": "cloudkitty:cloudkitty", + "perm": "0640", + "optional": true, + "merge": true + }, + { + "source": "/var/lib/config-data/mtls/private/*", + "dest": "/etc/pki/tls/private/", + "owner": "cloudkitty:cloudkitty", + "perm": "0640", + "optional": true, + "merge": true } ] } diff --git a/templates/cloudkitty/config/cloudkitty-storageinit-config.json b/templates/cloudkitty/config/cloudkitty-storageinit-config.json new file mode 100644 index 00000000..3d9561c5 --- /dev/null +++ b/templates/cloudkitty/config/cloudkitty-storageinit-config.json @@ -0,0 +1,11 @@ +{ + "command": "/usr/bin/cloudkitty-storage-init", + "config_files": [ + { + "source": "/var/lib/openstack/config/cloudkitty.conf", + "dest": "/etc/cloudkitty/cloudkitty.conf", + "owner": "cloudkitty", + "perm": "0600" + } + ] +} diff --git a/templates/cloudkitty/config/cloudkitty.conf b/templates/cloudkitty/config/cloudkitty.conf index e9896eb2..58531272 100644 --- a/templates/cloudkitty/config/cloudkitty.conf +++ b/templates/cloudkitty/config/cloudkitty.conf @@ -31,11 +31,11 @@ backend = keystone auth_section = authinfos [storage_influxdb] -port = 8086 -host = localhost -database = cloudkitty -password = cloudkitty -user = cloudkitty +version = 2 +token = cloudkitty +org = openstack +bucket = cloudkitty +url = http://influxdb:8086 [collect] period = 300 diff --git a/templates/cloudkitty/config/httpd.conf b/templates/cloudkitty/config/httpd.conf index b6c862a5..c742ea69 100644 --- a/templates/cloudkitty/config/httpd.conf +++ b/templates/cloudkitty/config/httpd.conf @@ -24,4 +24,4 @@ ErrorLog /dev/stdout # XXX: To disable SSL #Include conf.d/*.conf # If above include is commented include at least the cloudkitty wsgi file -Include conf.d/10-cloudkitty_wsgi.conf +Include conf.d/00wsgi-cloudkitty.conf diff --git a/templates/cloudkitty/config/10-cloudkitty_wsgi.conf b/templates/cloudkitty/config/wsgi-cloudkitty.conf similarity index 94% rename from templates/cloudkitty/config/10-cloudkitty_wsgi.conf rename to templates/cloudkitty/config/wsgi-cloudkitty.conf index 1fc084ae..b7407f3d 100644 --- a/templates/cloudkitty/config/10-cloudkitty_wsgi.conf +++ b/templates/cloudkitty/config/wsgi-cloudkitty.conf @@ -34,7 +34,7 @@ WSGIApplicationGroup %{GLOBAL} WSGIDaemonProcess {{ $endpt }} display-name={{ $endpt }} group=cloudkitty processes=4 threads=1 user=cloudkitty WSGIProcessGroup {{ $endpt }} - WSGIScriptAlias / "/var/www/cgi-bin/cloudkitty/cloudkitty-wsgi" + WSGIScriptAlias / "/var/www/cgi-bin/cloudkitty/cloudkitty-api" WSGIPassAuthorization On {{ end }} From aa300d93a602044063f4adc7fdf647112eb3c513 Mon Sep 17 00:00:00 2001 From: mgirgisf Date: Thu, 17 Jul 2025 13:13:41 +0200 Subject: [PATCH 15/87] update prometheus_collector --- api/v1beta1/cloudkitty_types.go | 24 +++++++++++ controllers/cloudkitty_controller.go | 46 +++++++++++++++++++++ pkg/cloudkitty/const.go | 3 ++ templates/cloudkitty/config/cloudkitty.conf | 8 +++- 4 files changed, 80 insertions(+), 1 deletion(-) diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index 1d2d1a7e..9559ffbd 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -90,6 +90,21 @@ type CloudKittySpecBase struct { // TopologyRef to apply the Topology defined by the associated CR referenced // by name TopologyRef *topologyv1.TopoRef `json:"topologyRef,omitempty"` + + // Host of user deployed prometheus + // +kubebuilder:validation:Optional + PrometheusHost string `json:"prometheusHost,omitempty"` + + // Port of user deployed prometheus + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + // +kubebuilder:validation:Optional + PrometheusPort int32 `json:"prometheusPort,omitempty"` + + // If defined, specifies which CA certificate to use for user deployed prometheus + // +kubebuilder:validation:Optional + // +nullable + PrometheusTLSCaCertSecret *corev1.SecretKeySelector `json:"prometheusTLSCaCertSecret,omitempty"` } // CloudKittySpecCore the same as CloudKittySpec without ContainerImage references @@ -211,6 +226,15 @@ type CloudKittyStatus struct { // controller has not started processing the latest changes, and the status // and its conditions are likely stale. ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // PrometheusHost - Hostname for prometheus used for autoscaling + PrometheusHost string `json:"prometheusHostname,omitempty"` + + // PrometheusPort - Port for prometheus used for autoscaling + PrometheusPort int32 `json:"prometheusPort,omitempty"` + + // PrometheusTLS - Determines if TLS should be used for accessing prometheus + PrometheusTLS bool `json:"prometheusTLS,omitempty"` } //+kubebuilder:object:root=true diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index 00cc5996..e032e60a 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -19,7 +19,10 @@ package controllers import ( "context" "fmt" + "strconv" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/metricstorage" k8s_errors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -813,6 +816,7 @@ func (r *CloudKittyReconciler) generateServiceConfigs( memcached *memcachedv1.Memcached, db *mariadbv1.Database, ) error { + Log := r.GetLogger(ctx) // // create Secret required for cloudkitty input // - %-scripts holds scripts to e.g. bootstrap the service @@ -855,6 +859,46 @@ func (r *CloudKittyReconciler) generateServiceConfigs( return err } + if instance.Spec.PrometheusHost == "" { + // We're using MetricStorage for Prometheus. + prometheusEndpointSecret := &corev1.Secret{} + err = r.Client.Get(ctx, client.ObjectKey{ + Name: cloudkitty.PrometheusEndpointSecret, + Namespace: instance.Namespace, + }, prometheusEndpointSecret) + if err != nil { + Log.Info("Prometheus Endpoint Secret not found") + } + if prometheusEndpointSecret.Data != nil { + instance.Status.PrometheusHost = string(prometheusEndpointSecret.Data[metricstorage.PrometheusHost]) + port, err := strconv.Atoi(string(prometheusEndpointSecret.Data[metricstorage.PrometheusPort])) + if err != nil { + return err + } + instance.Status.PrometheusPort = int32(port) + + metricStorage := &telemetryv1.MetricStorage{} + err = r.Client.Get(ctx, client.ObjectKey{ + Namespace: instance.Namespace, + Name: telemetryv1.DefaultServiceName, + }, metricStorage) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + } + instance.Status.PrometheusTLS = metricStorage.Spec.PrometheusTLS.Enabled() + } + } else { + // We're using user-deployed Prometheus. + instance.Status.PrometheusHost = instance.Spec.PrometheusHost + instance.Status.PrometheusPort = instance.Spec.PrometheusPort + instance.Status.PrometheusTLS = instance.Spec.PrometheusTLSCaCertSecret != nil + } + databaseAccount := db.GetAccount() dbSecret := db.GetSecret() @@ -864,6 +908,8 @@ func (r *CloudKittyReconciler) generateServiceConfigs( templateParameters["KeystoneInternalURL"] = keystoneInternalURL templateParameters["KeystonePublicURL"] = keystonePublicURL templateParameters["TransportURL"] = string(transportURLSecret.Data["transport_url"]) + templateParameters["PrometheusHost"] = instance.Status.PrometheusHost + templateParameters["PrometheusPort"] = instance.Status.PrometheusPort templateParameters["DatabaseConnection"] = fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf", databaseAccount.Spec.UserName, string(dbSecret.Data[mariadbv1.DatabasePasswordSelector]), diff --git a/pkg/cloudkitty/const.go b/pkg/cloudkitty/const.go index 5fb5269a..8b372471 100644 --- a/pkg/cloudkitty/const.go +++ b/pkg/cloudkitty/const.go @@ -49,6 +49,9 @@ const ( ShortDuration = time.Duration(5) * time.Second NormalDuration = time.Duration(10) * time.Second + + // PrometheusEndpointSecret - The name of the secret that contains the Prometheus endpoint configuration. + PrometheusEndpointSecret = "metric-storage-prometheus-endpoint" ) var ResultRequeue = ctrl.Result{RequeueAfter: NormalDuration} diff --git a/templates/cloudkitty/config/cloudkitty.conf b/templates/cloudkitty/config/cloudkitty.conf index 58531272..1aa611bf 100644 --- a/templates/cloudkitty/config/cloudkitty.conf +++ b/templates/cloudkitty/config/cloudkitty.conf @@ -45,8 +45,14 @@ collector = prometheus scope_key = project [collector_prometheus] -prometheus_url = https://metric-storage-prometheus.openstack.svc:9090 +{{- if .TLS }} +prometheus_url = https://{{ .PrometheusHost }}:{{ .PrometheusPort }} +cafile = {{ .CAFile }} +insecure = false +{{- else }} +prometheus_url = http://{{ .PrometheusHost }}:{{ .PrometheusPort }} insecure = true +{{- end }} [output] pipeline = osrf From ed8ab7faf2d38dce51b05627de6d0aaf1f275816 Mon Sep 17 00:00:00 2001 From: jlarriba Date: Thu, 24 Jul 2025 09:36:11 +0200 Subject: [PATCH 16/87] Make it compatible with previous oscp --- .../telemetry.openstack.org_cloudkitties.yaml | 44 ++++++++++++++++++- .../telemetry.openstack.org_telemetries.yaml | 36 ++++++++++++++- api/v1beta1/cloudkitty_types.go | 2 +- api/v1beta1/telemetry_types.go | 2 +- api/v1beta1/zz_generated.deepcopy.go | 5 +++ .../telemetry.openstack.org_cloudkitties.yaml | 44 ++++++++++++++++++- .../telemetry.openstack.org_telemetries.yaml | 36 ++++++++++++++- controllers/cloudkitty_controller.go | 2 +- .../config/cloudkitty-api-config.json | 22 ++++++---- templates/cloudkitty/config/cloudkitty.conf | 5 ++- 10 files changed, 180 insertions(+), 18 deletions(-) diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml index 387bf62d..07bd45f4 100644 --- a/api/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -42,7 +42,6 @@ spec: apiTimeout: default: 60 description: APITimeout for HAProxy, Apache, and rpc_response_timeout - minimum: 10 type: integer cloudKittyAPI: description: CloudKittyAPI - Spec definition for the API service of @@ -494,6 +493,7 @@ spec: DB, defaults to cloudkitty type: string databaseInstance: + default: openstack description: |- MariaDB instance name Right now required by the maridb-operator to get the credentials from the instance to create the DB @@ -538,6 +538,37 @@ spec: description: PreserveJobs - do not delete jobs after they finished e.g. to check logs type: boolean + prometheusHost: + description: Host of user deployed prometheus + type: string + prometheusPort: + description: Port of user deployed prometheus + format: int32 + maximum: 65535 + minimum: 1 + type: integer + prometheusTLSCaCertSecret: + description: If defined, specifies which CA certificate to use for + user deployed prometheus + nullable: true + properties: + key: + description: The key of the secret to select from. Must be a + valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic rabbitMqClusterName: default: rabbitmq description: |- @@ -657,6 +688,17 @@ spec: and its conditions are likely stale. format: int64 type: integer + prometheusHostname: + description: PrometheusHost - Hostname for prometheus used for autoscaling + type: string + prometheusPort: + description: PrometheusPort - Port for prometheus used for autoscaling + format: int32 + type: integer + prometheusTLS: + description: PrometheusTLS - Determines if TLS should be used for + accessing prometheus + type: boolean serviceIDs: additionalProperties: type: string diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index f69ad6d9..9798050d 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -600,7 +600,6 @@ spec: apiTimeout: default: 60 description: APITimeout for HAProxy, Apache, and rpc_response_timeout - minimum: 10 type: integer cloudKittyAPI: description: CloudKittyAPI - Spec definition for the API service @@ -1053,13 +1052,14 @@ spec: cloudkitty DB, defaults to cloudkitty type: string databaseInstance: + default: openstack description: |- MariaDB instance name Right now required by the maridb-operator to get the credentials from the instance to create the DB Might not be required in future type: string enabled: - default: true + default: false description: Enabled - Whether OpenStack CloudKitty service should be deployed and managed type: boolean @@ -1102,6 +1102,38 @@ spec: description: PreserveJobs - do not delete jobs after they finished e.g. to check logs type: boolean + prometheusHost: + description: Host of user deployed prometheus + type: string + prometheusPort: + description: Port of user deployed prometheus + format: int32 + maximum: 65535 + minimum: 1 + type: integer + prometheusTLSCaCertSecret: + description: If defined, specifies which CA certificate to use + for user deployed prometheus + nullable: true + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic rabbitMqClusterName: default: rabbitmq description: |- diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index 9559ffbd..e4e29684 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -47,6 +47,7 @@ type CloudKittySpecBase struct { CloudKittyTemplate `json:",inline"` // +kubebuilder:validation:Required + // +kubebuilder:default=openstack // MariaDB instance name // Right now required by the maridb-operator to get the credentials from the instance to create the DB // Might not be required in future @@ -82,7 +83,6 @@ type CloudKittySpecBase struct { // +kubebuilder:validation:Optional // +kubebuilder:default=60 - // +kubebuilder:validation:Minimum=10 // APITimeout for HAProxy, Apache, and rpc_response_timeout APITimeout int `json:"apiTimeout"` diff --git a/api/v1beta1/telemetry_types.go b/api/v1beta1/telemetry_types.go index 07621b5e..46373586 100644 --- a/api/v1beta1/telemetry_types.go +++ b/api/v1beta1/telemetry_types.go @@ -187,7 +187,7 @@ type LoggingSection struct { // CloudKittySpec defines the desired state of the cloudkitty service type CloudKittySection struct { // +kubebuilder:validation:Optional - // +kubebuilder:default=true + // +kubebuilder:default=false // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:booleanSwitch"} // Enabled - Whether OpenStack CloudKitty service should be deployed and managed Enabled *bool `json:"enabled"` diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index c9d622b6..f9a4c470 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1107,6 +1107,11 @@ func (in *CloudKittySpecBase) DeepCopyInto(out *CloudKittySpecBase) { *out = new(topologyv1beta1.TopoRef) **out = **in } + if in.PrometheusTLSCaCertSecret != nil { + in, out := &in.PrometheusTLSCaCertSecret, &out.PrometheusTLSCaCertSecret + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittySpecBase. diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml index 387bf62d..07bd45f4 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -42,7 +42,6 @@ spec: apiTimeout: default: 60 description: APITimeout for HAProxy, Apache, and rpc_response_timeout - minimum: 10 type: integer cloudKittyAPI: description: CloudKittyAPI - Spec definition for the API service of @@ -494,6 +493,7 @@ spec: DB, defaults to cloudkitty type: string databaseInstance: + default: openstack description: |- MariaDB instance name Right now required by the maridb-operator to get the credentials from the instance to create the DB @@ -538,6 +538,37 @@ spec: description: PreserveJobs - do not delete jobs after they finished e.g. to check logs type: boolean + prometheusHost: + description: Host of user deployed prometheus + type: string + prometheusPort: + description: Port of user deployed prometheus + format: int32 + maximum: 65535 + minimum: 1 + type: integer + prometheusTLSCaCertSecret: + description: If defined, specifies which CA certificate to use for + user deployed prometheus + nullable: true + properties: + key: + description: The key of the secret to select from. Must be a + valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic rabbitMqClusterName: default: rabbitmq description: |- @@ -657,6 +688,17 @@ spec: and its conditions are likely stale. format: int64 type: integer + prometheusHostname: + description: PrometheusHost - Hostname for prometheus used for autoscaling + type: string + prometheusPort: + description: PrometheusPort - Port for prometheus used for autoscaling + format: int32 + type: integer + prometheusTLS: + description: PrometheusTLS - Determines if TLS should be used for + accessing prometheus + type: boolean serviceIDs: additionalProperties: type: string diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index f69ad6d9..9798050d 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -600,7 +600,6 @@ spec: apiTimeout: default: 60 description: APITimeout for HAProxy, Apache, and rpc_response_timeout - minimum: 10 type: integer cloudKittyAPI: description: CloudKittyAPI - Spec definition for the API service @@ -1053,13 +1052,14 @@ spec: cloudkitty DB, defaults to cloudkitty type: string databaseInstance: + default: openstack description: |- MariaDB instance name Right now required by the maridb-operator to get the credentials from the instance to create the DB Might not be required in future type: string enabled: - default: true + default: false description: Enabled - Whether OpenStack CloudKitty service should be deployed and managed type: boolean @@ -1102,6 +1102,38 @@ spec: description: PreserveJobs - do not delete jobs after they finished e.g. to check logs type: boolean + prometheusHost: + description: Host of user deployed prometheus + type: string + prometheusPort: + description: Port of user deployed prometheus + format: int32 + maximum: 65535 + minimum: 1 + type: integer + prometheusTLSCaCertSecret: + description: If defined, specifies which CA certificate to use + for user deployed prometheus + nullable: true + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid? + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic rabbitMqClusterName: default: rabbitmq description: |- diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index e032e60a..5a98a71d 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -993,7 +993,7 @@ func (r *CloudKittyReconciler) transportURLCreateOrUpdate( ) (*rabbitmqv1.TransportURL, controllerutil.OperationResult, error) { transportURL := &rabbitmqv1.TransportURL{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-cloudkitty-transport", instance.Name), + Name: fmt.Sprintf("%s-transport", instance.Name), Namespace: instance.Namespace, Labels: serviceLabels, }, diff --git a/templates/cloudkitty/config/cloudkitty-api-config.json b/templates/cloudkitty/config/cloudkitty-api-config.json index 380bb3a0..107cfa1a 100644 --- a/templates/cloudkitty/config/cloudkitty-api-config.json +++ b/templates/cloudkitty/config/cloudkitty-api-config.json @@ -36,7 +36,7 @@ "source": "/var/lib/config-data/tls/certs/*", "dest": "/etc/pki/tls/certs/", "owner": "cloudkitty", - "perm": "0440", + "perm": "0640", "optional": true, "merge": true }, @@ -44,16 +44,10 @@ "source": "/var/lib/config-data/tls/private/*", "dest": "/etc/pki/tls/private/", "owner": "cloudkitty", - "perm": "0400", + "perm": "0600", "optional": true, "merge": true }, - { - "source": "/var/lib/openstack/config/my.cnf", - "dest": "/etc/my.cnf", - "owner": "cloudkitty", - "perm": "0644" - }, { "source": "/var/lib/config-data/mtls/certs/*", "dest": "/etc/pki/tls/certs/", @@ -70,5 +64,17 @@ "optional": true, "merge": true } + ], + "permissions": [ + { + "path": "/var/log/cinder", + "owner": "cloudkitty:apache", + "recurse": true + }, + { + "path": "/etc/httpd/run", + "owner": "cloudkitty:apache", + "recurse": true + } ] } diff --git a/templates/cloudkitty/config/cloudkitty.conf b/templates/cloudkitty/config/cloudkitty.conf index 1aa611bf..318ed398 100644 --- a/templates/cloudkitty/config/cloudkitty.conf +++ b/templates/cloudkitty/config/cloudkitty.conf @@ -61,7 +61,10 @@ backend = cloudkitty.backend.file.FileBackend [storage] version = 2 -backend = influxdb +backend = loki + +[storage_loki] +url = http://loki:3100/loki/api/v1 [database] connection = {{ .DatabaseConnection }} From 6c6efffdaf8614cd6e2a501fe24660cd07e7637e Mon Sep 17 00:00:00 2001 From: jlarriba Date: Tue, 26 Aug 2025 10:11:25 +0200 Subject: [PATCH 17/87] From mgirgis: Add Cloudkitty healthcheck.py --- pkg/cloudkittyproc/statefulset.go | 30 +++---- templates/cloudkitty/bin/healthcheck.py | 106 +++++++++++++++++++----- 2 files changed, 98 insertions(+), 38 deletions(-) diff --git a/pkg/cloudkittyproc/statefulset.go b/pkg/cloudkittyproc/statefulset.go index c5959e00..0c3f8b70 100644 --- a/pkg/cloudkittyproc/statefulset.go +++ b/pkg/cloudkittyproc/statefulset.go @@ -24,6 +24,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) const ( @@ -44,7 +45,7 @@ func StatefulSet( // cloudKittyGroup := int64(telemetryv1.CloudKittyGroupID) // TODO until we determine how to properly query for these - /*livenessProbe := &corev1.Probe{ + livenessProbe := &corev1.Probe{ // TODO might need tuning TimeoutSeconds: 5, PeriodSeconds: 3, @@ -56,19 +57,18 @@ func StatefulSet( FailureThreshold: 12, PeriodSeconds: 5, InitialDelaySeconds: 5, - }*/ + } args := []string{"-c", ServiceCommand} - /*var probeCommand []string + var probeCommand []string livenessProbe.HTTPGet = &corev1.HTTPGetAction{ Port: intstr.FromInt(8080), } startupProbe.HTTPGet = livenessProbe.HTTPGet probeCommand = []string{ - "/usr/local/bin/container-scripts/healthcheck.py", - "processor", - "/etc/cloudkitty/cloudkitty.conf.d", - }*/ + "/var/lib/openstack/bin/healthcheck.py", + "/etc/cloudkitty/cloudkitty.conf.d/cloudkitty.conf", + } envVars := map[string]env.Setter{} envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") @@ -112,13 +112,13 @@ func StatefulSet( SecurityContext: &corev1.SecurityContext{ RunAsUser: &cloudKittyUser, }, - Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: volumeMounts, - Resources: instance.Spec.Resources, - //LivenessProbe: livenessProbe, - //StartupProbe: startupProbe, + Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), + VolumeMounts: volumeMounts, + Resources: instance.Spec.Resources, + LivenessProbe: livenessProbe, + StartupProbe: startupProbe, }, - /*{ + { Name: "probe", Command: probeCommand, Image: instance.Spec.ContainerImage, @@ -127,7 +127,7 @@ func StatefulSet( //RunAsGroup: &cloudKittyGroup, }, VolumeMounts: volumeMounts, - },*/ + }, }, Volumes: volumes, }, @@ -149,4 +149,4 @@ func StatefulSet( } return statefulset -} +} \ No newline at end of file diff --git a/templates/cloudkitty/bin/healthcheck.py b/templates/cloudkitty/bin/healthcheck.py index 4bce1182..59639ccd 100755 --- a/templates/cloudkitty/bin/healthcheck.py +++ b/templates/cloudkitty/bin/healthcheck.py @@ -14,25 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -# Trivial HTTP server to check health of scheduler, backup and volume services. -# Cinder-API hast its own health check endpoint and does not need this. -# -# The only check this server currently does is using the heartbeat in the -# database service table, accessing the DB directly here using cinder's -# configuration options. -# -# The benefit of accessing the DB directly is that it doesn't depend on the -# Cinder-API service being up and we can also differentiate between the -# container not having a connection to the DB and the cinder service not doing -# the heartbeats. -# -# For volume services all enabled backends must be up to return 200, so it is -# recommended to use a different pod for each backend to avoid one backend -# affecting others. -# -# Requires the name of the service as the first argument (volume, backup, -# scheduler) and optionally a second argument with the location of the -# configuration directory (defaults to /etc/cinder/cinder.conf.d) from http import server import signal @@ -40,17 +21,67 @@ import sys import time import threading +import requests from oslo_config import cfg + SERVER_PORT = 8080 CONF = cfg.CONF + class HTTPServerV6(server.HTTPServer): - address_family = socket.AF_INET6 + address_family = socket.AF_INET6 + class HeartbeatServer(server.BaseHTTPRequestHandler): + + @staticmethod + def check_services(): + print("Starting health checks") + results = {} + + # Todo Database Endpoint Reachability + # Keystone Endpoint Reachability + try: + keystone_uri = CONF.keystone_authtoken.auth_url + response = requests.get(keystone_uri, timeout=5) + response.raise_for_status() + server_header = response.headers.get('Server', '').lower() + if 'keystone' in server_header: + results['keystone_endpoint'] = 'OK' + print("Keystone endpoint reachable and responsive.") + else: + results['keystone_endpoint'] = 'WARN' + print(f"Keystone endpoint reachable, but not a valid Keystone service: {keystone_uri}") + except requests.exceptions.RequestException as e: + results['keystone_endpoint'] = 'FAIL' + print(f"ERROR: Keystone endpoint check failed: {e}") + raise Exception('ERROR: Keystone check failed', e) + + # Prometheus Collector Endpoint Reachability + try: + prometheus_url = CONF.collector_prometheus.prometheus_url + insecure = CONF.collector_prometheus.insecure + cafile = CONF.collector_prometheus.cafile + verify_ssl = cafile if cafile and not insecure else not insecure + + response = requests.get(prometheus_url, timeout=5, verify=verify_ssl) + response.raise_for_status() + results['collector_endpoint'] = 'OK' + print("Prometheus collector endpoint reachable.") + except requests.exceptions.RequestException as e: + results['collector_endpoint'] = 'FAIL' + print(f"ERROR: Prometheus collector check failed: {e}") + raise Exception('ERROR: Prometheus collector check failed', e) + def do_GET(self): + try: + self.check_services() + except Exception as exc: + self.send_error(500, exc.args[0], exc.args[1]) + return + self.send_response(200) self.send_header("Content-type", "text/html") self.end_headers() @@ -68,12 +99,41 @@ def stopper(signal_number=None, frame=None): if __name__ == "__main__": + # Register config options + cfg.CONF.register_group(cfg.OptGroup(name='database', title='Database connection options')) + cfg.CONF.register_opt(cfg.StrOpt('connection', default=None), group='database') + + cfg.CONF.register_group(cfg.OptGroup(name='keystone_authtoken', title='Keystone Auth Token Options')) + cfg.CONF.register_opt(cfg.StrOpt('auth_url', + default='https://keystone-internal.openstack.svc:5000'), + group='keystone_authtoken') + + cfg.CONF.register_group(cfg.OptGroup(name='collector_prometheus', title='Prometheus Collector Options')) + cfg.CONF.register_opt(cfg.StrOpt('prometheus_url', + default='http://metric-storage-prometheus.openstack.svc:9090'), + group='collector_prometheus') + cfg.CONF.register_opt(cfg.BoolOpt('insecure', default=False), group='collector_prometheus') + cfg.CONF.register_opt(cfg.StrOpt('cafile', default=None), group='collector_prometheus') + + # Load configuration from file + try: + cfg.CONF(sys.argv[1:], default_config_files=['/etc/cloudkitty/cloudkitty.conf.d/cloudkitty.conf']) + except cfg.ConfigFilesNotFoundError as e: + print(f"Health check failed: {e}", file=sys.stderr) + sys.exit(1) + + # Detect IPv6 support for binding hostname = socket.gethostname() - ipv6_address = socket.getaddrinfo(hostname, None, socket.AF_INET6) + try: + ipv6_address = socket.getaddrinfo(hostname, None, socket.AF_INET6) + except socket.gaierror: + ipv6_address = None + if ipv6_address: - webServer = HTTPServerV6(("::",SERVER_PORT), HeartbeatServer) + webServer = HTTPServerV6(("::", SERVER_PORT), HeartbeatServer) else: webServer = server.HTTPServer(("0.0.0.0", SERVER_PORT), HeartbeatServer) + stop = get_stopper(webServer) # Need to run the server on a different thread because its shutdown method @@ -91,4 +151,4 @@ def stopper(signal_number=None, frame=None): except KeyboardInterrupt: pass finally: - stop() + stop() \ No newline at end of file From 2432fe249042e5f8c35efe135cf86230e68ffe13 Mon Sep 17 00:00:00 2001 From: jlarriba Date: Tue, 26 Aug 2025 10:39:10 +0200 Subject: [PATCH 18/87] Remove /v2 from path in cloudkitty, as the CLI automatically adds it --- controllers/cloudkittyapi_controller.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/controllers/cloudkittyapi_controller.go b/controllers/cloudkittyapi_controller.go index 082f6104..835b1395 100644 --- a/controllers/cloudkittyapi_controller.go +++ b/controllers/cloudkittyapi_controller.go @@ -460,21 +460,20 @@ func (r *CloudKittyAPIReconciler) reconcileInit( // expose the service (create service and return the created endpoint URLs) // - // V2 publicEndpointData := endpoint.Data{ Port: cloudkitty.CloudKittyPublicPort, - Path: "/v2", + Path: "", } internalEndpointData := endpoint.Data{ Port: cloudkitty.CloudKittyInternalPort, - Path: "/v2", + Path: "", } cloudkittyEndpoints := map[service.Endpoint]endpoint.Data{ service.EndpointPublic: publicEndpointData, service.EndpointInternal: internalEndpointData, } - apiEndpointsV3 := make(map[string]string) + apiEndpoints := make(map[string]string) for endpointType, data := range cloudkittyEndpoints { endpointTypeStr := string(endpointType) @@ -564,7 +563,7 @@ func (r *CloudKittyAPIReconciler) reconcileInit( data.Protocol = ptr.To(service.ProtocolHTTPS) } - apiEndpointsV3[string(endpointType)], err = svc.GetAPIEndpoint( + apiEndpoints[string(endpointType)], err = svc.GetAPIEndpoint( svcOverride.EndpointURL, data.Protocol, data.Path) if err != nil { instance.Status.Conditions.MarkFalse( From f066c86d1b2b852eb1fe25b0f4910eb2d662e2be Mon Sep 17 00:00:00 2001 From: jlarriba Date: Tue, 26 Aug 2025 11:08:12 +0200 Subject: [PATCH 19/87] Fix pre-commit --- .../telemetry.openstack.org_cloudkitties.yaml | 15 ++++----------- ...telemetry.openstack.org_cloudkittyapis.yaml | 8 ++------ ...elemetry.openstack.org_cloudkittyprocs.yaml | 8 ++------ .../telemetry.openstack.org_telemetries.yaml | 15 ++++----------- api/v1beta1/cloudkitty_types.go | 18 ++++++++++-------- api/v1beta1/cloudkittyapi_types.go | 8 ++++---- api/v1beta1/cloudkittyproc_types.go | 8 ++++---- .../telemetry.openstack.org_cloudkitties.yaml | 15 ++++----------- ...telemetry.openstack.org_cloudkittyapis.yaml | 8 ++------ ...elemetry.openstack.org_cloudkittyprocs.yaml | 8 ++------ .../telemetry.openstack.org_telemetries.yaml | 15 ++++----------- controllers/cloudkitty_controller.go | 12 ++++++------ controllers/cloudkittyapi_controller.go | 6 +++--- pkg/cloudkitty/dbsync.go | 5 ++++- pkg/cloudkitty/storageinit.go | 5 ++++- pkg/cloudkittyproc/statefulset.go | 2 +- templates/cloudkitty/bin/healthcheck.py | 2 +- .../cloudkitty/config/cloudkitty-api-uwsgi.ini | 17 ----------------- templates/cloudkitty/config/metrics.yaml | 2 +- 19 files changed, 62 insertions(+), 115 deletions(-) delete mode 100644 templates/cloudkitty/config/cloudkitty-api-uwsgi.ini diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml index 07bd45f4..6b58ee4d 100644 --- a/api/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -65,12 +65,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -348,8 +350,6 @@ spec: current project type: string type: object - required: - - containerImage type: object cloudKittyProc: description: CloudKittyProc - Spec definition for the Scheduler service @@ -373,12 +373,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -478,8 +480,6 @@ spec: current project type: string type: object - required: - - containerImage type: object customServiceConfig: description: |- @@ -600,13 +600,6 @@ spec: current project type: string type: object - required: - - cloudKittyAPI - - cloudKittyProc - - databaseInstance - - memcachedInstance - - rabbitMqClusterName - - secret type: object status: description: CloudKittyStatus defines the observed state of CloudKitty diff --git a/api/bases/telemetry.openstack.org_cloudkittyapis.yaml b/api/bases/telemetry.openstack.org_cloudkittyapis.yaml index d33b713c..bdd3309d 100644 --- a/api/bases/telemetry.openstack.org_cloudkittyapis.yaml +++ b/api/bases/telemetry.openstack.org_cloudkittyapis.yaml @@ -57,6 +57,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic databaseAccount: default: cloudkitty description: DatabaseAccount - optional MariaDBAccount used for cloudkitty @@ -71,6 +72,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -384,12 +386,6 @@ spec: transportURLSecret: description: Secret containing RabbitMq transport URL type: string - required: - - containerImage - - databaseHostname - - secret - - serviceAccount - - transportURLSecret type: object status: description: CloudKittyAPIStatus defines the observed state of CloudKittyAPI diff --git a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml index 6123f53c..178ceb15 100644 --- a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml +++ b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -71,6 +71,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic databaseAccount: default: cloudkitty description: DatabaseAccount - optional MariaDBAccount used for cloudkitty @@ -85,6 +86,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -220,12 +222,6 @@ spec: transportURLSecret: description: Secret containing RabbitMq transport URL type: string - required: - - containerImage - - databaseHostname - - secret - - serviceAccount - - transportURLSecret type: object status: description: CloudKittyProcStatus defines the observed state of CloudKitty diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index 9798050d..b3120d77 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -623,12 +623,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -906,8 +908,6 @@ spec: current project type: string type: object - required: - - containerImage type: object cloudKittyProc: description: CloudKittyProc - Spec definition for the Scheduler @@ -931,12 +931,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -1037,8 +1039,6 @@ spec: current project type: string type: object - required: - - containerImage type: object customServiceConfig: description: |- @@ -1166,13 +1166,6 @@ spec: current project type: string type: object - required: - - cloudKittyAPI - - cloudKittyProc - - databaseInstance - - memcachedInstance - - rabbitMqClusterName - - secret type: object logging: description: Logging - Parameters related to the logging diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index e4e29684..5a8cc27b 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -46,20 +46,20 @@ const ( type CloudKittySpecBase struct { CloudKittyTemplate `json:",inline"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // +kubebuilder:default=openstack // MariaDB instance name // Right now required by the maridb-operator to get the credentials from the instance to create the DB // Might not be required in future DatabaseInstance string `json:"databaseInstance"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // +kubebuilder:default=rabbitmq // RabbitMQ instance name // Needed to request a transportURL that is created and used in CloudKitty RabbitMqClusterName string `json:"rabbitMqClusterName"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // +kubebuilder:default=memcached // Memcached instance name. MemcachedInstance string `json:"memcachedInstance"` @@ -111,11 +111,11 @@ type CloudKittySpecBase struct { type CloudKittySpecCore struct { CloudKittySpecBase `json:",inline"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // CloudKittyAPI - Spec definition for the API service of this CloudKitty deployment CloudKittyAPI CloudKittyAPITemplateCore `json:"cloudKittyAPI"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // CloudKittyProc - Spec definition for the Scheduler service of this CloudKitty deployment CloudKittyProc CloudKittyProcTemplateCore `json:"cloudKittyProc"` } @@ -124,11 +124,11 @@ type CloudKittySpecCore struct { type CloudKittySpec struct { CloudKittySpecBase `json:",inline"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // CloudKittyAPI - Spec definition for the API service of this CloudKitty deployment CloudKittyAPI CloudKittyAPITemplate `json:"cloudKittyAPI"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // CloudKittyProc - Spec definition for the Scheduler service of this CloudKitty deployment CloudKittyProc CloudKittyProcTemplate `json:"cloudKittyProc"` } @@ -145,7 +145,7 @@ type CloudKittyTemplate struct { // DatabaseAccount - optional MariaDBAccount used for cloudkitty DB, defaults to cloudkitty DatabaseAccount string `json:"databaseAccount"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // Secret containing OpenStack password information Secret string `json:"secret"` @@ -171,6 +171,7 @@ type CloudKittyServiceTemplate struct { CustomServiceConfig string `json:"customServiceConfig,omitempty"` // +kubebuilder:validation:Optional + // +listType=atomic // CustomServiceConfigSecrets - customize the service config using this parameter to specify Secrets // that contain sensitive service config data. The content of each Secret gets added to the // /etc//.conf.d directory as a custom config file. @@ -182,6 +183,7 @@ type CloudKittyServiceTemplate struct { Resources corev1.ResourceRequirements `json:"resources,omitempty"` // +kubebuilder:validation:Optional + // +listType=atomic // NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network NetworkAttachments []string `json:"networkAttachments,omitempty"` diff --git a/api/v1beta1/cloudkittyapi_types.go b/api/v1beta1/cloudkittyapi_types.go index 42660c7e..3a71afb5 100644 --- a/api/v1beta1/cloudkittyapi_types.go +++ b/api/v1beta1/cloudkittyapi_types.go @@ -46,7 +46,7 @@ type CloudKittyAPITemplateCore struct { // CloudKittyAPITemplate defines the input parameters for the CloudKitty API service type CloudKittyAPITemplate struct { - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // ContainerImage - CloudKitty Container Image URL (will be set to environmental default if empty) ContainerImage string `json:"containerImage"` @@ -61,15 +61,15 @@ type CloudKittyAPISpec struct { // Input parameters for the CloudKitty API service CloudKittyAPITemplate `json:",inline"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // DatabaseHostname - CloudKitty Database Hostname DatabaseHostname string `json:"databaseHostname"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // Secret containing RabbitMq transport URL TransportURLSecret string `json:"transportURLSecret"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // ServiceAccount - service account name used internally to provide CloudKitty services the default SA name ServiceAccount string `json:"serviceAccount"` } diff --git a/api/v1beta1/cloudkittyproc_types.go b/api/v1beta1/cloudkittyproc_types.go index d7b80e19..83de10d1 100644 --- a/api/v1beta1/cloudkittyproc_types.go +++ b/api/v1beta1/cloudkittyproc_types.go @@ -42,7 +42,7 @@ type CloudKittyProcTemplateCore struct { // CloudKittyProcTemplate defines the input parameters for the CloudKitty Processor service type CloudKittyProcTemplate struct { - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // ContainerImage - CloudKitty Container Image URL (will be set to environmental default if empty) ContainerImage string `json:"containerImage"` @@ -57,15 +57,15 @@ type CloudKittyProcSpec struct { // Input parameters for the CloudKitty Processor service CloudKittyProcTemplate `json:",inline"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // DatabaseHostname - CloudKitty Database Hostname DatabaseHostname string `json:"databaseHostname"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // Secret containing RabbitMq transport URL TransportURLSecret string `json:"transportURLSecret"` - // +kubebuilder:validation:Required + // +kubebuilder:validation:Optional // ServiceAccount - service account name used internally to provide CloudKitty services the default SA name ServiceAccount string `json:"serviceAccount"` } diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml index 07bd45f4..6b58ee4d 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -65,12 +65,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -348,8 +350,6 @@ spec: current project type: string type: object - required: - - containerImage type: object cloudKittyProc: description: CloudKittyProc - Spec definition for the Scheduler service @@ -373,12 +373,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -478,8 +480,6 @@ spec: current project type: string type: object - required: - - containerImage type: object customServiceConfig: description: |- @@ -600,13 +600,6 @@ spec: current project type: string type: object - required: - - cloudKittyAPI - - cloudKittyProc - - databaseInstance - - memcachedInstance - - rabbitMqClusterName - - secret type: object status: description: CloudKittyStatus defines the observed state of CloudKitty diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml index d33b713c..bdd3309d 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml @@ -57,6 +57,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic databaseAccount: default: cloudkitty description: DatabaseAccount - optional MariaDBAccount used for cloudkitty @@ -71,6 +72,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -384,12 +386,6 @@ spec: transportURLSecret: description: Secret containing RabbitMq transport URL type: string - required: - - containerImage - - databaseHostname - - secret - - serviceAccount - - transportURLSecret type: object status: description: CloudKittyAPIStatus defines the observed state of CloudKittyAPI diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml index 6123f53c..178ceb15 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -71,6 +71,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic databaseAccount: default: cloudkitty description: DatabaseAccount - optional MariaDBAccount used for cloudkitty @@ -85,6 +86,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -220,12 +222,6 @@ spec: transportURLSecret: description: Secret containing RabbitMq transport URL type: string - required: - - containerImage - - databaseHostname - - secret - - serviceAccount - - transportURLSecret type: object status: description: CloudKittyProcStatus defines the observed state of CloudKitty diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index 9798050d..b3120d77 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -623,12 +623,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -906,8 +908,6 @@ spec: current project type: string type: object - required: - - containerImage type: object cloudKittyProc: description: CloudKittyProc - Spec definition for the Scheduler @@ -931,12 +931,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic networkAttachments: description: NetworkAttachments is a list of NetworkAttachment resource names to expose the services to the given network items: type: string type: array + x-kubernetes-list-type: atomic nodeSelector: additionalProperties: type: string @@ -1037,8 +1039,6 @@ spec: current project type: string type: object - required: - - containerImage type: object customServiceConfig: description: |- @@ -1166,13 +1166,6 @@ spec: current project type: string type: object - required: - - cloudKittyAPI - - cloudKittyProc - - databaseInstance - - memcachedInstance - - rabbitMqClusterName - - secret type: object logging: description: Logging - Parameters related to the logging diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index 5a98a71d..3020961a 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -227,8 +227,8 @@ func (r *CloudKittyReconciler) Reconcile(ctx context.Context, req ctrl.Request) const ( cloudKittyPasswordSecretField = ".spec.secret" cloudKittyCaBundleSecretNameField = ".spec.tls.caBundleSecretName" - cloudKittyTlsAPIInternalField = ".spec.tls.api.internal.secretName" - cloudKittyTlsAPIPublicField = ".spec.tls.api.public.secretName" + cloudKittyTLSAPIInternalField = ".spec.tls.api.internal.secretName" + cloudKittyTLSAPIPublicField = ".spec.tls.api.public.secretName" cloudKittyTopologyField = ".spec.topologyRef.Name" ) @@ -241,8 +241,8 @@ var ( cloudKittyAPIWatchFields = []string{ cloudKittyPasswordSecretField, cloudKittyCaBundleSecretNameField, - cloudKittyTlsAPIInternalField, - cloudKittyTlsAPIPublicField, + cloudKittyTLSAPIInternalField, + cloudKittyTLSAPIPublicField, cloudKittyTopologyField, } ) @@ -423,7 +423,7 @@ func (r *CloudKittyReconciler) reconcileInit( // run CloudKitty db sync // dbSyncHash := instance.Status.Hash[telemetryv1.CKDbSyncHash] - jobDbSyncDef := cloudkitty.DbSyncJob(instance, serviceLabels) + jobDbSyncDef := cloudkitty.DbSyncJob(instance, serviceLabels, serviceAnnotations) dbSyncjob := job.NewJob( jobDbSyncDef, @@ -465,7 +465,7 @@ func (r *CloudKittyReconciler) reconcileInit( // run CloudKitty Storage Init // ckStorageInitHash := instance.Status.Hash[telemetryv1.CKStorageInitHash] - jobStorageInitDef := cloudkitty.StorageInitJob(instance, serviceLabels) + jobStorageInitDef := cloudkitty.StorageInitJob(instance, serviceLabels, serviceAnnotations) storageInitjob := job.NewJob( jobStorageInitDef, diff --git a/controllers/cloudkittyapi_controller.go b/controllers/cloudkittyapi_controller.go index 835b1395..de2cb3ff 100644 --- a/controllers/cloudkittyapi_controller.go +++ b/controllers/cloudkittyapi_controller.go @@ -301,7 +301,7 @@ func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl } // index tlsAPIInternalField - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyTlsAPIInternalField, func(rawObj client.Object) []string { + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyTLSAPIInternalField, func(rawObj client.Object) []string { // Extract the secret name from the spec, if one is provided cr := rawObj.(*telemetryv1.CloudKittyAPI) if cr.Spec.TLS.API.Internal.SecretName == nil { @@ -313,7 +313,7 @@ func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl } // index tlsAPIPublicField - if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyTlsAPIPublicField, func(rawObj client.Object) []string { + if err := mgr.GetFieldIndexer().IndexField(context.Background(), &telemetryv1.CloudKittyAPI{}, cloudKittyTLSAPIPublicField, func(rawObj client.Object) []string { // Extract the secret name from the spec, if one is provided cr := rawObj.(*telemetryv1.CloudKittyAPI) if cr.Spec.TLS.API.Public.SecretName == nil { @@ -583,7 +583,7 @@ func (r *CloudKittyAPIReconciler) reconcileInit( if instance.Status.APIEndpoints == nil { instance.Status.APIEndpoints = map[string]map[string]string{} } - instance.Status.APIEndpoints[cloudkitty.ServiceName] = apiEndpointsV3 + instance.Status.APIEndpoints[cloudkitty.ServiceName] = apiEndpoints // V2 - end // expose service - end diff --git a/pkg/cloudkitty/dbsync.go b/pkg/cloudkitty/dbsync.go index 3cd213e5..5a84523a 100644 --- a/pkg/cloudkitty/dbsync.go +++ b/pkg/cloudkitty/dbsync.go @@ -36,7 +36,7 @@ const ( ) // DbSyncJob func -func DbSyncJob(instance *telemetryv1.CloudKitty, labels map[string]string) *batchv1.Job { +func DbSyncJob(instance *telemetryv1.CloudKitty, labels map[string]string, annotations map[string]string) *batchv1.Job { args := []string{"-c"} args = append(args, dbSyncCommand) @@ -75,6 +75,9 @@ func DbSyncJob(instance *telemetryv1.CloudKitty, labels map[string]string) *batc }, Spec: batchv1.JobSpec{ Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + }, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyOnFailure, ServiceAccountName: instance.RbacResourceName(), diff --git a/pkg/cloudkitty/storageinit.go b/pkg/cloudkitty/storageinit.go index 283c938d..73b05cd8 100644 --- a/pkg/cloudkitty/storageinit.go +++ b/pkg/cloudkitty/storageinit.go @@ -36,7 +36,7 @@ const ( ) // StorageInitJob func -func StorageInitJob(instance *telemetryv1.CloudKitty, labels map[string]string) *batchv1.Job { +func StorageInitJob(instance *telemetryv1.CloudKitty, labels map[string]string, annotations map[string]string) *batchv1.Job { args := []string{"-c", storageInitCommand} // create Volume and VolumeMounts @@ -74,6 +74,9 @@ func StorageInitJob(instance *telemetryv1.CloudKitty, labels map[string]string) }, Spec: batchv1.JobSpec{ Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: annotations, + }, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyOnFailure, ServiceAccountName: instance.RbacResourceName(), diff --git a/pkg/cloudkittyproc/statefulset.go b/pkg/cloudkittyproc/statefulset.go index 0c3f8b70..a568be4d 100644 --- a/pkg/cloudkittyproc/statefulset.go +++ b/pkg/cloudkittyproc/statefulset.go @@ -149,4 +149,4 @@ func StatefulSet( } return statefulset -} \ No newline at end of file +} diff --git a/templates/cloudkitty/bin/healthcheck.py b/templates/cloudkitty/bin/healthcheck.py index 59639ccd..906f688d 100755 --- a/templates/cloudkitty/bin/healthcheck.py +++ b/templates/cloudkitty/bin/healthcheck.py @@ -151,4 +151,4 @@ def stopper(signal_number=None, frame=None): except KeyboardInterrupt: pass finally: - stop() \ No newline at end of file + stop() diff --git a/templates/cloudkitty/config/cloudkitty-api-uwsgi.ini b/templates/cloudkitty/config/cloudkitty-api-uwsgi.ini deleted file mode 100644 index 2cd8f602..00000000 --- a/templates/cloudkitty/config/cloudkitty-api-uwsgi.ini +++ /dev/null @@ -1,17 +0,0 @@ -[uwsgi] -chmod-socket = 666 -socket = /var/run/uwsgi/cloudkitty.socket -start-time = %t -lazy-apps = true -add-header = Connection: close -buffer-size = 65535 -hook-master-start = unix_signal:15 gracefully_kill_them_all -thunder-lock = true -plugins = http,python3 -enable-threads = true -worker-reload-mercy = 80 -exit-on-reload = false -die-on-term = true -master = true -processes = 2 -wsgi-file = /opt/stack/data/venv/bin/cloudkitty-api \ No newline at end of file diff --git a/templates/cloudkitty/config/metrics.yaml b/templates/cloudkitty/config/metrics.yaml index c42701ed..6bb078af 100644 --- a/templates/cloudkitty/config/metrics.yaml +++ b/templates/cloudkitty/config/metrics.yaml @@ -74,4 +74,4 @@ metrics: - state mutate: NUMBOOL extra_args: - aggregation_method: max \ No newline at end of file + aggregation_method: max From e1891b50a159cd572e6810a28ae74ee3b039418c Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 29 Aug 2025 16:18:34 +0100 Subject: [PATCH 20/87] [CloudKitty] Add osp-secret as the default for secret When the secret is unset, cloudkitty will not start, since there was no default set. The osp-secret is used as the default secret for other services. Thes default value for the cloudkitty secret is updated to match the other service defaults, since the required information (CK password) is expected to be in the same secret as the other services. --- api/bases/telemetry.openstack.org_cloudkitties.yaml | 1 + api/bases/telemetry.openstack.org_cloudkittyapis.yaml | 1 + api/bases/telemetry.openstack.org_cloudkittyprocs.yaml | 1 + api/bases/telemetry.openstack.org_telemetries.yaml | 1 + api/v1beta1/cloudkitty_types.go | 1 + config/crd/bases/telemetry.openstack.org_cloudkitties.yaml | 1 + config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml | 1 + config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml | 1 + config/crd/bases/telemetry.openstack.org_telemetries.yaml | 1 + 9 files changed, 9 insertions(+) diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml index 6b58ee4d..07ee7e0f 100644 --- a/api/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -576,6 +576,7 @@ spec: Needed to request a transportURL that is created and used in CloudKitty type: string secret: + default: osp-secret description: Secret containing OpenStack password information type: string serviceUser: diff --git a/api/bases/telemetry.openstack.org_cloudkittyapis.yaml b/api/bases/telemetry.openstack.org_cloudkittyapis.yaml index bdd3309d..2c475c7c 100644 --- a/api/bases/telemetry.openstack.org_cloudkittyapis.yaml +++ b/api/bases/telemetry.openstack.org_cloudkittyapis.yaml @@ -325,6 +325,7 @@ spec: type: object type: object secret: + default: osp-secret description: Secret containing OpenStack password information type: string serviceAccount: diff --git a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml index 178ceb15..2024e7ca 100644 --- a/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml +++ b/api/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -180,6 +180,7 @@ spec: type: object type: object secret: + default: osp-secret description: Secret containing OpenStack password information type: string serviceAccount: diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index b3120d77..ec1d2893 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -1141,6 +1141,7 @@ spec: Needed to request a transportURL that is created and used in CloudKitty type: string secret: + default: osp-secret description: Secret containing OpenStack password information type: string serviceUser: diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index 5a8cc27b..b24d4ccc 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -146,6 +146,7 @@ type CloudKittyTemplate struct { DatabaseAccount string `json:"databaseAccount"` // +kubebuilder:validation:Optional + // +kubebuilder:default=osp-secret // Secret containing OpenStack password information Secret string `json:"secret"` diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml index 6b58ee4d..07ee7e0f 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -576,6 +576,7 @@ spec: Needed to request a transportURL that is created and used in CloudKitty type: string secret: + default: osp-secret description: Secret containing OpenStack password information type: string serviceUser: diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml index bdd3309d..2c475c7c 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyapis.yaml @@ -325,6 +325,7 @@ spec: type: object type: object secret: + default: osp-secret description: Secret containing OpenStack password information type: string serviceAccount: diff --git a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml index 178ceb15..2024e7ca 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkittyprocs.yaml @@ -180,6 +180,7 @@ spec: type: object type: object secret: + default: osp-secret description: Secret containing OpenStack password information type: string serviceAccount: diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index b3120d77..ec1d2893 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -1141,6 +1141,7 @@ spec: Needed to request a transportURL that is created and used in CloudKitty type: string secret: + default: osp-secret description: Secret containing OpenStack password information type: string serviceUser: From fba52debafdd943804ee781222209e4ad7d88eb0 Mon Sep 17 00:00:00 2001 From: Jaromir Wysoglad Date: Fri, 19 Sep 2025 05:13:28 -0400 Subject: [PATCH 21/87] [cloudkitty] Add LokiStack deployment Also add mTLS certificates deployment needed for communication between Loki and CloudKitty. All is tested with kuttl-tests, some minor fixes across CloudKitty related code included (other needed fixes were discussed and will follow in the future). EnsureWatches was moved from metricstorage_controller into pkg/utils. All calls to this function throughout the whole metric storage controller needed to be modified to point to the new location, but other than that there aren't any changes to the metric storage controller. --- Makefile | 6 + .../telemetry.openstack.org_cloudkitties.yaml | 92 ++++++++ .../telemetry.openstack.org_telemetries.yaml | 92 ++++++++ api/go.mod | 11 +- api/go.sum | 22 +- api/v1beta1/cloudkitty_types.go | 9 + api/v1beta1/conditions.go | 38 +++ api/v1beta1/zz_generated.deepcopy.go | 1 + .../telemetry.openstack.org_cloudkitties.yaml | 92 ++++++++ .../telemetry.openstack.org_telemetries.yaml | 92 ++++++++ controllers/cloudkitty_controller.go | 223 +++++++++++++++++- controllers/cloudkittyapi_controller.go | 67 ++++++ controllers/cloudkittyproc_controller.go | 66 ++++++ controllers/metricstorage_controller.go | 90 +++---- go.mod | 14 +- go.sum | 28 ++- main.go | 12 +- pkg/cloudkitty/cert.go | 67 ++++++ pkg/cloudkitty/const.go | 5 + pkg/cloudkitty/dbsync.go | 2 +- pkg/cloudkitty/lokistack.go | 118 +++++++++ pkg/cloudkitty/storageinit.go | 2 +- pkg/cloudkitty/volumes.go | 29 +++ pkg/cloudkittyapi/statefulset.go | 2 +- pkg/cloudkittyapi/volumes.go | 4 +- pkg/cloudkittyproc/statefulset.go | 2 +- pkg/cloudkittyproc/volumes.go | 4 +- pkg/utils/utils.go | 59 +++++ .../config/cloudkitty-api-config.json | 7 + .../config/cloudkitty-proc-config.json | 7 + templates/cloudkitty/config/cloudkitty.conf | 5 +- tests/kuttl/suites/cloudkitty/config.yaml | 14 ++ .../deps/OpenStackControlPlane.yaml | 27 +++ tests/kuttl/suites/cloudkitty/deps/infra.yaml | 42 ++++ .../suites/cloudkitty/deps/kustomization.yaml | 50 ++++ .../suites/cloudkitty/deps/loki-operator.yaml | 27 +++ .../cloudkitty/deps/loki-s3-secret.yaml | 10 + tests/kuttl/suites/cloudkitty/deps/minio.yaml | 99 ++++++++ .../suites/cloudkitty/deps/namespace.yaml | 4 + .../suites/cloudkitty/deps/telemetry.yaml | 7 + tests/kuttl/suites/cloudkitty/output/.keep | 0 .../suites/cloudkitty/tests/00-deps.yaml | 9 + .../suites/cloudkitty/tests/01-assert.yaml | 81 +++++++ .../suites/cloudkitty/tests/01-deploy.yaml | 54 +++++ 44 files changed, 1582 insertions(+), 110 deletions(-) create mode 100644 pkg/cloudkitty/cert.go create mode 100644 pkg/cloudkitty/lokistack.go create mode 100644 tests/kuttl/suites/cloudkitty/config.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/OpenStackControlPlane.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/infra.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/kustomization.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/loki-operator.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/loki-s3-secret.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/minio.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/namespace.yaml create mode 100644 tests/kuttl/suites/cloudkitty/deps/telemetry.yaml create mode 100644 tests/kuttl/suites/cloudkitty/output/.keep create mode 100644 tests/kuttl/suites/cloudkitty/tests/00-deps.yaml create mode 100644 tests/kuttl/suites/cloudkitty/tests/01-assert.yaml create mode 100644 tests/kuttl/suites/cloudkitty/tests/01-deploy.yaml diff --git a/Makefile b/Makefile index aa0d4c18..834a4473 100644 --- a/Makefile +++ b/Makefile @@ -393,6 +393,12 @@ kuttl-test-cleanup: if [ "$(KUTTL_SUITE)" == "ceilometer" ]; then \ oc delete --wait=true --all=true -n $(KUTTL_NAMESPACE) --timeout=120s Ceilometer; \ fi; \ + if [ "$(KUTTL_SUITE)" == "metric-storage" ]; then \ + oc delete --wait=true --all=true -n $(KUTTL_NAMESPACE) --timeout=120s MetricStorage; \ + fi; \ + if [ "$(KUTTL_SUITE)" == "cloudkitty" ]; then \ + oc delete --wait=true --all=true -n $(KUTTL_NAMESPACE) --timeout=120s CloudKitty; \ + fi; \ if [ "$(KUTTL_SUITE)" == "default" ]; then \ oc delete --wait=true --all=true -n $(KUTTL_NAMESPACE) --timeout=120s Telemetry; \ fi; \ diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml index 07ee7e0f..a39f82a7 100644 --- a/api/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -575,6 +575,95 @@ spec: RabbitMQ instance name Needed to request a transportURL that is created and used in CloudKitty type: string + s3StorageConfig: + description: S3 related configuration passed to Loki + properties: + schemas: + default: + - effectiveDate: "2020-10-11" + version: v11 + description: Schemas for reading and writing logs. + items: + description: ObjectStorageSchema defines a schema version and + the date when it will become effective. + properties: + effectiveDate: + description: |- + EffectiveDate contains a date in YYYY-MM-DD format which is interpreted in the UTC time zone. + + + The configuration always needs at least one schema that is currently valid. This means that when creating a new + LokiStack it is recommended to add a schema with the latest available version and an effective date of "yesterday". + New schema versions added to the configuration always needs to be placed "in the future", so that Loki can start + using it once the day rolls over. + pattern: ^([0-9]{4,})([-]([0-9]{2})){2}$ + type: string + version: + description: Version for writing and reading logs. + enum: + - v11 + - v12 + - v13 + type: string + required: + - effectiveDate + - version + type: object + minItems: 1 + type: array + secret: + description: |- + Secret for object storage authentication. + Name of a secret in the same namespace as the LokiStack custom resource. + properties: + credentialMode: + description: |- + CredentialMode can be used to set the desired credential mode for authenticating with the object storage. + If this is not set, then the operator tries to infer the credential mode from the provided secret and its + own configuration. + enum: + - static + - token + - token-cco + type: string + name: + description: Name of a secret in the namespace configured + for object storage secrets. + type: string + type: + description: Type of object storage that should be used + enum: + - azure + - gcs + - s3 + - swift + - alibabacloud + type: string + required: + - name + - type + type: object + tls: + description: TLS configuration for reaching the object storage + endpoint. + properties: + caKey: + description: |- + Key is the data key of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + If empty, it defaults to "service-ca.crt". + type: string + caName: + description: |- + CA is the name of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + type: string + required: + - caName + type: object + required: + - secret + type: object secret: default: osp-secret description: Secret containing OpenStack password information @@ -584,6 +673,9 @@ spec: description: ServiceUser - optional username used for this service to register in cloudkitty type: string + storageClass: + description: Storage class used for Loki + type: string topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index ec1d2893..1a81843e 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -1140,6 +1140,95 @@ spec: RabbitMQ instance name Needed to request a transportURL that is created and used in CloudKitty type: string + s3StorageConfig: + description: S3 related configuration passed to Loki + properties: + schemas: + default: + - effectiveDate: "2020-10-11" + version: v11 + description: Schemas for reading and writing logs. + items: + description: ObjectStorageSchema defines a schema version + and the date when it will become effective. + properties: + effectiveDate: + description: |- + EffectiveDate contains a date in YYYY-MM-DD format which is interpreted in the UTC time zone. + + + The configuration always needs at least one schema that is currently valid. This means that when creating a new + LokiStack it is recommended to add a schema with the latest available version and an effective date of "yesterday". + New schema versions added to the configuration always needs to be placed "in the future", so that Loki can start + using it once the day rolls over. + pattern: ^([0-9]{4,})([-]([0-9]{2})){2}$ + type: string + version: + description: Version for writing and reading logs. + enum: + - v11 + - v12 + - v13 + type: string + required: + - effectiveDate + - version + type: object + minItems: 1 + type: array + secret: + description: |- + Secret for object storage authentication. + Name of a secret in the same namespace as the LokiStack custom resource. + properties: + credentialMode: + description: |- + CredentialMode can be used to set the desired credential mode for authenticating with the object storage. + If this is not set, then the operator tries to infer the credential mode from the provided secret and its + own configuration. + enum: + - static + - token + - token-cco + type: string + name: + description: Name of a secret in the namespace configured + for object storage secrets. + type: string + type: + description: Type of object storage that should be used + enum: + - azure + - gcs + - s3 + - swift + - alibabacloud + type: string + required: + - name + - type + type: object + tls: + description: TLS configuration for reaching the object storage + endpoint. + properties: + caKey: + description: |- + Key is the data key of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + If empty, it defaults to "service-ca.crt". + type: string + caName: + description: |- + CA is the name of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + type: string + required: + - caName + type: object + required: + - secret + type: object secret: default: osp-secret description: Secret containing OpenStack password information @@ -1149,6 +1238,9 @@ spec: description: ServiceUser - optional username used for this service to register in cloudkitty type: string + storageClass: + description: Storage class used for Loki + type: string topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/api/go.mod b/api/go.mod index 671c2b3a..fb2d24a3 100644 --- a/api/go.mod +++ b/api/go.mod @@ -3,6 +3,7 @@ module github.com/openstack-k8s-operators/telemetry-operator/api go 1.21 require ( + github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba github.com/onsi/ginkgo/v2 v2.20.1 github.com/onsi/gomega v1.34.1 github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20250513115636-b549982a5d8f @@ -53,11 +54,11 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/net v0.28.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.18.0 // indirect - golang.org/x/sys v0.23.0 // indirect - golang.org/x/term v0.23.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.24.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect @@ -70,7 +71,7 @@ require ( k8s.io/component-base v0.29.15 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240322212309-b815d8309940 // indirect - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect + k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/api/go.sum b/api/go.sum index 5580678d..d969c0f2 100644 --- a/api/go.sum +++ b/api/go.sum @@ -47,6 +47,8 @@ github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQu github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba h1:P5Wgp2HfGfNPLCPpS+YqquKdrrl4tW0El7VX23D6vtg= +github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba/go.mod h1:OBAgJh0mLYRvziBzBKr4/anrPHqGY9qEfuNXCpnUNi0= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -125,8 +127,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -140,18 +142,18 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -197,8 +199,8 @@ k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240322212309-b815d8309940 h1:qVoMaQV5t62UUvHe16Q3eb2c5HPzLHYzsi0Tu/xLndo= k8s.io/kube-openapi v0.0.0-20240322212309-b815d8309940/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.17.6 h1:12IXsozEsIXWAMRpgRlYS1jjAHQXHtWEOMdULh3DbEw= sigs.k8s.io/controller-runtime v0.17.6/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index b24d4ccc..0eca7214 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -22,6 +22,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/util" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + lokistackv1 "github.com/grafana/loki/operator/api/loki/v1" ) const ( @@ -105,6 +106,14 @@ type CloudKittySpecBase struct { // +kubebuilder:validation:Optional // +nullable PrometheusTLSCaCertSecret *corev1.SecretKeySelector `json:"prometheusTLSCaCertSecret,omitempty"` + + // S3 related configuration passed to Loki + // +kubebuilder:validation:Optional + S3StorageConfig lokistackv1.ObjectStorageSpec `json:"s3StorageConfig"` + + // Storage class used for Loki + // +kubebuilder:validation:Optional + StorageClass string `json:"storageClass,omitempty"` } // CloudKittySpecCore the same as CloudKittySpec without ContainerImage references diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index 709bd1cd..d2ccd731 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -57,6 +57,12 @@ const ( // CloudKittyStorageInitReadyCondition Status=True condition which indicates if the CloudKitty Storage Init process has ran CloudKittyStorageInitReadyCondition condition.Type = "CloudKittyStorageInitReady" + // CloudKittyClientCertReadyCondition Status=True condition which indicates if the CloudKitty client certificate is ready for use + CloudKittyClientCertReadyCondition condition.Type = "CloudKittyClientCertReady" + + // CloudKittyLokiStackReadyCondition Status=True condition which indicates if the CloudKitty LokiStack is ready + CloudKittyLokiStackReadyCondition condition.Type = "CloudKittyLokiStackReady" + // LoggingCLONamespaceReadyCondition Status=True condition which indicates if the cluster-logging-operator namespace is created LoggingCLONamespaceReadyCondition condition.Type = "LoggingCLONamespaceReady" @@ -271,6 +277,38 @@ const ( // CloudKittyProcReadyRunningMessage CloudKittyProcReadyRunningMessage = "CloudKittyProc in progress" + // + // CloudKittyClientCertReady condition messages + // + // CloudKittyClientCertReadyInitMessage + CloudKittyClientCertReadyInitMessage = "CloudKittyClientCert not created" + + // CloudKittyClientCertReadyMessage + CloudKittyClientCertReadyMessage = "CloudKittyClientCert ready for use" + + // CloudKittyClientCertReadyErrorMessage + CloudKittyClientCertReadyErrorMessage = "CloudKittyClientCert error occured %s" + + // CloudKittyClientCertReadyRunningMessage + CloudKittyClientCertReadyRunningMessage = "CloudKittyClientCert in progress" + + // + // CloudKittyLokiStackReady condition messages + // + // CloudKittyLokiStackReadyInitMessage + CloudKittyLokiStackReadyInitMessage = "CloudKittyLokiStack not created" + + // CloudKittyLokiStackReadyMessage + CloudKittyLokiStackReadyMessage = "CloudKittyLokiStack ready for use" + + // CloudKittyLokiStackReadyErrorMessage + CloudKittyLokiStackReadyErrorMessage = "CloudKittyLokiStack error occured %s" + + // CloudKittyLokiStackReadyRunningMessage + CloudKittyLokiStackReadyRunningMessage = "CloudKittyLokiStack in progress" + // CloudKittyLokiStackReadyRunningMessage + CloudKittyLokiStackUnableToOwnMessage = "Error occured when trying to own %s" + DashboardsNotEnabledMessage = "Dashboarding was not enabled, so no actions required" DashboardPrometheusRuleReadyInitMessage = "Dashboard PrometheusRule not started" diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index f9a4c470..e9f328c9 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1112,6 +1112,7 @@ func (in *CloudKittySpecBase) DeepCopyInto(out *CloudKittySpecBase) { *out = new(v1.SecretKeySelector) (*in).DeepCopyInto(*out) } + in.S3StorageConfig.DeepCopyInto(&out.S3StorageConfig) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittySpecBase. diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml index 07ee7e0f..a39f82a7 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -575,6 +575,95 @@ spec: RabbitMQ instance name Needed to request a transportURL that is created and used in CloudKitty type: string + s3StorageConfig: + description: S3 related configuration passed to Loki + properties: + schemas: + default: + - effectiveDate: "2020-10-11" + version: v11 + description: Schemas for reading and writing logs. + items: + description: ObjectStorageSchema defines a schema version and + the date when it will become effective. + properties: + effectiveDate: + description: |- + EffectiveDate contains a date in YYYY-MM-DD format which is interpreted in the UTC time zone. + + + The configuration always needs at least one schema that is currently valid. This means that when creating a new + LokiStack it is recommended to add a schema with the latest available version and an effective date of "yesterday". + New schema versions added to the configuration always needs to be placed "in the future", so that Loki can start + using it once the day rolls over. + pattern: ^([0-9]{4,})([-]([0-9]{2})){2}$ + type: string + version: + description: Version for writing and reading logs. + enum: + - v11 + - v12 + - v13 + type: string + required: + - effectiveDate + - version + type: object + minItems: 1 + type: array + secret: + description: |- + Secret for object storage authentication. + Name of a secret in the same namespace as the LokiStack custom resource. + properties: + credentialMode: + description: |- + CredentialMode can be used to set the desired credential mode for authenticating with the object storage. + If this is not set, then the operator tries to infer the credential mode from the provided secret and its + own configuration. + enum: + - static + - token + - token-cco + type: string + name: + description: Name of a secret in the namespace configured + for object storage secrets. + type: string + type: + description: Type of object storage that should be used + enum: + - azure + - gcs + - s3 + - swift + - alibabacloud + type: string + required: + - name + - type + type: object + tls: + description: TLS configuration for reaching the object storage + endpoint. + properties: + caKey: + description: |- + Key is the data key of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + If empty, it defaults to "service-ca.crt". + type: string + caName: + description: |- + CA is the name of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + type: string + required: + - caName + type: object + required: + - secret + type: object secret: default: osp-secret description: Secret containing OpenStack password information @@ -584,6 +673,9 @@ spec: description: ServiceUser - optional username used for this service to register in cloudkitty type: string + storageClass: + description: Storage class used for Loki + type: string topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index ec1d2893..1a81843e 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -1140,6 +1140,95 @@ spec: RabbitMQ instance name Needed to request a transportURL that is created and used in CloudKitty type: string + s3StorageConfig: + description: S3 related configuration passed to Loki + properties: + schemas: + default: + - effectiveDate: "2020-10-11" + version: v11 + description: Schemas for reading and writing logs. + items: + description: ObjectStorageSchema defines a schema version + and the date when it will become effective. + properties: + effectiveDate: + description: |- + EffectiveDate contains a date in YYYY-MM-DD format which is interpreted in the UTC time zone. + + + The configuration always needs at least one schema that is currently valid. This means that when creating a new + LokiStack it is recommended to add a schema with the latest available version and an effective date of "yesterday". + New schema versions added to the configuration always needs to be placed "in the future", so that Loki can start + using it once the day rolls over. + pattern: ^([0-9]{4,})([-]([0-9]{2})){2}$ + type: string + version: + description: Version for writing and reading logs. + enum: + - v11 + - v12 + - v13 + type: string + required: + - effectiveDate + - version + type: object + minItems: 1 + type: array + secret: + description: |- + Secret for object storage authentication. + Name of a secret in the same namespace as the LokiStack custom resource. + properties: + credentialMode: + description: |- + CredentialMode can be used to set the desired credential mode for authenticating with the object storage. + If this is not set, then the operator tries to infer the credential mode from the provided secret and its + own configuration. + enum: + - static + - token + - token-cco + type: string + name: + description: Name of a secret in the namespace configured + for object storage secrets. + type: string + type: + description: Type of object storage that should be used + enum: + - azure + - gcs + - s3 + - swift + - alibabacloud + type: string + required: + - name + - type + type: object + tls: + description: TLS configuration for reaching the object storage + endpoint. + properties: + caKey: + description: |- + Key is the data key of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + If empty, it defaults to "service-ca.crt". + type: string + caName: + description: |- + CA is the name of a ConfigMap containing a CA certificate. + It needs to be in the same namespace as the LokiStack custom resource. + type: string + required: + - caName + type: object + required: + - secret + type: object secret: default: osp-secret description: Secret containing OpenStack password information @@ -1149,6 +1238,9 @@ spec: description: ServiceUser - optional username used for this service to register in cloudkitty type: string + storageClass: + description: Storage class used for Loki + type: string topologyRef: description: |- TopologyRef to apply the Topology defined by the associated CR referenced diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index 3020961a..cb7a9ea8 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -19,10 +19,15 @@ package controllers import ( "context" "fmt" + "slices" "strconv" + "time" + + lokistackv1 "github.com/grafana/loki/operator/api/loki/v1" "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" "github.com/openstack-k8s-operators/telemetry-operator/pkg/metricstorage" + "github.com/openstack-k8s-operators/telemetry-operator/pkg/utils" k8s_errors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -36,13 +41,16 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" + certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" "github.com/go-logr/logr" networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" + "github.com/openstack-k8s-operators/lib-common/modules/certmanager" "github.com/openstack-k8s-operators/lib-common/modules/common" "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/configmap" "github.com/openstack-k8s-operators/lib-common/modules/common/endpoint" "github.com/openstack-k8s-operators/lib-common/modules/common/env" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" @@ -79,11 +87,7 @@ func (r *CloudKittyReconciler) GetScheme() *runtime.Scheme { } // CloudKittyReconciler reconciles a CloudKitty object -type CloudKittyReconciler struct { - client.Client - Kclient kubernetes.Interface - Scheme *runtime.Scheme -} +type CloudKittyReconciler utils.ConditionalWatchingReconciler // GetLogger returns a logger object with a logging prefix of "controller.name" and additional controller context fields func (r *CloudKittyReconciler) GetLogger(ctx context.Context) logr.Logger { @@ -191,6 +195,8 @@ func (r *CloudKittyReconciler) Reconcile(ctx context.Context, req ctrl.Request) condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), condition.UnknownCondition(telemetryv1.CloudKittyAPIReadyCondition, condition.InitReason, telemetryv1.CloudKittyAPIReadyInitMessage), condition.UnknownCondition(telemetryv1.CloudKittyProcReadyCondition, condition.InitReason, telemetryv1.CloudKittyProcReadyInitMessage), + condition.UnknownCondition(telemetryv1.CloudKittyClientCertReadyCondition, condition.InitReason, telemetryv1.CloudKittyClientCertReadyInitMessage), + condition.UnknownCondition(telemetryv1.CloudKittyLokiStackReadyCondition, condition.InitReason, telemetryv1.CloudKittyLokiStackReadyInitMessage), condition.UnknownCondition(condition.NetworkAttachmentsReadyCondition, condition.InitReason, condition.NetworkAttachmentsReadyInitMessage), // service account, role, rolebinding conditions condition.UnknownCondition(condition.ServiceAccountReadyCondition, condition.InitReason, condition.ServiceAccountReadyInitMessage), @@ -327,7 +333,7 @@ func (r *CloudKittyReconciler) SetupWithManager(mgr ctrl.Manager) error { return nil } - return ctrl.NewControllerManagedBy(mgr). + control, err := ctrl.NewControllerManagedBy(mgr). For(&telemetryv1.CloudKitty{}). Owns(&mariadbv1.MariaDBDatabase{}). Owns(&mariadbv1.MariaDBAccount{}). @@ -336,9 +342,11 @@ func (r *CloudKittyReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&rabbitmqv1.TransportURL{}). Owns(&batchv1.Job{}). Owns(&corev1.Secret{}). + Owns(&corev1.ConfigMap{}). Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.Role{}). Owns(&rbacv1.RoleBinding{}). + Owns(&certmgrv1.Certificate{}). // Watch for TransportURL Secrets which belong to any TransportURLs created by CloudKitty CRs Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(transportURLSecretFn)). @@ -347,7 +355,10 @@ func (r *CloudKittyReconciler) SetupWithManager(mgr ctrl.Manager) error { Watches(&keystonev1.KeystoneAPI{}, handler.EnqueueRequestsFromMapFunc(r.findObjectForSrc), builder.WithPredicates(keystonev1.KeystoneAPIStatusChangedPredicate)). - Complete(r) + // LokiStack watch added dynamically inside the controller code. + Build(r) + r.Controller = control + return err } func (r *CloudKittyReconciler) findObjectForSrc(ctx context.Context, src client.Object) []reconcile.Request { @@ -507,11 +518,203 @@ func (r *CloudKittyReconciler) reconcileInit( return ctrl.Result{}, nil } +// Original source: +// https://github.com/openstack-k8s-operators/openstack-operator/blob/cf133b39e91c05f53c57725d7c6f5a627d98dccd/pkg/openstack/ca.go#L687 +func getCAFromSecret( + ctx context.Context, + instance *telemetryv1.CloudKitty, + helper *helper.Helper, + secretName string, +) (string, ctrl.Result, error) { + caSecret, ctrlResult, err := secret.GetDataFromSecret(ctx, helper, secretName, time.Duration(5), "ca.crt") + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyLokiStackReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + telemetryv1.CloudKittyLokiStackReadyErrorMessage, + err.Error())) + + return "", ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyLokiStackReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + telemetryv1.CloudKittyLokiStackReadyRunningMessage)) + + return "", ctrlResult, nil + } + + return caSecret, ctrl.Result{}, nil +} + func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *telemetryv1.CloudKitty, helper *helper.Helper) (ctrl.Result, error) { Log := r.GetLogger(ctx) Log.Info(fmt.Sprintf("Reconciling Service '%s'", instance.Name)) + // Create cloudkitty client cert / key + certIssuer, err := certmanager.GetIssuerByLabels( + ctx, helper, instance.Namespace, + map[string]string{certmanager.RootCAIssuerInternalLabel: ""}, + ) + if err != nil { + Log.Error(err, "Failed to determine certificate issuer") + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyClientCertReadyCondition, + condition.ErrorReason, + condition.SeverityError, + telemetryv1.CloudKittyClientCertReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + certDefinition := cloudkitty.Certificate( + instance, serviceLabels, certIssuer, + ) + cert := certmanager.NewCertificate(certDefinition, 5) + ctrlResult, _, err := cert.CreateOrPatch(ctx, helper, nil) + + if err != nil { + Log.Error(err, "Failed to create or patch cloudkitty client certificate") + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyClientCertReadyCondition, + condition.ErrorReason, + condition.SeverityError, + telemetryv1.CloudKittyClientCertReadyErrorMessage, + err.Error())) + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + Log.Info("CloudKitty client certificate is being created") + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyClientCertReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + telemetryv1.CloudKittyClientCertReadyRunningMessage)) + return ctrlResult, nil + } + + caData, ctrlResult, err := getCAFromSecret( + ctx, instance, helper, certDefinition.Spec.SecretName, + ) + if err != nil { + Log.Error(err, "Failed to get cloudkitty client certificate CA data") + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyClientCertReadyCondition, + condition.ErrorReason, + condition.SeverityError, + telemetryv1.CloudKittyClientCertReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + cms := []util.Template{ + { + Name: fmt.Sprintf("%s-%s", instance.Name, cloudkitty.CaConfigmapName), + Namespace: instance.Namespace, + Type: util.TemplateTypeNone, + InstanceType: "cloudkitty", + CustomData: map[string]string{ + cloudkitty.CaConfigmapKey: caData, + }, + }, + } + + err = configmap.EnsureConfigMaps( + ctx, helper, instance, cms, nil, + ) + if err != nil { + Log.Error(err, "Failed to create CA configmap for cloudkitty client cert verification") + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyClientCertReadyCondition, + condition.ErrorReason, + condition.SeverityError, + telemetryv1.CloudKittyClientCertReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + + instance.Status.Conditions.MarkTrue(telemetryv1.CloudKittyClientCertReadyCondition, telemetryv1.CloudKittyClientCertReadyMessage) + + // Deploy Loki + var eventHandler handler.EventHandler = handler.EnqueueRequestForOwner( + r.Scheme, + r.RESTMapper, + &telemetryv1.CloudKitty{}, + handler.OnlyControllerOwner(), + ) + + err = utils.EnsureWatches( + (*utils.ConditionalWatchingReconciler)(r), ctx, + "lokistacks.loki.grafana.com", + &lokistackv1.LokiStack{}, eventHandler, helper, + ) + if err != nil { + instance.Status.Conditions.MarkFalse(telemetryv1.CloudKittyLokiStackReadyCondition, + condition.Reason("Can't own LokiStack resource. The loki-operator probably isn't installed"), + condition.SeverityError, + telemetryv1.CloudKittyLokiStackUnableToOwnMessage, err) + Log.Info("Can't own LokiStack resource. The loki-operator probably isn't installed") + return ctrl.Result{RequeueAfter: telemetryv1.PauseBetweenWatchAttempts}, nil + } + + lokiStack := &lokistackv1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-lokistack", instance.Name), + Namespace: instance.Namespace, + }, + } + op, err := controllerutil.CreateOrPatch(ctx, r.Client, lokiStack, func() error { + desiredLokiStack := cloudkitty.LokiStack(instance, serviceLabels) + desiredLokiStack.Spec.DeepCopyInto(&lokiStack.Spec) + lokiStack.ObjectMeta.Labels = serviceLabels + err = controllerutil.SetControllerReference(instance, lokiStack, r.Scheme) + return err + }) + if err != nil { + Log.Error(err, "Failed to create or patch LokiStack") + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyLokiStackReadyCondition, + condition.ErrorReason, + condition.SeverityError, + telemetryv1.CloudKittyLokiStackReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + if op != controllerutil.OperationResultNone { + Log.Info(fmt.Sprintf("LokiStack %s successfully changed - operation: %s", lokiStack.Name, string(op))) + } + + // Mirror LokiStacks's condition here. LokiStack uses conditions + // a little differently than o-k-o. Whats more, it can have + // multiple 'active' conditions, while we have only one 'master' + // condition LokiStack here. So we mirror hopefully the one most + // relevant active condition in this order of + // priority: Ready > Failed > Degraded > Pending > Warning. + + order := []string{"Ready", "Failed", "Degraded", "Pending", "Warning"} + index := len(order) + reason := condition.InitReason + message := telemetryv1.CloudKittyLokiStackReadyInitMessage + for _, c := range lokiStack.Status.Conditions { + conditionIndex := slices.Index(order, c.Type) + if c.Status == "True" && conditionIndex < index { + index = conditionIndex + reason = c.Reason + message = c.Message + } + } + if index < len(order) && order[index] == "Ready" { + instance.Status.Conditions.MarkTrue(telemetryv1.CloudKittyLokiStackReadyCondition, telemetryv1.CloudKittyLokiStackReadyMessage) + } else { + Log.Info("LokiStack not ready") + instance.Status.Conditions.Set(condition.FalseCondition( + telemetryv1.CloudKittyLokiStackReadyCondition, + condition.Reason(reason), + condition.SeverityWarning, + fmt.Sprintf("LokiStack issue: %s", message))) + } + // Service account, role, binding rbacRules := []rbacv1.PolicyRule{ { @@ -726,7 +929,7 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te } // Handle service init - ctrlResult, err := r.reconcileInit(ctx, instance, helper, serviceLabels, serviceAnnotations) + ctrlResult, err = r.reconcileInit(ctx, instance, helper, serviceLabels, serviceAnnotations) if err != nil { return ctrlResult, err } else if (ctrlResult != ctrl.Result{}) { @@ -902,6 +1105,8 @@ func (r *CloudKittyReconciler) generateServiceConfigs( databaseAccount := db.GetAccount() dbSecret := db.GetSecret() + lokiHost := fmt.Sprintf("%s-lokistack-gateway-http.%s.svc", instance.Name, instance.Namespace) + templateParameters := make(map[string]interface{}) templateParameters["ServiceUser"] = instance.Spec.ServiceUser templateParameters["ServicePassword"] = string(ospSecret.Data[instance.Spec.PasswordSelectors.CloudKittyService]) @@ -910,6 +1115,8 @@ func (r *CloudKittyReconciler) generateServiceConfigs( templateParameters["TransportURL"] = string(transportURLSecret.Data["transport_url"]) templateParameters["PrometheusHost"] = instance.Status.PrometheusHost templateParameters["PrometheusPort"] = instance.Status.PrometheusPort + templateParameters["LokiHost"] = lokiHost + templateParameters["LokiPort"] = 8080 templateParameters["DatabaseConnection"] = fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf", databaseAccount.Spec.UserName, string(dbSecret.Data[mariadbv1.DatabasePasswordSelector]), diff --git a/controllers/cloudkittyapi_controller.go b/controllers/cloudkittyapi_controller.go index de2cb3ff..a47014b1 100644 --- a/controllers/cloudkittyapi_controller.go +++ b/controllers/cloudkittyapi_controller.go @@ -270,6 +270,51 @@ func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl } } } + + // Watch for changes to the client cert secret + if secretName == cloudkitty.ClientCertSecretName { + for _, cr := range apis.Items { + name := client.ObjectKey{ + Namespace: namespace, + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Secret %s is used by CloudKittyAPI CR %s", secretName, cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + if len(result) > 0 { + return result + } + return nil + } + + // Watch for changes to configmaps we don't own. + configMapFn := func(_ context.Context, o client.Object) []reconcile.Request { + var namespace string = o.GetNamespace() + var configMapName string = o.GetName() + result := []reconcile.Request{} + + // get all API CRs + apis := &telemetryv1.CloudKittyAPIList{} + listOpts := []client.ListOption{ + client.InNamespace(namespace), + } + if err := r.Client.List(context.Background(), apis, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve API CRs %v") + return nil + } + + // Watch for changes to the ca cert config map + for _, cr := range apis.Items { + if configMapName == fmt.Sprintf("%s-lokistack-gateway-ca-bundle", cloudkitty.GetOwningCloudKittyName(&cr)) { + name := client.ObjectKey{ + Namespace: namespace, + Name: cr.Name, + } + Log.Info(fmt.Sprintf("ConfigMap %s is used by CloudKittyAPI CR %s", configMapName, cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } if len(result) > 0 { return result } @@ -350,6 +395,8 @@ func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), ). + Watches(&corev1.ConfigMap{}, + handler.EnqueueRequestsFromMapFunc(configMapFn)). Watches(&topologyv1.Topology{}, handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), builder.WithPredicates(predicate.GenerationChangedPredicate{})). @@ -889,6 +936,26 @@ func (r *CloudKittyAPIReconciler) reconcileNormal(ctx context.Context, instance // normal reconcile tasks // + // Add client cert secret hash to all the other config data, so that + // restart is triggered on certificate changes (e.g. when they + // rotate) + _, clientCertHash, err := secret.GetSecret( + ctx, + helper, + cloudkitty.ClientCertSecretName, + instance.Namespace, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + configVars["client-cert"] = env.SetValue(clientCertHash) + // // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. diff --git a/controllers/cloudkittyproc_controller.go b/controllers/cloudkittyproc_controller.go index fbccef63..495f3332 100644 --- a/controllers/cloudkittyproc_controller.go +++ b/controllers/cloudkittyproc_controller.go @@ -244,6 +244,50 @@ func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctr } } } + // Watch for changes to the client cert secret + if secretName == cloudkitty.ClientCertSecretName { + for _, cr := range schedulers.Items { + name := client.ObjectKey{ + Namespace: namespace, + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Secret %s is used by CloudKittyProc CR %s", secretName, cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + if len(result) > 0 { + return result + } + return nil + } + + // Watch for changes to configmaps we don't own. + configMapFn := func(_ context.Context, o client.Object) []reconcile.Request { + var namespace string = o.GetNamespace() + var configMapName string = o.GetName() + result := []reconcile.Request{} + + // get all Proc CRs + procs := &telemetryv1.CloudKittyProcList{} + listOpts := []client.ListOption{ + client.InNamespace(namespace), + } + if err := r.Client.List(context.Background(), procs, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve Proc CRs %v") + return nil + } + + // Watch for changes to the ca cert config map + for _, cr := range procs.Items { + if configMapName == fmt.Sprintf("%s-lokistack-gateway-ca-bundle", cloudkitty.GetOwningCloudKittyName(&cr)) { + name := client.ObjectKey{ + Namespace: namespace, + Name: cr.Name, + } + Log.Info(fmt.Sprintf("ConfigMap %s is used by CloudKittyProc CR %s", configMapName, cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } if len(result) > 0 { return result } @@ -297,6 +341,8 @@ func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctr handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), ). + Watches(&corev1.ConfigMap{}, + handler.EnqueueRequestsFromMapFunc(configMapFn)). Watches(&topologyv1.Topology{}, handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), builder.WithPredicates(predicate.GenerationChangedPredicate{})). @@ -475,6 +521,26 @@ func (r *CloudKittyProcReconciler) reconcileNormal(ctx context.Context, instance return ctrl.Result{}, err } + // Add client cert secret hash to all the other config data, so that + // restart is triggered on certificate changes (e.g. when they + // rotate) + _, clientCertHash, err := secret.GetSecret( + ctx, + helper, + cloudkitty.ClientCertSecretName, + instance.Namespace, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + configVars["client-cert"] = env.SetValue(clientCertHash) + // // create hash over all the different input resources to identify if any those changed // and a restart/recreate is required. diff --git a/controllers/metricstorage_controller.go b/controllers/metricstorage_controller.go index a05b4ba8..6db87531 100644 --- a/controllers/metricstorage_controller.go +++ b/controllers/metricstorage_controller.go @@ -31,25 +31,16 @@ import ( discoveryv1 "k8s.io/api/discovery/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" - "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - - "k8s.io/apimachinery/pkg/api/meta" logr "github.com/go-logr/logr" "github.com/openstack-k8s-operators/lib-common/modules/ansible" @@ -93,15 +84,7 @@ var ( ) // MetricStorageReconciler reconciles a MetricStorage object -type MetricStorageReconciler struct { - client.Client - Kclient kubernetes.Interface - Scheme *runtime.Scheme - Controller controller.Controller - Watching []string - RESTMapper meta.RESTMapper - Cache cache.Cache -} +type MetricStorageReconciler utils.ConditionalWatchingReconciler // ConnectionInfo holds information about connection to a compute node type ConnectionInfo struct { @@ -318,7 +301,11 @@ func (r *MetricStorageReconciler) reconcileNormal( // Deploy monitoring stack - err := r.ensureWatches(ctx, "monitoringstacks.monitoring.rhobs", &obov1.MonitoringStack{}, eventHandler) + err := utils.EnsureWatches( + (*utils.ConditionalWatchingReconciler)(r), ctx, + "monitoringstacks.monitoring.rhobs", + &obov1.MonitoringStack{}, eventHandler, helper, + ) if err != nil { instance.Status.Conditions.MarkFalse(telemetryv1.MonitoringStackReadyCondition, condition.Reason("Can't own MonitoringStack resource. The Cluster Observability Operator probably isn't installed"), @@ -366,7 +353,13 @@ func (r *MetricStorageReconciler) reconcileNormal( } return []reconcile.Request{{NamespacedName: name}} } - err = r.ensureWatches(ctx, "prometheuses.monitoring.rhobs", &monv1.Prometheus{}, handler.EnqueueRequestsFromMapFunc(prometheusWatchFn)) + err = utils.EnsureWatches( + (*utils.ConditionalWatchingReconciler)(r), + ctx, "prometheuses.monitoring.rhobs", + &monv1.Prometheus{}, + handler.EnqueueRequestsFromMapFunc(prometheusWatchFn), + helper, + ) if err != nil { instance.Status.Conditions.MarkFalse(telemetryv1.PrometheusReadyCondition, condition.Reason("Can't watch prometheus resource. The Cluster Observability Operator probably isn't installed"), @@ -575,7 +568,13 @@ func (r *MetricStorageReconciler) reconcileNormal( } return []reconcile.Request{{NamespacedName: name}} } - err = r.ensureWatches(ctx, "prometheuses.monitoring.rhobs", &monv1.Prometheus{}, handler.EnqueueRequestsFromMapFunc(prometheusWatchFn)) + err = utils.EnsureWatches( + (*utils.ConditionalWatchingReconciler)(r), + ctx, "prometheuses.monitoring.rhobs", + &monv1.Prometheus{}, + handler.EnqueueRequestsFromMapFunc(prometheusWatchFn), + helper, + ) if err != nil { instance.Status.Conditions.MarkFalse(telemetryv1.PrometheusReadyCondition, condition.Reason("Can't watch prometheus resource. The Cluster Observability Operator probably isn't installed"), @@ -710,7 +709,11 @@ func (r *MetricStorageReconciler) createScrapeConfigs( helper *helper.Helper, ) (ctrl.Result, error) { Log := r.GetLogger(ctx) - err := r.ensureWatches(ctx, "scrapeconfigs.monitoring.rhobs", &monv1alpha1.ScrapeConfig{}, eventHandler) + err := utils.EnsureWatches( + (*utils.ConditionalWatchingReconciler)(r), + ctx, "scrapeconfigs.monitoring.rhobs", + &monv1alpha1.ScrapeConfig{}, eventHandler, helper, + ) if err != nil { instance.Status.Conditions.MarkFalse(telemetryv1.ScrapeConfigReadyCondition, condition.Reason("Can't own ScrapeConfig resource. The Cluster Observability Operator probably isn't installed"), @@ -959,7 +962,11 @@ func (r *MetricStorageReconciler) createDashboardObjects(ctx context.Context, in } // Deploy PrometheusRule for dashboards - err = r.ensureWatches(ctx, "prometheusrules.monitoring.rhobs", &monv1.PrometheusRule{}, eventHandler) + err = utils.EnsureWatches( + (*utils.ConditionalWatchingReconciler)(r), + ctx, "prometheusrules.monitoring.rhobs", + &monv1.PrometheusRule{}, eventHandler, helper, + ) if err != nil { instance.Status.Conditions.MarkFalse(telemetryv1.DashboardPrometheusRuleReadyCondition, condition.Reason("Can't own PrometheusRule resource. The Cluster Observability Operator probably isn't installed"), @@ -1079,43 +1086,6 @@ func (r *MetricStorageReconciler) createDashboardObjects(ctx context.Context, in return ctrl.Result{}, err } -func (r *MetricStorageReconciler) ensureWatches( - ctx context.Context, - name string, - kind client.Object, - handler handler.EventHandler, -) error { - Log := r.GetLogger(ctx) - for _, item := range r.Watching { - if item == name { - // We are already watching the resource - return nil - } - } - u := &unstructured.Unstructured{} - u.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "apiextensions.k8s.io", - Kind: "CustomResourceDefinition", - Version: "v1", - }) - - err := r.Client.Get(context.Background(), client.ObjectKey{ - Name: name, - }, u) - if err != nil { - return err - } - - Log.Info(fmt.Sprintf("Starting to watch %s", name)) - err = r.Controller.Watch(source.Kind(r.Cache, kind), - handler, - ) - if err == nil { - r.Watching = append(r.Watching, name) - } - return err -} - func getComputeNodesConnectionInfo( instance *telemetryv1.MetricStorage, helper *helper.Helper, diff --git a/go.mod b/go.mod index 466c105c..36b2976c 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,9 @@ go 1.21 replace github.com/openstack-k8s-operators/telemetry-operator/api => ./api require ( + github.com/cert-manager/cert-manager v1.14.7 github.com/go-logr/logr v1.4.2 + github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.6 github.com/onsi/ginkgo/v2 v2.20.1 github.com/onsi/gomega v1.34.1 @@ -13,6 +15,7 @@ require ( github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20250513115636-b549982a5d8f github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20250604143452-2260c431b9f1 github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20250423055245-3cb2ae8df6f0 + github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.0 github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20250508141203-be026d3164f7 github.com/openstack-k8s-operators/mariadb-operator/api v0.6.1-0.20250415060817-dc849adfa27e github.com/openstack-k8s-operators/telemetry-operator/api v0.3.1-0.20240529090522-c780bd90b147 @@ -22,7 +25,7 @@ require ( k8s.io/api v0.29.15 k8s.io/apimachinery v0.29.15 k8s.io/client-go v0.29.15 - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 + k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 sigs.k8s.io/controller-runtime v0.17.6 ) @@ -67,11 +70,11 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/net v0.28.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.18.0 // indirect - golang.org/x/sys v0.23.0 // indirect - golang.org/x/term v0.23.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.24.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect @@ -84,6 +87,7 @@ require ( k8s.io/component-base v0.29.15 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240322212309-b815d8309940 // indirect + sigs.k8s.io/gateway-api v1.0.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/go.sum b/go.sum index 6b71026f..a77e2c08 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cert-manager/cert-manager v1.14.7 h1:C2L59sMGMdSpd8SPx5qfPAL7ejZaNxJBRd24S7Ws5Ek= +github.com/cert-manager/cert-manager v1.14.7/go.mod h1:0QE/Hzfs2SxNrFFYgFh/d0c0cDfNv9qSrAev2LFt5nM= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -49,6 +51,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gophercloud/gophercloud v1.14.1 h1:DTCNaTVGl8/cFu58O1JwWgis9gtISAFONqpMKNg/Vpw= github.com/gophercloud/gophercloud v1.14.1/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= +github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba h1:P5Wgp2HfGfNPLCPpS+YqquKdrrl4tW0El7VX23D6vtg= +github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba/go.mod h1:OBAgJh0mLYRvziBzBKr4/anrPHqGY9qEfuNXCpnUNi0= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -86,6 +90,8 @@ github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20250604143452 github.com/openstack-k8s-operators/keystone-operator/api v0.6.1-0.20250604143452-2260c431b9f1/go.mod h1:dgYQJbbVaRuP98yZZB3K1rNpqnF54I1HM1ZTaOzPKBY= github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20250423055245-3cb2ae8df6f0 h1:QKmpBfn9zIYcmODrvhnnrOx2CV1Y2t6M8DwNgLbaRbI= github.com/openstack-k8s-operators/lib-common/modules/ansible v0.6.1-0.20250423055245-3cb2ae8df6f0/go.mod h1:0bajRHochTUT6Ecfriw27l3vL0yezVrnUmt3bcIpu4w= +github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.0 h1:cFOyP37qQ9T1D6mVTCwuPGt86LB4sTErpHT+L1e+VKY= +github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.6.0/go.mod h1:jgfvFeljXxot0LODLYCmjESxoMXbClXcBcf0DaX4zA0= github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20250508141203-be026d3164f7 h1:c3h1q3fDoit3NmvNL89xUL9A12bJivaTF+IOPEOAwLc= github.com/openstack-k8s-operators/lib-common/modules/common v0.6.1-0.20250508141203-be026d3164f7/go.mod h1:UwHXRIrMSPJD3lFqrA4oKmRXVLFQCRkLAj9x6KLEHiQ= github.com/openstack-k8s-operators/lib-common/modules/openstack v0.6.1-0.20250508141203-be026d3164f7 h1:IybBq3PrxwdvzAF19TjdMCqbEVkX2p3gIkme/Fju6do= @@ -149,8 +155,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -165,19 +171,19 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -223,10 +229,12 @@ k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240322212309-b815d8309940 h1:qVoMaQV5t62UUvHe16Q3eb2c5HPzLHYzsi0Tu/xLndo= k8s.io/kube-openapi v0.0.0-20240322212309-b815d8309940/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.17.6 h1:12IXsozEsIXWAMRpgRlYS1jjAHQXHtWEOMdULh3DbEw= sigs.k8s.io/controller-runtime v0.17.6/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY= +sigs.k8s.io/gateway-api v1.0.0 h1:iPTStSv41+d9p0xFydll6d7f7MOBGuqXM6p2/zVYMAs= +sigs.k8s.io/gateway-api v1.0.0/go.mod h1:4cUgr0Lnp5FZ0Cdq8FdRwCvpiWws7LVhLHGIudLlf4c= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= diff --git a/main.go b/main.go index 7905d299..80690fa9 100644 --- a/main.go +++ b/main.go @@ -28,6 +28,7 @@ import ( "k8s.io/client-go/kubernetes" _ "k8s.io/client-go/plugin/pkg/client/auth" + certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -39,6 +40,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + lokistackv1 "github.com/grafana/loki/operator/api/loki/v1" networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" heatv1 "github.com/openstack-k8s-operators/heat-operator/api/v1beta1" memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" @@ -81,6 +83,8 @@ func init() { utilruntime.Must(rabbitmqclusterv1.AddToScheme(scheme)) utilruntime.Must(networkv1.AddToScheme(scheme)) utilruntime.Must(topologyv1.AddToScheme(scheme)) + utilruntime.Must(lokistackv1.AddToScheme(scheme)) + utilruntime.Must(certmgrv1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } @@ -199,9 +203,11 @@ func main() { } if err = (&controllers.CloudKittyReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Kclient: kclient, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Kclient: kclient, + RESTMapper: mgr.GetRESTMapper(), + Cache: mgr.GetCache(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create CloudKitty controller") os.Exit(1) diff --git a/pkg/cloudkitty/cert.go b/pkg/cloudkitty/cert.go new file mode 100644 index 00000000..4a72e4cb --- /dev/null +++ b/pkg/cloudkitty/cert.go @@ -0,0 +1,67 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkitty + +import ( + "fmt" + + certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + certmgrmetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + "github.com/openstack-k8s-operators/lib-common/modules/certmanager" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" +) + +// Certificate defines a client certificate for communication between cloudkitty and loki +func Certificate( + instance *telemetryv1.CloudKitty, + labels map[string]string, + issuer *certmgrv1.Issuer, +) *certmgrv1.Certificate { + cert := certmanager.Cert( + ClientCertSecretName, + instance.Namespace, + labels, + certmgrv1.CertificateSpec{ + CommonName: fmt.Sprintf("%s.%s.svc", instance.Name, instance.Namespace), + DNSNames: []string{ + fmt.Sprintf("%s.%s.svc", instance.Name, instance.Namespace), + fmt.Sprintf("%s.%s.svc.cluster.local", instance.Name, instance.Namespace), + }, + SecretName: ClientCertSecretName, + Subject: &certmgrv1.X509Subject{ + OrganizationalUnits: []string{ + instance.Name, + }, + }, + PrivateKey: &certmgrv1.CertificatePrivateKey{ + Algorithm: "RSA", + Size: 3072, + }, + Usages: []certmgrv1.KeyUsage{ + certmgrv1.UsageDigitalSignature, + certmgrv1.UsageKeyEncipherment, + certmgrv1.UsageClientAuth, + }, + IssuerRef: certmgrmetav1.ObjectReference{ + Name: issuer.Name, + Kind: issuer.Kind, + Group: issuer.GroupVersionKind().Group, + }, + }, + ) + return cert +} diff --git a/pkg/cloudkitty/const.go b/pkg/cloudkitty/const.go index 8b372471..dc6589ba 100644 --- a/pkg/cloudkitty/const.go +++ b/pkg/cloudkitty/const.go @@ -52,6 +52,11 @@ const ( // PrometheusEndpointSecret - The name of the secret that contains the Prometheus endpoint configuration. PrometheusEndpointSecret = "metric-storage-prometheus-endpoint" + + ClientCertSecretName = "cert-cloudkitty-client-internal" + + CaConfigmapName = "lokistack-ca" + CaConfigmapKey = "ca.crt" ) var ResultRequeue = ctrl.Result{RequeueAfter: NormalDuration} diff --git a/pkg/cloudkitty/dbsync.go b/pkg/cloudkitty/dbsync.go index 5a84523a..28fb3161 100644 --- a/pkg/cloudkitty/dbsync.go +++ b/pkg/cloudkitty/dbsync.go @@ -41,7 +41,7 @@ func DbSyncJob(instance *telemetryv1.CloudKitty, labels map[string]string, annot args = append(args, dbSyncCommand) // create Volume and VolumeMounts - volumes := GetVolumes("cloudkitty") + volumes := GetVolumes(instance.Name) volumeMounts := GetVolumeMounts("cloudkitty-dbsync") // add CA cert if defined if instance.Spec.CloudKittyAPI.TLS.CaBundleSecretName != "" { diff --git a/pkg/cloudkitty/lokistack.go b/pkg/cloudkitty/lokistack.go new file mode 100644 index 00000000..c7604dc1 --- /dev/null +++ b/pkg/cloudkitty/lokistack.go @@ -0,0 +1,118 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloudkitty + +import ( + "fmt" + + lokistackv1 "github.com/grafana/loki/operator/api/loki/v1" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// LokiStack defines a lokistack for cloudkitty +func LokiStack( + instance *telemetryv1.CloudKitty, + labels map[string]string, +) *lokistackv1.LokiStack { + lokiStack := &lokistackv1.LokiStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-lokistack", instance.Name), + Namespace: instance.Namespace, + Labels: labels, + }, + Spec: lokistackv1.LokiStackSpec{ + // TODO: What size do we even want? I assume something + // smallish since only rating interact with this + Size: lokistackv1.LokiStackSizeType("1x.demo"), + Storage: instance.Spec.S3StorageConfig, + StorageClassName: instance.Spec.StorageClass, + Tenants: &lokistackv1.TenantsSpec{ + Mode: lokistackv1.Static, + Authentication: []lokistackv1.AuthenticationSpec{ + { + TenantName: instance.Name, + TenantID: instance.Name, + MTLS: &lokistackv1.MTLSSpec{ + CA: &lokistackv1.CASpec{ + CAKey: CaConfigmapKey, + CA: fmt.Sprintf("%s-%s", instance.Name, CaConfigmapName), + }, + }, + }, + }, + Authorization: &lokistackv1.AuthorizationSpec{ + // TODO: Determine what exactly this does and what's needed here + Roles: []lokistackv1.RoleSpec{ + { + Name: "cloudkitty-logs", + Resources: []string{ + "logs", + }, + Tenants: []string{ + "cloudkitty", + }, + Permissions: []lokistackv1.PermissionType{ + lokistackv1.Write, + lokistackv1.Read, + }, + }, + { + Name: "cluster-reader", + Resources: []string{ + "logs", + }, + Tenants: []string{ + "cloudkitty", + }, + Permissions: []lokistackv1.PermissionType{ + lokistackv1.Read, + }, + }, + }, + RoleBindings: []lokistackv1.RoleBindingsSpec{ + { + Name: "cloudkitty-logs", + Subjects: []lokistackv1.Subject{ + { + Name: "cloudkitty", + Kind: lokistackv1.Group, + }, + }, + Roles: []string{ + "cloudkitty-logs", + }, + }, + { + Name: "cluster-reader", + Subjects: []lokistackv1.Subject{ + { + Name: "cloudkitty-logs-admin", + Kind: lokistackv1.Group, + }, + }, + Roles: []string{ + "cluster-reader", + }, + }, + }, + }, + }, + }, + } + return lokiStack +} diff --git a/pkg/cloudkitty/storageinit.go b/pkg/cloudkitty/storageinit.go index 73b05cd8..05418f99 100644 --- a/pkg/cloudkitty/storageinit.go +++ b/pkg/cloudkitty/storageinit.go @@ -40,7 +40,7 @@ func StorageInitJob(instance *telemetryv1.CloudKitty, labels map[string]string, args := []string{"-c", storageInitCommand} // create Volume and VolumeMounts - volumes := GetVolumes("cloudkitty") + volumes := GetVolumes(instance.Name) volumeMounts := GetVolumeMounts("cloudkitty-storageinit") // add CA cert if defined if instance.Spec.CloudKittyAPI.TLS.CaBundleSecretName != "" { diff --git a/pkg/cloudkitty/volumes.go b/pkg/cloudkitty/volumes.go index 97ed4ab5..82b3d738 100644 --- a/pkg/cloudkitty/volumes.go +++ b/pkg/cloudkitty/volumes.go @@ -9,6 +9,8 @@ var ( scriptMode int32 = 0740 // configMode is the 640 permissions mode configMode int32 = 0640 + // certMode is the 400 permissions mode + certMode int32 = 0400 ) // GetVolumes - service volumes @@ -30,6 +32,28 @@ func GetVolumes(name string) []corev1.Volume { SecretName: name + "-config-data", }, }, + }, { + Name: "certs", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: ClientCertSecretName, + }, + }, + }, { + ConfigMap: &corev1.ConfigMapProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: name + "-lokistack-gateway-ca-bundle", + }, + }, + }, + }, + DefaultMode: &certMode, + }, + }, }, } } @@ -53,5 +77,10 @@ func GetVolumeMounts(serviceName string) []corev1.VolumeMount { SubPath: serviceName + "-config.json", ReadOnly: true, }, + { + Name: "certs", + MountPath: "/var/lib/openstack/loki-certs", + ReadOnly: true, + }, } } diff --git a/pkg/cloudkittyapi/statefulset.go b/pkg/cloudkittyapi/statefulset.go index 94d4346c..431a662d 100644 --- a/pkg/cloudkittyapi/statefulset.go +++ b/pkg/cloudkittyapi/statefulset.go @@ -75,7 +75,7 @@ func StatefulSet( // create Volume and VolumeMounts volumes := GetVolumes(cloudkitty.GetOwningCloudKittyName(instance), instance.Name) - volumeMounts := GetVolumeMounts(instance.Name) + volumeMounts := GetVolumeMounts() // add CA cert if defined if instance.Spec.TLS.CaBundleSecretName != "" { diff --git a/pkg/cloudkittyapi/volumes.go b/pkg/cloudkittyapi/volumes.go index 6d9f36ab..4108fc3d 100644 --- a/pkg/cloudkittyapi/volumes.go +++ b/pkg/cloudkittyapi/volumes.go @@ -31,7 +31,7 @@ func GetVolumes(parentName string, name string) []corev1.Volume { } // GetVolumeMounts - CloudKitty API VolumeMounts -func GetVolumeMounts(parentName string) []corev1.VolumeMount { +func GetVolumeMounts() []corev1.VolumeMount { volumeMounts := []corev1.VolumeMount{ { Name: "config-data-custom", @@ -41,7 +41,7 @@ func GetVolumeMounts(parentName string) []corev1.VolumeMount { GetLogVolumeMount(), } - return append(cloudkitty.GetVolumeMounts(parentName), volumeMounts...) + return append(cloudkitty.GetVolumeMounts(cloudkitty.ServiceName+"-api"), volumeMounts...) } // GetLogVolumeMount - CloudKitty API LogVolumeMount diff --git a/pkg/cloudkittyproc/statefulset.go b/pkg/cloudkittyproc/statefulset.go index a568be4d..b04baaa0 100644 --- a/pkg/cloudkittyproc/statefulset.go +++ b/pkg/cloudkittyproc/statefulset.go @@ -75,7 +75,7 @@ func StatefulSet( envVars["CONFIG_HASH"] = env.SetValue(configHash) volumes := GetVolumes(cloudkitty.GetOwningCloudKittyName(instance), instance.Name) - volumeMounts := GetVolumeMounts(instance.Name) + volumeMounts := GetVolumeMounts() // Add the CA bundle if instance.Spec.TLS.CaBundleSecretName != "" { diff --git a/pkg/cloudkittyproc/volumes.go b/pkg/cloudkittyproc/volumes.go index 51ff46a7..22fdc822 100644 --- a/pkg/cloudkittyproc/volumes.go +++ b/pkg/cloudkittyproc/volumes.go @@ -25,7 +25,7 @@ func GetVolumes(parentName string, name string) []corev1.Volume { } // GetVolumeMounts - CloudKitty API VolumeMounts -func GetVolumeMounts(parentName string) []corev1.VolumeMount { +func GetVolumeMounts() []corev1.VolumeMount { volumeMounts := []corev1.VolumeMount{ { Name: "config-data-custom", @@ -34,5 +34,5 @@ func GetVolumeMounts(parentName string) []corev1.VolumeMount { }, } - return append(cloudkitty.GetVolumeMounts(parentName), volumeMounts...) + return append(cloudkitty.GetVolumeMounts(cloudkitty.ServiceName+"-proc"), volumeMounts...) } diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index e19c58ca..0cde4c6e 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -18,14 +18,34 @@ package utils import ( "context" + "fmt" k8s_errors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/source" "github.com/openstack-k8s-operators/lib-common/modules/common/helper" ) +type ConditionalWatchingReconciler struct { + client.Client + Kclient kubernetes.Interface + Scheme *runtime.Scheme + Controller controller.Controller + Watching []string + RESTMapper meta.RESTMapper + Cache cache.Cache +} + // EnsureDeleted - Delete the object which in turn will clean the sub resources func EnsureDeleted(ctx context.Context, helper *helper.Helper, obj client.Object) (ctrl.Result, error) { key := client.ObjectKeyFromObject(obj) @@ -44,3 +64,42 @@ func EnsureDeleted(ctx context.Context, helper *helper.Helper, obj client.Object return ctrl.Result{}, nil } + +func EnsureWatches( + r *ConditionalWatchingReconciler, + ctx context.Context, + name string, + kind client.Object, + handler handler.EventHandler, + helper *helper.Helper, +) error { + Log := helper.GetLogger() + for _, item := range r.Watching { + if item == name { + // We are already watching the resource + return nil + } + } + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "apiextensions.k8s.io", + Kind: "CustomResourceDefinition", + Version: "v1", + }) + + err := r.Client.Get(context.Background(), client.ObjectKey{ + Name: name, + }, u) + if err != nil { + return err + } + + Log.Info(fmt.Sprintf("Starting to watch %s", name)) + err = r.Controller.Watch(source.Kind(r.Cache, kind), + handler, + ) + if err == nil { + r.Watching = append(r.Watching, name) + } + return err +} diff --git a/templates/cloudkitty/config/cloudkitty-api-config.json b/templates/cloudkitty/config/cloudkitty-api-config.json index 107cfa1a..9ff96965 100644 --- a/templates/cloudkitty/config/cloudkitty-api-config.json +++ b/templates/cloudkitty/config/cloudkitty-api-config.json @@ -63,6 +63,13 @@ "perm": "0640", "optional": true, "merge": true + }, + { + "source": "/var/lib/openstack/loki-certs/*", + "dest": "/etc/cloudkitty/certs/", + "owner": "cloudkitty:cloudkitty", + "perm": "0400", + "merge": true } ], "permissions": [ diff --git a/templates/cloudkitty/config/cloudkitty-proc-config.json b/templates/cloudkitty/config/cloudkitty-proc-config.json index 410ed9d3..b6d83aef 100644 --- a/templates/cloudkitty/config/cloudkitty-proc-config.json +++ b/templates/cloudkitty/config/cloudkitty-proc-config.json @@ -12,6 +12,13 @@ "dest": "/etc/cloudkitty/metrics.yaml", "owner": "cloudkitty", "perm": "0600" + }, + { + "source": "/var/lib/openstack/loki-certs/*", + "dest": "/etc/cloudkitty/certs/", + "owner": "cloudkitty:cloudkitty", + "perm": "0400", + "merge": true } ] } diff --git a/templates/cloudkitty/config/cloudkitty.conf b/templates/cloudkitty/config/cloudkitty.conf index 318ed398..5faf0864 100644 --- a/templates/cloudkitty/config/cloudkitty.conf +++ b/templates/cloudkitty/config/cloudkitty.conf @@ -64,7 +64,10 @@ version = 2 backend = loki [storage_loki] -url = http://loki:3100/loki/api/v1 +url = https://{{ .LokiHost }}:{{ .LokiPort }}/api/logs/v1/cloudkitty/loki/api/v1 +ca_file = /etc/cloudkitty/certs/service-ca.crt +cert_file = /etc/cloudkitty/certs/tls.crt +key_file = /etc/cloudkitty/certs/tls.key [database] connection = {{ .DatabaseConnection }} diff --git a/tests/kuttl/suites/cloudkitty/config.yaml b/tests/kuttl/suites/cloudkitty/config.yaml new file mode 100644 index 00000000..bd1d84da --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/config.yaml @@ -0,0 +1,14 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestSuite +reportFormat: JSON +reportName: kuttl-cloudkitty-results +namespace: telemetry-kuttl-cloudkitty +# we could set this lower, but the initial image pull can take a while +timeout: 300 +parallel: 1 +skipDelete: true +testDirs: + - tests/kuttl/suites/cloudkitty/ +suppress: + - events +artifactsDir: tests/kuttl/suites/cloudkitty/output diff --git a/tests/kuttl/suites/cloudkitty/deps/OpenStackControlPlane.yaml b/tests/kuttl/suites/cloudkitty/deps/OpenStackControlPlane.yaml new file mode 100644 index 00000000..f1b9f207 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/OpenStackControlPlane.yaml @@ -0,0 +1,27 @@ +apiVersion: core.openstack.org/v1beta1 +kind: OpenStackControlPlane +metadata: + name: openstack +spec: + storageClass: "crc-csi-hostpath-provisioner" + keystone: + template: + databaseInstance: openstack + secret: osp-secret + ironic: + enabled: false + template: + ironicConductors: [] + manila: + enabled: false + template: + manilaShares: {} + horizon: + enabled: false + nova: + enabled: false + placement: + template: + databaseInstance: openstack + secret: osp-secret + dataplane: diff --git a/tests/kuttl/suites/cloudkitty/deps/infra.yaml b/tests/kuttl/suites/cloudkitty/deps/infra.yaml new file mode 100644 index 00000000..7c47e716 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/infra.yaml @@ -0,0 +1,42 @@ +apiVersion: core.openstack.org/v1beta1 +kind: OpenStackControlPlane +metadata: + name: openstack +spec: + mariadb: + enabled: false + templates: + openstack: + replicas: 0 + openstack-cell1: + replicas: 0 + galera: + enabled: true + templates: + openstack: + replicas: 1 + storageRequest: 500M + openstack-cell1: + replicas: 1 + storageRequest: 500M + secret: osp-secret + secret: osp-secret + rabbitmq: + templates: + rabbitmq: + replicas: 1 + image: quay.io/podified-antelope-centos9/openstack-rabbitmq@sha256:41c36935b8b8cd3c5e490d1c03549ba2c0e8ddff50238fb2400d74613aa2e087 + rabbitmq-cell1: + replicas: 1 + memcached: + templates: + memcached: + replicas: 1 + ovn: + enabled: false + template: + ovnController: + external-ids: + ovn-encap-type: geneve + ovs: + enabled: false diff --git a/tests/kuttl/suites/cloudkitty/deps/kustomization.yaml b/tests/kuttl/suites/cloudkitty/deps/kustomization.yaml new file mode 100644 index 00000000..b366cb73 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/kustomization.yaml @@ -0,0 +1,50 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: telemetry-kuttl-cloudkitty + +secretGenerator: +- literals: + - AdminPassword=password + - DbRootPassword=password + - DatabasePassword=password + - KeystoneDatabasePassword=password + - PlacementPassword=password + - PlacementDatabasePassword=password + - GlancePassword=password + - GlanceDatabasePassword=password + - NeutronPassword=password + - NeutronDatabasePassword=password + - NovaPassword=password + - NovaAPIDatabasePassword=password + - NovaCell0DatabasePassword=password + - NovaCell1DatabasePassword=password + - AodhPassword=password + - AodhDatabasePassword=password + - CeilometerPassword=password + - CeilometerDatabasePassword=password + - HeatPassword=password + - HeatDatabasePassword=password + - HeatAuthEncryptionKey=66699966699966600666999666999666 + - MetadataSecret=42 + - CloudKittyPassword=password + name: osp-secret +generatorOptions: + disableNameSuffixHash: true + labels: + type: osp-secret + +resources: +- namespace.yaml +- OpenStackControlPlane.yaml +- loki-s3-secret.yaml + +patches: +- patch: |- + apiVersion: core.openstack.org/v1beta1 + kind: OpenStackControlPlane + metadata: + name: openstack + spec: + secret: osp-secret +- path: infra.yaml +- path: telemetry.yaml diff --git a/tests/kuttl/suites/cloudkitty/deps/loki-operator.yaml b/tests/kuttl/suites/cloudkitty/deps/loki-operator.yaml new file mode 100644 index 00000000..5b3d937a --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/loki-operator.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-operators-redhat + labels: + name: openshift-operators-redhat +--- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: loki-operator + namespace: openshift-operators-redhat +spec: + upgradeStrategy: Default +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: loki-operator + namespace: openshift-operators-redhat +spec: + channel: stable-6.1 + installPlanApproval: Automatic + name: loki-operator + source: redhat-operators + sourceNamespace: openshift-marketplace diff --git a/tests/kuttl/suites/cloudkitty/deps/loki-s3-secret.yaml b/tests/kuttl/suites/cloudkitty/deps/loki-s3-secret.yaml new file mode 100644 index 00000000..b176d7f9 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/loki-s3-secret.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: logging-loki-s3 +stringData: + access_key_id: minio + access_key_secret: minio123 + bucketnames: loki + endpoint: http://minio.minio-dev.svc.cluster.local:9000 diff --git a/tests/kuttl/suites/cloudkitty/deps/minio.yaml b/tests/kuttl/suites/cloudkitty/deps/minio.yaml new file mode 100644 index 00000000..f1854ff4 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/minio.yaml @@ -0,0 +1,99 @@ +--- +# Deploys a new Namespace for the MinIO Pod +apiVersion: v1 +kind: Namespace +metadata: + name: minio-dev # Change this value if you want a different namespace name + labels: + name: minio-dev # Change this value to match metadata.name +--- +# Deploys a new MinIO Pod into the metadata.namespace Kubernetes namespace +# +apiVersion: v1 +kind: Pod +metadata: + labels: + app: minio + name: minio + namespace: minio-dev # Change this value to match the namespace metadata.name +spec: + containers: + - name: minio + image: quay.io/minio/minio:latest + command: + - /bin/bash + - -c + - | + mkdir -p /data/loki && \ + minio server /data + env: + - name: MINIO_ACCESS_KEY + value: minio + - name: MINIO_SECRET_KEY + value: minio123 + volumeMounts: + - mountPath: /data + name: storage # Corresponds to the `spec.volumes` Persistent Volume + volumes: + - name: storage + persistentVolumeClaim: + claimName: minio-pvc +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio-pvc + namespace: minio-dev +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: crc-csi-hostpath-provisioner +--- +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: minio-dev +spec: + selector: + app: minio + ports: + - name: api + protocol: TCP + port: 9000 + - name: console + protocol: TCP + port: 9090 +--- +kind: Route +apiVersion: route.openshift.io/v1 +metadata: + name: minio-console + namespace: minio-dev +spec: + host: console-minio-dev.apps-crc.testing + to: + kind: Service + name: minio + weight: 100 + port: + targetPort: console + wildcardPolicy: None +--- +kind: Route +apiVersion: route.openshift.io/v1 +metadata: + name: minio-api + namespace: minio-dev +spec: + host: api-minio-dev.apps-crc.testing + to: + kind: Service + name: minio + weight: 100 + port: + targetPort: api + wildcardPolicy: None diff --git a/tests/kuttl/suites/cloudkitty/deps/namespace.yaml b/tests/kuttl/suites/cloudkitty/deps/namespace.yaml new file mode 100644 index 00000000..63c85762 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: telemetry-kuttl diff --git a/tests/kuttl/suites/cloudkitty/deps/telemetry.yaml b/tests/kuttl/suites/cloudkitty/deps/telemetry.yaml new file mode 100644 index 00000000..5c2714a2 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/deps/telemetry.yaml @@ -0,0 +1,7 @@ +apiVersion: core.openstack.org/v1beta1 +kind: OpenStackControlPlane +metadata: + name: openstack +spec: + telemetry: + enabled: false diff --git a/tests/kuttl/suites/cloudkitty/output/.keep b/tests/kuttl/suites/cloudkitty/output/.keep new file mode 100644 index 00000000..e69de29b diff --git a/tests/kuttl/suites/cloudkitty/tests/00-deps.yaml b/tests/kuttl/suites/cloudkitty/tests/00-deps.yaml new file mode 100644 index 00000000..f850b416 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/tests/00-deps.yaml @@ -0,0 +1,9 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + oc apply -f ../deps/loki-operator.yaml + until oc api-resources | grep -q grafana; do sleep 1; done + - script: | + oc apply -f ../deps/minio.yaml + oc wait --for='jsonpath={.status.conditions[?(@.type=="Ready")].status}=True' pod/minio -n minio-dev diff --git a/tests/kuttl/suites/cloudkitty/tests/01-assert.yaml b/tests/kuttl/suites/cloudkitty/tests/01-assert.yaml new file mode 100644 index 00000000..ad927419 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/tests/01-assert.yaml @@ -0,0 +1,81 @@ +apiVersion: loki.grafana.com/v1 +kind: LokiStack +metadata: + name: telemetry-kuttl-cloudkitty-lokistack + ownerReferences: + - kind: CloudKitty + name: telemetry-kuttl-cloudkitty +spec: + tenants: + authentication: + - tenantId: telemetry-kuttl-cloudkitty + tenantName: telemetry-kuttl-cloudkitty +# NOTE: It's hard to assert LokiStack condition with kuttl-tests, because +# their number can change even when LokiStack as whole is actually ready. +# And because kuttl-tests will compare the number of conditions defined +# here vs. the number of conditions inside the real CR, it could fail +# even when it shouldn't. +# But LokiStack condition is being copied into the master CloudKitty CR, +# so we can ensure that LokiStack is ready by checking that Cloudkitty +# is Ready +--- +apiVersion: telemetry.openstack.org/v1beta1 +kind: CloudKitty +metadata: + name: telemetry-kuttl-cloudkitty +status: + conditions: + - status: "True" + type: Ready + - status: "True" + type: CloudKittyAPIReady + - status: "True" + type: CloudKittyClientCertReady + - status: "True" + type: CloudKittyLokiStackReady + - status: "True" + type: CloudKittyProcReady + - status: "True" + type: CloudKittyStorageInitReady + - status: "True" + type: DBReady + - status: "True" + type: DBSyncReady + - status: "True" + type: InputReady + - status: "True" + type: MariaDBAccountReady + - status: "True" + type: MemcachedReady + - status: "True" + type: NetworkAttachmentsReady + - status: "True" + type: RabbitMqTransportURLReady + - status: "True" + type: RoleBindingReady + - status: "True" + type: RoleReady + - status: "True" + type: ServiceAccountReady + - status: "True" + type: ServiceConfigReady +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: cert-cloudkitty-client-internal + ownerReferences: + - kind: CloudKitty + name: telemetry-kuttl-cloudkitty +spec: + subject: + organizationalUnits: + - telemetry-kuttl-cloudkitty + usages: + - digital signature + - key encipherment + - client auth +status: + conditions: + - status: "True" + type: Ready diff --git a/tests/kuttl/suites/cloudkitty/tests/01-deploy.yaml b/tests/kuttl/suites/cloudkitty/tests/01-deploy.yaml new file mode 100644 index 00000000..135146e2 --- /dev/null +++ b/tests/kuttl/suites/cloudkitty/tests/01-deploy.yaml @@ -0,0 +1,54 @@ +apiVersion: telemetry.openstack.org/v1beta1 +kind: CloudKitty +metadata: + name: telemetry-kuttl-cloudkitty +spec: + apiTimeout: 0 + cloudKittyAPI: + # TODO: This should go away once CK images are taken from openstackversions + containerImage: quay.io/jwysogla/cloudkitty-api:latest + override: + service: + internal: + metadata: + labels: + osctlplane: "" + osctlplane-service: telemetry + public: + metadata: + labels: + osctlplane: "" + osctlplane-service: telemetry + replicas: 1 + resources: {} + tls: + api: + internal: {} + public: {} + caBundleSecretName: combined-ca-bundle + cloudKittyProc: + # TODO: This should go away once CK images are taken from openstackversions + containerImage: quay.io/jwysogla/cloudkitty-processor:latest + replicas: 1 + resources: {} + tls: + caBundleSecretName: combined-ca-bundle + databaseAccount: cloudkitty + databaseInstance: openstack + memcachedInstance: memcached + passwordSelector: + aodhService: AodhPassword + ceilometerService: CeilometerPassword + cloudKittyService: CloudKittyPassword + preserveJobs: false + rabbitMqClusterName: rabbitmq + s3StorageConfig: + schemas: + - effectiveDate: "2024-11-18" + version: v13 + secret: + name: logging-loki-s3 + type: s3 + secret: osp-secret + serviceUser: cloudkitty + storageClass: local-storage From bc01f18b34ca8988bc821fe925913ed54a825fef Mon Sep 17 00:00:00 2001 From: Jaromir Wysoglad Date: Thu, 18 Sep 2025 11:55:16 -0400 Subject: [PATCH 22/87] [cloudkitty] Add missing rbac annotations --- config/rbac/role.yaml | 32 ++++++++++++++++++++++++++++ controllers/cloudkitty_controller.go | 4 ++++ 2 files changed, 36 insertions(+) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6fabb4df..7bbc14ea 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -58,6 +58,26 @@ rules: - patch - update - watch +- apiGroups: + - cert-manager.io + resources: + - certificates + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - cert-manager.io + resources: + - issuers + verbs: + - get + - list + - watch - apiGroups: - cloudkitty.openstack.org resources: @@ -181,6 +201,18 @@ rules: - patch - update - watch +- apiGroups: + - loki.grafana.com + resources: + - lokistacks + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - mariadb.openstack.org resources: diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index cb7a9ea8..3d431be1 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -104,6 +104,7 @@ func (r *CloudKittyReconciler) GetLogger(ctx context.Context) logr.Logger { // +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyprocs/status,verbs=get;update;patch // +kubebuilder:rbac:groups=telemetry.openstack.org,resources=cloudkittyprocs/finalizers,verbs=update;patch // +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;create;update;patch;delete;watch +// +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;create;update;patch;delete;watch // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;create;update;patch;delete;watch // +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbdatabases,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=mariadb.openstack.org,resources=mariadbaccounts,verbs=get;list;watch;create;update;patch;delete @@ -112,6 +113,9 @@ func (r *CloudKittyReconciler) GetLogger(ctx context.Context) logr.Logger { // +kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneapis,verbs=get;list;watch // +kubebuilder:rbac:groups=rabbitmq.openstack.org,resources=transporturls,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch +// +kubebuilder:rbac:groups=cert-manager.io,resources=certificates,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=cert-manager.io,resources=issuers,verbs=get;list;watch +// +kubebuilder:rbac:groups=loki.grafana.com,resources=lokistacks,verbs=get;list;watch;create;update;patch;delete // service account, role, rolebinding // +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch From 5893a8f103cc136a8620974606e5cfb178e04d11 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Wed, 17 Sep 2025 15:34:10 -0400 Subject: [PATCH 23/87] [cloudkitty] Add a default value for cloudkitty.s3StorageConfig The s3StorageConfig.secret is passed to Lokistack, which requires some value be set. The default is needed, or else there will be an error when it is not configured. The default is set to a reasonable value for the secret name (cloudkitty-loki-s3) and the type (s3), which match the values we're planning on using for the CI environment. Lines added: 1 Lines generated: 16 --- api/bases/telemetry.openstack.org_cloudkitties.yaml | 4 ++++ api/bases/telemetry.openstack.org_telemetries.yaml | 4 ++++ api/v1beta1/cloudkitty_types.go | 1 + config/crd/bases/telemetry.openstack.org_cloudkitties.yaml | 4 ++++ config/crd/bases/telemetry.openstack.org_telemetries.yaml | 4 ++++ 5 files changed, 17 insertions(+) diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml index a39f82a7..cb26429f 100644 --- a/api/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -576,6 +576,10 @@ spec: Needed to request a transportURL that is created and used in CloudKitty type: string s3StorageConfig: + default: + secret: + name: cloudkitty-loki-s3 + type: s3 description: S3 related configuration passed to Loki properties: schemas: diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index 1a81843e..8c262f16 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -1141,6 +1141,10 @@ spec: Needed to request a transportURL that is created and used in CloudKitty type: string s3StorageConfig: + default: + secret: + name: cloudkitty-loki-s3 + type: s3 description: S3 related configuration passed to Loki properties: schemas: diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index 0eca7214..17587f8c 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -109,6 +109,7 @@ type CloudKittySpecBase struct { // S3 related configuration passed to Loki // +kubebuilder:validation:Optional + // +kubebuilder:default={secret: {name: "cloudkitty-loki-s3", type: "s3"}} S3StorageConfig lokistackv1.ObjectStorageSpec `json:"s3StorageConfig"` // Storage class used for Loki diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml index a39f82a7..cb26429f 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -576,6 +576,10 @@ spec: Needed to request a transportURL that is created and used in CloudKitty type: string s3StorageConfig: + default: + secret: + name: cloudkitty-loki-s3 + type: s3 description: S3 related configuration passed to Loki properties: schemas: diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index 1a81843e..8c262f16 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -1141,6 +1141,10 @@ spec: Needed to request a transportURL that is created and used in CloudKitty type: string s3StorageConfig: + default: + secret: + name: cloudkitty-loki-s3 + type: s3 description: S3 related configuration passed to Loki properties: schemas: From 447e1bf9f6c11d80fc11c48b89dd8fc233ba41ed Mon Sep 17 00:00:00 2001 From: Jaromir Wysoglad Date: Tue, 23 Sep 2025 04:29:58 -0400 Subject: [PATCH 24/87] Fix issues after golang update Fix "non-constant format string in call to github.com/openstack-k8s-operators/lib-common/modules/common/condition.FalseCondition" The condition.FalseCondition takes a , as its last arguments, similarly to fmt.Sprintf already. So there is no need to use Sprintf. --- controllers/cloudkitty_controller.go | 2 +- controllers/cloudkittyapi_controller.go | 6 ++++-- controllers/cloudkittyproc_controller.go | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index 3d431be1..b4dd4531 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -716,7 +716,7 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te telemetryv1.CloudKittyLokiStackReadyCondition, condition.Reason(reason), condition.SeverityWarning, - fmt.Sprintf("LokiStack issue: %s", message))) + "LokiStack issue: %s", message)) } // Service account, role, binding diff --git a/controllers/cloudkittyapi_controller.go b/controllers/cloudkittyapi_controller.go index a47014b1..ba7f98db 100644 --- a/controllers/cloudkittyapi_controller.go +++ b/controllers/cloudkittyapi_controller.go @@ -793,7 +793,8 @@ func (r *CloudKittyAPIReconciler) reconcileNormal(ctx context.Context, instance condition.TLSInputReadyCondition, condition.RequestedReason, condition.SeverityInfo, - fmt.Sprintf(condition.TLSInputReadyWaitingMessage, instance.Spec.TLS.CaBundleSecretName))) + condition.TLSInputReadyWaitingMessage, + instance.Spec.TLS.CaBundleSecretName)) return ctrl.Result{}, nil } instance.Status.Conditions.Set(condition.FalseCondition( @@ -818,7 +819,8 @@ func (r *CloudKittyAPIReconciler) reconcileNormal(ctx context.Context, instance condition.TLSInputReadyCondition, condition.RequestedReason, condition.SeverityInfo, - fmt.Sprintf(condition.TLSInputReadyWaitingMessage, err.Error()))) + condition.TLSInputReadyWaitingMessage, + err.Error())) return ctrl.Result{}, nil } instance.Status.Conditions.Set(condition.FalseCondition( diff --git a/controllers/cloudkittyproc_controller.go b/controllers/cloudkittyproc_controller.go index 495f3332..aebfb4c2 100644 --- a/controllers/cloudkittyproc_controller.go +++ b/controllers/cloudkittyproc_controller.go @@ -480,7 +480,8 @@ func (r *CloudKittyProcReconciler) reconcileNormal(ctx context.Context, instance condition.TLSInputReadyCondition, condition.RequestedReason, condition.SeverityInfo, - fmt.Sprintf(condition.TLSInputReadyWaitingMessage, instance.Spec.TLS.CaBundleSecretName))) + condition.TLSInputReadyWaitingMessage, + instance.Spec.TLS.CaBundleSecretName)) return ctrl.Result{}, nil } instance.Status.Conditions.Set(condition.FalseCondition( From 186405486b283196a872053d625e7c41a082f261 Mon Sep 17 00:00:00 2001 From: jlarriba Date: Tue, 23 Sep 2025 11:23:00 +0200 Subject: [PATCH 25/87] Address review comments --- controllers/cloudkitty_controller.go | 2 +- controllers/cloudkittyproc_controller.go | 14 +++++++------- .../cloudkitty/config/cloudkitty-api-config.json | 6 ++++++ templates/cloudkitty/config/cloudkitty.conf | 5 +++-- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index b4dd4531..bf1e0d44 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -243,7 +243,7 @@ const ( ) var ( - commonWatchFields = []string{ + cloudKittyProcWatchFields = []string{ cloudKittyPasswordSecretField, cloudKittyCaBundleSecretNameField, cloudKittyTopologyField, diff --git a/controllers/cloudkittyproc_controller.go b/controllers/cloudkittyproc_controller.go index aebfb4c2..0c00369d 100644 --- a/controllers/cloudkittyproc_controller.go +++ b/controllers/cloudkittyproc_controller.go @@ -202,12 +202,12 @@ func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctr var secretName string = o.GetName() result := []reconcile.Request{} - // get all scheduler CRs - schedulers := &telemetryv1.CloudKittyProcList{} + // get all CloudKittyProc CRs + cloudKittyProcs := &telemetryv1.CloudKittyProcList{} listOpts := []client.ListOption{ client.InNamespace(namespace), } - if err := r.Client.List(context.Background(), schedulers, listOpts...); err != nil { + if err := r.Client.List(context.Background(), cloudKittyProcs, listOpts...); err != nil { Log.Error(err, "Unable to retrieve scheduler CRs %v") return nil } @@ -216,7 +216,7 @@ func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctr // CR.Spec.ManagingCrName label matches label := o.GetLabels() if l, ok := label[labels.GetOwnerNameLabelSelector(labels.GetGroupLabel(cloudkitty.ServiceName))]; ok { - for _, cr := range schedulers.Items { + for _, cr := range cloudKittyProcs.Items { // return reconcile event for the CR where the owner label AND the parentCloudKittyName matches if l == cloudkitty.GetOwningCloudKittyName(&cr) { // return namespace and Name of CR @@ -232,7 +232,7 @@ func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctr } // Watch for changes to any CustomServiceConfigSecrets - for _, cr := range schedulers.Items { + for _, cr := range cloudKittyProcs.Items { for _, v := range cr.Spec.CustomServiceConfigSecrets { if v == secretName { name := client.ObjectKey{ @@ -246,7 +246,7 @@ func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctr } // Watch for changes to the client cert secret if secretName == cloudkitty.ClientCertSecretName { - for _, cr := range schedulers.Items { + for _, cr := range cloudKittyProcs.Items { name := client.ObjectKey{ Namespace: namespace, Name: cr.Name, @@ -354,7 +354,7 @@ func (r *CloudKittyProcReconciler) findObjectsForSrc(ctx context.Context, src cl l := log.FromContext(ctx).WithName("Controllers").WithName("CloudKittyProc") - for _, field := range commonWatchFields { + for _, field := range cloudKittyProcWatchFields { crList := &telemetryv1.CloudKittyProcList{} listOps := &client.ListOptions{ FieldSelector: fields.OneTermEqualSelector(field, src.GetName()), diff --git a/templates/cloudkitty/config/cloudkitty-api-config.json b/templates/cloudkitty/config/cloudkitty-api-config.json index f74eae28..2c691726 100644 --- a/templates/cloudkitty/config/cloudkitty-api-config.json +++ b/templates/cloudkitty/config/cloudkitty-api-config.json @@ -13,6 +13,12 @@ "owner": "cloudkitty", "perm": "0600" }, + { + "source": "/var/lib/openstack/config/metrics.yaml", + "dest": "/etc/cloudkitty/metrics.yaml", + "owner": "cloudkitty", + "perm": "0600" + }, { "source": "/var/lib/openstack/service-config/0*.conf", "dest": "/etc/cloudkitty/cloudkitty.conf.d/", diff --git a/templates/cloudkitty/config/cloudkitty.conf b/templates/cloudkitty/config/cloudkitty.conf index 5faf0864..37bd1849 100644 --- a/templates/cloudkitty/config/cloudkitty.conf +++ b/templates/cloudkitty/config/cloudkitty.conf @@ -29,6 +29,7 @@ backend = keystone [fetcher_keystone] auth_section = authinfos +ignore_rating_role = True [storage_influxdb] version = 2 @@ -46,11 +47,11 @@ scope_key = project [collector_prometheus] {{- if .TLS }} -prometheus_url = https://{{ .PrometheusHost }}:{{ .PrometheusPort }} +prometheus_url = https://{{ .PrometheusHost }}:{{ .PrometheusPort }}/api/v1 cafile = {{ .CAFile }} insecure = false {{- else }} -prometheus_url = http://{{ .PrometheusHost }}:{{ .PrometheusPort }} +prometheus_url = http://{{ .PrometheusHost }}:{{ .PrometheusPort }}/api/v1 insecure = true {{- end }} From f9fb2fac9f21a04d14711a87ea17a822d7a28503 Mon Sep 17 00:00:00 2001 From: Juan Larriba Date: Tue, 23 Sep 2025 12:03:44 +0200 Subject: [PATCH 26/87] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jaromír Wysoglad --- controllers/telemetry_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/telemetry_controller.go b/controllers/telemetry_controller.go index a38c1002..7a6e7b10 100644 --- a/controllers/telemetry_controller.go +++ b/controllers/telemetry_controller.go @@ -634,7 +634,7 @@ func (r TelemetryReconciler) reconcileCloudKitty(ctx context.Context, instance * telemetryv1.AutoscalingReadyRunningMessage, )) } else { - // Mirror Autoscaling condition status + // Mirror Cloudkitty condition status c := cloudKittyInstance.Status.Conditions.Mirror(telemetryv1.CloudKittyReadyCondition) if c != nil { instance.Status.Conditions.Set(c) From 518d756d092d3e13417a3990722ccce3bda42f91 Mon Sep 17 00:00:00 2001 From: Juan Larriba Date: Tue, 23 Sep 2025 12:03:44 +0200 Subject: [PATCH 27/87] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jaromír Wysoglad --- controllers/cloudkittyapi_controller.go | 26 +++++++++++++++++++++++ controllers/cloudkittyproc_controller.go | 27 ++++++++++++++++++++++++ controllers/telemetry_controller.go | 2 +- 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/controllers/cloudkittyapi_controller.go b/controllers/cloudkittyapi_controller.go index ba7f98db..949b4ee4 100644 --- a/controllers/cloudkittyapi_controller.go +++ b/controllers/cloudkittyapi_controller.go @@ -282,6 +282,32 @@ func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl result = append(result, reconcile.Request{NamespacedName: name}) } } + + // Watch for changes to the prometheus secret + if secretName == cloudkitty.PrometheusEndpointSecret { + for _, cr := range apis.Items { + // Get the parent CloudKitty name + parentCloudKittyName := cloudkitty.GetOwningCloudKittyName(&cr) + + // Fetch the parent CloudKitty instance + parentCloudKitty := &telemetryv1.CloudKitty{} + _ = r.Client.Get(ctx, types.NamespacedName{ + Name: parentCloudKittyName, + Namespace: cr.Namespace, + }, parentCloudKitty) + + // Only return a reconcile event if we are using the prometheus secret + if parentCloudKitty.Spec.PrometheusHost == "" { + name := client.ObjectKey{ + Namespace: namespace, + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Secret %s is used by CloudKittyAPI CR %s", secretName, cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + } + if len(result) > 0 { return result } diff --git a/controllers/cloudkittyproc_controller.go b/controllers/cloudkittyproc_controller.go index 0c00369d..af542394 100644 --- a/controllers/cloudkittyproc_controller.go +++ b/controllers/cloudkittyproc_controller.go @@ -244,6 +244,7 @@ func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctr } } } + // Watch for changes to the client cert secret if secretName == cloudkitty.ClientCertSecretName { for _, cr := range cloudKittyProcs.Items { @@ -255,6 +256,32 @@ func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctr result = append(result, reconcile.Request{NamespacedName: name}) } } + + // Watch for changes to the prometheus secret + if secretName == cloudkitty.PrometheusEndpointSecret { + for _, cr := range cloudKittyProcs.Items { + // Get the parent CloudKitty name + parentCloudKittyName := cloudkitty.GetOwningCloudKittyName(&cr) + + // Fetch the parent CloudKitty instance + parentCloudKitty := &telemetryv1.CloudKitty{} + _ = r.Client.Get(ctx, types.NamespacedName{ + Name: parentCloudKittyName, + Namespace: cr.Namespace, + }, parentCloudKitty) + + // Only return a reconcile event if we are using the prometheus secret + if parentCloudKitty.Spec.PrometheusHost == "" { + name := client.ObjectKey{ + Namespace: namespace, + Name: cr.Name, + } + Log.Info(fmt.Sprintf("Secret %s is used by CloudKittyAPI CR %s", secretName, cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + } + if len(result) > 0 { return result } diff --git a/controllers/telemetry_controller.go b/controllers/telemetry_controller.go index a38c1002..7a6e7b10 100644 --- a/controllers/telemetry_controller.go +++ b/controllers/telemetry_controller.go @@ -634,7 +634,7 @@ func (r TelemetryReconciler) reconcileCloudKitty(ctx context.Context, instance * telemetryv1.AutoscalingReadyRunningMessage, )) } else { - // Mirror Autoscaling condition status + // Mirror Cloudkitty condition status c := cloudKittyInstance.Status.Conditions.Mirror(telemetryv1.CloudKittyReadyCondition) if c != nil { instance.Status.Conditions.Set(c) From 15f44d2156c1ed2ba9d31edbf9393a6800c92d1e Mon Sep 17 00:00:00 2001 From: Jaromir Wysoglad Date: Tue, 23 Sep 2025 17:35:39 -0400 Subject: [PATCH 28/87] Make cloudkitty s3 config optional Make all S3 related fields optional to ensure telemetry and openstackcontrolplane CRs are valid even when cloudkitty is disabled and isn't valid. --- .../telemetry.openstack.org_cloudkitties.yaml | 33 +------ .../telemetry.openstack.org_telemetries.yaml | 33 +------ api/go.mod | 1 - api/go.sum | 2 - api/v1beta1/cloudkitty_types.go | 82 ++++++++++++++++- api/v1beta1/zz_generated.deepcopy.go | 87 +++++++++++++++++++ .../telemetry.openstack.org_cloudkitties.yaml | 33 +------ .../telemetry.openstack.org_telemetries.yaml | 33 +------ controllers/cloudkitty_controller.go | 5 +- pkg/cloudkitty/lokistack.go | 80 ++++++++++++++++- 10 files changed, 264 insertions(+), 125 deletions(-) diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml index 0787ea58..5d64d83f 100644 --- a/api/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -599,46 +599,32 @@ spec: version: v11 description: Schemas for reading and writing logs. items: - description: ObjectStorageSchema defines a schema version and - the date when it will become effective. properties: effectiveDate: description: |- EffectiveDate contains a date in YYYY-MM-DD format which is interpreted in the UTC time zone. The configuration always needs at least one schema that is currently valid. This means that when creating a new - LokiStack it is recommended to add a schema with the latest available version and an effective date of "yesterday". + CloudKitty it is recommended to add a schema with the latest available version and an effective date of "yesterday". New schema versions added to the configuration always needs to be placed "in the future", so that Loki can start using it once the day rolls over. - pattern: ^([0-9]{4,})([-]([0-9]{2})){2}$ type: string version: description: Version for writing and reading logs. - enum: - - v11 - - v12 - - v13 type: string - required: - - effectiveDate - - version type: object minItems: 1 type: array secret: description: |- Secret for object storage authentication. - Name of a secret in the same namespace as the LokiStack custom resource. + Name of a secret in the same namespace as the CloudKitty custom resource. properties: credentialMode: description: |- CredentialMode can be used to set the desired credential mode for authenticating with the object storage. If this is not set, then the operator tries to infer the credential mode from the provided secret and its own configuration. - enum: - - static - - token - - token-cco type: string name: description: Name of a secret in the namespace configured @@ -646,16 +632,7 @@ spec: type: string type: description: Type of object storage that should be used - enum: - - azure - - gcs - - s3 - - swift - - alibabacloud type: string - required: - - name - - type type: object tls: description: TLS configuration for reaching the object storage @@ -664,19 +641,17 @@ spec: caKey: description: |- Key is the data key of a ConfigMap containing a CA certificate. - It needs to be in the same namespace as the LokiStack custom resource. + It needs to be in the same namespace as the CloudKitty custom resource. If empty, it defaults to "service-ca.crt". type: string caName: description: |- CA is the name of a ConfigMap containing a CA certificate. - It needs to be in the same namespace as the LokiStack custom resource. + It needs to be in the same namespace as the CloudKitty custom resource. type: string required: - caName type: object - required: - - secret type: object secret: default: osp-secret diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index af9bbd3c..d7c5e451 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -1165,46 +1165,32 @@ spec: version: v11 description: Schemas for reading and writing logs. items: - description: ObjectStorageSchema defines a schema version - and the date when it will become effective. properties: effectiveDate: description: |- EffectiveDate contains a date in YYYY-MM-DD format which is interpreted in the UTC time zone. The configuration always needs at least one schema that is currently valid. This means that when creating a new - LokiStack it is recommended to add a schema with the latest available version and an effective date of "yesterday". + CloudKitty it is recommended to add a schema with the latest available version and an effective date of "yesterday". New schema versions added to the configuration always needs to be placed "in the future", so that Loki can start using it once the day rolls over. - pattern: ^([0-9]{4,})([-]([0-9]{2})){2}$ type: string version: description: Version for writing and reading logs. - enum: - - v11 - - v12 - - v13 type: string - required: - - effectiveDate - - version type: object minItems: 1 type: array secret: description: |- Secret for object storage authentication. - Name of a secret in the same namespace as the LokiStack custom resource. + Name of a secret in the same namespace as the CloudKitty custom resource. properties: credentialMode: description: |- CredentialMode can be used to set the desired credential mode for authenticating with the object storage. If this is not set, then the operator tries to infer the credential mode from the provided secret and its own configuration. - enum: - - static - - token - - token-cco type: string name: description: Name of a secret in the namespace configured @@ -1212,16 +1198,7 @@ spec: type: string type: description: Type of object storage that should be used - enum: - - azure - - gcs - - s3 - - swift - - alibabacloud type: string - required: - - name - - type type: object tls: description: TLS configuration for reaching the object storage @@ -1230,19 +1207,17 @@ spec: caKey: description: |- Key is the data key of a ConfigMap containing a CA certificate. - It needs to be in the same namespace as the LokiStack custom resource. + It needs to be in the same namespace as the CloudKitty custom resource. If empty, it defaults to "service-ca.crt". type: string caName: description: |- CA is the name of a ConfigMap containing a CA certificate. - It needs to be in the same namespace as the LokiStack custom resource. + It needs to be in the same namespace as the CloudKitty custom resource. type: string required: - caName type: object - required: - - secret type: object secret: default: osp-secret diff --git a/api/go.mod b/api/go.mod index 67cea330..4bee65d3 100644 --- a/api/go.mod +++ b/api/go.mod @@ -3,7 +3,6 @@ module github.com/openstack-k8s-operators/telemetry-operator/api go 1.24 require ( - github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba github.com/onsi/ginkgo/v2 v2.20.1 github.com/onsi/gomega v1.34.1 github.com/openstack-k8s-operators/infra-operator/apis v0.6.1-0.20250922155301-057562fb7182 diff --git a/api/go.sum b/api/go.sum index f1eebe53..e356f67e 100644 --- a/api/go.sum +++ b/api/go.sum @@ -46,8 +46,6 @@ github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQu github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba h1:P5Wgp2HfGfNPLCPpS+YqquKdrrl4tW0El7VX23D6vtg= -github.com/grafana/loki/operator/api/loki v0.0.0-20250910094332-a082b8a061ba/go.mod h1:OBAgJh0mLYRvziBzBKr4/anrPHqGY9qEfuNXCpnUNi0= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index 17587f8c..52f86702 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -22,7 +22,6 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/util" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - lokistackv1 "github.com/grafana/loki/operator/api/loki/v1" ) const ( @@ -44,6 +43,85 @@ const ( CloudKittyReplicas = 1 ) +// Reimplementation of loki-operator's Object storage related API. +// By doing this, we don't need to have a dependency on the loki-operator in +// the API module and it allows us to have all the fields optional due to +// worries about possible issues with upgrades when having required fields here + +type CASpec struct { + // Key is the data key of a ConfigMap containing a CA certificate. + // It needs to be in the same namespace as the CloudKitty custom resource. + // If empty, it defaults to "service-ca.crt". + // + // +kubebuilder:validation:optional + CAKey string `json:"caKey,omitempty"` + // CA is the name of a ConfigMap containing a CA certificate. + // It needs to be in the same namespace as the CloudKitty custom resource. + // + // +kubebuilder:validation:optional + CA string `json:"caName"` +} + +type ObjectStorageSchema struct { + // Version for writing and reading logs. + // + // +kubebuilder:validation:Optional + Version string `json:"version"` + + // EffectiveDate contains a date in YYYY-MM-DD format which is interpreted in the UTC time zone. + // + // The configuration always needs at least one schema that is currently valid. This means that when creating a new + // CloudKitty it is recommended to add a schema with the latest available version and an effective date of "yesterday". + // New schema versions added to the configuration always needs to be placed "in the future", so that Loki can start + // using it once the day rolls over. + // + // +kubebuilder:validation:Optional + EffectiveDate string `json:"effectiveDate"` +} + +type ObjectStorageSecretSpec struct { + // Type of object storage that should be used + // + // +kubebuilder:validation:Optional + Type string `json:"type"` + + // Name of a secret in the namespace configured for object storage secrets. + // + // +kubebuilder:validation:Optional + Name string `json:"name"` + + // CredentialMode can be used to set the desired credential mode for authenticating with the object storage. + // If this is not set, then the operator tries to infer the credential mode from the provided secret and its + // own configuration. + // + // +kubebuilder:validation:Optional + CredentialMode string `json:"credentialMode,omitempty"` +} + +type ObjectStorageTLSSpec struct { + CASpec `json:",inline"` +} + +type ObjectStorageSpec struct { + // Schemas for reading and writing logs. + // + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MinItems:=1 + // +kubebuilder:default:={{version:v11,effectiveDate:"2020-10-11"}} + Schemas []ObjectStorageSchema `json:"schemas"` + + // Secret for object storage authentication. + // Name of a secret in the same namespace as the CloudKitty custom resource. + // + // +kubebuilder:validation:Optional + Secret ObjectStorageSecretSpec `json:"secret"` + + // TLS configuration for reaching the object storage endpoint. + // + // +kubebuilder:validation:Optional + TLS *ObjectStorageTLSSpec `json:"tls,omitempty"` +} + type CloudKittySpecBase struct { CloudKittyTemplate `json:",inline"` @@ -110,7 +188,7 @@ type CloudKittySpecBase struct { // S3 related configuration passed to Loki // +kubebuilder:validation:Optional // +kubebuilder:default={secret: {name: "cloudkitty-loki-s3", type: "s3"}} - S3StorageConfig lokistackv1.ObjectStorageSpec `json:"s3StorageConfig"` + S3StorageConfig ObjectStorageSpec `json:"s3StorageConfig,omitempty"` // Storage class used for Loki // +kubebuilder:validation:Optional diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index e9f328c9..5ced2cd3 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -330,6 +330,21 @@ func (in *AutoscalingStatus) DeepCopy() *AutoscalingStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CASpec) DeepCopyInto(out *CASpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CASpec. +func (in *CASpec) DeepCopy() *CASpec { + if in == nil { + return nil + } + out := new(CASpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Ceilometer) DeepCopyInto(out *Ceilometer) { *out = *in @@ -1534,6 +1549,78 @@ func (in *MonitoringStack) DeepCopy() *MonitoringStack { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectStorageSchema) DeepCopyInto(out *ObjectStorageSchema) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectStorageSchema. +func (in *ObjectStorageSchema) DeepCopy() *ObjectStorageSchema { + if in == nil { + return nil + } + out := new(ObjectStorageSchema) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectStorageSecretSpec) DeepCopyInto(out *ObjectStorageSecretSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectStorageSecretSpec. +func (in *ObjectStorageSecretSpec) DeepCopy() *ObjectStorageSecretSpec { + if in == nil { + return nil + } + out := new(ObjectStorageSecretSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectStorageSpec) DeepCopyInto(out *ObjectStorageSpec) { + *out = *in + if in.Schemas != nil { + in, out := &in.Schemas, &out.Schemas + *out = make([]ObjectStorageSchema, len(*in)) + copy(*out, *in) + } + out.Secret = in.Secret + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(ObjectStorageTLSSpec) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectStorageSpec. +func (in *ObjectStorageSpec) DeepCopy() *ObjectStorageSpec { + if in == nil { + return nil + } + out := new(ObjectStorageSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectStorageTLSSpec) DeepCopyInto(out *ObjectStorageTLSSpec) { + *out = *in + out.CASpec = in.CASpec +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectStorageTLSSpec. +func (in *ObjectStorageTLSSpec) DeepCopy() *ObjectStorageTLSSpec { + if in == nil { + return nil + } + out := new(ObjectStorageTLSSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PasswordsSelector) DeepCopyInto(out *PasswordsSelector) { *out = *in diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml index 0787ea58..5d64d83f 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -599,46 +599,32 @@ spec: version: v11 description: Schemas for reading and writing logs. items: - description: ObjectStorageSchema defines a schema version and - the date when it will become effective. properties: effectiveDate: description: |- EffectiveDate contains a date in YYYY-MM-DD format which is interpreted in the UTC time zone. The configuration always needs at least one schema that is currently valid. This means that when creating a new - LokiStack it is recommended to add a schema with the latest available version and an effective date of "yesterday". + CloudKitty it is recommended to add a schema with the latest available version and an effective date of "yesterday". New schema versions added to the configuration always needs to be placed "in the future", so that Loki can start using it once the day rolls over. - pattern: ^([0-9]{4,})([-]([0-9]{2})){2}$ type: string version: description: Version for writing and reading logs. - enum: - - v11 - - v12 - - v13 type: string - required: - - effectiveDate - - version type: object minItems: 1 type: array secret: description: |- Secret for object storage authentication. - Name of a secret in the same namespace as the LokiStack custom resource. + Name of a secret in the same namespace as the CloudKitty custom resource. properties: credentialMode: description: |- CredentialMode can be used to set the desired credential mode for authenticating with the object storage. If this is not set, then the operator tries to infer the credential mode from the provided secret and its own configuration. - enum: - - static - - token - - token-cco type: string name: description: Name of a secret in the namespace configured @@ -646,16 +632,7 @@ spec: type: string type: description: Type of object storage that should be used - enum: - - azure - - gcs - - s3 - - swift - - alibabacloud type: string - required: - - name - - type type: object tls: description: TLS configuration for reaching the object storage @@ -664,19 +641,17 @@ spec: caKey: description: |- Key is the data key of a ConfigMap containing a CA certificate. - It needs to be in the same namespace as the LokiStack custom resource. + It needs to be in the same namespace as the CloudKitty custom resource. If empty, it defaults to "service-ca.crt". type: string caName: description: |- CA is the name of a ConfigMap containing a CA certificate. - It needs to be in the same namespace as the LokiStack custom resource. + It needs to be in the same namespace as the CloudKitty custom resource. type: string required: - caName type: object - required: - - secret type: object secret: default: osp-secret diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index af9bbd3c..d7c5e451 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -1165,46 +1165,32 @@ spec: version: v11 description: Schemas for reading and writing logs. items: - description: ObjectStorageSchema defines a schema version - and the date when it will become effective. properties: effectiveDate: description: |- EffectiveDate contains a date in YYYY-MM-DD format which is interpreted in the UTC time zone. The configuration always needs at least one schema that is currently valid. This means that when creating a new - LokiStack it is recommended to add a schema with the latest available version and an effective date of "yesterday". + CloudKitty it is recommended to add a schema with the latest available version and an effective date of "yesterday". New schema versions added to the configuration always needs to be placed "in the future", so that Loki can start using it once the day rolls over. - pattern: ^([0-9]{4,})([-]([0-9]{2})){2}$ type: string version: description: Version for writing and reading logs. - enum: - - v11 - - v12 - - v13 type: string - required: - - effectiveDate - - version type: object minItems: 1 type: array secret: description: |- Secret for object storage authentication. - Name of a secret in the same namespace as the LokiStack custom resource. + Name of a secret in the same namespace as the CloudKitty custom resource. properties: credentialMode: description: |- CredentialMode can be used to set the desired credential mode for authenticating with the object storage. If this is not set, then the operator tries to infer the credential mode from the provided secret and its own configuration. - enum: - - static - - token - - token-cco type: string name: description: Name of a secret in the namespace configured @@ -1212,16 +1198,7 @@ spec: type: string type: description: Type of object storage that should be used - enum: - - azure - - gcs - - s3 - - swift - - alibabacloud type: string - required: - - name - - type type: object tls: description: TLS configuration for reaching the object storage @@ -1230,19 +1207,17 @@ spec: caKey: description: |- Key is the data key of a ConfigMap containing a CA certificate. - It needs to be in the same namespace as the LokiStack custom resource. + It needs to be in the same namespace as the CloudKitty custom resource. If empty, it defaults to "service-ca.crt". type: string caName: description: |- CA is the name of a ConfigMap containing a CA certificate. - It needs to be in the same namespace as the LokiStack custom resource. + It needs to be in the same namespace as the CloudKitty custom resource. type: string required: - caName type: object - required: - - secret type: object secret: default: osp-secret diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index b4dd4531..d4e2980f 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -669,7 +669,10 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te }, } op, err := controllerutil.CreateOrPatch(ctx, r.Client, lokiStack, func() error { - desiredLokiStack := cloudkitty.LokiStack(instance, serviceLabels) + desiredLokiStack, err := cloudkitty.LokiStack(instance, serviceLabels) + if err != nil { + return err + } desiredLokiStack.Spec.DeepCopyInto(&lokiStack.Spec) lokiStack.ObjectMeta.Labels = serviceLabels err = controllerutil.SetControllerReference(instance, lokiStack, r.Scheme) diff --git a/pkg/cloudkitty/lokistack.go b/pkg/cloudkitty/lokistack.go index c7604dc1..216ed160 100644 --- a/pkg/cloudkitty/lokistack.go +++ b/pkg/cloudkitty/lokistack.go @@ -18,17 +18,91 @@ package cloudkitty import ( "fmt" + "slices" lokistackv1 "github.com/grafana/loki/operator/api/loki/v1" telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +func validateObjectStorageSpec(spec telemetryv1.ObjectStorageSpec) error { + for _, schema := range spec.Schemas { + if schema.EffectiveDate == "" { + return fmt.Errorf("Invalid CloudKitty spec. Field .spec.s3StorageConfig.schema.effectiveDate is required") + } + if schema.Version == "" { + return fmt.Errorf("Invalid CloudKitty spec. Field .spec.s3StorageConfig.schema.version is required") + } + } + + if spec.Secret.Name == "" { + return fmt.Errorf("Invalid CloudKitty spec. Field .spec.s3StorageConfig.secret.name is required") + } + + if spec.Secret.Type == "" { + return fmt.Errorf("Invalid CloudKitty spec. Field .spec.s3StorageConfig.secret.type is required") + } + validTypes := []string{"azure", "gcs", "s3", "swift", "alibabacloud"} + if !slices.Contains(validTypes, spec.Secret.Type) { + return fmt.Errorf("Invalid CloudKitty spec. Field .spec.s3StorageConfig.secret.type needs to be one of %s", validTypes) + } + + if spec.TLS != nil && spec.TLS.CASpec.CA == "" { + return fmt.Errorf("Invalid CloudKitty spec. Field .spec.s3StorageConfig.tls.caName is required") + } + + return nil +} + +func getLokiStackObjectStorageSpec(telemetryObjectStorageSpec telemetryv1.ObjectStorageSpec) lokistackv1.ObjectStorageSpec { + var result lokistackv1.ObjectStorageSpec + + if len(telemetryObjectStorageSpec.Schemas) == 0 { + // NOTE: if no schema is defined, use the same as defined in loki-operator. + result.Schemas = []lokistackv1.ObjectStorageSchema{ + { + Version: lokistackv1.ObjectStorageSchemaVersion("v11"), + EffectiveDate: lokistackv1.StorageSchemaEffectiveDate("2020-10-11"), + }, + } + } else { + for _, schema := range telemetryObjectStorageSpec.Schemas { + result.Schemas = append(result.Schemas, lokistackv1.ObjectStorageSchema{ + Version: lokistackv1.ObjectStorageSchemaVersion(schema.Version), + EffectiveDate: lokistackv1.StorageSchemaEffectiveDate(schema.EffectiveDate), + }) + } + } + + result.Secret.Type = lokistackv1.ObjectStorageSecretType(telemetryObjectStorageSpec.Secret.Type) + result.Secret.Name = telemetryObjectStorageSpec.Secret.Name + result.Secret.CredentialMode = lokistackv1.CredentialMode(telemetryObjectStorageSpec.Secret.CredentialMode) + + if telemetryObjectStorageSpec.TLS != nil { + result.TLS = &lokistackv1.ObjectStorageTLSSpec{ + CASpec: lokistackv1.CASpec{ + CAKey: telemetryObjectStorageSpec.TLS.CASpec.CAKey, + CA: telemetryObjectStorageSpec.TLS.CASpec.CA, + }, + } + if result.TLS.CASpec.CAKey == "" { + // NOTE: if no CAKey is defined, use the same as defined in loki-operator + result.TLS.CASpec.CAKey = "service-ca.crt" + } + } + + return result +} + // LokiStack defines a lokistack for cloudkitty func LokiStack( instance *telemetryv1.CloudKitty, labels map[string]string, -) *lokistackv1.LokiStack { +) (*lokistackv1.LokiStack, error) { + err := validateObjectStorageSpec(instance.Spec.S3StorageConfig) + if err != nil { + return nil, err + } lokiStack := &lokistackv1.LokiStack{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-lokistack", instance.Name), @@ -39,7 +113,7 @@ func LokiStack( // TODO: What size do we even want? I assume something // smallish since only rating interact with this Size: lokistackv1.LokiStackSizeType("1x.demo"), - Storage: instance.Spec.S3StorageConfig, + Storage: getLokiStackObjectStorageSpec(instance.Spec.S3StorageConfig), StorageClassName: instance.Spec.StorageClass, Tenants: &lokistackv1.TenantsSpec{ Mode: lokistackv1.Static, @@ -114,5 +188,5 @@ func LokiStack( }, }, } - return lokiStack + return lokiStack, nil } From d647fbf56c4f154cec02063292a402dd9de08143 Mon Sep 17 00:00:00 2001 From: mgirgisf Date: Wed, 24 Sep 2025 14:16:25 +0300 Subject: [PATCH 29/87] Update the Healthcheck.py --- controllers/cloudkittyproc_controller.go | 10 +- pkg/cloudkittyproc/statefulset.go | 42 ++--- templates/cloudkitty/bin/healthcheck.py | 189 +++++++---------------- 3 files changed, 78 insertions(+), 163 deletions(-) diff --git a/controllers/cloudkittyproc_controller.go b/controllers/cloudkittyproc_controller.go index af542394..cca14372 100644 --- a/controllers/cloudkittyproc_controller.go +++ b/controllers/cloudkittyproc_controller.go @@ -19,6 +19,7 @@ package controllers import ( "context" "fmt" + telemetryv1 "github.com/openstack-k8s-operators/telemetry-operator/api/v1beta1" "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkitty" "github.com/openstack-k8s-operators/telemetry-operator/pkg/cloudkittyproc" @@ -269,7 +270,7 @@ func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctr Name: parentCloudKittyName, Namespace: cr.Namespace, }, parentCloudKitty) - + // Only return a reconcile event if we are using the prometheus secret if parentCloudKitty.Spec.PrometheusHost == "" { name := client.ObjectKey{ @@ -820,8 +821,11 @@ func (r *CloudKittyProcReconciler) generateServiceConfigs( Namespace: instance.Namespace, Type: util.TemplateTypeConfig, InstanceType: instance.Kind, - CustomData: customData, - Labels: labels, + AdditionalTemplate: map[string]string{ + "healthcheck.py": "/cloudkitty/bin/healthcheck.py", + }, + CustomData: customData, + Labels: labels, }, } diff --git a/pkg/cloudkittyproc/statefulset.go b/pkg/cloudkittyproc/statefulset.go index cf4235b7..1fcae81f 100644 --- a/pkg/cloudkittyproc/statefulset.go +++ b/pkg/cloudkittyproc/statefulset.go @@ -24,12 +24,13 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" ) const ( // ServiceCommand - ServiceCommand = "/usr/local/bin/kolla_set_configs && /usr/local/bin/kolla_start" + // CloudKittyHCScript is the path to the health check script + CloudKittyHCScript = "/var/lib/openstack/bin/healthcheck.py" ) // StatefulSet func @@ -52,20 +53,19 @@ func StatefulSet( InitialDelaySeconds: 3, } - startupProbe := &corev1.Probe{ - TimeoutSeconds: 5, - FailureThreshold: 12, - PeriodSeconds: 5, - InitialDelaySeconds: 5, - } - args := []string{"-c", ServiceCommand} - var probeCommand string - livenessProbe.HTTPGet = &corev1.HTTPGetAction{ - Port: intstr.FromInt(8080), + //var probeCommand string + + //probeCommand = "/usr/local/bin/kolla_set_configs && /var/lib/openstack/bin/healthcheck.py --config-dir /etc/cloudkitty/cloudkitty.conf.d/" + + livenessProbe.Exec = &corev1.ExecAction{ + Command: []string{ + "/usr/bin/python3", + CloudKittyHCScript, + "--config-dir", + "/etc/cloudkitty/cloudkitty.conf.d/", + }, } - startupProbe.HTTPGet = livenessProbe.HTTPGet - probeCommand = "/usr/local/bin/kolla_set_configs && /var/lib/openstack/bin/healthcheck.py --config-dir /etc/cloudkitty/cloudkitty.conf.d/" envVars := map[string]env.Setter{} envVars["KOLLA_CONFIG_STRATEGY"] = env.SetValue("COPY_ALWAYS") @@ -113,22 +113,6 @@ func StatefulSet( VolumeMounts: volumeMounts, Resources: instance.Spec.Resources, LivenessProbe: livenessProbe, - StartupProbe: startupProbe, - }, - { - Name: "probe", - Command: []string{ - "/bin/bash", - }, - Args: []string{"-c", probeCommand}, - Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - Image: instance.Spec.ContainerImage, - SecurityContext: &corev1.SecurityContext{ - RunAsUser: &cloudKittyUser, - //RunAsGroup: &cloudKittyGroup, - }, - VolumeMounts: volumeMounts, - Resources: instance.Spec.Resources, }, }, Volumes: volumes, diff --git a/templates/cloudkitty/bin/healthcheck.py b/templates/cloudkitty/bin/healthcheck.py index 3ab36078..8daf7c39 100755 --- a/templates/cloudkitty/bin/healthcheck.py +++ b/templates/cloudkitty/bin/healthcheck.py @@ -1,154 +1,81 @@ #!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2025 Inc. +# All Rights Reserved. # -# Copyright 2022 Red Hat Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -from http import server -import signal -import socket import sys -import time -import threading import requests - from oslo_config import cfg +import psutil -SERVER_PORT = 8080 CONF = cfg.CONF +PROCESS_NAME = "cloudkitty-proc" - -class HTTPServerV6(server.HTTPServer): - address_family = socket.AF_INET6 - - -class HeartbeatServer(server.BaseHTTPRequestHandler): - - @staticmethod - def check_services(): - print("Starting health checks") - results = {} - - # Todo Database Endpoint Reachability - # Keystone Endpoint Reachability - try: - keystone_uri = CONF.keystone_authtoken.auth_url - response = requests.get(keystone_uri, timeout=5) - response.raise_for_status() - server_header = response.headers.get('Server', '').lower() - if 'keystone' in server_header: - results['keystone_endpoint'] = 'OK' - print("Keystone endpoint reachable and responsive.") - else: - results['keystone_endpoint'] = 'WARN' - print(f"Keystone endpoint reachable, but not a valid Keystone service: {keystone_uri}") - except requests.exceptions.RequestException as e: - results['keystone_endpoint'] = 'FAIL' - print(f"ERROR: Keystone endpoint check failed: {e}") - raise Exception('ERROR: Keystone check failed', e) - - # Prometheus Collector Endpoint Reachability +def check_process() -> tuple[int, str]: + # Return 0 if process with given name exists, else 1 with reason. + for proc in psutil.process_iter(attrs=["name"]): try: - prometheus_url = CONF.collector_prometheus.prometheus_url - insecure = CONF.collector_prometheus.insecure - cafile = CONF.collector_prometheus.cafile - verify_ssl = cafile if cafile and not insecure else not insecure - - response = requests.get(prometheus_url, timeout=5, verify=verify_ssl) - response.raise_for_status() - results['collector_endpoint'] = 'OK' - print("Prometheus collector endpoint reachable.") - except requests.exceptions.RequestException as e: - results['collector_endpoint'] = 'FAIL' - print(f"ERROR: Prometheus collector check failed: {e}") - raise Exception('ERROR: Prometheus collector check failed', e) - - def do_GET(self): - try: - self.check_services() - except Exception as exc: - self.send_error(500, exc.args[0], exc.args[1]) - return - - self.send_response(200) - self.send_header("Content-type", "text/html") - self.end_headers() - self.wfile.write('OK'.encode('utf-8')) - - -def get_stopper(server): - def stopper(signal_number=None, frame=None): - print("Stopping server.") - server.shutdown() - server.server_close() - print("Server stopped.") - sys.exit(0) - return stopper + if proc.info["name"] == PROCESS_NAME: + return 0, "" + except (psutil.NoSuchProcess, psutil.AccessDenied): + continue + return 1, f"Process {PROCESS_NAME} not found" + +def check_keystone() -> tuple[int, str]: + # Check Keystone endpoint reachability + try: + keystone_uri = CONF.keystone_authtoken.auth_url + response = requests.get(keystone_uri, timeout=5) + response.raise_for_status() + server_header = response.headers.get("Server", "").lower() + if "keystone" in server_header: + return 0, "" + return 1, f"Keystone reachable but not identified as Keystone: {keystone_uri}" + except requests.exceptions.RequestException as e: + return 1, f"Keystone endpoint check failed: {e}" + + +def run_checks() -> tuple[int, str]: + # Run all health checks and return aggregated result + checks = [check_process, check_keystone] + for check in checks: + rc, reason = check() + if rc != 0: + return rc, reason + return 0, "" if __name__ == "__main__": - # Register config options - cfg.CONF.register_group(cfg.OptGroup(name='database', title='Database connection options')) - cfg.CONF.register_opt(cfg.StrOpt('connection', default=None), group='database') - - cfg.CONF.register_group(cfg.OptGroup(name='keystone_authtoken', title='Keystone Auth Token Options')) - cfg.CONF.register_opt(cfg.StrOpt('auth_url', - default='https://keystone-internal.openstack.svc:5000'), - group='keystone_authtoken') + cfg.CONF.register_group(cfg.OptGroup(name="keystone_authtoken", title="Keystone Auth Token Options")) + cfg.CONF.register_opt( + cfg.StrOpt("auth_url", default="https://keystone-internal.openstack.svc:5000"), + group="keystone_authtoken", + ) - cfg.CONF.register_group(cfg.OptGroup(name='collector_prometheus', title='Prometheus Collector Options')) - cfg.CONF.register_opt(cfg.StrOpt('prometheus_url', - default='http://metric-storage-prometheus.openstack.svc:9090'), - group='collector_prometheus') - cfg.CONF.register_opt(cfg.BoolOpt('insecure', default=False), group='collector_prometheus') - cfg.CONF.register_opt(cfg.StrOpt('cafile', default=None), group='collector_prometheus') - - # Load configuration from file try: cfg.CONF(sys.argv[1:]) except cfg.ConfigFilesNotFoundError as e: - print(f"Health check failed: {e}", file=sys.stderr) - sys.exit(1) + print(f"Config load failed: {e}") + sys.exit(2) - # Detect IPv6 support for binding - hostname = socket.gethostname() try: - ipv6_address = socket.getaddrinfo(hostname, None, socket.AF_INET6) - except socket.gaierror: - ipv6_address = None - - if ipv6_address: - webServer = HTTPServerV6(("::", SERVER_PORT), HeartbeatServer) - else: - webServer = server.HTTPServer(("0.0.0.0", SERVER_PORT), HeartbeatServer) - - stop = get_stopper(webServer) + rc, reason = run_checks() + except Exception as ex: + rc, reason = 2, f"Unknown error: {ex}" - # Need to run the server on a different thread because its shutdown method - # will block if called from the same thread, and the signal handler must be - # on the main thread in Python. - thread = threading.Thread(target=webServer.serve_forever) - thread.daemon = True - thread.start() - print(f"CloudKitty Healthcheck Server started http://{hostname}:{SERVER_PORT}") - signal.signal(signal.SIGTERM, stop) - - try: - while True: - time.sleep(60) - except KeyboardInterrupt: - pass - finally: - stop() + if rc != 0: + print(reason) + sys.exit(rc) \ No newline at end of file From a3b4d44abea744f6aafa53661251c8967a356653 Mon Sep 17 00:00:00 2001 From: Jaromir Wysoglad Date: Fri, 26 Sep 2025 04:42:56 -0400 Subject: [PATCH 30/87] Add s3 config webhook validation --- api/v1beta1/cloudkitty_webhook.go | 129 +++++++++++++++++++++++++++++- api/v1beta1/telemetry_webhook.go | 40 ++++++++- 2 files changed, 165 insertions(+), 4 deletions(-) diff --git a/api/v1beta1/cloudkitty_webhook.go b/api/v1beta1/cloudkitty_webhook.go index 5ebd644a..dc3e0517 100644 --- a/api/v1beta1/cloudkitty_webhook.go +++ b/api/v1beta1/cloudkitty_webhook.go @@ -17,7 +17,13 @@ limitations under the License. package v1beta1 import ( + "fmt" + "slices" + + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -77,20 +83,139 @@ func (spec *CloudKittySpec) Default() { var _ webhook.Validator = &CloudKitty{} +func (r *ObjectStorageSpec) Validate(basePath *field.Path) field.ErrorList { + var allErrs field.ErrorList + + // NOTE: Having 0 schemas is allowed. LokiStack has a default for that + for _, s := range r.Schemas { + if s.EffectiveDate == "" { + allErrs = append( + allErrs, + field.Invalid( + basePath.Child("schemas").Child("effectiveDate"), "", "effectiveDate field should not be empty"), + ) + } + if s.Version == "" { + allErrs = append( + allErrs, + field.Invalid( + basePath.Child("schemas").Child("version"), "", "version field should not be empty"), + ) + } + } + + if r.Secret.Name == "" { + allErrs = append( + allErrs, + field.Invalid( + basePath.Child("secret").Child("name"), "", "name field should not be empty"), + ) + } + + if r.Secret.Type == "" { + allErrs = append( + allErrs, + field.Invalid( + basePath.Child("secret").Child("type"), "", "type field should not be empty"), + ) + } + validTypes := []string{"azure", "gcs", "s3", "swift", "alibabacloud"} + if !slices.Contains(validTypes, r.Secret.Type) { + allErrs = append( + allErrs, + field.Invalid( + basePath.Child("secret").Child("type"), r.Secret.Type, fmt.Sprintf("type field needs to be one of %s", validTypes)), + ) + } + + if r.TLS != nil && r.TLS.CASpec.CA == "" { + allErrs = append( + allErrs, + field.Invalid( + basePath.Child("tls").Child("caName"), "", "caName field should not be empty"), + ) + } + return allErrs +} + // ValidateCreate implements webhook.Validator so a webhook will be registered for the type func (r *CloudKitty) ValidateCreate() (admission.Warnings, error) { cloudKittyLog.Info("validate create", "name", r.Name) - // TODO(user): fill in your validation logic upon object creation. + allErrs := r.Spec.ValidateCreate(field.NewPath("spec"), r.Namespace) + + if len(allErrs) != 0 { + return nil, apierrors.NewInvalid( + schema.GroupKind{Group: "cloudkitties.telemetry.openstack.org", Kind: "CloudKitty"}, + r.Name, allErrs) + } + return nil, nil } +// ValidateCreate validates the CloudKittySpec during the webhook invocation. +func (r *CloudKittySpec) ValidateCreate(basePath *field.Path, namespace string) field.ErrorList { + return r.CloudKittySpecBase.ValidateCreate(basePath, namespace) +} + +// ValidateCreate validates the CloudKittySpecCore during the webhook invocation. It is +// expected to be called by the validation webhook in the higher level telemetry webhook +func (r *CloudKittySpecCore) ValidateCreate(basePath *field.Path, namespace string) field.ErrorList { + return r.CloudKittySpecBase.ValidateCreate(basePath, namespace) +} + +// ValidateCreate validates the CloudKittySpecBase during the webhook invocation. +func (r *CloudKittySpecBase) ValidateCreate(basePath *field.Path, namespace string) field.ErrorList { + var allErrs field.ErrorList + + allErrs = append(allErrs, r.S3StorageConfig.Validate(basePath.Child("s3StorageConfig"))...) + + // TODO: Add other CK spec field validations as needed + + return allErrs +} + // ValidateUpdate implements webhook.Validator so a webhook will be registered for the type func (r *CloudKitty) ValidateUpdate(old runtime.Object) (admission.Warnings, error) { + cloudKittyLog.Info("validate update", "name", r.Name) + oldCloudKitty, ok := old.(*CloudKitty) + if !ok || oldCloudKitty == nil { + return nil, apierrors.NewInternalError(fmt.Errorf("unable to convert existing object")) + } + + allErrs := r.Spec.ValidateUpdate(oldCloudKitty.Spec, field.NewPath("spec"), r.Namespace) + + if len(allErrs) != 0 { + return nil, apierrors.NewInvalid( + schema.GroupKind{Group: "cloudkitties.telemetry.openstack.org", Kind: "CloudKitty"}, + r.Name, allErrs) + } - // TODO(user): fill in your validation logic upon object update. return nil, nil + +} + +// ValidateCreate validates the CloudKittySpec during the webhook invocation. +func (r *CloudKittySpec) ValidateUpdate(old CloudKittySpec, basePath *field.Path, namespace string) field.ErrorList { + return r.CloudKittySpecBase.ValidateUpdate(old.CloudKittySpecBase, basePath, namespace) +} + +// ValidateUpdate validates the CloudKittySpecCore during the webhook invocation. It is +// expected to be called by the validation webhook in the higher level telemetry webhook +func (r *CloudKittySpecCore) ValidateUpdate(old CloudKittySpecCore, basePath *field.Path, namespace string) field.ErrorList { + return r.CloudKittySpecBase.ValidateUpdate(old.CloudKittySpecBase, basePath, namespace) +} + +// ValidateCreate validates the CloudKittySpecBase during the webhook invocation. +func (r *CloudKittySpecBase) ValidateUpdate(old CloudKittySpecBase, basePath *field.Path, namespace string) field.ErrorList { + var allErrs field.ErrorList + + allErrs = append(allErrs, r.S3StorageConfig.Validate(basePath.Child("s3StorageConfig"))...) + + // TODO: Add other CK spec field validations as needed + + return allErrs } // ValidateDelete implements webhook.Validator so a webhook will be registered for the type diff --git a/api/v1beta1/telemetry_webhook.go b/api/v1beta1/telemetry_webhook.go index 6d1d084b..c4d00805 100644 --- a/api/v1beta1/telemetry_webhook.go +++ b/api/v1beta1/telemetry_webhook.go @@ -148,12 +148,26 @@ func (r TelemetrySpec) ValidateCreate(basePath *field.Path, namespace string) fi var allErrs field.ErrorList allErrs = append(allErrs, r.ValidateTelemetryTopology(basePath, namespace)...) + if r.TelemetrySpecBase.CloudKitty.Enabled != nil && *r.TelemetrySpecBase.CloudKitty.Enabled { + allErrs = append(allErrs, r.TelemetrySpecBase.CloudKitty.CloudKittySpec.ValidateCreate(basePath.Child("cloudkitty"), namespace)...) + } + // TODO: Once we have CloudKittySectionCore, which uses the CloudKittySpecCore, the snippet above should be: + // if r.CloudKitty.Enabled != nil && *r.CloudKitty.Enabled { + // allErrs = append(allErrs, r.CloudKitty.CloudKittySpec.ValidateCreate(basePath.Child("cloudkitty"), namespace)...) + // } return allErrs } func (r TelemetrySpecCore) ValidateCreate(basePath *field.Path, namespace string) field.ErrorList { var allErrs field.ErrorList allErrs = append(allErrs, r.ValidateTelemetryTopology(basePath, namespace)...) + if r.TelemetrySpecBase.CloudKitty.Enabled != nil && *r.TelemetrySpecBase.CloudKitty.Enabled { + allErrs = append(allErrs, r.TelemetrySpecBase.CloudKitty.CloudKittySpec.ValidateCreate(basePath.Child("cloudkitty"), namespace)...) + } + // TODO: Once we have CloudKittySectionCore, which uses the CloudKittySpecCore, the snippet above should be: + // if r.CloudKitty.Enabled != nil && *r.CloudKitty.Enabled { + // allErrs = append(allErrs, r.CloudKitty.CloudKittySpecCore.ValidateCreate(basePath.Child("cloudkitty"), namespace)...) + // } return allErrs } @@ -178,12 +192,30 @@ func (r *Telemetry) ValidateUpdate(old runtime.Object) (admission.Warnings, erro } func (r TelemetrySpec) ValidateUpdate(old TelemetrySpec, basePath *field.Path, namespace string) field.ErrorList { - return r.ValidateCreate(basePath, namespace) + var allErrs field.ErrorList + + allErrs = append(allErrs, r.ValidateTelemetryTopology(basePath, namespace)...) + + if r.TelemetrySpecBase.CloudKitty.Enabled != nil && *r.TelemetrySpecBase.CloudKitty.Enabled { + allErrs = append(allErrs, r.TelemetrySpecBase.CloudKitty.CloudKittySpec.ValidateUpdate(old.TelemetrySpecBase.CloudKitty.CloudKittySpec, basePath.Child("cloudkitty"), namespace)...) + } + // TODO: Once we have CloudKittySectionCore, which uses the CloudKittySpecCore, the snippet above should be: + // if r.CloudKitty.Enabled != nil && *r.TelemetrySpecBase.CloudKitty.Enabled { + // allErrs = append(allErrs, r.CloudKitty.CloudKittySpec.ValidateUpdate(old.CloudKitty.CloudKittySpec, basePath.Child("cloudkitty"), namespace)...) + // } + return allErrs } -func (r TelemetrySpecCore) ValidateUpdate(old TelemetrySpec, basePath *field.Path, namespace string) field.ErrorList { +func (r TelemetrySpecCore) ValidateUpdate(old TelemetrySpecCore, basePath *field.Path, namespace string) field.ErrorList { var allErrs field.ErrorList allErrs = append(allErrs, r.ValidateTelemetryTopology(basePath, namespace)...) + if r.TelemetrySpecBase.CloudKitty.Enabled != nil && *r.TelemetrySpecBase.CloudKitty.Enabled { + allErrs = append(allErrs, r.TelemetrySpecBase.CloudKitty.CloudKittySpec.ValidateUpdate(old.TelemetrySpecBase.CloudKitty.CloudKittySpec, basePath.Child("cloudkitty"), namespace)...) + } + // TODO: Once we have CloudKittySectionCore, which uses the CloudKittySpecCore, the snippet above should be: + // if r.CloudKitty.Enabled != nil && *r.CloudKitty.Enabled { + // allErrs = append(allErrs, r.CloudKitty.CloudKittySpecCore.ValidateUpdate(old.CloudKitty.CloudKittySpecCore, basePath.Child("cloudkitty"), namespace)...) + // } return allErrs } @@ -217,6 +249,8 @@ func (spec *TelemetrySpecCore) ValidateTelemetryTopology(basePath *field.Path, n allErrs = append(allErrs, spec.Ceilometer.ValidateTopology(ceilPath, namespace)...) + // TODO: investigate whether a topology validation is needed for CloudKitty or MetricStorage + return allErrs } // ValidateTelemetryTopology - Returns an ErrorList if the Topology is referenced @@ -241,5 +275,7 @@ func (spec *TelemetrySpec) ValidateTelemetryTopology(basePath *field.Path, names allErrs = append(allErrs, spec.Ceilometer.ValidateTopology(ceilPath, namespace)...) + // TODO: investigate whether a topology validation is needed for CloudKitty or MetricStorage + return allErrs } From ac359d938872c47e1f3d7d8466b12f9d1f8a5236 Mon Sep 17 00:00:00 2001 From: jlarriba Date: Tue, 30 Sep 2025 15:30:19 +0200 Subject: [PATCH 31/87] Fix CloudKittySpecCore issues to avoid exposing the images to the control plane --- api/v1beta1/cloudkitty_types.go | 4 +-- api/v1beta1/telemetry_types.go | 30 ++++++++++++++++++--- api/v1beta1/telemetry_webhook.go | 40 +++++++++++----------------- api/v1beta1/zz_generated.deepcopy.go | 24 ++++++++++++++++- 4 files changed, 67 insertions(+), 31 deletions(-) diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index 52f86702..ad3e2c77 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -32,9 +32,9 @@ const ( CloudKittyGroupID = 42408 // CloudKittyAPIContainerImage - default fall-back image for CloudKitty API - CloudKittyAPIContainerImage = "quay.io/podified-master-centos9/openstack-cloudkitty-api:current-podified" + CloudKittyAPIContainerImage = "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-api:current" // CloudKittyProcContainerImage - default fall-back image for CloudKitty Processor - CloudKittyProcContainerImage = "quay.io/podified-master-centos9/openstack-cloudkitty-processor:current-podified" + CloudKittyProcContainerImage = "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-processor:current" // CloudKittyDbSyncHash hash CKDbSyncHash = "ckdbsync" // CKStorageInitHash hash diff --git a/api/v1beta1/telemetry_types.go b/api/v1beta1/telemetry_types.go index 46373586..9e8e895a 100644 --- a/api/v1beta1/telemetry_types.go +++ b/api/v1beta1/telemetry_types.go @@ -61,6 +61,10 @@ type TelemetrySpec struct { // +kubebuilder:validation:Optional // Ceilometer - Parameters related to the ceilometer service Ceilometer CeilometerSection `json:"ceilometer,omitempty"` + + // +kubebuilder:validation:Optional + // CloudKitty - Parameters related to the cloudkitty service + CloudKitty CloudKittySection `json:"cloudkitty,omitempty"` } // TelemetrySpecCore defines the desired state of Telemetry. This version has no image parameters and is used by OpenStackControlplane @@ -74,6 +78,10 @@ type TelemetrySpecCore struct { // +kubebuilder:validation:Optional // Ceilometer - Parameters related to the ceilometer service Ceilometer CeilometerSectionCore `json:"ceilometer,omitempty"` + + // +kubebuilder:validation:Optional + // CloudKitty - Parameters related to the cloudkitty service + CloudKitty CloudKittySectionCore `json:"cloudkitty,omitempty"` } // TelemetrySpecBase - @@ -86,10 +94,6 @@ type TelemetrySpecBase struct { // Logging - Parameters related to the logging Logging LoggingSection `json:"logging,omitempty"` - // +kubebuilder:validation:Optional - // CloudKitty - Parameters related to the cloudkitty service - CloudKitty CloudKittySection `json:"cloudkitty,omitempty"` - // +kubebuilder:validation:Optional // NodeSelector to target subset of worker nodes running this service NodeSelector *map[string]string `json:"nodeSelector,omitempty"` @@ -198,6 +202,20 @@ type CloudKittySection struct { CloudKittySpec `json:",inline"` } +// CloudKittySpec defines the desired state of the cloudkitty service +type CloudKittySectionCore struct { + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:booleanSwitch"} + // Enabled - Whether OpenStack CloudKitty service should be deployed and managed + Enabled *bool `json:"enabled"` + + // +kubebuilder:validation:Optional + //+operator-sdk:csv:customresourcedefinitions:type=spec + // Template - Overrides to use when creating the OpenStack CloudKitty service + CloudKittySpecCore `json:",inline"` +} + // TelemetryStatus defines the observed state of Telemetry type TelemetryStatus struct { // Map of hashes to track e.g. job status @@ -265,6 +283,10 @@ func SetupDefaultsTelemetry() { AodhEvaluatorContainerImageURL: util.GetEnvVar("RELATED_IMAGE_AODH_EVALUATOR_IMAGE_URL_DEFAULT", AodhEvaluatorContainerImage), AodhNotifierContainerImageURL: util.GetEnvVar("RELATED_IMAGE_AODH_NOTIFIER_IMAGE_URL_DEFAULT", AodhNotifierContainerImage), AodhListenerContainerImageURL: util.GetEnvVar("RELATED_IMAGE_AODH_LISTENER_IMAGE_URL_DEFAULT", AodhListenerContainerImage), + + // CloudKitty + CloudKittyAPIContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CLOUDKITTY_API_IMAGE_URL_DEFAULT", CloudKittyAPIContainerImage), + CloudKittyProcContainerImageURL: util.GetEnvVar("RELATED_IMAGE_CLOUDKITTY_PROC_IMAGE_URL_DEFAULT", CloudKittyProcContainerImage), } SetupTelemetryDefaults(telemetryDefaults) diff --git a/api/v1beta1/telemetry_webhook.go b/api/v1beta1/telemetry_webhook.go index c4d00805..5a9c3e20 100644 --- a/api/v1beta1/telemetry_webhook.go +++ b/api/v1beta1/telemetry_webhook.go @@ -46,6 +46,8 @@ type TelemetryDefaults struct { AodhEvaluatorContainerImageURL string AodhNotifierContainerImageURL string AodhListenerContainerImageURL string + CloudKittyAPIContainerImageURL string + CloudKittyProcContainerImageURL string } var telemetryDefaults TelemetryDefaults @@ -115,6 +117,12 @@ func (spec *TelemetrySpec) Default() { if spec.Autoscaling.AutoscalingSpec.Aodh.ListenerImage == "" { spec.Autoscaling.AutoscalingSpec.Aodh.ListenerImage = telemetryDefaults.AodhListenerContainerImageURL } + if spec.CloudKitty.CloudKittyAPI.ContainerImage == "" { + spec.CloudKitty.CloudKittyAPI.ContainerImage = telemetryDefaults.CloudKittyAPIContainerImageURL + } + if spec.CloudKitty.CloudKittyProc.ContainerImage == "" { + spec.CloudKitty.CloudKittyProc.ContainerImage = telemetryDefaults.CloudKittyProcContainerImageURL + } } // Default - set defaults for this Telemetry spec core @@ -148,26 +156,18 @@ func (r TelemetrySpec) ValidateCreate(basePath *field.Path, namespace string) fi var allErrs field.ErrorList allErrs = append(allErrs, r.ValidateTelemetryTopology(basePath, namespace)...) - if r.TelemetrySpecBase.CloudKitty.Enabled != nil && *r.TelemetrySpecBase.CloudKitty.Enabled { - allErrs = append(allErrs, r.TelemetrySpecBase.CloudKitty.CloudKittySpec.ValidateCreate(basePath.Child("cloudkitty"), namespace)...) + if r.CloudKitty.Enabled != nil && *r.CloudKitty.Enabled { + allErrs = append(allErrs, r.CloudKitty.CloudKittySpec.ValidateCreate(basePath.Child("cloudkitty"), namespace)...) } - // TODO: Once we have CloudKittySectionCore, which uses the CloudKittySpecCore, the snippet above should be: - // if r.CloudKitty.Enabled != nil && *r.CloudKitty.Enabled { - // allErrs = append(allErrs, r.CloudKitty.CloudKittySpec.ValidateCreate(basePath.Child("cloudkitty"), namespace)...) - // } return allErrs } func (r TelemetrySpecCore) ValidateCreate(basePath *field.Path, namespace string) field.ErrorList { var allErrs field.ErrorList allErrs = append(allErrs, r.ValidateTelemetryTopology(basePath, namespace)...) - if r.TelemetrySpecBase.CloudKitty.Enabled != nil && *r.TelemetrySpecBase.CloudKitty.Enabled { - allErrs = append(allErrs, r.TelemetrySpecBase.CloudKitty.CloudKittySpec.ValidateCreate(basePath.Child("cloudkitty"), namespace)...) + if r.CloudKitty.Enabled != nil && *r.CloudKitty.Enabled { + allErrs = append(allErrs, r.CloudKitty.CloudKittySpecCore.ValidateCreate(basePath.Child("cloudkitty"), namespace)...) } - // TODO: Once we have CloudKittySectionCore, which uses the CloudKittySpecCore, the snippet above should be: - // if r.CloudKitty.Enabled != nil && *r.CloudKitty.Enabled { - // allErrs = append(allErrs, r.CloudKitty.CloudKittySpecCore.ValidateCreate(basePath.Child("cloudkitty"), namespace)...) - // } return allErrs } @@ -196,26 +196,18 @@ func (r TelemetrySpec) ValidateUpdate(old TelemetrySpec, basePath *field.Path, n allErrs = append(allErrs, r.ValidateTelemetryTopology(basePath, namespace)...) - if r.TelemetrySpecBase.CloudKitty.Enabled != nil && *r.TelemetrySpecBase.CloudKitty.Enabled { - allErrs = append(allErrs, r.TelemetrySpecBase.CloudKitty.CloudKittySpec.ValidateUpdate(old.TelemetrySpecBase.CloudKitty.CloudKittySpec, basePath.Child("cloudkitty"), namespace)...) + if r.CloudKitty.Enabled != nil && *r.CloudKitty.Enabled { + allErrs = append(allErrs, r.CloudKitty.CloudKittySpec.ValidateUpdate(old.CloudKitty.CloudKittySpec, basePath.Child("cloudkitty"), namespace)...) } - // TODO: Once we have CloudKittySectionCore, which uses the CloudKittySpecCore, the snippet above should be: - // if r.CloudKitty.Enabled != nil && *r.TelemetrySpecBase.CloudKitty.Enabled { - // allErrs = append(allErrs, r.CloudKitty.CloudKittySpec.ValidateUpdate(old.CloudKitty.CloudKittySpec, basePath.Child("cloudkitty"), namespace)...) - // } return allErrs } func (r TelemetrySpecCore) ValidateUpdate(old TelemetrySpecCore, basePath *field.Path, namespace string) field.ErrorList { var allErrs field.ErrorList allErrs = append(allErrs, r.ValidateTelemetryTopology(basePath, namespace)...) - if r.TelemetrySpecBase.CloudKitty.Enabled != nil && *r.TelemetrySpecBase.CloudKitty.Enabled { - allErrs = append(allErrs, r.TelemetrySpecBase.CloudKitty.CloudKittySpec.ValidateUpdate(old.TelemetrySpecBase.CloudKitty.CloudKittySpec, basePath.Child("cloudkitty"), namespace)...) + if r.CloudKitty.Enabled != nil && *r.CloudKitty.Enabled { + allErrs = append(allErrs, r.CloudKitty.CloudKittySpecCore.ValidateUpdate(old.CloudKitty.CloudKittySpecCore, basePath.Child("cloudkitty"), namespace)...) } - // TODO: Once we have CloudKittySectionCore, which uses the CloudKittySpecCore, the snippet above should be: - // if r.CloudKitty.Enabled != nil && *r.CloudKitty.Enabled { - // allErrs = append(allErrs, r.CloudKitty.CloudKittySpecCore.ValidateUpdate(old.CloudKitty.CloudKittySpecCore, basePath.Child("cloudkitty"), namespace)...) - // } return allErrs } diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 5ced2cd3..3a99ea0f 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1042,6 +1042,27 @@ func (in *CloudKittySection) DeepCopy() *CloudKittySection { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CloudKittySectionCore) DeepCopyInto(out *CloudKittySectionCore) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + in.CloudKittySpecCore.DeepCopyInto(&out.CloudKittySpecCore) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudKittySectionCore. +func (in *CloudKittySectionCore) DeepCopy() *CloudKittySectionCore { + if in == nil { + return nil + } + out := new(CloudKittySectionCore) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CloudKittyServiceTemplate) DeepCopyInto(out *CloudKittyServiceTemplate) { *out = *in @@ -1752,6 +1773,7 @@ func (in *TelemetrySpec) DeepCopyInto(out *TelemetrySpec) { in.TelemetrySpecBase.DeepCopyInto(&out.TelemetrySpecBase) in.Autoscaling.DeepCopyInto(&out.Autoscaling) in.Ceilometer.DeepCopyInto(&out.Ceilometer) + in.CloudKitty.DeepCopyInto(&out.CloudKitty) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TelemetrySpec. @@ -1769,7 +1791,6 @@ func (in *TelemetrySpecBase) DeepCopyInto(out *TelemetrySpecBase) { *out = *in in.MetricStorage.DeepCopyInto(&out.MetricStorage) in.Logging.DeepCopyInto(&out.Logging) - in.CloudKitty.DeepCopyInto(&out.CloudKitty) if in.NodeSelector != nil { in, out := &in.NodeSelector, &out.NodeSelector *out = new(map[string]string) @@ -1804,6 +1825,7 @@ func (in *TelemetrySpecCore) DeepCopyInto(out *TelemetrySpecCore) { in.TelemetrySpecBase.DeepCopyInto(&out.TelemetrySpecBase) in.Autoscaling.DeepCopyInto(&out.Autoscaling) in.Ceilometer.DeepCopyInto(&out.Ceilometer) + in.CloudKitty.DeepCopyInto(&out.CloudKitty) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TelemetrySpecCore. From 020bd70758b454f4f03f761350467eb9337e2b6e Mon Sep 17 00:00:00 2001 From: jlarriba Date: Wed, 1 Oct 2025 17:33:26 +0200 Subject: [PATCH 32/87] Fix PrometheusTLS connection --- controllers/cloudkitty_controller.go | 8 ++++++++ controllers/cloudkittyapi_controller.go | 2 +- controllers/cloudkittyproc_controller.go | 2 +- templates/cloudkitty/config/cloudkitty.conf | 4 ++-- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index 77f0e780..63334ed8 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -1138,6 +1138,14 @@ func (r *CloudKittyReconciler) generateServiceConfigs( templateParameters["CAFile"] = tls.DownstreamTLSCABundlePath } + // Set Prometheus TLS configuration + templateParameters["PrometheusTLS"] = instance.Status.PrometheusTLS + if instance.Status.PrometheusTLS { + // For operator-managed Prometheus or user-deployed Prometheus with custom CA, + // use the downstream TLS CA bundle path + templateParameters["PrometheusCAFile"] = tls.DownstreamTLSCABundlePath + } + // create httpd vhost template parameters httpdVhostConfig := map[string]interface{}{} for _, endpt := range []service.Endpoint{service.EndpointInternal, service.EndpointPublic} { diff --git a/controllers/cloudkittyapi_controller.go b/controllers/cloudkittyapi_controller.go index 949b4ee4..0171dbe4 100644 --- a/controllers/cloudkittyapi_controller.go +++ b/controllers/cloudkittyapi_controller.go @@ -295,7 +295,7 @@ func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl Name: parentCloudKittyName, Namespace: cr.Namespace, }, parentCloudKitty) - + // Only return a reconcile event if we are using the prometheus secret if parentCloudKitty.Spec.PrometheusHost == "" { name := client.ObjectKey{ diff --git a/controllers/cloudkittyproc_controller.go b/controllers/cloudkittyproc_controller.go index cca14372..e92ace6f 100644 --- a/controllers/cloudkittyproc_controller.go +++ b/controllers/cloudkittyproc_controller.go @@ -277,7 +277,7 @@ func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctr Namespace: namespace, Name: cr.Name, } - Log.Info(fmt.Sprintf("Secret %s is used by CloudKittyAPI CR %s", secretName, cr.Name)) + Log.Info(fmt.Sprintf("Secret %s is used by CloudKittyProc CR %s", secretName, cr.Name)) result = append(result, reconcile.Request{NamespacedName: name}) } } diff --git a/templates/cloudkitty/config/cloudkitty.conf b/templates/cloudkitty/config/cloudkitty.conf index 37bd1849..1e549bef 100644 --- a/templates/cloudkitty/config/cloudkitty.conf +++ b/templates/cloudkitty/config/cloudkitty.conf @@ -46,9 +46,9 @@ collector = prometheus scope_key = project [collector_prometheus] -{{- if .TLS }} +{{- if .PrometheusTLS }} prometheus_url = https://{{ .PrometheusHost }}:{{ .PrometheusPort }}/api/v1 -cafile = {{ .CAFile }} +cafile = {{ .PrometheusCAFile }} insecure = false {{- else }} prometheus_url = http://{{ .PrometheusHost }}:{{ .PrometheusPort }}/api/v1 From bed9eb4c01052f999c6425f1c43972e08ac46456 Mon Sep 17 00:00:00 2001 From: jlarriba Date: Thu, 2 Oct 2025 09:58:20 +0200 Subject: [PATCH 33/87] Fix healthchecks --- pkg/cloudkittyproc/statefulset.go | 4 +-- templates/cloudkitty/bin/healthcheck.py | 41 ++++--------------------- 2 files changed, 7 insertions(+), 38 deletions(-) diff --git a/pkg/cloudkittyproc/statefulset.go b/pkg/cloudkittyproc/statefulset.go index 1fcae81f..48e90200 100644 --- a/pkg/cloudkittyproc/statefulset.go +++ b/pkg/cloudkittyproc/statefulset.go @@ -49,7 +49,7 @@ func StatefulSet( livenessProbe := &corev1.Probe{ // TODO might need tuning TimeoutSeconds: 5, - PeriodSeconds: 3, + PeriodSeconds: 5, InitialDelaySeconds: 3, } @@ -62,8 +62,6 @@ func StatefulSet( Command: []string{ "/usr/bin/python3", CloudKittyHCScript, - "--config-dir", - "/etc/cloudkitty/cloudkitty.conf.d/", }, } diff --git a/templates/cloudkitty/bin/healthcheck.py b/templates/cloudkitty/bin/healthcheck.py index 8daf7c39..29995114 100755 --- a/templates/cloudkitty/bin/healthcheck.py +++ b/templates/cloudkitty/bin/healthcheck.py @@ -16,41 +16,24 @@ # limitations under the License. import sys -import requests -from oslo_config import cfg import psutil -CONF = cfg.CONF -PROCESS_NAME = "cloudkitty-proc" - def check_process() -> tuple[int, str]: - # Return 0 if process with given name exists, else 1 with reason. - for proc in psutil.process_iter(attrs=["name"]): + # Return 0 if cloudkitty-processor process with given cmdline exists, else 1 with reason. + for proc in psutil.process_iter(attrs=["name", "cmdline"]): try: - if proc.info["name"] == PROCESS_NAME: + cmdline = proc.info.get("cmdline", []) + if cmdline and any("cloudkitty-processor" in arg for arg in cmdline): return 0, "" except (psutil.NoSuchProcess, psutil.AccessDenied): continue - return 1, f"Process {PROCESS_NAME} not found" - -def check_keystone() -> tuple[int, str]: - # Check Keystone endpoint reachability - try: - keystone_uri = CONF.keystone_authtoken.auth_url - response = requests.get(keystone_uri, timeout=5) - response.raise_for_status() - server_header = response.headers.get("Server", "").lower() - if "keystone" in server_header: - return 0, "" - return 1, f"Keystone reachable but not identified as Keystone: {keystone_uri}" - except requests.exceptions.RequestException as e: - return 1, f"Keystone endpoint check failed: {e}" + return 1, "CloudKitty processor process not found" def run_checks() -> tuple[int, str]: # Run all health checks and return aggregated result - checks = [check_process, check_keystone] + checks = [check_process] for check in checks: rc, reason = check() if rc != 0: @@ -59,18 +42,6 @@ def run_checks() -> tuple[int, str]: if __name__ == "__main__": - cfg.CONF.register_group(cfg.OptGroup(name="keystone_authtoken", title="Keystone Auth Token Options")) - cfg.CONF.register_opt( - cfg.StrOpt("auth_url", default="https://keystone-internal.openstack.svc:5000"), - group="keystone_authtoken", - ) - - try: - cfg.CONF(sys.argv[1:]) - except cfg.ConfigFilesNotFoundError as e: - print(f"Config load failed: {e}") - sys.exit(2) - try: rc, reason = run_checks() except Exception as ex: From 04198b2088ad4d324d5de8d2f3b6c1219470538e Mon Sep 17 00:00:00 2001 From: jlarriba Date: Tue, 7 Oct 2025 15:57:00 +0200 Subject: [PATCH 34/87] Fix pre-commit --- .../telemetry.openstack.org_cloudkitties.yaml | 3 +- .../telemetry.openstack.org_telemetries.yaml | 3 +- api/v1beta1/cloudkitty_types.go | 5 +- .../telemetry.openstack.org_cloudkitties.yaml | 3 +- .../telemetry.openstack.org_telemetries.yaml | 3 +- controllers/cloudkitty_controller.go | 29 ++++++---- controllers/cloudkittyapi_controller.go | 19 ++++--- controllers/cloudkittyproc_controller.go | 19 ++++--- controllers/metricstorage_controller.go | 55 +++---------------- controllers/telemetry_controller.go | 2 +- pkg/cloudkitty/const.go | 10 +++- pkg/cloudkitty/dbsync.go | 2 + pkg/cloudkitty/lokistack.go | 38 +++++++++---- pkg/cloudkitty/storageinit.go | 2 + pkg/cloudkittyapi/const.go | 1 + pkg/cloudkittyproc/const.go | 1 + pkg/utils/utils.go | 6 +- templates/cloudkitty/bin/healthcheck.py | 2 +- 18 files changed, 102 insertions(+), 101 deletions(-) diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml index 5d64d83f..13359220 100644 --- a/api/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -615,6 +615,7 @@ spec: type: object minItems: 1 type: array + x-kubernetes-list-type: atomic secret: description: |- Secret for object storage authentication. @@ -649,8 +650,6 @@ spec: CA is the name of a ConfigMap containing a CA certificate. It needs to be in the same namespace as the CloudKitty custom resource. type: string - required: - - caName type: object type: object secret: diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index d7c5e451..fb0bb10c 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -1181,6 +1181,7 @@ spec: type: object minItems: 1 type: array + x-kubernetes-list-type: atomic secret: description: |- Secret for object storage authentication. @@ -1215,8 +1216,6 @@ spec: CA is the name of a ConfigMap containing a CA certificate. It needs to be in the same namespace as the CloudKitty custom resource. type: string - required: - - caName type: object type: object secret: diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index ad3e2c77..957c7b7e 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -59,7 +59,7 @@ type CASpec struct { // It needs to be in the same namespace as the CloudKitty custom resource. // // +kubebuilder:validation:optional - CA string `json:"caName"` + CA string `json:"caName,omitempty"` } type ObjectStorageSchema struct { @@ -108,6 +108,7 @@ type ObjectStorageSpec struct { // +kubebuilder:validation:Optional // +kubebuilder:validation:MinItems:=1 // +kubebuilder:default:={{version:v11,effectiveDate:"2020-10-11"}} + // +listType=atomic Schemas []ObjectStorageSchema `json:"schemas"` // Secret for object storage authentication. @@ -188,7 +189,7 @@ type CloudKittySpecBase struct { // S3 related configuration passed to Loki // +kubebuilder:validation:Optional // +kubebuilder:default={secret: {name: "cloudkitty-loki-s3", type: "s3"}} - S3StorageConfig ObjectStorageSpec `json:"s3StorageConfig,omitempty"` + S3StorageConfig ObjectStorageSpec `json:"s3StorageConfig"` // Storage class used for Loki // +kubebuilder:validation:Optional diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml index 5d64d83f..13359220 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -615,6 +615,7 @@ spec: type: object minItems: 1 type: array + x-kubernetes-list-type: atomic secret: description: |- Secret for object storage authentication. @@ -649,8 +650,6 @@ spec: CA is the name of a ConfigMap containing a CA certificate. It needs to be in the same namespace as the CloudKitty custom resource. type: string - required: - - caName type: object type: object secret: diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index d7c5e451..fb0bb10c 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -1181,6 +1181,7 @@ spec: type: object minItems: 1 type: array + x-kubernetes-list-type: atomic secret: description: |- Secret for object storage authentication. @@ -1215,8 +1216,6 @@ spec: CA is the name of a ConfigMap containing a CA certificate. It needs to be in the same namespace as the CloudKitty custom resource. type: string - required: - - caName type: object type: object secret: diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index 63334ed8..7f19874f 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -131,7 +131,7 @@ func (r *CloudKittyReconciler) Reconcile(ctx context.Context, req ctrl.Request) // Fetch the CloudKitty instance instance := &telemetryv1.CloudKitty{} - err := r.Client.Get(ctx, req.NamespacedName, instance) + err := r.Get(ctx, req.NamespacedName, instance) if err != nil { if k8s_errors.IsNotFound(err) { // Request object not found, could have been deleted after reconcile request. @@ -235,7 +235,8 @@ func (r *CloudKittyReconciler) Reconcile(ctx context.Context, req ctrl.Request) // fields to index to reconcile when change const ( - cloudKittyPasswordSecretField = ".spec.secret" + cloudKittyPasswordSecretField = ".spec.secret" + //nolint:gosec // Not hardcoded credentials, just field name cloudKittyCaBundleSecretNameField = ".spec.tls.caBundleSecretName" cloudKittyTLSAPIInternalField = ".spec.tls.api.internal.secretName" cloudKittyTLSAPIPublicField = ".spec.tls.api.public.secretName" @@ -280,7 +281,7 @@ func (r *CloudKittyReconciler) SetupWithManager(mgr ctrl.Manager) error { listOpts := []client.ListOption{ client.InNamespace(o.GetNamespace()), } - if err := r.Client.List(ctx, cloudkitties, listOpts...); err != nil { + if err := r.List(ctx, cloudkitties, listOpts...); err != nil { Log.Error(err, "Unable to retrieve CloudKitty CRs %v") return nil } @@ -316,7 +317,7 @@ func (r *CloudKittyReconciler) SetupWithManager(mgr ctrl.Manager) error { listOpts := []client.ListOption{ client.InNamespace(o.GetNamespace()), } - if err := r.Client.List(ctx, cloudkitties, listOpts...); err != nil { + if err := r.List(ctx, cloudkitties, listOpts...); err != nil { Log.Error(err, "Unable to retrieve CloudKitty CRs %w") return nil } @@ -374,7 +375,7 @@ func (r *CloudKittyReconciler) findObjectForSrc(ctx context.Context, src client. listOps := &client.ListOptions{ Namespace: src.GetNamespace(), } - err := r.Client.List(ctx, crList, listOps) + err := r.List(ctx, crList, listOps) if err != nil { l.Error(err, fmt.Sprintf("listing %s for namespace: %s", crList.GroupVersionKind().Kind, src.GetNamespace())) return requests @@ -609,7 +610,9 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te condition.SeverityError, telemetryv1.CloudKittyClientCertReadyErrorMessage, err.Error())) - return ctrl.Result{}, err + return ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + return ctrlResult, nil } cms := []util.Template{ @@ -641,7 +644,7 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te instance.Status.Conditions.MarkTrue(telemetryv1.CloudKittyClientCertReadyCondition, telemetryv1.CloudKittyClientCertReadyMessage) // Deploy Loki - var eventHandler handler.EventHandler = handler.EnqueueRequestForOwner( + var eventHandler = handler.EnqueueRequestForOwner( r.Scheme, r.RESTMapper, &telemetryv1.CloudKitty{}, @@ -649,7 +652,7 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te ) err = utils.EnsureWatches( - (*utils.ConditionalWatchingReconciler)(r), ctx, + ctx, (*utils.ConditionalWatchingReconciler)(r), "lokistacks.loki.grafana.com", &lokistackv1.LokiStack{}, eventHandler, helper, ) @@ -674,7 +677,7 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te return err } desiredLokiStack.Spec.DeepCopyInto(&lokiStack.Spec) - lokiStack.ObjectMeta.Labels = serviceLabels + lokiStack.Labels = serviceLabels err = controllerutil.SetControllerReference(instance, lokiStack, r.Scheme) return err }) @@ -911,6 +914,7 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te condition.SeverityInfo, condition.NetworkAttachmentsReadyWaitingMessage, netAtt)) + //nolint:err113 // Using condition message format from lib-common return cloudkitty.ResultRequeue, fmt.Errorf(condition.NetworkAttachmentsReadyWaitingMessage, netAtt) } instance.Status.Conditions.Set(condition.FalseCondition( @@ -1036,7 +1040,7 @@ func (r *CloudKittyReconciler) generateServiceConfigs( labels := labels.GetLabels(instance, labels.GetGroupLabel(cloudkitty.ServiceName), serviceLabels) var tlsCfg *tls.Service - if instance.Spec.CloudKittyAPI.TLS.Ca.CaBundleSecretName != "" { + if instance.Spec.CloudKittyAPI.TLS.CaBundleSecretName != "" { tlsCfg = &tls.Service{} } @@ -1072,7 +1076,7 @@ func (r *CloudKittyReconciler) generateServiceConfigs( if instance.Spec.PrometheusHost == "" { // We're using MetricStorage for Prometheus. prometheusEndpointSecret := &corev1.Secret{} - err = r.Client.Get(ctx, client.ObjectKey{ + err = r.Get(ctx, client.ObjectKey{ Name: cloudkitty.PrometheusEndpointSecret, Namespace: instance.Namespace, }, prometheusEndpointSecret) @@ -1085,10 +1089,11 @@ func (r *CloudKittyReconciler) generateServiceConfigs( if err != nil { return err } + //nolint:gosec // G109: Port number is read from a secret and validated to be within valid range instance.Status.PrometheusPort = int32(port) metricStorage := &telemetryv1.MetricStorage{} - err = r.Client.Get(ctx, client.ObjectKey{ + err = r.Get(ctx, client.ObjectKey{ Namespace: instance.Namespace, Name: telemetryv1.DefaultServiceName, }, metricStorage) diff --git a/controllers/cloudkittyapi_controller.go b/controllers/cloudkittyapi_controller.go index 0171dbe4..d4921e1a 100644 --- a/controllers/cloudkittyapi_controller.go +++ b/controllers/cloudkittyapi_controller.go @@ -112,7 +112,7 @@ func (r *CloudKittyAPIReconciler) Reconcile(ctx context.Context, req ctrl.Reques // Fetch the CloudKittyAPI instance instance := &telemetryv1.CloudKittyAPI{} - err := r.Client.Get(ctx, req.NamespacedName, instance) + err := r.Get(ctx, req.NamespacedName, instance) if err != nil { if k8s_errors.IsNotFound(err) { // Request object not found, could have been deleted after reconcile request. @@ -224,8 +224,8 @@ func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl // Watch for changes to secrets we don't own. Global secrets // (e.g. TransportURLSecret) are handled by the main cloudkitty controller. secretFn := func(_ context.Context, o client.Object) []reconcile.Request { - var namespace string = o.GetNamespace() - var secretName string = o.GetName() + var namespace = o.GetNamespace() + var secretName = o.GetName() result := []reconcile.Request{} // get all API CRs @@ -233,7 +233,7 @@ func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl listOpts := []client.ListOption{ client.InNamespace(namespace), } - if err := r.Client.List(context.Background(), apis, listOpts...); err != nil { + if err := r.List(context.Background(), apis, listOpts...); err != nil { Log.Error(err, "Unable to retrieve API CRs %v") return nil } @@ -291,7 +291,7 @@ func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl // Fetch the parent CloudKitty instance parentCloudKitty := &telemetryv1.CloudKitty{} - _ = r.Client.Get(ctx, types.NamespacedName{ + _ = r.Get(ctx, types.NamespacedName{ Name: parentCloudKittyName, Namespace: cr.Namespace, }, parentCloudKitty) @@ -316,8 +316,8 @@ func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl // Watch for changes to configmaps we don't own. configMapFn := func(_ context.Context, o client.Object) []reconcile.Request { - var namespace string = o.GetNamespace() - var configMapName string = o.GetName() + var namespace = o.GetNamespace() + var configMapName = o.GetName() result := []reconcile.Request{} // get all API CRs @@ -325,7 +325,7 @@ func (r *CloudKittyAPIReconciler) SetupWithManager(ctx context.Context, mgr ctrl listOpts := []client.ListOption{ client.InNamespace(namespace), } - if err := r.Client.List(context.Background(), apis, listOpts...); err != nil { + if err := r.List(context.Background(), apis, listOpts...); err != nil { Log.Error(err, "Unable to retrieve API CRs %v") return nil } @@ -903,6 +903,7 @@ func (r *CloudKittyAPIReconciler) reconcileNormal(ctx context.Context, instance condition.SeverityInfo, condition.NetworkAttachmentsReadyWaitingMessage, netAtt)) + //nolint:err113 // Dynamic error message with network attachment name return cloudkitty.ResultRequeue, fmt.Errorf("network-attachment-definition %s not found", netAtt) } instance.Status.Conditions.Set(condition.FalseCondition( @@ -1038,6 +1039,7 @@ func (r *CloudKittyAPIReconciler) reconcileNormal(ctx context.Context, instance ssData = ss.GetStatefulSet() if ssData.Generation != ssData.Status.ObservedGeneration { ctrlResult = cloudkitty.ResultRequeue + //nolint:err113 // Dynamic error message with statefulset name err = fmt.Errorf("waiting for Statefulset %s to start reconciling", ssData.Name) } } @@ -1088,6 +1090,7 @@ func (r *CloudKittyAPIReconciler) reconcileNormal(ctx context.Context, instance if networkReady { instance.Status.Conditions.MarkTrue(condition.NetworkAttachmentsReadyCondition, condition.NetworkAttachmentsReadyMessage) } else { + //nolint:err113 // Dynamic error message with network attachments err := fmt.Errorf("not all pods have interfaces with ips as configured in NetworkAttachments: %s", instance.Spec.NetworkAttachments) instance.Status.Conditions.Set(condition.FalseCondition( condition.NetworkAttachmentsReadyCondition, diff --git a/controllers/cloudkittyproc_controller.go b/controllers/cloudkittyproc_controller.go index e92ace6f..7eb5df36 100644 --- a/controllers/cloudkittyproc_controller.go +++ b/controllers/cloudkittyproc_controller.go @@ -97,7 +97,7 @@ func (r *CloudKittyProcReconciler) Reconcile(ctx context.Context, req ctrl.Reque // Fetch the CloudKittyProc instance instance := &telemetryv1.CloudKittyProc{} - err := r.Client.Get(ctx, req.NamespacedName, instance) + err := r.Get(ctx, req.NamespacedName, instance) if err != nil { if k8s_errors.IsNotFound(err) { // Request object not found, could have been deleted after reconcile request. @@ -199,8 +199,8 @@ func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctr // Watch for changes to secrets we don't own. Global secrets // (e.g. TransportURLSecret) are handled by the main cloudkitty controller. secretFn := func(_ context.Context, o client.Object) []reconcile.Request { - var namespace string = o.GetNamespace() - var secretName string = o.GetName() + var namespace = o.GetNamespace() + var secretName = o.GetName() result := []reconcile.Request{} // get all CloudKittyProc CRs @@ -208,7 +208,7 @@ func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctr listOpts := []client.ListOption{ client.InNamespace(namespace), } - if err := r.Client.List(context.Background(), cloudKittyProcs, listOpts...); err != nil { + if err := r.List(context.Background(), cloudKittyProcs, listOpts...); err != nil { Log.Error(err, "Unable to retrieve scheduler CRs %v") return nil } @@ -266,7 +266,7 @@ func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctr // Fetch the parent CloudKitty instance parentCloudKitty := &telemetryv1.CloudKitty{} - _ = r.Client.Get(ctx, types.NamespacedName{ + _ = r.Get(ctx, types.NamespacedName{ Name: parentCloudKittyName, Namespace: cr.Namespace, }, parentCloudKitty) @@ -291,8 +291,8 @@ func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctr // Watch for changes to configmaps we don't own. configMapFn := func(_ context.Context, o client.Object) []reconcile.Request { - var namespace string = o.GetNamespace() - var configMapName string = o.GetName() + var namespace = o.GetNamespace() + var configMapName = o.GetName() result := []reconcile.Request{} // get all Proc CRs @@ -300,7 +300,7 @@ func (r *CloudKittyProcReconciler) SetupWithManager(ctx context.Context, mgr ctr listOpts := []client.ListOption{ client.InNamespace(namespace), } - if err := r.Client.List(context.Background(), procs, listOpts...); err != nil { + if err := r.List(context.Background(), procs, listOpts...); err != nil { Log.Error(err, "Unable to retrieve Proc CRs %v") return nil } @@ -613,6 +613,7 @@ func (r *CloudKittyProcReconciler) reconcileNormal(ctx context.Context, instance condition.SeverityInfo, condition.NetworkAttachmentsReadyWaitingMessage, netAtt)) + //nolint:err113 // Dynamic error message with network attachment name return cloudkitty.ResultRequeue, fmt.Errorf("network-attachment-definition %s not found", netAtt) } instance.Status.Conditions.Set(condition.FalseCondition( @@ -686,6 +687,7 @@ func (r *CloudKittyProcReconciler) reconcileNormal(ctx context.Context, instance ssData = ss.GetStatefulSet() if ssData.Generation != ssData.Status.ObservedGeneration { ctrlResult = cloudkitty.ResultRequeue + //nolint:err113 // Dynamic error message with statefulset name err = fmt.Errorf("waiting for Statefulset %s to start reconciling", ssData.Name) } } @@ -736,6 +738,7 @@ func (r *CloudKittyProcReconciler) reconcileNormal(ctx context.Context, instance if networkReady { instance.Status.Conditions.MarkTrue(condition.NetworkAttachmentsReadyCondition, condition.NetworkAttachmentsReadyMessage) } else { + //nolint:err113 // Dynamic error message with network attachments err := fmt.Errorf("not all pods have interfaces with ips as configured in NetworkAttachments: %s", instance.Spec.NetworkAttachments) instance.Status.Conditions.Set(condition.FalseCondition( condition.NetworkAttachmentsReadyCondition, diff --git a/controllers/metricstorage_controller.go b/controllers/metricstorage_controller.go index ac0a95bb..379c3975 100644 --- a/controllers/metricstorage_controller.go +++ b/controllers/metricstorage_controller.go @@ -23,7 +23,6 @@ import ( "net" "reflect" "regexp" - "slices" "strconv" "strings" "time" @@ -32,9 +31,7 @@ import ( discoveryv1 "k8s.io/api/discovery/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -44,7 +41,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" logr "github.com/go-logr/logr" "github.com/openstack-k8s-operators/lib-common/modules/ansible" @@ -308,7 +304,7 @@ func (r *MetricStorageReconciler) reconcileNormal( // Deploy monitoring stack err := utils.EnsureWatches( - (*utils.ConditionalWatchingReconciler)(r), ctx, + ctx, (*utils.ConditionalWatchingReconciler)(r), "monitoringstacks.monitoring.rhobs", &obov1.MonitoringStack{}, eventHandler, helper, ) @@ -360,8 +356,8 @@ func (r *MetricStorageReconciler) reconcileNormal( return []reconcile.Request{{NamespacedName: name}} } err = utils.EnsureWatches( - (*utils.ConditionalWatchingReconciler)(r), - ctx, "prometheuses.monitoring.rhobs", + ctx, (*utils.ConditionalWatchingReconciler)(r), + "prometheuses.monitoring.rhobs", &monv1.Prometheus{}, handler.EnqueueRequestsFromMapFunc(prometheusWatchFn), helper, @@ -580,8 +576,8 @@ func (r *MetricStorageReconciler) reconcileNormal( return []reconcile.Request{{NamespacedName: name}} } err = utils.EnsureWatches( - (*utils.ConditionalWatchingReconciler)(r), - ctx, "prometheuses.monitoring.rhobs", + ctx, (*utils.ConditionalWatchingReconciler)(r), + "prometheuses.monitoring.rhobs", &monv1.Prometheus{}, handler.EnqueueRequestsFromMapFunc(prometheusWatchFn), helper, @@ -721,8 +717,8 @@ func (r *MetricStorageReconciler) createScrapeConfigs( ) (ctrl.Result, error) { Log := r.GetLogger(ctx) err := utils.EnsureWatches( - (*utils.ConditionalWatchingReconciler)(r), - ctx, "scrapeconfigs.monitoring.rhobs", + ctx, (*utils.ConditionalWatchingReconciler)(r), + "scrapeconfigs.monitoring.rhobs", &monv1alpha1.ScrapeConfig{}, eventHandler, helper, ) if err != nil { @@ -1138,8 +1134,8 @@ func (r *MetricStorageReconciler) createDashboardObjects(ctx context.Context, in // Deploy PrometheusRule for dashboards err = utils.EnsureWatches( - (*utils.ConditionalWatchingReconciler)(r), - ctx, "prometheusrules.monitoring.rhobs", + ctx, (*utils.ConditionalWatchingReconciler)(r), + "prometheusrules.monitoring.rhobs", &monv1.PrometheusRule{}, eventHandler, helper, ) if err != nil { @@ -1261,39 +1257,6 @@ func (r *MetricStorageReconciler) createDashboardObjects(ctx context.Context, in return ctrl.Result{}, err } -func (r *MetricStorageReconciler) ensureWatches( - ctx context.Context, - name string, - kind client.Object, - handler handler.EventHandler, -) error { - Log := r.GetLogger(ctx) - if slices.Contains(r.Watching, name) { - // We are already watching the resource - return nil - } - u := &unstructured.Unstructured{} - u.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "apiextensions.k8s.io", - Kind: "CustomResourceDefinition", - Version: "v1", - }) - - err := r.Get(context.Background(), client.ObjectKey{ - Name: name, - }, u) - if err != nil { - return err - } - - Log.Info(fmt.Sprintf("Starting to watch %s", name)) - err = r.Controller.Watch(source.Kind(r.Cache, kind, handler)) - if err == nil { - r.Watching = append(r.Watching, name) - } - return err -} - func getComputeNodesConnectionInfo( instance *telemetryv1.MetricStorage, helper *helper.Helper, diff --git a/controllers/telemetry_controller.go b/controllers/telemetry_controller.go index 7a6e7b10..03091b95 100644 --- a/controllers/telemetry_controller.go +++ b/controllers/telemetry_controller.go @@ -752,7 +752,7 @@ func (r *TelemetryReconciler) checkCloudKittyGeneration( listOpts := []client.ListOption{ client.InNamespace(instance.Namespace), } - if err := r.Client.List(context.Background(), clm, listOpts...); err != nil { + if err := r.List(context.Background(), clm, listOpts...); err != nil { Log.Error(err, "Unable to retrieve CloudKitty CR %w") return false, err } diff --git a/pkg/cloudkitty/const.go b/pkg/cloudkitty/const.go index dc6589ba..164438e3 100644 --- a/pkg/cloudkitty/const.go +++ b/pkg/cloudkitty/const.go @@ -47,16 +47,22 @@ const ( // CloudKittyInternalPort - CloudKittyInternalPort int32 = 8889 - ShortDuration = time.Duration(5) * time.Second + // ShortDuration is the duration for short requeues + ShortDuration = time.Duration(5) * time.Second + // NormalDuration is the duration for normal requeues NormalDuration = time.Duration(10) * time.Second // PrometheusEndpointSecret - The name of the secret that contains the Prometheus endpoint configuration. PrometheusEndpointSecret = "metric-storage-prometheus-endpoint" + // ClientCertSecretName is the name of the client certificate secret ClientCertSecretName = "cert-cloudkitty-client-internal" + // CaConfigmapName is the name of the CA configmap CaConfigmapName = "lokistack-ca" - CaConfigmapKey = "ca.crt" + // CaConfigmapKey is the key in the CA configmap + CaConfigmapKey = "ca.crt" ) +// ResultRequeue is a ctrl.Result that requeues after NormalDuration var ResultRequeue = ctrl.Result{RequeueAfter: NormalDuration} diff --git a/pkg/cloudkitty/dbsync.go b/pkg/cloudkitty/dbsync.go index 28fb3161..7677b8a8 100644 --- a/pkg/cloudkitty/dbsync.go +++ b/pkg/cloudkitty/dbsync.go @@ -13,6 +13,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + +// Package cloudkitty provides CloudKitty service configuration and management utilities package cloudkitty import ( diff --git a/pkg/cloudkitty/lokistack.go b/pkg/cloudkitty/lokistack.go index 216ed160..6e18f0d7 100644 --- a/pkg/cloudkitty/lokistack.go +++ b/pkg/cloudkitty/lokistack.go @@ -17,6 +17,7 @@ limitations under the License. package cloudkitty import ( + "errors" "fmt" "slices" @@ -25,30 +26,45 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +var ( + // ErrEffectiveDateRequired is returned when schema effectiveDate is missing + ErrEffectiveDateRequired = errors.New("invalid CloudKitty spec. Field .spec.s3StorageConfig.schema.effectiveDate is required") + // ErrSchemaVersionRequired is returned when schema version is missing + ErrSchemaVersionRequired = errors.New("invalid CloudKitty spec. Field .spec.s3StorageConfig.schema.version is required") + // ErrSecretNameRequired is returned when secret name is missing + ErrSecretNameRequired = errors.New("invalid CloudKitty spec. Field .spec.s3StorageConfig.secret.name is required") + // ErrSecretTypeRequired is returned when secret type is missing + ErrSecretTypeRequired = errors.New("invalid CloudKitty spec. Field .spec.s3StorageConfig.secret.type is required") + // ErrInvalidSecretType is returned when secret type is not valid + ErrInvalidSecretType = errors.New("invalid CloudKitty spec. Field .spec.s3StorageConfig.secret.type needs to be one of: azure, gcs, s3, swift, alibabacloud") + // ErrCANameRequired is returned when TLS CA name is missing + ErrCANameRequired = errors.New("invalid CloudKitty spec. Field .spec.s3StorageConfig.tls.caName is required") +) + func validateObjectStorageSpec(spec telemetryv1.ObjectStorageSpec) error { for _, schema := range spec.Schemas { if schema.EffectiveDate == "" { - return fmt.Errorf("Invalid CloudKitty spec. Field .spec.s3StorageConfig.schema.effectiveDate is required") + return ErrEffectiveDateRequired } if schema.Version == "" { - return fmt.Errorf("Invalid CloudKitty spec. Field .spec.s3StorageConfig.schema.version is required") + return ErrSchemaVersionRequired } } if spec.Secret.Name == "" { - return fmt.Errorf("Invalid CloudKitty spec. Field .spec.s3StorageConfig.secret.name is required") + return ErrSecretNameRequired } if spec.Secret.Type == "" { - return fmt.Errorf("Invalid CloudKitty spec. Field .spec.s3StorageConfig.secret.type is required") + return ErrSecretTypeRequired } validTypes := []string{"azure", "gcs", "s3", "swift", "alibabacloud"} if !slices.Contains(validTypes, spec.Secret.Type) { - return fmt.Errorf("Invalid CloudKitty spec. Field .spec.s3StorageConfig.secret.type needs to be one of %s", validTypes) + return ErrInvalidSecretType } - if spec.TLS != nil && spec.TLS.CASpec.CA == "" { - return fmt.Errorf("Invalid CloudKitty spec. Field .spec.s3StorageConfig.tls.caName is required") + if spec.TLS != nil && spec.TLS.CA == "" { + return ErrCANameRequired } return nil @@ -81,13 +97,13 @@ func getLokiStackObjectStorageSpec(telemetryObjectStorageSpec telemetryv1.Object if telemetryObjectStorageSpec.TLS != nil { result.TLS = &lokistackv1.ObjectStorageTLSSpec{ CASpec: lokistackv1.CASpec{ - CAKey: telemetryObjectStorageSpec.TLS.CASpec.CAKey, - CA: telemetryObjectStorageSpec.TLS.CASpec.CA, + CAKey: telemetryObjectStorageSpec.TLS.CAKey, + CA: telemetryObjectStorageSpec.TLS.CA, }, } - if result.TLS.CASpec.CAKey == "" { + if result.TLS.CAKey == "" { // NOTE: if no CAKey is defined, use the same as defined in loki-operator - result.TLS.CASpec.CAKey = "service-ca.crt" + result.TLS.CAKey = "service-ca.crt" } } diff --git a/pkg/cloudkitty/storageinit.go b/pkg/cloudkitty/storageinit.go index 05418f99..859a8735 100644 --- a/pkg/cloudkitty/storageinit.go +++ b/pkg/cloudkitty/storageinit.go @@ -13,6 +13,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + +// Package cloudkitty provides CloudKitty service configuration and management utilities package cloudkitty import ( diff --git a/pkg/cloudkittyapi/const.go b/pkg/cloudkittyapi/const.go index 7d9a378b..e82c5afa 100644 --- a/pkg/cloudkittyapi/const.go +++ b/pkg/cloudkittyapi/const.go @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ +// Package cloudkittyapi provides CloudKitty API service configuration and management utilities package cloudkittyapi const ( diff --git a/pkg/cloudkittyproc/const.go b/pkg/cloudkittyproc/const.go index 8bda7978..c8fda2aa 100644 --- a/pkg/cloudkittyproc/const.go +++ b/pkg/cloudkittyproc/const.go @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ +// Package cloudkittyproc provides CloudKitty processor service configuration and management utilities package cloudkittyproc const ( diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index b8dd9bad..0461d056 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -37,6 +37,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/helper" ) +// ConditionalWatchingReconciler is a reconciler that can conditionally watch resources type ConditionalWatchingReconciler struct { client.Client Kclient kubernetes.Interface @@ -66,9 +67,10 @@ func EnsureDeleted(ctx context.Context, helper *helper.Helper, obj client.Object } +// EnsureWatches ensures that a watch is set up for a given resource func EnsureWatches( + _ context.Context, r *ConditionalWatchingReconciler, - ctx context.Context, name string, kind client.Object, handler handler.EventHandler, @@ -88,7 +90,7 @@ func EnsureWatches( Version: "v1", }) - err := r.Client.Get(context.Background(), client.ObjectKey{ + err := r.Get(context.Background(), client.ObjectKey{ Name: name, }, u) if err != nil { diff --git a/templates/cloudkitty/bin/healthcheck.py b/templates/cloudkitty/bin/healthcheck.py index 29995114..a1110d7a 100755 --- a/templates/cloudkitty/bin/healthcheck.py +++ b/templates/cloudkitty/bin/healthcheck.py @@ -49,4 +49,4 @@ def run_checks() -> tuple[int, str]: if rc != 0: print(reason) - sys.exit(rc) \ No newline at end of file + sys.exit(rc) From 36cb4c9cab50da2057c3f4fd5b413483bc7bf989 Mon Sep 17 00:00:00 2001 From: jlarriba Date: Fri, 10 Oct 2025 09:22:37 +0200 Subject: [PATCH 35/87] Add kuttl assertions for CloudKitty resources --- .../suites/cloudkitty/tests/01-assert.yaml | 116 ++++++++++++++++++ .../suites/cloudkitty/tests/01-deploy.yaml | 4 - 2 files changed, 116 insertions(+), 4 deletions(-) diff --git a/tests/kuttl/suites/cloudkitty/tests/01-assert.yaml b/tests/kuttl/suites/cloudkitty/tests/01-assert.yaml index ad927419..a9e56416 100644 --- a/tests/kuttl/suites/cloudkitty/tests/01-assert.yaml +++ b/tests/kuttl/suites/cloudkitty/tests/01-assert.yaml @@ -79,3 +79,119 @@ status: conditions: - status: "True" type: Ready +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + component: cloudkitty-api + service: cloudkitty + name: telemetry-kuttl-cloudkitty-api + ownerReferences: + - kind: CloudKittyAPI + name: telemetry-kuttl-cloudkitty-api +spec: + replicas: 1 + selector: + matchLabels: + component: cloudkitty-api + service: cloudkitty +status: + availableReplicas: 1 + currentReplicas: 1 + readyReplicas: 1 + replicas: 1 +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + component: cloudkitty-api + service: cloudkitty + name: telemetry-kuttl-cloudkitty-api-0 + ownerReferences: + - kind: StatefulSet + name: telemetry-kuttl-cloudkitty-api +spec: + containers: + - name: cloudkitty-api + hostname: telemetry-kuttl-cloudkitty-api-0 +status: + containerStatuses: + - name: cloudkitty-api + ready: true + started: true +--- +apiVersion: v1 +kind: Service +metadata: + labels: + component: cloudkitty-api + service: cloudkitty + name: cloudkitty-internal + ownerReferences: + - kind: CloudKittyAPI + name: telemetry-kuttl-cloudkitty-api +spec: + ports: + - port: 8889 + protocol: TCP + targetPort: 8889 +--- +apiVersion: v1 +kind: Service +metadata: + labels: + component: cloudkitty-api + service: cloudkitty + name: cloudkitty-public + ownerReferences: + - kind: CloudKittyAPI + name: telemetry-kuttl-cloudkitty-api +spec: + ports: + - port: 8889 + protocol: TCP + targetPort: 8889 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + component: cloudkitty-proc + service: cloudkitty + name: telemetry-kuttl-cloudkitty-proc + ownerReferences: + - kind: CloudKittyProc + name: telemetry-kuttl-cloudkitty-proc +spec: + replicas: 1 + selector: + matchLabels: + component: cloudkitty-proc + service: cloudkitty +status: + availableReplicas: 1 + currentReplicas: 1 + readyReplicas: 1 + replicas: 1 +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + component: cloudkitty-proc + service: cloudkitty + name: telemetry-kuttl-cloudkitty-proc-0 + ownerReferences: + - kind: StatefulSet + name: telemetry-kuttl-cloudkitty-proc +spec: + containers: + - name: cloudkitty-processor + hostname: telemetry-kuttl-cloudkitty-proc-0 +status: + containerStatuses: + - name: cloudkitty-processor + ready: true + started: true diff --git a/tests/kuttl/suites/cloudkitty/tests/01-deploy.yaml b/tests/kuttl/suites/cloudkitty/tests/01-deploy.yaml index 135146e2..3b5af6dd 100644 --- a/tests/kuttl/suites/cloudkitty/tests/01-deploy.yaml +++ b/tests/kuttl/suites/cloudkitty/tests/01-deploy.yaml @@ -5,8 +5,6 @@ metadata: spec: apiTimeout: 0 cloudKittyAPI: - # TODO: This should go away once CK images are taken from openstackversions - containerImage: quay.io/jwysogla/cloudkitty-api:latest override: service: internal: @@ -27,8 +25,6 @@ spec: public: {} caBundleSecretName: combined-ca-bundle cloudKittyProc: - # TODO: This should go away once CK images are taken from openstackversions - containerImage: quay.io/jwysogla/cloudkitty-processor:latest replicas: 1 resources: {} tls: From 43605379fc623c3061089f341620e56979e361d8 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Wed, 9 Jul 2025 18:11:56 +0100 Subject: [PATCH 36/87] [zuul] Add a CloudKitty CI job --- ci/vars-cloudkitty-tempest.yml | 19 ++++++++++ zuul.d/projects.yaml | 65 ++++++++++++++++++++++++---------- 2 files changed, 65 insertions(+), 19 deletions(-) create mode 100644 ci/vars-cloudkitty-tempest.yml diff --git a/ci/vars-cloudkitty-tempest.yml b/ci/vars-cloudkitty-tempest.yml new file mode 100644 index 00000000..0a8a475f --- /dev/null +++ b/ci/vars-cloudkitty-tempest.yml @@ -0,0 +1,19 @@ +--- +cifmw_run_tests: true +cifmw_run_test_role: test_operator +cifmw_test_operator_tempest_namespace: podified-antelope-centos9 +cifmw_test_operator_tempest_container: openstack-tempest-all +cifmw_test_operator_tempest_image_tag: 'current-podified' +# This value is used to populate the `tempestconfRun` parameter of the Tempest CR: https://openstack-k8s-operators.github.io/test-operator/crds.html#tempest-custom-resource +# https://github.com/openstack-k8s-operators/ci-framework/blob/main/roles/test_operator/defaults/main.yml +tempest_conf: + overrides: | + validation.run_validation true +cifmw_test_operator_tempest_tempestconf_config: "{{ tempest_conf }}" +cifmw_test_operator_tempest_include_list: | + ^tempest.*\[.*\bsmoke\b.*\] + cloudkitty_tempest_plugin.* + +external_plugin: "opendev.org/openstack/cloudkitty-tempest-plugin" +change_item: "{{ zuul['items'] | selectattr('project.canonical_name', 'equalto', external_plugin) }}" +cifmw_test_operator_tempest_external_plugin: "{{ [] if change_item | length < 1 else [ { 'repository': 'https://' + external_plugin + '.git', 'changeRepository': 'https://review' + external_plugin, 'changeRefspec': [ 'refs/changes', change_item[0].change[-2:], change_item[0].change, change_item[0].patchset ] | join('/') } ] }}" diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index 1a85a567..00d88ddc 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -104,6 +104,30 @@ - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-power-monitoring.yml" irrelevant-files: *irrelevant_files +- job: + name: telemetry-operator-multinode-cloudkitty + dependencies: ["telemetry-openstack-meta-content-provider-master"] + parent: telemetry-operator-multinode-autoscaling + description: | + Deploy CloudKitty and run tempest tests + required-projects: + - name: infrawatch/feature-verification-tests + override-checkout: master + extra-vars: *mcp_extra_vars + vars: + #patch_observabilityclient: true + cifmw_update_containers: false + cifmw_extras: + - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir }}/scenarios/centos-9/multinode-ci.yml" + # Deploys logging dependencies (e.g. loki) + - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-logging.yml" + # Need a config for CK + - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-cloudkitty-tempest.yml" + - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-use-master-containers.yml" + roles: + - zuul: github.com/openstack-k8s-operators/ci-framework + irrelevant-files: *irrelevant_files + - project-template: name: rdo-telemetry-tempest-plugin-jobs openstack-experimental: @@ -133,25 +157,28 @@ - project: name: openstack-k8s-operators/telemetry-operator - templates: - - podified-multinode-edpm-pipeline + #templates: + # - podified-multinode-edpm-pipeline github-check: + #debug: true jobs: + - telemetry-operator-multinode-cloudkitty - telemetry-openstack-meta-content-provider-master - - telemetry-operator-multinode-default-telemetry - - functional-tests-osp18: &fvt_jobs_config - voting: true - required-projects: - - name: infrawatch/feature-verification-tests - override-checkout: master - irrelevant-files: *irrelevant_files - - feature-verification-tests-noop: - files: *irrelevant_files - - functional-periodic-telemetry-with-ceph: - required-projects: - - name: infrawatch/feature-verification-tests - override-checkout: master - files: - - ci/deploy-telemetry-with-ceph.yml - - ci/vars-telemetry-with-ceph.yml - - zuul.d/projects.yaml + #- telemetry-operator-multinode-default-telemetry + #- functional-tests-osp18: &fvt_jobs_config + # voting: true + # required-projects: + # - name: infrawatch/feature-verification-tests + # override-checkout: master + # irrelevant-files: *irrelevant_files + #- feature-verification-tests-noop: + # files: *irrelevant_files + #- functional-periodic-telemetry-with-ceph: + # required-projects: + # - name: infrawatch/feature-verification-tests + # override-checkout: master + # files: + # - ci/deploy-telemetry-with-ceph.yml + # - ci/vars-telemetry-with-ceph.yml + # - zuul.d/projects.yaml + From defda0c754aca8157a89abf2c1005112e0aeb212 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Thu, 17 Jul 2025 13:55:41 +0100 Subject: [PATCH 37/87] update ci/vars-cloudkitty-tempest.yml --- ci/vars-cloudkitty-tempest.yml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/ci/vars-cloudkitty-tempest.yml b/ci/vars-cloudkitty-tempest.yml index 0a8a475f..2f21b010 100644 --- a/ci/vars-cloudkitty-tempest.yml +++ b/ci/vars-cloudkitty-tempest.yml @@ -2,6 +2,11 @@ cifmw_run_tests: true cifmw_run_test_role: test_operator cifmw_test_operator_tempest_namespace: podified-antelope-centos9 +# cloudkitty tempest plugin is not part of the tempest rpm. +# https://review.rdoproject.org/cgit/openstack/tempest-distgit/tree/openstack-tempest.spec +# We need to add the cloudkitty-tempest-plugin package to RDO, same as TTTP +# https://review.rdoproject.org/cgit/openstack/telemetry-tempest-plugin-distgit/# +# For now, we can force install using the cifmw_test_operator_tempest_external_plugin below. cifmw_test_operator_tempest_container: openstack-tempest-all cifmw_test_operator_tempest_image_tag: 'current-podified' # This value is used to populate the `tempestconfRun` parameter of the Tempest CR: https://openstack-k8s-operators.github.io/test-operator/crds.html#tempest-custom-resource @@ -9,11 +14,25 @@ cifmw_test_operator_tempest_image_tag: 'current-podified' tempest_conf: overrides: | validation.run_validation true + identity.v3_endpoint_type public + service_available.ceilometer true + service_available.sg_core true + service_available.aodh true + service_available.cinder false + telemetry.sg_core_service_url "https://ceilometer-internal.openstack.svc.cluster.local:3000" + telemetry.prometheus_service_url "https://metric-storage-prometheus.openstack.svc.cluster.local:9090" + telemetry.ceilometer_polling_interval 120 + telemetry.prometheus_scrape_interval 30 + telemetry.alarm_threshold 50000000000 cifmw_test_operator_tempest_tempestconf_config: "{{ tempest_conf }}" cifmw_test_operator_tempest_include_list: | ^tempest.*\[.*\bsmoke\b.*\] cloudkitty_tempest_plugin.* + telemetry_tempest_plugin.* +cifmw_test_operator_tempest_exclude_list: | + telemetry_tempest_plugin.scenario.test_telemetry_integration_prometheus.PrometheusGabbiTest.test_autoscaling external_plugin: "opendev.org/openstack/cloudkitty-tempest-plugin" change_item: "{{ zuul['items'] | selectattr('project.canonical_name', 'equalto', external_plugin) }}" -cifmw_test_operator_tempest_external_plugin: "{{ [] if change_item | length < 1 else [ { 'repository': 'https://' + external_plugin + '.git', 'changeRepository': 'https://review' + external_plugin, 'changeRefspec': [ 'refs/changes', change_item[0].change[-2:], change_item[0].change, change_item[0].patchset ] | join('/') } ] }}" +# This is a workaround because CK tempest is not packages in RDO. Typically, the default would be [], since we would not require an external installation. +cifmw_test_operator_tempest_external_plugin: "{{ [ {'repository': 'https://' + external_plugin + '.git'} ] if change_item | length < 1 else [ { 'repository': 'https://' + external_plugin + '.git', 'changeRepository': 'https://review' + external_plugin, 'changeRefspec': [ 'refs/changes', change_item[0].change[-2:], change_item[0].change, change_item[0].patchset ] | join('/') } ] }}" From f265a6491e531678f940f3af4501781175ccea14 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Thu, 17 Jul 2025 18:31:55 +0100 Subject: [PATCH 38/87] Add deployment config for cloudkitty --- ci/configure-autoscaling.yml | 30 ++++++++++++++++-------------- ci/vars-cloudkitty-tempest.yml | 21 +++++++++++++++++++++ 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/ci/configure-autoscaling.yml b/ci/configure-autoscaling.yml index 959b8925..eac0f6c3 100644 --- a/ci/configure-autoscaling.yml +++ b/ci/configure-autoscaling.yml @@ -21,7 +21,7 @@ name: unused spec: heat: - enabled: true + enabled: false telemetry: enabled: true template: @@ -30,21 +30,23 @@ monitoringStack: alertingEnabled: false autoscaling: - enabled: true + enabled: false ceilometer: enabled: true mysqldExporterEnabled: true + cloudkitty: + enabled: true target: kind: OpenStackControlPlane - - patch: |- - apiVersion: core.openstack.org/v1beta1 - kind: OpenStackControlPlane - metadata: - name: unused - spec: - telemetry: - template: - metricStorage: - dashboardsEnabled: true - target: - kind: OpenStackControlPlane +# - patch: |- +# apiVersion: core.openstack.org/v1beta1 +# kind: OpenStackControlPlane +# metadata: +# name: unused +# spec: +# telemetry: +# template: +# metricStorage: +# dashboardsEnabled: true +# target: +# kind: OpenStackControlPlane diff --git a/ci/vars-cloudkitty-tempest.yml b/ci/vars-cloudkitty-tempest.yml index 2f21b010..22111866 100644 --- a/ci/vars-cloudkitty-tempest.yml +++ b/ci/vars-cloudkitty-tempest.yml @@ -1,4 +1,25 @@ --- +# Deploy cloudkitty +cifmw_deploy_obs: true +cifmw_openshift_obs_definition: + apiVersion: operators.coreos.com/v1alpha1 + kind: Subscription + metadata: + name: observability-operator + namespace: openshift-operators + spec: + channel: stable + installPlanApproval: Automatic + name: cluster-observability-operator + source: redhat-operators + sourceNamespace: openshift-marketplace + +# TODO: Add a kustomization for CK, most likely separate from the autoscaling one. +pre_deploy_kustomize_autoscaling: + source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/configure-autoscaling.yml" + type: playbook + +# test cloudkitty cifmw_run_tests: true cifmw_run_test_role: test_operator cifmw_test_operator_tempest_namespace: podified-antelope-centos9 From 8cacfe975640f744b4b3b733d7cef4018b33d46d Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 18 Jul 2025 15:23:02 +0100 Subject: [PATCH 39/87] Add a configure-cloudkitty hook --- ci/configure-autoscaling.yml | 30 +++++++++++++-------------- ci/configure-cloudkitty.yml | 37 ++++++++++++++++++++++++++++++++++ ci/vars-cloudkitty-tempest.yml | 13 +++++++----- 3 files changed, 59 insertions(+), 21 deletions(-) create mode 100644 ci/configure-cloudkitty.yml diff --git a/ci/configure-autoscaling.yml b/ci/configure-autoscaling.yml index eac0f6c3..959b8925 100644 --- a/ci/configure-autoscaling.yml +++ b/ci/configure-autoscaling.yml @@ -21,7 +21,7 @@ name: unused spec: heat: - enabled: false + enabled: true telemetry: enabled: true template: @@ -30,23 +30,21 @@ monitoringStack: alertingEnabled: false autoscaling: - enabled: false + enabled: true ceilometer: enabled: true mysqldExporterEnabled: true - cloudkitty: - enabled: true target: kind: OpenStackControlPlane -# - patch: |- -# apiVersion: core.openstack.org/v1beta1 -# kind: OpenStackControlPlane -# metadata: -# name: unused -# spec: -# telemetry: -# template: -# metricStorage: -# dashboardsEnabled: true -# target: -# kind: OpenStackControlPlane + - patch: |- + apiVersion: core.openstack.org/v1beta1 + kind: OpenStackControlPlane + metadata: + name: unused + spec: + telemetry: + template: + metricStorage: + dashboardsEnabled: true + target: + kind: OpenStackControlPlane diff --git a/ci/configure-cloudkitty.yml b/ci/configure-cloudkitty.yml new file mode 100644 index 00000000..aaf3ec12 --- /dev/null +++ b/ci/configure-cloudkitty.yml @@ -0,0 +1,37 @@ +--- +- name: "Create the kustomization for deploying CloudKitty" + hosts: "{{ cifmw_target_hook_host | default('localhost') }}" + gather_facts: false + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + tasks: + - name: Copy controlplane kustomization + ansible.builtin.copy: + dest: "{{ cifmw_basedir }}/artifacts/manifests/kustomizations/controlplane/90-kustomize-controlplane-cloudkitty.yaml" + content: |- + apiVersion: kustomize.config.k8s.io/v1beta1 + kind: Kustomization + namespace: openstack + patches: + - patch: |- + apiVersion: core.openstack.org/v1beta1 + kind: OpenStackControlPlane + metadata: + name: unused + spec: + telemetry: + enabled: true + template: + metricStorage: + enabled: true + monitoringStack: + alertingEnabled: false + autoscaling: + enabled: false + ceilometer: + enabled: true + cloudkitty: + enabled: true + target: + kind: OpenStackControlPlane diff --git a/ci/vars-cloudkitty-tempest.yml b/ci/vars-cloudkitty-tempest.yml index 22111866..47003593 100644 --- a/ci/vars-cloudkitty-tempest.yml +++ b/ci/vars-cloudkitty-tempest.yml @@ -15,13 +15,14 @@ cifmw_openshift_obs_definition: sourceNamespace: openshift-marketplace # TODO: Add a kustomization for CK, most likely separate from the autoscaling one. -pre_deploy_kustomize_autoscaling: - source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/configure-autoscaling.yml" +pre_deploy_kustomize_cloudkitty: + source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/configure-cloudkitty.yml" type: playbook # test cloudkitty cifmw_run_tests: true cifmw_run_test_role: test_operator +# TODO: Consider switching to podified-master-centos10 for features that patch master cifmw_test_operator_tempest_namespace: podified-antelope-centos9 # cloudkitty tempest plugin is not part of the tempest rpm. # https://review.rdoproject.org/cgit/openstack/tempest-distgit/tree/openstack-tempest.spec @@ -32,6 +33,7 @@ cifmw_test_operator_tempest_container: openstack-tempest-all cifmw_test_operator_tempest_image_tag: 'current-podified' # This value is used to populate the `tempestconfRun` parameter of the Tempest CR: https://openstack-k8s-operators.github.io/test-operator/crds.html#tempest-custom-resource # https://github.com/openstack-k8s-operators/ci-framework/blob/main/roles/test_operator/defaults/main.yml +# TODO: Refine this tempest config tempest_conf: overrides: | validation.run_validation true @@ -50,9 +52,10 @@ cifmw_test_operator_tempest_include_list: | ^tempest.*\[.*\bsmoke\b.*\] cloudkitty_tempest_plugin.* telemetry_tempest_plugin.* -cifmw_test_operator_tempest_exclude_list: | - telemetry_tempest_plugin.scenario.test_telemetry_integration_prometheus.PrometheusGabbiTest.test_autoscaling - +#cifmw_test_operator_tempest_exclude_list: | +# telemetry_tempest_plugin.scenario.test_telemetry_integration_prometheus.PrometheusGabbiTest.test_autoscaling +# TODO: update this to allow multiple external plugins to be listed with Depends-On. +# Potentially, this can be done via the meta content provider, by adding the tempest images to the list. external_plugin: "opendev.org/openstack/cloudkitty-tempest-plugin" change_item: "{{ zuul['items'] | selectattr('project.canonical_name', 'equalto', external_plugin) }}" # This is a workaround because CK tempest is not packages in RDO. Typically, the default would be [], since we would not require an external installation. From 87ac2b63df3d438db67afdf68a81e123b93d69d6 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Mon, 21 Jul 2025 10:37:03 +0100 Subject: [PATCH 40/87] Add some comments --- ci/configure-cloudkitty.yml | 8 -------- ci/vars-cloudkitty-tempest.yml | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/ci/configure-cloudkitty.yml b/ci/configure-cloudkitty.yml index aaf3ec12..a62771c6 100644 --- a/ci/configure-cloudkitty.yml +++ b/ci/configure-cloudkitty.yml @@ -23,14 +23,6 @@ telemetry: enabled: true template: - metricStorage: - enabled: true - monitoringStack: - alertingEnabled: false - autoscaling: - enabled: false - ceilometer: - enabled: true cloudkitty: enabled: true target: diff --git a/ci/vars-cloudkitty-tempest.yml b/ci/vars-cloudkitty-tempest.yml index 47003593..c3b8d98a 100644 --- a/ci/vars-cloudkitty-tempest.yml +++ b/ci/vars-cloudkitty-tempest.yml @@ -14,7 +14,6 @@ cifmw_openshift_obs_definition: source: redhat-operators sourceNamespace: openshift-marketplace -# TODO: Add a kustomization for CK, most likely separate from the autoscaling one. pre_deploy_kustomize_cloudkitty: source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/configure-cloudkitty.yml" type: playbook @@ -59,4 +58,5 @@ cifmw_test_operator_tempest_include_list: | external_plugin: "opendev.org/openstack/cloudkitty-tempest-plugin" change_item: "{{ zuul['items'] | selectattr('project.canonical_name', 'equalto', external_plugin) }}" # This is a workaround because CK tempest is not packages in RDO. Typically, the default would be [], since we would not require an external installation. +# TODO: add the cloudkitty-tempest-plugin to RDO cifmw_test_operator_tempest_external_plugin: "{{ [ {'repository': 'https://' + external_plugin + '.git'} ] if change_item | length < 1 else [ { 'repository': 'https://' + external_plugin + '.git', 'changeRepository': 'https://review' + external_plugin, 'changeRefspec': [ 'refs/changes', change_item[0].change[-2:], change_item[0].change, change_item[0].patchset ] | join('/') } ] }}" From 4326afc4a4bc8133c53cadb4d08f4f4a330a63a0 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Thu, 28 Aug 2025 14:15:00 +0100 Subject: [PATCH 41/87] Add Telemetry kustomization to enable CK --- ci/configure-cloudkitty.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ci/configure-cloudkitty.yml b/ci/configure-cloudkitty.yml index a62771c6..3c95ff03 100644 --- a/ci/configure-cloudkitty.yml +++ b/ci/configure-cloudkitty.yml @@ -27,3 +27,21 @@ enabled: true target: kind: OpenStackControlPlane + - name: Copy telemetry kustomization + ansible.builtin.copy: + dest: "{{ cifmw_basedir }}/artifacts/manifests/kustomizations/controlplane/90-kustomize-telemetry-cloudkitty.yaml" + content: |- + apiVersion: kustomize.config.k8s.io/v1beta1 + kind: Kustomization + namespace: openstack + patches: + - patch: |- + apiVersion: core.openstack.org/v1beta1 + kind: Telemetry + metadata: + name: unused + spec: + cloudkitty: + enabled: true + target: + kind: Telemetry From 8d13a1a38419a6a73f0faa1d56758d38acce1bcc Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Thu, 28 Aug 2025 14:15:27 +0100 Subject: [PATCH 42/87] Use podified-master-centos10 tempest image --- ci/vars-cloudkitty-tempest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/vars-cloudkitty-tempest.yml b/ci/vars-cloudkitty-tempest.yml index c3b8d98a..2630ebdd 100644 --- a/ci/vars-cloudkitty-tempest.yml +++ b/ci/vars-cloudkitty-tempest.yml @@ -22,7 +22,7 @@ pre_deploy_kustomize_cloudkitty: cifmw_run_tests: true cifmw_run_test_role: test_operator # TODO: Consider switching to podified-master-centos10 for features that patch master -cifmw_test_operator_tempest_namespace: podified-antelope-centos9 +cifmw_test_operator_tempest_namespace: podified-master-centos10 # cloudkitty tempest plugin is not part of the tempest rpm. # https://review.rdoproject.org/cgit/openstack/tempest-distgit/tree/openstack-tempest.spec # We need to add the cloudkitty-tempest-plugin package to RDO, same as TTTP From d49b82e81109a82942af83d7722c84b20d7e0835 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Thu, 28 Aug 2025 18:25:43 +0100 Subject: [PATCH 43/87] Add osp-secret, and password workaround TODO: Add CloudKittyPassword to install_yamls/scripts/gen-input-kustomize.sh --- ci/configure-cloudkitty.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci/configure-cloudkitty.yml b/ci/configure-cloudkitty.yml index 3c95ff03..83d9329e 100644 --- a/ci/configure-cloudkitty.yml +++ b/ci/configure-cloudkitty.yml @@ -43,5 +43,10 @@ spec: cloudkitty: enabled: true + secret: osp-secret + passwordSelector: + aodhService: AodhPassword + ceilometerService: CeilometerPassword + cloudKittyService: CloudKittyPassword target: kind: Telemetry From 477bb17415e3e547cab830b853ba6260d1784fd8 Mon Sep 17 00:00:00 2001 From: "Chandan Kumar (raukadah)" Date: Fri, 4 Jul 2025 13:15:29 +0530 Subject: [PATCH 44/87] Add and Use meta content provider in telemetry-operator This pr adds following things: - Adds a new job telemetry meta content provider to build telemetry containers. - Update existing job to use meta content provider Signed-off-by: Chandan Kumar (raukadah) --- zuul.d/projects.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index 00d88ddc..4b4b54e6 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -5,15 +5,21 @@ A meta content provider zuul job to build telemetry specific containers. vars: +<<<<<<< HEAD # Note(Chandan Kumar): image_base is the operator name without -operator. # It is needed by openstack-operator to include telemetry operator images # in the openstack-operator catalog image. +======= +>>>>>>> a45c3ae (Add and Use meta content provider in telemetry-operator) cifmw_operator_build_operators: - name: telemetry-operator src: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/telemetry-operator" - name: openstack-operator src: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/openstack-operator" +<<<<<<< HEAD image_base: telemetry +======= +>>>>>>> a45c3ae (Add and Use meta content provider in telemetry-operator) cifmw_build_containers_image_tag: telemetry_latest zuul_project_container_path: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/telemetry-operator/ci/files/containers.yaml" cifmw_build_containers_config_file: "{{ ansible_user_dir }}/containers.yaml" From b8a31ecb8101f771fe6850f0dc4eb304f0f92968 Mon Sep 17 00:00:00 2001 From: "Chandan Kumar (raukadah)" Date: Mon, 14 Jul 2025 20:39:29 +0530 Subject: [PATCH 45/87] Set image_base in cifmw_operator_build_operators for meta operator image_base is used to specify the operator project name. It is used openstack operator to include telemetry operator catalog image into the openstack-operator catalog image. Note: It is not needed for standalone operator. Resolves: OSPCIX-965 Signed-off-by: Chandan Kumar (raukadah) --- zuul.d/projects.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index 4b4b54e6..00d88ddc 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -5,21 +5,15 @@ A meta content provider zuul job to build telemetry specific containers. vars: -<<<<<<< HEAD # Note(Chandan Kumar): image_base is the operator name without -operator. # It is needed by openstack-operator to include telemetry operator images # in the openstack-operator catalog image. -======= ->>>>>>> a45c3ae (Add and Use meta content provider in telemetry-operator) cifmw_operator_build_operators: - name: telemetry-operator src: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/telemetry-operator" - name: openstack-operator src: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/openstack-operator" -<<<<<<< HEAD image_base: telemetry -======= ->>>>>>> a45c3ae (Add and Use meta content provider in telemetry-operator) cifmw_build_containers_image_tag: telemetry_latest zuul_project_container_path: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/telemetry-operator/ci/files/containers.yaml" cifmw_build_containers_config_file: "{{ ansible_user_dir }}/containers.yaml" From e01adb2c12243550e99d6205a2e5112be68929a4 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 29 Aug 2025 15:55:09 +0100 Subject: [PATCH 46/87] Add CK CR kustomiation the telemetry one dd not set the expected values in CK Revert "Add CK CR kustomiation" This reverts commit 9cb243a0e15a00af2d80aed7dec1e9a39bb9c91e. From 5a807f90f48b493eadaa235729cd8d7caeaa7fb8 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Mon, 1 Sep 2025 13:52:55 +0100 Subject: [PATCH 47/87] [cloudkitty] Add in the images for CK services --- ci/configure-cloudkitty.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ci/configure-cloudkitty.yml b/ci/configure-cloudkitty.yml index 83d9329e..952c2a8f 100644 --- a/ci/configure-cloudkitty.yml +++ b/ci/configure-cloudkitty.yml @@ -48,5 +48,16 @@ aodhService: AodhPassword ceilometerService: CeilometerPassword cloudKittyService: CloudKittyPassword + # WORKAROUND + # Need to set the images, since the defaulting is not working at the moment. + # The container image is not being set, so the dbsync is failing. + # The defaulting will not be done in the webhook until CK is added to the openstack-operator. + # If I change the name of the CK, I can just configure using the CK CRD. + # If I don't change the name, and disable it in telemetry, then the telemetry-operator will delete the telemetry telemetry. + cloudKittyAPI: + containerImage: "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-api" + cloudKittyProc: + containerImage: "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-processor" target: kind: Telemetry + From bd9ce85e9f0f748b90ffc8271e271e083ff82a72 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Tue, 2 Sep 2025 14:12:10 +0100 Subject: [PATCH 48/87] Update CK to be configured post-deploy This is done to try and get some more information about the failures, so that must-gather might be running by the time we try to apply the change. There should be some more information available in the job-output, since the patch will be applied on its own with no other changes --- ci/cloudkitty-post_deploy.yml | 43 ++++++++++++++++++++++++++++++++++ ci/configure-cloudkitty.yml | 4 ++++ ci/vars-cloudkitty-tempest.yml | 8 ++++--- 3 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 ci/cloudkitty-post_deploy.yml diff --git a/ci/cloudkitty-post_deploy.yml b/ci/cloudkitty-post_deploy.yml new file mode 100644 index 00000000..b8c4ac7b --- /dev/null +++ b/ci/cloudkitty-post_deploy.yml @@ -0,0 +1,43 @@ +--- +# This should get to the point where we have must-gather running so that there is more information available. +- name: "Create the kustomization for deploying CloudKitty" + hosts: "{{ cifmw_target_hook_host | default('localhost') }}" + gather_facts: false + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + tasks: + # WORKAROUND + # Need to set the images, since the defaulting is not working at the moment. + # The container image is not being set, so the dbsync is failing. + # The defaulting will not be done in the webhook until CK is added to the openstack-operator. + # If I change the name of the CK, I can just configure using the CK CRD. + # If I don't change the name, and disable it in telemetry, then the telemetry-operator will delete the telemetry telemetry. + - name: Patch telemetry CR + ansible.builtin.shell: + cmd: | + oc apply -f - < Date: Thu, 4 Sep 2025 19:28:01 +0100 Subject: [PATCH 49/87] Update some image URLs --- ci/cloudkitty-post_deploy.yml | 4 ++-- ci/configure-cloudkitty.yml | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ci/cloudkitty-post_deploy.yml b/ci/cloudkitty-post_deploy.yml index b8c4ac7b..18b29de5 100644 --- a/ci/cloudkitty-post_deploy.yml +++ b/ci/cloudkitty-post_deploy.yml @@ -31,9 +31,9 @@ ceilometerService: CeilometerPassword cloudKittyService: CloudKittyPassword cloudKittyAPI: - containerImage: "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-api" + containerImage: "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-api:current-tested" cloudKittyProc: - containerImage: "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-processor" + containerImage: "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-processor:current-tested" EOF register: output - name: wait for the update to telemetry diff --git a/ci/configure-cloudkitty.yml b/ci/configure-cloudkitty.yml index 123c72ef..7a696b01 100644 --- a/ci/configure-cloudkitty.yml +++ b/ci/configure-cloudkitty.yml @@ -31,6 +31,8 @@ enabled: true target: kind: OpenStackControlPlane + # TODO: To use the images created by the content provider, we need to add in the images here until there's support in openstack operator to pull them in the expected way. + - name: Copy telemetry kustomization ansible.builtin.copy: dest: "{{ cifmw_basedir }}/artifacts/manifests/kustomizations/controlplane/90-kustomize-telemetry-cloudkitty.yaml" @@ -59,9 +61,9 @@ # If I change the name of the CK, I can just configure using the CK CRD. # If I don't change the name, and disable it in telemetry, then the telemetry-operator will delete the telemetry telemetry. cloudKittyAPI: - containerImage: "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-api" + containerImage: "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-api:current-tested" cloudKittyProc: - containerImage: "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-processor" + containerImage: "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-processor:current-tested" target: kind: Telemetry From a1cd0e89be8fdb17b780ab0062376773288b76c0 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Thu, 4 Sep 2025 21:30:35 +0100 Subject: [PATCH 50/87] Update CR patch to use the right API and add a var for images Pass the ck images to the hooks --- ci/cloudkitty-post_deploy.yml | 6 +++--- ci/configure-cloudkitty.yml | 7 ++++--- ci/vars-cloudkitty-tempest.yml | 19 +++++++++++++++++-- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/ci/cloudkitty-post_deploy.yml b/ci/cloudkitty-post_deploy.yml index 18b29de5..a727ecc9 100644 --- a/ci/cloudkitty-post_deploy.yml +++ b/ci/cloudkitty-post_deploy.yml @@ -17,7 +17,7 @@ ansible.builtin.shell: cmd: | oc apply -f - < Date: Fri, 5 Sep 2025 15:55:08 +0100 Subject: [PATCH 51/87] Disable the kustomisation for enabling cloudkitty; use the post-deploy instead --- ci/vars-cloudkitty-tempest.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ci/vars-cloudkitty-tempest.yml b/ci/vars-cloudkitty-tempest.yml index eb79ad70..de54e1fe 100644 --- a/ci/vars-cloudkitty-tempest.yml +++ b/ci/vars-cloudkitty-tempest.yml @@ -18,12 +18,13 @@ cifmw_openshift_obs_definition: cloudkitty_api_image: "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-api:current-tested" cloudkitty_proc_image: "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-processor:current-tested" -pre_deploy_kustomize_cloudkitty: - source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/configure-cloudkitty.yml" - type: playbook - extra_vars: - cloudkitty_api_image: "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-api:current-tested" - cloudkitty_proc_image: "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-processor:current-tested" +# This is removed for now. +#pre_deploy_kustomize_cloudkitty: +# source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/configure-cloudkitty.yml" +# type: playbook +# extra_vars: +# cloudkitty_api_image: "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-api:current-tested" +# cloudkitty_proc_image: "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-processor:current-tested" # TODO: Adapt the deploy-logging-dependencies role to allow config necessary for CK # # I can remove the vars-logging from the job definition. #pre_deploy_deploy_logging_dependencies: From e10d009131a97e5678a6f3480d3beca405c955ae Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 5 Sep 2025 18:57:02 +0100 Subject: [PATCH 52/87] [ci] Add vars so that content provider image can be consumed by CK --- ci/vars-cloudkitty-tempest.yml | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/ci/vars-cloudkitty-tempest.yml b/ci/vars-cloudkitty-tempest.yml index de54e1fe..e755b5e6 100644 --- a/ci/vars-cloudkitty-tempest.yml +++ b/ci/vars-cloudkitty-tempest.yml @@ -15,16 +15,32 @@ cifmw_openshift_obs_definition: sourceNamespace: openshift-marketplace # TODO: To use the images created by the content provider, we need to add in the images here until there's support in openstack operator to pull them in the expected way. -cloudkitty_api_image: "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-api:current-tested" -cloudkitty_proc_image: "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-processor:current-tested" + +# from fvt/ci/vars/common.yml should be used from there eventually. +telemetry_registry: >- + {%- if content_provider_os_registry_url is defined and content_provider_os_registry_url != 'null' -%} + {{ content_provider_os_registry_url }} + {%- else -%} + quay.rdoproject.org/podified-master-centos10 + {%- endif -%} + +telemetry_tag: >- + {%- if content_provider_os_registry_url is defined and content_provider_os_registry_url != 'null' -%} + telemetry_latest + {%- else -%} + current-tested + {%- endif -%} + +_ck_api_image: "{{ telemetry_registry }}/openstack-cloudkitty-api:{{ telemetry_tag }}" +_ck_proc_image: "{{ telemetry_registry }}/openstack-cloudkitty-processor:{{ telemetry_tag }}" # This is removed for now. #pre_deploy_kustomize_cloudkitty: # source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/configure-cloudkitty.yml" # type: playbook # extra_vars: -# cloudkitty_api_image: "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-api:current-tested" -# cloudkitty_proc_image: "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-processor:current-tested" +# cloudkitty_api_image: "{{ _ck_api_image }} +# cloudkitty_proc_image: "{{ _ck_proc_image }}" # TODO: Adapt the deploy-logging-dependencies role to allow config necessary for CK # # I can remove the vars-logging from the job definition. #pre_deploy_deploy_logging_dependencies: @@ -34,8 +50,9 @@ post_deploy_enable_cloudkitty: source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/cloudkitty-post_deploy.yml" type: playbook extra_vars: - cloudkitty_api_image: "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-api:current-tested" - cloudkitty_proc_image: "quay.rdoproject.org/podified-master-centos10/openstack-cloudkitty-processor:current-tested" + cloudkitty_api_image: "{{ _ck_api_image }}" + cloudkitty_proc_image: "{{ _ck_proc_image }}" + # test cloudkitty cifmw_run_tests: true cifmw_run_test_role: test_operator From b3f10fac844e268d92020b23dda1d540293d6bd7 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Mon, 8 Sep 2025 08:49:53 -0400 Subject: [PATCH 53/87] [ci] Set values for params that were not defaulted correctly databaseaccount, rabbitmqInstsance, memcachedInstance, serviceAccount and servicePassword were defaulted to "" when left out from the CR definition. This patch sets the expected values --- ci/cloudkitty-post_deploy.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/ci/cloudkitty-post_deploy.yml b/ci/cloudkitty-post_deploy.yml index a727ecc9..9a7ebe6a 100644 --- a/ci/cloudkitty-post_deploy.yml +++ b/ci/cloudkitty-post_deploy.yml @@ -13,6 +13,25 @@ # The defaulting will not be done in the webhook until CK is added to the openstack-operator. # If I change the name of the CK, I can just configure using the CK CRD. # If I don't change the name, and disable it in telemetry, then the telemetry-operator will delete the telemetry telemetry. + # WORKAROUND + # The databaseAccount, databaseInstance, memcachedInstance, rabbitMqClusterName and serviceUser do not set default values + # They should, IINM, since these are also defaulted/set for ceilometer, autoscaling, etc + # The databaseInstance needs to be created.. + # The databaseAccount needs to be created... + # I am re-using the existing memcached and rabbitmq instances + # The serviceuser should be cloudkitty, and that needs to be configured/created as well. + # databaseAccount defaults to cloudkitty, according to api/bases/openstack.org_cloudkittyes.yaml + # BUG: cloudkitty.databaseAccount is not defaulting correctly + # databaseInstance defaults to openstack, according to api/bases/openstack.org_cloudkittyes.yaml + # BUG: databaseInstance is not defaulting to the right value + # memcachedInstance is supposed to default to memcached, but does not + # serviceUser is optional + # NOTE: The failure to set the default might be some issue between the Telemetry CR and CloudKitty, since cloudkitty_types.go defines a default for memcachedInstance, rabbitmqInstance, databaseInstance. + # HOWEVER, the fact that they are all optional could be why they are not defaulting when created via telemetry CR + # They are not set to omitEmpty, which is why the empty values are propogating. + # Since the memcached instance is not omitted when empty, CK tries to resolve it and connect. + # The serviceUser should not be omitted when it is unset, since the values is used to create the cloudkitty.conf file. + # BUG: serviceUser value is not being defaulted correctly when it is omitted from the CR: expected "cloudkitty", got "" (same for the other config values mentioned above, all were set to "". - name: Patch telemetry CR ansible.builtin.shell: cmd: | @@ -34,6 +53,11 @@ containerImage: "{{ cloudkitty_api_image }}" cloudKittyProc: containerImage: "{{ cloudkitty_proc_image }}" + databaseAccount: "cloudkitty" + databaseInstance: "openstack" + memcachedInstance: "memcached" + rabbitMqClusterName: "rabbitmq" + serviceUser: "cloudkitty" EOF register: output - name: wait for the update to telemetry From dabac89fa7751e871204eaa86b10a50df6af0b5b Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Mon, 8 Sep 2025 15:05:17 -0400 Subject: [PATCH 54/87] Deploy CK with a second telemetry object --- ci/cloudkitty-post_deploy.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/ci/cloudkitty-post_deploy.yml b/ci/cloudkitty-post_deploy.yml index 9a7ebe6a..56f2e7c1 100644 --- a/ci/cloudkitty-post_deploy.yml +++ b/ci/cloudkitty-post_deploy.yml @@ -39,9 +39,15 @@ apiVersion: telemetry.openstack.org/v1beta1 kind: Telemetry metadata: - name: telemetry + name: telemetry-2 namespace: openstack spec: + ceilometer: + enabled: false + autoscaling: + enabled: false + logging: + enabled: false cloudkitty: enabled: true secret: osp-secret @@ -53,11 +59,12 @@ containerImage: "{{ cloudkitty_api_image }}" cloudKittyProc: containerImage: "{{ cloudkitty_proc_image }}" - databaseAccount: "cloudkitty" - databaseInstance: "openstack" - memcachedInstance: "memcached" - rabbitMqClusterName: "rabbitmq" - serviceUser: "cloudkitty" + preserveJobs: true + databaseAccount: cloudkitty + databaseInstance: openstack + memcachedInstance: memcached + rabbitMqClusterName: rabbitmq + serviceUser: cloudkitty EOF register: output - name: wait for the update to telemetry From 84ea94582f6310e731ba11f9f04dfa14ae604b70 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Wed, 10 Sep 2025 07:11:51 -0400 Subject: [PATCH 55/87] Update telemetry CR for enabling CK --- ci/cloudkitty-post_deploy.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/ci/cloudkitty-post_deploy.yml b/ci/cloudkitty-post_deploy.yml index 56f2e7c1..c52fb07f 100644 --- a/ci/cloudkitty-post_deploy.yml +++ b/ci/cloudkitty-post_deploy.yml @@ -32,22 +32,17 @@ # Since the memcached instance is not omitted when empty, CK tries to resolve it and connect. # The serviceUser should not be omitted when it is unset, since the values is used to create the cloudkitty.conf file. # BUG: serviceUser value is not being defaulted correctly when it is omitted from the CR: expected "cloudkitty", got "" (same for the other config values mentioned above, all were set to "". - - name: Patch telemetry CR + # # set the CK enabled to false in telemetry and create a CloudKitty CR + - name: Patch telemetry CR to enable CK ansible.builtin.shell: cmd: | oc apply -f - < Date: Thu, 11 Sep 2025 13:39:53 -0400 Subject: [PATCH 56/87] Update post deploy hook to deploy CK --- ci/cloudkitty-post_deploy.yml | 39 +++++++++++++++++------------------ 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/ci/cloudkitty-post_deploy.yml b/ci/cloudkitty-post_deploy.yml index c52fb07f..8d0c0f62 100644 --- a/ci/cloudkitty-post_deploy.yml +++ b/ci/cloudkitty-post_deploy.yml @@ -38,29 +38,28 @@ cmd: | oc apply -f - < Date: Thu, 11 Sep 2025 14:03:39 -0400 Subject: [PATCH 57/87] [ci] Add CRs for running Tempest and AnsibleTest --- ci/ansible-tests-cloudkitty.yaml | 47 ++++++++++++++ ci/cloudkitty-tempest.yaml | 103 +++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 ci/ansible-tests-cloudkitty.yaml create mode 100644 ci/cloudkitty-tempest.yaml diff --git a/ci/ansible-tests-cloudkitty.yaml b/ci/ansible-tests-cloudkitty.yaml new file mode 100644 index 00000000..a1d83530 --- /dev/null +++ b/ci/ansible-tests-cloudkitty.yaml @@ -0,0 +1,47 @@ +apiVersion: test.openstack.org/v1beta1 +kind: AnsibleTest +metadata: + name: ansible-tests-cloudkitty + namespace: openstack +spec: + ansibleGitRepo: 'http://github.com/elfiesmelfie/feature-verification-tests' + debug: true + #containerImage: quay.io/podified-antelope-centos9/openstack-ansible-tests:current-podified + containerImage: quay.io/efoley/openstack-ansible-tests@sha256:9080fa1a359de1d824585ae99bcc4f34a7400fc0a29550ea6b92c44f2975901c + #ansiblePlaybookPath: ci/run_verify_metrics_osp18.yml + storageClass: crc-csi-hostpath-provisioner + computeSSHKeySecretName: dataplane-ansible-ssh-private-key-secret + workloadSSHKeySecretName: dataplane-ansible-ssh-private-key-secret + ansibleVarFiles: | + cifmw_openshift_kubeconfig: ~/.kube.config + cifmw_path: "{{ ansible_env.PATH }}:/var/lib/ansible/bin" + cifmw_openshift_api: api.crc.testing:6443 + cifmw_openshift_password: '12345678' + cifmw_openshift_user: kubeadmin + openstack_cmd: "oc -n openstack rsh openstackclient openstack" + patch_observabilityclient: true + ansibleCollections: 'git+https://github.com/elfiesmelfie/feature-verification-tests.git,master' + privileged: false + #extraMounts: + # - extraVol: + # - mounts: + # - name: test-operator-logs + # mountPath: /var/lib/ansible/ci-framework-data/tests/feature-verification-tests + # volumes: [] + ansibleInventory: | + localhost ansible_connection=local ansible_python_interpreter=python3 + compute-0 ansible_host=192.168.122.100 ansible_user=root ansible_ssh_private_key_file=/var/lib/ansible/.ssh/compute_id + compute-1 ansible_host=192.168.122.101 ansible_user=root ansible_ssh_private_key_file=/var/lib/ansible/.ssh/compute_id + [compute] + compute-0 + compute-1 + [controller] + localhost + [local] + localhost + workflow: + - stepName: cloudkitty + ansiblePlaybookPath: ci/run_cloudkitty_tests.yml + debug: true + #- stepName: report-results + # ansiblePlaybookPath: ci/report_result.yml diff --git a/ci/cloudkitty-tempest.yaml b/ci/cloudkitty-tempest.yaml new file mode 100644 index 00000000..51428371 --- /dev/null +++ b/ci/cloudkitty-tempest.yaml @@ -0,0 +1,103 @@ +apiVersion: test.openstack.org/v1beta1 +kind: Tempest +metadata: + name: tempest-tests-cloudkitty + namespace: openstack +spec: + containerImage: quay.rdoproject.org/podified-master-centos10/openstack-tempest-all:current-tested + debug: true + openStackConfigMap: openstack-config + openStackConfigSecret: openstack-config-secret + parallel: false + privileged: true + rerunFailedTests: false + storageClass: local-storage + tempestRun: + concurrency: 2 + excludeList: | + tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_mtu_sized_frames + + externalPlugin: + - repository: https://opendev.org/openstack/cloudkitty-tempest-plugin.git + includeList: | + #^tempest.*\[.*\bsmoke\b.*\] + cloudkitty_tempest_plugin.* + #telemetry_tempest_plugin.* + excludeList: | + ^tempest* + ^*tempest* + parallel: true + serial: false + smoke: false + workerFile: "" + tempestconfRun: + append: "" + collectTiming: false + convertToRaw: false + create: true + createAccountsFile: "" + debug: false + deployerInput: | + [auth] + tempest_roles = + + [enforce_scope] + barbican = true + cinder = true + designate = true + glance = true + ironic = true + ironic_inspector = true + neutron = true + nova = true + octavia = true + keystone = true + manila = true + placement = true + + [identity-feature-enabled] + enforce_scope = true + + [compute-feature-enabled] + dhcp_domain = '' + + [load_balancer] + member_role = load-balancer_member + admin_role = load-balancer_admin + RBAC_test_type = keystone_default_roles + enforce_new_defaults = true + enforce_scope = false + + [volume] + catalog_type = volumev3 + flavorMinDisk: 0 + flavorMinMem: 0 + generateProfile: "" + image: "" + imageDiskFormat: "" + insecure: false + networkID: "" + noDefaultDeployer: false + nonAdmin: false + out: "" + overrides: |- + validation.run_validation true + identity.v3_endpoint_type public + service_available.ceilometer true + service_available.sg_core true + service_available.aodh true + service_available.cinder false + telemetry.sg_core_service_url "https://ceilometer-internal.openstack.svc.cluster.local:3000" + telemetry.prometheus_service_url "https://metric-storage-prometheus.openstack.svc.cluster.local:9090" + telemetry.ceilometer_polling_interval 120 + telemetry.prometheus_scrape_interval 30 + telemetry.alarm_threshold 50000000000 + whitebox_neutron_plugin_options.proxy_host_address 38.102.83.182 + profile: "" + remove: "" + retryImage: false + testAccounts: "" + timeout: 0 + verbose: false + workflow: [] + From a409c1ea30eb15328ee8acef29338272ceaff35c Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 12 Sep 2025 11:59:38 -0400 Subject: [PATCH 58/87] [ci] Update AnsibleTest Cr for cloudkitty --- ci/ansible-tests-cloudkitty.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/ansible-tests-cloudkitty.yaml b/ci/ansible-tests-cloudkitty.yaml index a1d83530..459d4609 100644 --- a/ci/ansible-tests-cloudkitty.yaml +++ b/ci/ansible-tests-cloudkitty.yaml @@ -6,8 +6,8 @@ metadata: spec: ansibleGitRepo: 'http://github.com/elfiesmelfie/feature-verification-tests' debug: true - #containerImage: quay.io/podified-antelope-centos9/openstack-ansible-tests:current-podified - containerImage: quay.io/efoley/openstack-ansible-tests@sha256:9080fa1a359de1d824585ae99bcc4f34a7400fc0a29550ea6b92c44f2975901c + containerImage: quay.io/podified-antelope-centos9/openstack-ansible-tests:current-podified + #containerImage: quay.io/efoley/openstack-ansible-tests@sha256:9080fa1a359de1d824585ae99bcc4f34a7400fc0a29550ea6b92c44f2975901c #ansiblePlaybookPath: ci/run_verify_metrics_osp18.yml storageClass: crc-csi-hostpath-provisioner computeSSHKeySecretName: dataplane-ansible-ssh-private-key-secret From 02c8fe41c31d77b25c7f0cc6790ba36e12bde880 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 12 Sep 2025 12:09:22 -0400 Subject: [PATCH 59/87] Add notes file --- notes | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 notes diff --git a/notes b/notes new file mode 100644 index 00000000..acd030da --- /dev/null +++ b/notes @@ -0,0 +1,75 @@ +* I can get to trying to init storage. + I don't have the right version of CK + +oc logs job/cloudkitty-storageinit + + sudo -E kolla_set_configs + sudo: unable to send audit message: Operation not permitted + + ++ cat /run_command + + CMD=/usr/bin/cloudkitty-storage-init + + ARGS= + + sudo kolla_copy_cacerts + sudo: unable to send audit message: Operation not permitted + + + exec /usr/bin/cloudkitty-storage-init + 2025-09-08 11:47:36.433 1 WARNING cloudkitty.storage [-] V2 Storage is in beta. Its API is considered stable but its implementation may still evolve. + 2025-09-08 11:47:36.641 1 WARNING stevedore.named [-] Could not load loki + 2025-09-08 11:47:36.641 1 CRITICAL cloudkitty [-] Unhandled error: stevedore.exception.NoMatches: No 'cloudkitty.storage.v2.backends' driver found, looking for 'loki' + 2025-09-08 11:47:36.641 1 ERROR cloudkitty Traceback (most recent call last): + 2025-09-08 11:47:36.641 1 ERROR cloudkitty File "/usr/bin/cloudkitty-storage-init", line 8, in + 2025-09-08 11:47:36.641 1 ERROR cloudkitty sys.exit(main()) + 2025-09-08 11:47:36.641 1 ERROR cloudkitty ^^^^^^ + 2025-09-08 11:47:36.641 1 ERROR cloudkitty File "/usr/lib/python3.12/site-packages/cloudkitty/cli/storage.py", line 30, in main + 2025-09-08 11:47:36.641 1 ERROR cloudkitty init_storage_backend() + 2025-09-08 11:47:36.641 1 ERROR cloudkitty File "/usr/lib/python3.12/site-packages/cloudkitty/cli/storage.py", line 22, in init_storage_backend + 2025-09-08 11:47:36.641 1 ERROR cloudkitty backend = storage.get_storage() + 2025-09-08 11:47:36.641 1 ERROR cloudkitty ^^^^^^^^^^^^^^^^^^^^^ + 2025-09-08 11:47:36.641 1 ERROR cloudkitty File "/usr/lib/python3.12/site-packages/cloudkitty/storage/__init__.py", line 211, in get_storage + 2025-09-08 11:47:36.641 1 ERROR cloudkitty return _get_storage_instance( + 2025-09-08 11:47:36.641 1 ERROR cloudkitty ^^^^^^^^^^^^^^^^^^^^^^ + 2025-09-08 11:47:36.641 1 ERROR cloudkitty File "/usr/lib/python3.12/site-packages/cloudkitty/storage/__init__.py", line 56, in _get_storage_instance + 2025-09-08 11:47:36.641 1 ERROR cloudkitty return driver.DriverManager( + 2025-09-08 11:47:36.641 1 ERROR cloudkitty ^^^^^^^^^^^^^^^^^^^^^ + 2025-09-08 11:47:36.641 1 ERROR cloudkitty File "/usr/lib/python3.12/site-packages/stevedore/driver.py", line 54, in __init__ + 2025-09-08 11:47:36.641 1 ERROR cloudkitty super().__init__( + 2025-09-08 11:47:36.641 1 ERROR cloudkitty File "/usr/lib/python3.12/site-packages/stevedore/named.py", line 89, in __init__ + 2025-09-08 11:47:36.641 1 ERROR cloudkitty self._init_plugins(extensions) + 2025-09-08 11:47:36.641 1 ERROR cloudkitty File "/usr/lib/python3.12/site-packages/stevedore/driver.py", line 113, in _init_plugins + 2025-09-08 11:47:36.641 1 ERROR cloudkitty raise NoMatches('No %r driver found, looking for %r' % + 2025-09-08 11:47:36.641 1 ERROR cloudkitty stevedore.exception.NoMatches: No 'cloudkitty.storage.v2.backends' driver found, looking for 'loki' + 2025-09-08 11:47:36.641 1 ERROR cloudkitty + + + +Tempest +======= +To run tempest, you need to do a hack until https://github.com/openstack-k8s-operators/tcib/pull/328 is merged. +This is required because the upper constraints file is incorrect for the master-centos10 containers. + +To work around this locally, you can create the tempest CR to get the pods running. + [telemetry-operator] $ oc apply -f telemetry-with-cloudkitty.yaml +Once the pod is up and running,rsh into the container + + $ oc rsh tempest-tests-cloudkitty + +Then, edit the run_tempest.sh script as root and update it to match this: https://github.com/openstack-k8s-operators/tcib/pull/328/files + +To re-run the tests, you need to clean up a little: + + $ rm -rf cloudkitty-tempest-plugin/ /var/lib/tempest/openshift + +Then re-run the tests: + $ /usr/local/bin/run_tempest.sh + +The results will be available in /var/lib/tempest/external_files//tempest-tests-cloudkitty//tempest_results.xml + +Updating the config +=================== +BUG +When the config for CK is updated, the pods don't automatically get restarted with the new config. + +Manually test by editing the telemetry to add e.g. a new custom service config value +Automatically test with envtest + +Either changes top the configmap/secret is not being added to the hash, or there needs to be a watch added for changes to configmap/secret. + From 76990f5bb66ba1c524c972b07eea11336680627d Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 12 Sep 2025 12:12:33 -0400 Subject: [PATCH 60/87] [ci] Update teh storage class for CK tempest --- ci/cloudkitty-tempest.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/cloudkitty-tempest.yaml b/ci/cloudkitty-tempest.yaml index 51428371..255a7680 100644 --- a/ci/cloudkitty-tempest.yaml +++ b/ci/cloudkitty-tempest.yaml @@ -11,7 +11,8 @@ spec: parallel: false privileged: true rerunFailedTests: false - storageClass: local-storage + #storageClass: local-storage + storageClass: crc-csi-hostpath-provisioner tempestRun: concurrency: 2 excludeList: | From 322f17ceeb6cfd27cdb877e2b495a4c5c29bd56a Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 12 Sep 2025 13:26:32 -0400 Subject: [PATCH 61/87] [ci] Update cloudkitty-tempest --- ci/cloudkitty-tempest.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/cloudkitty-tempest.yaml b/ci/cloudkitty-tempest.yaml index 255a7680..7167f821 100644 --- a/ci/cloudkitty-tempest.yaml +++ b/ci/cloudkitty-tempest.yaml @@ -4,7 +4,9 @@ metadata: name: tempest-tests-cloudkitty namespace: openstack spec: - containerImage: quay.rdoproject.org/podified-master-centos10/openstack-tempest-all:current-tested + # currently doesn't work with c10s containers, but works cith c9s + #containerImage: quay.rdoproject.org/podified-master-centos10/openstack-tempest-all:current-tested + containerImage: quay.io/podified-antelope-centos9/openstack-tempest-all:current-podified debug: true openStackConfigMap: openstack-config openStackConfigSecret: openstack-config-secret @@ -24,9 +26,8 @@ spec: #^tempest.*\[.*\bsmoke\b.*\] cloudkitty_tempest_plugin.* #telemetry_tempest_plugin.* - excludeList: | - ^tempest* - ^*tempest* + #excludeList: | + # ^tempest* parallel: true serial: false smoke: false From 96b588be28710ed30a9fb14451fb24cb57544eac Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 12 Sep 2025 13:46:02 -0400 Subject: [PATCH 62/87] Add CR for deploying telemetry with CloudKitty --- telemetry-with-cloudkitty.yaml | 60 ++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 telemetry-with-cloudkitty.yaml diff --git a/telemetry-with-cloudkitty.yaml b/telemetry-with-cloudkitty.yaml new file mode 100644 index 00000000..cac13442 --- /dev/null +++ b/telemetry-with-cloudkitty.yaml @@ -0,0 +1,60 @@ +apiVersion: telemetry.openstack.org/v1beta1 +kind: Telemetry +metadata: + name: telemetry-ck +spec: + metricStorage: + enabled: true + monitoringStack: + alertingEnabled: true + scrapeInterval: 30s + storage: + strategy: persistent + retention: 24h + persistent: + pvcStorageRequest: 20G + autoscaling: + enabled: false + aodh: + passwordSelectors: + databaseAccount: aodh + databaseInstance: openstack + memcachedInstance: memcached + secret: osp-secret + heatInstance: heat + ceilometer: + enabled: true + secret: osp-secret + cloudkitty: + enabled: true + cloudKittyAPI: + containerImage: "quay.io/efoley/cloudkitty-api@sha256:9cfa630cd297b09fe77c0eedfcdf464bb9c3f5398357dbb1d8343a494900e6d1" + #containerImage: "quay.io/efoley/cloudkitty-api:epoxy-loki" + customServiceConfig: | + [storage_loki] + url = lokistack-openstack.apps-crc.testing + tls: + caBundleSecretName: combined-ca-bundle + cloudKittyProc: + containerImage: "quay.io/efoley/cloudkitty-processor@sha256:a6c9caed1b0baeda3df710686ae9e8cf4c915eb59c845072eb91c13de0f68450" + #containerImage: "quay.io/efoley/cloudkitty-processor:epoxy-loki" + customServiceConfig: | + [storage_loki] + url = lokistack-openstack.apps-crc.testing + tls: + caBundleSecretName: combined-ca-bundle + databaseAccount: cloudkitty + databaseInstance: openstack + memcachedInstance: memcached + passwordSelector: + aodhService: AodhPassword + ceilometerService: CeilometerPassword + cloudKittyService: CloudKittyPassword + preserveJobs: false + rabbitMqClusterName: rabbitmq + secret: "osp-secret" + serviceUser: cloudkitty + + logging: + enabled: false + port: 10514 From 3f02868cb204f7a4db27c17d88deacf14d9e959d Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 12 Sep 2025 14:08:22 -0400 Subject: [PATCH 63/87] Add loki files rename lokistack --- cloudkitty-loki/README | 42 +++++++++++++++++ cloudkitty-loki/certificate-internal.yaml | 27 +++++++++++ cloudkitty-loki/certificate-public.yaml | 26 +++++++++++ cloudkitty-loki/loki-secret.yaml | 11 +++++ cloudkitty-loki/loki-subscription.yaml | 24 ++++++++++ cloudkitty-loki/lokistack.yaml | 56 +++++++++++++++++++++++ 6 files changed, 186 insertions(+) create mode 100644 cloudkitty-loki/README create mode 100644 cloudkitty-loki/certificate-internal.yaml create mode 100644 cloudkitty-loki/certificate-public.yaml create mode 100644 cloudkitty-loki/loki-secret.yaml create mode 100644 cloudkitty-loki/loki-subscription.yaml create mode 100644 cloudkitty-loki/lokistack.yaml diff --git a/cloudkitty-loki/README b/cloudkitty-loki/README new file mode 100644 index 00000000..7c42639c --- /dev/null +++ b/cloudkitty-loki/README @@ -0,0 +1,42 @@ +# CloudKitty Loki Deployment with Red Hat Loki Operator + +1. Create the internal certificate +``` +$ oc apply -f certificate-internal.yaml +$ oc apply -f certificate-public.yaml +``` + +2. Create ConfigMap +``` +$ kubectl create configmap lokistack-mtls --from-literal=ca.crt="$(kubectl get secret cert-loki-public-route -n openstack -o jsonpath='{.data.ca\.crt}' | base64 -d)" -n openstack +``` + +# Need to set up S3 +oc apply -f /ci/deploy-logging-dependencies/files/minio-dev.yaml + +oc apply -f loki-subscription.yaml + +3. Edit loki-secret.yaml with the appropiate data to connect to S3: +``` + access_key_id: + access_key_secret: + bucketnames: + endpoint: +``` + +4. Create LokiStack: +``` +$ oc apply -f lokistack.yaml +``` + +5. Check that Loki pods have been created in the openstack namespace: +``` +lokistack-compactor-0 1/1 Running 0 18m +lokistack-distributor-65c994c58-c84rw 1/1 Running 0 18m +lokistack-gateway-6dbbc4d789-lnjdv 1/1 Running 0 18m +lokistack-gateway-6dbbc4d789-s8pwx 1/1 Running 0 18m +lokistack-index-gateway-0 1/1 Running 0 18m +lokistack-ingester-0 1/1 Running 0 18m +lokistack-querier-7f887c4f89-rw7qb 1/1 Running 0 18m +lokistack-query-frontend-57576979bf-qbxw7 1/1 Running 0 18m +``` diff --git a/cloudkitty-loki/certificate-internal.yaml b/cloudkitty-loki/certificate-internal.yaml new file mode 100644 index 00000000..a21c1a8c --- /dev/null +++ b/cloudkitty-loki/certificate-internal.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: cert-loki-internal + namespace: openstack +spec: + secretName: cert-loki-internal-svc + commonName: cloudkitty-lokistack-gateway-http.openstack.svc + dnsNames: + - cloudkitty-lokistack-gateway-http.openstack.svc + - cloudkitty-lokistack-gateway-http.openstack.svc.cluster.local + subject: + ## This is what binds the client to the role cloudkitty definedi in LokiStack + organizationalUnits: + - cloudkitty + privateKey: + algorithm: RSA + size: 2048 + encoding: PKCS8 + usages: + - digital signature + - key encipherment + - client auth + issuerRef: + kind: Issuer + name: rootca-internal diff --git a/cloudkitty-loki/certificate-public.yaml b/cloudkitty-loki/certificate-public.yaml new file mode 100644 index 00000000..42528c30 --- /dev/null +++ b/cloudkitty-loki/certificate-public.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: cert-loki-public + namespace: openstack +spec: + secretName: cert-loki-public-route + commonName: lokistack-openstack.apps-crc.testing + dnsNames: + - lokistack-openstack.apps-crc.testing + subject: + ## This is what binds the client to the role cloudkitty defined in LokiStack + organizationalUnits: + - cloudkitty + privateKey: + algorithm: RSA + size: 2048 + encoding: PKCS8 + usages: + - digital signature + - key encipherment + - client auth + issuerRef: + kind: Issuer + name: rootca-public diff --git a/cloudkitty-loki/loki-secret.yaml b/cloudkitty-loki/loki-secret.yaml new file mode 100644 index 00000000..18249eba --- /dev/null +++ b/cloudkitty-loki/loki-secret.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: loki-secret-s3 + namespace: openstack +stringData: + access_key_id: minio + access_key_secret: minio123 + bucketnames: cloudkitty-bucket + endpoint: http://api-minio-dev.apps-crc.testing + diff --git a/cloudkitty-loki/loki-subscription.yaml b/cloudkitty-loki/loki-subscription.yaml new file mode 100644 index 00000000..f57a525a --- /dev/null +++ b/cloudkitty-loki/loki-subscription.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-operators-redhat + annotations: + openshift.io/node-selector: "" + labels: + openshift.io/cluster-monitoring: "true" +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + labels: + operators.coreos.com/loki-operator.openshift-operators-redhat: "" + name: loki-operator + namespace: openshift-operators-redhat +spec: + channel: stable-6.3 + installPlanApproval: Automatic + name: loki-operator + source: redhat-operators + sourceNamespace: openshift-marketplace + # startingCSV: loki-operator.v6.3.0 + diff --git a/cloudkitty-loki/lokistack.yaml b/cloudkitty-loki/lokistack.yaml new file mode 100644 index 00000000..a0bc0a46 --- /dev/null +++ b/cloudkitty-loki/lokistack.yaml @@ -0,0 +1,56 @@ +--- +apiVersion: loki.grafana.com/v1 +kind: LokiStack +metadata: + name: cloudkitty-lokistack + namespace: openstack +spec: + size: 1x.demo + storage: + schemas: + - version: v13 + effectiveDate: "2024-11-18" + secret: + name: loki-secret-s3 + type: s3 + storageClassName: crc-csi-hostpath-provisioner + tenants: + mode: static + authentication: + - tenantName: cloudkitty + tenantId: cloudkitty + mTLS: + ca: + caKey: ca.crt + caName: lokistack-mtls + authorization: + roleBindings: + - name: cloudkitty-dataframes + roles: + - cloudkitty-dataframes + subjects: + - kind: group + name: cloudkitty + - name: cluster-reader + roles: + - cluster-reader + subjects: + - kind: group + name: cloudkitty-dataframes-admin + roles: + - name: cloudkitty-dataframes + permissions: + - read + - write + resources: + - logs + tenants: + - cloudkitty + - name: cluster-reader + permissions: + - read + resources: + - logs + tenants: + - cloudkitty + From 59abc06fc3b8201561155299ebb7135a3dc6e2f6 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 12 Sep 2025 17:10:47 -0400 Subject: [PATCH 64/87] Update telemetry+cloudkitty CR to use different loki URL --- telemetry-with-cloudkitty.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/telemetry-with-cloudkitty.yaml b/telemetry-with-cloudkitty.yaml index cac13442..47a4ce0f 100644 --- a/telemetry-with-cloudkitty.yaml +++ b/telemetry-with-cloudkitty.yaml @@ -32,7 +32,8 @@ spec: #containerImage: "quay.io/efoley/cloudkitty-api:epoxy-loki" customServiceConfig: | [storage_loki] - url = lokistack-openstack.apps-crc.testing + url = https://cloudkitty-lokistack-gateway-http.openstack.svc:8080 + #url = https://lokistack-openstack.apps-crc.testing tls: caBundleSecretName: combined-ca-bundle cloudKittyProc: @@ -40,7 +41,8 @@ spec: #containerImage: "quay.io/efoley/cloudkitty-processor:epoxy-loki" customServiceConfig: | [storage_loki] - url = lokistack-openstack.apps-crc.testing + url = https://cloudkitty-lokistack-gateway-http.openstack.svc:8080 + #url = https://lokistack-openstack.apps-crc.testing tls: caBundleSecretName: combined-ca-bundle databaseAccount: cloudkitty From a83f9d708da48ec0ad379465776214b3eb909c16 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Mon, 15 Sep 2025 12:17:09 -0400 Subject: [PATCH 65/87] [WORKAROUND] The CK RPMS are not building in metacontent provider To workaround this issue, the regular content provider is used and hardcoded images are used for deploying CK ref: https://github.com/openstack-k8s-operators/telemetry-operator/pull/720#issuecomment-3292814710 --- ci/cloudkitty-post_deploy.yml | 6 ++++-- zuul.d/projects.yaml | 7 ++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ci/cloudkitty-post_deploy.yml b/ci/cloudkitty-post_deploy.yml index 8d0c0f62..0bef3965 100644 --- a/ci/cloudkitty-post_deploy.yml +++ b/ci/cloudkitty-post_deploy.yml @@ -50,9 +50,11 @@ ceilometerService: CeilometerPassword cloudKittyService: CloudKittyPassword cloudKittyAPI: - containerImage: "{{ cloudkitty_api_image }}" + # containerImage: "{{ cloudkitty_api_image }}" + containerImage: "quay.io/efoley/cloudkitty-api" cloudKittyProc: - containerImage: "{{ cloudkitty_proc_image }}" + # containerImage: "{{ cloudkitty_proc_image }}" + containerImage: "quay.io/efoley/cloudkitty-processor" preserveJobs: true databaseAccount: cloudkitty databaseInstance: openstack diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index 00d88ddc..ca869992 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -162,7 +162,12 @@ github-check: #debug: true jobs: - - telemetry-operator-multinode-cloudkitty + - telemetry-operator-multinode-cloudkitty: + dependencies: + - openstack-k8s-operators-content-provider + # temperorily use the regular content provider since CK rpm builds are failing + - openstack-k8s-operators-content-provider + # Keep the metacontent provider in the pipeline so we know when it is working again - telemetry-openstack-meta-content-provider-master #- telemetry-operator-multinode-default-telemetry #- functional-tests-osp18: &fvt_jobs_config From 8b2cc8b426232075a8b9d8f5ce956888f005ae7d Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Tue, 16 Sep 2025 06:49:12 -0400 Subject: [PATCH 66/87] Update CK post deploy hook --- ci/cloudkitty-post_deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/cloudkitty-post_deploy.yml b/ci/cloudkitty-post_deploy.yml index 0bef3965..3ec9c2a5 100644 --- a/ci/cloudkitty-post_deploy.yml +++ b/ci/cloudkitty-post_deploy.yml @@ -61,7 +61,7 @@ memcachedInstance: memcached rabbitMqClusterName: rabbitmq serviceUser: cloudkitty - EOF + EOF register: output - name: wait for the update to telemetry From 13841a1d2761cf4167eede80900aff2fc0a73b6e Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Wed, 17 Sep 2025 08:55:17 -0400 Subject: [PATCH 67/87] [ci] Use the antelope/centos9 image for tempest The master image uses the antelope constraints and the CK plugin installation fails at the moment. The antelope tempest image can be used until https://github.com/openstack-k8s-operators/tcib/pull/328 is merged --- ci/vars-cloudkitty-tempest.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/vars-cloudkitty-tempest.yml b/ci/vars-cloudkitty-tempest.yml index e755b5e6..3f5835a7 100644 --- a/ci/vars-cloudkitty-tempest.yml +++ b/ci/vars-cloudkitty-tempest.yml @@ -57,7 +57,10 @@ post_deploy_enable_cloudkitty: cifmw_run_tests: true cifmw_run_test_role: test_operator # TODO: Consider switching to podified-master-centos10 for features that patch master -cifmw_test_operator_tempest_namespace: podified-master-centos10 +# This also needs the registry_url configured. +# The master container is using an old constraints file, so revertng to antelope until https://github.com/openstack-k8s-operators/tcib/pull/328 is merged +#cifmw_test_operator_tempest_namespace: podified-master-centos10 +# # cloudkitty tempest plugin is not part of the tempest rpm. # https://review.rdoproject.org/cgit/openstack/tempest-distgit/tree/openstack-tempest.spec # We need to add the cloudkitty-tempest-plugin package to RDO, same as TTTP From bc6f91038b15301af10fb2f80744dfcc4bb33e56 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Wed, 17 Sep 2025 09:36:35 -0400 Subject: [PATCH 68/87] Update post-deploy hook to create new telemetry --- ci/cloudkitty-post_deploy.yml | 78 ++++++++++++++++++++-------------- telemetry-with-cloudkitty.yaml | 31 +++++++++----- 2 files changed, 67 insertions(+), 42 deletions(-) diff --git a/ci/cloudkitty-post_deploy.yml b/ci/cloudkitty-post_deploy.yml index 3ec9c2a5..d96f84b6 100644 --- a/ci/cloudkitty-post_deploy.yml +++ b/ci/cloudkitty-post_deploy.yml @@ -7,6 +7,17 @@ KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" PATH: "{{ cifmw_path }}" tasks: + - name: Disable telemetry in the openstack controlplane + ansible.builtin.shell: + cmd: | + oc patch oscp $(oc get oscp -ojsonpath='{ .items[].metadata.name }') --type=json -p='[{"op": "replace", "path": "/spec/telemetry/enabled", "value": false} ]' + + - name: Create the new telemetry + ansible.builtin.shell: + cmd: | + oc apply -f {{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/telemetry-with-cloudkitty.yaml + + # WORKAROUND # Need to set the images, since the defaulting is not working at the moment. # The container image is not being set, so the dbsync is failing. @@ -32,40 +43,45 @@ # Since the memcached instance is not omitted when empty, CK tries to resolve it and connect. # The serviceUser should not be omitted when it is unset, since the values is used to create the cloudkitty.conf file. # BUG: serviceUser value is not being defaulted correctly when it is omitted from the CR: expected "cloudkitty", got "" (same for the other config values mentioned above, all were set to "". + # I think this is because of the openstack operator. I need to create a new telemetry, with a different name... But I tried that before... # # set the CK enabled to false in telemetry and create a CloudKitty CR - - name: Patch telemetry CR to enable CK - ansible.builtin.shell: - cmd: | - oc apply -f - < Date: Tue, 7 Oct 2025 11:27:32 -0400 Subject: [PATCH 69/87] Update the timeout on check for telemetry/telemetry-ck readiness --- ci/cloudkitty-post_deploy.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/cloudkitty-post_deploy.yml b/ci/cloudkitty-post_deploy.yml index d96f84b6..7128f5eb 100644 --- a/ci/cloudkitty-post_deploy.yml +++ b/ci/cloudkitty-post_deploy.yml @@ -79,7 +79,8 @@ - name: wait for the update to telemetry ansible.builtin.shell: cmd: | - oc wait --for=condition=Ready telemetry telemetry-ck + oc wait --for=condition=Ready --timeout=600s telemetry telemetry-ck + changed_when: false - name: Check for secrets ansible.builtin.shell: From a5ac6cdfad62bca6d78c7964a96c3ffa1cfb7fa5 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Wed, 17 Sep 2025 13:57:08 -0400 Subject: [PATCH 70/87] [ci] Update OSCP kustomisation and loki secret --- ci/configure-cloudkitty.yml | 77 +++++++++++++++++--------------- ci/vars-cloudkitty-tempest.yml | 7 ++- cloudkitty-loki/loki-secret.yaml | 2 +- cloudkitty-loki/lokistack.yaml | 2 +- telemetry-with-cloudkitty.yaml | 2 +- 5 files changed, 49 insertions(+), 41 deletions(-) diff --git a/ci/configure-cloudkitty.yml b/ci/configure-cloudkitty.yml index 547fe14a..a830348c 100644 --- a/ci/configure-cloudkitty.yml +++ b/ci/configure-cloudkitty.yml @@ -25,46 +25,51 @@ name: unused spec: telemetry: - enabled: true + enabled: false template: cloudkitty: - enabled: true + s3StorageConfig: + secret: + type: "s3" + name: "cloudkitty-loki-s3" target: kind: OpenStackControlPlane # This probably won't actually work, since there's no telemetry object yet at this point. # The kustomization needs to be done post_deploy - - name: Copy telemetry kustomization - ansible.builtin.copy: - dest: "{{ cifmw_basedir }}/artifacts/manifests/kustomizations/controlplane/90-kustomize-telemetry-cloudkitty.yaml" - content: |- - apiVersion: kustomize.config.k8s.io/v1beta1 - kind: Kustomization - namespace: openstack - patches: - - patch: |- - apiVersion: core.openstack.org/v1beta1 - kind: Telemetry - metadata: - name: unused - spec: - cloudkitty: - enabled: true - secret: osp-secret - passwordSelector: - aodhService: AodhPassword - ceilometerService: CeilometerPassword - cloudKittyService: CloudKittyPassword - # WORKAROUND - # Need to set the images, since the defaulting is not working at the moment. - # The container image is not being set, so the dbsync is failing. - # The defaulting will not be done in the webhook until CK is added to the openstack-operator. - # If I change the name of the CK, I can just configure using the CK CRD. - # If I don't change the name, and disable it in telemetry, then the telemetry-operator will delete the telemetry telemetry. - cloudKittyAPI: - containerImage: "{{ cloudkitty_api_image }}" - cloudKittyProc: - containerImage: "{{ cloudkitty_proc_image }}" - target: - kind: Telemetry - + # - name: Copy telemetry kustomization + # ansible.builtin.copy: + # dest: "{{ cifmw_basedir }}/artifacts/manifests/kustomizations/controlplane/90-kustomize-telemetry-cloudkitty.yaml" + # content: |- + # apiVersion: kustomize.config.k8s.io/v1beta1 + # kind: Kustomization + # namespace: openstack + # patches: + # - patch: |- + # apiVersion: core.openstack.org/v1beta1 + # kind: Telemetry + # metadata: + # name: telemetry-ck + # spec: + # cloudkitty: + # enabled: true + # secret: osp-secret + # passwordSelector: + # aodhService: AodhPassword + # ceilometerService: CeilometerPassword + # cloudKittyService: CloudKittyPassword + # # WORKAROUND + # # Need to set the images, since the defaulting is not working at the moment. + # # The container image is not being set, so the dbsync is failing. + # # The defaulting will not be done in the webhook until CK is added to the openstack-operator. + # # If I change the name of the CK, I can just configure using the CK CRD. + # # If I don't change the name, and disable it in telemetry, then the telemetry-operator will delete the telemetry telemetry. + # cloudKittyAPI: + # # containerImage: "{{ cloudkitty_api_image }}" + # containerImage: "quay.io/efoley/cloudkitty-api@sha256:baf3f44225be4ac55eef6e45ab4095a8284fc186da5916408dbf95767a1eb5e9" + # cloudKittyProc: + # # containerImage: "{{ cloudkitty_proc_image }}" + # containerImage: "quay.io/efoley/cloudkitty-processor@sha256:4ddd2f824b17c0c2fb39363369a16af49cb2d95c07916f52e2c509ceca677de9" + # target: + # kind: Telemetry + # diff --git a/ci/vars-cloudkitty-tempest.yml b/ci/vars-cloudkitty-tempest.yml index 3f5835a7..d5cd7397 100644 --- a/ci/vars-cloudkitty-tempest.yml +++ b/ci/vars-cloudkitty-tempest.yml @@ -36,13 +36,16 @@ _ck_proc_image: "{{ telemetry_registry }}/openstack-cloudkitty-processor:{{ tele # This is removed for now. #pre_deploy_kustomize_cloudkitty: -# source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/configure-cloudkitty.yml" -# type: playbook +pre_deploy_disable_telemetry: + source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/configure-cloudkitty.yml" + type: playbook # extra_vars: # cloudkitty_api_image: "{{ _ck_api_image }} # cloudkitty_proc_image: "{{ _ck_proc_image }}" # TODO: Adapt the deploy-logging-dependencies role to allow config necessary for CK # # I can remove the vars-logging from the job definition. +# Need to skip creating the lokistack for CK, and add another secret for CK/lokistack +# This is already being run in the CI job in the vars-logging cifmw-extras #pre_deploy_deploy_logging_dependencies: # source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/deploy-logging-dependencies.yml" # type: playbook diff --git a/cloudkitty-loki/loki-secret.yaml b/cloudkitty-loki/loki-secret.yaml index 18249eba..9c65c464 100644 --- a/cloudkitty-loki/loki-secret.yaml +++ b/cloudkitty-loki/loki-secret.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Secret metadata: - name: loki-secret-s3 + name: cloudkitty-loki-s3 namespace: openstack stringData: access_key_id: minio diff --git a/cloudkitty-loki/lokistack.yaml b/cloudkitty-loki/lokistack.yaml index a0bc0a46..3e1955b0 100644 --- a/cloudkitty-loki/lokistack.yaml +++ b/cloudkitty-loki/lokistack.yaml @@ -11,7 +11,7 @@ spec: - version: v13 effectiveDate: "2024-11-18" secret: - name: loki-secret-s3 + name: cloudkitty-loki-s3 type: s3 storageClassName: crc-csi-hostpath-provisioner tenants: diff --git a/telemetry-with-cloudkitty.yaml b/telemetry-with-cloudkitty.yaml index fb55ce74..50f79c6a 100644 --- a/telemetry-with-cloudkitty.yaml +++ b/telemetry-with-cloudkitty.yaml @@ -60,7 +60,7 @@ spec: - effectiveDate: "2024-11-18" version: v13 secret: - name: loki-secret-s3 + name: cloudkitty-loki-s3 type: "s3" secret: "osp-secret" serviceUser: cloudkitty From a0478e4e98ebc0774277869f9ef244e885fd2b82 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Thu, 18 Sep 2025 07:13:02 -0400 Subject: [PATCH 71/87] [ci] Update oscp patch to keep telemetry enabled pre-deploy If telemetry is disabled, the secrets that the dataplane deployment expects are not created i.e. ceilometer-compute-config-data --- ci/configure-cloudkitty.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/configure-cloudkitty.yml b/ci/configure-cloudkitty.yml index a830348c..b9428f52 100644 --- a/ci/configure-cloudkitty.yml +++ b/ci/configure-cloudkitty.yml @@ -25,9 +25,11 @@ name: unused spec: telemetry: - enabled: false + # this is needed so the secrets are created for dataplane deployment + enabled: true template: cloudkitty: + enabled: false s3StorageConfig: secret: type: "s3" From 9a6ddd6fce245e09c80391fe8e9a8d62ff8c61ac Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Thu, 18 Sep 2025 09:21:15 -0400 Subject: [PATCH 72/87] [ci] Update oscp patch to disable autoscaling and logging --- ci/configure-cloudkitty.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/configure-cloudkitty.yml b/ci/configure-cloudkitty.yml index b9428f52..36204900 100644 --- a/ci/configure-cloudkitty.yml +++ b/ci/configure-cloudkitty.yml @@ -28,6 +28,10 @@ # this is needed so the secrets are created for dataplane deployment enabled: true template: + logging: + enabled: false + autoscaling: + enabled: false cloudkitty: enabled: false s3StorageConfig: From 4109c5fe63c58df82773d8bb44ab45ed99cde735 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Thu, 18 Sep 2025 14:38:15 -0400 Subject: [PATCH 73/87] [ci] Remove logging deps from ck job TODO: update the deploy_logging_dependencies role --- zuul.d/projects.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index ca869992..9e4c73ea 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -119,8 +119,6 @@ cifmw_update_containers: false cifmw_extras: - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir }}/scenarios/centos-9/multinode-ci.yml" - # Deploys logging dependencies (e.g. loki) - - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-logging.yml" # Need a config for CK - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-cloudkitty-tempest.yml" - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-use-master-containers.yml" From 8e3191a5581bcd6606470a350b560b561e416359 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 19 Sep 2025 08:19:13 -0400 Subject: [PATCH 74/87] WORKAROUND CI env is missing required values set for telemetry-ck When running in CI, with a differently named CR, the defaults are not set for Telemetry - 'The Telemetry "telemetry-ck" is invalid: ' - '* spec.autoscaling.aodh.apiImage: Required value' - '* spec.autoscaling.aodh.evaluatorImage: Required value' - '* spec.autoscaling.aodh.listenerImage: Required value' - '* spec.autoscaling.aodh.notifierImage: Required value' - '* spec.ceilometer.centralImage: Required value' - '* spec.ceilometer.computeImage: Required value' - '* spec.ceilometer.ipmiImage: Required value' - '* spec.ceilometer.notificationImage: Required value' - '* spec.ceilometer.proxyImage: Required value' - '* spec.ceilometer.sgCoreImage: Required value' WORKAROUND: telemetry-with-cloudkitty is updated to explicitly add these values --- ci/cloudkitty-post_deploy.yml | 14 ++++++++++++++ telemetry-with-cloudkitty.yaml | 13 +++++++++++++ 2 files changed, 27 insertions(+) diff --git a/ci/cloudkitty-post_deploy.yml b/ci/cloudkitty-post_deploy.yml index 7128f5eb..611b37e0 100644 --- a/ci/cloudkitty-post_deploy.yml +++ b/ci/cloudkitty-post_deploy.yml @@ -1,4 +1,18 @@ --- +# When running in CI, with a differently named CR, the defaults are not set for Telemetry +# - 'The Telemetry "telemetry-ck" is invalid: ' +# - '* spec.autoscaling.aodh.apiImage: Required value' +# - '* spec.autoscaling.aodh.evaluatorImage: Required value' +# - '* spec.autoscaling.aodh.listenerImage: Required value' +# - '* spec.autoscaling.aodh.notifierImage: Required value' +# - '* spec.ceilometer.centralImage: Required value' +# - '* spec.ceilometer.computeImage: Required value' +# - '* spec.ceilometer.ipmiImage: Required value' +# - '* spec.ceilometer.notificationImage: Required value' +# - '* spec.ceilometer.proxyImage: Required value' +# - '* spec.ceilometer.sgCoreImage: Required value' +# WORKAROUND: telemetry-with-cloudkitty is updated to explicitly add these values + # This should get to the point where we have must-gather running so that there is more information available. - name: "Create the kustomization for deploying CloudKitty" hosts: "{{ cifmw_target_hook_host | default('localhost') }}" diff --git a/telemetry-with-cloudkitty.yaml b/telemetry-with-cloudkitty.yaml index 50f79c6a..217d1d98 100644 --- a/telemetry-with-cloudkitty.yaml +++ b/telemetry-with-cloudkitty.yaml @@ -16,6 +16,12 @@ spec: autoscaling: enabled: false aodh: + # The images are added because when they are omitted in the CI env, telemetry-operator complains that the required values are missing. + # This doesn't occur locally when runnign with the webhook + apiImage: "quay.io/podified-antelope-centos9/openstack-aodh-api:current-podified" + evaluatorImage: "quay.io/podified-antelope-centos9/openstack-aodh-evaluator:current-podified" + listenerImage: "quay.io/podified-antelope-centos9/openstack-aodh-listener:current-podified" + notifierImage: "quay.io/podified-antelope-centos9/openstack-aodh-notifier:current-podified" passwordSelectors: databaseAccount: aodh databaseInstance: openstack @@ -25,6 +31,13 @@ spec: ceilometer: enabled: true secret: osp-secret + # The images are added because when they are omitted in the CI env, telemetry-operator complains that the required values are missing. + centralImage: "quay.io/podified-antelope-centos9/openstack-ceilometer-central:current-podified" + computeImage: "quay.io/podified-antelope-centos9/openstack-ceilometer-compute:current-podified" + ipmiImage: "quay.io/podified-antelope-centos9/openstack-ceilometer-ipmi:current-podified" + notificationImage: "quay.io/podified-antelope-centos9/openstack-ceilometer-notification:current-podified" + proxyImage: "quay.io/podified-antelope-centos9/openstack-aodh-api:current-podified" + sgCoreImage: "quay.io/openstack-k8s-operators/sg-core:latest" cloudkitty: enabled: true cloudKittyAPI: From d7ac1d6711c21df89e643413b3361bf5221bdfd7 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Tue, 23 Sep 2025 13:36:46 -0400 Subject: [PATCH 75/87] [ci] Add loki-operator deps to cloudkitty deployment Loki-operator needs to be pinned to 6.3.0, because 6.3.1 has this issue: https://issues.redhat.com/browse/LOG-7752 --- ci/cloudkitty-post_deploy.yml | 11 ++ cloudkitty-loki/README | 42 ------- cloudkitty-loki/certificate-internal.yaml | 27 ----- cloudkitty-loki/certificate-public.yaml | 26 ---- cloudkitty-loki/loki-secret.yaml | 11 -- cloudkitty-loki/loki-subscription.yaml | 24 ---- cloudkitty-loki/lokistack.yaml | 56 --------- deploy-loki-for-ck.yaml | 138 ++++++++++++++++++++++ telemetry-with-cloudkitty.yaml | 4 +- 9 files changed, 152 insertions(+), 187 deletions(-) delete mode 100644 cloudkitty-loki/README delete mode 100644 cloudkitty-loki/certificate-internal.yaml delete mode 100644 cloudkitty-loki/certificate-public.yaml delete mode 100644 cloudkitty-loki/loki-secret.yaml delete mode 100644 cloudkitty-loki/loki-subscription.yaml delete mode 100644 cloudkitty-loki/lokistack.yaml create mode 100644 deploy-loki-for-ck.yaml diff --git a/ci/cloudkitty-post_deploy.yml b/ci/cloudkitty-post_deploy.yml index 611b37e0..e8a5e2af 100644 --- a/ci/cloudkitty-post_deploy.yml +++ b/ci/cloudkitty-post_deploy.yml @@ -26,6 +26,17 @@ cmd: | oc patch oscp $(oc get oscp -ojsonpath='{ .items[].metadata.name }') --type=json -p='[{"op": "replace", "path": "/spec/telemetry/enabled", "value": false} ]' + - name: Deploy loki operator + ansible.builtin.shell: + cmd: | + oc apply -f {{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/deploy-loki-for-ck.yaml + + - name: Get install plan and approve the installation + ansible.builtin.shell: + cmd: | + export installplan=$(oc get installplan -n openshift-operators-redhat -l operators.coreos.com/loki-operator.openshift-operators-redhat="" | grep "loki-operator.v6.3.0" | awk '{print $1}') + oc patch -n openshift-operators-redhat installplan $installplan --type='json' -p='[{"op": "replace", "path": "/spec/approved", "value":true}]' + - name: Create the new telemetry ansible.builtin.shell: cmd: | diff --git a/cloudkitty-loki/README b/cloudkitty-loki/README deleted file mode 100644 index 7c42639c..00000000 --- a/cloudkitty-loki/README +++ /dev/null @@ -1,42 +0,0 @@ -# CloudKitty Loki Deployment with Red Hat Loki Operator - -1. Create the internal certificate -``` -$ oc apply -f certificate-internal.yaml -$ oc apply -f certificate-public.yaml -``` - -2. Create ConfigMap -``` -$ kubectl create configmap lokistack-mtls --from-literal=ca.crt="$(kubectl get secret cert-loki-public-route -n openstack -o jsonpath='{.data.ca\.crt}' | base64 -d)" -n openstack -``` - -# Need to set up S3 -oc apply -f /ci/deploy-logging-dependencies/files/minio-dev.yaml - -oc apply -f loki-subscription.yaml - -3. Edit loki-secret.yaml with the appropiate data to connect to S3: -``` - access_key_id: - access_key_secret: - bucketnames: - endpoint: -``` - -4. Create LokiStack: -``` -$ oc apply -f lokistack.yaml -``` - -5. Check that Loki pods have been created in the openstack namespace: -``` -lokistack-compactor-0 1/1 Running 0 18m -lokistack-distributor-65c994c58-c84rw 1/1 Running 0 18m -lokistack-gateway-6dbbc4d789-lnjdv 1/1 Running 0 18m -lokistack-gateway-6dbbc4d789-s8pwx 1/1 Running 0 18m -lokistack-index-gateway-0 1/1 Running 0 18m -lokistack-ingester-0 1/1 Running 0 18m -lokistack-querier-7f887c4f89-rw7qb 1/1 Running 0 18m -lokistack-query-frontend-57576979bf-qbxw7 1/1 Running 0 18m -``` diff --git a/cloudkitty-loki/certificate-internal.yaml b/cloudkitty-loki/certificate-internal.yaml deleted file mode 100644 index a21c1a8c..00000000 --- a/cloudkitty-loki/certificate-internal.yaml +++ /dev/null @@ -1,27 +0,0 @@ ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: cert-loki-internal - namespace: openstack -spec: - secretName: cert-loki-internal-svc - commonName: cloudkitty-lokistack-gateway-http.openstack.svc - dnsNames: - - cloudkitty-lokistack-gateway-http.openstack.svc - - cloudkitty-lokistack-gateway-http.openstack.svc.cluster.local - subject: - ## This is what binds the client to the role cloudkitty definedi in LokiStack - organizationalUnits: - - cloudkitty - privateKey: - algorithm: RSA - size: 2048 - encoding: PKCS8 - usages: - - digital signature - - key encipherment - - client auth - issuerRef: - kind: Issuer - name: rootca-internal diff --git a/cloudkitty-loki/certificate-public.yaml b/cloudkitty-loki/certificate-public.yaml deleted file mode 100644 index 42528c30..00000000 --- a/cloudkitty-loki/certificate-public.yaml +++ /dev/null @@ -1,26 +0,0 @@ ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: cert-loki-public - namespace: openstack -spec: - secretName: cert-loki-public-route - commonName: lokistack-openstack.apps-crc.testing - dnsNames: - - lokistack-openstack.apps-crc.testing - subject: - ## This is what binds the client to the role cloudkitty defined in LokiStack - organizationalUnits: - - cloudkitty - privateKey: - algorithm: RSA - size: 2048 - encoding: PKCS8 - usages: - - digital signature - - key encipherment - - client auth - issuerRef: - kind: Issuer - name: rootca-public diff --git a/cloudkitty-loki/loki-secret.yaml b/cloudkitty-loki/loki-secret.yaml deleted file mode 100644 index 9c65c464..00000000 --- a/cloudkitty-loki/loki-secret.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: cloudkitty-loki-s3 - namespace: openstack -stringData: - access_key_id: minio - access_key_secret: minio123 - bucketnames: cloudkitty-bucket - endpoint: http://api-minio-dev.apps-crc.testing - diff --git a/cloudkitty-loki/loki-subscription.yaml b/cloudkitty-loki/loki-subscription.yaml deleted file mode 100644 index f57a525a..00000000 --- a/cloudkitty-loki/loki-subscription.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: openshift-operators-redhat - annotations: - openshift.io/node-selector: "" - labels: - openshift.io/cluster-monitoring: "true" ---- -apiVersion: operators.coreos.com/v1alpha1 -kind: Subscription -metadata: - labels: - operators.coreos.com/loki-operator.openshift-operators-redhat: "" - name: loki-operator - namespace: openshift-operators-redhat -spec: - channel: stable-6.3 - installPlanApproval: Automatic - name: loki-operator - source: redhat-operators - sourceNamespace: openshift-marketplace - # startingCSV: loki-operator.v6.3.0 - diff --git a/cloudkitty-loki/lokistack.yaml b/cloudkitty-loki/lokistack.yaml deleted file mode 100644 index 3e1955b0..00000000 --- a/cloudkitty-loki/lokistack.yaml +++ /dev/null @@ -1,56 +0,0 @@ ---- -apiVersion: loki.grafana.com/v1 -kind: LokiStack -metadata: - name: cloudkitty-lokistack - namespace: openstack -spec: - size: 1x.demo - storage: - schemas: - - version: v13 - effectiveDate: "2024-11-18" - secret: - name: cloudkitty-loki-s3 - type: s3 - storageClassName: crc-csi-hostpath-provisioner - tenants: - mode: static - authentication: - - tenantName: cloudkitty - tenantId: cloudkitty - mTLS: - ca: - caKey: ca.crt - caName: lokistack-mtls - authorization: - roleBindings: - - name: cloudkitty-dataframes - roles: - - cloudkitty-dataframes - subjects: - - kind: group - name: cloudkitty - - name: cluster-reader - roles: - - cluster-reader - subjects: - - kind: group - name: cloudkitty-dataframes-admin - roles: - - name: cloudkitty-dataframes - permissions: - - read - - write - resources: - - logs - tenants: - - cloudkitty - - name: cluster-reader - permissions: - - read - resources: - - logs - tenants: - - cloudkitty - diff --git a/deploy-loki-for-ck.yaml b/deploy-loki-for-ck.yaml new file mode 100644 index 00000000..c59a0947 --- /dev/null +++ b/deploy-loki-for-ck.yaml @@ -0,0 +1,138 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-operators-redhat + labels: + name: openshift-operators-redhat +--- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: loki-operator + namespace: openshift-operators-redhat +spec: + upgradeStrategy: Default +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: loki-operator + namespace: openshift-operators-redhat +spec: + channel: stable-6.3 + installPlanApproval: Manual + startingCSV: loki-operator.v6.3.0 + name: loki-operator + source: redhat-operators + sourceNamespace: openshift-marketplace +--- +# Deploys a new Namespace for the MinIO Pod +apiVersion: v1 +kind: Namespace +metadata: + name: minio-dev # Change this value if you want a different namespace name + labels: + name: minio-dev # Change this value to match metadata.name +--- +# Deploys a new MinIO Pod into the metadata.namespace Kubernetes namespace +# +apiVersion: v1 +kind: Pod +metadata: + labels: + app: minio + name: minio + namespace: minio-dev # Change this value to match the namespace metadata.name +spec: + containers: + - name: minio + image: quay.io/minio/minio:latest + command: + - /bin/bash + - -c + - | + mkdir -p /data/loki && \ + minio server /data + env: + - name: MINIO_ACCESS_KEY + value: minio + - name: MINIO_SECRET_KEY + value: minio123 + volumeMounts: + - mountPath: /data + name: storage # Corresponds to the `spec.volumes` Persistent Volume + volumes: + - name: storage + persistentVolumeClaim: + claimName: minio-pvc +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio-pvc + namespace: minio-dev +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: crc-csi-hostpath-provisioner +--- +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: minio-dev +spec: + selector: + app: minio + ports: + - name: api + protocol: TCP + port: 9000 + - name: console + protocol: TCP + port: 9090 +--- +kind: Route +apiVersion: route.openshift.io/v1 +metadata: + name: minio-console + namespace: minio-dev +spec: + host: console-minio-dev.apps-crc.testing + to: + kind: Service + name: minio + weight: 100 + port: + targetPort: console + wildcardPolicy: None +--- +kind: Route +apiVersion: route.openshift.io/v1 +metadata: + name: minio-api + namespace: minio-dev +spec: + host: api-minio-dev.apps-crc.testing + to: + kind: Service + name: minio + weight: 100 + port: + targetPort: api + wildcardPolicy: None +--- +apiVersion: v1 +kind: Secret +metadata: + name: logging-loki-s3 +stringData: + access_key_id: minio + access_key_secret: minio123 + bucketnames: loki + endpoint: http://minio.minio-dev.svc.cluster.local:9000 + diff --git a/telemetry-with-cloudkitty.yaml b/telemetry-with-cloudkitty.yaml index 217d1d98..e422f8a2 100644 --- a/telemetry-with-cloudkitty.yaml +++ b/telemetry-with-cloudkitty.yaml @@ -73,7 +73,9 @@ spec: - effectiveDate: "2024-11-18" version: v13 secret: - name: cloudkitty-loki-s3 + #name: cloudkitty-loki-s3 + #Q: Is the intention to have a separate instance of loki for logging and cloudkitty? + name: logging-loki-s3 type: "s3" secret: "osp-secret" serviceUser: cloudkitty From 2fb63ca0a2c28909889a62b29450d9b84ea7f298 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 26 Sep 2025 12:58:11 -0400 Subject: [PATCH 76/87] Update installplan approval to include some debug Try alternative way to get installplan Add retries to installplan check If there's no value returned, the rc is still 0, so we need to explicitly state the success criteria --- ci/cloudkitty-post_deploy.yml | 48 ++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/ci/cloudkitty-post_deploy.yml b/ci/cloudkitty-post_deploy.yml index e8a5e2af..8f6a1706 100644 --- a/ci/cloudkitty-post_deploy.yml +++ b/ci/cloudkitty-post_deploy.yml @@ -31,11 +31,53 @@ cmd: | oc apply -f {{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/deploy-loki-for-ck.yaml + + - name: Get install plan from the list of installplans (not used) + ansible.builtin.shell: + cmd: | + oc get installplan -n openshift-operators-redhat | grep "loki-operator.v6.3.0" | awk '{print $1}' + retries: 10 + delay: 10 + register: install_plan + until: install_plan.stdout_lines != 0 + + # This is the better option if it produces an installplan var, since it filters by loki version + - name: Show the install_plan from oc get installplan + ansible.builtin.debug: + var: install_plan + + - name: Get the installplan from the loki-operator subscription + ansible.builtin.shell: + cmd: | + oc get subscription -n openshift-operators-redhat loki-operator -ojsonpath='{ .status.installplan.name }' + until: loki_installplan.stdout_lines | length > 0 + retries: 12 + delay: 10 + register: loki_installplan + + - name: Show the loki_installplan + ansible.builtin.debug: + var: loki_installplan.stdout + + - name: Get details of loki installplan + ansible.builtin.shell: + cmd: | + oc describe -n openshift-operators-redhat installplan {{ loki_installplan.stdout }} + + - name: Show installplans + ansible.builtin.shell: + cmd: | + oc get installplan --all-namespaces + register: all_install_plans + + - name: Show all install plans + ansible.builtin.debug: + var: all_install_plans + - name: Get install plan and approve the installation ansible.builtin.shell: cmd: | - export installplan=$(oc get installplan -n openshift-operators-redhat -l operators.coreos.com/loki-operator.openshift-operators-redhat="" | grep "loki-operator.v6.3.0" | awk '{print $1}') - oc patch -n openshift-operators-redhat installplan $installplan --type='json' -p='[{"op": "replace", "path": "/spec/approved", "value":true}]' + oc patch -n openshift-operators-redhat installplan {{ loki_installplan.stdout }} --type='json' -p='[{"op": "replace", "path": "/spec/approved", "value":true}]' - name: Create the new telemetry ansible.builtin.shell: @@ -101,7 +143,7 @@ # EOF # register: output - - name: wait for the update to telemetry + - name: Wait for the update to telemetry ansible.builtin.shell: cmd: | oc wait --for=condition=Ready --timeout=600s telemetry telemetry-ck From 258405c076dcab98d3b444a0a7dcb8df9555530f Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Thu, 25 Sep 2025 12:10:42 -0400 Subject: [PATCH 77/87] [ci] Use meta-content-provider plus custom images for CK --- telemetry-with-cloudkitty.yaml | 6 ++---- zuul.d/projects.yaml | 6 +++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/telemetry-with-cloudkitty.yaml b/telemetry-with-cloudkitty.yaml index e422f8a2..e6af91ec 100644 --- a/telemetry-with-cloudkitty.yaml +++ b/telemetry-with-cloudkitty.yaml @@ -42,8 +42,7 @@ spec: enabled: true cloudKittyAPI: # latest, with mTLS patch to CK - containerImage: "quay.io/efoley/cloudkitty-api@sha256:baf3f44225be4ac55eef6e45ab4095a8284fc186da5916408dbf95767a1eb5e9" - #containerImage: "quay.io/efoley/cloudkitty-api:epoxy-loki" + containerImage: quay.io/jwysogla/cloudkitty-api@sha256:5541d1160f777174a00982fde3c26a9b32ba156f9f140c9628f66d0eef834c86 customServiceConfig: | #[storage_loki] #url = https://cloudkitty-lokistack-gateway-http.openstack.svc:8080 @@ -51,8 +50,7 @@ spec: tls: caBundleSecretName: combined-ca-bundle cloudKittyProc: - containerImage: "quay.io/efoley/cloudkitty-processor@sha256:4ddd2f824b17c0c2fb39363369a16af49cb2d95c07916f52e2c509ceca677de9" - #containerImage: "quay.io/efoley/cloudkitty-processor:epoxy-loki" + containerImage: "quay.io/jwysogla/cloudkitty-processor@sha256:949f386cbf54366d492d6528156f9916e44089b26c7496fc5ee89dd286633208" customServiceConfig: | #[storage_loki] #url = https://cloudkitty-lokistack-gateway-http.openstack.svc:8080 diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index 9e4c73ea..833ceb7e 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -162,9 +162,9 @@ jobs: - telemetry-operator-multinode-cloudkitty: dependencies: - - openstack-k8s-operators-content-provider - # temperorily use the regular content provider since CK rpm builds are failing - - openstack-k8s-operators-content-provider + - telemetry-openstack-meta-content-provider-master + # temperorily use the regular content provider since CK rpm builds are failing + #- openstack-k8s-operators-content-provider # Keep the metacontent provider in the pipeline so we know when it is working again - telemetry-openstack-meta-content-provider-master #- telemetry-operator-multinode-default-telemetry From 46c3a378f43d547c0b571d90ce80919a7e2db7ed Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Thu, 9 Oct 2025 14:25:54 -0400 Subject: [PATCH 78/87] [ci][CLoudkitty] Add pre-test hook to run ansibletests --- ci/vars-cloudkitty-tempest.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/vars-cloudkitty-tempest.yml b/ci/vars-cloudkitty-tempest.yml index d5cd7397..8d40129d 100644 --- a/ci/vars-cloudkitty-tempest.yml +++ b/ci/vars-cloudkitty-tempest.yml @@ -56,6 +56,9 @@ post_deploy_enable_cloudkitty: cloudkitty_api_image: "{{ _ck_api_image }}" cloudkitty_proc_image: "{{ _ck_proc_image }}" +pre_tests_ansible_tests: + source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/ansible-tests-cloudkitty.yaml" + type: cr # test cloudkitty cifmw_run_tests: true cifmw_run_test_role: test_operator From 601334e5f22e0285347dbd932b08430f5ded4730 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Tue, 14 Oct 2025 06:50:02 -0400 Subject: [PATCH 79/87] [ci] Add a cloudkitty-with-telemetry.yaml template so built images can be passed through --- ci/cloudkitty-post_deploy.yml | 5 +++++ .../telemetry-with-cloudkitty.yaml.j2 | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) rename telemetry-with-cloudkitty.yaml => ci/telemetry-with-cloudkitty.yaml.j2 (92%) diff --git a/ci/cloudkitty-post_deploy.yml b/ci/cloudkitty-post_deploy.yml index 8f6a1706..c590c29a 100644 --- a/ci/cloudkitty-post_deploy.yml +++ b/ci/cloudkitty-post_deploy.yml @@ -79,6 +79,11 @@ cmd: | oc patch -n openshift-operators-redhat installplan {{ loki_installplan.stdout }} --type='json' -p='[{"op": "replace", "path": "/spec/approved", "value":true}]' + - name: Render telemetry-with-cloudkitty template + ansible.builtin.template: + src: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/telemetry-with-cloudkitty.yaml.j2" + dest: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/telemetry-with-cloudkitty.yaml" + - name: Create the new telemetry ansible.builtin.shell: cmd: | diff --git a/telemetry-with-cloudkitty.yaml b/ci/telemetry-with-cloudkitty.yaml.j2 similarity index 92% rename from telemetry-with-cloudkitty.yaml rename to ci/telemetry-with-cloudkitty.yaml.j2 index e6af91ec..35700ccb 100644 --- a/telemetry-with-cloudkitty.yaml +++ b/ci/telemetry-with-cloudkitty.yaml.j2 @@ -42,7 +42,7 @@ spec: enabled: true cloudKittyAPI: # latest, with mTLS patch to CK - containerImage: quay.io/jwysogla/cloudkitty-api@sha256:5541d1160f777174a00982fde3c26a9b32ba156f9f140c9628f66d0eef834c86 + containerImage: {{ cloudkitty_api_image }} customServiceConfig: | #[storage_loki] #url = https://cloudkitty-lokistack-gateway-http.openstack.svc:8080 @@ -50,7 +50,7 @@ spec: tls: caBundleSecretName: combined-ca-bundle cloudKittyProc: - containerImage: "quay.io/jwysogla/cloudkitty-processor@sha256:949f386cbf54366d492d6528156f9916e44089b26c7496fc5ee89dd286633208" + containerImage: {{ cloudkitty_proc_image }} customServiceConfig: | #[storage_loki] #url = https://cloudkitty-lokistack-gateway-http.openstack.svc:8080 From 16c445e430499f8d270cb5bd86ff190309dfcc50 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Tue, 14 Oct 2025 07:36:14 -0400 Subject: [PATCH 80/87] [ci] Update CK deployment to use oscp Add kustomization for oscp to enable CK pre-deploy Move loki installation to pre_deploy. --- ci/cloudkitty-post_deploy.yml | 44 +++++++++++++++++----------------- ci/configure-cloudkitty.yml | 2 +- ci/vars-cloudkitty-tempest.yml | 2 +- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/ci/cloudkitty-post_deploy.yml b/ci/cloudkitty-post_deploy.yml index c590c29a..45f34d8c 100644 --- a/ci/cloudkitty-post_deploy.yml +++ b/ci/cloudkitty-post_deploy.yml @@ -14,17 +14,17 @@ # WORKAROUND: telemetry-with-cloudkitty is updated to explicitly add these values # This should get to the point where we have must-gather running so that there is more information available. -- name: "Create the kustomization for deploying CloudKitty" +- name: "Install loki for cloudkitty" hosts: "{{ cifmw_target_hook_host | default('localhost') }}" gather_facts: false environment: KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" PATH: "{{ cifmw_path }}" tasks: - - name: Disable telemetry in the openstack controlplane - ansible.builtin.shell: - cmd: | - oc patch oscp $(oc get oscp -ojsonpath='{ .items[].metadata.name }') --type=json -p='[{"op": "replace", "path": "/spec/telemetry/enabled", "value": false} ]' + #- name: Disable telemetry in the openstack controlplane + # ansible.builtin.shell: + # cmd: | + # oc patch oscp $(oc get oscp -ojsonpath='{ .items[].metadata.name }') --type=json -p='[{"op": "replace", "path": "/spec/telemetry/enabled", "value": false} ]' - name: Deploy loki operator ansible.builtin.shell: @@ -79,15 +79,15 @@ cmd: | oc patch -n openshift-operators-redhat installplan {{ loki_installplan.stdout }} --type='json' -p='[{"op": "replace", "path": "/spec/approved", "value":true}]' - - name: Render telemetry-with-cloudkitty template - ansible.builtin.template: - src: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/telemetry-with-cloudkitty.yaml.j2" - dest: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/telemetry-with-cloudkitty.yaml" + #- name: Render telemetry-with-cloudkitty template + # ansible.builtin.template: + # src: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/telemetry-with-cloudkitty.yaml.j2" + # dest: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/telemetry-with-cloudkitty.yaml" - - name: Create the new telemetry - ansible.builtin.shell: - cmd: | - oc apply -f {{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/telemetry-with-cloudkitty.yaml + #- name: Create the new telemetry + # ansible.builtin.shell: + # cmd: | + # oc apply -f {{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/telemetry-with-cloudkitty.yaml # WORKAROUND @@ -148,13 +148,13 @@ # EOF # register: output - - name: Wait for the update to telemetry - ansible.builtin.shell: - cmd: | - oc wait --for=condition=Ready --timeout=600s telemetry telemetry-ck - changed_when: false + # - name: Wait for the update to telemetry + # ansible.builtin.shell: + # cmd: | + # oc wait --for=condition=Ready --timeout=600s telemetry telemetry-ck + # changed_when: false - - name: Check for secrets - ansible.builtin.shell: - cmd: | - oc get secrets -n openstack | grep "cloudkitty" + # - name: Check for secrets + # ansible.builtin.shell: + # cmd: | + # oc get secrets -n openstack | grep "cloudkitty" diff --git a/ci/configure-cloudkitty.yml b/ci/configure-cloudkitty.yml index 36204900..73b834da 100644 --- a/ci/configure-cloudkitty.yml +++ b/ci/configure-cloudkitty.yml @@ -33,7 +33,7 @@ autoscaling: enabled: false cloudkitty: - enabled: false + enabled: true s3StorageConfig: secret: type: "s3" diff --git a/ci/vars-cloudkitty-tempest.yml b/ci/vars-cloudkitty-tempest.yml index 8d40129d..958a982e 100644 --- a/ci/vars-cloudkitty-tempest.yml +++ b/ci/vars-cloudkitty-tempest.yml @@ -49,7 +49,7 @@ pre_deploy_disable_telemetry: #pre_deploy_deploy_logging_dependencies: # source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/deploy-logging-dependencies.yml" # type: playbook -post_deploy_enable_cloudkitty: +pre_deploy_loki_setup: source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/cloudkitty-post_deploy.yml" type: playbook extra_vars: From 6531f131d3a5608d77843570c88b9b41288f5b46 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Tue, 14 Oct 2025 09:54:32 -0400 Subject: [PATCH 81/87] Rename secret --- deploy-loki-for-ck.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy-loki-for-ck.yaml b/deploy-loki-for-ck.yaml index c59a0947..800b62ca 100644 --- a/deploy-loki-for-ck.yaml +++ b/deploy-loki-for-ck.yaml @@ -129,7 +129,8 @@ spec: apiVersion: v1 kind: Secret metadata: - name: logging-loki-s3 + name: cloudkitty-loki-s3 + namespace: openstack stringData: access_key_id: minio access_key_secret: minio123 From 168abfebe6aad46efa34ff20ab9c65a35d91455f Mon Sep 17 00:00:00 2001 From: jlarriba Date: Wed, 15 Oct 2025 12:40:20 +0200 Subject: [PATCH 82/87] Do not allow for CloudKitty deployment if the PrometheusEndpoint secret is not present --- controllers/cloudkitty_controller.go | 76 +++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 2 deletions(-) diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index 7f19874f..fd3dc817 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -338,6 +338,43 @@ func (r *CloudKittyReconciler) SetupWithManager(mgr ctrl.Manager) error { return nil } + prometheusEndpointSecretFn := func(ctx context.Context, o client.Object) []reconcile.Request { + Log := r.GetLogger(ctx) + + result := []reconcile.Request{} + + // Only reconcile if this is the PrometheusEndpoint secret + if o.GetName() != cloudkitty.PrometheusEndpointSecret { + return nil + } + + // get all CloudKitty CRs + cloudkitties := &telemetryv1.CloudKittyList{} + listOpts := []client.ListOption{ + client.InNamespace(o.GetNamespace()), + } + if err := r.List(ctx, cloudkitties, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve CloudKitty CRs %w") + return nil + } + + for _, cr := range cloudkitties.Items { + // Only reconcile CloudKitty CRs that are using MetricStorage (PrometheusHost is empty) + if cr.Spec.PrometheusHost == "" { + name := client.ObjectKey{ + Namespace: o.GetNamespace(), + Name: cr.Name, + } + Log.Info(fmt.Sprintf("PrometheusEndpoint Secret %s is used by CloudKitty CR %s", o.GetName(), cr.Name)) + result = append(result, reconcile.Request{NamespacedName: name}) + } + } + if len(result) > 0 { + return result + } + return nil + } + control, err := ctrl.NewControllerManagedBy(mgr). For(&telemetryv1.CloudKitty{}). Owns(&mariadbv1.MariaDBDatabase{}). @@ -355,6 +392,9 @@ func (r *CloudKittyReconciler) SetupWithManager(mgr ctrl.Manager) error { // Watch for TransportURL Secrets which belong to any TransportURLs created by CloudKitty CRs Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(transportURLSecretFn)). + // Watch for PrometheusEndpoint Secret created by MetricStorage + Watches(&corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(prometheusEndpointSecretFn)). Watches(&memcachedv1.Memcached{}, handler.EnqueueRequestsFromMapFunc(memcachedFn)). Watches(&keystonev1.KeystoneAPI{}, @@ -825,6 +865,37 @@ func (r *CloudKittyReconciler) reconcileNormal(ctx context.Context, instance *te condition.MemcachedReadyCondition, condition.MemcachedReadyMessage) // run check memcached - end + // + // Check for PrometheusEndpoint secret if using MetricStorage + // + if instance.Spec.PrometheusHost == "" { + prometheusEndpointSecret := &corev1.Secret{} + err = r.Get(ctx, client.ObjectKey{ + Name: cloudkitty.PrometheusEndpointSecret, + Namespace: instance.Namespace, + }, prometheusEndpointSecret) + if err != nil { + if k8s_errors.IsNotFound(err) { + Log.Info("PrometheusEndpoint Secret not found. CloudKitty will not be deployed until MetricStorage creates it.") + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.Reason("PrometheusEndpoint secret not found. The MetricStorage probably hasn't been created yet or isn't ready"), + condition.SeverityError, + "PrometheusEndpoint secret %s not found. Waiting for MetricStorage to create it", + cloudkitty.PrometheusEndpointSecret)) + return ctrl.Result{RequeueAfter: telemetryv1.PauseBetweenWatchAttempts}, nil + } + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + } + // run check PrometheusEndpoint secret - end + // // check for required OpenStack secret holding passwords for service/admin user and add hash to the vars map // @@ -1030,7 +1101,6 @@ func (r *CloudKittyReconciler) generateServiceConfigs( memcached *memcachedv1.Memcached, db *mariadbv1.Database, ) error { - Log := r.GetLogger(ctx) // // create Secret required for cloudkitty input // - %-scripts holds scripts to e.g. bootstrap the service @@ -1075,13 +1145,14 @@ func (r *CloudKittyReconciler) generateServiceConfigs( if instance.Spec.PrometheusHost == "" { // We're using MetricStorage for Prometheus. + // Note: The secret existence is already checked in reconcileNormal(), so we can safely get it here prometheusEndpointSecret := &corev1.Secret{} err = r.Get(ctx, client.ObjectKey{ Name: cloudkitty.PrometheusEndpointSecret, Namespace: instance.Namespace, }, prometheusEndpointSecret) if err != nil { - Log.Info("Prometheus Endpoint Secret not found") + return err } if prometheusEndpointSecret.Data != nil { instance.Status.PrometheusHost = string(prometheusEndpointSecret.Data[metricstorage.PrometheusHost]) @@ -1104,6 +1175,7 @@ func (r *CloudKittyReconciler) generateServiceConfigs( condition.SeverityWarning, condition.ServiceConfigReadyErrorMessage, err.Error())) + return err } instance.Status.PrometheusTLS = metricStorage.Spec.PrometheusTLS.Enabled() } From ce96df1684a277a3aae19696108a311a4545d671 Mon Sep 17 00:00:00 2001 From: jlarriba Date: Tue, 21 Oct 2025 10:33:32 +0200 Subject: [PATCH 83/87] Allow for configuration of the period CloudKitty config option --- api/bases/telemetry.openstack.org_cloudkitties.yaml | 6 ++++++ api/bases/telemetry.openstack.org_telemetries.yaml | 6 ++++++ api/v1beta1/cloudkitty_types.go | 6 ++++++ config/crd/bases/telemetry.openstack.org_cloudkitties.yaml | 6 ++++++ config/crd/bases/telemetry.openstack.org_telemetries.yaml | 6 ++++++ controllers/cloudkitty_controller.go | 1 + templates/cloudkitty/config/cloudkitty.conf | 2 +- 7 files changed, 32 insertions(+), 1 deletion(-) diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml index 13359220..763c6ee5 100644 --- a/api/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -541,6 +541,12 @@ spec: service password from the Secret type: string type: object + period: + default: 300 + description: Period for collecting metrics in seconds + format: int32 + minimum: 1 + type: integer preserveJobs: default: false description: PreserveJobs - do not delete jobs after they finished diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index fb0bb10c..790d8fb5 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -1106,6 +1106,12 @@ spec: service password from the Secret type: string type: object + period: + default: 300 + description: Period for collecting metrics in seconds + format: int32 + minimum: 1 + type: integer preserveJobs: default: false description: PreserveJobs - do not delete jobs after they finished diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index 957c7b7e..529e76f8 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -186,6 +186,12 @@ type CloudKittySpecBase struct { // +nullable PrometheusTLSCaCertSecret *corev1.SecretKeySelector `json:"prometheusTLSCaCertSecret,omitempty"` + // Period for collecting metrics in seconds + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Optional + // +kubebuilder:default=300 + Period int32 `json:"period"` + // S3 related configuration passed to Loki // +kubebuilder:validation:Optional // +kubebuilder:default={secret: {name: "cloudkitty-loki-s3", type: "s3"}} diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml index 13359220..763c6ee5 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -541,6 +541,12 @@ spec: service password from the Secret type: string type: object + period: + default: 300 + description: Period for collecting metrics in seconds + format: int32 + minimum: 1 + type: integer preserveJobs: default: false description: PreserveJobs - do not delete jobs after they finished diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index fb0bb10c..790d8fb5 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -1106,6 +1106,12 @@ spec: service password from the Secret type: string type: object + period: + default: 300 + description: Period for collecting metrics in seconds + format: int32 + minimum: 1 + type: integer preserveJobs: default: false description: PreserveJobs - do not delete jobs after they finished diff --git a/controllers/cloudkitty_controller.go b/controllers/cloudkitty_controller.go index fd3dc817..c318d52b 100644 --- a/controllers/cloudkitty_controller.go +++ b/controllers/cloudkitty_controller.go @@ -1199,6 +1199,7 @@ func (r *CloudKittyReconciler) generateServiceConfigs( templateParameters["TransportURL"] = string(transportURLSecret.Data["transport_url"]) templateParameters["PrometheusHost"] = instance.Status.PrometheusHost templateParameters["PrometheusPort"] = instance.Status.PrometheusPort + templateParameters["Period"] = instance.Spec.Period templateParameters["LokiHost"] = lokiHost templateParameters["LokiPort"] = 8080 templateParameters["DatabaseConnection"] = fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf", diff --git a/templates/cloudkitty/config/cloudkitty.conf b/templates/cloudkitty/config/cloudkitty.conf index 1e549bef..0b497160 100644 --- a/templates/cloudkitty/config/cloudkitty.conf +++ b/templates/cloudkitty/config/cloudkitty.conf @@ -39,7 +39,7 @@ bucket = cloudkitty url = http://influxdb:8086 [collect] -period = 300 +period = {{ .Period }} wait_periods = 0 metrics_conf = /etc/cloudkitty/metrics.yaml collector = prometheus From dc42c6c82bcfc660ac53a2dc81ce311539c31653 Mon Sep 17 00:00:00 2001 From: jlarriba Date: Tue, 21 Oct 2025 10:35:27 +0200 Subject: [PATCH 84/87] Removed unused storage_influxdb section, as we only support Loki storage --- api/bases/telemetry.openstack.org_cloudkitties.yaml | 1 - api/bases/telemetry.openstack.org_telemetries.yaml | 1 - api/v1beta1/cloudkitty_types.go | 1 - config/crd/bases/telemetry.openstack.org_cloudkitties.yaml | 1 - config/crd/bases/telemetry.openstack.org_telemetries.yaml | 1 - templates/cloudkitty/config/cloudkitty.conf | 7 ------- tests/kuttl/suites/cloudkitty/tests/01-deploy.yaml | 1 + 7 files changed, 1 insertion(+), 12 deletions(-) diff --git a/api/bases/telemetry.openstack.org_cloudkitties.yaml b/api/bases/telemetry.openstack.org_cloudkitties.yaml index 763c6ee5..a228b2de 100644 --- a/api/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/api/bases/telemetry.openstack.org_cloudkitties.yaml @@ -545,7 +545,6 @@ spec: default: 300 description: Period for collecting metrics in seconds format: int32 - minimum: 1 type: integer preserveJobs: default: false diff --git a/api/bases/telemetry.openstack.org_telemetries.yaml b/api/bases/telemetry.openstack.org_telemetries.yaml index 790d8fb5..f59937f9 100644 --- a/api/bases/telemetry.openstack.org_telemetries.yaml +++ b/api/bases/telemetry.openstack.org_telemetries.yaml @@ -1110,7 +1110,6 @@ spec: default: 300 description: Period for collecting metrics in seconds format: int32 - minimum: 1 type: integer preserveJobs: default: false diff --git a/api/v1beta1/cloudkitty_types.go b/api/v1beta1/cloudkitty_types.go index 529e76f8..cf1702aa 100644 --- a/api/v1beta1/cloudkitty_types.go +++ b/api/v1beta1/cloudkitty_types.go @@ -187,7 +187,6 @@ type CloudKittySpecBase struct { PrometheusTLSCaCertSecret *corev1.SecretKeySelector `json:"prometheusTLSCaCertSecret,omitempty"` // Period for collecting metrics in seconds - // +kubebuilder:validation:Minimum=1 // +kubebuilder:validation:Optional // +kubebuilder:default=300 Period int32 `json:"period"` diff --git a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml index 763c6ee5..a228b2de 100644 --- a/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml +++ b/config/crd/bases/telemetry.openstack.org_cloudkitties.yaml @@ -545,7 +545,6 @@ spec: default: 300 description: Period for collecting metrics in seconds format: int32 - minimum: 1 type: integer preserveJobs: default: false diff --git a/config/crd/bases/telemetry.openstack.org_telemetries.yaml b/config/crd/bases/telemetry.openstack.org_telemetries.yaml index 790d8fb5..f59937f9 100644 --- a/config/crd/bases/telemetry.openstack.org_telemetries.yaml +++ b/config/crd/bases/telemetry.openstack.org_telemetries.yaml @@ -1110,7 +1110,6 @@ spec: default: 300 description: Period for collecting metrics in seconds format: int32 - minimum: 1 type: integer preserveJobs: default: false diff --git a/templates/cloudkitty/config/cloudkitty.conf b/templates/cloudkitty/config/cloudkitty.conf index 0b497160..80ba122e 100644 --- a/templates/cloudkitty/config/cloudkitty.conf +++ b/templates/cloudkitty/config/cloudkitty.conf @@ -31,13 +31,6 @@ backend = keystone auth_section = authinfos ignore_rating_role = True -[storage_influxdb] -version = 2 -token = cloudkitty -org = openstack -bucket = cloudkitty -url = http://influxdb:8086 - [collect] period = {{ .Period }} wait_periods = 0 diff --git a/tests/kuttl/suites/cloudkitty/tests/01-deploy.yaml b/tests/kuttl/suites/cloudkitty/tests/01-deploy.yaml index 3b5af6dd..b9c06e71 100644 --- a/tests/kuttl/suites/cloudkitty/tests/01-deploy.yaml +++ b/tests/kuttl/suites/cloudkitty/tests/01-deploy.yaml @@ -36,6 +36,7 @@ spec: aodhService: AodhPassword ceilometerService: CeilometerPassword cloudKittyService: CloudKittyPassword + period: 300 preserveJobs: false rabbitMqClusterName: rabbitmq s3StorageConfig: From 89ef600c21da3e8fd5549cb3c1f2e46298cbacba Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Thu, 23 Oct 2025 09:10:34 -0400 Subject: [PATCH 85/87] [ci] Enable metrics storage in telemetry --- ci/configure-cloudkitty.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/configure-cloudkitty.yml b/ci/configure-cloudkitty.yml index 73b834da..477d1a7e 100644 --- a/ci/configure-cloudkitty.yml +++ b/ci/configure-cloudkitty.yml @@ -38,6 +38,8 @@ secret: type: "s3" name: "cloudkitty-loki-s3" + metricStorage: + enabled: true target: kind: OpenStackControlPlane From 3e500f3013e1591f2adb3c0e5280138886013a33 Mon Sep 17 00:00:00 2001 From: Emma Foley Date: Fri, 24 Oct 2025 10:22:24 -0400 Subject: [PATCH 86/87] [ci] Configure storageClass for CloudKitty in CRC environment Set the storageClass to crc-csi-hostpath-provisioner for CloudKitty deployment in CRC-based CI jobs. This prevents the need to increase PVC allocations in install_yamls by using the default storage class available in CRC, which allocates storage as required. --- ci/configure-cloudkitty.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci/configure-cloudkitty.yml b/ci/configure-cloudkitty.yml index 477d1a7e..54034c28 100644 --- a/ci/configure-cloudkitty.yml +++ b/ci/configure-cloudkitty.yml @@ -24,6 +24,11 @@ metadata: name: unused spec: + # Set overall storage class so we don't need to increase the + # number of PVCs that install_yamls creates + # this is only applicable to crc-based jobs, it is not in + # openshift by default, but is included in the crc distribution + storageClass: crc-csi-hostpath-provisioner telemetry: # this is needed so the secrets are created for dataplane deployment enabled: true From daa1bfe4f9b0b03fe8d046b398499cf31f6a5e86 Mon Sep 17 00:00:00 2001 From: ayefimov-1 Date: Mon, 27 Oct 2025 14:12:40 -0400 Subject: [PATCH 87/87] Add a Cloudkitty CI job that runs tempest tests --- ci/vars-cloudkitty-tempest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/vars-cloudkitty-tempest.yml b/ci/vars-cloudkitty-tempest.yml index 958a982e..ec352648 100644 --- a/ci/vars-cloudkitty-tempest.yml +++ b/ci/vars-cloudkitty-tempest.yml @@ -5,7 +5,7 @@ cifmw_openshift_obs_definition: apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: - name: observability-operator +# name: observability-operator # removed per Juan namespace: openshift-operators spec: channel: stable