(ruka) cp obc(s) & cephbobjectstoreuser(s) from elqui

Replacing the previous example CRs.

Author: jhoblitt

fleet/lib/rook-ceph-conf/charts/ruka/templates/cephobjectstore-lfa.yaml

@@ -1,29 +1,65 @@
 ---
 apiVersion: ceph.rook.io/v1
+kind: CephObjectRealm
+metadata:
+  name: lfa
+  namespace: rook-ceph
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectZoneGroup
+metadata:
+  name: lfa
+  namespace: rook-ceph
+spec:
+  realm: lfa
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectZone
+metadata:
+  name: lfa
+  namespace: rook-ceph
+spec:
+  zoneGroup: lfa
+---
+apiVersion: ceph.rook.io/v1
 kind: CephObjectStore
 metadata:
   name: lfa
   namespace: rook-ceph
 spec:
-  metadataPool:
-    failureDomain: host
-    replicated:
-      size: 3
-    quotas:
-      maxSize: 10Gi
-  dataPool:
-    failureDomain: host
-    erasureCoded:
-      dataChunks: 2
-      codingChunks: 1
-    quotas:
-      maxSize: 1Ti
   preservePoolsOnDelete: false
   gateway:
-    sslCertificateRef:
     port: 80
-    # securePort: 443
     instances: 3
+    # resources:
+    #   limits:
+    #     cpu: "16"
+    #     memory: 32Gi
+    #   requests:
+    #     cpu: "16"
+    #     memory: 8Gi
+  zone:
+    name: lfa
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStore
+metadata:
+  name: lfagc  # gc only
+  namespace: rook-ceph
+spec:
+  preservePoolsOnDelete: true
+  gateway:
+    port: 80
+    instances: 3
+    # resources:
+    #   limits:
+    #     cpu: "16"
+    #     memory: 32Gi
+    #   requests:
+    #     cpu: "16"
+    #     memory: 8Gi
+  zone:
+    name: lfa
 ---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
@@ -37,16 +73,183 @@ spec:
   ingressClassName: nginx
   tls:
   - hosts:
-    - s3.ruka.dev.lsst.org
+    -  &host s3.ruka.dev.lsst.org
     secretName: rook-ceph-rgw-ingress-tls
   rules:
-  - host: s3.ruka.dev.lsst.org
+  - host: *host
+    http:
+      paths:
+        - path: /
+          pathType: Prefix
+          backend:
+            service:
+              name: rook-ceph-rgw-lfa
+              port:
+                number: 80
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: rook-ceph-rgw-ingress-o11y
+  namespace: rook-ceph
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt
+    nginx.ingress.kubernetes.io/proxy-body-size: 1024m
+spec:
+  ingressClassName: nginx
+  tls:
+  - hosts:
+    - s3.o11y.ruka.dev.lsst.org
+    secretName: rook-ceph-rgw-ingress-tls-o11y
+  rules:
+  - host: s3.o11y.ruka.dev.lsst.org
     http:
       paths:
       - path: /
         pathType: Prefix
         backend:
           service:
-            name: rook-ceph-rgw-lfa
+            name: rook-ceph-rgw-o11y
             port:
               number: 80
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: rgw.root
+  namespace: rook-ceph
+spec:
+  application: rgw
+  failureDomain: host
+  name: .rgw.root
+  parameters:
+    nodelete: "true"
+    nosizechange: "true"
+    pg_autoscale_mode: "off"
+    pg_num: "4"
+  replicated:
+    size: 3
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: lfa.rgw.control
+  namespace: rook-ceph
+spec:
+  application: rgw
+  failureDomain: host
+  parameters:
+    nodelete: "true"
+    nosizechange: "true"
+    pg_autoscale_mode: "off"
+    pg_num: "4"
+  replicated:
+    size: 3
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: lfa.rgw.meta
+  namespace: rook-ceph
+spec:
+  application: rgw
+  failureDomain: host
+  parameters:
+    nodelete: "true"
+    nosizechange: "true"
+    pg_autoscale_mode: "off"
+    pg_num: "4"
+  replicated:
+    size: 3
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: lfa.rgw.log
+  namespace: rook-ceph
+spec:
+  application: rgw
+  failureDomain: host
+  parameters:
+    nodelete: "true"
+    nosizechange: "true"
+    pg_autoscale_mode: "off"
+    pg_num: "32"
+  replicated:
+    size: 3
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: lfa.rgw.buckets.index
+  namespace: rook-ceph
+spec:
+  application: rgw
+  failureDomain: host
+  parameters:
+    nodelete: "true"
+    nosizechange: "true"
+    pg_autoscale_mode: "off"
+    pg_num: "256"
+  replicated:
+    size: 3
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: lfa.rgw.buckets.non-ec
+  namespace: rook-ceph
+spec:
+  application: rgw
+  failureDomain: host
+  parameters:
+    nodelete: "true"
+    nosizechange: "true"
+    pg_autoscale_mode: "off"
+    pg_num: "4"
+  replicated:
+    size: 3
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: lfa.rgw.otp
+  namespace: rook-ceph
+spec:
+  application: rgw
+  failureDomain: host
+  parameters:
+    nodelete: "true"
+    nosizechange: "true"
+    pg_autoscale_mode: "off"
+    pg_num: "1"
+  replicated:
+    size: 3
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: lfa.rgw.buckets.data
+  namespace: rook-ceph
+spec:
+  application: rgw
+  erasureCoded:
+    dataChunks: 8
+    codingChunks: 3
+  failureDomain: host
+  parameters:
+    nodelete: "true"
+    nosizechange: "true"
+    pg_autoscale_mode: "off"
+    bulk: "true"
+    pg_num: "512"
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: lfa
+provisioner: rook-ceph.ceph.rook.io/bucket
+parameters:
+  objectStoreName: lfa
+  objectStoreNamespace: rook-ceph
+reclaimPolicy: Retain

fleet/lib/rook-ceph-conf/charts/ruka/templates/cephobjectstore-o11y.yaml

@@ -9,6 +9,7 @@ spec:
   clusterNamespace: rook-ceph
   quotas:
     maxBuckets: 2
+    maxSize: 2Pi
 ---
 apiVersion: external-secrets.io/v1alpha1
 kind: PushSecret

fleet/lib/rook-ceph-conf/charts/ruka/templates/cephobjectstoreuser-butler.yaml

@@ -0,0 +1,33 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStoreUser
+metadata:
+  name: calib
+  namespace: rook-ceph
+spec:
+  store: lfa
+  clusterNamespace: rook-ceph
+---
+apiVersion: external-secrets.io/v1alpha1
+kind: PushSecret
+metadata:
+  name: rook-ceph-object-user-lfa-calib
+  namespace: rook-ceph
+spec:
+  secretStoreRefs:
+    - kind: ClusterSecretStore
+      name: onepassword-oods
+  selector:
+    secret:
+      name: rook-ceph-object-user-lfa-calib
+  data:
+    - match:
+        secretKey: AccessKey
+        remoteRef:
+          remoteKey: calib
+          property: AWS_ACCESS_KEY_ID
+    - match:
+        secretKey: SecretKey
+        remoteRef:
+          remoteKey: calib
+          property: AWS_SECRET_ACCESS_KEY