Merge pull request #909 from lsst-it/IT-6108/ayekan-rook-kafka-secrets

(ayekan) add lfa cephobjectstore w/ kafka auth from secrets

Author: jhoblitt

fleet/lib/fleet-conf/overlays/dev/gitrepo-ayekan.yaml

@@ -12,6 +12,11 @@ spec:
     - fleet/s/dev/c/ayekan/*
   targets:
     - name: ayekan
-      clusterName: ayekan
+      clusterSelector:
+        matchExpressions:
+          - key: management.cattle.io/cluster-display-name
+            operator: In
+            values:
+              - ayekan
   correctDrift:
     enabled: true

fleet/lib/rook-ceph-cluster/overlays/ayekan/values.yaml

@@ -1,7 +1,5 @@
 ---
 cephClusterSpec:
-  network:
-    provider: host
   mon:
     count: 3
   cephConfig:
@@ -15,6 +13,13 @@ cephClusterSpec:
       osd_max_pg_per_osd_hard_ratio: "10"
       osd_op_queue: wpq
       osd_scrub_auto_repair: "true"
+    client.rgw.lfa.a:
+      rgw_enable_usage_log: "false"
+      rgw_enable_lc_threads: "false"  # disable object gc
+    client.rgw.lfagc.a:
+      rgw_enable_usage_log: "false"
+      rgw_enable_lc_threads: "true"  # enable object gc
+
   storage:
     useAllNodes: false
     useAllDevices: false
@@ -46,7 +51,7 @@ cephBlockPools:
         nodelete: "true"
         nosizechange: "true"
         pg_autoscale_mode: "off"
-        pg_num: "128"
+        pg_num: "32"
     storageClass:
       name: rook-ceph-block
       enabled: true

fleet/lib/rook-ceph-conf/charts/ayekan/templates/cephbuckettopic-lsst.s3.raw.comcam.yaml

@@ -0,0 +1,22 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBucketTopic
+metadata:
+  name: lsst.s3.raw.comcam
+  namespace: rook-ceph
+spec:
+  objectStoreName: lfa
+  objectStoreNamespace: rook-ceph
+  persistent: false
+  endpoint:
+    kafka:
+      uri: kafka://sasquatch-summit-kafka-bootstrap.lsst.codes:9094
+      ackLevel: broker
+      useSSL: true
+      mechanism: SCRAM-SHA-512
+      UserSecretRef:  # lowercase for rook >= 1.17.2
+        name: &item kafka-bucket-notifications
+        key: username
+      PasswordSecretRef:  # lowercase for rook >= 1.17.2
+        name: *item
+        key: password

fleet/lib/rook-ceph-conf/charts/ayekan/templates/cephbuckettopic-lsst.s3.raw.latiss.yaml

@@ -0,0 +1,22 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBucketTopic
+metadata:
+  name: lsst.s3.raw.latiss
+  namespace: rook-ceph
+spec:
+  objectStoreName: lfa
+  objectStoreNamespace: rook-ceph
+  persistent: false
+  endpoint:
+    kafka:
+      uri: kafka://sasquatch-summit-kafka-bootstrap.lsst.codes:9094
+      ackLevel: broker
+      useSSL: true
+      mechanism: SCRAM-SHA-512
+      UserSecretRef:  # lowercase for rook >= 1.17.2
+        name: &item kafka-bucket-notifications
+        key: username
+      PasswordSecretRef:  # lowercase for rook >= 1.17.2
+        name: *item
+        key: password

fleet/lib/rook-ceph-conf/charts/ayekan/templates/cephbuckettopic-lsst.s3.raw.lsstcam.yaml

@@ -0,0 +1,22 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBucketTopic
+metadata:
+  name: lsst.s3.raw.lsstcam
+  namespace: rook-ceph
+spec:
+  objectStoreName: lfa
+  objectStoreNamespace: rook-ceph
+  persistent: false
+  endpoint:
+    kafka:
+      uri: kafka://sasquatch-summit-kafka-bootstrap.lsst.codes:9094
+      ackLevel: broker
+      useSSL: true
+      mechanism: SCRAM-SHA-512
+      UserSecretRef:  # lowercase for rook >= 1.17.2
+        name: &item kafka-bucket-notifications
+        key: username
+      PasswordSecretRef:  # lowercase for rook >= 1.17.2
+        name: *item
+        key: password

fleet/lib/rook-ceph-conf/charts/ayekan/templates/cephobjectstore-lfa.yaml

@@ -0,0 +1,229 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectRealm
+metadata:
+  name: lfa
+  namespace: rook-ceph
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectZoneGroup
+metadata:
+  name: lfa
+  namespace: rook-ceph
+spec:
+  realm: lfa
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectZone
+metadata:
+  name: lfa
+  namespace: rook-ceph
+spec:
+  zoneGroup: lfa
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStore
+metadata:
+  name: lfa
+  namespace: rook-ceph
+spec:
+  preservePoolsOnDelete: true
+  gateway:
+    port: 80
+    instances: 3
+    resources:
+      limits:
+        cpu: "16"
+        memory: 32Gi
+      requests:
+        cpu: "1"
+        memory: 8Gi
+  zone:
+    name: lfa
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStore
+metadata:
+  name: lfagc  # gc only
+  namespace: rook-ceph
+spec:
+  preservePoolsOnDelete: true
+  gateway:
+    port: 80
+    instances: 3
+    resources:
+      limits:
+        cpu: "16"
+        memory: 32Gi
+      requests:
+        cpu: "1"
+        memory: 8Gi
+  zone:
+    name: lfa
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: rook-ceph-rgw-lfa
+  namespace: rook-ceph
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt
+    nginx.ingress.kubernetes.io/proxy-body-size: 1024m
+spec:
+  ingressClassName: nginx
+  tls:
+    - hosts:
+        - &host s3.ayekan.dev.lsst.org
+      secretName: rook-ceph-rgw-lfa-ingress-tls
+  rules:
+    - host: *host
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: rook-ceph-rgw-lfa
+                port:
+                  number: 80
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: rgw.root
+  namespace: rook-ceph
+spec:
+  application: rgw
+  failureDomain: host
+  name: .rgw.root
+  parameters:
+    nodelete: "true"
+    nosizechange: "true"
+    pg_autoscale_mode: "off"
+    pg_num: "16"
+  replicated:
+    size: 3
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: lfa.rgw.control
+  namespace: rook-ceph
+spec:
+  application: rgw
+  failureDomain: host
+  parameters:
+    nodelete: "true"
+    nosizechange: "true"
+    pg_autoscale_mode: "off"
+    pg_num: "16"
+  replicated:
+    size: 3
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: lfa.rgw.meta
+  namespace: rook-ceph
+spec:
+  application: rgw
+  failureDomain: host
+  parameters:
+    nodelete: "true"
+    nosizechange: "true"
+    pg_autoscale_mode: "off"
+    pg_num: "16"
+  replicated:
+    size: 3
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: lfa.rgw.log
+  namespace: rook-ceph
+spec:
+  application: rgw
+  failureDomain: host
+  parameters:
+    nodelete: "true"
+    nosizechange: "true"
+    pg_autoscale_mode: "off"
+    pg_num: "16"
+  replicated:
+    size: 3
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: lfa.rgw.buckets.index
+  namespace: rook-ceph
+spec:
+  application: rgw
+  failureDomain: host
+  parameters:
+    nodelete: "true"
+    nosizechange: "true"
+    pg_autoscale_mode: "off"
+    pg_num: "32"
+  replicated:
+    size: 3
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: lfa.rgw.buckets.non-ec
+  namespace: rook-ceph
+spec:
+  application: rgw
+  failureDomain: host
+  parameters:
+    nodelete: "true"
+    nosizechange: "true"
+    pg_autoscale_mode: "off"
+    pg_num: "16"
+  replicated:
+    size: 3
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: lfa.rgw.otp
+  namespace: rook-ceph
+spec:
+  application: rgw
+  failureDomain: host
+  parameters:
+    nodelete: "true"
+    nosizechange: "true"
+    pg_autoscale_mode: "off"
+    pg_num: "1"
+  replicated:
+    size: 3
+---
+apiVersion: ceph.rook.io/v1
+kind: CephBlockPool
+metadata:
+  name: lfa.rgw.buckets.data
+  namespace: rook-ceph
+spec:
+  application: rgw
+  erasureCoded:
+    dataChunks: 2
+    codingChunks: 1
+  failureDomain: host
+  parameters:
+    nodelete: "true"
+    nosizechange: "true"
+    pg_autoscale_mode: "off"
+    bulk: "true"
+    pg_num: "256"
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: lfa
+provisioner: rook-ceph.ceph.rook.io/bucket
+parameters:
+  objectStoreName: lfa
+  objectStoreNamespace: rook-ceph
+reclaimPolicy: Retain

fleet/lib/rook-ceph-conf/charts/ayekan/templates/cephobjectstoreuser-butler.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStoreUser
+metadata:
+  name: butler
+  namespace: rook-ceph
+spec:
+  store: lfa
+  clusterNamespace: rook-ceph
+  quotas:
+    maxBuckets: 2
+    maxSize: 2Pi

fleet/lib/rook-ceph-conf/charts/ayekan/templates/cephobjectstoreuser-calib.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStoreUser
+metadata:
+  name: calib
+  namespace: rook-ceph
+spec:
+  store: lfa
+  clusterNamespace: rook-ceph

fleet/lib/rook-ceph-conf/charts/ayekan/templates/cephobjectstoreuser-comcam.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStoreUser
+metadata:
+  name: comcam
+  namespace: rook-ceph
+spec:
+  store: lfa
+  clusterNamespace: rook-ceph
+  quotas:
+    maxBuckets: 2

fleet/lib/rook-ceph-conf/charts/ayekan/templates/cephobjectstoreuser-extended-ceph-exporter.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: ceph.rook.io/v1
+kind: CephObjectStoreUser
+metadata:
+  name: extended-ceph-exporter
+  namespace: rook-ceph
+spec:
+  store: lfa
+  clusterNamespace: rook-ceph
+  displayName: extended-ceph-exporter
+  capabilities:
+    buckets: read
+    users: read
+    usage: read
+    metadata: read
+    zone: read