(cluster/lukay) migration rke1 -> rke2

fleet/lib/rook-ceph-cluster/overlays/lukay/values.yaml

@@ -1,68 +1,73 @@
 ---
 cephClusterSpec:
-  # XXX allow upgrade even if not healthy because lukay05 was removed
-  continueUpgradeAfterChecksEvenIfNotHealthy: true
   mon:
     count: 3
+  cephConfig:
+    global:
+      osd_pool_default_pg_autoscale_mode: warn
+      rgw_override_bucket_index_max_shards: "401"
+      rgw_enable_usage_log: "false"
+    mgr:
+      mgr/balancer/upmap_max_deviation: "1"
+    osd:
+      osd_max_pg_per_osd_hard_ratio: "10"
+      osd_op_queue: wpq
+      osd_scrub_auto_repair: "true"
   storage:
     useAllNodes: false
     useAllDevices: false
     config:
-      osdsPerDevice: "3"
+      osdsPerDevice: "1"
+      encryptedDevice: "true"
     nodes:
       - name: lukay01
         devices:
-          - name: sda
-          - name: sdb
-          - name: sdc
-          - name: sdd
-          - name: sde
-          - name: sdf
-          - name: sdg
-          - name: sdh
-          - name: sdi
-          - name: sdj
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R702912N
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R702901H
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R702902M
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R702906Y
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R702936A
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R704221R
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R702953P
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R804983Z
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R702951K
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R702385N
       - name: lukay02
         devices:
-          - name: sda
-          - name: sdb
-          - name: sdc
-          - name: sdd
-          - name: sde
-          - name: sdf
-          - name: sdg
-          - name: sdh
-          - name: sdi
-          - name: sdj
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R705147R
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R705139Z
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R705153V
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R705479L
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R705487X
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R705492F
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R705485Y
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R705109K
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R705141T
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R805000T
       - name: lukay03
         devices:
-          - name: sda
-          - name: sdb
-          - name: sdc
-          - name: sdd
-          - name: sde
-          - name: sdf
-          - name: sdg
-          - name: sdh
-          - name: sdi
-          - name: sdj
-      - name: lukay04
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R704207V
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R804245D
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R802783P
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R802782B
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R804846V
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R802794M
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0RA02831V
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0RA02160A
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R801805Z
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R804287J
+      - name: lukay05
         devices:
-          - name: sda
-          - name: sdb
-          - name: sdc
-          - name: sdd
-          - name: sde
-          - name: sdf
-          - name: sdg
-          - name: sdh
-          - name: sdi
-          - name: sdj
-  resources:
-    osd:
-      requests:
-        cpu: 100m
-        memory: 4Gi
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R703848Z
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R704195Z
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R704225B
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R705482V
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R804292R
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R804293V
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R804996Z
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R804998H
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R805002E
+          - name: /dev/disk/by-id/scsi-SATA_Samsung_SSD_870_S5VUNJ0R805003M
 
 cephBlockPools:
   - name: replicapool
@@ -74,12 +79,19 @@ cephBlockPools:
       quotas:
         maxSize: 2Ti
       enableRBDStats: true
+      parameters:
+        nodelete: "true"
+        nosizechange: "true"
+        pg_autoscale_mode: "off"
+        pg_num: "32"
     storageClass:
       name: rook-ceph-block
       enabled: true
       isDefault: true
       reclaimPolicy: Delete
       allowVolumeExpansion: true
+      mountOptions:
+        - discard
       parameters:
         clusterID: rook-ceph
         pool: replicapool

fleet/lib/rook-ceph-conf/charts/lukay/templates/cephobjectstore-it.yaml → fleet/lib/rook-ceph-conf/charts/lukay/templates/cephobjectstore-backups.yaml

@@ -2,32 +2,28 @@
 apiVersion: ceph.rook.io/v1
 kind: CephObjectRealm
 metadata:
-  name: it
-  namespace: rook-ceph
+  name: backups
 spec:
   defaultRealm: true
 ---
 apiVersion: ceph.rook.io/v1
 kind: CephObjectZoneGroup
 metadata:
-  name: it
-  namespace: rook-ceph
+  name: backups
 spec:
-  realm: it
+  realm: backups
 ---
 apiVersion: ceph.rook.io/v1
 kind: CephObjectZone
 metadata:
-  name: it
-  namespace: rook-ceph
+  name: backups
 spec:
-  zoneGroup: it
+  zoneGroup: backups
 ---
 apiVersion: ceph.rook.io/v1
 kind: CephObjectStore
 metadata:
-  name: it
-  namespace: rook-ceph
+  name: backups
 spec:
   metadataPool:
     failureDomain: host
@@ -49,30 +45,29 @@ spec:
     # securePort: 443
     instances: 3
   zone:
-    name: it
+    name: backups
 ---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
-  name: rook-ceph-rgw-it
-  namespace: rook-ceph
+  name: rook-ceph-rgw-backups
   annotations:
     cert-manager.io/cluster-issuer: letsencrypt
     nginx.ingress.kubernetes.io/proxy-body-size: 1024m
 spec:
   ingressClassName: nginx
   tls:
   - hosts:
-    - it-s3.ls.lsst.org
-    secretName: rook-ceph-rgw-it-ingress-tls
+    - s3-backups.ls.lsst.org
+    secretName: rook-ceph-rgw-backups-ingress-tls
   rules:
-  - host: it-s3.ls.lsst.org
+  - host: s3-backups.ls.lsst.org
     http:
       paths:
       - path: /
         pathType: Prefix
         backend:
           service:
-            name: rook-ceph-rgw-it
+            name: rook-ceph-rgw-backups
             port:
               number: 80

fleet/lib/rook-ceph-conf/charts/lukay/templates/cm-cephcli.yaml

@@ -10,10 +10,6 @@ data:
     ceph orch set backend ""
     ceph mgr module disable rook
 
-    waitfornfs backup
-    ceph nfs export rm backup /backup
-    ceph nfs export create cephfs backup /backup backup
-
     ceph mgr module enable rook
     ceph orch set backend rook
     ceph device monitoring on