feat: add selectable regions to object storage wizard

package.json

@@ -12,6 +12,7 @@
     "@apidevtools/json-schema-ref-parser": "^9.0.9",
     "@casl/ability": "^6.3.2",
     "@kubernetes/client-node": "^0.22.0",
+    "@linode/api-v4": "^0.129.0",
     "@types/json-schema": "^7.0.7",
     "@types/jsonwebtoken": "^9.0.1",
     "aws-sdk": "^2.879.0",
@@ -95,6 +96,7 @@
     "openapi-schema-validator": "3.0.3",
     "openapi-typescript": "5.3.0",
     "prettier": "2.6.2",
+    "proxyquire": "^2.1.3",
     "sinon": "8.1.1",
     "sinon-chai": "3.7.0",
     "standard-version": "9.5.0",

src/api/objwizard.ts

@@ -5,13 +5,6 @@ import { ObjWizard, OpenApiRequestExt } from 'src/otomi-models'
 const debug = Debug('otomi:api:objwizard')
 
 export default function (): OperationHandlerArray {
-  const get: Operation = [
-    ({ otomi }: OpenApiRequestExt, res): void => {
-      debug('getObjWizard')
-      const v = otomi.getObjWizard()
-      res.json(v)
-    },
-  ]
   const post: Operation = [
     async ({ otomi, body }: OpenApiRequestExt, res): Promise<void> => {
       debug('createObjWizard')
@@ -20,7 +13,6 @@ export default function (): OperationHandlerArray {
     },
   ]
   const api = {
-    get,
     post,
   }
   return api

src/openapi/api.yaml

@@ -1418,17 +1418,6 @@ paths:
                 $ref: '#/components/schemas/SettingsInfo'
 
   '/objwizard':
-    get:
-      operationId: getObjWizard
-      x-aclSchema: ObjWizard
-      responses:
-        <<: *DefaultGetResponses
-        '200':
-          description: Successfully obtained obj wizard configuration
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ObjWizard'
     post:
       operationId: createObjWizard
       description: Create a obj wizard configuration

src/openapi/objwizard.yaml

@@ -13,4 +13,7 @@ ObjWizard:
       type: string
       description: The Linode API token for creating object storage access key and buckets.
       $ref: definitions.yaml#/wordCharacterPattern
+    regionId:
+      type: string
+      description: The region where the object storage buckets will be created.
   type: object

src/openapi/session.yaml

@@ -20,10 +20,29 @@ Session:
       # readOnly: true
     defaultPlatformAdminEmail:
       type: string
-    objStorageApps:
-      type: array
-      items:
-        type: object
+    objectStorage:
+      type: object
+      properties:
+        showWizard:
+          type: boolean
+        objStorageApps:
+          type: array
+          items:
+            type: object
+            properties:
+              appId:
+                type: string
+              required:
+                type: boolean
+        objStorageRegions:
+          type: array
+          items:
+            type: object
+            properties:
+              id:
+                type: string
+              label:
+                type: string
     versions:
       type: object
       # readOnly: true

src/otomi-stack.ts

@@ -3,6 +3,7 @@ import * as k8s from '@kubernetes/client-node'
 import { V1ObjectReference } from '@kubernetes/client-node'
 import Debug from 'debug'
 
+import { ObjectStorageKeyRegions, getRegions } from '@linode/api-v4'
 import { emptyDir, pathExists, unlink } from 'fs-extra'
 import { readFile, readdir, writeFile } from 'fs/promises'
 import { generate as generatePassword } from 'generate-password'
@@ -72,7 +73,7 @@ import { validateBackupFields } from './utils/backupUtils'
 import { getPolicies } from './utils/policiesUtils'
 import { encryptSecretItem } from './utils/sealedSecretUtils'
 import { getKeycloakUsers } from './utils/userUtils'
-import { createObjectStorageAccessKey, createObjectStorageBucket, getClusterRegion } from './utils/wizardUtils'
+import { ObjectStorageClient } from './utils/wizardUtils'
 import { fetchWorkloadCatalog } from './utils/workloadUtils'
 
 interface ExcludedApp extends App {
@@ -276,45 +277,51 @@ export default class OtomiStack {
     return settingsInfo
   }
 
-  getObjWizard(): ObjWizard {
-    const { obj } = this.getSettings(['obj'])
-    return { showWizard: obj?.showWizard ?? true } as ObjWizard
-  }
-
   async createObjWizard(data: ObjWizard): Promise<void> {
     const { obj } = this.getSettings(['obj'])
     const settingsdata = { obj: { ...obj, showWizard: data.showWizard } }
-    if (data?.apiToken) {
+    if (data?.apiToken && data?.regionId) {
       const { cluster } = this.getSettings(['cluster'])
-      const clusterId = cluster?.name?.replace('aplinstall', '')
-      const clusterRegion = await getClusterRegion(data.apiToken, clusterId)
-      const { access_key, secret_key, regions } = await createObjectStorageAccessKey(
-        data.apiToken,
-        clusterId,
-        clusterRegion,
-      )
-      const { s3_endpoint } = regions.find((region) => region.id === clusterRegion)
-      const objStorageRegion = s3_endpoint.split('.')[0] as string
-      const buckets = ['cnpg', 'harbor', 'loki', 'tempo', 'velero', 'gitea', 'thanos']
-      for (const bucket of buckets) {
-        const res = await createObjectStorageBucket(data.apiToken, `lke${clusterId}-${bucket}`, clusterRegion)
-        debug(`${res.label} is created!`)
+      const lkeClusterId = Number(cluster?.name?.replace('aplinstall', ''))
+      if (!lkeClusterId) throw new OtomiError('Cluster ID is not found in the cluster name')
+      const bucketNames = {
+        cnpg: `lke${lkeClusterId}-cnpg`,
+        harbor: `lke${lkeClusterId}-harbor`,
+        loki: `lke${lkeClusterId}-loki`,
+        tempo: `lke${lkeClusterId}-tempo`,
+        velero: `lke${lkeClusterId}-velero`,
+        gitea: `lke${lkeClusterId}-gitea`,
+        thanos: `lke${lkeClusterId}-thanos`,
       }
+      const objectStorageClient = new ObjectStorageClient(data.apiToken)
+      // Create object storage buckets
+      for (const bucket in bucketNames) {
+        const bucketLabel = await objectStorageClient.createObjectStorageBucket(
+          bucketNames[bucket] as string,
+          data.regionId,
+        )
+        debug(`${bucketLabel} bucket is created.`)
+      }
+      // Create object storage keys
+      const { access_key, secret_key, regions } = await objectStorageClient.createObjectStorageKey(
+        lkeClusterId,
+        data.regionId,
+        Object.values(bucketNames),
+      )
+      // The data.regionId (for example 'eu-central') does not include the zone.
+      // However, we need to add the region with the zone suffix (for example 'eu-central-1') in the object storage values.
+      // Therefore, we need to extract the region with the zone suffix from the s3_endpoint.
+      const { s3_endpoint } = regions.find((region) => region.id === data.regionId) as ObjectStorageKeyRegions
+      const [objStorageRegion] = s3_endpoint.split('.')
+      debug(`Object Storage keys are created.`)
+      // Modify object storage settings
       settingsdata.obj = {
         showWizard: false,
         provider: {
           type: 'linode',
           linode: {
             accessKeyId: access_key,
-            buckets: {
-              cnpg: `lke${clusterId}-cnpg`,
-              harbor: `lke${clusterId}-harbor`,
-              loki: `lke${clusterId}-loki`,
-              tempo: `lke${clusterId}-tempo`,
-              velero: `lke${clusterId}-velero`,
-              gitea: `lke${clusterId}-gitea`,
-              thanos: `lke${clusterId}-thanos`,
-            },
+            buckets: bucketNames,
             region: objStorageRegion,
             secretAccessKey: secret_key,
           },
@@ -323,6 +330,7 @@ export default class OtomiStack {
     }
     await this.editSettings(settingsdata as Settings, 'obj')
     await this.doDeployment()
+    debug('Object storage settings have been configured.')
   }
 
   getSettings(keys?: string[]): Settings {
@@ -1889,6 +1897,13 @@ export default class OtomiStack {
     const rootStack = await getSessionStack()
     const valuesSchema = await getValuesSchema()
     const currentSha = rootStack.repo.commitSha
+    const { obj } = this.getSettings(['obj'])
+    const regions = await getRegions()
+    const objStorageRegions =
+      regions.data
+        .filter((region) => region.capabilities.includes('Object Storage'))
+        .map(({ id, label }) => ({ id, label }))
+        .sort((a, b) => a.label.localeCompare(b.label)) || []
     const data: Session = {
       ca: env.CUSTOM_ROOT_CA,
       core: this.getCore() as Record<string, any>,
@@ -1897,7 +1912,11 @@ export default class OtomiStack {
       inactivityTimeout: env.EDITOR_INACTIVITY_TIMEOUT,
       user: user as SessionUser,
       defaultPlatformAdminEmail: env.DEFAULT_PLATFORM_ADMIN_EMAIL,
-      objStorageApps: env.OBJ_STORAGE_APPS,
+      objectStorage: {
+        showWizard: obj?.showWizard ?? true,
+        objStorageApps: env.OBJ_STORAGE_APPS,
+        objStorageRegions,
+      },
       versions: {
         core: env.VERSIONS.core,
         api: env.VERSIONS.api ?? process.env.npm_package_version,

src/utils/wizardUtils.test.ts

@@ -0,0 +1,157 @@
+import { ObjectStorageKey } from '@linode/api-v4'
+import { expect } from 'chai'
+import proxyquire from 'proxyquire'
+import sinon from 'sinon'
+import { OtomiError } from 'src/error'
+
+describe('ObjectStorageClient', () => {
+  let ObjectStorageClient: any
+  let setTokenStub: sinon.SinonStub
+  let getKubernetesClusterStub: sinon.SinonStub
+  let createObjectStorageKeysStub: sinon.SinonStub
+  let createBucketStub: sinon.SinonStub
+  let client: any
+  const clusterId = 12345
+
+  beforeEach(() => {
+    setTokenStub = sinon.stub()
+    getKubernetesClusterStub = sinon.stub()
+    createObjectStorageKeysStub = sinon.stub()
+    createBucketStub = sinon.stub()
+
+    // Use proxyquire to mock the module imports
+    const module = proxyquire('./wizardUtils.ts', {
+      '@linode/api-v4': {
+        setToken: setTokenStub,
+        getKubernetesCluster: getKubernetesClusterStub,
+        createObjectStorageKeys: createObjectStorageKeysStub,
+        createBucket: createBucketStub,
+      },
+    })
+
+    ObjectStorageClient = module.ObjectStorageClient
+    client = new ObjectStorageClient('test-token')
+  })
+
+  afterEach(() => {
+    sinon.restore()
+  })
+
+  describe('constructor', () => {
+    it('should set token when initialized', () => {
+      expect(setTokenStub.calledOnceWith('test-token')).to.be.true
+    })
+  })
+
+  describe('createObjectStorageBucket', () => {
+    const label = 'test-bucket'
+    const region = 'us-east'
+
+    it('should successfully create bucket', async () => {
+      const mockResponse = { label: 'test-bucket' }
+      createBucketStub.resolves(mockResponse)
+
+      const result = await client.createObjectStorageBucket(label, region)
+
+      expect(
+        createBucketStub.calledOnceWith({
+          label,
+          region,
+        }),
+      ).to.be.true
+      expect(result).to.equal('test-bucket')
+    })
+
+    it('should throw OtomiError when bucket creation fails', async () => {
+      const mockError = {
+        response: {
+          status: 401,
+          data: { errors: [{ reason: 'Your OAuth token is not authorized to use this endpoint' }] },
+        },
+      }
+      createBucketStub.rejects(mockError)
+
+      try {
+        await client.createObjectStorageBucket(label, region)
+        expect.fail('Should have thrown an error')
+      } catch (error) {
+        expect(error).to.be.instanceOf(OtomiError)
+        expect(error.publicMessage).to.equal('Your OAuth token is not authorized to use this endpoint')
+        expect(error.code).to.equal(401)
+      }
+    })
+
+    it('should throw OtomiError with default message when no specific error info', async () => {
+      const mockError = {
+        response: {
+          status: 500,
+        },
+      }
+      createBucketStub.rejects(mockError)
+
+      try {
+        await client.createObjectStorageBucket(label, region)
+        expect.fail('Should have thrown an error')
+      } catch (error) {
+        expect(error).to.be.instanceOf(OtomiError)
+        expect(error.publicMessage).to.equal('Error creating object storage bucket')
+        expect(error.code).to.equal(500)
+      }
+    })
+  })
+
+  describe('createObjectStorageKey', () => {
+    const region = 'us-east'
+    const bucketNames = ['bucket1', 'bucket2']
+    let clock: sinon.SinonFakeTimers
+
+    beforeEach(() => {
+      const fixedDate = new Date('2024-01-01T12:00:00.000Z')
+      clock = sinon.useFakeTimers(fixedDate.getTime())
+    })
+
+    afterEach(() => {
+      clock.restore()
+    })
+
+    it('should successfully create object storage keys', async () => {
+      const mockResponse: Pick<ObjectStorageKey, 'access_key' | 'secret_key' | 'regions'> = {
+        access_key: 'test-access-key',
+        secret_key: 'test-secret-key',
+        regions: [{ id: 'us-east', s3_endpoint: 'us-east-1.linodeobjects.com' }],
+      }
+      createObjectStorageKeysStub.resolves(mockResponse)
+      const result = await client.createObjectStorageKey(clusterId, region, bucketNames)
+
+      expect(createObjectStorageKeysStub.calledOnce).to.be.true
+      expect(createObjectStorageKeysStub.firstCall.args[0]).to.deep.equal({
+        label: `lke${clusterId}-key-1704110400000`,
+        regions: [region],
+        bucket_access: [
+          { bucket_name: 'bucket1', permissions: 'read_write', region },
+          { bucket_name: 'bucket2', permissions: 'read_write', region },
+        ],
+      })
+      expect(result).to.deep.equal(mockResponse)
+    })
+
+    it('should throw OtomiError when keys creation fails', async () => {
+      const mockError = {
+        response: {
+          data: { errors: [{ reason: 'Your OAuth token is not authorized to use this endpoint' }] },
+          status: 401,
+        },
+      }
+      createObjectStorageKeysStub.rejects(mockError)
+
+      try {
+        await client.createObjectStorageKey(clusterId, region, bucketNames)
+        expect.fail('Should have thrown an error')
+      } catch (error) {
+        expect(error).to.be.instanceOf(OtomiError)
+        expect(error.publicMessage).to.equal('Your OAuth token is not authorized to use this endpoint')
+        expect(error.code).to.equal(401)
+      }
+    })
+  })
+})