Merge remote-tracking branch 'origin/main' into APL-1260

Author: svcAPLBot

eslint.config.mjs

@@ -75,6 +75,7 @@ export default defineConfig([
           varsIgnorePattern: '^_',
         },
       ],
+      
       '@typescript-eslint/no-use-before-define': ['error'],
       '@typescript-eslint/restrict-template-expressions': 'off',
       '@typescript-eslint/naming-convention': [

src/git.ts

@@ -203,6 +203,13 @@ export class Git {
     await writeFile(absolutePath, content, 'utf8')
   }
 
+  async writeTextFile(file: string, content: string): Promise<void> {
+    const absolutePath = join(this.path, file)
+    debug(`Writing to file: ${absolutePath}`)
+    await ensureDir(dirname(absolutePath))
+    await writeFile(absolutePath, content, 'utf8')
+  }
+
   async fileExists(relativePath: string): Promise<boolean> {
     const absolutePath = join(this.path, relativePath)
     return await pathExists(absolutePath)

src/otomi-models.ts

@@ -268,6 +268,7 @@ export interface Repo {
   users: User[]
   versions: Versions
   teamConfig: Record<string, TeamConfig>
+  files: Record<string, string>
 }
 
 export interface TeamConfig {

src/otomi-stack.ts

@@ -1,4 +1,4 @@
-import { CoreV1Api, KubeConfig, User as k8sUser, V1ObjectReference } from '@kubernetes/client-node'
+import { CoreV1Api, User as k8sUser, KubeConfig, V1ObjectReference } from '@kubernetes/client-node'
 import Debug from 'debug'
 
 import { getRegions, ObjectStorageKeyRegions } from '@linode/api-v4'
@@ -94,6 +94,10 @@ import {
 } from 'src/validators'
 import { v4 as uuidv4 } from 'uuid'
 import { parse as parseYaml, stringify as stringifyYaml } from 'yaml'
+import { getAIModels } from './ai/aiModelHandler'
+import { AkamaiAgentCR } from './ai/AkamaiAgentCR'
+import { AkamaiKnowledgeBaseCR } from './ai/AkamaiKnowledgeBaseCR'
+import { DatabaseCR } from './ai/DatabaseCR'
 import {
   apply,
   checkPodExists,
@@ -121,10 +125,6 @@ import { getSealedSecretsPEM, sealedSecretManifest, SealedSecretManifestType } f
 import { getKeycloakUsers, isValidUsername } from './utils/userUtils'
 import { ObjectStorageClient } from './utils/wizardUtils'
 import { fetchChartYaml, fetchWorkloadCatalog, NewHelmChartValues, sparseCloneChart } from './utils/workloadUtils'
-import { getAIModels } from './ai/aiModelHandler'
-import { AkamaiKnowledgeBaseCR } from './ai/AkamaiKnowledgeBaseCR'
-import { AkamaiAgentCR } from './ai/AkamaiAgentCR'
-import { DatabaseCR } from './ai/DatabaseCR'
 
 interface ExcludedApp extends App {
   managed: boolean
@@ -159,6 +159,14 @@ function getTeamSealedSecretsValuesFilePath(teamId: string, sealedSecretsName: s
   return `env/teams/${teamId}/sealedsecrets/${sealedSecretsName}`
 }
 
+function getTeamWorkloadValuesManagedFilePath(teamId: string, workloadName: string): string {
+  return `env/teams/${teamId}/workloadValues/${workloadName}.managed.yaml`
+}
+
+function getTeamWorkloadValuesFilePath(teamId: string, workloadName: string): string {
+  return `env/teams/${teamId}/workloadValues/${workloadName}.yaml`
+}
+
 function getTeamKnowledgeBaseValuesFilePath(teamId: string, knowledgeBaseName: string): string {
   return `env/teams/${teamId}/knowledgebases/${knowledgeBaseName}`
 }
@@ -272,16 +280,13 @@ export default class OtomiStack {
     })
   }
 
-  transformWorkloads(workloads: AplWorkloadResponse[], workloadValues: Record<string, any>[]): AplWorkloadResponse[] {
-    const values = {}
-    workloadValues.forEach((value) => {
-      const workloadName = value.name
-      if (workloadName) {
-        values[workloadName] = value.values
-      }
-    })
+  transformWorkloads(workloads: AplWorkloadResponse[], files: string[]): AplWorkloadResponse[] {
     return workloads.map((workload) => {
-      return merge(workload, { spec: { values: values[workload.metadata.name] } })
+      const workloadName = workload.metadata.name
+      const teamId = workload.metadata.labels?.['apl.io/teamId']
+
+      const filePath = getTeamWorkloadValuesFilePath(teamId, workloadName)
+      return merge(workload, { spec: { values: files[filePath] } })
     })
   }
 
@@ -314,10 +319,9 @@ export default class OtomiStack {
         apps: this.transformApps(teamConfig.apps),
         policies: this.transformPolicies(teamName, teamConfig.policies || {}),
         sealedsecrets: this.transformSecrets(teamName, teamConfig.sealedsecrets || []),
-        workloads: this.transformWorkloads(teamConfig.workloads || [], teamConfig.workloadValues || []),
+        workloads: this.transformWorkloads(teamConfig.workloads || [], rawRepo.files || {}),
         settings: this.transformTeamSettings(teamConfig.settings),
       }))
-
       const repo = rawRepo as Repo
       this.repoService = new RepoService(repo)
     }
@@ -834,18 +838,16 @@ export default class OtomiStack {
     await this.git.saveConfig(repo, fileMap)
   }
 
-  async saveTeamWorkloadValues(data: AplWorkloadResponse) {
+  async saveTeamWorkloadValues(data: AplWorkloadResponse, createManagedFile = false) {
     const { metadata } = data
     const teamId = metadata.labels['apl.io/teamId']!
     debug(`Saving AplTeamWorkloadValues ${metadata.name} for team ${teamId}`)
-    const values = {
-      name: metadata.name,
-      values: data.spec.values,
+    const filePath = getTeamWorkloadValuesFilePath(teamId, metadata.name)
+    await this.git.writeTextFile(filePath, data.spec.values || '{}')
+    if (createManagedFile) {
+      const filePathValuesManaged = getTeamWorkloadValuesManagedFilePath(teamId, metadata.name)
+      await this.git.writeTextFile(filePathValuesManaged, '')
     }
-    const configKey = this.getConfigKey('AplTeamWorkloadValues')
-    const repo = this.createTeamConfigInRepo(teamId, configKey, [values])
-    const fileMap = getFileMaps('').find((fm) => fm.kind === 'AplTeamWorkloadValues')!
-    await this.git.saveConfig(repo, fileMap, false)
   }
 
   async saveTeamPolicy(teamId: string, data: AplPolicyResponse): Promise<void> {
@@ -884,8 +886,10 @@ export default class OtomiStack {
     const fileMapWorkload = getFileMaps('').find((fm) => fm.kind === 'AplTeamWorkload')!
     const repoValues = this.createTeamConfigInRepo(teamId, 'workloads', [data])
     const fileMapValues = getFileMaps('').find((fm) => fm.kind === 'AplTeamWorkloadValues')!
+    const filePathValuesManaged = getTeamWorkloadValuesManagedFilePath(teamId, metadata.name)
     await this.git.deleteConfig(repoWorkload, fileMapWorkload)
     await this.git.deleteConfig(repoValues, fileMapValues)
+    await this.git.removeFile(filePathValuesManaged)
   }
 
   getTeamBackups(teamId: string): Backup[] {
@@ -1786,7 +1790,7 @@ export default class OtomiStack {
     try {
       const workload = this.repoService.getTeamConfigService(teamId).createWorkload(data)
       await this.saveTeamWorkload(workload)
-      await this.saveTeamWorkloadValues(workload)
+      await this.saveTeamWorkloadValues(workload, true)
       await this.doTeamDeployment(
         teamId,
         (teamService) => {

src/repo.ts

@@ -419,10 +419,6 @@ export function getFileMap(kind: AplKind, envDir: string): FileMap {
 }
 
 export function renderManifest(fileMap: FileMap, jsonPath: jsonpath.PathComponent[], data: Record<string, any>) {
-  //TODO remove this custom workaround for workloadValues
-  if (fileMap.kind === 'AplTeamWorkloadValues') {
-    return { values: data.values }
-  }
   let spec = data
   const labels = {}
   if (fileMap.resourceGroup === 'team') {
@@ -489,6 +485,10 @@ export async function loadFileToSpec(
   const jsonPath = getJsonPath(fileMap, filePath)
   try {
     const data = (await deps.loadYaml(filePath)) || {}
+    // ensure that local path does not include envDir and the leading slash
+    const localFilePath = filePath.replace(fileMap.envDir, '').replace(/^\/+/, '')
+    // eslint-disable-next-line no-param-reassign
+    spec.files[localFilePath] = data
     if (fileMap.processAs === 'arrayItem') {
       const ref: Record<string, any>[] = get(spec, jsonPath)
       const name = filePath.match(/\/([^/]+)\.yaml$/)?.[1]
@@ -577,7 +577,9 @@ export async function loadToSpec(
 export async function loadValues(envDir: string, deps = { loadToSpec }): Promise<Record<string, any>> {
   //We need everything to load to spec for the API
   const fileMaps = getFileMaps(envDir)
-  const spec = {}
+  const spec = {
+    files: {},
+  }
 
   await Promise.all(
     fileMaps.map(async (fileMap) => {

src/services/RepoService.test.ts

@@ -39,6 +39,7 @@ describe('RepoService', () => {
       obj: {},
       oidc: { issuer: 'https://issuer.com', clientID: 'client-id', clientSecret: 'client-secret' },
       versions: { version: '1.0.0' },
+      files: {},
     } as Repo
     service = new RepoService(repo)
     service.createTeamConfig(teamSettings)

src/services/RepoService.ts

@@ -52,6 +52,7 @@ export class RepoService {
     this.repo.users ??= []
     this.repo.versions ??= {} as Versions
     this.repo.teamConfig ??= {}
+    this.repo.files ??= {}
   }
 
   public getTeamConfigService(teamName: string): TeamConfigService {