fix: load workload values (#860)

* fix: load workload values

* fix: load comment lines

* fix: get all resources

* fix: get all resources

* fix: loading platform and team resources by kind

* fix: loading raw yaml and gracefully handle errors

* fix: loading raw yaml and gracefully handle errors

---------

Co-authored-by: svcAPLBot <[email protected]>
Co-authored-by: Cas Lubbers <[email protected]>

Author: 3 people

src/fileStore/file-store.ts

@@ -1,15 +1,15 @@
 // The in-memory key-value store: file path -> parsed content
-import path from 'path'
-import { globSync } from 'glob'
+import Debug from 'debug'
 import { ensureDir } from 'fs-extra'
 import { writeFile } from 'fs/promises'
+import { globSync } from 'glob'
+import { merge } from 'lodash'
+import path from 'path'
 import { stringify as stringifyYaml } from 'yaml'
 import { z } from 'zod'
-import { merge } from 'lodash'
-import { loadYaml } from '../utils'
-import { getFileMapForKind, getFileMaps, getResourceFilePath } from './file-map'
 import { APL_KINDS, AplKind, AplObject, AplPlatformObject, AplRecord, AplTeamObject } from '../otomi-models'
-import Debug from 'debug'
+import { loadRawYaml, loadYaml } from '../utils'
+import { getFileMapForKind, getFileMaps, getResourceFilePath } from './file-map'
 
 const debug = Debug('otomi:file-store')
 
@@ -44,6 +44,10 @@ function shouldSkipValidation(filePath: string): boolean {
   return filePath.includes('/sealedsecrets/') || filePath.includes('/workloadValues/')
 }
 
+function isRawContent(filePath: string): boolean {
+  return filePath.includes('/workloadValues/')
+}
+
 export class FileStore {
   private store: Map<string, AplObject> = new Map()
 
@@ -68,29 +72,33 @@ export class FileStore {
 
     await Promise.all(
       filesToLoad.map(async (filePath) => {
-        const rawContent = await loadYaml(filePath)
-        const relativePath = path.relative(envDir, filePath).replace(/\.dec$/, '')
-
-        // Skip validation for specific file paths
-        if (shouldSkipValidation(filePath)) {
-          allFiles.set(relativePath, rawContent as AplObject)
-          return
+        try {
+          const rawContent = isRawContent(filePath) ? await loadRawYaml(filePath) : await loadYaml(filePath)
+          const relativePath = path.relative(envDir, filePath).replace(/\.dec$/, '')
+
+          // Skip validation for specific file paths
+          if (shouldSkipValidation(filePath)) {
+            allFiles.set(relativePath, rawContent as AplObject)
+            return
+          }
+
+          // Validate all other kinds
+          const result = AplObjectSchema.safeParse(rawContent)
+
+          if (!result.success) {
+            debug(`Validation failed for ${relativePath}:`, result.error.message)
+            return
+          }
+
+          if (!result.data) {
+            debug(`No content found for ${relativePath}`)
+            return
+          }
+
+          allFiles.set(relativePath, result.data as AplObject)
+        } catch (error) {
+          debug(`Failed to load file ${filePath}:`, error)
         }
-
-        // Validate all other kinds
-        const result = AplObjectSchema.safeParse(rawContent)
-
-        if (!result.success) {
-          debug(`Validation failed for ${relativePath}:`, result.error.message)
-          return
-        }
-
-        if (!result.data) {
-          debug(`No content found for ${relativePath}`)
-          return
-        }
-
-        allFiles.set(relativePath, result.data as AplObject)
       }),
     )
 
@@ -171,23 +179,72 @@ export class FileStore {
     return filePath
   }
 
-  // Generic method for all resources (platform and team)
-  getByKind(kind: AplKind, teamId?: string): Map<string, AplObject> {
+  // Get platform resources (no team scope - e.g., AplUser, settings)
+  getPlatformResourcesByKind(kind: AplKind): Map<string, AplObject> {
     const fileMap = getFileMapForKind(kind)
     if (!fileMap) {
       throw new Error(`Unknown kind: ${kind}`)
     }
 
-    // Generate path prefix from template (e.g., 'env/teams/team1/workloads/')
-    const prefix = fileMap.pathTemplate.replace('{teamId}', teamId || '').replace('{name}.yaml', '')
+    // Platform resources don't have {teamId} in their path
+    // e.g., 'env/settings/{name}.yaml' → 'env/settings/'
+    const prefix = fileMap.pathTemplate.replace('{name}.yaml', '')
+    const result = new Map<string, AplObject>()
+
+    for (const filePath of this.store.keys()) {
+      if (filePath.startsWith(prefix) && filePath.endsWith('.yaml')) {
+        const content = this.store.get(filePath)
+        if (content) result.set(filePath, content)
+      }
+    }
 
+    return result
+  }
+
+  // Get ALL team resources across all teams (e.g., all workloads, all services)
+  getAllTeamResourcesByKind(kind: AplKind): Map<string, AplObject> {
+    const fileMap = getFileMapForKind(kind)
+    if (!fileMap) {
+      throw new Error(`Unknown kind: ${kind}`)
+    }
+
+    // Extract resource path segment from template
+    // e.g., 'env/teams/{teamId}/workloads/{name}.yaml' → '/workloads'
+    const parts = fileMap.pathTemplate.split('{teamId}')
+    const resourcePath = parts[1].replace('/{name}.yaml', '')
+    const result = new Map<string, AplObject>()
+
+    // Match any path containing this resource segment
+    // e.g., matches 'env/teams/*/workloads/*.yaml'
+    for (const filePath of this.store.keys()) {
+      if (filePath.includes(resourcePath) && filePath.endsWith('.yaml')) {
+        const content = this.store.get(filePath)
+        if (content) result.set(filePath, content)
+      }
+    }
+
+    return result
+  }
+
+  getTeamResourcesByKindAndTeamId(kind: AplKind, teamId: string): Map<string, AplObject> {
+    const fileMap = getFileMapForKind(kind)
+    if (!fileMap) {
+      throw new Error(`Unknown kind: ${kind}`)
+    }
+
+    // Generate exact prefix for this team
+    // e.g., 'env/teams/team1/workloads/'
+    const prefix = fileMap.pathTemplate.replace('{teamId}', teamId).replace('{name}.yaml', '')
     const result = new Map<string, AplObject>()
+
+    // Exact prefix match
     for (const filePath of this.store.keys()) {
       if (filePath.startsWith(prefix) && filePath.endsWith('.yaml')) {
         const content = this.store.get(filePath)
         if (content) result.set(filePath, content)
       }
     }
+
     return result
   }
 

src/otomi-stack.ts

@@ -383,7 +383,7 @@ export default class OtomiStack {
         const fileMap = settingsFileMaps.get(key)
         if (!fileMap) return // Skip unknown keys
 
-        const files = this.fileStore.getByKind(fileMap.kind)
+        const files = this.fileStore.getPlatformResourcesByKind(fileMap.kind)
         for (const [, content] of files) {
           settings[key] = content?.spec || content
         }
@@ -399,7 +399,7 @@ export default class OtomiStack {
 
     // No keys specified: fetch all settings
     for (const [name, fileMap] of settingsFileMaps.entries()) {
-      const files = this.fileStore.getByKind(fileMap.kind)
+      const files = this.fileStore.getPlatformResourcesByKind(fileMap.kind)
       for (const [, content] of files) {
         settings[name] = content?.spec || content
       }
@@ -643,7 +643,7 @@ export default class OtomiStack {
     const teamIds = this.fileStore.getTeamIds()
 
     for (const teamId of teamIds) {
-      const settingsFiles = this.fileStore.getByKind('AplTeamSettingSet', teamId)
+      const settingsFiles = this.fileStore.getTeamResourcesByKindAndTeamId('AplTeamSettingSet', teamId)
       for (const [, content] of settingsFiles) {
         // v1 format: return spec directly
         const team = getV1ObjectFromApl(content as AplTeamSettingsResponse) as Team
@@ -667,7 +667,7 @@ export default class OtomiStack {
 
     for (const teamId of teamIds) {
       if (teamId === 'admin') continue
-      const settingsFiles = this.fileStore.getByKind('AplTeamSettingSet', teamId)
+      const settingsFiles = this.fileStore.getTeamResourcesByKindAndTeamId('AplTeamSettingSet', teamId)
       for (const [, content] of settingsFiles) {
         if (content) {
           // Return full v2 object with password removed
@@ -859,7 +859,7 @@ export default class OtomiStack {
   }
 
   getTeamAplNetpols(teamId: string): AplNetpolResponse[] {
-    const files = this.fileStore.getByKind('AplTeamNetworkControl', teamId)
+    const files = this.fileStore.getTeamResourcesByKindAndTeamId('AplTeamNetworkControl', teamId)
     return Array.from(files.values()) as AplNetpolResponse[]
   }
 
@@ -868,7 +868,7 @@ export default class OtomiStack {
   }
 
   getAllAplNetpols(): AplNetpolResponse[] {
-    const files = this.fileStore.getByKind('AplTeamNetworkControl')
+    const files = this.fileStore.getAllTeamResourcesByKind('AplTeamNetworkControl')
     return Array.from(files.values()) as AplNetpolResponse[]
   }
 
@@ -936,7 +936,7 @@ export default class OtomiStack {
   }
 
   getAllUsers(sessionUser: SessionUser): Array<User> {
-    const files = this.fileStore.getByKind('AplUser')
+    const files = this.fileStore.getPlatformResourcesByKind('AplUser')
     const aplObjects = Array.from(files.values()) as AplObject[]
     const users = aplObjects.map((aplObject) => {
       return { ...aplObject.spec, id: aplObject.metadata.name } as User
@@ -970,7 +970,7 @@ export default class OtomiStack {
     const user: User = { ...data, id: userId, initialPassword }
 
     // Get existing users' emails
-    const files = this.fileStore.getByKind('AplUser')
+    const files = this.fileStore.getPlatformResourcesByKind('AplUser')
     let existingUsersEmail = Array.from(files.values()).map((aplObject: AplObject) => aplObject.spec.email)
 
     if (!env.isDev) {
@@ -1120,7 +1120,7 @@ export default class OtomiStack {
   }
 
   getTeamAplCodeRepos(teamId: string): AplCodeRepoResponse[] {
-    const files = this.fileStore.getByKind('AplTeamCodeRepo', teamId)
+    const files = this.fileStore.getTeamResourcesByKindAndTeamId('AplTeamCodeRepo', teamId)
     return Array.from(files.values()) as AplCodeRepoResponse[]
   }
 
@@ -1129,7 +1129,7 @@ export default class OtomiStack {
   }
 
   getAllAplCodeRepos(): AplCodeRepoResponse[] {
-    const files = this.fileStore.getByKind('AplTeamCodeRepo')
+    const files = this.fileStore.getAllTeamResourcesByKind('AplTeamCodeRepo')
     return Array.from(files.values()) as AplCodeRepoResponse[]
   }
 
@@ -1298,7 +1298,7 @@ export default class OtomiStack {
   }
 
   getTeamAplBuilds(teamId: string): AplBuildResponse[] {
-    const files = this.fileStore.getByKind('AplTeamBuild', teamId)
+    const files = this.fileStore.getTeamResourcesByKindAndTeamId('AplTeamBuild', teamId)
     return Array.from(files.values()) as AplBuildResponse[]
   }
 
@@ -1307,7 +1307,7 @@ export default class OtomiStack {
   }
 
   getAllAplBuilds(): AplBuildResponse[] {
-    const files = this.fileStore.getByKind('AplTeamBuild')
+    const files = this.fileStore.getAllTeamResourcesByKind('AplTeamBuild')
     return Array.from(files.values()) as AplBuildResponse[]
   }
 
@@ -1387,7 +1387,7 @@ export default class OtomiStack {
   }
 
   getTeamAplPolicies(teamId: string): AplPolicyResponse[] {
-    const files = this.fileStore.getByKind('AplTeamPolicy', teamId)
+    const files = this.fileStore.getTeamResourcesByKindAndTeamId('AplTeamPolicy', teamId)
     return Array.from(files.values()) as AplPolicyResponse[]
   }
 
@@ -1401,7 +1401,7 @@ export default class OtomiStack {
   }
 
   getAllAplPolicies(): AplPolicyResponse[] {
-    const files = this.fileStore.getByKind('AplTeamPolicy')
+    const files = this.fileStore.getAllTeamResourcesByKind('AplTeamPolicy')
     return Array.from(files.values()) as AplPolicyResponse[]
   }
 
@@ -1610,7 +1610,7 @@ export default class OtomiStack {
   }
 
   getTeamAplWorkloads(teamId: string): AplWorkloadResponse[] {
-    const files = this.fileStore.getByKind('AplTeamWorkload', teamId)
+    const files = this.fileStore.getTeamResourcesByKindAndTeamId('AplTeamWorkload', teamId)
     return Array.from(files.values()) as AplWorkloadResponse[]
   }
 
@@ -1632,7 +1632,7 @@ export default class OtomiStack {
   }
 
   getAllAplWorkloads(): AplWorkloadResponse[] {
-    const files = this.fileStore.getByKind('AplTeamWorkload')
+    const files = this.fileStore.getAllTeamResourcesByKind('AplTeamWorkload')
     return Array.from(files.values()) as AplWorkloadResponse[]
   }
 
@@ -1738,7 +1738,7 @@ export default class OtomiStack {
   }
 
   getAllAplServices(): AplServiceResponse[] {
-    const files = this.fileStore.getByKind('AplTeamService')
+    const files = this.fileStore.getAllTeamResourcesByKind('AplTeamService')
     return Array.from(files.values()) as AplServiceResponse[]
   }
 
@@ -1747,7 +1747,7 @@ export default class OtomiStack {
   }
 
   getTeamAplServices(teamId: string): AplServiceResponse[] {
-    const files = this.fileStore.getByKind('AplTeamService', teamId)
+    const files = this.fileStore.getTeamResourcesByKindAndTeamId('AplTeamService', teamId)
     return Array.from(files.values()) as AplServiceResponse[]
   }
 
@@ -2155,7 +2155,7 @@ export default class OtomiStack {
   }
 
   getAllAplSealedSecrets(): AplSecretResponse[] {
-    const files = this.fileStore.getByKind('AplTeamSecret')
+    const files = this.fileStore.getAllTeamResourcesByKind('AplTeamSecret')
     return Array.from(files.values()) as AplSecretResponse[]
   }
 
@@ -2167,7 +2167,7 @@ export default class OtomiStack {
   }
 
   getAplSealedSecrets(teamId: string): AplSecretResponse[] {
-    const files = this.fileStore.getByKind('AplTeamSecret', teamId)
+    const files = this.fileStore.getTeamResourcesByKindAndTeamId('AplTeamSecret', teamId)
     return Array.from(files.values()) as AplSecretResponse[]
   }
 
@@ -2229,7 +2229,7 @@ export default class OtomiStack {
   }
 
   getAplKnowledgeBases(teamId: string): AplKnowledgeBaseResponse[] {
-    const files = this.fileStore.getByKind('AkamaiKnowledgeBase', teamId)
+    const files = this.fileStore.getTeamResourcesByKindAndTeamId('AkamaiKnowledgeBase', teamId)
     return Array.from(files.values()) as AplKnowledgeBaseResponse[]
   }
 
@@ -2294,12 +2294,12 @@ export default class OtomiStack {
   }
 
   getAplAgents(teamId: string): AplAgentResponse[] {
-    const files = this.fileStore.getByKind('AkamaiAgent', teamId)
+    const files = this.fileStore.getTeamResourcesByKindAndTeamId('AkamaiAgent', teamId)
     return Array.from(files.values()) as AplAgentResponse[]
   }
 
   getAllAplAgents(): AplAgentResponse[] {
-    const files = this.fileStore.getByKind('AkamaiAgent')
+    const files = this.fileStore.getAllTeamResourcesByKind('AkamaiAgent')
     return Array.from(files.values()) as AplAgentResponse[]
   }
 

src/utils.ts

@@ -65,12 +65,20 @@ export const flattenObject = (
     }, {})
 }
 
-export const loadYaml = async (path: string, opts?: { noError: boolean }): Promise<Record<string, any> | undefined> => {
+export const loadYaml = async (path: string): Promise<Record<string, any> | undefined> => {
   if (!(await pathExists(path))) {
-    if (opts?.noError) return undefined
     throw new Error(`${path} does not exist`)
   }
-  return parse(await readFile(path, 'utf-8')) as Record<string, any>
+  const rawFile = await readFile(path, 'utf-8')
+  return parse(rawFile) as Record<string, any>
+}
+
+export async function loadRawYaml(path: string) {
+  if (!(await pathExists(path))) {
+    throw new Error(`${path} does not exist`)
+  }
+  const rawFile = await readFile(path, 'utf-8')
+  return rawFile as any
 }
 
 const valuesSchemaEndpointUrl = `${BASEURL}/apl/schema`