fix: apl-operator upgrade

Author: Ani1357

package-lock.json

@@ -54,6 +54,7 @@
     "@types/async-retry": "1.4.9",
     "@types/debug": "4.1.12",
     "@types/express": "5.0.3",
+    "@types/fs-extra": "^11.0.4",
     "@types/ignore-walk": "4.0.3",
     "@types/jest": "^30.0.0",
     "@types/js-yaml": "4.0.9",

package.json

@@ -1,17 +1,16 @@
-import { mkdirSync, rmdirSync } from 'fs'
-import { pathExists } from 'fs-extra'
-import { writeFile } from 'fs/promises'
+import { mkdirSync, rmdirSync, existsSync } from 'fs'
+import { readFile, writeFile } from 'fs/promises'
 import { cleanupHandler, prepareEnvironment } from 'src/common/cli'
 import { logLevelString, terminal } from 'src/common/debug'
 import { hf } from 'src/common/hf'
-import { isResourcePresent, k8s, patchContainerResourcesOfSts } from 'src/common/k8s'
+import { appRevisionMatches, k8s, patchArgoCdApp, patchContainerResourcesOfSts } from 'src/common/k8s'
 import { getFilename, loadYaml } from 'src/common/utils'
 import { getImageTag, objectToYaml } from 'src/common/values'
 import { appPatches, genericPatch } from 'src/applicationPatches.json'
 import { getParsedArgs, HelmArguments, helmOptions, setParsedArgs } from 'src/common/yargs'
 import { Argv, CommandModule } from 'yargs'
 import { $ } from 'zx'
-import { V1ResourceRequirements } from '@kubernetes/client-node'
+import { ApiException, V1ResourceRequirements } from '@kubernetes/client-node'
 import { env } from '../common/envalid'
 
 const cmdName = getFilename(__filename)
@@ -92,27 +91,24 @@ const getArgocdAppManifest = (release: HelmRelease, values: Record<string, any>,
 const setFinalizers = async (name: string) => {
   d.info(`Setting finalizers for ${name}`)
   const resPatch =
-    await $`kubectl -n argocd patch application ${name} -p '{"metadata": {"finalizers": ["resources-finalizer.argocd.argoproj.io"]}}' --type merge`
+    await $`kubectl -n argocd patch applications.argoproj.io ${name} -p '{"metadata": {"finalizers": ["resources-finalizer.argocd.argoproj.io"]}}' --type merge`
   if (resPatch.exitCode !== 0) {
     throw new Error(`Failed to set finalizers for ${name}: ${resPatch.stderr}`)
   }
 }
 
 const getFinalizers = async (name: string): Promise<string[]> => {
-  const res = await $`kubectl -n argocd get application ${name} -o jsonpath='{.metadata.finalizers}'`
+  const res = await $`kubectl -n argocd get applications.argoproj.io ${name} -o jsonpath='{.metadata.finalizers}'`
   return res.stdout ? JSON.parse(res.stdout) : []
 }
 
-const removeApplication = async (release: HelmRelease): Promise<void> => {
-  const name = getAppName(release)
-  if (!(await isResourcePresent('application', name, 'argocd'))) return
-
+const removeApplication = async (name: string): Promise<void> => {
   try {
     const finalizers = await getFinalizers(name)
     if (!finalizers.includes('resources-finalizer.argocd.argoproj.io')) {
       await setFinalizers(name)
     }
-    const resDelete = await $`kubectl -n argocd delete application ${name}`
+    const resDelete = await $`kubectl -n argocd delete applications.argoproj.io ${name}`
     d.info(resDelete.stdout.toString().trim())
   } catch (e) {
     d.error(`Failed to delete application ${name}: ${e.message}`)
@@ -149,25 +145,62 @@ async function patchArgocdResources(release: HelmRelease, values: Record<string,
   }
 }
 
+const getApplications = async (): Promise<string[]> => {
+  const res = await $`kubectl get application.argoproj.io -n argocd -oname`
+  return res.stdout.split('\n')
+}
+
 const writeApplicationManifest = async (release: HelmRelease, otomiVersion: string): Promise<void> => {
   const appName = `${release.namespace}-${release.name}`
   const applicationPath = `${appsDir}/${appName}.yaml`
   const valuesPath = `${valuesDir}/${appName}.yaml`
   let values = {}
 
-  if (await pathExists(valuesPath)) values = (await loadYaml(valuesPath)) || {}
+  if (existsSync(valuesPath)) values = (await loadYaml(valuesPath)) || {}
   const manifest = getArgocdAppManifest(release, values, otomiVersion)
   await writeFile(applicationPath, objectToYaml(manifest))
 
   await patchArgocdResources(release, values)
 }
 
-export const applyAsApps = async (argv: HelmArguments): Promise<void> => {
+const getAplOperatorValues = async (): Promise<string> => {
+  await hf({
+    labelOpts: ['name=apl-operator'],
+    logLevel: logLevelString(),
+    args: ['write-values', `--output-file-template=${valuesDir}/{{.Release.Namespace}}-{{.Release.Name}}.yaml`],
+  })
+  return await readFile(`${valuesDir}/apl-operator-apl-operator.yaml`, 'utf-8')
+}
+
+export const applyAsApps = async (argv: HelmArguments): Promise<boolean> => {
   const helmfileSource = argv.file?.toString() || 'helmfile.d/'
   d.info(`Parsing helm releases defined in ${helmfileSource}`)
   setup()
   const otomiVersion = await getImageTag()
-
+  try {
+    const expectedRevision = env.APPS_REVISION || otomiVersion
+    d.info('Checking running revision of apl-operator...')
+    const operatorRevisionMatches = await appRevisionMatches(
+      'apl-operator-apl-operator',
+      env.APPS_REVISION || otomiVersion,
+      k8s.custom(),
+    )
+    if (operatorRevisionMatches) {
+      d.info(`Expected revision ${expectedRevision} found for apl-operator.`)
+    } else {
+      const values = await getAplOperatorValues()
+      d.info(`Updating apl-operator application to revision ${expectedRevision}.`)
+      await patchArgoCdApp('apl-operator-apl-operator', expectedRevision, values, k8s.custom())
+      d.info('Skipping further updates until apl-operator has restarted.')
+      return false
+    }
+  } catch (error) {
+    if (error instanceof ApiException && error.code === 404) {
+      d.info('apl-operator application not found, continuing')
+    } else {
+      throw error
+    }
+  }
   const res = await hf({
     fileOpts: argv.file,
     labelOpts: argv.label,
@@ -186,6 +219,7 @@ export const applyAsApps = async (argv: HelmArguments): Promise<void> => {
   const errors: Array<any> = []
   // Generate JSON object with all helmfile releases defined in helmfile.d
   const releases: [] = JSON.parse(res.stdout.toString())
+  const currentApplications = await getApplications()
   await Promise.allSettled(
     releases.map(async (release: HelmRelease) => {
       try {
@@ -197,7 +231,11 @@ export const applyAsApps = async (argv: HelmArguments): Promise<void> => {
 
         if (release.installed) await writeApplicationManifest(release, otomiVersion)
         else {
-          await removeApplication(release)
+          const appName = getAppName(release)
+          const resourceName = `application.argoproj.io/${appName}`
+          if (currentApplications.includes(resourceName)) {
+            await removeApplication(appName)
+          }
         }
       } catch (e) {
         errors.push(e)
@@ -218,6 +256,7 @@ export const applyAsApps = async (argv: HelmArguments): Promise<void> => {
     errors.map((e) => d.error(e))
     d.error(`Not all applications has been deployed successfully`)
   }
+  return true
 }
 
 export const module: CommandModule = {

src/cmd/apply-as-apps.ts

@@ -86,6 +86,7 @@ const applyAll = async () => {
   await $`kubectl apply -f charts/tekton-triggers/crds --server-side`
   d.info('Deploying essential manifests')
   await $`kubectl apply -f ${templateFile}`
+  let applyCompleted = false
   if (initialInstall) {
     d.info('Deploying charts containing label stage=prep')
     await hf(
@@ -116,13 +117,15 @@ const applyAll = async () => {
     // We still need to deploy all teams because some settings depend on platform apps.
     // Note that team-ns-admin contains ingress for platform apps.
     const params = cloneDeep(argv)
-    //TODO here happens the real installation of the apps
-    await applyAsApps(params)
+    applyCompleted = await applyAsApps(params)
   }
-
   if (!initialInstall) {
-    await upgrade({ when: 'post' })
-    await runtimeUpgrade({ when: 'post' })
+    if (applyCompleted) {
+      await upgrade({ when: 'post' })
+      await runtimeUpgrade({ when: 'post' })
+    } else {
+      d.info('Apply step not completed, skipping upgrade checks')
+    }
   }
   if (!(env.isDev && env.DISABLE_SYNC)) {
     await commit(initialInstall)

src/cmd/apply.ts

@@ -1,6 +1,6 @@
 import { randomUUID } from 'crypto'
-import { copy, pathExists } from 'fs-extra'
-import { copyFile, mkdir, readFile, writeFile } from 'fs/promises'
+import { existsSync } from 'fs'
+import { copyFile, cp, mkdir, readFile, writeFile } from 'fs/promises'
 import { generate as generatePassword } from 'generate-password'
 import { cloneDeep, get, isEmpty, merge, set } from 'lodash'
 import { pki } from 'node-forge'
@@ -39,7 +39,7 @@ export const bootstrapSops = async (
     encrypt,
     gucci,
     loadYaml,
-    pathExists,
+    pathExists: existsSync,
     getKmsSettings,
     readFile,
     terminal,
@@ -78,7 +78,7 @@ export const bootstrapSops = async (
     }
   }
 
-  const exists = await deps.pathExists(targetPath)
+  const exists = deps.pathExists(targetPath)
   d.log(`Creating sops file for provider ${provider}`)
   const output = (await deps.gucci(templatePath, obj, true)) as string
   await deps.writeFile(targetPath, output)
@@ -230,7 +230,7 @@ export const getUsers = (originalInput: any, deps = { generatePassword, addIniti
 }
 
 export const copyBasicFiles = async (
-  deps = { copy, copyFile, copySchema, mkdir, pathExists, terminal },
+  deps = { copy: cp, copyFile, copySchema, mkdir, pathExists: existsSync, terminal },
 ): Promise<void> => {
   const d = deps.terminal(`cmd:${cmdName}:copyBasicFiles`)
   const { ENV_DIR } = env
@@ -242,15 +242,15 @@ export const copyBasicFiles = async (
   ])
   d.info('Copied bin files')
   await deps.mkdir(`${ENV_DIR}/.vscode`, { recursive: true })
-  await deps.copy(`${rootDir}/.values/.vscode`, `${ENV_DIR}/.vscode`)
+  await deps.copy(`${rootDir}/.values/.vscode`, `${ENV_DIR}/.vscode`, { recursive: true })
   d.info('Copied vscode folder')
 
   await deps.copySchema()
 
   // only copy sample files if a real one is not found
   await Promise.allSettled(
     ['.secrets.sample']
-      .filter(async (val) => !(await deps.pathExists(`${ENV_DIR}/${val.replace(/\.sample$/g, '')}`)))
+      .filter((val) => !deps.pathExists(`${ENV_DIR}/${val.replace(/\.sample$/g, '')}`))
       .map(async (val) => deps.copyFile(`${rootDir}/.values/${val}`, `${ENV_DIR}/${val}`)),
   )
 
@@ -276,7 +276,7 @@ export const processValues = async (
     getStoredClusterSecrets,
     getKmsValues,
     writeValues,
-    pathExists,
+    pathExists: existsSync,
     hfValues,
     validateValues,
     generateSecrets,
@@ -301,7 +301,7 @@ export const processValues = async (
     d.log(`Loading repo values from ${ENV_DIR}`)
     // we can only read values from ENV_DIR if we can determine cluster.providers
     storedSecrets = {}
-    if (await deps.pathExists(`${ENV_DIR}/env/settings/cluster.yaml`)) {
+    if (deps.pathExists(`${ENV_DIR}/env/settings/cluster.yaml`)) {
       await deps.decrypt()
       originalInput = (await deps.hfValues({ defaultValues: true })) || {}
     }
@@ -429,7 +429,7 @@ export const createCustomCA = (deps = { terminal, pki, writeValues }): Record<st
 
 export const bootstrap = async (
   deps = {
-    pathExists,
+    pathExists: existsSync,
     getDeploymentState,
     getImageTag,
     getCurrentVersion,
@@ -458,7 +458,7 @@ export const bootstrap = async (
     if (prevVersion && prevTag && version === prevVersion && tag === prevTag) return
   }
   const { ENV_DIR } = env
-  const hasOtomi = await deps.pathExists(`${ENV_DIR}/bin/otomi`)
+  const hasOtomi = deps.pathExists(`${ENV_DIR}/bin/otomi`)
 
   const otomiImage = `linode/apl-core:${tag}`
   d.log(`Installing artifacts from ${otomiImage}`)

src/cmd/bootstrap.ts

@@ -1,8 +1,8 @@
 import { ApiException } from '@kubernetes/client-node'
 import { randomUUID } from 'crypto'
 import { diff } from 'deep-diff'
-import { copy, createFileSync, move, pathExists, renameSync, rm } from 'fs-extra'
-import { mkdir, readFile, writeFile } from 'fs/promises'
+import { existsSync, renameSync, rmSync, writeFileSync } from 'fs'
+import { cp, mkdir, readFile, rename as fsRename, writeFile } from 'fs/promises'
 import { glob, globSync } from 'glob'
 import { cloneDeep, each, get, isObject, isUndefined, mapKeys, mapValues, omit, pick, pull, set, unset } from 'lodash'
 import { basename, dirname, join } from 'path'
@@ -20,6 +20,7 @@ import { parse } from 'yaml'
 import { Argv } from 'yargs'
 import { $, cd } from 'zx'
 import { k8s } from '../common/k8s'
+import { ARGOCD_APP_PARAMS } from '../common/constants'
 
 const cmdName = getFilename(__filename)
 
@@ -62,27 +63,27 @@ export type Changes = Array<Change>
 export const deleteFile = async (
   relativeFilePath: string,
   dryRun = false,
-  deps = { pathExists, renameSync, terminal, copy, rm },
+  deps = { existsSync, renameSync, terminal, rmSync },
 ): Promise<void> => {
   const d = deps.terminal(`cmd:${cmdName}:rename`)
   const path = `${env.ENV_DIR}/${relativeFilePath}`
-  if (!(await deps.pathExists(path))) {
+  if (!deps.existsSync(path)) {
     d.warn(`File does not exist: "${path}". Already removed?`)
     return
   }
   if (!dryRun) {
-    await deps.rm(path)
+    deps.rmSync(path)
   }
 }
 
 export const rename = async (
   oldName: string,
   newName: string,
   dryRun = false,
-  deps = { pathExists, renameSync, terminal, move, copy, rm },
+  deps = { pathExists: existsSync, renameSync, terminal, move: fsRename, cp, rmSync },
 ): Promise<void> => {
   const d = deps.terminal(`cmd:${cmdName}:rename`)
-  if (!(await deps.pathExists(`${env.ENV_DIR}/${oldName}`))) {
+  if (!deps.pathExists(`${env.ENV_DIR}/${oldName}`)) {
     d.warn(`File does not exist: "${env.ENV_DIR}/${oldName}". Already renamed?`)
     return
   }
@@ -92,7 +93,7 @@ export const rename = async (
   if (oldName.includes('.yaml') && !oldName.includes('secrets.')) {
     const lastSlashPosOld = oldName.lastIndexOf('/') + 1
     const tmpOld = `${oldName.substring(0, lastSlashPosOld)}secrets.${oldName.substring(lastSlashPosOld)}`
-    if (await deps.pathExists(`${env.ENV_DIR}/${secretsCompanionOld}`)) {
+    if (deps.pathExists(`${env.ENV_DIR}/${secretsCompanionOld}`)) {
       secretsCompanionOld = tmpOld
       const lastSlashPosNew = oldName.lastIndexOf('/') + 1
       secretsCompanionNew = `${newName.substring(0, lastSlashPosNew)}secrets.${newName.substring(lastSlashPosNew)}`
@@ -105,16 +106,16 @@ export const rename = async (
       if (secretsCompanionOld) {
         // we also rename the secret companion
         await deps.move(`${env.ENV_DIR}/${secretsCompanionOld}`, `${env.ENV_DIR}/${secretsCompanionNew}`)
-        if (await deps.pathExists(`${env.ENV_DIR}/${secretsCompanionOld}.dec`))
+        if (deps.pathExists(`${env.ENV_DIR}/${secretsCompanionOld}.dec`))
           // and remove the old decrypted file
-          await deps.rm(`${env.ENV_DIR}/${secretsCompanionOld}.dec`)
+          deps.rmSync(`${env.ENV_DIR}/${secretsCompanionOld}.dec`)
       }
     } catch (e) {
       if (e.message === 'dest already exists.') {
         // we were given a folder that already exists, which is allowed,
         // so we defer to copying the contents and remove the source
-        await deps.copy(`${env.ENV_DIR}/${oldName}`, `${env.ENV_DIR}/${newName}`, { preserveTimestamps: true })
-        await deps.rm(`${env.ENV_DIR}/${oldName}`, { recursive: true, force: true })
+        await deps.cp(`${env.ENV_DIR}/${oldName}`, `${env.ENV_DIR}/${newName}`, { preserveTimestamps: true })
+        deps.rmSync(`${env.ENV_DIR}/${oldName}`, { recursive: true, force: true })
       }
     }
   }
@@ -274,7 +275,7 @@ const networkPoliciesMigration = async (values: Record<string, any>): Promise<vo
           servicePermissions.filter((s: any) => s !== 'networkPolicy'),
         )
 
-      createFileSync(`${env.ENV_DIR}/env/teams/netpols.${teamName}.yaml`)
+      writeFileSync(`${env.ENV_DIR}/env/teams/netpols.${teamName}.yaml`, '')
       let services = get(values, `teamConfig.${teamName}.services`)
       if (!services || services.length === 0) return
       const valuesToWrite = {
@@ -466,10 +467,7 @@ async function appExists(name: string): Promise<boolean> {
   return await checkExists(
     async () =>
       await k8s.custom().getNamespacedCustomObject({
-        group: 'argoproj.io',
-        version: 'v1alpha1',
-        namespace: 'argocd',
-        plural: 'applications',
+        ...ARGOCD_APP_PARAMS,
         name,
       }),
   )
@@ -650,7 +648,7 @@ export const applyChanges = async (
   for (const c of changes) {
     c.renamings?.forEach((entry) => each(entry, async (newName, oldName) => deps.rename(oldName, newName, dryRun)))
     // same for any new file additions
-    c.fileAdditions?.forEach((path) => createFileSync(`${env.ENV_DIR}/${path}`))
+    c.fileAdditions?.forEach((path) => writeFileSync(`${env.ENV_DIR}/${path}`, ''))
   }
   // only then can we get the values and do mutations on them
   const prevValues = (await deps.hfValues({ filesOnly: true })) as Record<string, any>
@@ -879,7 +877,7 @@ export const migrate = async (): Promise<boolean> => {
   const d = terminal(`cmd:${cmdName}:migrate`)
   const argv: Arguments = getParsedArgs()
   d.log('Migrating values')
-  if (await pathExists(`${env.ENV_DIR}/env/settings.yaml`)) {
+  if (existsSync(`${env.ENV_DIR}/env/settings.yaml`)) {
     d.log('Detected the old values file structure')
     await migrateLegacyValues(env.ENV_DIR)
   }