feat: ensure manifest folder in values repo

src/cmd/apply-as-apps.ts

@@ -5,9 +5,9 @@ import {
   setHeaderOptions,
   V1ResourceRequirements,
 } from '@kubernetes/client-node'
-import { existsSync, statSync, mkdirSync, rmSync } from 'fs'
-import { glob } from 'glob'
+import { existsSync, mkdirSync, rmSync, statSync } from 'fs'
 import { readFile } from 'fs/promises'
+import { glob } from 'glob'
 import { appPatches, genericPatch } from 'src/applicationPatches.json'
 import { cleanupHandler, prepareEnvironment } from 'src/common/cli'
 import { logLevelString, terminal } from 'src/common/debug'
@@ -16,12 +16,11 @@ import { appRevisionMatches, k8s, patchArgoCdApp, patchContainerResourcesOfSts }
 import { getFilename, loadYaml } from 'src/common/utils'
 import { getImageTagFromValues, objectToYaml } from 'src/common/values'
 import { getParsedArgs, HelmArguments, helmOptions, setParsedArgs } from 'src/common/yargs'
+import { operatorEnv } from 'src/operator/validators'
 import { Argv, CommandModule } from 'yargs'
 import { ARGOCD_APP_DEFAULT_SYNC_POLICY, ARGOCD_APP_PARAMS } from '../common/constants'
 import { env } from '../common/envalid'
 
-export const GITOPS_MANIFESTS_NS_PATH = 'env/manifests/namespaces'
-export const GITOPS_MANIFESTS_GLOBAL_PATH = 'env/manifests/global'
 export const ARGOCD_APP_DEFAULT_LABEL = 'managed'
 export const ARGOCD_APP_GITOPS_LABEL = 'generic-gitops'
 export const ARGOCD_APP_GITOPS_NS_PREFIX = 'gitops-ns'
@@ -153,7 +152,9 @@ export const getArgocdGitopsManifest = (name: string, targetNamespace?: string)
     syncPolicy.syncOptions.push('CreateNamespace=true')
   }
   const repoURL = `${env.GIT_PROTOCOL}://${env.GIT_URL}:${env.GIT_PORT}/otomi/values.git`
-  const path = targetNamespace ? `${GITOPS_MANIFESTS_NS_PATH}/${targetNamespace}` : GITOPS_MANIFESTS_GLOBAL_PATH
+  const path = targetNamespace
+    ? `${operatorEnv.GITOPS_NS_MANIFESTS_RELATIVE_PATH}/${targetNamespace}`
+    : operatorEnv.GITOPS_GLOBAL_MANIFESTS_RELATIVE_PATH
   return getArgoCdAppManifest(name, ARGOCD_APP_GITOPS_LABEL, {
     project: 'default',
     syncPolicy,
@@ -460,13 +461,17 @@ export const calculateGitOpsAppsDiff = async (
   deps = { getApplications },
 ): Promise<{ toAdd: Set<string>; toRemove: Set<string>; namespaceDirs: string[] }> => {
   const envDir = env.ENV_DIR
-  const namespaceListing = await glob(`${envDir}/${GITOPS_MANIFESTS_NS_PATH}/*`, { withFileTypes: true })
+  const namespaceListing = await glob(`${envDir}/${operatorEnv.GITOPS_NS_MANIFESTS_RELATIVE_PATH}/*`, {
+    withFileTypes: true,
+  })
   const namespaceDirs = namespaceListing.filter((path) => path.isDirectory()).map((path) => path.name)
   const existingGitOpsApps = new Set(await deps.getApplications(`otomi.io/app=${ARGOCD_APP_GITOPS_LABEL}`))
 
   // First create sets of Applications to be updated
   const requiredGitOpsApps = new Set(namespaceDirs.map((dirName) => `${ARGOCD_APP_GITOPS_NS_PREFIX}-${dirName}`))
-  const globalPath = statSync(`${envDir}/${GITOPS_MANIFESTS_GLOBAL_PATH}`, { throwIfNoEntry: false })
+  const globalPath = statSync(`${envDir}/${operatorEnv.GITOPS_GLOBAL_MANIFESTS_RELATIVE_PATH}`, {
+    throwIfNoEntry: false,
+  })
   if (globalPath && globalPath.isDirectory()) {
     requiredGitOpsApps.add(ARGOCD_APP_GITOPS_GLOBAL_NAME)
   }

src/cmd/bootstrap.test.ts

@@ -40,6 +40,7 @@ describe('Bootstrapping values', () => {
       copyBasicFiles: jest.fn(),
       copyFile: jest.fn(),
       createCustomCA: jest.fn(),
+      ensureManifestDirectories: jest.fn(),
       handleFileEntry: jest.fn(),
       decrypt: jest.fn(),
       encrypt: jest.fn(),

src/cmd/bootstrap.ts

@@ -21,7 +21,15 @@ import {
   secretId,
 } from 'src/common/k8s'
 import { getKmsSettings } from 'src/common/repo'
-import { ensureTeamGitOpsDirectories, getFilename, gucci, isCore, loadYaml, rootDir } from 'src/common/utils'
+import {
+  ensureManifestDirectories,
+  ensureTeamGitOpsDirectories,
+  getFilename,
+  gucci,
+  isCore,
+  loadYaml,
+  rootDir,
+} from 'src/common/utils'
 import { generateSecrets, writeValues } from 'src/common/values'
 import { BasicArguments, setParsedArgs } from 'src/common/yargs'
 import { Argv } from 'yargs'
@@ -438,6 +446,7 @@ export const bootstrap = async (
     encrypt,
     decrypt,
     handleFileEntry,
+    ensureManifestDirectories,
   },
 ): Promise<void> => {
   const d = deps.terminal(`cmd:${cmdName}:bootstrap`)
@@ -452,6 +461,7 @@ export const bootstrap = async (
   const originalValues = await deps.processValues()
   await deps.handleFileEntry()
   await deps.bootstrapSops()
+  await deps.ensureManifestDirectories()
   await ensureTeamGitOpsDirectories(ENV_DIR, originalValues)
   d.log(`Done bootstrapping values`)
 }

src/common/utils.ts

@@ -2,13 +2,14 @@ import $RefParser, { JSONSchema } from '@apidevtools/json-schema-ref-parser'
 import cleanDeep, { CleanOptions } from 'clean-deep'
 import { createHash } from 'crypto'
 import { existsSync, readFileSync } from 'fs'
-import { readdir, readFile, writeFile } from 'fs/promises'
+import { access, mkdir, readdir, readFile, writeFile } from 'fs/promises'
 import { glob } from 'glob'
 import walk from 'ignore-walk'
 import { dump, load } from 'js-yaml'
 import { omit } from 'lodash'
 import { dirname, join, resolve } from 'path'
 import { $, ProcessOutput, within } from 'zx'
+import { operatorEnv } from '../operator/validators'
 import { terminal } from './debug'
 import { env } from './envalid'
 
@@ -205,21 +206,23 @@ export const getSchemaSecretsPaths = async (teams: string[]): Promise<string[]>
   return cleanSecretPaths
 }
 
-async function ensureKeepFile(keepFilePath: string, deps = { writeFile }): Promise<void> {
-  const dirPath = dirname(keepFilePath)
-  if (!existsSync(dirPath)) {
-    await $`mkdir -p ${dirname(keepFilePath)}`
+export async function ensureManifestDirectories(): Promise<void> {
+  await ensureDirectoryWithGitkeepAsync(join(env.ENV_DIR, operatorEnv.GITOPS_NS_MANIFESTS_RELATIVE_PATH))
+  await ensureDirectoryWithGitkeepAsync(join(env.ENV_DIR, operatorEnv.GITOPS_GLOBAL_MANIFESTS_RELATIVE_PATH))
+}
+
+async function ensureDirectoryWithGitkeepAsync(dirPath: string, deps = { access, writeFile, mkdir }): Promise<void> {
+  await deps.mkdir(dirPath, { recursive: true })
+  const gitkeepPath = join(dirPath, '.gitkeep')
+
+  try {
+    await deps.access(gitkeepPath)
+  } catch {
+    await deps.writeFile(gitkeepPath, '')
   }
-  if (existsSync(keepFilePath)) return
-  // create the .keep file
-  await deps.writeFile(keepFilePath, '')
 }
 
-export async function ensureTeamGitOpsDirectories(
-  envDir: string,
-  values: Record<string, any>,
-  deps = { writeFile, glob },
-) {
+export async function ensureTeamGitOpsDirectories(envDir: string, values: Record<string, any>, deps = { glob }) {
   const dirs = await deps.glob(`${envDir}/env/teams/*`)
   const baseGitOpsDirs = ['sealedsecrets', 'workloadValues']
   const aiGitOpsDirs = ['databases', 'knowledgebases', 'agents']
@@ -236,10 +239,7 @@ export async function ensureTeamGitOpsDirectories(
 
   await Promise.allSettled(
     keepFilePaths.map(async (keepFilePath) => {
-      await ensureKeepFile(keepFilePath, deps)
-      if (!existsSync(dirname(keepFilePath))) {
-        await $`mkdir -p ${dirname(keepFilePath)}`
-      }
+      await ensureDirectoryWithGitkeepAsync(dirname(keepFilePath))
     }),
   )
   return keepFilePaths

src/operator/apl-operator.ts

@@ -3,8 +3,8 @@ import { commit } from '../cmd/commit'
 import { terminal } from '../common/debug'
 import { env } from '../common/envalid'
 import { GitRepoConfig } from '../common/git-config'
-import { hfValues } from '../common/hf'
 import { waitTillGitRepoAvailable } from '../common/gitea'
+import { hfValues } from '../common/hf'
 import { ensureTeamGitOpsDirectories } from '../common/utils'
 import { writeValues } from '../common/values'
 import { HelmArguments } from '../common/yargs'

src/operator/validators.ts

@@ -6,6 +6,14 @@ dotenv.config()
 export const operatorEnv = cleanEnv(process.env, {
   GIT_ORG: str({ desc: 'Git organisation', default: 'otomi' }),
   GIT_REPO: str({ desc: 'Git repository', default: 'values' }),
+  GITOPS_NS_MANIFESTS_RELATIVE_PATH: str({
+    desc: 'Path to the gitops manifests namespace',
+    default: 'env/manifests/namespaces',
+  }),
+  GITOPS_GLOBAL_MANIFESTS_RELATIVE_PATH: str({
+    desc: 'Path to the gitops manifests global',
+    default: 'env/manifests/global',
+  }),
   POLL_INTERVAL_MS: num({ desc: 'Interval in which the operator polls Git', default: 1000 }),
   RECONCILE_INTERVAL_MS: num({ desc: 'Interval in which the operator reconciles the cluster in', default: 300_000 }),
   INSTALL_RETRIES: num({ desc: 'Number of installation retry attempts', default: 1000 }),