feat: ensure manifest folder in values repo

src/cmd/apply-as-apps.ts

@@ -5,9 +5,9 @@ import {
   setHeaderOptions,
   V1ResourceRequirements,
 } from '@kubernetes/client-node'
-import { existsSync, statSync, mkdirSync, rmSync } from 'fs'
-import { glob } from 'glob'
+import { existsSync, mkdirSync, rmSync, statSync } from 'fs'
 import { readFile } from 'fs/promises'
+import { glob } from 'glob'
 import { appPatches, genericPatch } from 'src/applicationPatches.json'
 import { cleanupHandler, prepareEnvironment } from 'src/common/cli'
 import { logLevelString, terminal } from 'src/common/debug'
@@ -16,12 +16,11 @@ import { appRevisionMatches, k8s, patchArgoCdApp, patchContainerResourcesOfSts }
 import { getFilename, loadYaml } from 'src/common/utils'
 import { getImageTagFromValues, objectToYaml } from 'src/common/values'
 import { getParsedArgs, HelmArguments, helmOptions, setParsedArgs } from 'src/common/yargs'
+import { operatorEnv } from 'src/operator/validators'
 import { Argv, CommandModule } from 'yargs'
 import { ARGOCD_APP_DEFAULT_SYNC_POLICY, ARGOCD_APP_PARAMS } from '../common/constants'
 import { env } from '../common/envalid'
 
-export const GITOPS_MANIFESTS_NS_PATH = 'env/manifests/namespaces'
-export const GITOPS_MANIFESTS_GLOBAL_PATH = 'env/manifests/global'
 export const ARGOCD_APP_DEFAULT_LABEL = 'managed'
 export const ARGOCD_APP_GITOPS_LABEL = 'generic-gitops'
 export const ARGOCD_APP_GITOPS_NS_PREFIX = 'gitops-ns'
@@ -153,7 +152,9 @@ export const getArgocdGitopsManifest = (name: string, targetNamespace?: string)
     syncPolicy.syncOptions.push('CreateNamespace=true')
   }
   const repoURL = `${env.GIT_PROTOCOL}://${env.GIT_URL}:${env.GIT_PORT}/otomi/values.git`
-  const path = targetNamespace ? `${GITOPS_MANIFESTS_NS_PATH}/${targetNamespace}` : GITOPS_MANIFESTS_GLOBAL_PATH
+  const path = targetNamespace
+    ? `${operatorEnv.GITOPS_MANIFESTS_NS_PATH}/${targetNamespace}`
+    : operatorEnv.GITOPS_MANIFESTS_GLOBAL_PATH
   return getArgoCdAppManifest(name, ARGOCD_APP_GITOPS_LABEL, {
     project: 'default',
     syncPolicy,
@@ -460,13 +461,13 @@ export const calculateGitOpsAppsDiff = async (
   deps = { getApplications },
 ): Promise<{ toAdd: Set<string>; toRemove: Set<string>; namespaceDirs: string[] }> => {
   const envDir = env.ENV_DIR
-  const namespaceListing = await glob(`${envDir}/${GITOPS_MANIFESTS_NS_PATH}/*`, { withFileTypes: true })
+  const namespaceListing = await glob(`${envDir}/${operatorEnv.GITOPS_MANIFESTS_NS_PATH}/*`, { withFileTypes: true })
   const namespaceDirs = namespaceListing.filter((path) => path.isDirectory()).map((path) => path.name)
   const existingGitOpsApps = new Set(await deps.getApplications(`otomi.io/app=${ARGOCD_APP_GITOPS_LABEL}`))
 
   // First create sets of Applications to be updated
   const requiredGitOpsApps = new Set(namespaceDirs.map((dirName) => `${ARGOCD_APP_GITOPS_NS_PREFIX}-${dirName}`))
-  const globalPath = statSync(`${envDir}/${GITOPS_MANIFESTS_GLOBAL_PATH}`, { throwIfNoEntry: false })
+  const globalPath = statSync(`${envDir}/${operatorEnv.GITOPS_MANIFESTS_GLOBAL_PATH}`, { throwIfNoEntry: false })
   if (globalPath && globalPath.isDirectory()) {
     requiredGitOpsApps.add(ARGOCD_APP_GITOPS_GLOBAL_NAME)
   }

src/cmd/bootstrap.test.ts

@@ -40,6 +40,7 @@ describe('Bootstrapping values', () => {
       copyBasicFiles: jest.fn(),
       copyFile: jest.fn(),
       createCustomCA: jest.fn(),
+      ensureManifestDirectories: jest.fn(),
       handleFileEntry: jest.fn(),
       decrypt: jest.fn(),
       encrypt: jest.fn(),

src/cmd/bootstrap.ts

@@ -21,7 +21,15 @@ import {
   secretId,
 } from 'src/common/k8s'
 import { getKmsSettings } from 'src/common/repo'
-import { ensureTeamGitOpsDirectories, getFilename, gucci, isCore, loadYaml, rootDir } from 'src/common/utils'
+import {
+  ensureManifestDirectories,
+  ensureTeamGitOpsDirectories,
+  getFilename,
+  gucci,
+  isCore,
+  loadYaml,
+  rootDir,
+} from 'src/common/utils'
 import { generateSecrets, writeValues } from 'src/common/values'
 import { BasicArguments, setParsedArgs } from 'src/common/yargs'
 import { Argv } from 'yargs'
@@ -438,6 +446,7 @@ export const bootstrap = async (
     encrypt,
     decrypt,
     handleFileEntry,
+    ensureManifestDirectories,
   },
 ): Promise<void> => {
   const d = deps.terminal(`cmd:${cmdName}:bootstrap`)
@@ -452,6 +461,7 @@ export const bootstrap = async (
   const originalValues = await deps.processValues()
   await deps.handleFileEntry()
   await deps.bootstrapSops()
+  deps.ensureManifestDirectories()
   await ensureTeamGitOpsDirectories(ENV_DIR, originalValues)
   d.log(`Done bootstrapping values`)
 }

src/common/utils.ts

@@ -1,14 +1,16 @@
 import $RefParser, { JSONSchema } from '@apidevtools/json-schema-ref-parser'
 import cleanDeep, { CleanOptions } from 'clean-deep'
 import { createHash } from 'crypto'
-import { existsSync, readFileSync } from 'fs'
+import fs, { existsSync, readFileSync } from 'fs'
 import { readdir, readFile, writeFile } from 'fs/promises'
 import { glob } from 'glob'
 import walk from 'ignore-walk'
 import { dump, load } from 'js-yaml'
 import { omit } from 'lodash'
+import * as pathModule from 'path'
 import { dirname, join, resolve } from 'path'
 import { $, ProcessOutput, within } from 'zx'
+import { operatorEnv } from '../operator/validators'
 import { terminal } from './debug'
 import { env } from './envalid'
 
@@ -215,10 +217,27 @@ async function ensureKeepFile(keepFilePath: string, deps = { writeFile }): Promi
   await deps.writeFile(keepFilePath, '')
 }
 
+export function ensureManifestDirectories(): void {
+  ;[operatorEnv.GITOPS_MANIFESTS_NS_PATH, operatorEnv.GITOPS_MANIFESTS_GLOBAL_PATH].forEach((p) =>
+    ensureDirectoryWithGitkeepAsync(pathModule.join(env.ENV_DIR, p)),
+  )
+}
+
+async function ensureDirectoryWithGitkeepAsync(dirPath: string, deps = { fs, path: pathModule }) {
+  await deps.fs.promises.mkdir(dirPath, { recursive: true })
+  const gitkeepPath = deps.path.join(dirPath, '.gitkeep')
+
+  try {
+    await deps.fs.promises.access(gitkeepPath)
+  } catch {
+    await deps.fs.promises.writeFile(gitkeepPath, '')
+  }
+}
+
 export async function ensureTeamGitOpsDirectories(
   envDir: string,
   values: Record<string, any>,
-  deps = { writeFile, glob },
+  deps = { fs, path: pathModule, glob },
 ) {
   const dirs = await deps.glob(`${envDir}/env/teams/*`)
   const baseGitOpsDirs = ['sealedsecrets', 'workloadValues']
@@ -236,10 +255,7 @@ export async function ensureTeamGitOpsDirectories(
 
   await Promise.allSettled(
     keepFilePaths.map(async (keepFilePath) => {
-      await ensureKeepFile(keepFilePath, deps)
-      if (!existsSync(dirname(keepFilePath))) {
-        await $`mkdir -p ${dirname(keepFilePath)}`
-      }
+      await ensureDirectoryWithGitkeepAsync(dirname(keepFilePath), deps)
     }),
   )
   return keepFilePaths

src/operator/apl-operator.test.ts

@@ -35,6 +35,12 @@ jest.mock('../common/debug', () => ({
   })),
 }))
 
+jest.mock('fs', () => ({
+  existsSync: jest.fn(),
+  mkdirSync: jest.fn(),
+  writeFileSync: jest.fn(),
+}))
+
 jest.mock('../common/gitea', () => ({
   waitTillGitRepoAvailable: jest.fn().mockResolvedValue(undefined),
 }))
@@ -70,9 +76,11 @@ jest.mock('./apl-operations', () => ({
 describe('AplOperator', () => {
   let aplOperator: AplOperator
   let defaultConfig: AplOperatorConfig
+  const fs = require('fs')
 
   beforeEach(() => {
     jest.clearAllMocks()
+    fs.existsSync.mockReturnValue(false)
 
     defaultConfig = {
       gitConfig: {} as GitRepoConfig,

src/operator/apl-operator.ts

@@ -3,8 +3,8 @@ import { commit } from '../cmd/commit'
 import { terminal } from '../common/debug'
 import { env } from '../common/envalid'
 import { GitRepoConfig } from '../common/git-config'
-import { hfValues } from '../common/hf'
 import { waitTillGitRepoAvailable } from '../common/gitea'
+import { hfValues } from '../common/hf'
 import { ensureTeamGitOpsDirectories } from '../common/utils'
 import { writeValues } from '../common/values'
 import { HelmArguments } from '../common/yargs'

src/operator/validators.ts

@@ -6,6 +6,14 @@ dotenv.config()
 export const operatorEnv = cleanEnv(process.env, {
   GIT_ORG: str({ desc: 'Git organisation', default: 'otomi' }),
   GIT_REPO: str({ desc: 'Git repository', default: 'values' }),
+  GITOPS_MANIFESTS_NS_PATH: str({
+    desc: 'Path to the gitops manifests namespace',
+    default: 'env/manifests/namespaces',
+  }),
+  GITOPS_MANIFESTS_GLOBAL_PATH: str({
+    desc: 'Path to the gitops manifests global',
+    default: 'env/manifests/global',
+  }),
   POLL_INTERVAL_MS: num({ desc: 'Interval in which the operator polls Git', default: 1000 }),
   RECONCILE_INTERVAL_MS: num({ desc: 'Interval in which the operator reconciles the cluster in', default: 300_000 }),
   INSTALL_RETRIES: num({ desc: 'Number of installation retry attempts', default: 1000 }),