feat: ✨ manage project observability repository

Author: mathieulaude

README.md

@@ -4,3 +4,9 @@ Ceci est un plugin additionnel pour la console [Cloud-Pi-Native](https://github.
 Il permet :
 - d'alimenter un fichier de values qui pilote la création d'instance Grafana et de ses datasources (prometheus, alert-manager, loki)
 - de propager les permissions des utilisateurs via Keycloak
+
+## Usage
+
+Les variables d'environnement suivantes doivent être définies :
+- `GRAFANA_URL` : l'URL racine à utiliser pour accéder aux instances déployées pour chaque projet
+- `DSO_OBSERVABILITY_CHART_VERSION` : la version du Chart dso-observability à utiliser dans les dépôts de dashboards et alertes projet.

package.json

@@ -1,7 +1,7 @@
 {
   "name": "@cpn-console/observability-plugin",
   "type": "module",
-  "version": "1.1.0",
+  "version": "1.1.1",
   "description": "Observability plugin for DSO console",
   "exports": {
     ".": {

src/function.ts

@@ -1,9 +1,10 @@
+import type { GitlabProjectApi } from '@cpn-console/gitlab-plugin/types/class.js'
 import type { Environment, PluginResult, Project, StepCall, UserObject } from '@cpn-console/hooks'
 import type { KeycloakProjectApi } from '@cpn-console/keycloak-plugin/types/class.js'
-import { parseError, specificallyDisabled } from '@cpn-console/hooks'
+import { okStatus, parseError, specificallyDisabled } from '@cpn-console/hooks'
 import { compressUUID } from '@cpn-console/shared'
 import { deleteKeycloakGroup, ensureKeycloakGroups } from './keycloak.js'
-import { type EnvType, type ObservabilityProject, ObservabilityRepoManager } from './observability-repo-manager.js'
+import { type EnvType, type ObservabilityProject, ObservabilityRepoManager, observabilityRepository } from './observability-repo-manager.js'
 
 const okSkipped: PluginResult = {
   status: {
@@ -58,14 +59,32 @@ function getListPerms(environments: Environment[]): ListPerms {
   return listPerms
 }
 
+// Create and update (if needed) the project repository for custom dashboards and alerts
+export const ensureProjectRepository: StepCall<Project> = async (payload) => {
+  const gitlabProjectApi = payload.apis.gitlab as GitlabProjectApi
+  try {
+    await gitlabProjectApi.getProjectId(observabilityRepository)
+  } catch (e) {
+    console.log('Repository not fond', e)
+    await gitlabProjectApi.createEmptyProjectRepository({
+      repoName: observabilityRepository,
+      description: 'Respository for custom Observability infrastructure resources',
+      clone: false,
+    })
+  }
+  // Reference to avoid deletion
+  gitlabProjectApi.addSpecialRepositories(observabilityRepository)
+  return okStatus
+}
+
 export const upsertProject: StepCall<Project> = async (payload) => {
   try {
     if (specificallyDisabled(payload.config.observability?.enabled)) {
       return okSkipped
     }
-    // init args
     const project = payload.args
     const keycloakApi = payload.apis.keycloak as KeycloakProjectApi
+    const gitlabApi = payload.apis.gitlab as GitlabProjectApi
 
     const keycloakRootGroupPath = await keycloakApi.getProjectGroupPath()
     const tenantRbacProd = [`${keycloakRootGroupPath}/grafana/prod-RW`, `${keycloakRootGroupPath}/grafana/prod-RO`]
@@ -75,6 +94,10 @@ export const upsertProject: StepCall<Project> = async (payload) => {
 
     const projectValue: ObservabilityProject = {
       projectName: project.slug,
+      projectRepository: {
+        url: await gitlabApi.getRepoUrl(observabilityRepository),
+        path: '.',
+      },
       envs: {
         hprod: {
           groups: tenantRbacHProd,
@@ -110,7 +133,7 @@ export const upsertProject: StepCall<Project> = async (payload) => {
     const listPerms = getListPerms(project.environments)
 
     // Upsert or delete Gitlab config based on prod/non-prod environment
-    const observabilityRepoManager = new ObservabilityRepoManager()
+    const observabilityRepoManager = new ObservabilityRepoManager(gitlabApi)
     const yamlResult = await observabilityRepoManager.updateProjectConfig(project, projectValue)
 
     await ensureKeycloakGroups(listPerms, keycloakApi)
@@ -142,7 +165,7 @@ export const deleteProject: StepCall<Project> = async (payload) => {
     }
     const project = payload.args
     const keycloakApi = payload.apis.keycloak as KeycloakProjectApi
-    const observabilityRepoManager = new ObservabilityRepoManager()
+    const observabilityRepoManager = new ObservabilityRepoManager(payload.apis.gitlab)
 
     await Promise.all([
       deleteKeycloakGroup(keycloakApi),

src/index.ts

@@ -1,14 +1,15 @@
 import type { DeclareModuleGenerator, Plugin } from '@cpn-console/hooks'
 import { requiredEnv } from '@cpn-console/shared'
-import { deleteProject, upsertProject } from './function.js'
+import { deleteProject, ensureProjectRepository, upsertProject } from './function.js'
 import infos from './infos.js'
 
 export const plugin: Plugin = {
   infos,
   subscribedHooks: {
     upsertProject: {
       steps: {
-        post: upsertProject,
+        pre: ensureProjectRepository,
+        main: upsertProject,
       },
     },
     deleteProject: {
@@ -17,7 +18,7 @@ export const plugin: Plugin = {
       },
     },
   },
-  start: () => { requiredEnv('GRAFANA_URL') }, // to check is the variable is set, unless it crashes the app
+  start: () => { requiredEnv('GRAFANA_URL') && requiredEnv('DSO_OBSERVABILITY_CHART_VERSION') }, // to check is the variable is set, unless it crashes the app
 }
 
 declare module '@cpn-console/hooks' {

src/observability-repo-manager.ts

@@ -1,12 +1,31 @@
+import type { GitlabProjectApi } from '@cpn-console/gitlab-plugin/types/class.js'
 import type { Project } from '@cpn-console/hooks'
+import type { Gitlab as IGitlab, ProjectSchema } from '@gitbeaker/core'
 import { removeTrailingSlash, requiredEnv } from '@cpn-console/shared'
-import { Gitlab, type ProjectSchema } from '@gitbeaker/core'
+import { Gitlab } from '@gitbeaker/rest'
 import yaml from 'js-yaml'
 
 const valuesPath = 'helm/values.yaml'
 const valuesBranch = 'main'
 const groupName = 'observability'
 const repoName = 'observability'
+export const observabilityRepository = 'infra-observability'
+const observabilityChartVersion = requiredEnv('DSO_OBSERVABILITY_CHART_VERSION')
+const observabilityChartContent = `
+apiVersion: v2
+name: dso-observability
+type: application
+version: 0.1.0
+appVersion: "0.0.1"
+dependencies:
+  - name: charts/dso-observability
+    version: ${observabilityChartVersion}
+    repository: https://cloud-pi-native.github.io/helm-charts/
+`
+const observabilityTemplateContent = `
+{{- include "grafana-dashboards.dashboards" . -}}
+{{- include "grafana-dashboards.rules" . -}}
+`
 
 export type EnvType = 'prod' | 'hprod'
 interface Tenant {}
@@ -19,6 +38,10 @@ interface Env {
 }
 export interface ObservabilityProject {
   projectName: string // slug
+  projectRepository: {
+    url: string
+    path: string
+  }
   envs: {
     prod: Env
     hprod: Env
@@ -39,20 +62,22 @@ const yamlInitData = `
   `
 
 export class ObservabilityRepoManager {
-  private gitlabApi: Gitlab
+  private gitlabApi: IGitlab
+  private gitlabProjectApi: GitlabProjectApi
 
-  constructor() {
+  constructor(gitlabProjectApi: GitlabProjectApi) {
     const gitlabUrl = removeTrailingSlash(requiredEnv('GITLAB_URL'))
     const gitlabToken = requiredEnv('GITLAB_TOKEN')
     this.gitlabApi = new Gitlab({ token: gitlabToken, host: gitlabUrl })
+    this.gitlabProjectApi = gitlabProjectApi
   }
 
   private async findOrCreateRepo(): Promise<ProjectSchema> {
     try {
       // Find or create parent Gitlab group
       const groups = await this.gitlabApi.Groups.search(groupName)
       let group = groups.find(g => g.full_path === groupName || g.name === groupName)
-      if(!group) {
+      if (!group) {
         group = await this.gitlabApi.Groups.create(groupName, groupName)
       }
       // Find or create parent Gitlab repository
@@ -119,16 +144,33 @@ export class ObservabilityRepoManager {
   }
 
   public async updateProjectConfig(project: Project, projectValue: ObservabilityProject): Promise<string> {
-    // Déplacer toute la logique de création ou de récupération de groupe et de repo ici
+    // Repository created during 'pre' step if needed
+    const projectId = await this.gitlabProjectApi.getProjectId(observabilityRepository)
+    const observabilityProjectRepository = await this.gitlabProjectApi.getProjectById(projectId)
+
+    // Add or update chart files
+    const chartUpdated = await this.gitlabProjectApi.commitCreateOrUpdate(
+      observabilityProjectRepository.id,
+      observabilityChartContent,
+      'Chart.yaml',
+    )
+    const templateUpdated = await this.gitlabProjectApi.commitCreateOrUpdate(
+      observabilityProjectRepository.id,
+      observabilityTemplateContent,
+      'templates/includes.yaml',
+    )
+
+    // Dépôt d'infra scruté par ArgoCD (charts dso-grafana et dso-observatorium)
     const gitlabRepo = await this.findOrCreateRepo()
 
     // Récupérer le fichier values.yaml
-    const yamlFile = await this.getValuesFile(gitlabRepo) 
+    const yamlFile = await this.getValuesFile(gitlabRepo)
       || yaml.load(Buffer.from(yamlInitData, 'base64').toString('utf-8')) as ObservabilityData
 
     const projects = yamlFile.global?.projects || {}
 
-    if (JSON.stringify(projects[project.id]) === JSON.stringify(projectValue)) {
+    if (!chartUpdated && !templateUpdated
+      && JSON.stringify(projects[project.id]) === JSON.stringify(projectValue)) {
       return 'Already up-to-date'
     }