diff --git a/apps/client/nginx/default.conf b/apps/client/nginx/default.conf index 133437fd6..8ff31b9a7 100644 --- a/apps/client/nginx/default.conf +++ b/apps/client/nginx/default.conf @@ -5,6 +5,8 @@ server { index index.html; large_client_header_buffers 4 32k; + proxy_read_timeout 3600; + proxy_send_timeout 3600; location / { try_files $uri $uri/ @rewrites; diff --git a/apps/client/nginx/default.docker.conf b/apps/client/nginx/default.docker.conf index 2b94c3bc4..91798df9e 100644 --- a/apps/client/nginx/default.docker.conf +++ b/apps/client/nginx/default.docker.conf @@ -27,9 +27,11 @@ server { proxy_redirect off; proxy_buffering off; proxy_http_version 1.1; + proxy_read_timeout 3600; + proxy_send_timeout 3600; # For WebSockets proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; } -} \ No newline at end of file +} diff --git a/apps/nginx-strangler/conf.d/routing.conf b/apps/nginx-strangler/conf.d/routing.conf index a5ffb7e09..62d2dd38f 100644 --- a/apps/nginx-strangler/conf.d/routing.conf +++ b/apps/nginx-strangler/conf.d/routing.conf @@ -61,6 +61,8 @@ server { proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; + proxy_read_timeout 3600; + proxy_send_timeout 3600; proxy_buffering off; proxy_redirect off; } @@ -72,5 +74,7 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 3600; + proxy_send_timeout 3600; } } diff --git a/apps/nginx-strangler/nginx.conf b/apps/nginx-strangler/nginx.conf index 657e2100d..b8c146801 100644 --- a/apps/nginx-strangler/nginx.conf +++ b/apps/nginx-strangler/nginx.conf @@ -19,7 +19,9 @@ http { access_log /var/log/nginx/access.log main; sendfile on; - keepalive_timeout 65; + keepalive_timeout 180; + proxy_read_timeout 180; + proxy_send_timeout 180; # Taille des headers (nécessaire pour les tokens Keycloak) large_client_header_buffers 4 32k; diff --git a/apps/server-nestjs/src/modules/healthz/healthz.module.ts b/apps/server-nestjs/src/modules/healthz/healthz.module.ts index 7e7307126..5dbd41a00 100644 --- a/apps/server-nestjs/src/modules/healthz/healthz.module.ts +++ b/apps/server-nestjs/src/modules/healthz/healthz.module.ts @@ -1,7 +1,7 @@ import { Module } from '@nestjs/common' import { TerminusModule } from '@nestjs/terminus' -import { KeycloakModule } from '../keycloak/keycloak.module' import { DatabaseModule } from '../../cpin-module/infrastructure/database/database.module' +import { KeycloakModule } from '../keycloak/keycloak.module' import { HealthzController } from './healthz.controller' @Module({ diff --git a/packages/hooks/src/hooks/hook.ts b/packages/hooks/src/hooks/hook.ts index 4a80a4618..7c019e7e8 100644 --- a/packages/hooks/src/hooks/hook.ts +++ b/packages/hooks/src/hooks/hook.ts @@ -77,20 +77,16 @@ export async function executeStep(step: HookStep, payl payload.results[name].executionTime[stepName] = Date.now() - payload.results[name].executionTime[stepName] return fnResult }) - const results = await Promise.allSettled(fns) + const results = await Promise.all(fns) names.forEach((name, index) => { - const settled = results[index] - const result = settled.status === 'fulfilled' - ? settled.value - : { status: { result: 'KO', message: String(settled.reason) } } satisfies PluginResult - if (result.status.result === 'KO') { + if (results[index].status.result === 'KO') { payload.failed = Array.isArray(payload.failed) ? [...payload.failed, name] : [name] - } else if (result.status.result === 'WARNING' && !payload.warning.includes(name)) { + } else if (results[index].status.result === 'WARNING' && !payload.warning.includes(name)) { payload.warning.push(name) } - payload.results[name] = { ...result, executionTime: payload.results[name].executionTime } + payload.results[name] = { ...results[index], executionTime: payload.results[name].executionTime } }) return payload } diff --git a/plugins/gitlab/src/class.ts b/plugins/gitlab/src/class.ts index fa0a4a3e4..0d02014bb 100644 --- a/plugins/gitlab/src/class.ts +++ b/plugins/gitlab/src/class.ts @@ -464,7 +464,7 @@ export class GitlabProjectApi extends GitlabApi { public async addGroupMember(userId: number, accessLevel: AccessLevelAllowed = AccessLevel.DEVELOPER): Promise { console.log(`[GITLAB] addGroupMember ${userId} ${accessLevel}`) const group = await this.getOrCreateProjectGroup() - return this.api.GroupMembers.add(group.id, userId, accessLevel) + return this.api.GroupMembers.add(group.id, userId, accessLevel as any) } public async editGroupMember(userId: number, accessLevel: AccessLevelAllowed = AccessLevel.DEVELOPER): Promise { diff --git a/plugins/gitlab/src/functions.ts b/plugins/gitlab/src/functions.ts index e7658b332..07c4fb2af 100644 --- a/plugins/gitlab/src/functions.ts +++ b/plugins/gitlab/src/functions.ts @@ -15,6 +15,7 @@ import { cleanGitlabError } from './utils.js' // Check export const checkApi: StepCall = async (payload) => { + console.log(`[GITLAB] checkApi`) try { const { users } = payload.args for (const user of users) { @@ -47,6 +48,7 @@ export const checkApi: StepCall = async (payload) => { } export const getDsoProjectSecrets: StepCall = async (payload) => { + console.log(`[GITLAB] getDsoProjectSecrets`) try { if (!specificallyDisabled(payload.config.gitlab?.displayTriggerHint)) { // TODO déplacer les secrets dans un dossier pour tout lister plutôt que de sélectionner dans le code @@ -93,6 +95,7 @@ export const getDsoProjectSecrets: StepCall = async (payload) => { } export const upsertDsoProject: StepCall = async (payload) => { + console.log(`[GITLAB] upsertDsoProject`) const returnResult: PluginResult = { status: { result: 'OK', @@ -136,6 +139,7 @@ export const upsertDsoProject: StepCall = async (payload) => { return returnResult } catch (error) { returnResult.error = parseError(cleanGitlabError(error)) + console.log(`[GITLAB:ERROR] upsertDsoProject: ${returnResult.error}}`) returnResult.status.result = 'KO' returnResult.status.message = 'Can\'t reconcile please inspect logs' return returnResult @@ -143,6 +147,7 @@ export const upsertDsoProject: StepCall = async (payload) => { } export const deleteDsoProject: StepCall = async (payload) => { + console.log(`[GITLAB] deleteDsoProject`) try { const group = await payload.apis.gitlab.getProjectGroup() if (group) await deleteGroup(group.id, group.full_path) @@ -165,6 +170,7 @@ export const deleteDsoProject: StepCall = async (payload) => { } export const syncRepository: StepCall = async (payload) => { + console.log(`[GITLAB] syncRepository`) const targetRepo = payload.args.repo const gitlabApi = payload.apis.gitlab try { @@ -187,6 +193,7 @@ export const syncRepository: StepCall = async (payload) => { } export const upsertZone: StepCall = async (payload) => { + console.log(`[GITLAB] upsertZone`) const returnResult: PluginResult = okStatus try { const gitlabApi = payload.apis.gitlab @@ -194,6 +201,7 @@ export const upsertZone: StepCall = async (payload) => { return returnResult } catch (error) { returnResult.error = parseError(cleanGitlabError(error)) + console.log(`[GITLAB:ERROR] upsertZone: ${returnResult.error}}`) returnResult.status.result = 'KO' returnResult.status.message = 'Can\'t reconcile please inspect logs' return returnResult @@ -201,6 +209,7 @@ export const upsertZone: StepCall = async (payload) => { } export const deleteZone: StepCall = async (payload) => { + console.log(`[GITLAB] deleteZone`) const returnResult: PluginResult = { status: { result: 'OK', @@ -214,6 +223,7 @@ export const deleteZone: StepCall = async (payload) => { return returnResult } catch (error) { returnResult.error = parseError(cleanGitlabError(error)) + console.log(`[GITLAB:ERROR] deleteZone: ${returnResult.error}}`) returnResult.status.result = 'KO' returnResult.status.message = 'Can\'t reconcile please inspect logs' return returnResult @@ -221,6 +231,7 @@ export const deleteZone: StepCall = async (payload) => { } export const commitFiles: StepCall = async (payload) => { + console.log(`[GITLAB] commitFiles`) const returnResult = payload.results.gitlab try { const filesUpdated = await payload.apis.gitlab.commitFiles() @@ -236,6 +247,7 @@ export const commitFiles: StepCall = async (payload) => { + console.log(`[GITLAB] upsertAdminRole`) try { const role = payload.args const adminGroupPath = payload.config.gitlab?.adminGroupPath ?? DEFAULT_ADMIN_GROUP_PATH @@ -275,6 +287,7 @@ export const upsertAdminRole: StepCall = async (payload) => { } export const deleteAdminRole: StepCall = async (payload) => { + console.log(`[GITLAB] deleteAdminRole`) try { const role = payload.args const adminGroupPath = payload.config.gitlab?.adminGroupPath ?? DEFAULT_ADMIN_GROUP_PATH @@ -314,6 +327,7 @@ export const deleteAdminRole: StepCall = async (payload) => { } export const upsertProjectMember: StepCall = async (payload) => { + console.log(`[GITLAB] upsertProjectMember`) const member = payload.args const { gitlab: gitlabApi } = payload.apis as { gitlab: GitlabProjectApi } // TODO: apis is never type for some resaon @@ -340,6 +354,7 @@ export const upsertProjectMember: StepCall = async (payload) => { } export const deleteProjectMember: StepCall = async (payload) => { + console.log(`[GITLAB] deleteProjectMember`) const member = payload.args const { gitlab: gitlabApi } = payload.apis as { gitlab: GitlabProjectApi } // TODO: apis is never type for some resaon diff --git a/plugins/gitlab/src/repositories.ts b/plugins/gitlab/src/repositories.ts index 00797cf44..ea116fccd 100644 --- a/plugins/gitlab/src/repositories.ts +++ b/plugins/gitlab/src/repositories.ts @@ -13,6 +13,7 @@ interface ProjectMirrorCreds { } export async function ensureRepositories(gitlabApi: GitlabProjectApi, project: Project, vaultApi: VaultProjectApi, projectMirrorCreds: ProjectMirrorCreds) { + console.log(`[GITLAB] ensureRepositories`) const specialRepos = await gitlabApi.getSpecialRepositories() const gitlabRepositories = await gitlabApi.listRepositories() @@ -46,6 +47,7 @@ export async function ensureRepositories(gitlabApi: GitlabProjectApi, project: P const urnRegexp = /:\/\/(.*)/s async function ensureRepositoryExists(gitlabRepositories: CondensedProjectSchema[], repository: Repository, gitlabApi: GitlabProjectApi, projectMirrorCreds: ProjectMirrorCreds, vaultApi: VaultProjectApi) { + console.log(`[GITLAB] ensureRepositoryExists`) const gitlabRepository: CondensedProjectSchema | ProjectSchema | void = gitlabRepositories.find(gitlabRepository => gitlabRepository.name === repository.internalRepoName) const externalRepoUrn = repository.externalRepoUrl.split(urnRegexp)[1] const vaultCredsPath = `${repository.internalRepoName}-mirror` diff --git a/plugins/gitlab/src/utils.ts b/plugins/gitlab/src/utils.ts index f0fca23a0..0a1b3ae2b 100644 --- a/plugins/gitlab/src/utils.ts +++ b/plugins/gitlab/src/utils.ts @@ -1,4 +1,9 @@ -import type { BaseRequestOptions, Gitlab as IGitlab, OffsetPagination, PaginationRequestOptions } from '@gitbeaker/core' +import type { + BaseRequestOptions, + Gitlab as IGitlab, + OffsetPagination, + PaginationRequestOptions, +} from '@gitbeaker/core' import { GitbeakerRequestError } from '@gitbeaker/requester-utils' import { Gitlab } from '@gitbeaker/rest' import config from './config.js' @@ -8,23 +13,34 @@ let api: IGitlab | undefined let groupRootId: number export async function getGroupRootId(throwIfNotFound?: true): Promise -export async function getGroupRootId(throwIfNotFound?: false): Promise -export async function getGroupRootId(throwIfNotFound?: boolean): Promise { +export async function getGroupRootId( + throwIfNotFound?: false, +): Promise +export async function getGroupRootId( + throwIfNotFound?: boolean, +): Promise { console.log(`[GITLAB] getGroupRootId`) const gitlabApi = getApi() const projectRootDir = config().projectsRootDir console.log(`[GITLAB] projectRootDir is ${projectRootDir}`) if (groupRootId) return groupRootId - const groupRoot = await find(offsetPaginate(opts => gitlabApi.Groups.all({ - search: projectRootDir, - orderBy: 'id', - ...opts, - })), grp => grp.full_path === projectRootDir) + const groupRoot = await find( + offsetPaginate(opts => + gitlabApi.Groups.all({ + search: projectRootDir, + orderBy: 'id', + ...opts, + }), + ), + grp => grp.full_path === projectRootDir, + ) console.log(`[GITLAB] groupRoot is ${JSON.stringify(groupRoot)}`) const searchId = groupRoot?.id if (typeof searchId === 'undefined') { if (throwIfNotFound) { - throw new Error(`Gitlab inaccessible, impossible de trouver le groupe RACINE ${projectRootDir}`) + throw new Error( + `Gitlab inaccessible, impossible de trouver le groupe RACINE ${projectRootDir}`, + ) } return searchId } @@ -43,11 +59,17 @@ async function createGroupRoot(): Promise { throw new Error('No projectRootDir available') } - let parentGroup = await find(offsetPaginate(opts => gitlabApi.Groups.all({ - search: rootGroupPath, - orderBy: 'id', - ...opts, - })), grp => grp.full_path === rootGroupPath) ?? await gitlabApi.Groups.create(rootGroupPath, rootGroupPath) + let parentGroup + = (await find( + offsetPaginate(opts => + gitlabApi.Groups.all({ + search: rootGroupPath, + orderBy: 'id', + ...opts, + }), + ), + grp => grp.full_path === rootGroupPath, + )) ?? (await gitlabApi.Groups.create(rootGroupPath, rootGroupPath)) if (parentGroup.full_path === projectRootDir) { return parentGroup.id @@ -55,11 +77,21 @@ async function createGroupRoot(): Promise { for (const path of projectRootDirArray) { const futureFullPath = `${parentGroup.full_path}/${path}` - parentGroup = await find(offsetPaginate(opts => gitlabApi.Groups.all({ - search: futureFullPath, - orderBy: 'id', - ...opts, - })), grp => grp.full_path === futureFullPath) ?? await gitlabApi.Groups.create(path, path, { parentId: parentGroup.id, visibility: 'internal' }) + parentGroup + = (await find( + offsetPaginate(opts => + gitlabApi.Groups.all({ + search: futureFullPath, + orderBy: 'id', + ...opts, + }), + ), + grp => grp.full_path === futureFullPath, + )) + ?? (await gitlabApi.Groups.create(path, path, { + parentId: parentGroup.id, + visibility: 'internal', + })) if (parentGroup.full_path === projectRootDir) { return parentGroup.id @@ -70,12 +102,18 @@ async function createGroupRoot(): Promise { export async function getOrCreateGroupRoot(): Promise { console.log(`[GITLAB] getOrCreateGroupRoot`) - return await getGroupRootId(false) ?? createGroupRoot() + return (await getGroupRootId(false)) ?? createGroupRoot() } export function getApi(): IGitlab { console.log(`[GITLAB] getApi`) - api ??= new Gitlab({ token: config().token, host: config().internalUrl }) + api ??= new Gitlab({ + token: config().token, + host: config().internalUrl, + rateLimits: { + '**': 1000, + }, + }) return api } @@ -95,21 +133,36 @@ const keyValueRegExp = /\/\/(.*):(.*)@/g export function cleanGitlabError(error: T): T { if (error instanceof GitbeakerRequestError && error.cause?.description) { - error.cause.description = String(error.cause.description).replaceAll(keyValueRegExp, '//MASKED:MASKED@') + error.cause.description = String(error.cause.description).replaceAll( + keyValueRegExp, + '//MASKED:MASKED@', + ) } return error } -export function matchRole(projectSlug: string, roleOidcGroup: string, configuredRolePath: string[]) { - return configuredRolePath.some(path => roleOidcGroup === `/${projectSlug}${path}`) +export function matchRole( + projectSlug: string, + roleOidcGroup: string, + configuredRolePath: string[], +) { + return configuredRolePath.some( + path => roleOidcGroup === `/${projectSlug}${path}`, + ) } export async function* offsetPaginate( - request: (options: PaginationRequestOptions<'offset'> & BaseRequestOptions) => Promise<{ data: T[], paginationInfo: OffsetPagination }>, + request: ( + options: PaginationRequestOptions<'offset'> & BaseRequestOptions, + ) => Promise<{ data: T[], paginationInfo: OffsetPagination }>, ): AsyncGenerator { let page: number | null = 1 while (page !== null) { - const { data, paginationInfo } = await request({ page, showExpanded: true, pagination: 'offset' }) + const { data, paginationInfo } = await request({ + page, + showExpanded: true, + pagination: 'offset', + }) for (const item of data) { yield item } @@ -117,9 +170,7 @@ export async function* offsetPaginate( } } -export async function getAll( - iterable: AsyncIterable, -): Promise { +export async function getAll(iterable: AsyncIterable): Promise { const items: T[] = [] for await (const item of iterable) { items.push(item)