diff --git a/src/api/v1/repoBranches.ts b/src/api/v1/repoBranches.ts new file mode 100644 index 000000000..e4d5a597b --- /dev/null +++ b/src/api/v1/repoBranches.ts @@ -0,0 +1,19 @@ +import Debug from 'debug' +import { Operation, OperationHandlerArray } from 'express-openapi' +import { OpenApiRequestExt } from 'src/otomi-models' + +const debug = Debug('otomi:api:repoBranches') + +export default function (): OperationHandlerArray { + const get: Operation = [ + async ({ otomi, query }: OpenApiRequestExt, res): Promise => { + debug(`getRepoBranches`, query) + const { codeRepoName, teamId }: { codeRepoName: string; teamId: string } = query as any + res.json(await otomi.getRepoBranches(codeRepoName, teamId)) + }, + ] + const api = { + get, + } + return api +} diff --git a/src/openapi/api.yaml b/src/openapi/api.yaml index c407e0ca5..d8e3697dc 100644 --- a/src/openapi/api.yaml +++ b/src/openapi/api.yaml @@ -2247,10 +2247,36 @@ paths: schema: $ref: '#/components/schemas/SettingsInfo' + /v1/repoBranches: + get: + operationId: getRepoBranches + description: Get repo branches from the git providers. + x-aclSchema: RepoBranches + parameters: + - name: codeRepoName + in: query + description: Name of the code repository + schema: + type: string + - name: teamId + in: query + description: Id of the team + schema: + type: string + responses: + '200': + description: The request is successful. + content: + application/json: + schema: + type: array + items: + type: string + /v1/testRepoConnect: get: operationId: getTestRepoConnect - description: Get settings information from the 'settings.yaml' file. + description: Get test repo connect information from the git providers. x-aclSchema: TestRepoConnect parameters: - name: url @@ -2912,6 +2938,8 @@ components: $ref: 'settings.yaml#/Settings' SettingsInfo: $ref: 'settingsinfo.yaml#/SettingsInfo' + RepoBranches: + $ref: 'repobranches.yaml#/RepoBranches' TestRepoConnect: $ref: 'testrepoconnect.yaml#/TestRepoConnect' InternalRepoUrls: diff --git a/src/openapi/build.yaml b/src/openapi/build.yaml index 11bcc542a..4af18fa96 100644 --- a/src/openapi/build.yaml +++ b/src/openapi/build.yaml @@ -26,6 +26,9 @@ Build: title: Name $ref: 'definitions.yaml#/idName' description: Results in image name harbor.//name. + imageName: + title: Image name + $ref: 'definitions.yaml#/idName' tag: $ref: 'definitions.yaml#/imageTag' default: latest @@ -53,6 +56,9 @@ Build: AplBuildSpec: type: object properties: + imageName: + title: Image name + $ref: 'definitions.yaml#/idName' tag: $ref: 'definitions.yaml#/imageTag' default: latest diff --git a/src/openapi/repobranches.yaml b/src/openapi/repobranches.yaml new file mode 100644 index 000000000..a39889a17 --- /dev/null +++ b/src/openapi/repobranches.yaml @@ -0,0 +1,7 @@ +RepoBranches: + x-acl: + platformAdmin: [read-any] + teamAdmin: [read-any] + teamMember: [read] + properties: {} + type: object diff --git a/src/otomi-models.ts b/src/otomi-models.ts index 140e1d5f1..0d614af66 100644 --- a/src/otomi-models.ts +++ b/src/otomi-models.ts @@ -30,6 +30,7 @@ export type Session = components['schemas']['Session'] export type ObjWizard = components['schemas']['ObjWizard'] export type Settings = components['schemas']['Settings'] export type SettingsInfo = components['schemas']['SettingsInfo'] +export type RepoBranches = components['schemas']['RepoBranches'] export type TestRepoConnect = components['schemas']['TestRepoConnect'] export type InternalRepoUrls = components['schemas']['InternalRepoUrls'] export type Team = components['schemas']['Team'] diff --git a/src/otomi-stack.ts b/src/otomi-stack.ts index 3b8c5e3d2..d44799cd2 100644 --- a/src/otomi-stack.ts +++ b/src/otomi-stack.ts @@ -105,15 +105,17 @@ import { TeamConfigService } from './services/TeamConfigService' import { validateBackupFields } from './utils/backupUtils' import { getGiteaRepoUrls, + getPrivateRepoBranches, + getPublicRepoBranches, normalizeRepoUrl, testPrivateRepoConnect, testPublicRepoConnect, } from './utils/codeRepoUtils' +import { getAplObjectFromV1, getV1MergeObject, getV1ObjectFromApl, getV1SealedSecretFromApl } from './utils/manifests' import { getEncryptedData, sealedSecretManifest, SealedSecretManifestType } from './utils/sealedSecretUtils' import { getKeycloakUsers, isValidUsername } from './utils/userUtils' import { ObjectStorageClient } from './utils/wizardUtils' import { fetchChartYaml, fetchWorkloadCatalog, NewHelmChartValues, sparseCloneChart } from './utils/workloadUtils' -import { getAplObjectFromV1, getV1MergeObject, getV1ObjectFromApl, getV1SealedSecretFromApl } from './utils/manifests' interface ExcludedApp extends App { managed: boolean @@ -1275,8 +1277,11 @@ export default class OtomiStack { } async createAplCodeRepo(teamId: string, data: AplCodeRepoRequest): Promise { + const allRepoUrls = this.getAllAplCodeRepos().map((repo) => repo.spec.repositoryUrl) || [] + if (allRepoUrls.includes(data.spec.repositoryUrl)) throw new AlreadyExists('Code repository URL already exists') + if (!data.spec.private) unset(data.spec, 'secret') + if (data.spec.gitService === 'gitea') unset(data.spec, 'private') try { - if (!data.spec.private) unset(data.spec, 'secret') const codeRepo = this.repoService.getTeamConfigService(teamId).createCodeRepo(data) await this.saveTeamConfigItem(codeRepo) await this.doTeamDeployment( @@ -1288,7 +1293,9 @@ export default class OtomiStack { ) return codeRepo } catch (err) { - if (err.code === 409) err.publicMessage = 'Code repo label already exists' + if (err.code === 409) { + err.publicMessage = 'Code repo name already exists' + } throw err } } @@ -1314,6 +1321,7 @@ export default class OtomiStack { patch = false, ): Promise { if (!data.spec?.private) unset(data.spec, 'secret') + if (data.spec?.gitService === 'gitea') unset(data.spec, 'private') const codeRepo = patch ? this.repoService.getTeamConfigService(teamId).patchCodeRepo(name, data) : this.repoService.getTeamConfigService(teamId).updateCodeRepo(name, data as AplCodeRepoRequest) @@ -1341,7 +1349,10 @@ export default class OtomiStack { ) } - async getTestRepoConnect(url: string, teamId: string, secretName: string): Promise { + async getRepoBranches(codeRepoName: string, teamId: string): Promise { + if (!codeRepoName) return ['HEAD'] + const coderepo = this.getCodeRepo(teamId, codeRepoName) + const { repositoryUrl, secret: secretName } = coderepo try { let sshPrivateKey = '', username = '', @@ -1355,6 +1366,38 @@ export default class OtomiStack { } const isPrivate = !!secretName + const isSSH = !!sshPrivateKey + const repoUrl = repositoryUrl.startsWith('https://gitea') + ? repositoryUrl + : normalizeRepoUrl(repositoryUrl, isPrivate, isSSH) + + if (!repoUrl) return ['HEAD'] + + if (isPrivate) return await getPrivateRepoBranches(repoUrl, sshPrivateKey, username, accessToken) + + return await getPublicRepoBranches(repoUrl) + } catch (error) { + const errorMessage = error.response?.data?.message || error?.message || 'Failed to get repo branches' + debug('Error getting branches:', errorMessage) + return [] + } + } + + async getTestRepoConnect(url: string, teamId: string, secretName: string): Promise { + try { + let sshPrivateKey = '', + username = '', + accessToken = '' + + const isPrivate = !!secretName + + if (isPrivate) { + const secret = await getSecretValues(secretName, `team-${teamId}`) + sshPrivateKey = secret?.['ssh-privatekey'] || '' + username = secret?.username || '' + accessToken = secret?.password || '' + } + const isSSH = !!sshPrivateKey const repoUrl = normalizeRepoUrl(url, isPrivate, isSSH) @@ -1422,6 +1465,12 @@ export default class OtomiStack { } async createAplBuild(teamId: string, data: AplBuildRequest): Promise { + const buildName = `${data?.spec?.imageName}-${data?.spec?.tag}` + if (buildName.length > 128) + throw new HttpError( + 400, + 'Invalid container image name, the combined image name and tag must not exceed 128 characters.', + ) try { const build = this.repoService.getTeamConfigService(teamId).createBuild(data) await this.saveTeamConfigItem(build) @@ -1434,7 +1483,8 @@ export default class OtomiStack { ) return build } catch (err) { - if (err.code === 409) err.publicMessage = 'Build name already exists' + if (err.code === 409) + err.publicMessage = 'Container image name already exists, the combined image name and tag must be unique.' throw err } } diff --git a/src/services/TeamConfigService.ts b/src/services/TeamConfigService.ts index 000883abb..968b51952 100644 --- a/src/services/TeamConfigService.ts +++ b/src/services/TeamConfigService.ts @@ -35,6 +35,7 @@ function mergeCustomizer(prev, next) { export class TeamConfigService { constructor(private teamConfig: TeamConfig) { + this.teamConfig.codeRepos ??= [] this.teamConfig.builds ??= [] this.teamConfig.workloads ??= [] this.teamConfig.services ??= [] diff --git a/src/utils/codeRepoUtils.test.ts b/src/utils/codeRepoUtils.test.ts index efe0c967b..27dbf2c48 100644 --- a/src/utils/codeRepoUtils.test.ts +++ b/src/utils/codeRepoUtils.test.ts @@ -4,7 +4,18 @@ import { chmod, writeFile } from 'fs/promises' import simpleGit, { SimpleGit } from 'simple-git' import { OtomiError } from 'src/error' import { v4 as uuidv4 } from 'uuid' -import { getGiteaRepoUrls, normalizeRepoUrl, testPrivateRepoConnect, testPublicRepoConnect } from './codeRepoUtils' +import * as codeRepoUtils from './codeRepoUtils' +import { + extractRepositoryRefs, + getGiteaRepoUrls, + getPrivateRepoBranches, + getPublicRepoBranches, + normalizeRepoUrl, + normalizeSSHKey, + setupGitAuthentication, + testPrivateRepoConnect, + testPublicRepoConnect, +} from './codeRepoUtils' jest.mock('simple-git', () => ({ __esModule: true, @@ -142,4 +153,228 @@ describe('codeRepoUtils', () => { ) }) }) + + describe('extractRepositoryRefs', () => { + beforeEach(() => { + jest.clearAllMocks() + }) + + it('should extract branches and tags correctly', async () => { + const mockGit: Partial = { + listRemote: jest.fn().mockResolvedValueOnce(` + abcd1234 refs/heads/main + efgh5678 refs/heads/develop + ijkl9012 refs/tags/v1.0.0 + mnop3456 refs/tags/v1.1.0 + `), + } + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + + const result = await extractRepositoryRefs('https://github.com/user/repo.git') + + expect(result).toEqual(['main', 'develop', 'v1.0.0', 'v1.1.0']) + expect(mockGit.listRemote).toHaveBeenCalledWith(['--refs', 'https://github.com/user/repo.git']) + }) + + it('should handle empty repository with no refs', async () => { + const mockGit: Partial = { + listRemote: jest.fn().mockResolvedValueOnce(''), + } + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + + const result = await extractRepositoryRefs('https://github.com/user/repo.git') + + expect(result).toEqual([]) + }) + + it('should handle malformed ref lines', async () => { + const mockGit: Partial = { + listRemote: jest.fn().mockResolvedValueOnce(` + abcd1234 refs/heads/main + invalid-line + efgh5678 refs/tags/v1.0.0 + `), + } + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + + const result = await extractRepositoryRefs('https://github.com/user/repo.git') + + expect(result).toEqual(['main', 'v1.0.0']) + }) + + it('should return empty array on listRemote error', async () => { + const mockGit: Partial = { + listRemote: jest.fn().mockRejectedValueOnce(new Error('Network error')), + } + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + + const result = await extractRepositoryRefs('https://github.com/user/repo.git') + + expect(result).toEqual([]) + }) + }) + + describe('normalizeSSHKey', () => { + it('should normalize SSH key with multiple whitespaces', () => { + const sshKey = `-----BEGIN OPENSSH PRIVATE KEY----- + some key with multiple whitespaces + -----END OPENSSH PRIVATE KEY-----` + + const result = normalizeSSHKey(sshKey) + + expect(result).toEqual(`-----BEGIN OPENSSH PRIVATE KEY----- +some +key +with +multiple +whitespaces +-----END OPENSSH PRIVATE KEY-----`) + }) + + it('should throw error for invalid SSH key format', () => { + const invalidKey = 'just a random string' + + expect(() => normalizeSSHKey(invalidKey)).toThrow('Invalid SSH Key format') + }) + }) + + describe('setupGitAuthentication', () => { + it('should setup SSH authentication with private key', async () => { + const sshKey = '-----BEGIN OPENSSH PRIVATE KEY-----\nkey\n-----END OPENSSH PRIVATE KEY-----' + const repoUrl = 'git@github.com:user/repo.git' + + const mockGit: Partial = { + env: jest.fn(), + } + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + ;(uuidv4 as jest.Mock).mockReturnValue('test-uuid') + ;(writeFile as jest.Mock).mockResolvedValue(undefined) + ;(chmod as jest.Mock).mockResolvedValue(undefined) + + const result = await setupGitAuthentication(repoUrl, sshKey) + + expect(mockGit.env).toHaveBeenCalledWith( + 'GIT_SSH_COMMAND', + 'ssh -i /tmp/otomi/sshKey-test-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null', + ) + expect(result.url).toBe(repoUrl) + expect(result.keyPath).toBe('/tmp/otomi/sshKey-test-uuid') + }) + + it('should setup HTTPS authentication with username and token', async () => { + const repoUrl = 'https://github.com/user/repo.git' + const username = 'testuser' + const accessToken = 'test-token' + + const result = await setupGitAuthentication(repoUrl, undefined, username, accessToken) + + expect(result.url).toBe( + `https://${encodeURIComponent(username)}:${encodeURIComponent(accessToken)}@github.com/user/repo.git`, + ) + }) + + it('should throw error for missing HTTPS credentials', async () => { + const repoUrl = 'https://github.com/user/repo.git' + + await expect(setupGitAuthentication(repoUrl)).rejects.toThrow( + 'Username and access token are required for HTTPS authentication', + ) + }) + + it('should throw error for invalid repository URL', async () => { + const repoUrl = 'invalid-url' + + await expect(setupGitAuthentication(repoUrl)).rejects.toThrow('Invalid repository URL format') + }) + }) + + describe('getPrivateRepoBranches', () => { + it('should get branches from private repo using SSH', async () => { + const repoUrl = 'git@github.com:user/repo.git' + const sshKey = '-----BEGIN OPENSSH PRIVATE KEY-----\nkey\n-----END OPENSSH PRIVATE KEY-----' + + const mockRefs = ['main', 'develop', 'feature/test'] + const mockGit: Partial = { + listRemote: jest.fn().mockResolvedValueOnce(` + abcd1234\trefs/heads/main + efgh5678\trefs/heads/develop + ijkl9012\trefs/heads/feature/test + `), + } + + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + ;(uuidv4 as jest.Mock).mockReturnValue('test-uuid') + ;(pathExists as jest.Mock).mockResolvedValue(true) + + const mockSetupGitAuthentication = { + url: repoUrl, + sshKey, + git: mockGit as SimpleGit, + keyPath: '/mock/path/test-uuid', + } + jest.spyOn(codeRepoUtils, 'setupGitAuthentication').mockResolvedValueOnce(mockSetupGitAuthentication) + const result = await getPrivateRepoBranches(repoUrl) + + expect(result).toEqual(mockRefs) + }) + + it('should return empty array for failed private repo connection', async () => { + const repoUrl = 'git@github.com:user/repo.git' + const sshKey = '-----BEGIN OPENSSH PRIVATE KEY-----\nkey\n-----END OPENSSH PRIVATE KEY-----' + + ;(simpleGit as jest.Mock).mockImplementation(() => ({ + listRemote: jest.fn().mockRejectedValue(new Error('Connection failed')), + })) + + const result = await getPrivateRepoBranches(repoUrl, sshKey) + + expect(result).toEqual([]) + }) + }) + + describe('getPublicRepoBranches', () => { + it('should retrieve branches from a public repository', async () => { + const repoUrl = 'https://github.com/user/repo.git' + + const mockGit: Partial = { + listRemote: jest.fn().mockResolvedValueOnce(` + abcd1234 refs/heads/main + efgh5678 refs/heads/develop + ijkl9012 refs/tags/v1.0.0 + `), + } + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + + const result = await getPublicRepoBranches(repoUrl) + + expect(result).toEqual(['main', 'develop', 'v1.0.0']) + expect(mockGit.listRemote).toHaveBeenCalledWith(['--refs', repoUrl]) + }) + + it('should return empty array when retrieving refs fails', async () => { + const repoUrl = 'https://github.com/user/repo.git' + + const mockGit: Partial = { + listRemote: jest.fn().mockRejectedValueOnce(new Error('Network error')), + } + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + + const result = await getPublicRepoBranches(repoUrl) + + expect(result).toEqual([]) + }) + + it('should handle an empty repository with no refs', async () => { + const repoUrl = 'https://github.com/user/repo.git' + + const mockGit: Partial = { + listRemote: jest.fn().mockResolvedValueOnce(''), + } + ;(simpleGit as jest.Mock).mockReturnValue(mockGit) + + const result = await getPublicRepoBranches(repoUrl) + + expect(result).toEqual([]) + }) + }) }) diff --git a/src/utils/codeRepoUtils.ts b/src/utils/codeRepoUtils.ts index 9b0144b42..c9b0e60f2 100644 --- a/src/utils/codeRepoUtils.ts +++ b/src/utils/codeRepoUtils.ts @@ -81,7 +81,7 @@ export function normalizeRepoUrl(inputUrl: string, isPrivate: boolean, isSSH: bo } } -function normalizeSSHKey(sshPrivateKey) { +export function normalizeSSHKey(sshPrivateKey) { if ( !sshPrivateKey.includes('-----BEGIN OPENSSH PRIVATE KEY-----') || !sshPrivateKey.includes('-----END OPENSSH PRIVATE KEY-----') @@ -97,44 +97,32 @@ function normalizeSSHKey(sshPrivateKey) { return `-----BEGIN OPENSSH PRIVATE KEY-----\n${basePrivateKey}\n-----END OPENSSH PRIVATE KEY-----` } -async function connectPrivateRepo( +export async function setupGitAuthentication( repoUrl: string, - sshKey?: string, + sshPrivateKey?: string, username?: string, accessToken?: string, -): Promise<{ status: string }> { - const keyId = uuidv4() as string - const keyPath = `/tmp/otomi/sshKey-${keyId}` - try { - let git: SimpleGit - let url = repoUrl - - if (url.startsWith('git@') && sshKey) { - await writeFile(keyPath, `${sshKey}\n`, { mode: 0o600 }) +): Promise<{ git: SimpleGit; url: string; keyPath?: string }> { + let keyPath: string | undefined + const git: SimpleGit = simpleGit() + let url = repoUrl + + if (url.startsWith('git@')) { + const normalizedKey: string = sshPrivateKey ? normalizeSSHKey(sshPrivateKey) : '' + if (normalizedKey) { + const keyId = uuidv4() as string + keyPath = `/tmp/otomi/sshKey-${keyId}` + await writeFile(keyPath, `${normalizedKey}\n`, { mode: 0o600 }) await chmod(keyPath, 0o600) - const GIT_SSH_COMMAND = `ssh -i ${keyPath} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null` - git = simpleGit() git.env('GIT_SSH_COMMAND', GIT_SSH_COMMAND) - } else if (url.startsWith('https://')) { - if (!username || !accessToken) throw new Error('Username and access token are required for HTTPS authentication') - const urlWithAuth = repoUrl.replace( - 'https://', - `https://${encodeURIComponent(username)}:${encodeURIComponent(accessToken)}@`, - ) - - git = simpleGit() - url = urlWithAuth - } else throw new Error('Invalid repository URL format. Must be SSH or HTTPS.') + } + } else if (url.startsWith('https://')) { + if (!username || !accessToken) throw new Error('Username and access token are required for HTTPS authentication') + url = repoUrl.replace('https://', `https://${encodeURIComponent(username)}:${encodeURIComponent(accessToken)}@`) + } else throw new Error('Invalid repository URL format. Must be SSH or HTTPS.') - await git.listRemote([url]) - return { status: 'success' } - } catch (error) { - console.log('error', error) - return { status: 'failed' } - } finally { - if (repoUrl.startsWith('git@') && (await pathExists(keyPath))) await unlink(keyPath) - } + return { git, url, keyPath } } export async function testPrivateRepoConnect( @@ -143,8 +131,17 @@ export async function testPrivateRepoConnect( username?: string, accessToken?: string, ) { - const normalizedKey: string = sshPrivateKey ? normalizeSSHKey(sshPrivateKey) : '' - return connectPrivateRepo(repoUrl, normalizedKey, username, accessToken) + let keyPath: string | undefined + try { + const authResult = await setupGitAuthentication(repoUrl, sshPrivateKey, username, accessToken) + keyPath = authResult.keyPath + await authResult.git.listRemote([authResult.url]) + return { status: 'success' } + } catch (error) { + return { status: 'failed' } + } finally { + if (repoUrl.startsWith('git@') && keyPath && (await pathExists(keyPath))) await unlink(keyPath) + } } export async function testPublicRepoConnect(repoUrl: string) { @@ -156,3 +153,61 @@ export async function testPublicRepoConnect(repoUrl: string) { return { status: 'failed' } } } + +export async function extractRepositoryRefs(repoUrl: string, git: SimpleGit = simpleGit()): Promise { + try { + let formattedRepoUrl = repoUrl + if (repoUrl.startsWith('https://gitea')) { + git.env({ + GIT_ASKPASS: 'echo', + GIT_TERMINAL_PROMPT: '0', + GIT_SSL_NO_VERIFY: 'true', + }) + const username = process.env.GIT_USER as string + const accessToken = process.env.GIT_PASSWORD as string + formattedRepoUrl = repoUrl.replace( + 'https://', + `https://${encodeURIComponent(username)}:${encodeURIComponent(accessToken)}@`, + ) + } + + const rawData = await git.listRemote(['--refs', formattedRepoUrl]) + const branches: string[] = [] + const tags: string[] = [] + + rawData.split('\n').forEach((line) => { + const parts = line.split('\t') + if (parts.length !== 2) return + const ref = parts[1] + if (ref.startsWith('refs/heads/')) branches.push(ref.replace('refs/heads/', '')) + else if (ref.startsWith('refs/tags/')) tags.push(ref.replace('refs/tags/', '')) + }) + + return [...branches, ...tags] + } catch (error) { + return [] + } +} + +export async function getPrivateRepoBranches( + repoUrl: string, + sshPrivateKey?: string, + username?: string, + accessToken?: string, +) { + let keyPath: string | undefined + try { + const authResult = await setupGitAuthentication(repoUrl, sshPrivateKey, username, accessToken) + keyPath = authResult.keyPath + return await extractRepositoryRefs(authResult.url, authResult.git) + } catch (error) { + return [] + } finally { + if (repoUrl.startsWith('git@') && keyPath && (await pathExists(keyPath))) await unlink(keyPath) + } +} + +export async function getPublicRepoBranches(repoUrl: string) { + const branches = await extractRepositoryRefs(repoUrl) + return branches +}