diff --git a/server/package.json b/server/package.json index d6380d484..caf633b82 100644 --- a/server/package.json +++ b/server/package.json @@ -123,7 +123,8 @@ "^.+\\.(t|j)s$": "ts-jest" }, "moduleNameMapper": { - "^@octokit/core$": "/__mocks__/@octokit/core.js" + "^@octokit/core$": "/__mocks__/@octokit/core.js", + "^jose$": "/__mocks__/jose/core.js" }, "setupFilesAfterEnv": [ "/../jest-setup.js" diff --git a/server/prisma/schema.prisma b/server/prisma/schema.prisma index 4dc5d6b4d..c36f5de1d 100644 --- a/server/prisma/schema.prisma +++ b/server/prisma/schema.prisma @@ -260,4 +260,15 @@ enum NotificationType { slack webhook discord +} + +model RegistryUser { + id String @id @default(cuid()) + username String + password String + scope Json + + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + expiresAt DateTime? } \ No newline at end of file diff --git a/server/src/__mocks__/jose/core.js b/server/src/__mocks__/jose/core.js new file mode 100644 index 000000000..f5e9bdfa9 --- /dev/null +++ b/server/src/__mocks__/jose/core.js @@ -0,0 +1,17 @@ +module.exports = { + importJWK: jest.fn().mockResolvedValue('mock-key'), + exportJWK: jest.fn().mockImplementation((v) => v), + SignJWT: jest.fn().mockImplementation((claims) => ({ + setProtectedHeader: jest.fn().mockReturnThis(), + setIssuedAt: jest.fn().mockReturnThis(), + setIssuer: jest.fn().mockReturnThis(), + setSubject: jest.fn().mockReturnThis(), + setAudience: jest.fn().mockReturnThis(), + setExpirationTime: jest.fn().mockReturnThis(), + sign: jest.fn().mockResolvedValue('mock-jwt-token'), + })), + generateKeyPair: jest.fn().mockResolvedValue({ + privateKey: {"kty":"EC","x":"fake","y":"fake","crv":"P-256","d":"fake"}, + publicKey: {"kty":"EC","x":"fake","y":"fake","crv":"P-256"}, + }) +}; \ No newline at end of file diff --git a/server/src/app.module.ts b/server/src/app.module.ts index 7cef91a87..4fdbc0e6b 100644 --- a/server/src/app.module.ts +++ b/server/src/app.module.ts @@ -25,6 +25,7 @@ import { GroupModule } from './groups/groups.module'; import { RolesModule } from './roles/roles.module'; import { TokenModule } from './token/token.module'; import { CliModule } from './cli/cli.module'; +import { RegistryModule } from './registry/registry.module'; @Module({ imports: [ @@ -51,6 +52,7 @@ import { CliModule } from './cli/cli.module'; RolesModule, TokenModule, CliModule, + RegistryModule ], controllers: [AppController, TemplatesController], providers: [AppService, TemplatesService], diff --git a/server/src/apps/apps.module.ts b/server/src/apps/apps.module.ts index b547b3f24..b646dff28 100644 --- a/server/src/apps/apps.module.ts +++ b/server/src/apps/apps.module.ts @@ -3,10 +3,12 @@ import { AppsService } from './apps.service'; import { KubernetesModule } from '../kubernetes/kubernetes.module'; import { AppsController } from './apps.controller'; import { PipelinesService } from '../pipelines/pipelines.service'; +import { RegistryModule } from '../registry/registry.module'; @Module({ providers: [AppsService, KubernetesModule, PipelinesService], exports: [AppsService], controllers: [AppsController], + imports: [RegistryModule] }) export class AppsModule {} diff --git a/server/src/apps/apps.service.spec.ts b/server/src/apps/apps.service.spec.ts index 07344a7e9..0f41d08ef 100644 --- a/server/src/apps/apps.service.spec.ts +++ b/server/src/apps/apps.service.spec.ts @@ -10,6 +10,7 @@ import { IApp } from './apps.interface'; import { IPodSize, ISecurityContext } from 'src/config/config.interface'; import { IUser } from 'src/auth/auth.interface'; import { IKubectlApp } from 'src/kubernetes/kubernetes.interface'; +import { RegistryService } from '../registry/registry.service'; const podsize: IPodSize = { name: 'small', @@ -122,6 +123,7 @@ describe('AppsService', () => { let notificationsService: jest.Mocked; let configService: jest.Mocked; let eventsGateway: jest.Mocked; + let registryService: jest.Mocked; const user: IUser = { id: '1', username: 'testuser' } as IUser; beforeEach(async () => { @@ -153,6 +155,10 @@ describe('AppsService', () => { provide: EventsGateway, useValue: { execStreams: {}, sendTerminalLine: jest.fn() }, }, + { + provide: RegistryService, + useValue: { makeTemporaryPushCredentialsForImage: jest.fn() } + } ], }).compile(); @@ -162,6 +168,7 @@ describe('AppsService', () => { notificationsService = module.get(NotificationsService); configService = module.get(ConfigService); eventsGateway = module.get(EventsGateway); + registryService = module.get(RegistryService); }); it('should be defined', () => { @@ -225,6 +232,7 @@ describe('AppsService', () => { describe('triggerImageBuild', () => { it('should call createBuildJob', async () => { (pipelinesService.getContext as jest.Mock).mockResolvedValue('ctx'); + (registryService.makeTemporaryPushCredentialsForImage as jest.Mock).mockResolvedValue({ username: 'fake', password: 'fake'}); jest.spyOn(service, 'getApp').mockResolvedValue({ spec: { gitrepo: { admin: true, ssh_url: 'repo' }, @@ -234,6 +242,7 @@ describe('AppsService', () => { } as any); await service.triggerImageBuild('p', 'ph', 'a', mockUSerGroups); expect(kubectl.createBuildJob).toHaveBeenCalled(); + expect(registryService.makeTemporaryPushCredentialsForImage).toHaveBeenCalled(); }); }); @@ -334,6 +343,7 @@ describe('AppsService', () => { {} as any, // NotificationsService {} as any, // ConfigService mockEventsGateway, + {} as any, // RegistryService ); }); @@ -450,6 +460,7 @@ describe('AppsService', () => { {} as any, // NotificationsService {} as any, // ConfigService {} as any, // EventsGateway + {} as any, // RegistryService ); }); @@ -489,6 +500,7 @@ describe('AppsService', () => { {} as any, // NotificationsService {} as any, // configService {} as any, // eventsGateway + {} as any, // RegistryService ); jest.spyOn(service, 'triggerImageBuild').mockResolvedValue({ status: 'ok', @@ -550,6 +562,7 @@ describe('AppsService', () => { {} as any, // NotificationsService {} as any, // configService {} as any, // eventsGateway + {} as any, // RegistryService ); // Methoden ersetzen service.getAllAppsList = mockGetAllAppsList; @@ -672,6 +685,7 @@ describe('AppsService', () => { mockNotificationsService, mockConfigService, {} as any, + {} as any, // RegistryService ); service.createApp = jest.fn().mockResolvedValue(undefined); @@ -893,6 +907,7 @@ describe('AppsService', () => { {} as any, // NotificationsService {} as any, // ConfigService {} as any, // eventsGateway + {} as any, // RegistryService ); // Override the methods and properties @@ -939,6 +954,7 @@ describe('AppsService', () => { mockNotificationsService, {} as any, // configService {} as any, // eventsGateway + {} as any, // RegistryService ); service['logger'] = mockLogger; }); @@ -1026,6 +1042,7 @@ describe('AppsService', () => { {} as any, // NotificationsService {} as any, // ConfigService {} as any, // EventsGateway + {} as any, // RegistryService ); }); diff --git a/server/src/apps/apps.service.ts b/server/src/apps/apps.service.ts index 90b7bbb49..715c4865e 100644 --- a/server/src/apps/apps.service.ts +++ b/server/src/apps/apps.service.ts @@ -11,6 +11,7 @@ import { ConfigService } from '../config/config.service'; import { KubectlTemplate } from '../templates/template'; import { Stream } from 'stream'; import { EventsGateway } from '../events/events.gateway'; +import { RegistryService } from '../registry/registry.service'; @Injectable() export class AppsService { @@ -23,6 +24,7 @@ export class AppsService { private NotificationsService: NotificationsService, private configService: ConfigService, private eventsGateway: EventsGateway, + private registryService: RegistryService ) { //this.logger.log('AppsService initialized'); } @@ -160,7 +162,9 @@ export class AppsService { const timestamp = new Date().getTime(); if (contextName) { + const image = pipeline + '-' + appName; this.kubectl.setCurrentContext(contextName); + const registryCredentials = await this.registryService.makeTemporaryPushCredentialsForImage(image); this.kubectl.createBuildJob( namespace, @@ -173,8 +177,11 @@ export class AppsService { ref: app.spec.branch, //git commit reference }, { - image: `${process.env.KUBERO_BUILD_REGISTRY}/${pipeline}-${appName}`, + registry: process.env.KUBERO_BUILD_REGISTRY || "", + image: image, tag: app.spec.branch + '-' + timestamp, + registryUsername: registryCredentials.username, + registryPassword: registryCredentials.password }, ); } diff --git a/server/src/deployments/deployments.module.ts b/server/src/deployments/deployments.module.ts index 83287ff54..84ef9cc4f 100644 --- a/server/src/deployments/deployments.module.ts +++ b/server/src/deployments/deployments.module.ts @@ -1,11 +1,13 @@ import { Module } from '@nestjs/common'; import { DeploymentsController } from './deployments.controller'; import { DeploymentsService } from './deployments.service'; -import { AppsService } from '../apps/apps.service'; -import { LogsService } from '../logs/logs.service'; +import { AppsModule } from '../apps/apps.module'; +import { LogsModule } from '../logs/logs.module'; +import { RegistryModule } from '../registry/registry.module'; @Module({ controllers: [DeploymentsController], - providers: [DeploymentsService, AppsService, LogsService], + imports: [AppsModule, LogsModule, RegistryModule], + providers: [DeploymentsService], }) export class DeploymentsModule {} diff --git a/server/src/deployments/deployments.service.spec.ts b/server/src/deployments/deployments.service.spec.ts index 3121fb7c2..6f3d992c4 100644 --- a/server/src/deployments/deployments.service.spec.ts +++ b/server/src/deployments/deployments.service.spec.ts @@ -7,6 +7,7 @@ import { LogsService } from '../logs/logs.service'; import { IUser } from '../auth/auth.interface'; import { ILoglines } from 'src/logs/logs.interface'; import { mockKubectlApp as app } from '../apps/apps.controller.spec'; +import { RegistryService } from 'src/registry/registry.service'; const mockUserGroups = ['group1', 'group2']; @@ -26,6 +27,7 @@ describe('DeploymentsService', () => { let pipelinesService: jest.Mocked; let logsService: jest.Mocked; let logLine: jest.Mocked; + let registryService: jest.Mocked; beforeEach(() => { kubectl = { @@ -51,12 +53,17 @@ describe('DeploymentsService', () => { fetchLogs: jest.fn(), } as any; + registryService = { + makeTemporaryPushCredentialsForImage: jest.fn().mockResolvedValue({ username: 'fake', password: 'fake' }), + } as any; + service = new DeploymentsService( kubectl, appsService, notificationsService, pipelinesService, logsService, + registryService ); logLine = { diff --git a/server/src/deployments/deployments.service.ts b/server/src/deployments/deployments.service.ts index ed3df2813..a9162a97e 100644 --- a/server/src/deployments/deployments.service.ts +++ b/server/src/deployments/deployments.service.ts @@ -9,6 +9,7 @@ import { AppsService } from '../apps/apps.service'; import { PipelinesService } from '../pipelines/pipelines.service'; import { ILoglines } from '../logs/logs.interface'; import { LogsService } from '../logs/logs.service'; +import { RegistryService } from '../registry/registry.service'; import { V1JobList } from '@kubernetes/client-node'; @Injectable() @@ -22,6 +23,7 @@ export class DeploymentsService { private notificationService: NotificationsService, private pipelinesService: PipelinesService, private LogsService: LogsService, + private registryService: RegistryService, ) { //this.kubectl = options.kubectl //this._io = options.io @@ -35,7 +37,7 @@ export class DeploymentsService { pipelineName: string, phaseName: string, appName: string, - userGroups: string[] + userGroups: string[], ): Promise { const namespace = pipelineName + '-' + phaseName; const jobs = (await this.kubectl.getJobs(namespace)) as V1JobList; @@ -139,6 +141,9 @@ export class DeploymentsService { // Create the Build CRD try { + const image = pipeline + '-' + app; + const registryCredentials = + await this.registryService.makeTemporaryPushCredentialsForImage(image); await this.kubectl.createBuildJob( namespace, app, @@ -150,8 +155,11 @@ export class DeploymentsService { url: gitrepo, }, { - image: process.env.KUBERO_BUILD_REGISTRY + '/' + pipeline + '-' + app, + registry: process.env.KUBERO_BUILD_REGISTRY || '', + image: image, tag: reference, + registryUsername: registryCredentials.username, + registryPassword: registryCredentials.password, }, ); } catch (error) { diff --git a/server/src/deployments/templates/buildpacks.yaml.ts b/server/src/deployments/templates/buildpacks.yaml.ts index 8ad868202..458dcc5ae 100644 --- a/server/src/deployments/templates/buildpacks.yaml.ts +++ b/server/src/deployments/templates/buildpacks.yaml.ts @@ -40,11 +40,12 @@ spec: value: "123456" - name: APP value: example + - name: PATCH_JSON + value: '{"spec":{"image":{"repository":"$REPOSITORY","tag": "$TAG"}}}' command: - sh - -c - - 'kubectl patch kuberoapps $APP --type=merge -p "{\"spec\":{\"image\":{\"repository\": - \"$REPOSITORY\",\"tag\": \"$TAG\"}}}"' + - 'kubectl patch kuberoapps $APP --type=merge -p "$PATCH_JSON"' image: bitnami/kubectl:latest imagePullPolicy: Always resources: {} diff --git a/server/src/deployments/templates/dockerfile.yaml.ts b/server/src/deployments/templates/dockerfile.yaml.ts index a4d2bd4a7..6c7788dc8 100644 --- a/server/src/deployments/templates/dockerfile.yaml.ts +++ b/server/src/deployments/templates/dockerfile.yaml.ts @@ -41,11 +41,12 @@ spec: value: 123456 - name: APP value: example + - name: PATCH_JSON + value: '{"spec":{"image":{"repository":"$REPOSITORY","tag": "$TAG"}}}' command: - sh - -c - - 'kubectl patch kuberoapps $APP --type=merge -p "{\"spec\":{\"image\":{\"repository\": - \"$REPOSITORY\",\"tag\": \"$TAG\"}}}"' + - 'kubectl patch kuberoapps $APP --type=merge -p "$PATCH_JSON"' image: bitnami/kubectl:latest imagePullPolicy: Always name: deploy diff --git a/server/src/deployments/templates/nixpacks.yaml.ts b/server/src/deployments/templates/nixpacks.yaml.ts index 62a6ba63a..47868b062 100644 --- a/server/src/deployments/templates/nixpacks.yaml.ts +++ b/server/src/deployments/templates/nixpacks.yaml.ts @@ -41,11 +41,12 @@ spec: value: 123456 - name: APP value: example + - name: PATCH_JSON + value: '{"spec":{"image":{"repository":"$REPOSITORY","tag": "$TAG"}}}' command: - sh - -c - - 'kubectl patch kuberoapps $APP --type=merge -p "{\"spec\":{\"image\":{\"repository\": - \"$REPOSITORY\",\"tag\": \"$TAG\"}}}"' + - 'kubectl patch kuberoapps $APP --type=merge -p "$PATCH_JSON"' image: bitnami/kubectl:latest imagePullPolicy: Always name: deploy diff --git a/server/src/kubernetes/kubernetes.service.ts b/server/src/kubernetes/kubernetes.service.ts index 0749222f3..920787e4c 100644 --- a/server/src/kubernetes/kubernetes.service.ts +++ b/server/src/kubernetes/kubernetes.service.ts @@ -6,7 +6,7 @@ import { IKubectlApp, IStorageClass, } from './kubernetes.interface'; -import { IPipeline } from '../pipelines/pipelines.interface'; +import { IPipeline, IRegistry } from '../pipelines/pipelines.interface'; import { KubectlPipeline } from '../pipelines/pipeline/pipeline'; import { KubectlApp, App } from '../apps/app/app'; import { dockerfileTemplate } from '../deployments/templates/dockerfile.yaml'; @@ -1266,6 +1266,136 @@ export class KubernetesService { } } + private async makeDockerAuthConfig( + registry: IRegistry, + username: string, + password: string, + ) { + const registryHost = registry.host; + + const authString = `${username}:${password}`; + const authBase64 = Buffer.from(authString).toString('base64'); + + const authConfig = { + auths: { + [registryHost]: { + auth: authBase64, + }, + }, + }; + + return authConfig; + } + + public async getSecret( + secretName: string, + ): Promise<{ [key: string]: string } | null> { + const namespace = process.env.KUBERO_NAMESPACE || 'kubero'; + + try { + const secret = await this.coreV1Api.readNamespacedSecret( + secretName, + namespace, + ); + + if (!secret.body.data) { + return null; + } + + const decodedData: any = {}; + for (const [key, value] of Object.entries(secret.body.data)) { + decodedData[key] = Buffer.from(value as string, 'base64').toString( + 'utf-8', + ); + } + + return decodedData; + } catch (error) { + if (error.response?.body?.reason === 'NotFound') { + return null; + } + throw error; + } + } + + public async upsertSecret( + secretName: string, + secretData: { [key: string]: string }, + ) { + const namespace = process.env.KUBERO_NAMESPACE || 'kubero'; + const secretUpsertRequest = { + apiVersion: 'v1', + kind: 'Secret', + metadata: { + name: secretName, + }, + type: 'Opaque', + stringData: secretData, + }; + + try { + await this.coreV1Api.replaceNamespacedSecret( + secretName, + namespace, + secretUpsertRequest, + ); + } catch (error) { + if (error.response?.body?.reason === 'NotFound') { + await this.coreV1Api.createNamespacedSecret( + namespace, + secretUpsertRequest, + ); + } else { + throw error; + } + } + } + + private async createPushSecret( + namespace: string, + jobName: string, + registryUsername: string, + registryPassword: string, + ) { + const conf = await this.getKuberoConfig( + process.env.KUBERO_NAMESPACE || 'kubero', + ); + const secretName = 'push-' + jobName; + + const dockerauthconfig = await this.makeDockerAuthConfig( + conf.spec.registry, + registryUsername, + registryPassword, + ); + const pushSecret = { + apiVersion: 'v1', + kind: 'Secret', + metadata: { + name: secretName, + }, + type: 'kubernetes.io/dockerconfigjson', + stringData: { + '.dockerconfigjson': JSON.stringify(dockerauthconfig), + }, + }; + + try { + await this.coreV1Api.replaceNamespacedSecret( + secretName, + namespace, + pushSecret, + ); + } catch (error) { + if (error.response?.body?.reason === 'NotFound') { + await this.coreV1Api.createNamespacedSecret(namespace, pushSecret); + } else { + throw error; + } + } + + return secretName; + } + public async createBuildJob( namespace: string, appName: string, @@ -1277,12 +1407,14 @@ export class KubernetesService { ref: string; }, repository: { + registry: string; + registryUsername: string; + registryPassword: string; image: string; tag: string; }, //job: any, //V1Job, ): Promise { - this.logger.error('refactoring: loadJob not implemented'); const job = this.loadJob(buildstrategy) as any; //const job = new Object() as any; @@ -1292,6 +1424,12 @@ export class KubernetesService { .replace(/[T]/g, '-') .substring(0, 13); const name = appName + '-' + pipelineName + '-' + id; + const secretName = await this.createPushSecret( + namespace, + name, + repository.registryUsername, + repository.registryPassword, + ); job.metadata.name = name.substring(0, 53); // max 53 characters allowed within kubernetes //job.metadata.namespace = namespace; @@ -1308,26 +1446,31 @@ export class KubernetesService { job.spec.template.spec.serviceAccount = appName + '-kuberoapp'; job.spec.template.spec.initContainers[0].env[0].value = git.url; job.spec.template.spec.initContainers[0].env[1].value = git.ref; - job.spec.template.spec.containers[0].env[0].value = repository.image; - job.spec.template.spec.containers[0].env[1].value = - repository.tag + '-' + id; + const imageUrl = repository.registry + '/' + repository.image; + job.spec.template.spec.containers[0].env[0].value = imageUrl; + const tag = repository.tag + '-' + id; + job.spec.template.spec.containers[0].env[1].value = tag; job.spec.template.spec.containers[0].env[2].value = appName; + job.spec.template.spec.containers[0].env[3].value = JSON.stringify({ + spec: { image: { repository: imageUrl, tag: tag } }, + }); + job.spec.template.spec.volumes[2].secret.secretName = secretName; if (buildstrategy === 'buildpacks') { // configure build container job.spec.template.spec.initContainers[2].args[1] = - repository.image + ':' + repository.tag + '-' + id; + imageUrl + ':' + repository.tag + '-' + id; } if (buildstrategy === 'dockerfile') { // configure push container job.spec.template.spec.initContainers[1].env[1].value = - repository.image + ':' + repository.tag + '-' + id; + imageUrl + ':' + repository.tag + '-' + id; job.spec.template.spec.initContainers[1].env[2].value = dockerfilePath; } if (buildstrategy === 'nixpacks') { // configure push container job.spec.template.spec.initContainers[2].env[1].value = - repository.image + ':' + repository.tag + '-' + id; + imageUrl + ':' + repository.tag + '-' + id; job.spec.template.spec.initContainers[2].env[2].value = dockerfilePath; } diff --git a/server/src/logs/logs.module.ts b/server/src/logs/logs.module.ts index b09347fe8..645c72ebd 100644 --- a/server/src/logs/logs.module.ts +++ b/server/src/logs/logs.module.ts @@ -1,10 +1,12 @@ import { Module } from '@nestjs/common'; import { LogsService } from './logs.service'; -import { AppsService } from 'src/apps/apps.service'; import { LogsController } from './logs.controller'; +import { AppsModule } from '../apps/apps.module'; @Module({ - providers: [LogsService, AppsService], + providers: [LogsService], controllers: [LogsController], + imports: [AppsModule], + exports: [LogsService] }) export class LogsModule {} diff --git a/server/src/registry/registry.controller.ts b/server/src/registry/registry.controller.ts new file mode 100644 index 000000000..657ce130a --- /dev/null +++ b/server/src/registry/registry.controller.ts @@ -0,0 +1,80 @@ +import { + Body, + Controller, + Delete, + Get, + Headers, + UnauthorizedException, + HttpCode, + HttpException, + HttpStatus, + Logger, + Param, + Post, + Put, + UseGuards, + Request, + Req, + Query, +} from '@nestjs/common'; +import { RegistryService } from './registry.service'; +import { ConfigService } from 'src/config/config.service'; + +@Controller({ path: 'api/registry', version: '1' }) +export class RegistryController { + private readonly logger = new Logger(RegistryController.name); + + constructor( + private registryService: RegistryService, + private configService: ConfigService, + ) {} + + private parseAuthHeader(authHeader: string) { + if (!authHeader) { + this.logger.verbose('auth header missing'); + throw new UnauthorizedException('Authorization header is missing'); + } + + // Extract the base64-encoded credentials + const authData = authHeader.split(' '); + const authType = authData[0]; + const base64Credentials = authData[1]; + if (authType.toLowerCase() != 'basic' || !base64Credentials) { + this.logger.verbose('auth header invalid'); + throw new UnauthorizedException('Invalid authorization header'); + } + + // Decode the credentials + const credentials = Buffer.from(base64Credentials, 'base64').toString( + 'utf-8', + ); + const [username, password] = credentials.split(':'); + + if (!username || !password) { + this.logger.verbose('invalid credentials'); + throw new UnauthorizedException('Invalid credentials format'); + } + + return [username, password]; + } + + @Get('/token') + async getPipelineToken( + @Headers('authorization') authHeader: string, + @Query('service') service: string, + @Query('scope') scope: string | string[], + ) { + const [username, password] = this.parseAuthHeader(authHeader); + + const jwt = await this.registryService.generateToken( + username, + password, + service, + scope, + ); + + return { + token: jwt, + }; + } +} diff --git a/server/src/registry/registry.module.ts b/server/src/registry/registry.module.ts new file mode 100644 index 000000000..0dd56c8f7 --- /dev/null +++ b/server/src/registry/registry.module.ts @@ -0,0 +1,14 @@ +import { Global, Module } from '@nestjs/common'; +import { RegistryController } from './registry.controller'; +import { RegistryService } from './registry.service'; +import { ConfigModule } from '../config/config.module'; +import { PrismaClient } from '@prisma/client'; +import { KubernetesModule } from 'src/kubernetes/kubernetes.module'; + +@Module({ + controllers: [RegistryController], + providers: [RegistryService], + exports: [RegistryService], + imports: [ConfigModule, KubernetesModule, PrismaClient], +}) +export class RegistryModule {} diff --git a/server/src/registry/registry.service.spec.ts b/server/src/registry/registry.service.spec.ts new file mode 100644 index 000000000..4cfb837de --- /dev/null +++ b/server/src/registry/registry.service.spec.ts @@ -0,0 +1,291 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { Logger, UnauthorizedException } from '@nestjs/common'; +import { RegistryService } from './registry.service'; +import { ConfigService } from '../config/config.service'; +import { PrismaClient } from '@prisma/client'; +import { SignJWT } from 'jose'; +import { KubernetesService } from '../kubernetes/kubernetes.service'; + +describe('RegistryService', () => { + let service: RegistryService; + let configService: ConfigService; + let prismaMock = { + registryUser: { + findFirst: jest.fn(), + create: jest.fn(), + }, + }; + const registryConfigBase = { + host: 'someservice', + port: 5000, + enabled: true, + create: false, + storage: '1Gi', + storageClassName: null, + subpath: '', + account: { + username: 'admin', + password: 'password', + hash: 'hash', + }, + }; + const registryPullUser = { + username: 'fake', + password: '$2b$10$SVulDccp3nXVJJv7fNLYZOHqDW./xumFwMDX0MfH6X47dThPiWRLy', // password + expiresAt: null, + scope: { + allowedScopes: [ + { type: 'repository', name: 'test/image', actions: ['pull'] }, + ], + }, + }; + const fakePrivateKey = + '{"kty":"EC","x":"fake","y":"fake","crv":"P-256","d":"fake"}'; + const fakePublicKey = '{"kty":"EC","x":"fake","y":"fake","crv":"P-256"}'; + const fakeRegistryLogin = { + usename: 'fake', + password: 'password', + '.dockerconfigjson': JSON.stringify({ + auths: { + someservice: { + auth: Buffer.from('fake:password').toString('base64'), + }, + }, + }), + }; + let kubernetesMock = { + upsertSecret: jest.fn(), + getSecret: jest.fn().mockImplementation((secretName) => { + switch (secretName) { + case 'registry-jwt-pubkey': + return { + jwk: fakePublicKey, + }; + case 'registry-jwt-privkey': + return { + privateKey: fakePrivateKey, + publicKey: fakePublicKey, + }; + case 'registry-login': + return fakeRegistryLogin; + } + }), + }; + + beforeEach(async () => { + jest.clearAllMocks(); + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + RegistryService, + { + provide: PrismaClient, + useValue: prismaMock, + }, + { + provide: KubernetesService, + useValue: kubernetesMock, + }, + ], + }) + .setLogger(new Logger()) + .compile(); + + service = module.get(RegistryService); + process.env.KUBERO_BUILD_REGISTRY = 'someservice'; + }); + + describe('onApplicationBootstrap', () => { + it('should try to create private and public JWK when both are missing', async () => { + //jest.clearAllMocks(); + let upsertSecretSpy = jest.spyOn(kubernetesMock, 'upsertSecret'); + let getSecretSpy = jest.spyOn(kubernetesMock, 'getSecret'); + getSecretSpy.mockImplementation((secretName) => { + switch (secretName) { + case 'registry-jwt-privkey': + return null; + case 'registry-jwt-pubkey': + return null; + case 'registry-login': + return fakeRegistryLogin; + } + }); + await service.onApplicationBootstrap(); + + expect(upsertSecretSpy).toHaveBeenNthCalledWith( + 1, + 'registry-jwt-privkey', + expect.objectContaining({ + privateKey: expect.any(String), + publicKey: expect.any(String), + }), + ); + expect(upsertSecretSpy).toHaveBeenNthCalledWith( + 2, + 'registry-jwt-pubkey', + expect.objectContaining({ + jwk: expect.any(String), + }), + ); + jest.restoreAllMocks(); + }); + it('should create the public key for the registry when a private key is present', async () => { + //jest.clearAllMocks(); + let upsertSecretSpy = jest.spyOn(kubernetesMock, 'upsertSecret'); + let getSecretSpy = jest.spyOn(kubernetesMock, 'getSecret'); + getSecretSpy.mockImplementation((secretName) => { + switch (secretName) { + case 'registry-jwt-privkey': + return { + privateKey: fakePrivateKey, + publicKey: fakePublicKey, + }; + case 'registry-jwt-pubkey': + return null; + } + }); + await service.onApplicationBootstrap(); + expect(upsertSecretSpy).toHaveBeenCalledWith('registry-jwt-pubkey', { + jwk: fakePublicKey, + }); + }); + }); + + describe('generateToken', () => { + beforeEach(async () => { + await service.onApplicationBootstrap(); + }); + + it('should throw an exception when service does not match the registry host name', async () => { + await expect( + service.generateToken( + 'fake', + 'password', + 'wrongservice', + 'repository:test/image:pull', + ), + ).rejects.toThrow(UnauthorizedException); + expect(prismaMock.registryUser.findFirst).toHaveBeenCalledTimes(0); + }); + + it('should throw an unauthorized exception when the user is not found', async () => { + prismaMock.registryUser.findFirst.mockResolvedValueOnce(null); + await expect( + service.generateToken( + 'fake', + 'password', + 'someservice', + 'repository:test/image:pull', + ), + ).rejects.toThrow(UnauthorizedException); + expect(prismaMock.registryUser.findFirst).toHaveBeenCalled(); + }); + + it('should throw an unauthorized exception when the user is expired', async () => { + let expiredUser = { + username: 'fake', + password: + '$2b$10$SVulDccp3nXVJJv7fNLYZOHqDW./xumFwMDX0MfH6X47dThPiWRLy', // password + expiresAt: new Date(1991, 1, 1, 23, 59, 59), + scope: [], + }; + prismaMock.registryUser.findFirst.mockResolvedValueOnce(expiredUser); + await expect( + service.generateToken( + 'fake', + 'password', + 'someservice', + 'repository:test/image:pull', + ), + ).rejects.toThrow(UnauthorizedException); + expect(prismaMock.registryUser.findFirst).toHaveBeenCalled(); + }); + + var testTokenMismatch = async ( + permission: { type; name; action }, + requestedScope: string, + ) => { + const testUser = { + username: 'fake', + password: + '$2b$10$SVulDccp3nXVJJv7fNLYZOHqDW./xumFwMDX0MfH6X47dThPiWRLy', // password + expiresAt: null, + scope: [permission], + }; + prismaMock.registryUser.findFirst.mockResolvedValueOnce(testUser); + + const result = await service.generateToken( + 'fake', + 'password', + 'someservice', + requestedScope, + ); + + expect(result).toBe('mock-jwt-token'); + expect(SignJWT).toHaveBeenCalledWith({ access: [] }); + expect(prismaMock.registryUser.findFirst).toHaveBeenCalled(); + }; + + it('should return a JWT with empty scope when the image name in permissions and requestedScopes does not match', async () => { + await testTokenMismatch( + { + type: 'repository', + name: { kind: 'exact', name: 'test/image' }, + action: ['pull'], + }, + 'repository:test/otherimage:pull', + ); + }); + + it('should return a JWT with empty scope when the action in permissions and requestedScopes does not match', async () => { + await testTokenMismatch( + { + type: 'repository', + name: { kind: 'exact', name: 'test/image' }, + action: ['pull'], + }, + 'repository:test/image:push', + ); + }); + + it('should return a JWT with empty scope when the type in permissions and requestedScopes does not match', async () => { + await testTokenMismatch( + { + type: 'repository', + name: { kind: 'exact', name: 'test/image' }, + action: ['pull'], + }, + 'registry:test/image:pull', + ); + }); + + it('should return a JWT with correct scope when allowedScopes and requestedScopes match', async () => { + let testUser = { + username: 'fake', + password: + '$2b$10$SVulDccp3nXVJJv7fNLYZOHqDW./xumFwMDX0MfH6X47dThPiWRLy', // password + expiresAt: null, + scope: [ + { type: 'repository', name: 'test/image', actions: ['pull', 'push'] }, + ], + }; + prismaMock.registryUser.findFirst.mockResolvedValueOnce(testUser); + + // action is intentionally swapped + const result = await service.generateToken( + 'fake', + 'password', + 'someservice', + `repository:test/image:push,pull`, + ); + + expect(result).toBe('mock-jwt-token'); + expect(SignJWT).toHaveBeenCalledWith({ + access: [ + { type: 'repository', name: 'test/image', actions: ['pull', 'push'] }, + ], + }); + expect(prismaMock.registryUser.findFirst).toHaveBeenCalled(); + }); + }); +}); diff --git a/server/src/registry/registry.service.ts b/server/src/registry/registry.service.ts new file mode 100644 index 000000000..5290afdcf --- /dev/null +++ b/server/src/registry/registry.service.ts @@ -0,0 +1,545 @@ +import { + Injectable, + Logger, + NotImplementedException, + OnApplicationBootstrap, + UnauthorizedException, +} from '@nestjs/common'; +import { PrismaClient } from '@prisma/client'; +import * as bcrypt from 'bcrypt'; +import { randomBytes, randomUUID } from 'crypto'; +import { importJWK, exportJWK, JWK, SignJWT, generateKeyPair } from 'jose'; +import { KubernetesService } from '../kubernetes/kubernetes.service'; +import { truncate } from 'fs/promises'; +import { truncateSync } from 'fs'; + +interface NameMatchExact { + kind: 'exact'; + name: string; +} + +interface NameMatchAny { + kind: 'any'; +} + +type NameMatch = NameMatchExact | NameMatchAny; + +type PermissionType = 'repository'; + +/** + * This class models a permission that is granted to a RegistryUser. + */ +export class Permission { + public actions: Set; + + constructor( + public type: PermissionType, + public name: NameMatch, + actions: Set | string[], + ) { + this.actions = actions instanceof Set ? actions : new Set(actions); + } + + toObject() { + return { + type: this.type, + name: + this.name.kind === 'exact' + ? { kind: 'exact' as const, name: this.name.name } + : { kind: 'any' as const }, + actions: Array.from(this.actions), + }; + } + + equals(other: Permission) { + if (this.type !== other.type) { + return false; + } + + if (this.name.kind !== other.name.kind) { + return false; + } + + return ( + Array.from(this.actions.values()).sort() == + Array.from(other.actions.values()).sort() + ); + } + + static fromObject(obj: any): Permission { + const actions = Array.isArray(obj.actions) + ? new Set(obj.actions) + : new Set(); + + const name: NameMatch = + obj.name?.kind === 'exact' + ? { kind: 'exact', name: obj.name.name } + : { kind: 'any' }; + + return new Permission(obj.type, name, actions); + } + + static fromObjectArray(objArray: any): Permission[] { + if (!Array.isArray(objArray)) { + return []; + } + return objArray.map((obj) => Permission.fromObject(obj)); + } +} + +/** + * This class models a Scope that is requested by the registry. + */ +export class RegistryScope { + constructor( + public type: string, + public name: string, + public actions: Set, + ) {} + + toObject() { + return { + type: this.type, + name: this.name, + actions: Array.from(this.actions), + }; + } + + static fromObject(obj: any): RegistryScope { + const actions = Array.isArray(obj.actions) + ? new Set(obj.actions) + : new Set(); + return new RegistryScope(obj.type, obj.name, actions); + } + + static fromObjectArray(objArray: any): RegistryScope[] { + if (!Array.isArray(objArray)) { + return []; + } + return objArray.map((obj) => RegistryScope.fromObject(obj)); + } +} + +@Injectable() +export class RegistryService implements OnApplicationBootstrap { + private readonly logger = new Logger(RegistryService.name); + + private jwk: CryptoKeyPair; + + private registryHostname: string; + + private readonly jwk_algo = 'ES256'; + + constructor( + private prisma: PrismaClient, + private kubectl: KubernetesService, + ) {} + + public async onApplicationBootstrap() { + if (!process.env.KUBERO_BUILD_REGISTRY) { + return; + } + this.registryHostname = process.env.KUBERO_BUILD_REGISTRY; + await this.maybeProvisionRegistryJwk(); + await this.maybeProvisionPullCredential(); + } + + private async maybeProvisionRegistryJwk() { + const tryJwk = await this.tryGetRegistryJwkPrivate(); + if (!tryJwk) { + this.logger.log( + 'Private JWK for registry not found in kubernetes or invalid; generating one', + ); + this.jwk = await this.provisionRegistryJwkPrivate(); + } else { + this.jwk = tryJwk; + } + + if (!(await this.isRegistryJwkPublicValid())) { + this.logger.log( + "Public JWK not found in kubernetes, invalid or doesn't match private key; provisioning from privatekey", + ); + await this.kubectl.upsertSecret('registry-jwt-pubkey', { + jwk: JSON.stringify(this.jwk.publicKey), + }); + } + } + + private jwkPubkeysEqual(keyA, keyB) { + return ( + keyA.crv === keyB.crv && + keyA.kty === keyB.kty && + keyA.x === keyB.x && + keyA.y === keyB.y + ); + } + + private async isRegistryJwkPublicValid() { + let secretJwkPublic = await this.kubectl.getSecret('registry-jwt-pubkey'); + if (!secretJwkPublic || !secretJwkPublic.jwk) { + return false; + } + + let jwkPublic = JSON.parse(secretJwkPublic.jwk); + if (!jwkPublic) { + return false; + } + return this.jwkPubkeysEqual(jwkPublic, this.jwk.publicKey); + } + + private async tryGetRegistryJwkPrivate(): Promise { + let secretJwkPrivate = await this.kubectl.getSecret('registry-jwt-privkey'); + if (!secretJwkPrivate || !secretJwkPrivate.privateKey) { + return null; + } + const jwkPrivateJson = JSON.parse(secretJwkPrivate.privateKey); + if (!jwkPrivateJson) { + return null; + } + + if (!secretJwkPrivate.publicKey) { + return null; + } + + const jwkPublicJson = JSON.parse(secretJwkPrivate.publicKey); + if (!jwkPublicJson) { + return null; + } + + return { + publicKey: jwkPublicJson, + privateKey: jwkPrivateJson, + }; + } + + private async provisionRegistryJwkPrivate(): Promise<{ + privateKey: CryptoKey; + publicKey: CryptoKey; + }> { + const generatedKey = await generateKeyPair(this.jwk_algo, { + extractable: true, + }); + const exportedPrivateKey = await exportJWK(generatedKey.privateKey); + const exportedPublicKey = await exportJWK(generatedKey.publicKey); + + this.kubectl.upsertSecret('registry-jwt-privkey', { + privateKey: JSON.stringify(exportedPrivateKey), + publicKey: JSON.stringify(exportedPublicKey), + }); + + return generatedKey; + } + + public async addRegistryUser( + username: string, + password: string, + permissions: Permission[], + ) { + // TODO expiration + await this.prisma.registryUser.create({ + data: { + username: username, + password: bcrypt.hashSync(password, 10), + scope: permissions.map((p) => p.toObject()), + }, + }); + } + + private randomPassword() { + return randomBytes(30).toString('base64').substring(0, 30); + } + + public async makeTemporaryPushCredentialsForImage( + authorizedImage: string, + ): Promise<{ username: string; password: string }> { + const registryUsername = randomUUID(); + const registryPassword = this.randomPassword(); + const permission = new Permission( + 'repository', + { kind: 'exact', name: authorizedImage }, + new Set(['pull', 'push']), + ); + this.addRegistryUser(registryUsername, registryPassword, [permission]); + + return { + username: registryUsername, + password: registryPassword, + }; + } + + private parseScope(scope: String) { + const [ressourceType, ressourceName, ressourceActionStr] = scope.split(':'); + if (!ressourceType || !['repository', 'registry'].includes(ressourceType)) { + this.logger.debug( + `parseScope: missing or invalid ressourceType "${ressourceType}"`, + ); + return null; + } + + if (!ressourceName || !ressourceActionStr) { + this.logger.debug( + `parseScope: missing ressourceName "${ressourceName}" or ressourceActionStr "${ressourceActionStr}`, + ); + return null; + } + + const ressourceActions = new Set(ressourceActionStr.split(',')); + + return new RegistryScope(ressourceType, ressourceName, ressourceActions); + } + + private parseRequestedScopes(scope: string | string[]) { + const requestedScopes = Array.isArray(scope) ? scope : [scope]; + const parsedScopes: Array = []; + + for (const scopeStr of requestedScopes) { + const parsed = this.parseScope(scopeStr); + if (!parsed) { + throw new UnauthorizedException('Invalid scope format'); + } + parsedScopes.push(parsed); + } + + return parsedScopes; + } + + /** + * Find the scopes in requestedScopes that are authorized by the given permissions. + * @param requestedScopes the scopes to check the permissions against + * @param permissions the permissions the user has + * @returns the requestedScopes that are covered by the given permissions + */ + private findAuthorizedScopes( + requestedScopes: RegistryScope[], + permissions: Permission[], + ): Array { + const grantedScopes: RegistryScope[] = []; + + for (const requestedScope of requestedScopes) { + // Find matching permission by type and name + const matchingPermission = permissions.find((permission: Permission) => { + if (permission.type != requestedScope.type) { + return false; + } + switch (permission.name.kind) { + case 'any': + return true; + case 'exact': + return requestedScope.name == permission.name.name; + default: + throw new NotImplementedException(); + } + }); + + if (!matchingPermission) { + // Skip scopes that don't have a matching permission + continue; + } + + // Compute intersection of actions + const allowedActions = matchingPermission.actions || []; + let grantedActions = new Set(); + for (const a of allowedActions) { + if (requestedScope.actions.has(a)) { + grantedActions.add(a); + } + } + + // Only include scope if at least one action is granted + if (grantedActions.size > 0) { + grantedScopes.push( + new RegistryScope( + matchingPermission.type, + // use the name from the requestedScope since the permission may have a match-all for the name + requestedScope.name, + grantedActions, + ), + ); + } + } + + return grantedScopes; + } + + async generateToken( + username: string, + password: string, + service: string, + scope: string | string[], + ): Promise { + if (service !== this.registryHostname) { + this.logger.verbose( + `invalid service: ${service} != ${this.registryHostname}`, + ); + throw new UnauthorizedException('Invalid service'); + } + + const registryUser = await this.verifyPassword(username, password); + if (!registryUser) { + this.logger.verbose(`invalid credentials: ${registryUser}`); + throw new UnauthorizedException('Invalid credentials'); + } + + const parsedScopes = this.parseRequestedScopes(scope); + + const allowedScopes = Permission.fromObjectArray(registryUser.scope); + + const grantedScopes = this.findAuthorizedScopes( + parsedScopes, + allowedScopes, + ); + + return this.signJwt(username, grantedScopes); + } + + private async signJwt(subjectName: string, grantedScopes: RegistryScope[]) { + const jwtprivkeystr = JSON.parse( + process.env.KUBERO_REGISTRY_JWT_KEY_PRIVATE || '{}', + ); + const jwtprivkey = await importJWK(jwtprivkeystr, this.jwk_algo); + + var token = await new SignJWT({ + access: grantedScopes.map((scope) => scope.toObject()), + }) + .setProtectedHeader({ alg: this.jwk_algo, kid: '0' }) + .setIssuedAt() + .setIssuer('todo.kubero.dev') // TODO + .setSubject(subjectName) + .setAudience(this.registryHostname) + .setExpirationTime('3h') // TODO clamp to user expiration + .sign(jwtprivkey); + + return token; + } + + private async verifyPassword(username: string, password: string) { + const registryUser = await this.prisma.registryUser.findFirst({ + where: { username: username }, + }); + + if (!registryUser) { + return null; + } + const isUserExpired = + registryUser.expiresAt && registryUser.expiresAt < new Date(); + if (isUserExpired) { + return null; + } + + const isPasswordValid = await bcrypt.compare( + password, + registryUser.password, + ); + if (!isPasswordValid) { + return null; + } + + return registryUser; + } + + private async isPullCredentialValid() { + const pullCredentials = await this.kubectl.getSecret('registry-login'); + + if ( + !pullCredentials || + !pullCredentials.username || + !pullCredentials.password || + !pullCredentials['.dockerconfigjson'] + ) { + return false; + } + + const pullUser = await this.verifyPassword( + pullCredentials.username, + pullCredentials.password, + ); + if (!pullUser) { + return false; + } + + if (!Array.isArray(pullUser.scope) || !pullUser.scope.length) { + return false; + } + + const permission = Permission.fromObject(pullUser.scope[0]); + if ( + !permission || + !permission.equals( + new Permission( + 'repository', + { kind: 'any' }, + new Set(['pull']), + ), + ) + ) { + return false; + } + + const dockerconfig = JSON.parse(pullCredentials['.dockerconfigjson']); + if ( + !dockerconfig || + !dockerconfig.auths || + typeof dockerconfig.auths !== 'object' + ) { + return false; + } + + if (!dockerconfig.auths[this.registryHostname]) { + return false; + } + + const auth = dockerconfig.auths[this.registryHostname].auth; + const expectedAuth = this.encodeCredentialsDockerAuthConfig( + pullCredentials.username, + pullCredentials.password, + ); + return auth === expectedAuth; + } + + private encodeCredentialsDockerAuthConfig( + username: string, + password: string, + ) { + const authString = `${username}:${password}`; + return Buffer.from(authString).toString('base64'); + } + + private makeDockerAuthConfig(username: string, password: string) { + return { + auths: { + [this.registryHostname]: { + auth: this.encodeCredentialsDockerAuthConfig(username, password), + }, + }, + }; + } + + private async maybeProvisionPullCredential() { + if (await this.isPullCredentialValid()) { + return; + } + + this.logger.log( + 'No pull credentials for kubernetes found or credentials invalid, creating.', + ); + + const password = this.randomPassword(); + const username = `k8s-pull-${randomUUID()}`; + const permission = new Permission( + 'repository', + { kind: 'any' }, + new Set(['pull']), + ); + this.addRegistryUser(username, password, [permission]); + + const secret = { + username, + password, + '.dockerconfigjson': JSON.stringify( + this.makeDockerAuthConfig(username, password), + ), + }; + + await this.kubectl.upsertSecret('registry-login', secret); + } +} diff --git a/server/src/repo/repo.module.ts b/server/src/repo/repo.module.ts index 64631481e..a515d10f3 100644 --- a/server/src/repo/repo.module.ts +++ b/server/src/repo/repo.module.ts @@ -1,10 +1,11 @@ import { Module } from '@nestjs/common'; import { RepoController } from './repo.controller'; import { RepoService } from './repo.service'; -import { AppsService } from 'src/apps/apps.service'; +import { AppsModule } from '../apps/apps.module'; @Module({ controllers: [RepoController], - providers: [RepoService, AppsService], + providers: [RepoService], + imports: [AppsModule], }) export class RepoModule {} diff --git a/server/src/security/security.module.ts b/server/src/security/security.module.ts index c0b18d674..55c5cea96 100644 --- a/server/src/security/security.module.ts +++ b/server/src/security/security.module.ts @@ -3,9 +3,10 @@ import { SecurityController } from './security.controller'; import { SecurityService } from './security.service'; import { KubernetesModule } from '../kubernetes/kubernetes.module'; import { PipelinesModule } from '../pipelines/pipelines.module'; -import { AppsService } from '../apps/apps.service'; +import { AppsModule} from '../apps/apps.module'; @Module({ controllers: [SecurityController], - providers: [SecurityService, KubernetesModule, PipelinesModule, AppsService], + imports: [AppsModule], + providers: [SecurityService, KubernetesModule, PipelinesModule], }) export class SecurityModule {} diff --git a/server/src/status/status.module.ts b/server/src/status/status.module.ts index f61dd7519..cac0ce1a5 100644 --- a/server/src/status/status.module.ts +++ b/server/src/status/status.module.ts @@ -6,7 +6,7 @@ import { } from '@willsoto/nestjs-prometheus'; import { StatusService } from './status.service'; import { ScheduleModule } from '@nestjs/schedule'; -import { AppsService } from '../apps/apps.service'; +import { AppsModule } from '../apps/apps.module'; import { StatusController } from './status.controller'; @Module({ @@ -15,10 +15,10 @@ import { StatusController } from './status.controller'; controller: StatusController, }), ScheduleModule.forRoot(), + AppsModule ], providers: [ StatusService, - AppsService, makeGaugeProvider({ name: 'kubero_pipelines_total', help: 'Total number of pipelines',