diff --git a/backend/src/clients/artefactScan.ts b/backend/src/clients/artefactScan.ts index 20dbc6d2dc..0a4a553154 100644 --- a/backend/src/clients/artefactScan.ts +++ b/backend/src/clients/artefactScan.ts @@ -62,7 +62,7 @@ export const ModelScanResponseSchema = z.object({ }), ), }) -export type ModelScanResponseSchema = z.infer +export type ModelScanResponse = z.infer // There's no formal definition of the JSON schema so this must be permissive const ImageHistorySchema = z @@ -189,7 +189,7 @@ export const TrivyScanResultResponseSchema = z Results: z.array(ResultSchema).optional(), }) .passthrough() -export type TrivyScanResultResponseSchema = z.infer +export type TrivyScanResultResponse = z.infer async function getArtefactScanInfo() { const url = `${config.artefactScanning.artefactscan.protocol}://${config.artefactScanning.artefactscan.host}:${config.artefactScanning.artefactscan.port}` @@ -239,14 +239,11 @@ async function scanStream(stream: Readable, fileName: string, endpoint: 'file' | return await res.json() } -export async function scanFileStream(stream: Readable, fileName: string): Promise { +export async function scanFileStream(stream: Readable, fileName: string): Promise { return ModelScanResponseSchema.parse(await scanStream(stream, fileName, 'file')) } -export async function scanImageBlobStream( - stream: Readable, - blobDigest: string, -): Promise { +export async function scanImageBlobStream(stream: Readable, blobDigest: string): Promise { return TrivyScanResultResponseSchema.parse(await scanStream(stream, blobDigest, 'image')) } diff --git a/backend/src/clients/registry.ts b/backend/src/clients/registry.ts index 656a2344b6..66ef73d5c4 100644 --- a/backend/src/clients/registry.ts +++ b/backend/src/clients/registry.ts @@ -11,21 +11,22 @@ import config from '../utils/config.js' import { InternalError, RegistryError } from '../utils/error.js' import { AcceptManifestMediaTypeHeaderValue, - BaseApiCheckResponseBody, - BaseApiCheckResponseHeaders, - BlobResponseHeaders, - BlobUploadResponseHeaders, - CatalogBodyResponse, - CatalogResponseHeaders, + BaseApiCheckResponseBodySchema, + BaseApiCheckResponseHeadersSchema, + BlobResponseHeadersSchema, + BlobUploadResponseHeadersSchema, + CatalogBodyResponseSchema, + CatalogResponseHeadersSchema, CommonRegistryHeaders, - DeleteManifestResponseHeaders, - ImageManifestV2, - ManifestListMediaType, - ManifestMediaType, - ManifestResponseHeaders, - RegistryErrorResponseBody, - TagsListResponseBody, - TagsListResponseHeaders, + CommonRegistryHeadersSchema, + DeleteManifestResponseHeadersSchema, + ImageManifestV2Schema, + ManifestListMediaTypeSchema, + ManifestMediaTypeSchema, + ManifestResponseHeadersSchema, + RegistryErrorResponseBodySchema, + TagsListResponseBodySchema, + TagsListResponseHeadersSchema, } from '../utils/registryResponses.js' const registry = config.registry.connection.internal @@ -103,7 +104,7 @@ async function registryRequest, + headersSchema = CommonRegistryHeadersSchema as unknown as ZodSchema, expectStream = false, extraFetchOptions = {}, extraHeaders = {}, @@ -154,8 +155,8 @@ async function registryRequest { const result = await registryRequest(token, '_catalog?n=100', { - bodySchema: CatalogBodyResponse, - headersSchema: CatalogResponseHeaders, + bodySchema: CatalogBodyResponseSchema, + headersSchema: CatalogResponseHeadersSchema, pagination: { enabled: true, aggregate: (acc, next) => ({ @@ -256,8 +257,8 @@ export async function listModelRepos(token: string, modelId: string): Promise ({ @@ -279,7 +280,7 @@ export async function listImageTags(token: string, repoRef: RepoRefInterface) { export async function isImageTagManifestList(token: string, imageRef: ImageRefInterface): Promise { const result = await registryRequest(token, `${imageRef.repository}/${imageRef.name}/manifests/${imageRef.tag}`, { // do not validate the body here as we only care about Content-Type - headersSchema: ManifestResponseHeaders, + headersSchema: ManifestResponseHeadersSchema, extraHeaders: { Accept: AcceptManifestMediaTypeHeaderValue, }, @@ -294,10 +295,10 @@ export async function isImageTagManifestList(token: string, imageRef: ImageRefIn }) } - if ((ManifestListMediaType.options as string[]).includes(contentType)) { + if ((ManifestListMediaTypeSchema.options as string[]).includes(contentType)) { return true } - if ((ManifestMediaType.options as string[]).includes(contentType)) { + if ((ManifestMediaTypeSchema.options as string[]).includes(contentType)) { return false } @@ -310,8 +311,8 @@ export async function isImageTagManifestList(token: string, imageRef: ImageRefIn export async function getImageTagManifest(token: string, imageRef: ImageRefInterface) { // TODO: handle multi-platform images const result = await registryRequest(token, `${imageRef.repository}/${imageRef.name}/manifests/${imageRef.tag}`, { - bodySchema: ImageManifestV2, - headersSchema: ManifestResponseHeaders, + bodySchema: ImageManifestV2Schema, + headersSchema: ManifestResponseHeadersSchema, extraHeaders: { Accept: AcceptManifestMediaTypeHeaderValue, }, @@ -326,7 +327,7 @@ export async function getRegistryLayerStream( layerDigest: string, ): Promise<{ stream: Readable; abort: () => void }> { const result = await registryRequest(token, `${repoRef.repository}/${repoRef.name}/blobs/${layerDigest}`, { - headersSchema: BlobResponseHeaders, + headersSchema: BlobResponseHeadersSchema, expectStream: true, extraHeaders: { Accept: AcceptManifestMediaTypeHeaderValue, @@ -366,7 +367,7 @@ export async function doesLayerExist(token: string, repoRef: RepoRefInterface, d export async function initialiseUpload(token: string, repoRef: RepoRefInterface) { const result = await registryRequest(token, `${repoRef.repository}/${repoRef.name}/blobs/uploads/`, { - headersSchema: BlobUploadResponseHeaders, + headersSchema: BlobUploadResponseHeadersSchema, expectStream: true, extraFetchOptions: { method: 'POST', @@ -378,7 +379,7 @@ export async function initialiseUpload(token: string, repoRef: RepoRefInterface) export async function putManifest(token: string, imageRef: ImageRefInterface, manifest: BodyInit, contentType: string) { const result = await registryRequest(token, `${imageRef.repository}/${imageRef.name}/manifests/${imageRef.tag}`, { - headersSchema: ManifestResponseHeaders, + headersSchema: ManifestResponseHeadersSchema, expectStream: true, extraFetchOptions: { method: 'PUT', @@ -402,7 +403,7 @@ export async function uploadLayerMonolithic( size: string, ) { const result = await registryRequest(token, `${uploadURL}&digest=${digest}`.replace(/^(\/v2\/)/, ''), { - headersSchema: BlobUploadResponseHeaders, + headersSchema: BlobUploadResponseHeadersSchema, expectStream: true, extraFetchOptions: { method: 'PUT', @@ -430,7 +431,7 @@ export async function mountBlob( token, `${destinationRepoRef.repository}/${destinationRepoRef.name}/blobs/uploads/?from=${sourceRepoRef.repository}/${sourceRepoRef.name}&mount=${blobDigest}`, { - headersSchema: BlobUploadResponseHeaders, + headersSchema: BlobUploadResponseHeadersSchema, extraFetchOptions: { method: 'POST', }, @@ -443,7 +444,7 @@ export async function mountBlob( export async function deleteManifest(token: string, imageRef: ImageRefInterface) { const result = await registryRequest(token, `${imageRef.repository}/${imageRef.name}/manifests/${imageRef.tag}`, { - headersSchema: DeleteManifestResponseHeaders, + headersSchema: DeleteManifestResponseHeadersSchema, expectStream: true, extraFetchOptions: { method: 'DELETE', diff --git a/backend/src/models/Scan.ts b/backend/src/models/Scan.ts index 99b693d1ba..5880955bdb 100644 --- a/backend/src/models/Scan.ts +++ b/backend/src/models/Scan.ts @@ -1,6 +1,6 @@ import { model, type ObjectId, Schema } from 'mongoose' -import type { ModelScanResponseSchema, TrivyScanResultResponseSchema } from '../clients/artefactScan.js' +import type { ModelScanResponse, TrivyScanResultResponse } from '../clients/artefactScan.js' import { ArtefactScanState, type ArtefactScanStateKeys } from '../connectors/artefactScanning/Base.js' import { type SoftDeleteDocument, softDeletionPlugin } from './plugins/softDeletePlugin.js' @@ -11,7 +11,7 @@ export type ScanInterface = { scannerVersion?: string state: ArtefactScanStateKeys summary?: ScanSummary - additionalInfo?: TrivyScanResultResponseSchema | ModelScanResponseSchema + additionalInfo?: TrivyScanResultResponse | ModelScanResponse lastRunAt: Date diff --git a/backend/src/services/mirroredModel/importers/image.ts b/backend/src/services/mirroredModel/importers/image.ts index 51c4aa7d04..cfe56ebd69 100644 --- a/backend/src/services/mirroredModel/importers/image.ts +++ b/backend/src/services/mirroredModel/importers/image.ts @@ -11,7 +11,7 @@ import { getAccessToken } from '../../../routes/v1/registryAuth.js' import { MirrorImportLogData, MirrorKind, MirrorKindKeys } from '../../../types/types.js' import config from '../../../utils/config.js' import { InternalError } from '../../../utils/error.js' -import { ImageManifestV2, OCIEmptyMediaType } from '../../../utils/registryResponses.js' +import { ImageManifestV2, ImageManifestV2Schema, OCIEmptyMediaType } from '../../../utils/registryResponses.js' import log from '../../log.js' import { splitDistributionPackageName } from '../../registry.js' import { BaseImporter, BaseMirrorMetadata } from './base.js' @@ -68,7 +68,7 @@ export class ImageImporter extends BaseImporter { if (ImageImporter.manifestRegex.test(entry.name)) { // manifest.json must be uploaded after the other layers otherwise the registry will error as the referenced layers won't yet exist log.debug({ ...this.logData }, 'Extracting un-tarred manifest.') - this.manifestBody = ImageManifestV2.parse(await json(stream)) + this.manifestBody = ImageManifestV2Schema.parse(await json(stream)) } else if (ImageImporter.blobRegex.test(entry.name)) { // convert filename to digest format const layerDigest = `${entry.name.replace(new RegExp(String.raw`^(${config.modelMirror.contentDirectory}/blobs\/sha256\/)`), 'sha256:')}` diff --git a/backend/src/utils/registryResponses.ts b/backend/src/utils/registryResponses.ts index ada24fde8b..66c9ca8168 100644 --- a/backend/src/utils/registryResponses.ts +++ b/backend/src/utils/registryResponses.ts @@ -8,28 +8,28 @@ import { InternalError } from './error.js' * so definitions must also be lower case. */ -const HeaderValue = z.string() +const HeaderValueSchema = z.string() // Common headers -export const CommonRegistryHeaders = z +export const CommonRegistryHeadersSchema = z .object({ - 'docker-distribution-api-version': HeaderValue.optional(), - 'content-type': HeaderValue.optional(), - 'content-length': HeaderValue.optional(), - date: HeaderValue.optional(), + 'docker-distribution-api-version': HeaderValueSchema.optional(), + 'content-type': HeaderValueSchema.optional(), + 'content-length': HeaderValueSchema.optional(), + date: HeaderValueSchema.optional(), }) .passthrough() -export type CommonRegistryHeaders = z.infer +export type CommonRegistryHeaders = z.infer // Error response -const RegistryErrorDetail = z.object({ +const RegistryErrorDetailSchema = z.object({ code: z.string(), message: z.string(), detail: z.unknown().optional(), }) -export const RegistryErrorResponseBody = z.object({ - errors: z.array(RegistryErrorDetail), +export const RegistryErrorResponseBodySchema = z.object({ + errors: z.array(RegistryErrorDetailSchema), }) -export type RegistryErrorResponseBody = z.infer +export type RegistryErrorResponseBody = z.infer export function parseRegistryResponse( schema: ZodSchema, @@ -42,7 +42,7 @@ export function parseRegistryResponse( } // fallback on error - const error = RegistryErrorResponseBody.safeParse(body) + const error = RegistryErrorResponseBodySchema.safeParse(body) if (error.success) { return { ok: false, error: error.data } } @@ -52,107 +52,107 @@ export function parseRegistryResponse( } // GET /v2/ -export const BaseApiCheckResponseHeaders = CommonRegistryHeaders.extend({ +export const BaseApiCheckResponseHeadersSchema = CommonRegistryHeadersSchema.extend({ 'docker-distribution-api-version': z .string() .regex(/^registry\/2\.0$/) .optional(), }) -export const BaseApiCheckResponseBody = z.record(z.never()) +export const BaseApiCheckResponseBodySchema = z.record(z.never()) // GET /v2//tags/list -export const TagsListResponseBody = z.object({ +export const TagsListResponseBodySchema = z.object({ name: z.string(), tags: z.array(z.string()).nullable(), }) -export const TagsListResponseHeaders = CommonRegistryHeaders.extend({ - link: HeaderValue.optional(), // pagination +export const TagsListResponseHeadersSchema = CommonRegistryHeadersSchema.extend({ + link: HeaderValueSchema.optional(), // pagination }) // GET /v2/_catalog -export const CatalogBodyResponse = z.object({ +export const CatalogBodyResponseSchema = z.object({ repositories: z.array(z.string()), }) -export const CatalogResponseHeaders = TagsListResponseHeaders +export const CatalogResponseHeadersSchema = TagsListResponseHeadersSchema // GET /v2//blobs/ -export const BlobResponseHeaders = CommonRegistryHeaders.extend({ - 'docker-content-digest': HeaderValue, - etag: HeaderValue.optional(), +export const BlobResponseHeadersSchema = CommonRegistryHeadersSchema.extend({ + 'docker-content-digest': HeaderValueSchema, + etag: HeaderValueSchema.optional(), }) // POST/PATCH/PUT blob upload -export const BlobUploadResponseHeaders = CommonRegistryHeaders.extend({ - location: HeaderValue.optional(), - range: HeaderValue.optional(), - 'docker-upload-uuid': HeaderValue.optional(), +export const BlobUploadResponseHeadersSchema = CommonRegistryHeadersSchema.extend({ + location: HeaderValueSchema.optional(), + range: HeaderValueSchema.optional(), + 'docker-upload-uuid': HeaderValueSchema.optional(), }) // DELETE /v2//manifests/ -export const DeleteManifestResponseHeaders = CommonRegistryHeaders.extend({ - 'docker-content-digest': HeaderValue.optional(), +export const DeleteManifestResponseHeadersSchema = CommonRegistryHeadersSchema.extend({ + 'docker-content-digest': HeaderValueSchema.optional(), }) // GET /v2//manifests/ export const DockerManifestMediaType = 'application/vnd.docker.distribution.manifest.v2+json' export const OCIManifestMediaType = 'application/vnd.oci.image.manifest.v1+json' export const OCIEmptyMediaType = 'application/vnd.oci.empty.v1+json' -export const ManifestMediaType = z.enum([DockerManifestMediaType, OCIManifestMediaType]) -export const AcceptManifestMediaTypeHeaderValue = ManifestMediaType.options.join(',') -export const ManifestListMediaType = z.enum([ +export const ManifestMediaTypeSchema = z.enum([DockerManifestMediaType, OCIManifestMediaType]) +export const AcceptManifestMediaTypeHeaderValue = ManifestMediaTypeSchema.options.join(',') +export const ManifestListMediaTypeSchema = z.enum([ 'application/vnd.docker.distribution.manifest.list.v2+json', 'application/vnd.oci.image.index.v1+json', ]) -const BaseDescriptor = z.object({ +const BaseDescriptorSchema = z.object({ mediaType: z.string(), size: z.number().int().nonnegative(), digest: z.string(), }) -const DockerDescriptor = BaseDescriptor.extend({ +const DockerDescriptorSchema = BaseDescriptorSchema.extend({ urls: z.array(z.string()).optional(), }) -const OCIAnnotations = z.record(z.string(), z.string()) -const OCIDescriptor = BaseDescriptor.extend({ +const OCIAnnotationsSchema = z.record(z.string(), z.string()) +const OCIDescriptorSchema = BaseDescriptorSchema.extend({ urls: z.array(z.string()).optional(), - annotations: OCIAnnotations.optional(), + annotations: OCIAnnotationsSchema.optional(), data: z.string().optional(), artifactType: z.string().optional(), }) -export const Descriptors = z.union([BaseDescriptor, DockerDescriptor, OCIDescriptor]) -export type Descriptors = z.infer +export const DescriptorsSchema = z.union([BaseDescriptorSchema, DockerDescriptorSchema, OCIDescriptorSchema]) +export type Descriptors = z.infer -const DockerImageManifestV2 = z.object({ +const DockerImageManifestV2Schema = z.object({ schemaVersion: z.literal(2), mediaType: z.literal(DockerManifestMediaType), - config: BaseDescriptor, - layers: z.array(DockerDescriptor), + config: BaseDescriptorSchema, + layers: z.array(DockerDescriptorSchema), }) // helper for conditional setting -const OCIImageBaseManifestV2 = z.object({ +const OCIImageBaseManifestV2Schema = z.object({ schemaVersion: z.literal(2), mediaType: z.literal(OCIManifestMediaType).optional(), artifactType: z.string().optional(), - config: OCIDescriptor, - layers: z.array(OCIDescriptor), - subject: OCIDescriptor.optional(), - annotations: OCIAnnotations.optional(), + config: OCIDescriptorSchema, + layers: z.array(OCIDescriptorSchema), + subject: OCIDescriptorSchema.optional(), + annotations: OCIAnnotationsSchema.optional(), }) -const OCIImageManifestV2 = z.discriminatedUnion('mediaType', [ - OCIImageBaseManifestV2.extend({ +const OCIImageManifestV2Schema = z.discriminatedUnion('mediaType', [ + OCIImageBaseManifestV2Schema.extend({ mediaType: z.literal(OCIManifestMediaType), }), - OCIImageBaseManifestV2.extend({ + OCIImageBaseManifestV2Schema.extend({ mediaType: z.literal(OCIEmptyMediaType), artifactType: z.string(), }), ]) -export const ImageManifestV2 = z.union([DockerImageManifestV2, OCIImageManifestV2]) -export type ImageManifestV2 = z.infer +export const ImageManifestV2Schema = z.union([DockerImageManifestV2Schema, OCIImageManifestV2Schema]) +export type ImageManifestV2 = z.infer -const ManifestListDescriptor = BaseDescriptor.extend({ +const ManifestListDescriptorSchema = BaseDescriptorSchema.extend({ platform: z .object({ architecture: z.string(), @@ -164,15 +164,15 @@ const ManifestListDescriptor = BaseDescriptor.extend({ .optional(), }) -export const ManifestListV2 = z.object({ +export const ManifestListV2Schema = z.object({ schemaVersion: z.literal(2), - mediaType: ManifestListMediaType.optional(), - manifests: z.array(ManifestListDescriptor), + mediaType: ManifestListMediaTypeSchema.optional(), + manifests: z.array(ManifestListDescriptorSchema), }) // TODO: handle multi-platform images -export const ManifestResponseBody = z.union([ImageManifestV2, ManifestListV2]) +export const ManifestResponseBodySchema = z.union([ImageManifestV2Schema, ManifestListV2Schema]) -export const ManifestResponseHeaders = CommonRegistryHeaders.extend({ - 'docker-content-digest': HeaderValue, - etag: HeaderValue.optional(), +export const ManifestResponseHeadersSchema = CommonRegistryHeadersSchema.extend({ + 'docker-content-digest': HeaderValueSchema, + etag: HeaderValueSchema.optional(), }) diff --git a/backend/test/utils/registryResponses.spec.ts b/backend/test/utils/registryResponses.spec.ts index dd9698fc7a..3ae8d05a3d 100644 --- a/backend/test/utils/registryResponses.spec.ts +++ b/backend/test/utils/registryResponses.spec.ts @@ -1,15 +1,15 @@ import { describe, expect, test } from 'vitest' import { - BaseApiCheckResponseBody, - BaseApiCheckResponseHeaders, + BaseApiCheckResponseBodySchema, + BaseApiCheckResponseHeadersSchema, parseRegistryResponse, } from '../../src/utils/registryResponses.js' describe('clients > registryResponses', () => { test('parseRegistryResponse > success', () => { const header = { 'docker-distribution-api-version': 'registry/2.0' } - const result = parseRegistryResponse(BaseApiCheckResponseHeaders, header) + const result = parseRegistryResponse(BaseApiCheckResponseHeadersSchema, header) expect(result).toEqual({ ok: true, data: header }) }) @@ -18,14 +18,14 @@ describe('clients > registryResponses', () => { const header = { errors: [{ code: '404', message: 'Not Found' }], } - const result = parseRegistryResponse(BaseApiCheckResponseBody, header) + const result = parseRegistryResponse(BaseApiCheckResponseBodySchema, header) expect(result).toEqual({ ok: false, error: header }) }) test('parseRegistryResponse > unhandled error', () => { const header = { foo: 'bar' } - expect(() => parseRegistryResponse(BaseApiCheckResponseBody, header)).toThrowError( + expect(() => parseRegistryResponse(BaseApiCheckResponseBodySchema, header)).toThrowError( /^Response did not match expected schema or RegistryErrorResponse./, ) })