Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 4 additions & 7 deletions backend/src/clients/artefactScan.ts
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ export const ModelScanResponseSchema = z.object({
}),
),
})
export type ModelScanResponseSchema = z.infer<typeof ModelScanResponseSchema>
export type ModelScanResponse = z.infer<typeof ModelScanResponseSchema>

// There's no formal definition of the JSON schema so this must be permissive
const ImageHistorySchema = z
Expand Down Expand Up @@ -189,7 +189,7 @@ export const TrivyScanResultResponseSchema = z
Results: z.array(ResultSchema).optional(),
})
.passthrough()
export type TrivyScanResultResponseSchema = z.infer<typeof TrivyScanResultResponseSchema>
export type TrivyScanResultResponse = z.infer<typeof TrivyScanResultResponseSchema>

async function getArtefactScanInfo() {
const url = `${config.artefactScanning.artefactscan.protocol}://${config.artefactScanning.artefactscan.host}:${config.artefactScanning.artefactscan.port}`
Expand Down Expand Up @@ -239,14 +239,11 @@ async function scanStream(stream: Readable, fileName: string, endpoint: 'file' |
return await res.json()
}

export async function scanFileStream(stream: Readable, fileName: string): Promise<ModelScanResponseSchema> {
export async function scanFileStream(stream: Readable, fileName: string): Promise<ModelScanResponse> {
return ModelScanResponseSchema.parse(await scanStream(stream, fileName, 'file'))
}

export async function scanImageBlobStream(
stream: Readable,
blobDigest: string,
): Promise<TrivyScanResultResponseSchema> {
export async function scanImageBlobStream(stream: Readable, blobDigest: string): Promise<TrivyScanResultResponse> {
return TrivyScanResultResponseSchema.parse(await scanStream(stream, blobDigest, 'image'))
}

Expand Down
69 changes: 35 additions & 34 deletions backend/src/clients/registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,21 +11,22 @@ import config from '../utils/config.js'
import { InternalError, RegistryError } from '../utils/error.js'
import {
AcceptManifestMediaTypeHeaderValue,
BaseApiCheckResponseBody,
BaseApiCheckResponseHeaders,
BlobResponseHeaders,
BlobUploadResponseHeaders,
CatalogBodyResponse,
CatalogResponseHeaders,
BaseApiCheckResponseBodySchema,
BaseApiCheckResponseHeadersSchema,
BlobResponseHeadersSchema,
BlobUploadResponseHeadersSchema,
CatalogBodyResponseSchema,
CatalogResponseHeadersSchema,
CommonRegistryHeaders,
DeleteManifestResponseHeaders,
ImageManifestV2,
ManifestListMediaType,
ManifestMediaType,
ManifestResponseHeaders,
RegistryErrorResponseBody,
TagsListResponseBody,
TagsListResponseHeaders,
CommonRegistryHeadersSchema,
DeleteManifestResponseHeadersSchema,
ImageManifestV2Schema,
ManifestListMediaTypeSchema,
ManifestMediaTypeSchema,
ManifestResponseHeadersSchema,
RegistryErrorResponseBodySchema,
TagsListResponseBodySchema,
TagsListResponseHeadersSchema,
} from '../utils/registryResponses.js'

const registry = config.registry.connection.internal
Expand Down Expand Up @@ -103,7 +104,7 @@ async function registryRequest<TBody = unknown, THeaders = CommonRegistryHeaders
endpoint: string,
{
bodySchema,
headersSchema = CommonRegistryHeaders as unknown as ZodSchema<THeaders>,
headersSchema = CommonRegistryHeadersSchema as unknown as ZodSchema<THeaders>,
expectStream = false,
extraFetchOptions = {},
extraHeaders = {},
Expand Down Expand Up @@ -154,8 +155,8 @@ async function registryRequest<TBody = unknown, THeaders = CommonRegistryHeaders
// check response
if (!res.ok) {
controller.abort()
if (rawBody && RegistryErrorResponseBody.safeParse(rawBody).success) {
throw RegistryError(RegistryErrorResponseBody.parse(rawBody), context)
if (rawBody && RegistryErrorResponseBodySchema.safeParse(rawBody).success) {
throw RegistryError(RegistryErrorResponseBodySchema.parse(rawBody), context)
}

throw InternalError('Unrecognised registry error response.', {
Expand Down Expand Up @@ -230,17 +231,17 @@ async function registryRequest<TBody = unknown, THeaders = CommonRegistryHeaders

export async function getApiVersion(token: string) {
const result = await registryRequest(token, '', {
bodySchema: BaseApiCheckResponseBody,
headersSchema: BaseApiCheckResponseHeaders,
bodySchema: BaseApiCheckResponseBodySchema,
headersSchema: BaseApiCheckResponseHeadersSchema,
})

return result.headers['docker-distribution-api-version']
}

export async function listModelRepos(token: string, modelId: string): Promise<string[]> {
const result = await registryRequest(token, '_catalog?n=100', {
bodySchema: CatalogBodyResponse,
headersSchema: CatalogResponseHeaders,
bodySchema: CatalogBodyResponseSchema,
headersSchema: CatalogResponseHeadersSchema,
pagination: {
enabled: true,
aggregate: (acc, next) => ({
Expand All @@ -256,8 +257,8 @@ export async function listModelRepos(token: string, modelId: string): Promise<st
export async function listImageTags(token: string, repoRef: RepoRefInterface) {
try {
const result = await registryRequest(token, `${repoRef.repository}/${repoRef.name}/tags/list`, {
bodySchema: TagsListResponseBody,
headersSchema: TagsListResponseHeaders,
bodySchema: TagsListResponseBodySchema,
headersSchema: TagsListResponseHeadersSchema,
pagination: {
enabled: true,
aggregate: (acc, next) => ({
Expand All @@ -279,7 +280,7 @@ export async function listImageTags(token: string, repoRef: RepoRefInterface) {
export async function isImageTagManifestList(token: string, imageRef: ImageRefInterface): Promise<boolean> {
const result = await registryRequest(token, `${imageRef.repository}/${imageRef.name}/manifests/${imageRef.tag}`, {
// do not validate the body here as we only care about Content-Type
headersSchema: ManifestResponseHeaders,
headersSchema: ManifestResponseHeadersSchema,
extraHeaders: {
Accept: AcceptManifestMediaTypeHeaderValue,
},
Expand All @@ -294,10 +295,10 @@ export async function isImageTagManifestList(token: string, imageRef: ImageRefIn
})
}

if ((ManifestListMediaType.options as string[]).includes(contentType)) {
if ((ManifestListMediaTypeSchema.options as string[]).includes(contentType)) {
return true
}
if ((ManifestMediaType.options as string[]).includes(contentType)) {
if ((ManifestMediaTypeSchema.options as string[]).includes(contentType)) {
return false
}

Expand All @@ -310,8 +311,8 @@ export async function isImageTagManifestList(token: string, imageRef: ImageRefIn
export async function getImageTagManifest(token: string, imageRef: ImageRefInterface) {
// TODO: handle multi-platform images
const result = await registryRequest(token, `${imageRef.repository}/${imageRef.name}/manifests/${imageRef.tag}`, {
bodySchema: ImageManifestV2,
headersSchema: ManifestResponseHeaders,
bodySchema: ImageManifestV2Schema,
headersSchema: ManifestResponseHeadersSchema,
extraHeaders: {
Accept: AcceptManifestMediaTypeHeaderValue,
},
Expand All @@ -326,7 +327,7 @@ export async function getRegistryLayerStream(
layerDigest: string,
): Promise<{ stream: Readable; abort: () => void }> {
const result = await registryRequest(token, `${repoRef.repository}/${repoRef.name}/blobs/${layerDigest}`, {
headersSchema: BlobResponseHeaders,
headersSchema: BlobResponseHeadersSchema,
expectStream: true,
extraHeaders: {
Accept: AcceptManifestMediaTypeHeaderValue,
Expand Down Expand Up @@ -366,7 +367,7 @@ export async function doesLayerExist(token: string, repoRef: RepoRefInterface, d

export async function initialiseUpload(token: string, repoRef: RepoRefInterface) {
const result = await registryRequest(token, `${repoRef.repository}/${repoRef.name}/blobs/uploads/`, {
headersSchema: BlobUploadResponseHeaders,
headersSchema: BlobUploadResponseHeadersSchema,
expectStream: true,
extraFetchOptions: {
method: 'POST',
Expand All @@ -378,7 +379,7 @@ export async function initialiseUpload(token: string, repoRef: RepoRefInterface)

export async function putManifest(token: string, imageRef: ImageRefInterface, manifest: BodyInit, contentType: string) {
const result = await registryRequest(token, `${imageRef.repository}/${imageRef.name}/manifests/${imageRef.tag}`, {
headersSchema: ManifestResponseHeaders,
headersSchema: ManifestResponseHeadersSchema,
expectStream: true,
extraFetchOptions: {
method: 'PUT',
Expand All @@ -402,7 +403,7 @@ export async function uploadLayerMonolithic(
size: string,
) {
const result = await registryRequest(token, `${uploadURL}&digest=${digest}`.replace(/^(\/v2\/)/, ''), {
headersSchema: BlobUploadResponseHeaders,
headersSchema: BlobUploadResponseHeadersSchema,
expectStream: true,
extraFetchOptions: {
method: 'PUT',
Expand Down Expand Up @@ -430,7 +431,7 @@ export async function mountBlob(
token,
`${destinationRepoRef.repository}/${destinationRepoRef.name}/blobs/uploads/?from=${sourceRepoRef.repository}/${sourceRepoRef.name}&mount=${blobDigest}`,
{
headersSchema: BlobUploadResponseHeaders,
headersSchema: BlobUploadResponseHeadersSchema,
extraFetchOptions: {
method: 'POST',
},
Expand All @@ -443,7 +444,7 @@ export async function mountBlob(

export async function deleteManifest(token: string, imageRef: ImageRefInterface) {
const result = await registryRequest(token, `${imageRef.repository}/${imageRef.name}/manifests/${imageRef.tag}`, {
headersSchema: DeleteManifestResponseHeaders,
headersSchema: DeleteManifestResponseHeadersSchema,
expectStream: true,
extraFetchOptions: {
method: 'DELETE',
Expand Down
4 changes: 2 additions & 2 deletions backend/src/models/Scan.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { model, type ObjectId, Schema } from 'mongoose'

import type { ModelScanResponseSchema, TrivyScanResultResponseSchema } from '../clients/artefactScan.js'
import type { ModelScanResponse, TrivyScanResultResponse } from '../clients/artefactScan.js'
import { ArtefactScanState, type ArtefactScanStateKeys } from '../connectors/artefactScanning/Base.js'
import { type SoftDeleteDocument, softDeletionPlugin } from './plugins/softDeletePlugin.js'

Expand All @@ -11,7 +11,7 @@ export type ScanInterface = {
scannerVersion?: string
state: ArtefactScanStateKeys
summary?: ScanSummary
additionalInfo?: TrivyScanResultResponseSchema | ModelScanResponseSchema
additionalInfo?: TrivyScanResultResponse | ModelScanResponse

lastRunAt: Date

Expand Down
4 changes: 2 additions & 2 deletions backend/src/services/mirroredModel/importers/image.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { getAccessToken } from '../../../routes/v1/registryAuth.js'
import { MirrorImportLogData, MirrorKind, MirrorKindKeys } from '../../../types/types.js'
import config from '../../../utils/config.js'
import { InternalError } from '../../../utils/error.js'
import { ImageManifestV2, OCIEmptyMediaType } from '../../../utils/registryResponses.js'
import { ImageManifestV2, ImageManifestV2Schema, OCIEmptyMediaType } from '../../../utils/registryResponses.js'
import log from '../../log.js'
import { splitDistributionPackageName } from '../../registry.js'
import { BaseImporter, BaseMirrorMetadata } from './base.js'
Expand Down Expand Up @@ -68,7 +68,7 @@ export class ImageImporter extends BaseImporter {
if (ImageImporter.manifestRegex.test(entry.name)) {
// manifest.json must be uploaded after the other layers otherwise the registry will error as the referenced layers won't yet exist
log.debug({ ...this.logData }, 'Extracting un-tarred manifest.')
this.manifestBody = ImageManifestV2.parse(await json(stream))
this.manifestBody = ImageManifestV2Schema.parse(await json(stream))
} else if (ImageImporter.blobRegex.test(entry.name)) {
// convert filename to digest format
const layerDigest = `${entry.name.replace(new RegExp(String.raw`^(${config.modelMirror.contentDirectory}/blobs\/sha256\/)`), 'sha256:')}`
Expand Down
Loading
Loading