fix: proxy cache serve local on remote not found (#22153)

AYDEV-FR · Vad1mo · Copilot · web-flow · commit 3d45be3d03f2 · 2026-03-10T07:02:06.000Z
* fix: proxy cache serve local on remote not found

Signed-off-by: AYDEV-FR &lt;aymeric.deliencourt@aydev.fr&gt;

* tests: fix proxy controller test

Signed-off-by: AYDEV-FR &lt;aymeric.deliencourt@aydev.fr&gt;

* Update src/controller/proxy/controller.go

Co-authored-by: Copilot &lt;175728472+Copilot@users.noreply.github.com&gt;
Signed-off-by: Vadim Bauer &lt;Bauer.vadim@gmail.com&gt;

* feat: add ui

Signed-off-by: AYDEV-FR &lt;aymeric.deliencourt@aydev.fr&gt;

* chore: add checkbox in creation project modal

Signed-off-by: AYDEV-FR &lt;aymeric.deliencourt@aydev.fr&gt;

* chore: fix tooltip text

Signed-off-by: AYDEV-FR &lt;aymeric.deliencourt@aydev.fr&gt;

---------

Signed-off-by: AYDEV-FR &lt;aymeric.deliencourt@aydev.fr&gt;
Signed-off-by: Vadim Bauer &lt;Bauer.vadim@gmail.com&gt;
Co-authored-by: Vadim Bauer &lt;Bauer.vadim@gmail.com&gt;
Co-authored-by: Copilot &lt;175728472+Copilot@users.noreply.github.com&gt;
diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml
@@ -7336,6 +7336,10 @@ definitions:
         type: string
         description: 'The max connection per artifact to the upstream registry in current proxy cache project, if it is -1, no limit to upstream registry connections'
         x-nullable: true
+      proxy_cache_local_on_not_found:
+        type: string
+        description: 'Whether to serve images from local cache when they are removed from the upstream registry. The valid values are "true", "false".'
+        x-nullable: true
   ProjectSummary:
     type: object
     properties:
diff --git a/src/controller/proxy/controller.go b/src/controller/proxy/controller.go
@@ -58,7 +58,7 @@ type Controller interface {
 	// UseLocalBlob check if the blob should use local copy
 	UseLocalBlob(ctx context.Context, art lib.ArtifactInfo) bool
 	// UseLocalManifest check manifest should use local copy
-	UseLocalManifest(ctx context.Context, art lib.ArtifactInfo, remote RemoteInterface) (bool, *ManifestList, error)
+	UseLocalManifest(ctx context.Context, art lib.ArtifactInfo, remote RemoteInterface, p *proModels.Project) (bool, *ManifestList, error)
 	// ProxyBlob proxy the blob request to the remote server, p is the proxy project
 	// art is the ArtifactInfo which includes the digest of the blob
 	ProxyBlob(ctx context.Context, p *proModels.Project, art lib.ArtifactInfo) (int64, io.ReadCloser, error)
@@ -169,7 +169,7 @@ func (c *controller) getManifestDigestInLocal(ctx context.Context, art lib.Artif
 // the return error should be nil when it is not found in local and need to delegate to remote registry
 // the return error should be NotFoundError when it is not found in remote registry
 // the error will be captured by framework and return 404 to client
-func (c *controller) UseLocalManifest(ctx context.Context, art lib.ArtifactInfo, remote RemoteInterface) (bool, *ManifestList, error) {
+func (c *controller) UseLocalManifest(ctx context.Context, art lib.ArtifactInfo, remote RemoteInterface, p *proModels.Project) (bool, *ManifestList, error) {
 	a, err := c.local.GetManifest(ctx, art)
 	if err != nil {
 		return false, nil, err
@@ -188,15 +188,10 @@ func (c *controller) UseLocalManifest(ctx context.Context, art lib.ArtifactInfo,
 		return false, nil, err
 	}
 	if !exist || desc == nil {
-		dig, err := c.getManifestDigestInLocal(ctx, art)
-		if err != nil {
-			// skip to delete when error, use debug level log to avoid too many logs when the manifest is removed from upstream
-			log.Debugf("failed to get manifest digest in local, error: %v, skip to delete it, art %+v", err, art)
-		} else {
-			go func() {
-				c.local.DeleteManifest(art.Repository, dig)
-				log.Infof("delete manifest %s with digest %s", art.Repository, dig)
-			}()
+		// if not found in remote and local artifact exists, check if we should serve from local cache
+		if a != nil && p != nil && p.ProxyCacheLocalOnNotFound() {
+			log.Infof("Artifact not found in remote registry but exists in local cache, serving from local: %v:%v", art.Repository, art.Tag)
+			return true, nil, nil
 		}
 		return false, nil, errors.NotFoundError(fmt.Errorf("repo %v, tag %v not found", art.Repository, art.Tag))
 	}
diff --git a/src/controller/proxy/controller_test.go b/src/controller/proxy/controller_test.go
@@ -106,7 +106,7 @@ func (p *proxyControllerTestSuite) TestUseLocalManifest_True() {
 	art := lib.ArtifactInfo{Repository: "library/hello-world", Digest: dig}
 	p.local.On("GetManifest", mock.Anything, mock.Anything).Return(&artifact.Artifact{}, nil)
 
-	result, _, err := p.ctr.UseLocalManifest(ctx, art, p.remote)
+	result, _, err := p.ctr.UseLocalManifest(ctx, art, p.remote, p.proj)
 	p.Assert().Nil(err)
 	p.Assert().True(result)
 }
@@ -118,7 +118,7 @@ func (p *proxyControllerTestSuite) TestUseLocalManifest_False() {
 	art := lib.ArtifactInfo{Repository: "library/hello-world", Digest: dig}
 	p.remote.On("ManifestExist", mock.Anything, mock.Anything).Return(true, desc, nil)
 	p.local.On("GetManifest", mock.Anything, mock.Anything).Return(nil, nil)
-	result, _, err := p.ctr.UseLocalManifest(ctx, art, p.remote)
+	result, _, err := p.ctr.UseLocalManifest(ctx, art, p.remote, p.proj)
 	p.Assert().Nil(err)
 	p.Assert().False(result)
 }
@@ -130,7 +130,7 @@ func (p *proxyControllerTestSuite) TestUseLocalManifest_429() {
 	art := lib.ArtifactInfo{Repository: "library/hello-world", Digest: dig}
 	p.remote.On("ManifestExist", mock.Anything, mock.Anything).Return(false, desc, errors.New("too many requests").WithCode(errors.RateLimitCode))
 	p.local.On("GetManifest", mock.Anything, mock.Anything).Return(nil, nil)
-	_, _, err := p.ctr.UseLocalManifest(ctx, art, p.remote)
+	_, _, err := p.ctr.UseLocalManifest(ctx, art, p.remote, p.proj)
 	p.Assert().NotNil(err)
 	errors.IsRateLimitError(err)
 }
@@ -142,19 +142,40 @@ func (p *proxyControllerTestSuite) TestUseLocalManifest_429ToLocal() {
 	art := lib.ArtifactInfo{Repository: "library/hello-world", Digest: dig}
 	p.remote.On("ManifestExist", mock.Anything, mock.Anything).Return(false, desc, errors.New("too many requests").WithCode(errors.RateLimitCode))
 	p.local.On("GetManifest", mock.Anything, mock.Anything).Return(&artifact.Artifact{}, nil)
-	result, _, err := p.ctr.UseLocalManifest(ctx, art, p.remote)
+	result, _, err := p.ctr.UseLocalManifest(ctx, art, p.remote, p.proj)
 	p.Assert().Nil(err)
 	p.Assert().True(result)
 }
 
-func (p *proxyControllerTestSuite) TestUseLocalManifestWithTag_False() {
+func (p *proxyControllerTestSuite) TestUseLocalManifestWithTag_True() {
 	ctx := context.Background()
 	art := lib.ArtifactInfo{Repository: "library/hello-world", Tag: "latest"}
 	desc := &distribution.Descriptor{}
+	// Set up project with ProxyCacheLocalOnNotFound enabled
+	proj := &proModels.Project{
+		RegistryID: 1,
+		Metadata:   map[string]string{proModels.ProMetaProxyCacheLocalOnNotFound: "true"},
+	}
+	p.local.On("GetManifest", mock.Anything, mock.Anything).Return(&artifact.Artifact{}, nil)
+	p.remote.On("ManifestExist", mock.Anything, mock.Anything).Return(false, desc, nil)
+	result, _, err := p.ctr.UseLocalManifest(ctx, art, p.remote, proj)
+	p.Assert().Nil(err)
+	p.Assert().True(result)
+}
+
+func (p *proxyControllerTestSuite) TestUseLocalManifestWithTag_NotFoundWhenOptionDisabled() {
+	ctx := context.Background()
+	art := lib.ArtifactInfo{Repository: "library/hello-world", Tag: "latest"}
+	desc := &distribution.Descriptor{}
+	// Set up project without ProxyCacheLocalOnNotFound enabled
+	proj := &proModels.Project{
+		RegistryID: 1,
+		Metadata:   map[string]string{},
+	}
 	p.local.On("GetManifest", mock.Anything, mock.Anything).Return(&artifact.Artifact{}, nil)
 	p.remote.On("ManifestExist", mock.Anything, mock.Anything).Return(false, desc, nil)
-	result, _, err := p.ctr.UseLocalManifest(ctx, art, p.remote)
-	p.Assert().True(errors.IsNotFoundErr(err))
+	result, _, err := p.ctr.UseLocalManifest(ctx, art, p.remote, proj)
+	p.Assert().NotNil(err)
 	p.Assert().False(result)
 }
 
diff --git a/src/pkg/project/models/pro_meta.go b/src/pkg/project/models/pro_meta.go
@@ -16,14 +16,15 @@ package models
 
 // keys of project metadata and severity values
 const (
-	ProMetaPublic                   = "public"
-	ProMetaEnableContentTrust       = "enable_content_trust"
-	ProMetaEnableContentTrustCosign = "enable_content_trust_cosign"
-	ProMetaPreventVul               = "prevent_vul" // prevent vulnerable images from being pulled
-	ProMetaSeverity                 = "severity"
-	ProMetaAutoScan                 = "auto_scan"
-	ProMetaReuseSysCVEAllowlist     = "reuse_sys_cve_allowlist"
-	ProMetaAutoSBOMGen              = "auto_sbom_generation"
-	ProMetaProxySpeed               = "proxy_speed_kb"
-	ProMetaMaxUpstreamConn          = "max_upstream_conn"
+	ProMetaPublic                    = "public"
+	ProMetaEnableContentTrust        = "enable_content_trust"
+	ProMetaEnableContentTrustCosign  = "enable_content_trust_cosign"
+	ProMetaPreventVul                = "prevent_vul" // prevent vulnerable images from being pulled
+	ProMetaSeverity                  = "severity"
+	ProMetaAutoScan                  = "auto_scan"
+	ProMetaReuseSysCVEAllowlist      = "reuse_sys_cve_allowlist"
+	ProMetaAutoSBOMGen               = "auto_sbom_generation"
+	ProMetaProxySpeed                = "proxy_speed_kb"
+	ProMetaMaxUpstreamConn           = "max_upstream_conn"
+	ProMetaProxyCacheLocalOnNotFound = "proxy_cache_local_on_not_found"
 )
diff --git a/src/pkg/project/models/project.go b/src/pkg/project/models/project.go
@@ -184,6 +184,15 @@ func (p *Project) MaxUpstreamConnection() int {
 	return int(cnt)
 }
 
+// ProxyCacheLocalOnNotFound returns true if images should be served from local cache when removed from upstream
+func (p *Project) ProxyCacheLocalOnNotFound() bool {
+	val, exist := p.GetMetadata(ProMetaProxyCacheLocalOnNotFound)
+	if !exist {
+		return false
+	}
+	return isTrue(val)
+}
+
 // FilterByPublic returns orm.QuerySeter with public filter
 func (p *Project) FilterByPublic(_ context.Context, qs orm.QuerySeter, _ string, value any) orm.QuerySeter {
 	subQuery := `SELECT project_id FROM project_metadata WHERE name = 'public' AND value = '%s'`
diff --git a/src/portal/src/app/base/left-side-nav/projects/create-project/create-project.component.html b/src/portal/src/app/base/left-side-nav/projects/create-project/create-project.component.html
@@ -330,6 +330,39 @@ <h3 class="modal-title">{{ 'PROJECT.NEW_PROJECT' | translate }}</h3>
                     </clr-control-error>
                 </div>
             </div>
+            <clr-checkbox-container
+                [style.visibility]="
+                    isSystemAdmin && enableProxyCache ? 'visible' : 'hidden'
+                ">
+                <label class="clr-control-label">
+                    {{ 'PROJECT.PROXY_CACHE_LOCAL_ON_NOT_FOUND' | translate }}
+                    <clr-tooltip>
+                        <clr-icon
+                            clrTooltipTrigger
+                            shape="info-circle"
+                            size="24"></clr-icon>
+                        <clr-tooltip-content
+                            clrPosition="bottom-left"
+                            clrSize="lg"
+                            *clrIfOpen>
+                            <span>{{
+                                'PROJECT.PROXY_CACHE_LOCAL_ON_NOT_FOUND_TOOLTIP'
+                                    | translate
+                            }}</span>
+                        </clr-tooltip-content>
+                    </clr-tooltip>
+                </label>
+                <clr-checkbox-wrapper>
+                    <input
+                        clrCheckbox
+                        type="checkbox"
+                        id="proxy_cache_local_on_not_found"
+                        [(ngModel)]="
+                            project.metadata.proxy_cache_local_on_not_found
+                        "
+                        name="proxy_cache_local_on_not_found" />
+                </clr-checkbox-wrapper>
+            </clr-checkbox-container>
         </form>
     </div>
     <div class="modal-footer">
diff --git a/src/portal/src/app/base/left-side-nav/projects/create-project/create-project.component.ts b/src/portal/src/app/base/left-side-nav/projects/create-project/create-project.component.ts
@@ -395,6 +395,10 @@ export class CreateProjectComponent
                             this.project.metadata.bandwidth.toString(),
                         max_upstream_conn:
                             this.project.metadata.max_upstream_conn.toString(),
+                        proxy_cache_local_on_not_found: this.project.metadata
+                            .proxy_cache_local_on_not_found
+                            ? 'true'
+                            : 'false',
                     },
                     storage_limit: +storageByte,
                     registry_id: registryId,
@@ -440,6 +444,7 @@ export class CreateProjectComponent
         this.selectedSpeedLimitUnit = BandwidthUnit.KB;
         this.speedLimit = -1;
         this.project.metadata.max_upstream_conn = -1;
+        this.project.metadata.proxy_cache_local_on_not_found = false;
     }
 
     public get isValid(): boolean {
diff --git a/src/portal/src/app/base/project/project-config/project-policy-config/project-policy-config.component.html b/src/portal/src/app/base/project/project-config/project-policy-config/project-policy-config.component.html
@@ -122,6 +122,27 @@
                 </div>
             </div>
         </div>
+        <clr-checkbox-container
+            *ngIf="isSystemAdmin && projectPolicy.ProxyCacheEnabled">
+            <label class="clr-control-label"></label>
+            <clr-checkbox-wrapper id="proxy-cache-local-on-not-found">
+                <input
+                    type="checkbox"
+                    id="proxy-cache-local-on-not-found-input"
+                    clrCheckbox
+                    [(ngModel)]="projectPolicy.ProxyCacheLocalOnNotFound"
+                    name="proxy-cache-local-on-not-found"
+                    [disabled]="!hasChangeConfigRole" />
+                <label for="proxy-cache-local-on-not-found-input">
+                    {{ 'PROJECT.PROXY_CACHE_LOCAL_ON_NOT_FOUND' | translate }}
+                </label>
+            </clr-checkbox-wrapper>
+            <clr-control-helper class="config-subtext">
+                {{
+                    'PROJECT.PROXY_CACHE_LOCAL_ON_NOT_FOUND_TOOLTIP' | translate
+                }}
+            </clr-control-helper>
+        </clr-checkbox-container>
 
         <clr-checkbox-container *ngIf="!isProxyCacheProject" clrInline>
             <label
diff --git a/src/portal/src/app/base/project/project-config/project-policy-config/project-policy-config.component.ts b/src/portal/src/app/base/project/project-config/project-policy-config/project-policy-config.component.ts
@@ -59,6 +59,7 @@ export class ProjectPolicy {
     RegistryId?: number | null;
     ProxySpeedKb?: number | null;
     MaxUpstreamConn?: number | null;
+    ProxyCacheLocalOnNotFound?: boolean;
 
     constructor() {
         this.Public = false;
@@ -72,6 +73,7 @@ export class ProjectPolicy {
         this.RegistryId = null;
         this.ProxySpeedKb = -1;
         this.MaxUpstreamConn = -1;
+        this.ProxyCacheLocalOnNotFound = false;
     }
 
     initByProject(pro: Project) {
@@ -93,6 +95,8 @@ export class ProjectPolicy {
         this.MaxUpstreamConn = pro.metadata.max_upstream_conn
             ? pro.metadata.max_upstream_conn
             : -1;
+        this.ProxyCacheLocalOnNotFound =
+            pro.metadata.proxy_cache_local_on_not_found === 'true';
     }
 }
 const PAGE_SIZE: number = 100;
diff --git a/src/portal/src/app/base/project/project-config/project-policy-config/project.ts b/src/portal/src/app/base/project/project-config/project-policy-config/project.ts
@@ -37,6 +37,7 @@ export class Project {
         reuse_sys_cve_allowlist?: string;
         proxy_speed_kb?: number | null;
         max_upstream_conn?: number | null;
+        proxy_cache_local_on_not_found?: string | boolean;
     };
     cve_allowlist?: object;
     constructor() {
diff --git a/src/portal/src/app/base/project/project.ts b/src/portal/src/app/base/project/project.ts
@@ -37,6 +37,7 @@ export class Project {
         retention_id: number;
         bandwidth: number;
         max_upstream_conn: number;
+        proxy_cache_local_on_not_found?: string | boolean;
     };
     constructor() {
         this.metadata = <any>{};
diff --git a/src/portal/src/app/shared/services/project.service.ts b/src/portal/src/app/shared/services/project.service.ts
@@ -177,6 +177,10 @@ export class ProjectDefaultService extends ProjectService {
                             : 'false',
                         reuse_sys_cve_allowlist: reuseSysCVEVAllowlist,
                         proxy_speed_kb: projectPolicy.ProxySpeedKb.toString(),
+                        proxy_cache_local_on_not_found:
+                            projectPolicy.ProxyCacheLocalOnNotFound
+                                ? 'true'
+                                : 'false',
                     },
                     cve_allowlist: projectAllowlist,
                 },
diff --git a/src/portal/src/i18n/lang/en-us-lang.json b/src/portal/src/i18n/lang/en-us-lang.json
@@ -245,6 +245,8 @@
         "CONNECTION_LIMIT": "Max connection to upstream registry",
         "PROXY_CACHE_MAX_UPSTREAM_CONN_TIP": "The max connection to the upstream registry for this proxy cache project, if -1, then there is no limit",
         "PROXY_CACHE_MAX_UPSTREAM_CONN_INPUT_TIP": "Please enter -1 or an integer greater than 0. ",
+        "PROXY_CACHE_LOCAL_ON_NOT_FOUND": "Serve stale content when upstream is unavailable",
+        "PROXY_CACHE_LOCAL_ON_NOT_FOUND_TOOLTIP": "Enable this to allow this project to keep serving cached content even if the artifact is not found in the upstream registry or the upstream registry is unavailable. This can lead to stale content being served if the upstream registry is unavailable.",
         "NO_PROJECT": "We couldn't find any projects"
     },
     "PROJECT_DETAIL": {
diff --git a/src/portal/src/i18n/lang/fr-fr-lang.json b/src/portal/src/i18n/lang/fr-fr-lang.json
@@ -246,6 +246,8 @@
         "CONNECTION_LIMIT": "Connexion maximale au registre en amont",
         "PROXY_CACHE_MAX_UPSTREAM_CONN_TIP": "La connexion maximale au registre en amont pour ce projet de cache proxy, si -1, alors il n'y a pas de limite",
         "PROXY_CACHE_MAX_UPSTREAM_CONN_INPUT_TIP": "Veuillez entrer -1 ou un entier supérieur à 0.",
+        "PROXY_CACHE_LOCAL_ON_NOT_FOUND": "Servir le contenu mis en cache lorsque le registre amont est indisponible",
+        "PROXY_CACHE_LOCAL_ON_NOT_FOUND_TOOLTIP": "Activez cette option pour permettre à ce projet de continuer à servir le contenu mis en cache même si l'artefact n'est pas trouvé dans le registre amont ou si le registre amont est indisponible. Cela peut entraîner la diffusion de contenu obsolète si le registre amont est indisponible.",
         "NO_PROJECT": "Nous n'avons trouvé aucun projet"
     },
     "PROJECT_DETAIL": {
diff --git a/src/server/middleware/repoproxy/proxy.go b/src/server/middleware/repoproxy/proxy.go
@@ -227,7 +227,7 @@ func handleManifest(w http.ResponseWriter, r *http.Request, next http.Handler) e
 	if err != nil {
 		return err
 	}
-	useLocal, man, err := proxyCtl.UseLocalManifest(ctx, art, remote)
+	useLocal, man, err := proxyCtl.UseLocalManifest(ctx, art, remote, p)
 	if err != nil {
 		return err
 	}
diff --git a/src/server/v2.0/handler/project_metadata.go b/src/server/v2.0/handler/project_metadata.go
@@ -143,7 +143,8 @@ func (p *projectMetadataAPI) validate(metas map[string]string) (map[string]strin
 
 	switch key {
 	case proModels.ProMetaPublic, proModels.ProMetaEnableContentTrust, proModels.ProMetaEnableContentTrustCosign,
-		proModels.ProMetaAutoSBOMGen, proModels.ProMetaPreventVul, proModels.ProMetaAutoScan, proModels.ProMetaReuseSysCVEAllowlist:
+		proModels.ProMetaAutoSBOMGen, proModels.ProMetaPreventVul, proModels.ProMetaAutoScan, proModels.ProMetaReuseSysCVEAllowlist,
+		proModels.ProMetaProxyCacheLocalOnNotFound:
 		v, err := strconv.ParseBool(value)
 		if err != nil {
 			return nil, errors.New(nil).WithCode(errors.BadRequestCode).WithMessagef("invalid value: %s", value)
