Add optional TTL-based cleanup for httpcache to reduce disk and memory usage

abushwang · abushwang · commit 4ff54f23c11e · 2026-04-02T19:56:32.000+08:00
Signed-off-by: abushwang &lt;abushwang@tencent.com&gt;
diff --git a/cache/httpcache_ttl.go b/cache/httpcache_ttl.go
@@ -0,0 +1,124 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package cache
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/containerd/log"
+)
+
+var (
+	cleanupJanitorMu     sync.Mutex
+	globalCleanupJanitor *cleanupJanitor
+)
+
+type cleanupJanitor struct {
+	rootDir string
+	ttl     time.Duration
+
+	stopCh chan struct{}
+	wg     sync.WaitGroup
+}
+
+// StartCleanupJanitor starts the process-wide janitor for httpcache cleanup.
+func StartCleanupJanitor(rootDir string, ttl time.Duration) {
+	if ttl <= 0 {
+		return
+	}
+
+	cleanupJanitorMu.Lock()
+	defer cleanupJanitorMu.Unlock()
+
+	if globalCleanupJanitor != nil {
+		if globalCleanupJanitor.rootDir != rootDir || globalCleanupJanitor.ttl != ttl {
+			log.L.WithFields(map[string]any{
+				"root":          rootDir,
+				"configured":    ttl,
+				"existing_root": globalCleanupJanitor.rootDir,
+				"existing_ttl":  globalCleanupJanitor.ttl,
+			}).Warn("httpcache janitor already initialized; reusing existing janitor")
+		}
+		return
+	}
+
+	janitor := &cleanupJanitor{
+		rootDir: rootDir,
+		ttl:     ttl,
+		stopCh:  make(chan struct{}),
+	}
+	janitor.start()
+	globalCleanupJanitor = janitor
+}
+
+func (j *cleanupJanitor) start() {
+	j.wg.Add(1)
+	go func() {
+		defer j.wg.Done()
+
+		ticker := time.NewTicker(j.ttl)
+		defer ticker.Stop()
+
+		j.cleanupOnce()
+		for {
+			select {
+			case <-ticker.C:
+				j.cleanupOnce()
+			case <-j.stopCh:
+				return
+			}
+		}
+	}()
+}
+
+func (j *cleanupJanitor) cleanupOnce() {
+	if j.ttl <= 0 {
+		return
+	}
+
+	cutoff := time.Now().Add(-j.ttl)
+	_ = filepath.WalkDir(j.rootDir, func(path string, d fs.DirEntry, walkErr error) error {
+		if walkErr != nil {
+			return nil
+		}
+		select {
+		case <-j.stopCh:
+			return fs.SkipAll
+		default:
+		}
+		if d.IsDir() {
+			if d.Name() == "wip" {
+				return fs.SkipDir
+			}
+			return nil
+		}
+
+		info, err := d.Info()
+		if err != nil || info.ModTime().After(cutoff) {
+			return nil
+		}
+
+		if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+			log.L.WithError(err).Debugf("failed to remove expired cache entry %q", path)
+		}
+		return nil
+	})
+}
diff --git a/cache/httpcache_ttl_test.go b/cache/httpcache_ttl_test.go
@@ -0,0 +1,116 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package cache
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestStartCleanupJanitorInitializesOnce(t *testing.T) {
+	stopCleanupJanitorForTest()
+	t.Cleanup(stopCleanupJanitorForTest)
+
+	root := t.TempDir()
+	StartCleanupJanitor(root, time.Second)
+
+	cleanupJanitorMu.Lock()
+	first := globalCleanupJanitor
+	cleanupJanitorMu.Unlock()
+	if first == nil {
+		t.Fatalf("expected janitor to be initialized")
+	}
+
+	otherRoot := t.TempDir()
+	StartCleanupJanitor(otherRoot, 2*time.Second)
+
+	cleanupJanitorMu.Lock()
+	second := globalCleanupJanitor
+	cleanupJanitorMu.Unlock()
+	if second != first {
+		t.Fatalf("expected existing janitor to be reused")
+	}
+}
+
+func TestCleanupJanitorCleanupOnceRemovesExpiredFilesAcrossRoot(t *testing.T) {
+	root := t.TempDir()
+	digestDir := filepath.Join(root, "sha256-a")
+	wipDir := filepath.Join(digestDir, "wip")
+	if err := os.MkdirAll(wipDir, 0700); err != nil {
+		t.Fatalf("mkdir wip failed: %v", err)
+	}
+
+	expired := filepath.Join(digestDir, "aa", "expired")
+	if err := os.MkdirAll(filepath.Dir(expired), 0700); err != nil {
+		t.Fatalf("mkdir expired dir failed: %v", err)
+	}
+	if err := os.WriteFile(expired, []byte("x"), 0600); err != nil {
+		t.Fatalf("write expired failed: %v", err)
+	}
+
+	expiredWip := filepath.Join(wipDir, "tmp-expired")
+	if err := os.WriteFile(expiredWip, []byte("y"), 0600); err != nil {
+		t.Fatalf("write expired wip failed: %v", err)
+	}
+
+	fresh := filepath.Join(root, "sha256-b", "bb", "fresh")
+	if err := os.MkdirAll(filepath.Dir(fresh), 0700); err != nil {
+		t.Fatalf("mkdir fresh dir failed: %v", err)
+	}
+	if err := os.WriteFile(fresh, []byte("z"), 0600); err != nil {
+		t.Fatalf("write fresh failed: %v", err)
+	}
+
+	old := time.Now().Add(-2 * time.Second)
+	if err := os.Chtimes(expired, old, old); err != nil {
+		t.Fatalf("chtimes expired failed: %v", err)
+	}
+	if err := os.Chtimes(expiredWip, old, old); err != nil {
+		t.Fatalf("chtimes expired wip failed: %v", err)
+	}
+
+	janitor := &cleanupJanitor{
+		rootDir: root,
+		ttl:     100 * time.Millisecond,
+		stopCh:  make(chan struct{}),
+	}
+	janitor.cleanupOnce()
+
+	if _, err := os.Stat(expired); !os.IsNotExist(err) {
+		t.Fatalf("expected expired file to be removed; err=%v", err)
+	}
+	if _, err := os.Stat(expiredWip); err != nil {
+		t.Fatalf("expected wip file to remain; err=%v", err)
+	}
+	if _, err := os.Stat(fresh); err != nil {
+		t.Fatalf("expected fresh file to remain; err=%v", err)
+	}
+}
+
+func stopCleanupJanitorForTest() {
+	cleanupJanitorMu.Lock()
+	janitor := globalCleanupJanitor
+	globalCleanupJanitor = nil
+	cleanupJanitorMu.Unlock()
+
+	if janitor != nil {
+		close(janitor.stopCh)
+		janitor.wg.Wait()
+	}
+}
diff --git a/fs/config/config.go b/fs/config/config.go
@@ -39,6 +39,10 @@ type Config struct {
 	// Other values default to cache them on disk.
 	HTTPCacheType string `toml:"http_cache_type" json:"http_cache_type"`
 
+	// HTTPCacheChunkTTLSec specifies TTL (in sec) for each http cache chunk.
+	// Zero or negative values disable TTL-based cleanup.
+	HTTPCacheChunkTTLSec int `toml:"http_cache_chunk_ttl_sec" json:"http_cache_chunk_ttl_sec"`
+
 	// Type of cache for uncompressed files contents. "memory" stores them on memory. Other values
 	// default to cache them on disk.
 	FSCacheType string `toml:"filesystem_cache_type" json:"filesystem_cache_type"`
diff --git a/fs/layer/layer.go b/fs/layer/layer.go
@@ -181,6 +181,13 @@ func NewResolver(root string, backgroundTaskManager *task.BackgroundTaskManager,
 		return nil, err
 	}
 
+	if cfg.HTTPCacheType != memoryCacheType && cfg.HTTPCacheChunkTTLSec > 0 {
+		cache.StartCleanupJanitor(
+			filepath.Join(root, "httpcache"),
+			time.Duration(cfg.HTTPCacheChunkTTLSec)*time.Second,
+		)
+	}
+
 	return &Resolver{
 		rootDir:                 root,
 		resolver:                remote.NewResolver(cfg.BlobConfig, resolveHandlers),
