[pull] master from gohugoio:master

.github/workflows/aiwatchdog.yml

@@ -13,7 +13,7 @@ jobs:
     steps:
       - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1
       - name: AI Watchdog
-        uses: bep/ai-watchdog@995475dd81767f8035dc2c943e9230918d06caca # temporary fork (probably)
+        uses: bep/ai-watchdog@e85b3674edb3f174950e7622b87350ad54b2013e # temporary fork (probably)
         with:
           openai-api-key: ${{ secrets.OPENAI_API_KEY }}
           pr-label: AI Suspect

cache/filecache/filecache.go

@@ -1,4 +1,4 @@
-// Copyright 2024 The Hugo Authors. All rights reserved.
+// Copyright 2026 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -46,17 +46,9 @@ const (
 type Cache struct {
 	Fs afero.Fs
 
-	// Max age for items in this cache. Negative duration means forever,
-	// 0 is effectively turning this cache off.
-	maxAge time.Duration
+	cfg FileCacheConfig
 
-	// When set, we just remove this entire root directory on expiration.
-	pruneAllRootDir string
-
-	nlocker *lockTracker
-
-	isResourceDir bool
-	baseDir       string
+	entryLocker *lockTracker
 
 	initOnce sync.Once
 	isInited bool
@@ -93,12 +85,15 @@ type ItemInfo struct {
 }
 
 // NewCache creates a new file cache with the given filesystem and max age.
-func NewCache(fs afero.Fs, maxAge time.Duration, pruneAllRootDir string) *Cache {
+func NewCache(fs afero.Fs, cfg FileCacheConfig) *Cache {
+	if err := cfg.init(); err != nil {
+		panic(fmt.Sprintf("invalid cache config: %s", err))
+	}
+
 	return &Cache{
-		Fs:              fs,
-		nlocker:         &lockTracker{Locker: locker.NewLocker(), seen: make(map[string]struct{})},
-		maxAge:          maxAge,
-		pruneAllRootDir: pruneAllRootDir,
+		Fs:          fs,
+		entryLocker: &lockTracker{Locker: locker.NewLocker(), seen: make(map[string]struct{})},
+		cfg:         cfg,
 	}
 }
 
@@ -117,6 +112,7 @@ func (c *Cache) init() error {
 	if c == nil {
 		panic("cache is nil")
 	}
+
 	c.initOnce.Do(func() {
 		c.isInited = true
 		// Create the base dir if it does not exist.
@@ -136,19 +132,19 @@ func (c *Cache) WriteCloser(id string) (ItemInfo, io.WriteCloser, error) {
 	}
 
 	id = cleanID(id)
-	c.nlocker.Lock(id)
+	c.entryLocker.Lock(id)
 
 	info := ItemInfo{Name: id}
 
 	f, err := helpers.OpenFileForWriting(c.Fs, id)
 	if err != nil {
-		c.nlocker.Unlock(id)
+		c.entryLocker.Unlock(id)
 		return info, nil, err
 	}
 
 	return info, &lockedFile{
 		File:   f,
-		unlock: func() { c.nlocker.Unlock(id) },
+		unlock: func() { c.entryLocker.Unlock(id) },
 	}, nil
 }
 
@@ -166,8 +162,8 @@ func (c *Cache) ReadOrCreate(id string,
 
 	id = cleanID(id)
 
-	c.nlocker.Lock(id)
-	defer c.nlocker.Unlock(id)
+	c.entryLocker.Lock(id)
+	defer c.entryLocker.Unlock(id)
 
 	info = ItemInfo{Name: id}
 
@@ -196,9 +192,9 @@ func (c *Cache) ReadOrCreate(id string,
 // NamedLock locks the given id. The lock is released when the returned function is called.
 func (c *Cache) NamedLock(id string) func() {
 	id = cleanID(id)
-	c.nlocker.Lock(id)
+	c.entryLocker.Lock(id)
 	return func() {
-		c.nlocker.Unlock(id)
+		c.entryLocker.Unlock(id)
 	}
 }
 
@@ -211,8 +207,8 @@ func (c *Cache) GetOrCreate(id string, create func() (io.ReadCloser, error)) (It
 	}
 	id = cleanID(id)
 
-	c.nlocker.Lock(id)
-	defer c.nlocker.Unlock(id)
+	c.entryLocker.Lock(id)
+	defer c.entryLocker.Unlock(id)
 
 	info := ItemInfo{Name: id}
 
@@ -230,7 +226,7 @@ func (c *Cache) GetOrCreate(id string, create func() (io.ReadCloser, error)) (It
 		return info, nil, err
 	}
 
-	if c.maxAge == 0 {
+	if c.cfg.MaxAge == 0 {
 		// No caching.
 		return info, hugio.ToReadCloser(r), nil
 	}
@@ -241,6 +237,40 @@ func (c *Cache) GetOrCreate(id string, create func() (io.ReadCloser, error)) (It
 		c.writeReader(id, io.TeeReader(r, &buff))
 }
 
+// AbsFilenameFromID returns the filename for the given id in the cache.
+// This will be an absolute path.
+func (c *Cache) AbsFilenameFromID(id string) string {
+	return filepath.Join(c.cfg.DirCompiled, cleanID(id))
+}
+
+// GetOrCreateInfo tries to get the item info with the given id from cache. If not found or expired, create will
+// be invoked with the id. The create function is expected to create the cache item with the given id. The returned ItemInfo will have the id as Name.
+// This method is protected by a named lock using the given id as identifier.
+func (c *Cache) GetOrCreateInfo(id string, create func(id string) error) (ItemInfo, error) {
+	if err := c.init(); err != nil {
+		return ItemInfo{}, err
+	}
+
+	id = cleanID(id)
+
+	c.entryLocker.Lock(id)
+	defer c.entryLocker.Unlock(id)
+
+	info := ItemInfo{Name: id}
+
+	if !c.removeIfNeeded(id) {
+		// The file exists and is not expired, so we consider it a cache hit.
+		return info, nil
+	}
+
+	if err := create(id); err != nil {
+		c.remove(id)
+		return info, err
+	}
+
+	return info, nil
+}
+
 func (c *Cache) writeReader(id string, r io.Reader) error {
 	dir := filepath.Dir(id)
 	if dir != "" {
@@ -264,8 +294,8 @@ func (c *Cache) GetOrCreateBytes(id string, create func() ([]byte, error)) (Item
 	}
 	id = cleanID(id)
 
-	c.nlocker.Lock(id)
-	defer c.nlocker.Unlock(id)
+	c.entryLocker.Lock(id)
+	defer c.entryLocker.Unlock(id)
 
 	info := ItemInfo{Name: id}
 
@@ -285,7 +315,7 @@ func (c *Cache) GetOrCreateBytes(id string, create func() ([]byte, error)) (Item
 		return info, nil, err
 	}
 
-	if c.maxAge == 0 {
+	if c.cfg.MaxAge == 0 {
 		return info, b, nil
 	}
 
@@ -303,10 +333,10 @@ func (c *Cache) SetBytes(id string, data []byte) error {
 	}
 	id = cleanID(id)
 
-	c.nlocker.Lock(id)
-	defer c.nlocker.Unlock(id)
+	c.entryLocker.Lock(id)
+	defer c.entryLocker.Unlock(id)
 
-	if c.maxAge == 0 {
+	if c.cfg.MaxAge == 0 {
 		// No caching.
 		return nil
 	}
@@ -327,8 +357,8 @@ func (c *Cache) GetItemBytes(id string) (ItemInfo, []byte, error) {
 	}
 	id = cleanID(id)
 
-	c.nlocker.Lock(id)
-	defer c.nlocker.Unlock(id)
+	c.entryLocker.Lock(id)
+	defer c.entryLocker.Unlock(id)
 
 	info := ItemInfo{Name: id}
 
@@ -348,8 +378,8 @@ func (c *Cache) Get(id string) (ItemInfo, io.ReadCloser, error) {
 	}
 	id = cleanID(id)
 
-	c.nlocker.Lock(id)
-	defer c.nlocker.Unlock(id)
+	c.entryLocker.Lock(id)
+	defer c.entryLocker.Unlock(id)
 
 	info := ItemInfo{Name: id}
 
@@ -358,10 +388,23 @@ func (c *Cache) Get(id string) (ItemInfo, io.ReadCloser, error) {
 	return info, r, nil
 }
 
+// removeIfNeeded checks if the file with the given id should be re-created.
+func (c *Cache) removeIfNeeded(id string) bool {
+	if c.cfg.MaxAge == 0 {
+		// No caching, remove.
+		c.remove(id)
+		return true
+	}
+	if removed, err := c.removeIfExpired(id); err != nil || removed {
+		return true
+	}
+	return false
+}
+
 // getOrRemove gets the file with the given id. If it's expired, it will
 // be removed.
 func (c *Cache) getOrRemove(id string) hugio.ReadSeekCloser {
-	if c.maxAge == 0 {
+	if c.cfg.MaxAge == 0 {
 		// No caching.
 		return nil
 	}
@@ -379,7 +422,7 @@ func (c *Cache) getOrRemove(id string) hugio.ReadSeekCloser {
 }
 
 func (c *Cache) getBytesAndRemoveIfExpired(id string) ([]byte, bool) {
-	if c.maxAge == 0 {
+	if c.cfg.MaxAge == 0 {
 		// No caching.
 		return nil, false
 	}
@@ -404,7 +447,7 @@ func (c *Cache) getBytesAndRemoveIfExpired(id string) ([]byte, bool) {
 }
 
 func (c *Cache) removeIfExpired(id string) (bool, error) {
-	if c.maxAge <= 0 {
+	if c.cfg.MaxAge <= 0 {
 		return false, nil
 	}
 
@@ -414,29 +457,37 @@ func (c *Cache) removeIfExpired(id string) (bool, error) {
 	}
 
 	if c.isExpired(fi.ModTime()) {
-		c.Fs.Remove(id)
+		c.remove(id)
 		return true, nil
 	}
 
 	return false, nil
 }
 
+func (c *Cache) remove(id string) {
+	if c.cfg.entryIsDir {
+		c.Fs.RemoveAll(id)
+	} else {
+		c.Fs.Remove(id)
+	}
+}
+
 func (c *Cache) isExpired(modTime time.Time) bool {
-	if c.maxAge < 0 {
+	if c.cfg.MaxAge < 0 {
 		return false
 	}
 
 	// Note the use of time.Since here.
 	// We cannot use Hugo's global Clock for this.
-	return c.maxAge == 0 || time.Since(modTime) > c.maxAge
+	return c.cfg.MaxAge == 0 || time.Since(modTime) > c.cfg.MaxAge
 }
 
 // For testing
 func (c *Cache) GetString(id string) string {
 	id = cleanID(id)
 
-	c.nlocker.Lock(id)
-	defer c.nlocker.Unlock(id)
+	c.entryLocker.Lock(id)
+	defer c.entryLocker.Unlock(id)
 
 	f, err := c.Fs.Open(id)
 	if err != nil {
@@ -453,11 +504,11 @@ type Caches map[string]*Cache
 
 func (f Caches) SetResourceFs(fs afero.Fs) {
 	for _, c := range f {
-		if c.isResourceDir {
+		if c.cfg.IsResourceDir {
 			if c.isInited {
 				panic("cannot set resource fs after init")
 			}
-			c.Fs = hugofs.NewBasePathFs(fs, c.baseDir)
+			c.Fs = hugofs.NewBasePathFs(fs, c.cfg.DirCompiled)
 		}
 	}
 }
@@ -470,7 +521,6 @@ func (f Caches) Get(name string) *Cache {
 // NewCaches creates a new set of file caches from the given
 // configuration.
 func NewCaches(dcfg Configs, sourceFs afero.Fs) (Caches, error) {
-	// dcfg := p.Cfg.GetConfigSection("caches").(Configs)
 	fs := sourceFs
 
 	m := make(Caches)
@@ -479,21 +529,10 @@ func NewCaches(dcfg Configs, sourceFs afero.Fs) (Caches, error) {
 		if v.IsResourceDir {
 			cfs = nil // Set later. TODO(bep) this needs to be cleanded up.
 		} else {
-			cfs = fs
-		}
-
-		baseDir := v.DirCompiled
-
-		bfs := hugofs.NewBasePathFs(cfs, baseDir)
-
-		var pruneAllRootDir string
-		if k == CacheKeyModules {
-			pruneAllRootDir = "pkg"
+			cfs = hugofs.NewBasePathFs(fs, v.DirCompiled)
 		}
 
-		c := NewCache(bfs, v.MaxAge, pruneAllRootDir)
-		c.isResourceDir = v.IsResourceDir
-		c.baseDir = baseDir
+		c := NewCache(cfs, v)
 
 		m[k] = c
 	}
@@ -524,7 +563,7 @@ func (h *httpCache) Get(id string) (resp []byte, ok bool) {
 }
 
 func (h *httpCache) Set(id string, resp []byte) {
-	if h.c.maxAge == 0 {
+	if h.c.cfg.MaxAge == 0 {
 		return
 	}