Implemented a better include-cache

Author: cmaglie

internal/arduino/builder/internal/detector/cache.go

@@ -0,0 +1,122 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2024 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package detector
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/arduino/go-paths-helper"
+)
+
+type detectorCache struct {
+	curr    int
+	entries []*detectorCacheEntry
+}
+
+type detectorCacheEntry struct {
+	AddedIncludePath    *paths.Path `json:"added_include_path,omitempty"`
+	CompilingSourcePath *paths.Path `json:"compiling_source_path,omitempty"`
+	MissingIncludeH     *string     `json:"missing_include_h,omitempty"`
+}
+
+func (e *detectorCacheEntry) String() string {
+	if e.AddedIncludePath != nil {
+		return "Added include path: " + e.AddedIncludePath.String()
+	}
+	if e.CompilingSourcePath != nil {
+		return "Compiling source path: " + e.CompilingSourcePath.String()
+	}
+	if e.MissingIncludeH != nil {
+		if *e.MissingIncludeH == "" {
+			return "No missing include files detected"
+		}
+		return "Missing include file: " + *e.MissingIncludeH
+	}
+	return "No operation"
+}
+
+func (e *detectorCacheEntry) Equals(entry *detectorCacheEntry) bool {
+	return e.String() == entry.String()
+}
+
+func newDetectorCache() *detectorCache {
+	return &detectorCache{}
+}
+
+func (c *detectorCache) String() string {
+	res := ""
+	for _, entry := range c.entries {
+		res += fmt.Sprintln(entry)
+	}
+	return res
+}
+
+// Load reads a saved cache from the given file.
+// If the file do not exists, it does nothing.
+func (c *detectorCache) Load(cacheFile *paths.Path) error {
+	if exist, err := cacheFile.ExistCheck(); err != nil {
+		return err
+	} else if !exist {
+		return nil
+	}
+	data, err := cacheFile.ReadFile()
+	if err != nil {
+		return err
+	}
+	var entries []*detectorCacheEntry
+	if err := json.Unmarshal(data, &entries); err != nil {
+		return err
+	}
+	c.curr = 0
+	c.entries = entries
+	return nil
+}
+
+// Expect adds an entry to the cache and checks if it matches the next expected entry.
+func (c *detectorCache) Expect(entry *detectorCacheEntry) {
+	if c.curr < len(c.entries) {
+		if c.entries[c.curr].Equals(entry) {
+			// Cache hit, move to the next entry
+			c.curr++
+			return
+		}
+		// Cache mismatch, invalidate and cut the remainder of the cache
+		c.entries = c.entries[:c.curr]
+	}
+	c.curr++
+	c.entries = append(c.entries, entry)
+}
+
+// Peek returns the next cache entry to be expected or nil if the cache is fully consumed.
+func (c *detectorCache) Peek() *detectorCacheEntry {
+	if c.curr < len(c.entries) {
+		return c.entries[c.curr]
+	}
+	return nil
+}
+
+// Save writes the current cache to the given file.
+func (c *detectorCache) Save(cacheFile *paths.Path) error {
+	// Cut off the cache if it is not fully consumed
+	c.entries = c.entries[:c.curr]
+
+	data, err := json.MarshalIndent(c.entries, "", "  ")
+	if err != nil {
+		return err
+	}
+	return cacheFile.WriteFile(data)
+}

internal/arduino/builder/internal/detector/detector.go

@@ -52,7 +52,7 @@ type SketchLibrariesDetector struct {
 	librariesManager              *librariesmanager.LibrariesManager
 	librariesResolver             *librariesresolver.Cpp
 	useCachedLibrariesResolution  bool
-	cache                         *includeCache
+	cache                         *detectorCache
 	onlyUpdateCompilationDatabase bool
 	importedLibraries             libraries.List
 	librariesResolutionResults    map[string]libraryResolutionResult
@@ -74,6 +74,7 @@ func NewSketchLibrariesDetector(
 		librariesManager:              lm,
 		librariesResolver:             libsResolver,
 		useCachedLibrariesResolution:  useCachedLibrariesResolution,
+		cache:                         newDetectorCache(),
 		librariesResolutionResults:    map[string]libraryResolutionResult{},
 		importedLibraries:             libraries.List{},
 		includeFolders:                paths.PathList{},
@@ -175,21 +176,10 @@ func (l *SketchLibrariesDetector) IncludeFolders() paths.PathList {
 	return l.includeFolders
 }
 
-// appendIncludeFolder todo should rename this, probably after refactoring the
-// container_find_includes command.
-// Original comment:
-// Append the given folder to the include path and match or append it to
-// the cache. sourceFilePath and include indicate the source of this
-// include (e.g. what #include line in what file it was resolved from)
-// and should be the empty string for the default include folders, like
-// the core or variant.
-func (l *SketchLibrariesDetector) appendIncludeFolder(
-	sourceFilePath *paths.Path,
-	include string,
-	folder *paths.Path,
-) {
+// addIncludeFolder add the given folder to the include path.
+func (l *SketchLibrariesDetector) addIncludeFolder(folder *paths.Path) {
 	l.includeFolders = append(l.includeFolders, folder)
-	l.cache.ExpectEntry(sourceFilePath, include, folder)
+	l.cache.Expect(&detectorCacheEntry{AddedIncludePath: folder})
 }
 
 // FindIncludes todo
@@ -245,11 +235,13 @@ func (l *SketchLibrariesDetector) findIncludes(
 	}
 
 	cachePath := buildPath.Join("includes.cache")
-	l.cache = readCache(cachePath)
+	if err := l.cache.Load(cachePath); err != nil {
+		l.logger.Warn(i18n.Tr("Failed to load library discovery cache: %[1]s", err))
+	}
 
-	l.appendIncludeFolder(nil, "", buildCorePath)
+	l.addIncludeFolder(buildCorePath)
 	if buildVariantPath != nil {
-		l.appendIncludeFolder(nil, "", buildVariantPath)
+		l.addIncludeFolder(buildVariantPath)
 	}
 
 	sourceFileQueue := &uniqueSourceFileQueue{}
@@ -269,16 +261,15 @@ func (l *SketchLibrariesDetector) findIncludes(
 		}
 
 		for !sourceFileQueue.Empty() {
-			err := l.findIncludesUntilDone(ctx, sourceFileQueue, buildProperties, librariesBuildPath, platformArch)
+			err := l.findMissingIncludesInCompilationUnit(ctx, sourceFileQueue, buildProperties, librariesBuildPath, platformArch)
 			if err != nil {
 				cachePath.Remove()
 				return err
 			}
 		}
 
 		// Finalize the cache
-		l.cache.ExpectEnd()
-		if err := l.cache.write(cachePath); err != nil {
+		if err := l.cache.Save(cachePath); err != nil {
 			return err
 		}
 	}
@@ -296,7 +287,7 @@ func (l *SketchLibrariesDetector) findIncludes(
 	return nil
 }
 
-func (l *SketchLibrariesDetector) findIncludesUntilDone(
+func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
 	ctx context.Context,
 	sourceFileQueue *uniqueSourceFileQueue,
 	buildProperties *properties.Map,
@@ -328,7 +319,7 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone(
 
 	first := true
 	for {
-		l.cache.ExpectFile(sourcePath)
+		l.cache.Expect(&detectorCacheEntry{CompilingSourcePath: sourcePath})
 
 		// Libraries may require the "utility" directory to be added to the include
 		// search path, but only for the source code of the library, so we temporary
@@ -343,8 +334,8 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone(
 		var preprocFirstResult *runner.Result
 
 		var missingIncludeH string
-		if unchanged && l.cache.valid {
-			missingIncludeH = l.cache.Next().Include
+		if entry := l.cache.Peek(); unchanged && entry != nil && entry.MissingIncludeH != nil {
+			missingIncludeH = *entry.MissingIncludeH
 			if first && l.logger.VerbosityLevel() == logger.VerbosityVerbose {
 				l.logger.Info(i18n.Tr("Using cached library dependencies for file: %[1]s", sourcePath))
 			}
@@ -370,9 +361,10 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone(
 			}
 		}
 
+		l.cache.Expect(&detectorCacheEntry{MissingIncludeH: &missingIncludeH})
+
 		if missingIncludeH == "" {
 			// No missing includes found, we're done
-			l.cache.ExpectEntry(sourcePath, "", nil)
 			return nil
 		}
 
@@ -405,7 +397,7 @@ func (l *SketchLibrariesDetector) findIncludesUntilDone(
 		// include path and queue its source files for further
 		// include scanning
 		l.AppendImportedLibraries(library)
-		l.appendIncludeFolder(sourcePath, missingIncludeH, library.SourceDir)
+		l.addIncludeFolder(library.SourceDir)
 
 		if library.Precompiled && library.PrecompiledWithSources {
 			// Fully precompiled libraries should have no dependencies to avoid ABI breakage