Feat/4183/cache files in cc json

CC_JSON_SCHEMA_CHANGELOG.md

@@ -92,3 +92,34 @@ export interface Fixed {
   }
 }
 ```
+
+## 1.6
+
+- An additional optional property `checksum` has been added to elements of `nodes` which are of type `File`. This checksum is calculated based on the files content and can be used by analyzers that update a `cc.json` to check if they need to recalculate the metrics of a node.
+```json
+{
+  "nodes": [
+    {
+      "name": "root",
+      "type": "Folder",
+      "attributes": {},
+      "link": "",
+      "children": [
+        {
+          "name": "Samplefile",
+          "type": "File",
+          "attributes": {
+            "complexity": 32.0,
+            "comment_lines": 46.0,
+            "rloc": 140.0,
+            "loc": 203.0
+          },
+          "link": "",
+          "children": [],
+          "checksum": "3ccac3cacd32f1c7"
+        }
+      ]
+    }
+  ]
+}
+```

analysis/CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/)
 
 ## [unreleased] (Added 🚀 | Changed | Removed  | Fixed 🐞 | Chore 👨‍💻 👩‍💻)
 
+### Added 🚀
+
+- Add new '--base-file' flag to unifiedparser and rawtextparser
+
 ## [1.138.0] - 2025-10-08
 
 ### Added 🚀

analysis/analysers/AnalyserInterface/src/main/kotlin/de/maibornwolff/codecharta/analysers/analyserinterface/CommonAnalyserParameters.kt

@@ -3,8 +3,13 @@ package de.maibornwolff.codecharta.analysers.analyserinterface
 import de.maibornwolff.codecharta.analysers.analyserinterface.util.CommaSeparatedParameterPreprocessor
 import de.maibornwolff.codecharta.analysers.analyserinterface.util.CommaSeparatedStringToListConverter
 import de.maibornwolff.codecharta.analysers.analyserinterface.util.FileExtensionConverter
+import de.maibornwolff.codecharta.model.Node
+import de.maibornwolff.codecharta.model.NodeType
+import de.maibornwolff.codecharta.serialization.ProjectDeserializer
+import de.maibornwolff.codecharta.util.Logger
 import picocli.CommandLine
 import java.io.File
+import java.io.FileInputStream
 
 abstract class CommonAnalyserParameters {
     @CommandLine.Option(names = ["-h", "--help"], usageHelp = true, description = ["displays this help and exits"])
@@ -56,4 +61,51 @@ abstract class CommonAnalyserParameters {
         preprocessor = CommaSeparatedParameterPreprocessor::class
     )
     protected var fileExtensionsToAnalyse: List<String> = listOf()
+
+    @CommandLine.Option(
+        names = ["-bf", "--base-file"],
+        description = ["base cc.json file with checksums to skip unchanged files during analysis"]
+    )
+    protected var baseFile: File? = null
+
+    protected fun loadBaseFileNodes(): Map<String, Node> {
+        if (baseFile == null) {
+            return emptyMap()
+        }
+
+        if (!baseFile!!.exists()) {
+            Logger.warn { "Base file '${baseFile!!.absolutePath}' does not exist, continuing with normal analysis... " }
+            return emptyMap()
+        }
+
+        return try {
+            val baseProject = ProjectDeserializer.deserializeProject(FileInputStream(baseFile!!))
+            val nodeMap = mutableMapOf<String, Node>()
+            extractFileNodesWithPaths(baseProject.rootNode, "", nodeMap)
+            Logger.info { "Loaded ${nodeMap.size} file nodes from base file for checksum comparison" }
+            nodeMap
+        } catch (e: Exception) {
+            Logger.warn { "Failed to load base file: ${e.message}" }
+            emptyMap()
+        }
+    }
+
+    private fun extractFileNodesWithPaths(node: Node, currentPath: String, nodeMap: MutableMap<String, Node>) {
+        // Skip the root node name to match the relative paths generated by parsers
+        val nodePath = if (currentPath.isEmpty() && node.name == "root") {
+            ""
+        } else if (currentPath.isEmpty()) {
+            node.name
+        } else {
+            "$currentPath/${node.name}"
+        }
+
+        if (node.type == NodeType.File && node.checksum != null) {
+            nodeMap[nodePath] = node
+        }
+
+        node.children.forEach { child ->
+            extractFileNodesWithPaths(child, nodePath, nodeMap)
+        }
+    }
 }

analysis/analysers/filters/MergeFilter/src/main/kotlin/de/maibornwolff/codecharta/analysers/filters/mergefilter/ProjectMerger.kt

@@ -126,16 +126,16 @@ class ProjectMerger(
         newAttributeDescriptor: AttributeDescriptor
     ) {
         if (existingAttributeDescriptor.title != newAttributeDescriptor.title) {
-            Logger.warn { "Title of '$metric' metric differs between files! Using value of first file..." }
+            Logger.info { "Title of '$metric' metric differs between files! Using value of first file..." }
         }
         if (existingAttributeDescriptor.description != newAttributeDescriptor.description) {
-            Logger.warn { "Description of '$metric' metric differs between files! Using value of first file..." }
+            Logger.info { "Description of '$metric' metric differs between files! Using value of first file..." }
         }
         if (existingAttributeDescriptor.link != newAttributeDescriptor.link) {
-            Logger.warn { "Link of '$metric' metric differs between files! Using value of first file..." }
+            Logger.info { "Link of '$metric' metric differs between files! Using value of first file..." }
         }
         if (existingAttributeDescriptor.direction != newAttributeDescriptor.direction) {
-            Logger.warn { "Direction of '$metric' metric differs between files! Using value of first file..." }
+            Logger.info { "Direction of '$metric' metric differs between files! Using value of first file..." }
         }
     }
 

analysis/analysers/parsers/RawTextParser/README.md

@@ -15,23 +15,24 @@ This parser analyzes code, regardless of the programming language, to generate t
 
 ## Usage and Parameters
 
-| Parameter                                 | Description                                                                                                                                                                                        |
-|-------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `FILE or FOLDER`                          | file/project to parseProject                                                                                                                                                                       |
-| `-e, --exclude=<exclude>`                 | comma-separated list of regex patterns to exclude files/folders                                                                                                                                    |
-| `-e, --exclude=<exclude>`                 | comma-separated list of regex patterns to exclude files/folders                                                                                                                                    |
-| `-fe, --file-extensions=<fileExtensions>` | comma-separated list of file-extensions to parse only those files (default: any)                                                                                                                   |
-| `-h, --help`                              | displays this help and exits                                                                                                                                                                       |
-| `-m, --metrics=metrics`                   | comma-separated list of metrics to be computed (all available metrics are computed if not specified)                                                                                               |
-| `--max-indentation-level=<maxIndentLvl>`  | maximum Indentation Level (default 10)                                                                                                                                                             |
-| `-nc, --not-compressed`                   | save uncompressed output File                                                                                                                                                                      |
-| `-o, --output-file=<outputFile>`          | output File (or empty for stdout)                                                                                                                                                                  |
-| `--tab-width=<tabWidth>`                  | tab width used (estimated if not provided)                                                                                                                                                         |
-| `--verbose`                               | verbose mode                                                                                                                                                                                       |
-| `--without-default-excludes`              | (DEPRECATION WARNING: this flag will soon be disabled and replaced by '--include-build-folders') include build, target, dist, resources and out folders as well as files/folders starting with '.' |
+| Parameter                                 | Description                                                                                                                                                                                          |
+|-------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `FILE or FOLDER`                          | file/project to parseProject                                                                                                                                                                         |
+| `-bf, --base-file=<baseFile>`             | base cc.json file with checksums to skip unchanged files during analysis                                                                                                                             |
+| `-e, --exclude=<exclude>`                 | comma-separated list of regex patterns to exclude files/folders                                                                                                                                      |
+| `-fe, --file-extensions=<fileExtensions>` | comma-separated list of file-extensions to parse only those files (default: any)                                                                                                                     |
+| `-h, --help`                              | displays this help and exits                                                                                                                                                                         |
+| `-m, --metrics=metrics`                   | comma-separated list of metrics to be computed (all available metrics are computed if not specified)                                                                                                 |
+| `--max-indentation-level=<maxIndentLvl>`  | maximum Indentation Level (default 10)                                                                                                                                                               |
+| `-nc, --not-compressed`                   | save uncompressed output File                                                                                                                                                                        |
+| `-o, --output-file=<outputFile>`          | output File (or empty for stdout)                                                                                                                                                                    |
+| `--tab-width=<tabWidth>`                  | tab width used (estimated if not provided)                                                                                                                                                           |
+| `--verbose`                               | verbose mode                                                                                                                                                                                         |
+| `--without-default-excludes`              | ("DEPRECATION WARNING: this flag will soon be disabled and replaced by '--include-build-folders'") include build, target, dist, resources and out folders as well as files/folders starting with '.' |
 
 ```
 Usage: ccsh rawtextparser [-h] [-nc] [--verbose] [--without-default-excludes]
+                          [-bf=<baseFile>]
                           [--max-indentation-level=<maxIndentLvl>]
                           [-o=<outputFile>] [--tab-width=<tabWidth>]
                           [-e=<exclude>]... [-fe=<fileExtensions>]...

analysis/analysers/parsers/RawTextParser/src/main/kotlin/de/maibornwolff/codecharta/analysers/parsers/rawtext/FileMetrics.kt

@@ -2,6 +2,7 @@ package de.maibornwolff.codecharta.analysers.parsers.rawtext
 
 class FileMetrics {
     val metricsMap = mutableMapOf<String, Double>()
+    var checksum: String? = null
 
     fun addMetric(name: String, value: Number): FileMetrics {
         metricsMap[name] = value.toDouble()

analysis/analysers/parsers/RawTextParser/src/main/kotlin/de/maibornwolff/codecharta/analysers/parsers/rawtext/ProjectGenerator.kt

@@ -23,7 +23,7 @@ class ProjectGenerator(private val projectBuilder: ProjectBuilder = ProjectBuild
     private fun addMetricsAsNodes(metricsMap: Map<String, FileMetrics>) {
         metricsMap.forEach { (key, fileMetrics) ->
             val (directory, fileName) = splitDirectoryAndFileName(key)
-            val node = MutableNode(fileName, attributes = fileMetrics.metricsMap)
+            val node = MutableNode(fileName, attributes = fileMetrics.metricsMap, checksum = fileMetrics.checksum)
             val path = PathFactory.fromFileSystemPath(directory)
             projectBuilder.insertByPath(path, node)
         }

analysis/analysers/parsers/RawTextParser/src/main/kotlin/de/maibornwolff/codecharta/analysers/parsers/rawtext/ProjectMetricsCollector.kt

@@ -3,12 +3,16 @@ package de.maibornwolff.codecharta.analysers.parsers.rawtext
 import de.maibornwolff.codecharta.analysers.parsers.rawtext.metrics.IndentationMetric
 import de.maibornwolff.codecharta.analysers.parsers.rawtext.metrics.LinesOfCodeMetric
 import de.maibornwolff.codecharta.analysers.parsers.rawtext.metrics.Metric
+import de.maibornwolff.codecharta.model.ChecksumCalculator
+import de.maibornwolff.codecharta.model.Node
 import de.maibornwolff.codecharta.progresstracker.ParsingUnit
 import de.maibornwolff.codecharta.progresstracker.ProgressTracker
+import de.maibornwolff.codecharta.util.Logger
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
 import java.io.File
+import java.util.concurrent.atomic.AtomicLong
 
 class ProjectMetricsCollector(
     private var root: File,
@@ -17,10 +21,13 @@ class ProjectMetricsCollector(
     private val metricNames: List<String>,
     private val verbose: Boolean,
     private val maxIndentLvl: Int,
-    private val tabWidth: Int
+    private val tabWidth: Int,
+    private val baseFileNodeMap: Map<String, Node> = emptyMap()
 ) {
     private var totalFiles = 0L
     private var filesParsed = 0L
+    private val filesSkipped = AtomicLong(0)
+    private val filesAnalyzed = AtomicLong(0)
     private val parsingUnit = ParsingUnit.Files
     private val progressTracker = ProgressTracker()
     private var excludePatterns = createExcludePatterns()
@@ -44,14 +51,15 @@ class ProjectMetricsCollector(
                     ) {
                         filesParsed++
                         logProgress(it.name, filesParsed)
-                        projectMetrics.addFileMetrics(standardizedPath, parseFile(it))
+                        projectMetrics.addFileMetrics(standardizedPath, collectMetricsForFile(it, standardizedPath))
                         lastFileName = it.name
                     }
                 }
             }
         }
 
         logProgress(lastFileName, totalFiles)
+        logStatistics()
 
         return projectMetrics
     }
@@ -92,17 +100,41 @@ class ProjectMetricsCollector(
         return exclude.isNotEmpty() && excludePatterns.containsMatchIn(path)
     }
 
-    private fun parseFile(file: File): FileMetrics {
+    private fun collectMetricsForFile(file: File, standardizedPath: String): FileMetrics {
+        val fileContent = file.readText()
+        val currentChecksum = ChecksumCalculator.calculateChecksum(fileContent)
+        val baseNode = baseFileNodeMap[standardizedPath.removePrefix("/")]
+        return if (baseNode != null && baseNode.checksum == currentChecksum) {
+            reuseMetricsFromBaseFile(baseNode)
+        } else {
+            calculateMetricsForFile(file, currentChecksum)
+        }
+    }
+
+    private fun reuseMetricsFromBaseFile(baseNode: Node): FileMetrics {
+        filesSkipped.incrementAndGet()
+
+        val fileMetrics = FileMetrics()
+        baseNode.attributes.forEach { (key, value) ->
+            fileMetrics.addMetric(key, value.toString().toDoubleOrNull() ?: 0.0)
+        }
+        fileMetrics.checksum = baseNode.checksum
+        return fileMetrics
+    }
+
+    private fun calculateMetricsForFile(file: File, checksum: String?): FileMetrics {
+        filesAnalyzed.incrementAndGet()
+
         val metrics = createMetricsFromMetricNames()
-        file
-            .bufferedReader()
-            .useLines { lines -> lines.forEach { line -> metrics.forEach { it.parseLine(line) } } }
+        file.bufferedReader().useLines { lines -> lines.forEach { line -> metrics.forEach { it.parseLine(line) } } }
 
         if (metrics.isEmpty()) return FileMetrics()
-        return metrics.map { it.getValue() }.reduceRight { current: FileMetrics, acc: FileMetrics ->
+        val resultMetrics = metrics.map { it.getValue() }.reduceRight { current: FileMetrics, acc: FileMetrics ->
             acc.metricsMap.putAll(current.metricsMap)
             acc
         }
+        resultMetrics.checksum = checksum
+        return resultMetrics
     }
 
     private fun getStandardizedPath(file: File): String {
@@ -120,4 +152,16 @@ class ProjectMetricsCollector(
     private fun logProgress(fileName: String, parsedFiles: Long) {
         progressTracker.updateProgress(totalFiles, parsedFiles, parsingUnit.name, fileName)
     }
+
+    private fun logStatistics() {
+        if (baseFileNodeMap.isNotEmpty()) {
+            val skipped = filesSkipped.get()
+            val analyzed = filesAnalyzed.get()
+            val total = skipped + analyzed
+            Logger.info {
+                "Checksum comparison: $skipped files skipped, $analyzed files analyzed " +
+                    "(${skipped * 100 / total.coerceAtLeast(1)}% reused)"
+            }
+        }
+    }
 }