Issue #540 Incremental Builds

Author: Tobias Brennecke

core/lib/lineage_hunter.js

@@ -1,5 +1,7 @@
 "use strict";
 
+var extend = require("util")._extend;
+
 var lineage_hunter = function () {
 
   var pa = require('./pattern_assembler');
@@ -28,6 +30,10 @@ var lineage_hunter = function () {
             l.lineageState = ancestorPattern.patternState;
           }
 
+          patternlab.graph.add(ancestorPattern);
+          // Confusing: pattern includes "ancestorPattern", not the other way round
+          patternlab.graph.link(pattern, ancestorPattern);
+
           pattern.lineage.push(l);
 
           //also, add the lineageR entry if it doesn't exist
@@ -44,62 +50,70 @@ var lineage_hunter = function () {
             }
 
             ancestorPattern.lineageR.push(lr);
+            extend(patternlab.graph.node(ancestorPattern), lr);
           }
         }
       });
     }
   }
 
-  function setPatternState(direction, pattern, targetPattern) {
-    // if the request came from the past, apply target pattern state to current pattern lineage
+  /**
+   * Apply the target pattern state either to any predecessors or successors of the given
+   * pattern in the pattern graph.
+   * @param direction Either 'fromPast' or 'fromFuture'
+   * @param pattern {Pattern}
+   * @param targetPattern {Pattern}
+   * @param graph {PatternGraph}
+   */
+  function setPatternState(direction, pattern, targetPattern, graph) {
+    var index = null;
     if (direction === 'fromPast') {
-      for (var i = 0; i < pattern.lineageIndex.length; i++) {
-        if (pattern.lineageIndex[i] === targetPattern.patternPartial) {
-          pattern.lineage[i].lineageState = targetPattern.patternState;
-        }
-      }
+      index  = graph.lineage(pattern);
     } else {
-      //the request came from the future, apply target pattern state to current pattern reverse lineage
-      for (var i = 0; i < pattern.lineageRIndex.length; i++) {
-        if (pattern.lineageRIndex[i] === targetPattern.patternPartial) {
-          pattern.lineageR[i].lineageState = targetPattern.patternState;
-        }
+      index = graph.lineageR(pattern);
+    }
+    // if the request came from the past, apply target pattern state to current pattern lineage
+    for (var i = 0; i < index.length; i++) {
+      if (index[i].patternPartial === targetPattern.patternPartial) {
+        index[i].lineageState = targetPattern.patternState;
       }
     }
   }
 
 
   function cascadePatternStates(patternlab) {
 
-    var pattern_assembler = new pa();
-
     for (var i = 0; i < patternlab.patterns.length; i++) {
       var pattern = patternlab.patterns[i];
 
       //for each pattern with a defined state
       if (pattern.patternState) {
+        var lineage = patternlab.graph.lineage(pattern);
 
-        if (pattern.lineageIndex && pattern.lineageIndex.length > 0) {
+        if (lineage && lineage.length > 0) {
 
           //find all lineage - patterns being consumed by this one
-          for (var h = 0; h < pattern.lineageIndex.length; h++) {
-            var lineagePattern = pattern_assembler.getPartial(pattern.lineageIndex[h], patternlab);
-            setPatternState('fromFuture', lineagePattern, pattern);
+          for (var h = 0; h < lineage.length; h++) {
+            // Not needed, the graph already knows the concrete pattern
+            // var lineagePattern = pattern_assembler.getPartial(lineageIndex[h], patternlab);
+            setPatternState('fromFuture', lineage[h], pattern, patternlab.graph);
           }
         }
-
-        if (pattern.lineageRIndex && pattern.lineageRIndex.length > 0) {
+        var lineageR = patternlab.graph.lineageR(pattern);
+        if (lineageR && lineageR.length > 0) {
 
           //find all reverse lineage - that is, patterns consuming this one
-          for (var j = 0; j < pattern.lineageRIndex.length; j++) {
+          for (var j = 0; j < lineageR.length; j++) {
 
-            var lineageRPattern = pattern_assembler.getPartial(pattern.lineageRIndex[j], patternlab);
+            var lineageRPattern = lineageR[j];
 
             //only set patternState if pattern.patternState "is less than" the lineageRPattern.patternstate
             //or if lineageRPattern.patternstate (the consuming pattern) does not have a state
             //this makes patternlab apply the lowest common ancestor denominator
-            if (lineageRPattern.patternState === '' || (patternlab.config.patternStateCascade.indexOf(pattern.patternState)
-              < patternlab.config.patternStateCascade.indexOf(lineageRPattern.patternState))) {
+            let patternStateCascade = patternlab.config.patternStateCascade;
+            let patternStateIndex = patternStateCascade.indexOf(pattern.patternState);
+            let patternReverseStateIndex = patternStateCascade.indexOf(lineageRPattern.patternState);
+            if (lineageRPattern.patternState === '' || (patternStateIndex < patternReverseStateIndex)) {
 
               if (patternlab.config.debug) {
                 console.log('Found a lower common denominator pattern state: ' + pattern.patternState + ' on ' + pattern.patternPartial + '. Setting reverse lineage pattern ' + lineageRPattern.patternPartial + ' from ' + (lineageRPattern.patternState === '' ? '<<blank>>' : lineageRPattern.patternState));
@@ -108,9 +122,9 @@ var lineage_hunter = function () {
               lineageRPattern.patternState = pattern.patternState;
 
               //take this opportunity to overwrite the lineageRPattern's lineage state too
-              setPatternState('fromPast', lineageRPattern, pattern);
+              setPatternState('fromPast', lineageRPattern, pattern, patternlab.graph);
             } else {
-              setPatternState('fromPast', pattern, lineageRPattern);
+              setPatternState('fromPast', pattern, lineageRPattern, patternlab.graph);
             }
           }
         }

core/lib/object_factory.js

@@ -73,6 +73,11 @@ var Pattern = function (relPath, data, patternlab) {
   this.isPseudoPattern = false;
   this.order = Number.MAX_SAFE_INTEGER;
   this.engine = patternEngines.getEngineForPattern(this);
+  // For completeness
+  this.compileState = null;
+  // The unix timestamp when the pattern was last modified
+  this.lastModified = null;
+
 };
 
 // Pattern methods
@@ -153,6 +158,13 @@ Pattern.create = function (relPath, data, customProps, patternlab) {
   return extend(newPattern, customProps);
 };
 
+var CompileState = {
+  NEEDS_REBUILD: "needs rebuild",
+  BUILDING: "building",
+  CLEAN: "clean"
+};
+
 module.exports = {
-  Pattern: Pattern
+  Pattern: Pattern,
+  CompileState: CompileState
 };

core/lib/pattern_assembler.js

@@ -3,6 +3,7 @@
 var path = require('path'),
   fs = require('fs-extra'),
   Pattern = require('./object_factory').Pattern,
+  CompileState = require('./object_factory').CompileState,
   pph = require('./pseudopattern_hunter'),
   mp = require('./markdown_parser'),
   plutils = require('./utilities'),
@@ -123,7 +124,7 @@ var pattern_assembler = function () {
       } else {
         patternlab.partials[pattern.patternPartial] = pattern.patternDesc;
       }
-
+      patternlab.graph.add(pattern);
       patternlab.patterns.push(pattern);
 
     }
@@ -240,6 +241,30 @@ var pattern_assembler = function () {
     addPattern(pattern, patternlab);
   }
 
+  function checkBuildState (pattern, patternlab) {
+    //write the compiled template to the public patterns directory
+    var renderedTemplatePath =
+      patternlab.config.paths.public.patterns + pattern.getPatternLink(patternlab, 'rendered');
+
+    if (!pattern.compileState) {
+      pattern.compileState = CompileState.NEEDS_REBUILD;
+    }
+
+    try {
+      // Prevent error message if file does not exist
+      fs.accessSync(renderedTemplatePath, fs.F_OK);
+      var outputLastModified = fs.statSync(renderedTemplatePath).mtime.getTime();
+
+      if (pattern.lastModified && outputLastModified > pattern.lastModified) {
+        pattern.compileState = CompileState.CLEAN;
+      }
+    } catch (e) {
+      // Output does not exist yet, needs recompile
+    }
+    // Make the pattern known to the PatternGraph and remember its compileState
+    patternlab.graph.add(pattern);
+  }
+
   function processPatternIterative(relPath, patternlab) {
 
     var relativeDepth = (relPath.match(/\w(?=\\)|\w(?=\/)/g) || []).length;
@@ -309,7 +334,7 @@ var pattern_assembler = function () {
 
     //see if this file has a state
     setState(currentPattern, patternlab, true);
-
+    var jsonFileLastModified = null;
     //look for a json file for this template
     try {
       var jsonFilename = path.resolve(patternsPath, currentPattern.subdir, currentPattern.fileName + ".json");
@@ -320,6 +345,7 @@ var pattern_assembler = function () {
       }
       if (jsonFilenameStats && jsonFilenameStats.isFile()) {
         currentPattern.jsonFileData = fs.readJSONSync(jsonFilename);
+        jsonFileLastModified = jsonFilenameStats.mtime.getTime();
         if (patternlab.config.debug) {
           console.log('processPatternIterative: found pattern-specific data.json for ' + currentPattern.patternPartial);
         }
@@ -355,7 +381,20 @@ var pattern_assembler = function () {
     parsePatternMarkdown(currentPattern, patternlab);
 
     //add the raw template to memory
-    currentPattern.template = fs.readFileSync(path.resolve(patternsPath, relPath), 'utf8');
+    var templatePath = path.resolve(patternsPath, currentPattern.relPath);
+    if (templatePath) {
+      try {
+        var stat = fs.statSync(templatePath);
+        // Needs recompile whenever either the JSON or the source file has been changed
+        currentPattern.lastModified = Math.max(stat.mtime.getTime(), jsonFileLastModified);
+      } catch (e) {
+        // Ignore, not a regular file
+      }
+    }
+
+    checkBuildState(currentPattern, patternlab);
+
+    currentPattern.template = fs.readFileSync(templatePath, 'utf8');
 
     //find any stylemodifiers that may be in the current pattern
     currentPattern.stylePartials = currentPattern.findPartialsWithStyleModifiers();
@@ -393,6 +432,15 @@ var pattern_assembler = function () {
     decomposePattern(currentPattern, patternlab);
   }
 
+  function findModifiedPatterns(lastModified, patternlab) {
+    return patternlab.patterns.filter( p => {
+      if (p.compileState !== CompileState.CLEAN || ! p.lastModified) {
+        return true;
+      }
+      return p.lastModified >= lastModified;
+    });
+  }
+
   function expandPartials(foundPatternPartials, list_item_hunter, patternlab, currentPattern) {
 
     var style_modifier_hunter = new smh(),
@@ -506,6 +554,9 @@ var pattern_assembler = function () {
   }
 
   return {
+    find_modified_patterns: function (lastModified, patternlab) {
+      return findModifiedPatterns(lastModified, patternlab)
+    },
     find_pattern_partials: function (pattern) {
       return pattern.findPartials();
     },
@@ -533,6 +584,9 @@ var pattern_assembler = function () {
     renderPattern: function (template, data, partials) {
       return renderPattern(template, data, partials);
     },
+    check_build_state: function (pattern, patternlab) {
+      return checkBuildState(pattern, patternlab);
+    },
     process_pattern_iterative: function (file, patternlab) {
       return processPatternIterative(file, patternlab);
     },