Merge pull request #569 from pattern-lab/features/incremental-build

Features/incremental build

Author: bmuenzenmeyer

.eslintrc

@@ -1,7 +1,8 @@
 {
   "env": {
     "node": true,
-    "builtin": true
+    "builtin": true,
+    "es6": true
   },
   "parserOptions": {
     "ecmaVersion": 6,

.gitignore

@@ -8,3 +8,5 @@ Thumbs.db
 source/css/style.css.map
 .idea/
 public
+!test/patterns/public/.gitkeep
+!test/patterns/testDependencyGraph.json

core/lib/changes_hunter.js

@@ -0,0 +1,81 @@
+"use strict";
+const fs = require("fs-extra"),
+  CompileState = require('./object_factory').CompileState;
+
+/**
+ * For detecting changed patterns.
+ * @constructor
+ */
+let ChangesHunter = function () {
+};
+
+ChangesHunter.prototype = {
+
+  /**
+   * Checks the build state of a pattern by comparing the modification date of the rendered output
+   * file with the {@link Pattern.lastModified}. If the pattern was modified after the last
+   * time it has been rendered, it is flagged for rebuilding via {@link CompileState.NEEDS_REBUILD}.
+   *
+   * @param {Pattern} pattern
+   * @param patternlab
+   *
+   * @see {@link CompileState}
+   */
+  checkBuildState: function (pattern, patternlab) {
+
+    //write the compiled template to the public patterns directory
+    let renderedTemplatePath =
+      patternlab.config.paths.public.patterns + pattern.getPatternLink(patternlab, 'rendered');
+
+    if (!pattern.compileState) {
+      pattern.compileState = CompileState.NEEDS_REBUILD;
+    }
+
+    try {
+      // Prevent error message if file does not exist
+      fs.accessSync(renderedTemplatePath, fs.F_OK);
+      let outputLastModified = fs.statSync(renderedTemplatePath).mtime.getTime();
+
+      if (pattern.lastModified && outputLastModified > pattern.lastModified) {
+        pattern.compileState = CompileState.CLEAN;
+      }
+    } catch (e) {
+      // Output does not exist yet, needs recompile
+    }
+
+    let node = patternlab.graph.node(pattern);
+
+    // Make the pattern known to the PatternGraph and remember its compileState
+    if (!node) {
+      patternlab.graph.add(pattern);
+    } else {
+      // Works via object reference, so we directly manipulate the node data here
+      node.compileState = pattern.compileState;
+    }
+
+
+  },
+
+  /**
+   * Updates {Pattern#lastModified} to the files modification date if the file was modified
+   * after {Pattern#lastModified}.
+   *
+   * @param {Pattern} currentPattern
+   * @param {string} file
+   */
+  checkLastModified: function (currentPattern, file) {
+    if (file) {
+      try {
+        let stat = fs.statSync(file);
+
+        // Needs recompile whenever one of the patterns files (template, json, pseudopatterns) changed
+        currentPattern.lastModified =
+          Math.max(stat.mtime.getTime(), currentPattern.lastModified || 0);
+      } catch (e) {
+        // Ignore, not a regular file
+      }
+    }
+  }
+};
+
+module.exports = ChangesHunter;

core/lib/lineage_hunter.js

@@ -1,10 +1,14 @@
 "use strict";
 
+var extend = require("util")._extend;
+
 var lineage_hunter = function () {
 
   var pa = require('./pattern_assembler');
 
   function findlineage(pattern, patternlab) {
+    // As we are adding edges from pattern to ancestor patterns, ensure it is known to the graph
+    patternlab.graph.add(pattern);
 
     var pattern_assembler = new pa();
 
@@ -28,6 +32,11 @@ var lineage_hunter = function () {
             l.lineageState = ancestorPattern.patternState;
           }
 
+          patternlab.graph.add(ancestorPattern);
+
+          // Confusing: pattern includes "ancestorPattern", not the other way round
+          patternlab.graph.link(pattern, ancestorPattern);
+
           pattern.lineage.push(l);
 
           //also, add the lineageR entry if it doesn't exist
@@ -44,62 +53,71 @@ var lineage_hunter = function () {
             }
 
             ancestorPattern.lineageR.push(lr);
+            extend(patternlab.graph.node(ancestorPattern), lr);
           }
         }
       });
     }
   }
 
-  function setPatternState(direction, pattern, targetPattern) {
-    // if the request came from the past, apply target pattern state to current pattern lineage
+  /**
+   * Apply the target pattern state either to any predecessors or successors of the given
+   * pattern in the pattern graph.
+   * @param direction Either 'fromPast' or 'fromFuture'
+   * @param pattern {Pattern}
+   * @param targetPattern {Pattern}
+   * @param graph {PatternGraph}
+   */
+  function setPatternState(direction, pattern, targetPattern, graph) {
+    var index = null;
     if (direction === 'fromPast') {
-      for (var i = 0; i < pattern.lineageIndex.length; i++) {
-        if (pattern.lineageIndex[i] === targetPattern.patternPartial) {
-          pattern.lineage[i].lineageState = targetPattern.patternState;
-        }
-      }
+      index = graph.lineage(pattern);
     } else {
-      //the request came from the future, apply target pattern state to current pattern reverse lineage
-      for (var i = 0; i < pattern.lineageRIndex.length; i++) {
-        if (pattern.lineageRIndex[i] === targetPattern.patternPartial) {
-          pattern.lineageR[i].lineageState = targetPattern.patternState;
-        }
+      index = graph.lineageR(pattern);
+    }
+
+    // if the request came from the past, apply target pattern state to current pattern lineage
+    for (var i = 0; i < index.length; i++) {
+      if (index[i].patternPartial === targetPattern.patternPartial) {
+        index[i].lineageState = targetPattern.patternState;
       }
     }
   }
 
 
   function cascadePatternStates(patternlab) {
 
-    var pattern_assembler = new pa();
-
     for (var i = 0; i < patternlab.patterns.length; i++) {
       var pattern = patternlab.patterns[i];
 
       //for each pattern with a defined state
       if (pattern.patternState) {
+        var lineage = patternlab.graph.lineage(pattern);
 
-        if (pattern.lineageIndex && pattern.lineageIndex.length > 0) {
+        if (lineage && lineage.length > 0) {
 
           //find all lineage - patterns being consumed by this one
-          for (var h = 0; h < pattern.lineageIndex.length; h++) {
-            var lineagePattern = pattern_assembler.getPartial(pattern.lineageIndex[h], patternlab);
-            setPatternState('fromFuture', lineagePattern, pattern);
+          for (var h = 0; h < lineage.length; h++) {
+            // Not needed, the graph already knows the concrete pattern
+            // var lineagePattern = pattern_assembler.getPartial(lineageIndex[h], patternlab);
+            setPatternState('fromFuture', lineage[h], pattern, patternlab.graph);
           }
         }
-
-        if (pattern.lineageRIndex && pattern.lineageRIndex.length > 0) {
+        var lineageR = patternlab.graph.lineageR(pattern);
+        if (lineageR && lineageR.length > 0) {
 
           //find all reverse lineage - that is, patterns consuming this one
-          for (var j = 0; j < pattern.lineageRIndex.length; j++) {
+          for (var j = 0; j < lineageR.length; j++) {
 
-            var lineageRPattern = pattern_assembler.getPartial(pattern.lineageRIndex[j], patternlab);
+            var lineageRPattern = lineageR[j];
 
             //only set patternState if pattern.patternState "is less than" the lineageRPattern.patternstate
             //or if lineageRPattern.patternstate (the consuming pattern) does not have a state
             //this makes patternlab apply the lowest common ancestor denominator
-            if (lineageRPattern.patternState === '' || (patternlab.config.patternStateCascade.indexOf(pattern.patternState)
-              < patternlab.config.patternStateCascade.indexOf(lineageRPattern.patternState))) {
+            let patternStateCascade = patternlab.config.patternStateCascade;
+            let patternStateIndex = patternStateCascade.indexOf(pattern.patternState);
+            let patternReverseStateIndex = patternStateCascade.indexOf(lineageRPattern.patternState);
+            if (lineageRPattern.patternState === '' || (patternStateIndex < patternReverseStateIndex)) {
 
               if (patternlab.config.debug) {
                 console.log('Found a lower common denominator pattern state: ' + pattern.patternState + ' on ' + pattern.patternPartial + '. Setting reverse lineage pattern ' + lineageRPattern.patternPartial + ' from ' + (lineageRPattern.patternState === '' ? '<<blank>>' : lineageRPattern.patternState));
@@ -108,9 +126,9 @@ var lineage_hunter = function () {
               lineageRPattern.patternState = pattern.patternState;
 
               //take this opportunity to overwrite the lineageRPattern's lineage state too
-              setPatternState('fromPast', lineageRPattern, pattern);
+              setPatternState('fromPast', lineageRPattern, pattern, patternlab.graph);
             } else {
-              setPatternState('fromPast', pattern, lineageRPattern);
+              setPatternState('fromPast', pattern, lineageRPattern, patternlab.graph);
             }
           }
         }

core/lib/object_factory.js

@@ -73,6 +73,21 @@ var Pattern = function (relPath, data, patternlab) {
   this.isPseudoPattern = false;
   this.order = Number.MAX_SAFE_INTEGER;
   this.engine = patternEngines.getEngineForPattern(this);
+
+  /**
+   * Determines if this pattern needs to be recompiled.
+   *
+   * @ee {@link CompileState}*/
+  this.compileState = null;
+
+  /**
+   * Timestamp in milliseconds when the pattern template or auxilary file (e.g. json) were modified.
+   * If multiple files are affected, this is the timestamp of the most recent change.
+   *
+   * @see {@link pattern}
+   */
+  this.lastModified = null;
+
 };
 
 // Pattern methods
@@ -141,7 +156,16 @@ Pattern.prototype = {
 // factory: creates an empty Pattern for miscellaneous internal use, such as
 // by list_item_hunter
 Pattern.createEmpty = function (customProps, patternlab) {
-  var pattern = new Pattern('', null, patternlab);
+  let relPath = '';
+  if (customProps) {
+    if (customProps.relPath) {
+      relPath = customProps.relPath;
+    } else if (customProps.subdir && customProps.filename) {
+      relPath = customProps.subdir + path.sep + customProps.filename;
+    }
+  }
+
+  var pattern = new Pattern(relPath, null, patternlab);
   return extend(pattern, customProps);
 };
 
@@ -153,6 +177,13 @@ Pattern.create = function (relPath, data, customProps, patternlab) {
   return extend(newPattern, customProps);
 };
 
+var CompileState = {
+  NEEDS_REBUILD: "needs rebuild",
+  BUILDING: "building",
+  CLEAN: "clean"
+};
+
 module.exports = {
-  Pattern: Pattern
+  Pattern: Pattern,
+  CompileState: CompileState
 };

core/lib/pattern_assembler.js

@@ -3,6 +3,7 @@
 var path = require('path'),
   fs = require('fs-extra'),
   Pattern = require('./object_factory').Pattern,
+  CompileState = require('./object_factory').CompileState,
   pph = require('./pseudopattern_hunter'),
   mp = require('./markdown_parser'),
   plutils = require('./utilities'),
@@ -11,9 +12,11 @@ var path = require('path'),
   lih = require('./list_item_hunter'),
   smh = require('./style_modifier_hunter'),
   ph = require('./parameter_hunter'),
+  ch = require('./changes_hunter'),
   JSON5 = require('json5');
 
 var markdown_parser = new mp();
+var changes_hunter = new ch();
 
 var pattern_assembler = function () {
   // HELPER FUNCTIONS
@@ -123,7 +126,7 @@ var pattern_assembler = function () {
       } else {
         patternlab.partials[pattern.patternPartial] = pattern.patternDesc;
       }
-
+      patternlab.graph.add(pattern);
       patternlab.patterns.push(pattern);
 
     }
@@ -355,14 +358,22 @@ var pattern_assembler = function () {
     parsePatternMarkdown(currentPattern, patternlab);
 
     //add the raw template to memory
-    currentPattern.template = fs.readFileSync(path.resolve(patternsPath, relPath), 'utf8');
+    var templatePath = path.resolve(patternsPath, currentPattern.relPath);
+
+    currentPattern.template = fs.readFileSync(templatePath, 'utf8');
 
     //find any stylemodifiers that may be in the current pattern
     currentPattern.stylePartials = currentPattern.findPartialsWithStyleModifiers();
 
     //find any pattern parameters that may be in the current pattern
     currentPattern.parameteredPartials = currentPattern.findPartialsWithPatternParameters();
 
+    [templatePath, jsonFilename, listJsonFileName].forEach(file => {
+      changes_hunter.checkLastModified(currentPattern, file);
+    });
+
+    changes_hunter.checkBuildState(currentPattern, patternlab);
+
     //add currentPattern to patternlab.patterns array
     addPattern(currentPattern, patternlab);
 
@@ -393,6 +404,15 @@ var pattern_assembler = function () {
     decomposePattern(currentPattern, patternlab);
   }
 
+  function findModifiedPatterns(lastModified, patternlab) {
+    return patternlab.patterns.filter(p => {
+      if (p.compileState !== CompileState.CLEAN || ! p.lastModified) {
+        return true;
+      }
+      return p.lastModified >= lastModified;
+    });
+  }
+
   function expandPartials(foundPatternPartials, list_item_hunter, patternlab, currentPattern) {
 
     var style_modifier_hunter = new smh(),
@@ -506,6 +526,9 @@ var pattern_assembler = function () {
   }
 
   return {
+    find_modified_patterns: function (lastModified, patternlab) {
+      return findModifiedPatterns(lastModified, patternlab);
+    },
     find_pattern_partials: function (pattern) {
       return pattern.findPartials();
     },