Added several output and processing control keys

Author: daveaglick

RELEASE.md

@@ -1,5 +1,8 @@
 # 1.0.0-alpha.10
 
+- Added a new `RenderPostProcessTemplates` key that prevents running post-processing templates like Razor.
+- Added a new `ShouldOutput` key that controls outputting a particular document to disk (but doesn't remove it from the pipeline like `Excluded` does).
+- Added support for directory metadata (by default as `_directory.yaml` files).
 - Added new `ContentFiles` and `DataFiles` settings to control the file globbing patterns.
 - Added a new `GenerateSitemap` setting and `Sitemap` pipeline to generate sitemap files by default.
 - Added a new `Excluded` key that indicates a document should be filtered out of the content or data pipeline.

src/Statiq.Web/BootstrapperFactoryExtensions.cs

@@ -25,6 +25,7 @@ public static Bootstrapper CreateWeb(this BootstrapperFactory factory, string[]
                 {
                     { WebKeys.ContentFiles, "**/{!_,}*.{html,cshtml,md}" },
                     { WebKeys.DataFiles, "**/{!_,}*.{json,yaml,yml}" },
+                    { WebKeys.DirectoryMetadataFiles, "**/_{d,D}irectory.{json,yaml,yml}" },
                     { WebKeys.MirrorResources, true },
                     { WebKeys.ValidateRelativeLinks, true },
                     { WebKeys.GenerateSitemap, true },

src/Statiq.Web/Modules/ApplyDirectoryMetadata.cs

@@ -0,0 +1,76 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Linq;
+using Statiq.Common;
+using Statiq.Web.Pipelines;
+
+namespace Statiq.Web.Modules
+{
+    /// <summary>
+    /// Applies directory metadata documents to the input documents. Make sure this module
+    /// is executed before applying local metadata like front matter or it will get overwritten
+    /// by the directory metadata.
+    /// </summary>
+    public class ApplyDirectoryMetadata : SyncModule
+    {
+        protected override IEnumerable<IDocument> ExecuteContext(IExecutionContext context)
+        {
+            // Figure out relative input paths for all metadata documents
+            ImmutableDictionary<string, ImmutableArray<DirectoryMetadataData>> directoryMetadata = context
+                .Outputs[nameof(DirectoryMetadata)]
+                .GroupBy(x => x.Source.Parent.GetRelativeInputPath().FullPath)
+                .ToImmutableDictionary(
+                    x => x.Key,
+                    x => x
+                        .Select(doc => new DirectoryMetadataData
+                        {
+                            Recursive = doc.GetBool(WebKeys.Recursive, true),
+                            Metadata = doc.GetRawEnumerable().ToImmutableArray()
+                        })
+                        .ToImmutableArray());
+
+            // Iterate through all input documents in parallel
+            return context.Inputs.Select(input =>
+            {
+                IDocument merged = input;
+
+                // Get all matching directory metadata documents
+                NormalizedPath path = input.Source.Parent.GetRelativeInputPath();
+                Stack<DirectoryMetadataData> matchStack = new Stack<DirectoryMetadataData>();
+                bool local = true;
+                while (!path.IsNull)
+                {
+                    if (directoryMetadata.TryGetValue(path.FullPath, out ImmutableArray<DirectoryMetadataData> matches))
+                    {
+                        foreach (DirectoryMetadataData match in matches.Where(x => local || x.Recursive))
+                        {
+                            matchStack.Push(match);
+                        }
+                    }
+                    if (path.IsNullOrEmpty)
+                    {
+                        break;
+                    }
+                    local = false;
+                    path = path.Parent;
+                }
+
+                // Apply matches if there are any
+                while (matchStack.Count > 0)
+                {
+                    DirectoryMetadataData match = matchStack.Pop();
+                    merged = merged.Clone(match.Metadata);
+                }
+
+                return merged;
+            });
+        }
+
+        private class DirectoryMetadataData
+        {
+            public bool Recursive { get; set; }
+            public ImmutableArray<KeyValuePair<string, object>> Metadata { get; set; }
+        }
+    }
+}

src/Statiq.Web/Modules/RenderPostProcessTemplates.cs

@@ -12,17 +12,22 @@ public class RenderPostProcessTemplates : ForAllDocuments
     {
         public RenderPostProcessTemplates(Templates templates)
             : base(
-                templates.GetModules(Phase.PostProcess)
-                    .Concat(new IModule[]
+                new IModule[]
+                {
+                    new ExecuteIf(
+                        Config.FromDocument(WebKeys.RenderPostProcessTemplates, true),
+                        templates.GetModules(Phase.PostProcess))
+                }
+                .Concat(new IModule[]
+                {
+                    new ProcessShortcodes(),
+                    new ExecuteIf(Config.FromSetting<bool>(WebKeys.MirrorResources))
                     {
-                        new ProcessShortcodes(),
-                        new ExecuteIf(Config.FromSetting<bool>(WebKeys.MirrorResources))
-                        {
-                            new MirrorResources()
-                        },
-                        new ResolveXrefs()
-                    })
-                    .ToArray())
+                        new MirrorResources()
+                    },
+                    new ResolveXrefs()
+                })
+                .ToArray())
         {
         }
     }

src/Statiq.Web/Pipelines/Content.cs

@@ -1,24 +1,17 @@
-using System;
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.Linq;
-using System.Text;
-using Microsoft.AspNetCore.Mvc.Formatters;
-using Statiq.App;
 using Statiq.Common;
 using Statiq.Core;
 using Statiq.Html;
-using Statiq.Markdown;
-using Statiq.Razor;
 using Statiq.Web.Modules;
-using Statiq.Yaml;
 
 namespace Statiq.Web.Pipelines
 {
     public class Content : Pipeline
     {
         public Content(Templates templates)
         {
-            Dependencies.Add(nameof(Data));
+            Dependencies.AddRange(nameof(Data), nameof(DirectoryMetadata));
 
             InputModules = new ModuleList
             {
@@ -30,6 +23,9 @@ public Content(Templates templates)
                 // Concat all documents from externally declared dependencies (exclude explicit dependencies above like "Data")
                 new ConcatDocuments(Config.FromContext<IEnumerable<IDocument>>(ctx => ctx.Outputs.FromPipelines(ctx.Pipeline.GetAllDependencies(ctx).Except(Dependencies).ToArray()))),
 
+                // Apply directory metadata
+                new ApplyDirectoryMetadata(),
+
                 // Process front matter and sidecar files
                 new ProcessMetadata(),
 
@@ -62,6 +58,7 @@ public Content(Templates templates)
 
             OutputModules = new ModuleList
             {
+                new FilterDocuments(Config.FromDocument(WebKeys.ShouldOutput, true)),
                 new WriteFiles()
             };
         }

src/Statiq.Web/Pipelines/Data.cs

@@ -1,13 +1,8 @@
-using System;
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.Linq;
-using System.Text;
-using Statiq.App;
 using Statiq.Common;
 using Statiq.Core;
-using Statiq.Html;
-using Statiq.Markdown;
-using Statiq.Razor;
+using Statiq.Web.Modules;
 using Statiq.Yaml;
 
 namespace Statiq.Web.Pipelines
@@ -16,6 +11,8 @@ public class Data : Pipeline
     {
         public Data()
         {
+            Dependencies.Add(nameof(DirectoryMetadata));
+
             InputModules = new ModuleList
             {
                 new ReadFiles(Config.FromSetting<IEnumerable<string>>(WebKeys.DataFiles))
@@ -26,6 +23,9 @@ public Data()
                 // Concat all documents from externally declared dependencies (exclude explicit dependencies above)
                 new ConcatDocuments(Config.FromContext<IEnumerable<IDocument>>(ctx => ctx.Outputs.FromPipelines(ctx.Pipeline.GetAllDependencies(ctx).Except(Dependencies).ToArray()))),
 
+                // Apply directory metadata
+                new ApplyDirectoryMetadata(),
+
                 // Parse the content into metadata depending on the content type
                 new ExecuteSwitch(Config.FromDocument(doc => doc.ContentProvider.MediaType))
                     .Case(MediaTypes.Json, new ParseJson())
@@ -46,7 +46,7 @@ public Data()
 
             OutputModules = new ModuleList
             {
-                new FilterDocuments(Config.FromSetting<bool>(WebKeys.OutputData)),
+                new FilterDocuments(Config.FromDocument<bool>(WebKeys.ShouldOutput)),
                 new WriteFiles()
             };
         }

src/Statiq.Web/Pipelines/DirectoryMetadata.cs

@@ -0,0 +1,34 @@
+using System;
+using System.Collections.Generic;
+using System.IO.Pipelines;
+using System.Linq;
+using System.Text;
+using Statiq.App;
+using Statiq.Common;
+using Statiq.Core;
+using Statiq.Html;
+using Statiq.Markdown;
+using Statiq.Razor;
+using Statiq.Yaml;
+
+namespace Statiq.Web.Pipelines
+{
+    public class DirectoryMetadata : Pipeline
+    {
+        public DirectoryMetadata()
+        {
+            InputModules = new ModuleList
+            {
+                new ReadFiles(Config.FromSetting<IEnumerable<string>>(WebKeys.DirectoryMetadataFiles))
+            };
+
+            ProcessModules = new ModuleList
+            {
+                // Parse the content into metadata depending on the content type
+                new ExecuteSwitch(Config.FromDocument(doc => doc.ContentProvider.MediaType))
+                    .Case(MediaTypes.Json, new ParseJson())
+                    .Case(MediaTypes.Yaml, new ParseYaml())
+            };
+        }
+    }
+}

src/Statiq.Web/Pipelines/Sitemap.cs

@@ -16,10 +16,13 @@ public Sitemap()
             {
                 new ExecuteIf(Config.FromSetting<bool>(WebKeys.GenerateSitemap))
                 {
-                    new ConcatDocuments(nameof(Content)),
+                    new ConcatDocuments(nameof(Content))
+                    {
+                        new FilterDocuments(Config.FromDocument(WebKeys.ShouldOutput, true))
+                    },
                     new ConcatDocuments(nameof(Data))
                     {
-                        new FilterDocuments(Config.FromSetting<bool>(WebKeys.OutputData))
+                        new FilterDocuments(Config.FromDocument<bool>(WebKeys.ShouldOutput))
                     },
                     new ConcatDocuments(nameof(Archives)),
                     new FlattenTree(),

src/Statiq.Web/WebKeys.cs

@@ -25,6 +25,11 @@ public static class WebKeys
         /// </summary>
         public const string DataFiles = nameof(DataFiles);
 
+        /// <summary>
+        /// The globbing pattern(s) that will be used to read directory metadata.
+        /// </summary>
+        public const string DirectoryMetadataFiles = nameof(DirectoryMetadataFiles);
+
         public const string OptimizeContentFileNames = nameof(OptimizeContentFileNames);
 
         public const string OptimizeDataFileNames = nameof(OptimizeDataFileNames);
@@ -160,9 +165,10 @@ public static class WebKeys
         public const string FeedItemThreadUpdated = nameof(FeedItemThreadUpdated);
 
         /// <summary>
-        /// Indicates that the data file (.json, .yaml, etc.) should be output (by default data files are not output).
+        /// Indicates that the content or data file should be output.
+        /// By default content files are output and data files are not.
         /// </summary>
-        public const string OutputData = nameof(OutputData);
+        public const string ShouldOutput = nameof(ShouldOutput);
 
         /// <summary>
         /// Indicates the layout file that should be used for this document.
@@ -175,5 +181,21 @@ public static class WebKeys
         /// if no <see cref="Title"/> metadata is defined for the document).
         /// </summary>
         public const string Xref = nameof(Xref);
+
+        /// <summary>
+        /// Used with directory metadata to indicate if the metadata should be applied
+        /// recursively to files in child directories (the default is <c>true</c>).
+        /// </summary>
+        public const string Recursive = nameof(Recursive);
+
+        /// <summary>
+        /// Indicates that post-process templates should be rendered (the default is <c>true</c>).
+        /// </summary>
+        /// <remarks>
+        /// Set this to <c>false</c> for a document to prevent rendering post-process templates such as Razor.
+        /// This can be helpful when you have small bits of content like Markdown that you want to render
+        /// as HTML but not as an entire page so that it can be included in other pages.
+        /// </remarks>
+        public const string RenderPostProcessTemplates = nameof(RenderPostProcessTemplates);
     }
 }