diff --git a/packages/static_shock/lib/src/pipeline.dart b/packages/static_shock/lib/src/pipeline.dart index 8746a34..e4811f5 100644 --- a/packages/static_shock/lib/src/pipeline.dart +++ b/packages/static_shock/lib/src/pipeline.dart @@ -5,14 +5,23 @@ import 'package:mason_logger/mason_logger.dart'; import 'package:static_shock/src/data.dart'; import 'package:static_shock/src/files.dart'; import 'package:static_shock/src/finishers.dart'; +import 'package:static_shock/src/source_files.dart'; import 'package:static_shock/src/templates/components.dart'; import 'package:static_shock/src/templates/layouts.dart'; import 'package:static_shock/src/assets.dart'; import 'package:static_shock/src/pages.dart'; +import 'package:static_shock/src/themes.dart'; /// A pipeline that runs a series of steps to generate a static website. abstract class StaticShockPipeline { + /// Adds the given [SourceFiles] to the collection of source files that are + /// pushed through the pipeline. + /// + /// There's only one true source directory, within the project, but users + /// can supplement the collection of source files with extensions. + void addSourceExtension(SourceFiles extensionFiles); + /// Adds the given [picker] to the pipeline, which selects files that will /// be pushed through the pipeline. void pick(Picker picker); @@ -38,6 +47,10 @@ abstract class StaticShockPipeline { Set? pages, }); + /// Loads the given [theme], which is a collection of pages, assets, layouts, + /// and components. + void loadTheme(Theme theme); + /// Adds the given [DataLoader] to the pipeline, which loads external data before /// any assets or pages are loaded. void loadData(DataLoader dataLoader); @@ -112,6 +125,7 @@ class StaticShockPipelineContext { required Directory sourceDirectory, this.buildMode = StaticShockBuildMode.production, this.cliArguments = const [], + required this.buildCacheDirectory, required this.errorLog, Logger? log, }) : _sourceDirectory = sourceDirectory, @@ -123,6 +137,8 @@ class StaticShockPipelineContext { /// {@macro cli_arguments} final List cliArguments; + final Directory buildCacheDirectory; + /// The shared [Logger] for all CLI output. /// /// Plugins should log messages with this [Logger] so that verbosity output level @@ -191,6 +207,7 @@ class StaticShockPipelineContext { /// Returns the [Layout] template from the given file [path]. Layout? getLayout(FileRelativePath path) => _layouts[path]; + final _layouts = {}; /// Adds the given [layout] to the pipeline. diff --git a/packages/static_shock/lib/src/plugins/sass.dart b/packages/static_shock/lib/src/plugins/sass.dart index df247fc..db1a07f 100644 --- a/packages/static_shock/lib/src/plugins/sass.dart +++ b/packages/static_shock/lib/src/plugins/sass.dart @@ -74,6 +74,7 @@ class SassIndexTransformer implements AssetTransformer { } _log.detail("Adding Sass content to cache: ${asset.sourcePath}"); + _log.detail("Content:\n${asset.sourceContent!.text}"); _environment.cache["${asset.destinationPath!.filename}.${asset.destinationPath!.extension}"] = asset.sourceContent!.text!; } diff --git a/packages/static_shock/lib/src/source_files.dart b/packages/static_shock/lib/src/source_files.dart index 2ecea04..6b16fe3 100644 --- a/packages/static_shock/lib/src/source_files.dart +++ b/packages/static_shock/lib/src/source_files.dart @@ -57,6 +57,11 @@ class SourceFiles { file, file.path.substring(directory.path.length), )) // + .where( + (file) => + !file.subPath.startsWith("_includes") && // + !file.subPath.startsWith("/_includes"), + ) .where((file) => !_isExcluded(file.subPath)) .where((file) => filter?.passesFilter(file) ?? true); } diff --git a/packages/static_shock/lib/src/static_shock.dart b/packages/static_shock/lib/src/static_shock.dart index 9624973..84e955c 100644 --- a/packages/static_shock/lib/src/static_shock.dart +++ b/packages/static_shock/lib/src/static_shock.dart @@ -12,6 +12,7 @@ import 'package:static_shock/src/infrastructure/data.dart'; import 'package:static_shock/src/infrastructure/timer.dart'; import 'package:static_shock/src/templates/components.dart'; import 'package:static_shock/src/templates/layouts.dart'; +import 'package:static_shock/src/themes.dart'; import 'package:yaml/yaml.dart'; import 'package:static_shock/src/assets.dart'; @@ -47,6 +48,7 @@ class StaticShock implements StaticShockPipeline { Set? remoteDataPickers, Set? remoteAssetPickers, Set? remotePagePickers, + Set? themes, Set? dataLoaders, Set? assetLoaders, Set? assetTransformers, @@ -68,6 +70,7 @@ class StaticShock implements StaticShockPipeline { { const FilePrefixExcluder("."), }, + _themeLoaders = themes ?? {}, _dataLoaders = dataLoaders ?? {}, _assetLoaders = assetLoaders ?? {}, _assetTransformers = assetTransformers ?? {}, @@ -178,6 +181,14 @@ class StaticShock implements StaticShockPipeline { late Directory _sourceDirectory; late SourceFiles _sourceFiles; + @override + void addSourceExtension(SourceFiles extensionFiles) { + _context.log.detail("Adding extension source set: ${extensionFiles.directory.absolute.path}"); + _sourceExtensions.add(extensionFiles); + } + + final _sourceExtensions = {}; + late Directory _destinationDir; /// Adds the given [picker] to the pipeline, which selects files that will @@ -213,6 +224,12 @@ class StaticShock implements StaticShockPipeline { late final Set _remoteAssets; late final Set _remotePages; + /// Loads the given [theme], which is a collection of pages, assets, layouts, + /// and components. + @override + void loadTheme(Theme theme) => _themeLoaders.add(theme); + late final Set _themeLoaders; + /// Adds the given [DataLoader] to the pipeline, which loads external data before /// any assets or pages are loaded. @override @@ -304,7 +321,8 @@ class StaticShock implements StaticShockPipeline { late ErrorLog _errorLog; late CheckpointTimer _timer; late StaticShockPipelineContext _context; - final _files = []; + // final _files = []; + final _files = []; /// Generates a static site from content and assets. /// @@ -329,12 +347,20 @@ class StaticShock implements StaticShockPipeline { _clearDestination(); + // Ensure the build cache directory exists. This is a cache for intermediate + // artifacts that are used during build, but which are not meant to be + // version controlled, or distributed with the build. + final buildCacheDirectory = + Directory("${_sourceDirectory.path}${path.separator}..${path.separator}.shock${path.separator}build_cache"); + buildCacheDirectory.createSync(recursive: true); + //---- Run new pipeline ---- _errorLog = ErrorLog(); _context = StaticShockPipelineContext( sourceDirectory: _sourceDirectory, buildMode: _buildMode ?? StaticShockBuildMode.production, cliArguments: _cliArguments, + buildCacheDirectory: buildCacheDirectory, errorLog: _errorLog, log: _log, ); @@ -349,6 +375,10 @@ class StaticShock implements StaticShockPipeline { "basePath": _site.basePath, }); + // Load all themes. Themes will add source files, so this needs to happen + // before picking and/or processing. + await _loadThemes(); + // Run plugin configuration - we do this first so that plugins can contribute pickers. _applyPlugins(); @@ -484,31 +514,36 @@ class StaticShock implements StaticShockPipeline { // Load local layouts and components. We do this after loading remove values so // that local values can overwrite remote values. - for (final sourceFile in _sourceFiles.layouts()) { - _log.detail("Layout: ${sourceFile.subPath}"); - _context.putLayout( - Layout( - FileRelativePath.parse(sourceFile.subPath), - sourceFile.file.readAsStringSync(), - ), - ); - } - for (final sourceFile in _sourceFiles.components()) { - _log.detail("Component: ${sourceFile.subPath}"); - - final componentContent = front_matter.parse(sourceFile.file.readAsStringSync()); - - // TODO: process the component data, e.g., pull out CSS imports - _context.putComponent( - path.basenameWithoutExtension(sourceFile.subPath), - Component( - FileRelativePath.parse(sourceFile.subPath), - Map.from(componentContent.data), - // If there's no Front Matter, then `content` will be `null`. In that case, assume - // everything is a Jinja template, and pass the full `value`. - componentContent.content ?? componentContent.value, - ), - ); + final sourceSets = {_sourceFiles, ..._sourceExtensions}; + for (final sourceSet in sourceSets) { + _log.detail("Processing source set: ${sourceSet.directory}"); + for (final sourceFile in sourceSet.layouts()) { + _log.detail("Layout: ${sourceFile.subPath}"); + _context.putLayout( + Layout( + FileRelativePath.parse(sourceFile.subPath), + sourceFile.file.readAsStringSync(), + ), + ); + } + + for (final sourceFile in sourceSet.components()) { + _log.detail("Component: ${sourceFile.subPath}"); + + final componentContent = front_matter.parse(sourceFile.file.readAsStringSync()); + + // TODO: process the component data, e.g., pull out CSS imports + _context.putComponent( + path.basenameWithoutExtension(sourceFile.subPath), + Component( + FileRelativePath.parse(sourceFile.subPath), + Map.from(componentContent.data), + // If there's no Front Matter, then `content` will be `null`. In that case, assume + // everything is a Jinja template, and pass the full `value`. + componentContent.content ?? componentContent.value, + ), + ); + } } _timer.checkpoint("Load layouts & components", "Finds all layout and component files and loads them into memory"); @@ -658,23 +693,29 @@ class StaticShock implements StaticShockPipeline { void _pickAllSourceFiles() { _log.info("⚡ Picking files"); - for (final sourceFile in _sourceFiles.sourceFiles()) { - final relativePath = FileRelativePath.parse(sourceFile.subPath); - - pickerLoop: - for (final picker in _pickers) { - if (picker.shouldPick(relativePath)) { - for (final excluder in _excluders) { - if (excluder.shouldExclude(relativePath)) { - break pickerLoop; + // Order sources from extensions to main sources so that main sources + // overwrite extensions (like themes). + final sourceSets = {..._sourceExtensions, _sourceFiles}; + for (final sourceSet in sourceSets) { + _log.detail("Picking from source set: ${sourceSet.directory}"); + for (final sourceFile in sourceSet.sourceFiles()) { + final relativePath = FileRelativePath.parse(sourceFile.subPath); + + pickerLoop: + for (final picker in _pickers) { + if (picker.shouldPick(relativePath)) { + for (final excluder in _excluders) { + if (excluder.shouldExclude(relativePath)) { + break pickerLoop; + } } - } - _log.detail("Picked: $relativePath"); - _files.add(relativePath); + _log.detail("Picked: $relativePath"); + _files.add(sourceFile); - // We picked the file. No need to check more pickers. - break; + // We picked the file. No need to check more pickers. + break; + } } } } @@ -683,6 +724,18 @@ class StaticShock implements StaticShockPipeline { _log.info(""); } + Future _loadThemes() async { + _log.info("⚡ Loading themes"); + + for (final theme in _themeLoaders) { + _log.detail("Loading theme: ${theme.describe}"); + await theme.load(this, _context); + } + + _timer.checkpoint("Load themes", "Loads all themes, such as from git or the file system"); + _log.info(""); + } + Future _loadExternalData() async { _log.info("⚡ Loading external data"); @@ -755,7 +808,8 @@ class StaticShock implements StaticShockPipeline { for (final pickedFile in _files) { late AssetContent content; - final file = _resolveSourceFile(pickedFile); + final file = pickedFile.file; + final relativePath = FileRelativePath.parse(pickedFile.subPath); try { // Try to read as plain text, first. final textContent = file.readAsStringSync(); @@ -773,12 +827,12 @@ class StaticShock implements StaticShockPipeline { // Try to interpret the file as a page. If it is a page, load the page. for (final pageLoader in _pageLoaders) { - if (!pageLoader.canLoad(pickedFile)) { + if (!pageLoader.canLoad(relativePath)) { continue; } _log.detail("Loading page: $pickedFile"); - final page = await pageLoader.loadPage(pickedFile, content.text!); + final page = await pageLoader.loadPage(relativePath, content.text!); final inheritedData = _context.dataIndex.inheritDataForPath(page.sourcePath); page.data.addEntries({ @@ -819,11 +873,11 @@ class StaticShock implements StaticShockPipeline { // The file isn't a page, therefore it must be an asset. _log.detail("Loading asset: $pickedFile"); _context.addAsset(Asset( - sourcePath: pickedFile, + sourcePath: relativePath, sourceContent: content, // By default, we assume a direct copy of each asset. Asset transformers // can change this decision later. - destinationPath: pickedFile, + destinationPath: relativePath, destinationContent: content, )); } @@ -987,6 +1041,7 @@ class StaticShock implements StaticShockPipeline { for (final renderer in _pageRenderers) { if (renderer.id == rendererId) { _log.detail("Rendering page '${page.title}' content as '$rendererId'"); + _log.detail(" - page path: ${page.pagePath}"); didRender = true; await renderer.renderContent(_context, page); } diff --git a/packages/static_shock/lib/src/themes.dart b/packages/static_shock/lib/src/themes.dart new file mode 100644 index 0000000..eb1b127 --- /dev/null +++ b/packages/static_shock/lib/src/themes.dart @@ -0,0 +1,74 @@ +import 'dart:io'; + +import 'package:path/path.dart'; +import 'package:static_shock/src/files.dart'; +import 'package:static_shock/src/pipeline.dart'; +import 'package:static_shock/src/source_files.dart'; + +abstract class Theme { + static Theme fromGit({ + required String url, + String? path, + String? ref, + }) => + GitTheme(url: url, path: path, ref: ref); + + Future load(StaticShockPipeline pipeline, StaticShockPipelineContext context); + + String get describe; +} + +class GitTheme implements Theme { + const GitTheme({ + required this.url, + this.path, + this.ref, + }); + + final String url; + final String? path; + final String? ref; + + @override + Future load(StaticShockPipeline pipeline, StaticShockPipelineContext context) async { + final normalizedUrl = Uri.parse(url).normalizePath(); + final directoryName = "${normalizedUrl.host}/${normalizedUrl.path}".replaceAll("/", "_"); + + final cloneDirectory = context.buildCacheDirectory.subDir(["themes", "git", directoryName]).absolute; + cloneDirectory.createSync(recursive: true); + + // Clone the theme repo into the build cache. This repo might already exist + // here in the cache, but trying to clone again shouldn't cause any issues. + await Process.run("git", ["clone", url, cloneDirectory.path]); + + // Checkout the desired ref. + await Process.run("git", ["fetch", "origin"], workingDirectory: cloneDirectory.path); + if (ref != null) { + final result = await Process.run("git", ["checkout", ref!], workingDirectory: cloneDirectory.path); + if (result.exitCode != 0) { + context.errorLog.crash("Failed to checkout Git theme ref ($ref): ${result.stderr}"); + } + } else { + final result = await Process.run("git", ["checkout", "HEAD"], workingDirectory: cloneDirectory.path); + if (result.exitCode != 0) { + context.errorLog.crash("Failed to checkout Git theme ref (HEAD): ${result.stderr}"); + } + } + + // Pull the latest for the branch. + await Process.run("git", ["pull"], workingDirectory: cloneDirectory.path); + + // Walk all files within the desired path and add them to the pipeline. + final themeDirectory = path != null ? cloneDirectory.subDir(path!.split(separator)) : cloneDirectory; + + final sourceFiles = SourceFiles( + directory: themeDirectory, + excludedPaths: {}, + ); + + pipeline.addSourceExtension(sourceFiles); + } + + @override + String get describe => "Theme (from git) - url: $url, path: ${path ?? "none"}, ref: ${ref ?? "none"}"; +} diff --git a/packages/static_shock/lib/static_shock.dart b/packages/static_shock/lib/static_shock.dart index b44a3d3..f7a8fdb 100644 --- a/packages/static_shock/lib/static_shock.dart +++ b/packages/static_shock/lib/static_shock.dart @@ -8,6 +8,7 @@ export 'src/pages.dart'; export 'src/pipeline.dart'; export 'src/source_files.dart'; export 'src/static_shock.dart'; +export 'src/themes.dart'; export 'src/templates/components.dart'; export 'src/templates/layouts.dart'; diff --git a/packages/static_shock_docs/.gitignore b/packages/static_shock_docs/.gitignore index bd369ca..f722e7f 100644 --- a/packages/static_shock_docs/.gitignore +++ b/packages/static_shock_docs/.gitignore @@ -1,10 +1,8 @@ # Build output /build -# Shock cache and temp files -# (Jan 12, 2024) - I started committing the cache because we're able to generate screenshots -# locally, but puppeteer is broken on CI. -#/source/.shock/ +# Build cache +.shock # Puppeteer cached output .local-chrome diff --git a/packages/static_shock_docs/bin/static_shock_docs.dart b/packages/static_shock_docs/bin/static_shock_docs.dart index dd3ce6c..7b9110b 100644 --- a/packages/static_shock_docs/bin/static_shock_docs.dart +++ b/packages/static_shock_docs/bin/static_shock_docs.dart @@ -11,6 +11,12 @@ Future main(List arguments) async { ), cliArguments: arguments, ) + ..loadTheme( + Theme.fromGit( + url: "https://github.com/flutter-bounty-hunters/fbh_docs_theme", + path: "theme", + ), + ) ..pick(DirectoryPicker.parse("images")) ..plugin(const MarkdownPlugin()) ..plugin(JinjaPlugin( diff --git a/packages/static_shock_docs/source/test-layout.md b/packages/static_shock_docs/source/test-layout.md new file mode 100644 index 0000000..0aed46c --- /dev/null +++ b/packages/static_shock_docs/source/test-layout.md @@ -0,0 +1,25 @@ +--- +title: Test Docs Layout +layout: layouts/docs_page.jinja + +# Configuration for the package that this website documents. +package: + name: super_editor + title: Super Editor + description: Tools for easily simulating user behavior in widget tests + is_on_pub: true + github: + url: https://github.com/superlistapp/super_editor + organization: superlistapp + name: super_editor + discord: https://discord.gg/8hna2VD32s + sponsorship: https://flutterbountyhunters.com + +# Configuration of the GitHub plugin for loading info about GitHub repositories. +github: + contributors: + repositories: + - { organization: superlistapp, name: super_editor } +--- +# Docs Page Layout Test +This is a test of the remote layout from GitHub. diff --git a/template_sources/template_blog/.gitignore b/template_sources/template_blog/.gitignore index 035bb7d..299c241 100644 --- a/template_sources/template_blog/.gitignore +++ b/template_sources/template_blog/.gitignore @@ -1,8 +1,8 @@ # Build output /build -# Shock cache and temp files -/source/.shock/ +# Build cache +.shock # Puppeteer cached output .local-chrome diff --git a/template_sources/template_docs_multi_page/.gitignore b/template_sources/template_docs_multi_page/.gitignore index 035bb7d..299c241 100644 --- a/template_sources/template_docs_multi_page/.gitignore +++ b/template_sources/template_docs_multi_page/.gitignore @@ -1,8 +1,8 @@ # Build output /build -# Shock cache and temp files -/source/.shock/ +# Build cache +.shock # Puppeteer cached output .local-chrome diff --git a/template_sources/template_empty/.gitignore b/template_sources/template_empty/.gitignore index 035bb7d..299c241 100644 --- a/template_sources/template_empty/.gitignore +++ b/template_sources/template_empty/.gitignore @@ -1,8 +1,8 @@ # Build output /build -# Shock cache and temp files -/source/.shock/ +# Build cache +.shock # Puppeteer cached output .local-chrome