Merge pull request #11 from JuliaComputing/sp/release-ready

Get this ready for release

Author: pfitzseb

.github/workflows/CI.yml

@@ -0,0 +1,41 @@
+name: CI
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+    tags: '*'
+jobs:
+  test:
+    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - '1.6'
+          - '1.8'
+        os:
+          - ubuntu-latest
+        arch:
+          - x64
+    env:
+      PYTHON: ""
+    steps:
+      - uses: actions/checkout@v2
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: ${{ matrix.version }}
+          arch: ${{ matrix.arch }}
+      - uses: actions/cache@v1
+        env:
+          cache-name: cache-artifacts
+        with:
+          path: ~/.julia/artifacts
+          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-test-${{ env.cache-name }}-
+            ${{ runner.os }}-test-
+            ${{ runner.os }}-
+      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/julia-runtest@v1

Project.toml

@@ -5,15 +5,13 @@ version = "0.1.0"
 
 [deps]
 AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
-Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 Gumbo = "708ec375-b3d6-5a57-a7ce-8257bf98657a"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 NodeJS = "2bd173c7-0d6d-553b-b6af-13a54713934c"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 
 [compat]
 AbstractTrees = "0.4"
-Documenter = "0.27"
 Gumbo = "0.8"
 JSON = "0.20,0.21"
 NodeJS = "1"

README.md

@@ -4,31 +4,34 @@ Example usage:
 ```julia
 using MultiDocumenter
 
+clonedir = mktempdir()
+
 docs = [
-    MultiDocumenter.MultiDocRef(
-        upstream = "/home/pfitzseb/.julia/dev/Documenter/docs/build",
-        path = "documenter",
-        name = "Home"
+    ("JuliaDocs/Documenter.jl.git", "gh-pages") => MultiDocumenter.MultiDocRef(
+        upstream = joinpath(clonedir, "Documenter"),
+        path = "doc",
+        name = "Documenter"
     ),
-    MultiDocumenter.MultiDocRef(
-        upstream = "/home/pfitzseb/.julia/dev/Infiltrator/docs/build",
-        path = "infil",
+    ("JuliaDebug/Infiltrator.jl.git", "gh-pages") => MultiDocumenter.MultiDocRef(
+        upstream = joinpath(clonedir, "Infiltrator"),
+        path = "inf",
         name = "Infiltrator"
-    )
+    ),
 ]
 
+for ((remote, branch), docref) in docs
+    run(`git clone --depth 1 [email protected]:$remote --branch $branch --single-branch $(docref.upstream)`)
+end
+
+outpath = joinpath(@__DIR__, "out")
+
 MultiDocumenter.make(
-    joinpath(@__DIR__, "out"),
-    docs;
-    assets_dir = joinpath(@__DIR__, "assets"),
-    brand_image = MultiDocumenter.BrandImage(
-        "documenter/index.html",
-        "assets/brandimg.svg"
-    ),
-    custom_stylesheets = [
-        "assets/custom.css",
-        "assets/fonts/fonts.css",
-    ]
+    outpath,
+    collect(last.(docs));
+    search_engine = MultiDocumenter.SearchConfig(
+        index_versions = ["stable"],
+        engine = MultiDocumenter.FlexSearch
+    )
 )
 ```
 

sample.png

@@ -1,13 +1,13 @@
 module MultiDocumenter
 
-import Documenter, Gumbo, AbstractTrees
-
-include("search/flexsearch.jl")
-include("search/stork.jl")
+import Gumbo, AbstractTrees
 
 """
     SearchConfig(index_versions = ["stable"], engine = MultiDocumenter.FlexSearch)
 
+`index_versions` is a vector of relative paths used for generating the search index.
+`engine` may be `MultiDocumenter.FlexSearch`, `MultiDocumenter.Stork`, or a module that conforms
+to the expected API.
 """
 Base.@kwdef mutable struct SearchConfig
     index_versions = ["stable"]
@@ -30,6 +30,28 @@ function MultiDocRef(; upstream, name, path)
     MultiDocRef(upstream, path, name)
 end
 
+function walk_outputs(f, root, docs::Vector{MultiDocRef}, dirs::Vector{String})
+    for ref in docs
+        p = joinpath(root, ref.path)
+        for dir in dirs
+            dirpath = joinpath(p, dir)
+            isdir(dirpath) || continue
+            for (r, _, files) in walkdir(dirpath)
+                for file in files
+                    file == "index.html" || continue
+
+                    f(chop(r, head = length(root), tail = 0), joinpath(r, file))
+                end
+            end
+        end
+    end
+end
+
+include("search/flexsearch.jl")
+include("search/stork.jl")
+
+const DEFAULT_ENGINE = SearchConfig(index_versions = ["stable"], engine = FlexSearch)
+
 """
     make(
         outdir,
@@ -49,7 +71,7 @@ Aggregates multiple Documenter.jl-based documentation pages `docs` into `outdir`
   item in the global navigation
 - `custom_stylesheets` is a `Vector{String}` of stylesheets injected into each page.
 - `custom_scripts` is a `Vector{String}` of scripts injected into each page.
-- `search_engine` inserts a global search bar. See [`SearchConfig`](@ref) for more details.
+- `search_engine` inserts a global search bar if not `false`. See [`SearchConfig`](@ref) for more details.
 - `prettyurls` removes all `index.html` suffixes from links in the global navigation.
 """
 function make(
@@ -59,7 +81,7 @@ function make(
     brand_image::Union{Nothing,BrandImage} = nothing,
     custom_stylesheets = [],
     custom_scripts = [],
-    search_engine = SearchConfig(index_versions = ["stable"], engine = FlexSearch),
+    search_engine = DEFAULT_ENGINE,
     prettyurls = true
 )
 
@@ -71,6 +93,13 @@ function make(
     isdir(out_assets) || mkpath(out_assets)
     cp(joinpath(@__DIR__, "..", "assets", "__default"), joinpath(out_assets, "__default"))
 
+    if search_engine != false
+        if search_engine.engine == Stork && !Stork.has_stork()
+            @warn "stork binary not found. Falling back to flexsearch as search_engine."
+            search_engine = DEFAULT_ENGINE
+        end
+    end
+
     inject_styles_and_global_navigation(
         dir,
         docs,
@@ -82,7 +111,7 @@ function make(
     )
 
     if search_engine != false
-        search_engine.engine.build_search_index(dir, search_engine)
+        search_engine.engine.build_search_index(dir, docs, search_engine)
     end
 
     cp(dir, outdir; force = true)
@@ -215,10 +244,11 @@ function js_injector()
     return read(joinpath(@__DIR__, "..", "assets", "multidoc_injector.js"), String)
 end
 
+
 function inject_styles_and_global_navigation(
     dir,
     docs::Vector{MultiDocRef},
-    brand_image::BrandImage,
+    brand_image,
     custom_stylesheets,
     custom_scripts,
     search_engine,
@@ -240,8 +270,6 @@ function inject_styles_and_global_navigation(
                 end
                 continue
             end
-            # no need to do anything about /index.html
-            path == joinpath(dir, "index.html") && continue
 
             endswith(file, ".html") || continue
 
@@ -251,7 +279,11 @@ function inject_styles_and_global_navigation(
             stylesheets = make_global_stylesheet(custom_stylesheets, relpath(dir, root))
             scripts = make_global_scripts(custom_scripts, relpath(dir, root))
 
+
             page = read(path, String)
+            if startswith(page, "<!--This file is automatically generated by Documenter.jl-->")
+                continue
+            end
             doc = Gumbo.parsehtml(page)
             injected = 0
 
@@ -273,16 +305,15 @@ function inject_styles_and_global_navigation(
                         documenter_div = first(el.children)
                         if documenter_div isa Gumbo.HTMLElement &&
                            Gumbo.getattr(documenter_div, "id", "") == "documenter"
-                            # inject global navigation as first element in body
-
-                            global_nav =
-                                make_global_nav(dir, docs, root, brand_image, search_engine, prettyurls)
-                            global_nav.parent = el
-                            pushfirst!(el.children, global_nav)
-                            injected += 1
-                        else
-                            @warn "Could not inject global nav into $path."
+                           @debug "Could not detect Documenter page layout in $path. This may be due to an old version of Documenter."
                         end
+                        # inject global navigation as first element in body
+
+                        global_nav =
+                            make_global_nav(dir, docs, root, brand_image, search_engine, prettyurls)
+                        global_nav.parent = el
+                        pushfirst!(el.children, global_nav)
+                        injected += 1
                     end
                 end
             end

src/MultiDocumenter.jl

@@ -1,5 +1,6 @@
 module FlexSearch
 import Gumbo, JSON, AbstractTrees, NodeJS
+import ..walk_outputs
 
 const ID = Ref(0)
 
@@ -61,7 +62,7 @@ function add_fragment(doc, el)
     end
 end
 
-function add_to_index(index, ref, file)
+function add_to_index!(index, ref, file)
     content = read(file, String)
     html = Gumbo.parsehtml(content)
 
@@ -82,22 +83,12 @@ function add_to_index(index, ref, file)
     push!(index.documents, doc)
 end
 
-function generate_index(root, config)
+function generate_index(root, docs, config)
     search_index = SearchIndex()
-    for (r, _, files) in walkdir(root)
-        pathparts = splitpath(relpath(r, root))
-        if length(pathparts) >= 2 && pathparts[2] in config.index_versions
-            for file in files
-                if file == "index.html"
-                    add_to_index(
-                        search_index,
-                        chop(r, head = length(root), tail = 0),
-                        joinpath(r, file),
-                    )
-                end
-            end
-        end
+    walk_outputs(root, docs, config.index_versions) do path, file
+        add_to_index!(search_index, path, file)
     end
+
     return search_index
 end
 
@@ -160,9 +151,9 @@ function to_json_index(index::SearchIndex, file)
     end
 end
 
-function build_search_index(root, config)
+function build_search_index(root, docs, config)
     ID[] = 0
-    idx = generate_index(root, config)
+    idx = generate_index(root, docs, config)
     to_json_index(idx, joinpath(root, "index.json"))
     println("Writing flexsearch index:")
     cd(root) do

src/search/flexsearch.jl

@@ -1,9 +1,20 @@
 module Stork
 import Pkg.TOML
 import Gumbo, AbstractTrees
+import ..walk_outputs
 
-function build_search_index(root, config)
-    config = make_stork_config(root, config)
+function has_stork()
+    has_stork = false
+    try
+        has_stork = success(`stork -V`)
+    catch
+        has_stork = false
+    end
+    return has_stork
+end
+
+function build_search_index(root, docs, config)
+    config = make_stork_config(root, docs, config)
     config_path = joinpath(root, "stork.config.toml")
     index_path = joinpath(root, "stork.st")
     open(config_path, "w") do io
@@ -13,7 +24,7 @@ function build_search_index(root, config)
     return index_path
 end
 
-function make_stork_config(root, config)
+function make_stork_config(root, docs, config)
     files = []
     stork_config = Dict(
         "input" => Dict(
@@ -23,19 +34,9 @@ function make_stork_config(root, config)
         ),
         "output" => Dict("save_nearest_html_id" => true),
     )
-    for (r, _, fs) in walkdir(root)
-        pathparts = splitpath(relpath(r, root))
-        if length(pathparts) >= 2 && pathparts[2] in config.index_versions
-            for file in fs
-                if file == "index.html"
-                    add_to_index!(
-                        files,
-                        chop(r, head = length(root), tail = 0),
-                        joinpath(r, file),
-                    )
-                end
-            end
-        end
+
+    walk_outputs(root, docs, config.index_versions) do path, file
+        add_to_index!(files, path, file)
     end
 
     return stork_config

src/search/stork.jl

@@ -0,0 +1,2 @@
+out
+clones