diff --git a/docs/quickstart/usage_example/basics/configure_sec_and_page.md b/docs/quickstart/usage_example/basics/configure_sec_and_page.md index 7e65bac9..be1c4631 100644 --- a/docs/quickstart/usage_example/basics/configure_sec_and_page.md +++ b/docs/quickstart/usage_example/basics/configure_sec_and_page.md @@ -46,7 +46,7 @@ There are three possible options for `theme`: You can check the "Theme Gallery" to see how different themes look like. -## [Override default values with properties](@ref page_sec_properties) +## [Override default values with properties](@id page_sec_properties) Another item that could be useful is `properties`, it is written as a dictionary-like format, and provides a way to override some default values. Properties can be configured at any level and it diff --git a/docs/quickstart/usage_example/julia_demos/1.julia_demo.jl b/docs/quickstart/usage_example/julia_demos/1.julia_demo.jl index 416d55b1..cccce191 100644 --- a/docs/quickstart/usage_example/julia_demos/1.julia_demo.jl +++ b/docs/quickstart/usage_example/julia_demos/1.julia_demo.jl @@ -69,7 +69,7 @@ img = testimage("lena") # Another extra keyword is `notebook`, it allows you to control whether you needs to generate # jupyter notebook `.ipynb` for the demo. The valid values are `true` and `false`. See also -# [Override default values with properties](@id page_sec_properties) on how to disable all notebook +# [Override default values with properties](@ref page_sec_properties) on how to disable all notebook # generation via the properties entry. # !!! warning diff --git a/src/generate.jl b/src/generate.jl index cae36bbd..5989f965 100644 --- a/src/generate.jl +++ b/src/generate.jl @@ -143,13 +143,24 @@ function makedemos(source::String, templates::Union{Dict, Nothing} = nothing; # we can directly pass it to Documenter.makedocs if isnothing(templates) - out_path = walkpage(page; flatten=false) do item - splitext(joinpath(relative_root, relpath(item.path, page_root)))[1] * ".md" + out_path = walkpage(page; flatten=false) do dir, item + joinpath( + basename(source), + relpath(dir, page_root), + splitext(basename(item))[1] * ".md" + ) end else + # For themes that requires an index page + # This will not generate a multi-level structure in the sidebar out_path = joinpath(relative_root, "index.md") end + # Ensure every path exists before we actually do the work + walkpage(page) do dir, item + @assert isfile(item.path) || isdir(item.path) + end + @info "SetupDemoCardsDirectory: setting up \"$(source)\" directory." if isdir(absolute_root) # a typical and probably safe case -- that we're still in docs/ folder @@ -173,6 +184,7 @@ function makedemos(source::String, templates::Union{Dict, Nothing} = nothing; end mkpath(absolute_root) + local clean_up_temp_remote_files try # hard coded "covers" should be consistant to card template isnothing(templates) || mkpath(joinpath(absolute_root, "covers")) @@ -181,6 +193,10 @@ function makedemos(source::String, templates::Union{Dict, Nothing} = nothing; source_files = [x.path for x in walkpage(page)[2]] # pipeline + # prepare remote files into current folder structure, and provide a callback + # to clean it up after the generation + page, clean_up_temp_remote_files = prepare_remote_files(page) + # for themeless version, we don't need to generate covers and index page copy_assets(absolute_root, page) # WARNING: julia cards are reconfigured here @@ -198,6 +214,9 @@ function makedemos(source::String, templates::Union{Dict, Nothing} = nothing; @info "Redirect page URL: redirect docs-edit-link for demos in \"$(source)\" directory." isnothing(templates) || push!(source_files, joinpath(page_root, "index.md")) foreach(source_files) do source_file + # LocalRemoteCard is a virtual placeholder and does not exist here + isfile(source_file) && return + # only redirect to "real" files redirect_link(source_file, source, root, src, build, edit_branch) end @@ -229,6 +248,8 @@ function makedemos(source::String, templates::Union{Dict, Nothing} = nothing; rm(absolute_root; force=true, recursive=true) @error "Errors when building demo dir" pwd=pwd() source root src rethrow(err) + finally + clean_up_temp_remote_files() end end @@ -249,18 +270,18 @@ end function generate(cards::AbstractVector{<:AbstractDemoCard}, template; properties=Dict{String, Any}()) # for those hidden cards, only generate the necessary assets and files, but don't add them into # the index.md page - foreach(filter(x->x.hidden, cards)) do x + foreach(filter(ishidden, cards)) do x generate(x, template; properties=properties) end - mapreduce(*, filter(x->!x.hidden, cards); init="") do x + mapreduce(*, filter(x->!ishidden(x), cards); init="") do x generate(x, template; properties=properties) end end function generate(secs::AbstractVector{DemoSection}, templates; level=1, properties=Dict{String, Any}()) mapreduce(*, secs; init="") do x - properties = merge(properties, x.properties) # sec.properties has higher priority + properties = merge(properties, x.properties) generate(x, templates; level=level, properties=properties) end end @@ -323,7 +344,7 @@ function save_cover(path::String, sec::DemoSection) end """ - save_cover(path::String, card::AbstractDemoCard) + save_cover(path::String, card) process the cover image and save it. """ @@ -358,7 +379,7 @@ function save_cover(path::String, card::AbstractDemoCard) end end -function get_covername(card::AbstractDemoCard) +function get_covername(card) isnothing(card.cover) && return nothing is_remote_url(card.cover) && return card.cover @@ -410,6 +431,50 @@ function _copy_assets(dest_root::String, src_root::String) end end +### prepare_remote_files + +function prepare_remote_files(page) + # 1. copy all remote files into its corresponding folders + # 2. record all temporarily remote files + # 3. rebuild the demo page in no-remote mode + temp_entry_list = [] + for (root, dirs, files) in walkdir(page.root) + config_filename in files || continue + + config_path = joinpath(root, config_filename) + config = JSON.parsefile(config_path) + + if haskey(config, "remote") + remotes = config["remote"] + for (dst_entry_name, src_entry_path) in remotes + dst_entry_path = joinpath(root, dst_entry_name) + src_entry_path = normpath(root, src_entry_path) + if !ispath(src_entry_path) + @warn "File/folder doesn't exist, skip it." path=src_entry_path + continue + end + if ispath(dst_entry_path) + @warn "A file/folder already exists for remote path $dst_entry_name, skip it." path=dst_entry_path + continue + end + + cp(src_entry_path, dst_entry_path) + push!(temp_entry_list, dst_entry_path) + end + end + end + + page = isempty(temp_entry_list) ? page : DemoPage(page.root; ignore_remote=true) + function clean_up_temp_remote_files() + Base.Sys.iswindows() && GC.gc() + foreach(temp_entry_list) do x + rm(x; force=true, recursive=true) + end + end + return page, clean_up_temp_remote_files +end + + ### postprocess """ diff --git a/src/preview.jl b/src/preview.jl index 1163e94f..f0fc9a82 100644 --- a/src/preview.jl +++ b/src/preview.jl @@ -54,7 +54,7 @@ function preview_demos(demo_path::String; end page_dir = generate_or_copy_pagedir(demo_path, build_dir) - copy_assets_and_configs(page_dir, build_dir) + copy_assets(page_dir, build_dir) cd(build_dir) do page = @suppress_err DemoPage(page_dir) @@ -158,9 +158,12 @@ function generate_or_copy_pagedir(src_path, build_dir) end mkpath(dirname(dst_sec_dir)) cp(sec_dir, dst_sec_dir; force=true) + config_rewrite(sec_dir, dst_sec_dir) elseif is_demopage(src_path) page_dir = src_path - cp(page_dir, abspath(build_dir, basename(page_dir)); force=true) + dst_page_dir = abspath(build_dir, basename(page_dir)) + cp(page_dir, dst_page_dir; force=true) + config_rewrite(page_dir, dst_page_dir) else throw(ArgumentError("failed to parse demo page structure from path: $src_path. There might be some invalid demo files.")) end @@ -169,14 +172,12 @@ function generate_or_copy_pagedir(src_path, build_dir) end """ - copy_assets_and_configs(src_page_dir, dst_build_dir=pwd()) + copy_assets(src_page_dir, dst_build_dir=pwd()) -copy only assets, configs and templates from `src_page_dir` to `dst_build_dir`. The folder structure +copy only assets and templates from `src_page_dir` to `dst_build_dir`. The folder structure is preserved under `dst_build_dir/\$(basename(src_page_dir))` - -`order` in config files are modified accordingly. """ -function copy_assets_and_configs(src_page_dir, dst_build_dir=pwd()) +function copy_assets(src_page_dir, dst_build_dir=pwd()) (isabspath(src_page_dir) && isdir(src_page_dir)) || throw(ArgumentError("src_page_dir is expected to be absolute folder path: $src_page_dir")) for (root, dirs, files) in walkdir(src_page_dir) @@ -207,32 +208,40 @@ function copy_assets_and_configs(src_page_dir, dst_build_dir=pwd()) ispath(dst_template_path) || cp(src_template_path, dst_template_path) end end +end - # modify and copy config.json after the folder structure is already set up - for (root, dirs, files) in walkdir(src_page_dir) - if config_filename in files - src_config_path = abspath(root, config_filename) - dst_config_path = abspath(dst_build_dir, relpath(src_config_path, dirname(src_page_dir))) - mkpath(dirname(dst_config_path)) - - config = JSON.parsefile(src_config_path) - config_dir = dirname(dst_config_path) - if haskey(config, "order") - order = [x for x in config["order"] if x in readdir(config_dir)] - if isempty(order) - delete!(config, "order") - else - config["order"] = order - end - end +function config_rewrite(src_dir, dst_dir) + for (dst_root, dirs, files) in walkdir(dst_dir) + src_root = joinpath(dirname(src_dir), relpath(dst_root, dirname(dst_dir))) + + config_filename in files || continue - if !isempty(config) - # Ref: https://discourse.julialang.org/t/find-what-has-locked-held-a-file/23278/2 - Base.Sys.iswindows() && GC.gc() - open(dst_config_path, "w") do io - JSON.print(io, config) + dst_config_path = joinpath(dst_root, config_filename) + config = JSON.parsefile(dst_config_path) + + if haskey(config, "remote") + remotes = config["remote"] + for (dst_entry_name, src_entry_path) in remotes + src_entry_path = isabspath(src_entry_path) ? src_entry_path : normpath(src_root, src_entry_path) + if !ispath(src_entry_path) + @warn "File/folder doesn't exist, skip it." path=src_entry_path + continue + end + + dst_entry_path = joinpath(dst_root, dst_entry_name) + if ispath(dst_entry_path) + @warn "A file/folder already exists for remote path $dst_entry_name, skip it." path=dst_entry_path + continue end + cp(src_entry_path, dst_entry_path) end + delete!(config, "remote") + end + + # Ref: https://discourse.julialang.org/t/find-what-has-locked-held-a-file/23278/2 + Base.Sys.iswindows() && GC.gc() + open(dst_config_path, "w") do io + JSON.print(io, config) end end end diff --git a/src/show.jl b/src/show.jl index 37159d12..3a330155 100644 --- a/src/show.jl +++ b/src/show.jl @@ -3,26 +3,27 @@ import Base: show const indent_spaces = " " function show(io::IO, card::AbstractDemoCard) - println(io, basename(card)) + print(io, basename(card)) end -function show(io::IO, sec::DemoSection; level=1) +function show(io::IO, sec::AbstractDemoSection; level=1) print(io, repeat(indent_spaces, level-1)) - println(io, repeat("#", level), " ", sec.title) + println(io, repeat("#", level), " ", compact_title(sec)) # a section either holds cards or subsections # here it's a mere coincident to show cards first - foreach(sec.cards) do x + foreach(cards(sec)) do x print(io, repeat(indent_spaces, level)) show(io, x) + println(io) end - foreach(x->show(io, x; level=level+1), sec.subsections) + foreach(x->show(io, x; level=level+1), subsections(sec)) end function show(io::IO, page::DemoPage) page_root = replace(page.root, "\\" => "/") println(io, "DemoPage(\"", page_root, "\"):\n") println(io, "# ", page.title) - foreach(x->show(io, x; level=2), page.sections) + foreach(x->show(io, x; level=2), subsections(page)) end diff --git a/src/types/card.jl b/src/types/card.jl index dc47f11d..ada24528 100644 --- a/src/types/card.jl +++ b/src/types/card.jl @@ -1,5 +1,12 @@ abstract type AbstractDemoCard end +""" + UnmatchedCard(path) + +A dummy placeholder card for files that DemoCards doesn't support +yet. No operation will be applied on this type of file, except for +warnings. +""" struct UnmatchedCard <: AbstractDemoCard path::String end @@ -18,7 +25,6 @@ to your demofile. Currently supported types are: """ function democard(path::String)::AbstractDemoCard - validate_file(path) _, ext = splitext(path) if ext in markdown_exts return MarkdownDemoCard(path) @@ -28,6 +34,9 @@ function democard(path::String)::AbstractDemoCard return UnmatchedCard(path) end end +function democard((name, path)::Pair{String, String}) + LocalRemoteCard(name, path, democard(path)) +end basename(x::AbstractDemoCard) = basename(x.path) @@ -46,8 +55,8 @@ end function is_democard(file) try - @suppress_err democard(file) - return true + card = @suppress_err democard(file) + return !isa(card, UnmatchedCard) catch err @debug err return false @@ -122,3 +131,4 @@ end include("markdown.jl") include("julia.jl") +include("remote_card.jl") diff --git a/src/types/julia.jl b/src/types/julia.jl index 7cb75a5d..b44a1004 100644 --- a/src/types/julia.jl +++ b/src/types/julia.jl @@ -76,6 +76,21 @@ mutable struct JuliaDemoCard <: AbstractDemoCard julia::VersionNumber hidden::Bool notebook::Union{Nothing, Bool} + function JuliaDemoCard( + path::String, + cover::Union{String, Nothing}, + id::String, + title::String, + description::String, + author::String, + date::DateTime, + julia::VersionNumber, + hidden::Bool, + notebook::Union{Nothing, Bool} + ) + isfile(path) || throw(ArgumentError("$(path) is not a valid file")) + new(path, cover, id, title, description, author, date, julia, hidden, notebook) + end end function JuliaDemoCard(path::String)::JuliaDemoCard @@ -274,3 +289,5 @@ function make_badges(card::JuliaDemoCard; src, card_dir, nbviewer_root_url, proj join(badges, " ") end + +ishidden(x::JuliaDemoCard) = x.hidden diff --git a/src/types/markdown.jl b/src/types/markdown.jl index c853cdbb..14f522db 100644 --- a/src/types/markdown.jl +++ b/src/types/markdown.jl @@ -67,6 +67,19 @@ mutable struct MarkdownDemoCard <: AbstractDemoCard author::String date::DateTime hidden::Bool + function MarkdownDemoCard( + path::String, + cover::Union{String, Nothing}, + id::String, + title::String, + description::String, + author::String, + date::DateTime, + hidden::Bool + ) + isfile(path) || throw(ArgumentError("$(path) is not a valid file")) + new(path, cover, id, title, description, author, date, hidden) + end end function MarkdownDemoCard(path::String)::MarkdownDemoCard @@ -120,3 +133,5 @@ function save_democards(card_dir::String, footer = credit ? markdown_footer : "\n" write(markdown_path, header, make_badges(card)*"\n\n", body, footer) end + +ishidden(x::MarkdownDemoCard) = x.hidden diff --git a/src/types/page.jl b/src/types/page.jl index 2a00c68a..69e72576 100644 --- a/src/types/page.jl +++ b/src/types/page.jl @@ -92,7 +92,7 @@ See also: [`MarkdownDemoCard`](@ref DemoCards.MarkdownDemoCard), [`DemoSection`] """ mutable struct DemoPage root::String - sections::Vector{DemoSection} + sections::Vector template::String theme::Union{Nothing, String} title::String @@ -100,9 +100,10 @@ mutable struct DemoPage properties::Dict{String, Any} end -basename(page::DemoPage) = basename(page.root) +Base.basename(page::DemoPage) = basename(page.root) +subsections(sec::DemoPage) = sec.sections -function DemoPage(root::String)::DemoPage +function DemoPage(root::String; ignore_remote=false)::DemoPage root = replace(root, r"[/\\]" => Base.Filesystem.path_separator) # windows compatibility isdir(root) || throw(ArgumentError("page root does not exist: $(root)")) root = rstrip(root, '/') # otherwise basename(root) will returns `""` @@ -130,7 +131,10 @@ function DemoPage(root::String)::DemoPage config = parse(Val(:Markdown), template_file) config = merge(json_config, config) # template has higher priority over config file - sections = filter(map(DemoSection, section_paths)) do sec + sections = map(section_paths) do x + DemoSection(x, ignore_remote=ignore_remote) + end + sections = filter(sections) do sec empty_section = isempty(sec.cards) && isempty(sec.subsections) if empty_section @warn "Empty section detected, remove from the demo page tree." section=relpath(sec.root, root) @@ -139,6 +143,10 @@ function DemoPage(root::String)::DemoPage return true end end + if !ignore_remote + remote_sections = map(demosection, read_remote_sections(config, root)) + sections = [sections..., remote_sections...] + end isempty(sections) && error("Empty demo page, you have to add something.") page = DemoPage(root, sections, "", nothing, "", Dict{String, Any}()) @@ -146,7 +154,7 @@ function DemoPage(root::String)::DemoPage section_orders = load_config(page, "order"; config=config) section_orders = map(sections) do sec - findfirst(x-> x == basename(sec.root), section_orders) + findfirst(x-> x == basename(sec), section_orders) end ordered_sections = sections[section_orders] diff --git a/src/types/remote_card.jl b/src/types/remote_card.jl new file mode 100644 index 00000000..6a7ff227 --- /dev/null +++ b/src/types/remote_card.jl @@ -0,0 +1,21 @@ +struct LocalRemoteCard{T<:AbstractDemoCard} <: AbstractDemoCard + name::String + path::String + item::T + function LocalRemoteCard(name::String, path::String, item::T) where T<:AbstractDemoCard + basename(name) == name || throw(ArgumentError("`name` should not be a path, instead it is: \"$name\". Do you mean \"$(basename(name))\"")) + isempty(splitext(name)[2]) && throw(ArgumentError("Remote entry name `$name`for card should has extensions.")) + isfile(path) || throw(ArgumentError("file $path does not exist.")) + new{T}(name, path, item) + end +end +function LocalRemoteCard(name::String, path::String, card::LocalRemoteCard{T}) where T<:AbstractDemoCard + LocalRemoteCard(name, path, card.item) +end + +ishidden(card::LocalRemoteCard) = ishidden(card.item) +Base.basename(card::LocalRemoteCard) = card.name + +function Base.show(io::IO, card::LocalRemoteCard) + print(io, basename(card), " => ", card.path) +end diff --git a/src/types/section.jl b/src/types/section.jl index 9ee34920..c48053c4 100644 --- a/src/types/section.jl +++ b/src/types/section.jl @@ -1,3 +1,11 @@ +abstract type AbstractDemoSection end + +demosection(root::String) = DemoSection(root) +function demosection((name, path)::Pair{String, String}) + LocalRemoteSection(name, path, DemoSection(path)) +end + + """ struct DemoSection <: Any DemoSection(root::String) @@ -76,19 +84,22 @@ section See also: [`MarkdownDemoCard`](@ref DemoCards.MarkdownDemoCard), [`DemoPage`](@ref DemoCards.DemoPage) """ -struct DemoSection +struct DemoSection <: AbstractDemoSection root::String cards::Vector - subsections::Vector{DemoSection} + subsections::Vector title::String description::String # These properties will be shared by all children of it during build time properties::Dict{String, Any} end -basename(sec::DemoSection) = basename(sec.root) +Base.basename(sec::DemoSection) = basename(sec.root) +compact_title(sec::DemoSection) = sec.title +cards(sec::DemoSection) = sec.cards +subsections(sec::DemoSection) = sec.subsections -function DemoSection(root::String)::DemoSection +function DemoSection(root::String; ignore_remote=false)::DemoSection root = replace(root, r"[/\\]" => Base.Filesystem.path_separator) # windows compatibility isdir(root) || throw(ArgumentError("section root should be a valid dir, instead it's $(root)")) @@ -106,7 +117,7 @@ function DemoSection(root::String)::DemoSection # For files that `democard` fails to recognized, dummy # `UnmatchedCard` will be generated. Currently, we only # throw warnings for it. - cards = map(democard, card_paths) + cards = AbstractDemoCard[democard(x) for x in card_paths] unmatches = filter(cards) do x x isa UnmatchedCard end @@ -114,26 +125,23 @@ function DemoSection(root::String)::DemoSection msg = join(map(basename, unmatches), "\", \"") @warn "skip unmatched file: \"$msg\"" section_dir=root end + if !ignore_remote + remote_card_paths = read_remote_cards(config, root) + cards = [cards..., democard.(remote_card_paths)...] + end cards = filter!(cards) do x !(x isa UnmatchedCard) end - section = DemoSection(root, - cards, - map(DemoSection, section_paths), - "", - "", - Dict{String, Any}()) - - ordered_paths = joinpath.(root, load_config(section, "order"; config=config)) - if !isempty(section.cards) - cards = map(democard, ordered_paths) - subsections = [] - else - cards = [] - subsections = map(DemoSection, ordered_paths) + subsections = map(DemoSection, section_paths) + if !ignore_remote + remote_sections = map(demosection, read_remote_sections(config, root)) + subsections = [subsections..., remote_sections...] end + section = DemoSection(root, cards, subsections, "", "", Dict{String, Any}()) + cards, subsections = sort_by_order(section, config) + title = load_config(section, "title"; config=config) description = load_config(section, "description"; config=config) @@ -146,6 +154,31 @@ function DemoSection(root::String)::DemoSection DemoSection(root, cards, subsections, title, description, properties) end +function sort_by_order(sec::DemoSection, config) + cards = sec.cards + subsections = sec.subsections + + ordered_paths = load_config(sec, "order"; config=config) + if !isempty(sec.cards) + indices = map(ordered_paths) do ref + findfirst(cards) do card + basename(card) == ref + end + end + cards = cards[indices] + subsections = [] + else + indices = map(ordered_paths) do ref + findfirst(subsections) do sec + basename(sec) == ref + end + end + subsections = subsections[indices] + cards = [] + end + + return cards, subsections +end function load_config(sec::DemoSection, key; config=Dict()) if isempty(config) @@ -185,3 +218,26 @@ function is_demosection(dir) return false end end + +### +# LocalRemoteSection +### + +struct LocalRemoteSection{T<:AbstractDemoSection} <: AbstractDemoSection + name::String + path::String + item::T + function LocalRemoteSection(name::String, path::String, item::T) where T<:AbstractDemoSection + basename(name) == name || throw(ArgumentError("`name` should not be a path, instead it is: \"$name\". Do you mean \"$(basename(name))\"")) + isdir(path) || throw(ArgumentError("folder $path does not exist.")) + new{T}(name, path, item) + end +end +function LocalRemoteSection(name::String, path::String, card::LocalRemoteSection{T}) where T<:AbstractDemoCard + LocalRemoteSection(name, path, card.item) +end + +Base.basename(sec::LocalRemoteSection) = sec.name +compact_title(sec::LocalRemoteSection) = "$(sec.name) => $(sec.path)" +cards(sec::LocalRemoteSection) = cards(sec.item) +subsections(sec::LocalRemoteSection) = subsections(sec.item) diff --git a/src/utils.jl b/src/utils.jl index 9431de40..bdbf8c29 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -1,9 +1,3 @@ -function validate_file(path, filetype = :text) - isfile(path) || throw(ArgumentError("$(path) is not a valid file")) - check_ext(path, filetype) -end - - function check_ext(path, filetype = :text) _, ext = splitext(path) if filetype == :text @@ -19,6 +13,7 @@ end ### common utils for DemoPage and DemoSection function validate_order(order::AbstractArray, x::Union{DemoPage, DemoSection}) + length(unique(order)) == length(order) || throw(ArgumentError("`\"order\"` entry should be unique.")) default_order = get_default_order(x) if intersect(order, default_order) == union(order, default_order) return true @@ -35,6 +30,21 @@ function validate_order(order::AbstractArray, x::Union{DemoPage, DemoSection}) end end +read_remote_cards(config, root) = _read_remote_items(isfile, config, root) +read_remote_sections(config, root) = _read_remote_items(isdir, config, root) +function _read_remote_items(fn, config, root) + remote_items = Pair{String, String}[] + haskey(config, "remote") || return remote_items + + for (itemname, itempath) in config["remote"] + # if possible, store the abspath + itempath = isabspath(itempath) ? itempath : normpath(joinpath(root, itempath)) + fn(itempath) || continue + push!(remote_items, itemname=>itempath) + end + return remote_items +end + """ walkpage([f=identity], page; flatten=true, max_depth=Inf) @@ -50,7 +60,7 @@ This is particulaly useful to generate information for the page structure. For e ```julia julia> page = DemoPage("docs/quickstart/"); -julia> walkpage(page; flatten=true) do item +julia> walkpage(page; flatten=true) do dir, item basename(item.path) end @@ -85,7 +95,7 @@ If `flatten=false`, then it gives you something like this: ] ``` """ -walkpage(page::DemoPage; kwargs...) = walkpage(identity, page; kwargs...) +walkpage(page::DemoPage; kwargs...) = walkpage((dir, item)->item, page; kwargs...) function walkpage(f, page::DemoPage; flatten=true, kwargs...) nodes = _walkpage.(f, page.sections, 1; flatten=flatten, kwargs...) if flatten @@ -97,14 +107,15 @@ function walkpage(f, page::DemoPage; flatten=true, kwargs...) end function _walkpage(f, sec::DemoSection, depth; max_depth=Inf, flatten=true) - depth+1 > max_depth && return flatten ? f(sec) : [f(sec), ] + depth+1 > max_depth && return flatten ? f(src.root, sec) : [f(sec.root, sec), ] if !isempty(sec.cards) - return flatten ? mapreduce(f, vcat, sec.cards) : sec.title => f.(sec.cards) + return flatten ? mapreduce(x->f(sec.root, x), vcat, sec.cards) : sec.title => f.(sec.root, sec.cards) else map_fun(section) = _walkpage(f, section, depth+1; max_depth=max_depth, flatten=flatten) return flatten ? mapreduce(map_fun, vcat, sec.subsections) : sec.title => map_fun.(sec.subsections) end end +_walkpage(f, sec::LocalRemoteSection, depth; kwargs...) = _walkpage(f, sec.item, depth; kwargs...) ### regexes and configuration parsers diff --git a/test/assets/remote/config.json b/test/assets/remote/config.json new file mode 100644 index 00000000..3830ee93 --- /dev/null +++ b/test/assets/remote/config.json @@ -0,0 +1,12 @@ +{ + "remote": { + "hidden_section": "../page/hidden" + }, + "order": [ + "hidden_section", + "remote_card_mixed", + "remote_card_mixed_order", + "remote_card_simplest", + "remote_subfolder" + ] +} diff --git a/test/assets/remote/remote_card_mixed/card1.md b/test/assets/remote/remote_card_mixed/card1.md new file mode 100644 index 00000000..e69de29b diff --git a/test/assets/remote/remote_card_mixed/config.json b/test/assets/remote/remote_card_mixed/config.json new file mode 100644 index 00000000..6976605a --- /dev/null +++ b/test/assets/remote/remote_card_mixed/config.json @@ -0,0 +1,6 @@ +{ + "remote": { + "julia_card1.jl": "../../card/julia/cover_1.jl", + "markdown_card1.md": "../../card/markdown/cover_3.md" + } +} diff --git a/test/assets/remote/remote_card_mixed_order/card2.md b/test/assets/remote/remote_card_mixed_order/card2.md new file mode 100644 index 00000000..b96b50f4 --- /dev/null +++ b/test/assets/remote/remote_card_mixed_order/card2.md @@ -0,0 +1,3 @@ +# [Custom Title 2](@id custom_id_2) + +This is the content diff --git a/test/assets/remote/remote_card_mixed_order/config.json b/test/assets/remote/remote_card_mixed_order/config.json new file mode 100644 index 00000000..9008a028 --- /dev/null +++ b/test/assets/remote/remote_card_mixed_order/config.json @@ -0,0 +1,11 @@ +{ + "remote": { + "julia_card2.jl": "../../card/julia/version_1.jl", + "markdown_card2.md": "../../card/markdown/description_1.md" + }, + "order": [ + "julia_card2.jl", + "markdown_card2.md", + "card2.md" + ] +} diff --git a/test/assets/remote/remote_card_simplest/config.json b/test/assets/remote/remote_card_simplest/config.json new file mode 100644 index 00000000..5ebd095b --- /dev/null +++ b/test/assets/remote/remote_card_simplest/config.json @@ -0,0 +1,6 @@ +{ + "remote": { + "julia_card3.jl": "../../card/julia/simplest.jl", + "markdown_card3.md": "../../card/markdown/simplest.md" + } +} diff --git a/test/assets/remote/remote_subfolder/config.json b/test/assets/remote/remote_subfolder/config.json new file mode 100644 index 00000000..bc0ce020 --- /dev/null +++ b/test/assets/remote/remote_subfolder/config.json @@ -0,0 +1,5 @@ +{ + "remote": { + "hidden_section": "../../page/one_card" + } +} diff --git a/test/remote.jl b/test/remote.jl new file mode 100644 index 00000000..05918682 --- /dev/null +++ b/test/remote.jl @@ -0,0 +1,81 @@ +@testset "RemotePath" begin + root = "assets" + abs_root = joinpath(pwd(), root, "remote") + + @testset "LocalRemote" begin + @testset "LocalRemoteCard" begin + for (numcards, dir) in [ + # As long as it does not error, we believe the order is respected + (5, "remote_card_mixed_order"), + (5, "remote_card_mixed"), + (4, "remote_card_simplest") + ] + # there will be warnings due to lack of cover images, but we can just safely ignore them + @suppress_err preview_demos(joinpath(abs_root, dir); theme="grid") + page_dir = @suppress_err preview_demos(joinpath(abs_root, dir); require_html=false) + files = readdir(joinpath(page_dir, dir)) + @test numcards == map(files) do f + splitext(f)[1] + end |> length + end + end + + @testset "LocalRemoteSection" begin + # there will be warnings due to lack of cover images, but we can just safely ignore them + @suppress_err preview_demos(abs_root; theme="grid") + page_dir = @suppress_err preview_demos(abs_root; require_html=false) + @test readdir(page_dir) == ["hidden_section", "remote_card_mixed", "remote_card_mixed_order", "remote_card_simplest", "remote_subfolder"] + end + + # Because `preview_demos` eagerly converts local remote files as direct entries + # we also need to test it on plain "makedemos" to ensure that it works on `docs/make.jl` + @testset "application" begin + mktempdir() do root + tmp_root = joinpath(root, "remote") + mkpath(tmp_root) + cp(abs_root, tmp_root, force=true) + + # override relative paths as absolute paths + # -- I'm too lazy to figure a solution so just copy and paste an ad-hoc solution here :( + for (dir, entry_names) in [ + (".", ["hidden_section", ]), + ("remote_subfolder", ["hidden_section", ]), + ("remote_card_mixed", ["julia_card1.jl", "markdown_card1.md"]), + ("remote_card_mixed_order", ["julia_card2.jl", "markdown_card2.md"]), + ("remote_card_simplest", ["julia_card3.jl", "markdown_card3.md"]), + ] + config_file = joinpath(tmp_root, dir, "config.json") + config = JSON.parsefile(config_file) + for x in entry_names + config["remote"][x] = normpath(abs_root, dir, config["remote"][x]) + end + Base.Sys.iswindows() && GC.gc() + open(config_file, "w") do io + JSON.print(io, config) + end + end + + + function flatten_walkdir(root_dir) + contents = [] + for (root, dirs, files) in walkdir(root_dir) + push!(contents, root) + append!(contents, map(x->joinpath(root, x), dirs)) + append!(contents, map(x->joinpath(root, x), files)) + end + return unique(map(x->relpath(x, root_dir), contents)) + end + + # we generate twice using `makedemos` and `preview_demos` and compare the results + templates, theme = @suppress_err cardtheme(root=root) + path, post_process_cb = @suppress_err makedemos(tmp_root, templates, root=root) + page_dir = joinpath(root, "src", dirname(path)) + plain_contents = flatten_walkdir(page_dir) + + page_dir = @suppress_err preview_demos(abs_root; require_html=false) + preview_contents = flatten_walkdir(page_dir) + @test sort(setdiff(plain_contents, preview_contents)) == ["covers", "covers/democards_logo.svg", "index.md"] + end + end + end +end diff --git a/test/runtests.jl b/test/runtests.jl index f2e7d623..91ec3e18 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,7 +1,7 @@ using DemoCards using DemoCards: democard, MarkdownDemoCard, JuliaDemoCard, DemoSection, DemoPage using DemoCards: generate -using HTTP +using HTTP, JSON using Test, ReferenceTests, Suppressor using Dates @@ -19,6 +19,7 @@ cd(test_root) do include("generate.jl") include("preview.jl") + include("remote.jl") include("show.jl") include("utils.jl") diff --git a/test/utils.jl b/test/utils.jl index 752eb491..01f4af41 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -93,21 +93,21 @@ end @testset "walkpage" begin page = DemoPage(joinpath("assets", "page", "hidden")) reference = "Hidden" => ["hidden1.jl", "hidden2.md", "normal.md"] - @test reference == walkpage(page) do item + @test reference == walkpage(page) do dir, item basename(item.path) end reference = "Hidden" => ["Sec" => ["hidden1.jl", "hidden2.md", "normal.md"]] - @test reference == walkpage(page; flatten=false) do item + @test reference == walkpage(page; flatten=false) do dir, item basename(item.path) end page = DemoPage(joinpath("assets", "page", "one_card")) reference = "One card" => ["card.md"] - @test reference == walkpage(page) do item + @test reference == walkpage(page) do dir, item basename(item.path) end reference = "One card" => ["Section" => ["Subsection" => ["card.md"]]] - @test reference == walkpage(page; flatten=false) do item + @test reference == walkpage(page; flatten=false) do dir, item basename(item.path) end end