diff --git a/Project.toml b/Project.toml
index 2616c819..1329e933 100644
--- a/Project.toml
+++ b/Project.toml
@@ -17,6 +17,7 @@ HTMLSanitizer = "9a15a9f4-ddd5-46ee-89fc-c219f813dd6f"
Highlights = "eafb193a-b7ab-5a9e-9068-77385905fa72"
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
+Org = "587fedb0-ad84-11e9-2bd6-d15ea4be1f9e"
Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
@@ -24,8 +25,8 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
[compat]
AbstractTrees = "0.2, 0.3"
Documenter = "0.24, 0.25, 0.26, 0.27"
-GitHub = "5.1, 5.2, 5.3, 5.4"
GitForge = "0.4"
+GitHub = "5.1, 5.2, 5.3, 5.4"
GithubMarkdown = "0.2"
Gumbo = "0.8"
HTMLSanitizer = "0.2"
diff --git a/src/builders.jl b/src/builders.jl
index 37670643..9e80e005 100644
--- a/src/builders.jl
+++ b/src/builders.jl
@@ -4,6 +4,7 @@ using HTMLSanitizer
using Highlights
using Downloads
using Documenter
+import Org
function build_git_docs(packagespec, buildpath, uri; src_prefix="", href_prefix="")
pkgname = packagespec.name
return mktempdir() do dir
@@ -242,7 +243,11 @@ function build_readme_docs(pkgname, pkgroot, docsdir, mod, src_prefix, href_pref
mkpath(doc_src)
index = joinpath(doc_src, "index.md")
- render_html(readme, index, src_prefix, href_prefix; documenter = true)
+ if isnothing(readme)
+ @error "Readme missing, skipping generation." pkgroot
+ else
+ render_html(readme, index, src_prefix, href_prefix; documenter = true)
+ end
if !isfile(index)
open(index, "w") do io
@@ -285,7 +290,7 @@ function find_readme(pkgroot)
allfiles = readdir(pkgroot)
# look for readme.md/readme first
for file in allfiles
- if lowercase(file) in ("readme.md", "readme")
+ if lowercase(file) in ("readme.md", "readme", "readme.org")
readme = joinpath(pkgroot, file)
if isfile(readme)
return readme
@@ -387,27 +392,66 @@ function copy_package_source(packagespec, buildpath)
end
end
-function render_html(input, output, src_prefix="", href_prefix=""; documenter = false)
- if input === nothing
- @error("Package doesn't seem to have a readme. ")
-
- return
+abstract type ReadmeFormat end
+struct GFMFormat <: ReadmeFormat end
+struct OrgFormat <: ReadmeFormat end
+struct TextFormat <: ReadmeFormat end
+struct UnknownFormat <: ReadmeFormat end
+
+function readme_format(readme_path::AbstractString)
+ _, ext = splitext(readme_path)
+ ext = lowercase(ext)
+ return if ext == ".md"
+ GFMFormat()
+ elseif ext == ".org"
+ OrgFormat()
+ elseif ext == ""
+ TextFormat()
+ else
+ UnknownFormat()
end
+end
+function render_html(::GFMFormat, readme_path::AbstractString)
io = IOBuffer()
- GithubMarkdown.rendergfm(io, input; documenter = false)
+ GithubMarkdown.rendergfm(io, readme_path; documenter = false)
+ return String(take!(io))
+end
+
+function render_html(::OrgFormat, readme_path::AbstractString)
+ doc = parse(Org.OrgDoc, read(readme_path, String))
+ return sprint(show, "text/html", doc)
+end
+
+function render_html(format::Union{TextFormat,UnknownFormat}, readme_path::AbstractString)
+ out = IOBuffer()
+ if isa(format, UnknownFormat)
+ write(out, """
+ WARNING! Unknown README file format: $(escapehtml(basename(readme_path)))
+ """)
+ end
+ write(out, "")
+ write(out, escapehtml(read(readme_path, String)))
+ write(out, "
")
+ return String(take!(out))
+end
+function render_html(
+ input::AbstractString,
+ output::AbstractString,
+ src_prefix::AbstractString="",
+ href_prefix::AbstractString="";
+ documenter::Bool = false
+)
allow_class_on_code = deepcopy(HTMLSanitizer.WHITELIST)
allow_class_on_code[:attributes]["code"] = ["class"]
- out = sanitize(String(take!(io)), prettyprint = false, whitelist = allow_class_on_code)
-
+ readme_html = render_html(readme_format(input), input)
+ out = sanitize(readme_html, prettyprint = false, whitelist = allow_class_on_code)
out = postprocess_html_readme(out; src_prefix = src_prefix, href_prefix = href_prefix)
out = out.children[2]
- print(io, out)
- out = String(take!(io))
-
+ out = sprint(print, out)
out = replace(out, r"^" => "")
out = replace(out, r"$" => "")
@@ -421,15 +465,17 @@ end
function render_readme_html(pkgroot, buildpath, src_prefix="", href_prefix=""; documenter = false)
outpath = joinpath(buildpath, "_readme")
try
- readme = find_readme(pkgroot)
-
mkpath(outpath)
readmehtml = joinpath(outpath, "readme.html")
- @info("Rendering readme to HTML.")
- render_html(readme, readmehtml, src_prefix, href_prefix; documenter = false)
- @info("Done rendering readme to HTML.")
-
+ readme = find_readme(pkgroot)
+ if isnothing(readme)
+ @error "Readme missing, skipping generation." pkgroot
+ else
+ @info("Rendering readme to HTML.")
+ render_html(readme, readmehtml, src_prefix, href_prefix; documenter = false)
+ @info("Done rendering readme to HTML.")
+ end
return readmehtml
catch err
@error("Error trying to render readme to HTML.", exception=err)
diff --git a/src/utils/misc.jl b/src/utils/misc.jl
index 9bfcb1ab..fdc36a1e 100644
--- a/src/utils/misc.jl
+++ b/src/utils/misc.jl
@@ -18,3 +18,35 @@ function find_free_x_servernum()
return i
end
+
+# Borrowed from Documenter.jl (MIT)
+#
+# https://github.com/JuliaDocs/Documenter.jl/blob/5dafb6488f90d173ca4fcfeead0396332bdc6de6/src/utilities/DOM.jl#L269-L296
+"""
+Escape characters in the provided string. This converts the following characters:
+
+- `<` to `<`
+- `>` to `>`
+- `&` to `&`
+- `'` to `'`
+- `\"` to `"`
+
+When no escaping is needed then the same object is returned, otherwise a new
+string is constructed with the characters escaped. The returned object should
+always be treated as an immutable copy and compared using `==` rather than `===`.
+"""
+function escapehtml(text::AbstractString)
+ if occursin(r"[<>&'\"]", text)
+ buffer = IOBuffer()
+ for char in text
+ char === '<' ? write(buffer, "<") :
+ char === '>' ? write(buffer, ">") :
+ char === '&' ? write(buffer, "&") :
+ char === '\'' ? write(buffer, "'") :
+ char === '"' ? write(buffer, """) : write(buffer, char)
+ end
+ String(take!(buffer))
+ else
+ text
+ end
+end
diff --git a/test/fixtures/README b/test/fixtures/README
new file mode 100644
index 00000000..b3c605ed
--- /dev/null
+++ b/test/fixtures/README
@@ -0,0 +1,3 @@
+# This is a plaintext README
+
+This should render just as plaintext ().
diff --git a/test/fixtures/README.html b/test/fixtures/README.html
new file mode 100644
index 00000000..b5bb6760
--- /dev/null
+++ b/test/fixtures/README.html
@@ -0,0 +1,4 @@
+# This is a plaintext README
+
+This should render just as plaintext (<pre>).
+
diff --git a/test/fixtures/readme.ORG b/test/fixtures/readme.ORG
new file mode 100644
index 00000000..c3bfa310
--- /dev/null
+++ b/test/fixtures/readme.ORG
@@ -0,0 +1,100 @@
+#+title: Org.jl
+#+author: tecosaur
+
+#+html: 
+
+A library for working with Org files. Specifically, this provides utilities for:
+
++ Parsing Org text to an AST (Abstract Syntax Tree)
++ Regenerating Org text from an AST
++ Basic manipulation and analysis of an Org document AST
++ Generating basic representations Org content in the terminal, and a few other forms.
+
+With the exception of some of the /particularly fancy/ capabilities provided by
+=org-mode= (like Babel and Calc-based spreadsheeting), this project aims to
+exactly match the interpretation of Org mode markup --- specifically the AST
+generated by =org-element.el=. This goal is not yet achieved, however
+the bulk of the work is now complete.
+
+*Org.jl* implements the vast majority of the [[https://orgmode.org/worg/dev/org-syntax.html][org-syntax]] document (see the
+Progress table). This can be checked by looking at ~Org.compatability~ in
+Julia.
+
+* Basic usage
+
+#+begin_src julia
+# after installing with ~] add Org~ or ~Pkg.add("Org")~
+using Org
+text1 = org"Some *Org* markup, written with ease using the ~org\"\"~ macro."
+parsetree(text1) # show the generated parse tree
+
+text2 = parse(OrgDoc, "Some *Org* markup, written with ease using the ~parse~ function.")
+diff(text1, text2) # show the components of the parse trees that differ
+
+dochead = @doc Org.Heading # the documentation for the Heading component (::OrgDoc)
+org(dochead) # generate Org text that produces the Org.Heading object
+string(dochead) # as above, but produces a String
+
+parse(OrgDoc, string(dochead)) == dochead # round-trip equality
+
+# get the lang of each source block
+[c.lang for c in dochead.components if c isa Org.SourceBlock]
+#+end_src
+
+* Progress
+
+| Component | Type | Parse | Org | Term | HTML |
+|---------------------+------+-------+-----+------+------|
+| Heading | X | X | X | X | X |
+| Section | X | X | X | X | X |
+|---------------------+------+-------+-----+------+------|
+| Affiliated Keywords | X | X | X | X | |
+|---------------------+------+-------+-----+------+------|
+| GreaterBlock | X | X | X | X | X |
+| Drawer | X | X | X | X | X |
+| DynamicBlock | X | X | X | X | X |
+| FootnoteDefinition | X | X | X | X | |
+| InlineTask | X | | | | |
+| Item | X | X | X | X | X |
+| List | X | X | X | X | X |
+| PropertyDrawer | X | X | X | X | X |
+| Table | X | X | X | X | X |
+|---------------------+------+-------+-----+------+------|
+| BabelCall | X | X | X | X | X |
+| Block | X | X | X | X | |
+| Clock | X | X | X | X | |
+| DiarySexp | X | X | X | X | |
+| Planning | X | X | X | X | |
+| Comment | X | X | X | X | X |
+| FixedWidth | X | X | X | X | X |
+| HorizontalRule | X | X | X | X | X |
+| Keyword | X | X | X | X | |
+| LaTeXEnvironment | X | X | X | X | |
+| NodeProperty | X | X | X | X | X |
+| Paragraph | X | X | X | X | X |
+| TableRow | X | X | X | X | X |
+| TableHRule | X | X | X | X | X |
+| BlankLine | X | X | X | X | X |
+|---------------------+------+-------+-----+------+------|
+| OrgEntity | X | X | X | X | X |
+| LaTeXFragment | X | X | X | X | |
+| ExportSnippet | X | X | X | X | X |
+| FootnoteReference | X | X | X | X | X |
+| InlineBabelCall | X | X | X | X | X |
+| InlineSrcBlock | X | X | X | X | X |
+| RadioLink | X | X | X | X | X |
+| PlainLink | X | X | X | X | X |
+| AngleLink | X | X | X | X | X |
+| RegularLink | X | X | X | X | X |
+| LineBreak | X | X | X | X | X |
+| Macro | X | X | X | X | X |
+| Citation | X | X | X | X | X |
+| RadioTarget | X | X | X | X | X |
+| Target | X | X | X | X | X |
+| StatisticsCookie | X | X | X | X | X |
+| Subscript | X | X | X | X | X |
+| Superscript | X | X | X | X | X |
+| TableCell | X | X | X | X | X |
+| Timestamp | X | X | X | X | X |
+| TextPlain | X | X | X | X | X |
+| TextMarkup | X | X | X | X | X |
diff --git a/test/fixtures/readme.ORG.html b/test/fixtures/readme.ORG.html
new file mode 100644
index 00000000..9038deaf
--- /dev/null
+++ b/test/fixtures/readme.ORG.html
@@ -0,0 +1,88 @@
+
+tecosaur
+
+A library for working with Org files. Specifically, this provides utilities for:
+
+Parsing Org text to an AST (Abstract Syntax Tree)
Regenerating Org text from an AST
Basic manipulation and analysis of an Org document AST
Generating basic representations Org content in the terminal, and a few other forms.
+With the exception of some of the particularly fancy capabilities provided by org-mode (like Babel and Calc-based spreadsheeting), this project aims to exactly match the interpretation of Org mode markup — specifically the AST generated by org-element.el. This goal is not yet achieved, however the bulk of the work is now complete.
+Org.jl implements the vast majority of the org-syntax document (see the Progress table). This can be checked by looking at Org.compatability in Julia.
+
+
# after installing with ~] add Org~ or ~Pkg.add("Org")~
+using Org
+text1 = org"Some *Org* markup, written with ease using the ~org\"\"~ macro."
+parsetree(text1) # show the generated parse tree
+
+text2 = parse(OrgDoc, "Some *Org* markup, written with ease using the ~parse~ function.")
+diff(text1, text2) # show the components of the parse trees that differ
+
+dochead = @doc Org.Heading # the documentation for the Heading component (::OrgDoc)
+org(dochead) # generate Org text that produces the Org.Heading object
+string(dochead) # as above, but produces a String
+
+parse(OrgDoc, string(dochead)) == dochead # round-trip equality
+
+# get the lang of each source block
+[c.lang for c in dochead.components if c isa Org.SourceBlock]
+
+
+
+| Component | Type | Parse | Org | Term | HTML |
|---|
+
+
+| Heading | X | X | X | X | X |
+| Section | X | X | X | X | X |
+| Affiliated Keywords | X | X | X | X | |
+| GreaterBlock | X | X | X | X | X |
+| Drawer | X | X | X | X | X |
+| DynamicBlock | X | X | X | X | X |
+| FootnoteDefinition | X | X | X | X | |
+| InlineTask | X | | | | |
+| Item | X | X | X | X | X |
+| List | X | X | X | X | X |
+| PropertyDrawer | X | X | X | X | X |
+| Table | X | X | X | X | X |
+| BabelCall | X | X | X | X | X |
+| Block | X | X | X | X | |
+| Clock | X | X | X | X | |
+| DiarySexp | X | X | X | X | |
+| Planning | X | X | X | X | |
+| Comment | X | X | X | X | X |
+| FixedWidth | X | X | X | X | X |
+| HorizontalRule | X | X | X | X | X |
+| Keyword | X | X | X | X | |
+| LaTeXEnvironment | X | X | X | X | |
+| NodeProperty | X | X | X | X | X |
+| Paragraph | X | X | X | X | X |
+| TableRow | X | X | X | X | X |
+| TableHRule | X | X | X | X | X |
+| BlankLine | X | X | X | X | X |
+| OrgEntity | X | X | X | X | X |
+| LaTeXFragment | X | X | X | X | |
+| ExportSnippet | X | X | X | X | X |
+| FootnoteReference | X | X | X | X | X |
+| InlineBabelCall | X | X | X | X | X |
+| InlineSrcBlock | X | X | X | X | X |
+| RadioLink | X | X | X | X | X |
+| PlainLink | X | X | X | X | X |
+| AngleLink | X | X | X | X | X |
+| RegularLink | X | X | X | X | X |
+| LineBreak | X | X | X | X | X |
+| Macro | X | X | X | X | X |
+| Citation | X | X | X | X | X |
+| RadioTarget | X | X | X | X | X |
+| Target | X | X | X | X | X |
+| StatisticsCookie | X | X | X | X | X |
+| Subscript | X | X | X | X | X |
+| Superscript | X | X | X | X | X |
+| TableCell | X | X | X | X | X |
+| Timestamp | X | X | X | X | X |
+| TextPlain | X | X | X | X | X |
+| TextMarkup | X | X | X | X | X |
+
+
WARNING! Unknown README file format: readme.unknown.format
+Unknown format. Becomes text
++ warning box.
+
diff --git a/test/runtests.jl b/test/runtests.jl
index 0b1b715b..ef23d7b4 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -62,9 +62,20 @@ const julia = first(Base.julia_cmd())
end
@testset "Readme rendering" begin
- DocumentationGenerator.render_html(joinpath(@__DIR__, "fixtures", "readme.md"), joinpath(@__DIR__, "readme.html"), "/foo/", "/bar/")
-
- @test read(joinpath(@__DIR__, "fixtures", "readme.html"), String) == read(joinpath(@__DIR__, "readme.html"), String)
+ @testset "Readme rendering: $file" for file in ("readme.md", "README", "readme.ORG", "readme.unknown.format")
+ input_path = joinpath(@__DIR__, "fixtures", file)
+ reference_path = joinpath(@__DIR__, "fixtures", "$(file).html")
+ tmp_output = tempname()
+ s = DocumentationGenerator.render_html(input_path, tmp_output, "/foo/", "/bar/")
+ # Test that the generated READMEs match the reference files
+ if haskey(ENV, "DOCUMENTATIONGENERATOR_UPDATE_REFERENCE_FILES")
+ @warn "Overwriting reference file" reference_path
+ cp(tmp_output, reference_path; force=true)
+ end
+ output = read(tmp_output, String)
+ reference = read(reference_path, String)
+ @test output == reference
+ end
end
@testset "Documentation Generation" begin
@@ -127,6 +138,18 @@ end
api_url="",
doctype = ["fallback_autodocs"],
),
+ # Org.jl README
+ (
+ name = "Bumper",
+ uuid = "8ce10254-0962-460f-a3d8-1f77fea1446e",
+ url = "https://github.com/MasonProtter/Bumper.jl",
+ versions = [v"0.4.1"],
+ installs = [true],
+ success = [true],
+ server_type = "github",
+ api_url="",
+ doctype = ["fallback_autodocs"],
+ ),
# with hosted docs
(
name = "Juno",