work on DTD parsing, make escaping idempotent

Author: joshday

Project.toml

@@ -8,10 +8,3 @@ Mmap = "a63ad114-7e13-5084-954f-fe012c677804"
 
 [compat]
 julia = "1.7"
-
-[extras]
-Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
-Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-
-[targets]
-test = ["Test", "Downloads"]

README.md

@@ -193,3 +193,10 @@ Platform Info:
  EzXML.StreamReader  ■■■■■■■■■■■■■■■■■■■■■■■■■■■■ 165.21
       EzXML.readxml  ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■ 239.197
 ```
+
+<br>
+<br>
+
+# Possible Gotchas
+
+XML.jl doesn't escape special characters (`<`, `>`, `&`, `"`, and `'` ) for you, but it provides `XML.escape(::String)` and `XML.unescape(::String)` utility functions.

src/XML.jl

@@ -11,11 +11,24 @@ export
     parent, depth, next, prev
 
 #-----------------------------------------------------------------------------# escape/unescape
-# only used by Text nodes during `XML.write`
+# TODO: be smarter :
+# 1. to avoid escaping entities e.g. `&entity;`
+# 2. to avoid re-escaping already escaped entities e.g. `&`
 const escape_chars = ('&' => "&amp;", '<' => "&lt;", '>' => "&gt;", "'" => "&apos;", '"' => "&quot;")
-escape(x::AbstractString) = replace(x, escape_chars...)
+
 unescape(x::AbstractString) = replace(x, reverse.(escape_chars)...)
 
+# requires special handling of `&` to avoid double-escaping
+function escape(x::String)
+    s = replace(x,
+        r"&(?!(?:amp|lt|gt|quot|apos);)" => "&amp;",
+        '<' => "&lt;", '>' => "&gt;",
+        '"' => "&quot;",
+        ''' => "&apos;"
+    )
+    return s
+end
+
 #-----------------------------------------------------------------------------# NodeType
 """
     NodeType:
@@ -42,8 +55,9 @@ NodeTypes can be used to construct XML.Nodes:
 @enum(NodeType, CData, Comment, Declaration, Document, DTD, Element, ProcessingInstruction, Text)
 
 
-#-----------------------------------------------------------------------------# raw
+#-----------------------------------------------------------------------------# includes
 include("raw.jl")
+include("dtd.jl")
 
 abstract type AbstractXMLNode end
 
@@ -103,7 +117,7 @@ function prev(o::LazyNode)
     n.type === RawElementClose ? prev(LazyNode(n)) : LazyNode(n)
 end
 
-#-----------------------------------------------------------------------------?de
+#-----------------------------------------------------------------------------# Node
 """
     Node(nodetype, tag, attributes, value, children)
     Node(node::Node; kw...)  # copy node with keyword overrides
@@ -142,6 +156,17 @@ function Node(node::LazyNode)
     Node(nodetype, tag, attributes, value, isempty(c) ? nothing : map(Node, c))
 end
 
+# NOT in-place for Text Nodes
+function escape!(o::Node, warn::Bool=true)
+    if o.nodetype == Text
+        warn && @warn "escape!() called on a Text Node creates a new node."
+        return Text(escape(o.value))
+    end
+    isnothing(o.children) && return o
+    map!(x -> escape!(x, false), o.children, o.children)
+    o
+end
+
 
 Base.read(filename::AbstractString, ::Type{Node}) = Node(read(filename, Raw))
 Base.read(io::IO, ::Type{Node}) = Node(read(io, Raw))
@@ -314,7 +339,7 @@ function write(io::IO, x; indentsize::Int=2, depth::Union{Missing,Int}=depth(x))
     padding = indent ^ max(0, depth - 1)
     print(io, padding)
     if nodetype === Text
-        print(io, escape(value))
+        print(io, value)
     elseif nodetype === Element
         print(io, '<', tag)
         _print_attrs(io, x)

src/dtd.jl

@@ -0,0 +1,93 @@
+#-----------------------------------------------------------------------------# DeclaredElement
+struct DeclaredElement
+    name::String
+    content::String  # "ANY", "EMPTY", or "(children...)"
+    function DeclaredElement(name, content)
+        content in ("ANY", "EMPTY") || (content[1] == '('  && content[end] == ')') ||
+            error("DeclaredElement `content` must be 'ANY', 'EMPTY', or '(children...)'.  Got $content.")
+        new(name, content)
+    end
+end
+Base.show(io::IO, o::DeclaredElement) = print(io, "<!ELEMENT ", o.name, " ", o.content, ">")
+
+function get_declared_elements(data::Vector{UInt8})
+    i = findnext(Vector{UInt8}("<!ELEMENT"), data, 1)[end]
+    out = DeclaredElement[]
+    while !isnothing(i)
+        name, i = get_name(data, i + 1)
+        i = findnext(!isspace, data, i)
+        if data[i] == UInt8('(')
+            j = findnext(==(UInt8(')')), data, i + 1)
+            content = String(data[i:j])
+        else
+            content, i = get_name(data, i)
+        end
+        push!(out, DeclaredElement(name, content))
+        fn = findnext(Vector{UInt8}("<!ELEMENT"), data, i)
+        i = isnothing(fn) ? nothing : fn[end]
+    end
+    return out
+end
+
+#-----------------------------------------------------------------------------# DeclaredAttribute
+struct DeclaredAttribute
+    element_name::String
+    attribute_name::String
+    attribute_type::String
+    attribute_value::String
+end
+Base.show(io::IO, o::DeclaredAttribute) = print(io, "<!ATTLIST ", o.element_name, " ", o.attribute_name, " ", o.attribute_type, " ", o.attribute_value, ">")
+
+function get_declared_attributes(data)
+    []
+end
+
+#-----------------------------------------------------------------------------# DeclaredEntity
+struct DeclaredEntity
+    name::String
+    value::String
+end
+Base.show(io::IO, o::DeclaredEntity) = print(io, "<!ENTITY ", o.name, " ", o.value, ">")
+
+function get_declared_entities(data)
+    []
+end
+
+#-----------------------------------------------------------------------------# DTDBody
+struct DTDBody
+    root::String
+    elements::Vector{DeclaredElement}
+    attributes::Vector{DeclaredAttribute}
+    entities::Vector{DeclaredEntity}
+end
+
+function Base.show(io::IO, o::DTDBody)
+    println(io, "DTDBody(root=\"", o.root)
+    println(io, "  • DeclaredElements")
+    foreach(x -> println(io, "    ", x), o.elements)
+    println(io, "  • DeclaredAttributes")
+    println(io, "  • DeclaredEntities")
+end
+
+
+
+function DTDBody(data::Vector{UInt8})
+    start = "<!DOCTYPE"
+    data[1:length(start)] == Vector{UInt8}(start) || error("DTD must start with `<!DOCTYPE`.")
+    i = length(start) + 1
+    root, i = get_name(data, i)
+
+    i = findnext(==(UInt8('[')), data, i)
+    isnothing(i) && return DTDBody(root, [], [], [])
+
+    elements = get_declared_elements(data)
+    attributes = get_declared_attributes(data)
+    entities = get_declared_entities(data)
+    return DTDBody(root, elements, attributes, entities)
+end
+
+
+Base.read(filename::String, ::Type{DTDBody}) = DTDBody(read(filename))
+Base.read(io::IO, ::Type{DTDBody}) = Raw(read(io))
+Base.parse(s::AbstractString, ::Type{DTDBody}) = DTDBody(Vector{UInt8}(s))
+Base.parse(::Type{DTDBody}, s::AbstractString) = parse(s, DTDBody)

src/raw.jl

@@ -102,7 +102,7 @@ xml_nodes(o::Raw) = Iterators.Filter(is_node, o)
 
 #-----------------------------------------------------------------------------# get_name
 is_name_start_char(x::UInt8) = x in UInt8('A'):UInt8('Z') || x in UInt8('a'):UInt8('z') || x == UInt8('_')
-is_name_char(x::UInt8) = is_name_start_char(x) || x in UInt8('0'):UInt8('9') || x == UInt8('-') || x == UInt8('.')
+is_name_char(x::UInt8) = is_name_start_char(x) || x in UInt8('0'):UInt8('9') || x == UInt8('-') || x == UInt8('.') || x == UInt8(':')
 
 name_start(data, i) = findnext(is_name_start_char, data, i)
 name_stop(data, i) = findnext(!is_name_char, data, i) - 1
@@ -176,7 +176,7 @@ Return the value of `Text`, `CData`, `Comment`, or `DTD` nodes.
 """
 function value(o::Raw)
     if o.type === RawText
-        unescape(String(o))
+        String(o)
     elseif o.type === RawCData
         String(view(o.data, o.pos + length("<![CData[") : o.pos + o.len - 3))
     elseif o.type === RawComment

test/Project.toml

@@ -0,0 +1,4 @@
+[deps]
+AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
+Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

test/runtests.jl

@@ -1,7 +1,10 @@
 using XML
-using XML: Document, Element, Declaration, Comment, CData, DTD, ProcessingInstruction, Text
+using XML: Document, Element, Declaration, Comment, CData, DTD, ProcessingInstruction, Text, escape, unescape
 using Downloads: download
 using Test
+import AbstractTrees
+
+AbstractTrees.children(x::Node) = children(x)
 
 #-----------------------------------------------------------------------------# files
 xml_spec = download("http://www.w3.org/2001/xml.xsd")
@@ -16,6 +19,15 @@ all_files = [
     "example.kml" => example_kml
 ]
 
+#-----------------------------------------------------------------------------# utils
+@testset "utils" begin
+    s = "This > string < has & some \" special ' characters"
+    @test escape(s) == "This &gt; string &lt; has &amp; some &quot; special &apos; characters"
+    @test escape(escape(s)) == escape(s)
+    @test s == unescape(escape(s))
+    @test s == unescape(unescape(escape(s)))
+end
+
 #-----------------------------------------------------------------------------# Raw
 @testset "Raw tag/attributes/value" begin
     examples = [