Merge pull request #3 from JuliaComputing/sp/upgrade

pfitzseb · web-flow · commit e4890d879f0a · 2020-03-25T12:35:13.000+01:00
upgrade gumbo
diff --git a/Project.toml b/Project.toml
@@ -8,7 +8,7 @@ Gumbo = "708ec375-b3d6-5a57-a7ce-8257bf98657a"
 
 [compat]
 julia = "1"
-Gumbo = "0.5"
+Gumbo = "0.7"
 
 [extras]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
diff --git a/src/HTMLSanitizer.jl b/src/HTMLSanitizer.jl
@@ -14,16 +14,13 @@ Sanitizes the HTML input according to `whitelist`.
 - `prettyprint`: Returns a prettier multiline string instead of a somewhat minified version.
 """
 function sanitize(input::AbstractString; isfragment = true, whitelist = WHITELIST, prettyprint = false)
-    input_preserve_ws = replace(input, r"(\s+)"s => s" 🐑\1🐑 ")
-    doc = parsehtml(input_preserve_ws)
-
+    doc = parsehtml(input, preserve_whitespace=true)
     sanitize_bfs(doc.root, whitelist)
 
     out = IOBuffer()
     print(out, doc.root, pretty = prettyprint)
 
     out = String(take!(out))
-    out = replace(out, r"\s?🐑(\s+)🐑\s?"s => s"\1")
 
     if isfragment
         out = replace(out, r"^<HTML>" => "")
@@ -33,12 +30,7 @@ function sanitize(input::AbstractString; isfragment = true, whitelist = WHITELIS
     end
 end
 
-reparent!(_, _) = nothing
-
-reparent!(node::HTMLElement, parent) = node.parent = parent
-
-# HTMLText isn't mutable, so this does nothing. Will lead to inconsistencies, but ¯\_(ツ)_/¯.
-reparent!(node::HTMLText, parent) = nothing
+reparent!(node, parent) = node.parent = parent
 
 function sanitize_bfs(tree, whitelist)
     i = 1
@@ -50,7 +42,7 @@ function sanitize_bfs(tree, whitelist)
             # reparent all nodes
             reparent!.(sanitized, Ref(tree))
             splice!(tree.children, i, sanitized)
-            i += length(sanitized)
+            # don't increment i here so the newly inserted nodes are sanitized in the next iteration
         else
             # reparent node
             reparent!(sanitized, tree)
@@ -75,7 +67,7 @@ function sanitize_element(el::HTMLElement{TAG}, whitelist) where TAG
             return Gumbo.HTMLText("")
         end
         @debug("Replacing `$(tag)` with its contents.")
-        out = sanitize_element.(el.children, Ref(whitelist))
+        out = el.children
         return isempty(out) ? Gumbo.HTMLText("") : out
     end
 
@@ -90,6 +82,8 @@ sanitize_element(el::HTMLText, whitelist) = el
 
 const REGEX_PROTOCOL = r"\A\s*([^\/#]*?)(?:\:|&#0*58|&#x0*3a)"i
 
+sanitize_attributes(el, whitelist) = el
+
 function sanitize_attributes(el::HTMLElement{TAG}, whitelist) where TAG
     tag = string(TAG)
     attributes = attrs(el)
@@ -128,9 +122,12 @@ function sanitize_attributes(el::HTMLElement{TAG}, whitelist) where TAG
     return el
 end
 
-function is_relative_url(url)
-    startswith(url, "./")
-end
+# A relative URL either
+# 1. starts with `/` (root-relative).
+# 2. starts with `//` (protocol-relative).
+# 3. starts with `../`/`./` (relative directory traversal)
+# 4. doesn't start with either of the above and doesn't start with a protocol (e.g. `foo/bar.html`)
+is_relative_url(url) = occursin(r"\.?\.?//?"i, url) || !occursin(r"^\w+://"i, url)
 
 """
 Default whitelist. Allows many elements and attributes, but crucially removes `<script>` elements
@@ -142,7 +139,7 @@ const WHITELIST = Dict(
         "div","ins","del","sup","sub","p","ol","ul","table","thead","tbody","tfoot","blockquote",
         "dl","dt","dd","kbd","q","samp","var","hr","ruby","rt","rp","li","tr","td","th","s","strike",
         "summary","details","caption","figure","figcaption","abbr","bdo","cite","dfn","mark",
-        "small","span","time","wbr"
+        "small","span","time","wbr","center"
     ],
     :remove_contents => ["script"],
     :attributes => Dict(
diff --git a/test/runtests.jl b/test/runtests.jl
@@ -47,7 +47,7 @@ using Test
   end
 
   @testset "test_whitelisted_longdesc_schemes_are_allowed" begin
-    stuff = """<img longdesc="http://longdesc.com"src="./foo.jpg"></img>"""
+    stuff = """<img longdesc="http://longdesc.com" src="./foo.jpg"></img>"""
     html  = HTMLSanitizer.sanitize(stuff)
     @test stuff == html
   end
@@ -72,7 +72,12 @@ using Test
     @test stuff == html
   end
 
-  @testset "test_script_contents_are_removed" begin
+  @testset "test_script_contents_are_removed1" begin
+    orig = """<div><script>JavaScript!</script></div>"""
+    @test "<div></div>" == HTMLSanitizer.sanitize(orig)
+  end
+
+  @testset "test_script_contents_are_removed2" begin
     orig = """<script>JavaScript!</script>"""
     @test "" == HTMLSanitizer.sanitize(orig)
   end
@@ -142,8 +147,43 @@ end
     </body>
   </html>
   """
-  expected = "<HTML>\n\n  \n    \n  \n  \n    <p>A simple test page.</p>\n    <a></a>\n    <a></a>\n    <pre>\n        <code>\nfoo\nbar\nbaz\n        </code>\n    </pre>\n  \n\n</HTML>"
+  expected = "<HTML>\n    \n  \n  \n    <p>A simple test page.</p>\n    <a></a>\n    <a></a>\n    <pre>        <code>\nfoo\nbar\nbaz\n        </code>\n    </pre>\n  \n\n</HTML>"
   @test sanitize(orig, isfragment=false) == expected
 end
 
+@testset "urls" begin
+  @testset "relative" begin
+    orig = """<img src="foo/bar.html"></img>"""
+    @test sanitize(orig) == orig
+
+    orig = """<img src="/foo/bar.html"></img>"""
+    @test sanitize(orig) == orig
+
+    orig = """<img src="//foo/bar.html"></img>"""
+    @test sanitize(orig) == orig
+
+    orig = """<img src="./foo/bar.html"></img>"""
+    @test sanitize(orig) == orig
+
+    orig = """<img src="/asd://foo/bar.html"></img>"""
+    @test sanitize(orig) == orig
+  end
+
+  @testset "protocols" begin
+    orig = """<img src="asd://foo/bar.html"></img>"""
+    @test sanitize(orig) == "<img></img>"
+
+    orig = """<img src="http://foo/bar.html"></img>"""
+    @test sanitize(orig) == orig
+
+    orig = """<img src="https://foo/bar.html"></img>"""
+    @test sanitize(orig) == orig
+  end
+end
+
+@testset "edge case" begin
+  html = read(joinpath(@__DIR__, "testhtml.html"), String)
+  sanitize(html) == read(joinpath(@__DIR__, "testhtml_out.html"), String)
+end
+
 include("malicious_html.jl")
diff --git a/test/testhtml.html b/test/testhtml.html
@@ -0,0 +1,40 @@
+<h1>RiemannHilbert.jl</h1>
+<p>A Julia package for solving Riemann–Hilbert problems</p>
+<p><a href="https://travis-ci.org/JuliaHolomorphic/RiemannHilbert.jl"><img src="https://travis-ci.org/JuliaHolomorphic/RiemannHilbert.jl.svg?branch=master" alt="Build Status" /></a>
+<a href="https://codecov.io/gh/JuliaHolomorphic/RiemannHilbert.jl"><img src="https://codecov.io/gh/JuliaHolomorphic/RiemannHilbert.jl/branch/master/graph/badge.svg" alt="codecov" /></a>
+<a href="https://gitter.im/JuliaApproximation/ApproxFun.jl?utm_source=badge&amp;utm_medium=badge&amp;utm_campaign=pr-badge&amp;utm_content=badge"><img src="https://badges.gitter.im/JuliaApproximation/ApproxFun.jl.svg" alt="Join the chat at https://gitter.im/JuliaApproximation/ApproxFun.jl" /></a></p>
+<center>
+<img src="images/sixrays.jpg" height="250" alt=".">
+</center>
+<p>A Riemann–Hilbert problem is a certain type of boundary value problem in the complex plane where an analytic function has prescribed jumps.
+They arise in integrable systems, random matrices, spectral analysis, orthogonal polynomials, and elsewhere. This package implements
+the numerical method of [Olver 2011, Olver 2012] (see also review in [Trogodon &amp; Olver 2015]) for solving Riemann–Hilbert problems, and is very much related to <a href="https://github.com/dlfivefifty/RHPackage">RHPackage</a>.</p>
+<p>For an example, the following calculates the Hastings–McLeod solution to Painlev'e II at the origin,
+which is posed on 4 rays:</p>
+<pre><code class="language-julia"># Define the contour
+Γ = Segment(0, 2.5exp(im*π/6)) ∪ Segment(0, 2.5exp(5im*π/6)) ∪
+                Segment(0, 2.5exp(-5im*π/6)) ∪ Segment(0, 2.5exp(-im*π/6))
+
+# Defe the jump function
+G = Fun( z -&gt; if angle(z) ≈ π/6
+                                        [1 0; im*exp(8im/3*z^3) 1]
+                                elseif angle(z) ≈ 5π/6
+                                        [1 0; -im*exp(8im/3*z^3) 1]
+                                elseif angle(z) ≈ -π/6
+                                        [1 im*exp(-8im/3*z^3); 0 1]
+                                elseif angle(z) ≈ -5π/6
+                                        [1 -im*exp(-8im/3*z^3); 0 1]
+                                end, Γ)
+
+# Solve the Riemann–Hilbert problem. We transpose to recast a left
+# Riemann–Hilbert problem as a left one.
+Φ = transpose(rhsolve(transpose(G), 4*200)) # use 200 collocation points per ray
+z = Fun(ℂ) # The function z in the complex plane
+2(z*Φ[1,2])(Inf) # Evaluate 2lim_{z -&gt; ∞} zΦ(z)_{1,2}
+</code></pre>
+<h1>References</h1>
+<ol>
+<li>T. Trogdon &amp; S. Olver (2015), <a href="http://bookstore.siam.org/ot146/">Riemann–Hilbert Problems, Their Numerical Solution and the Computation of Nonlinear Special Functions</a>, SIAM.</li>
+<li>S. Olver (2012), <a href="https://link.springer.com/article/10.1007/s00211-012-0459-7">A general framework for solving Riemann–Hilbert problems numerically</a>, Numer. Math., 122: 305–340.</li>
+<li>S. Olver (2011), <a href="https://link.springer.com/article/10.1007/s10208-010-9079-8">Numerical solution of Riemann–Hilbert problems: Painlevé II</a>, Found. Comput. Maths, 11: 153–179.</li>
+</ol>
diff --git a/test/testhtml_out.html b/test/testhtml_out.html
@@ -0,0 +1,40 @@
+<h1>RiemannHilbert.jl</h1>
+<p>A Julia package for solving Riemann–Hilbert problems</p>
+<p><a href="https://travis-ci.org/JuliaHolomorphic/RiemannHilbert.jl"><img alt="Build Status" src="https://travis-ci.org/JuliaHolomorphic/RiemannHilbert.jl.svg?branch=master"></img></a>
+<a href="https://codecov.io/gh/JuliaHolomorphic/RiemannHilbert.jl"><img alt="codecov" src="https://codecov.io/gh/JuliaHolomorphic/RiemannHilbert.jl/branch/master/graph/badge.svg"></img></a>
+<a href="https://gitter.im/JuliaApproximation/ApproxFun.jl?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge"><img alt="Join the chat at https://gitter.im/JuliaApproximation/ApproxFun.jl" src="https://badges.gitter.im/JuliaApproximation/ApproxFun.jl.svg"></img></a></p>
+<center>
+<img alt="." height="250" src="images/sixrays.jpg"></img>
+</center>
+<p>A Riemann–Hilbert problem is a certain type of boundary value problem in the complex plane where an analytic function has prescribed jumps.
+They arise in integrable systems, random matrices, spectral analysis, orthogonal polynomials, and elsewhere. This package implements
+the numerical method of [Olver 2011, Olver 2012] (see also review in [Trogodon & Olver 2015]) for solving Riemann–Hilbert problems, and is very much related to <a href="https://github.com/dlfivefifty/RHPackage">RHPackage</a>.</p>
+<p>For an example, the following calculates the Hastings–McLeod solution to Painlev'e II at the origin,
+which is posed on 4 rays:</p>
+<pre><code># Define the contour
+Γ = Segment(0, 2.5exp(im*π/6)) ∪ Segment(0, 2.5exp(5im*π/6)) ∪
+                Segment(0, 2.5exp(-5im*π/6)) ∪ Segment(0, 2.5exp(-im*π/6))
+
+# Defe the jump function
+G = Fun( z -> if angle(z) ≈ π/6
+                                        [1 0; im*exp(8im/3*z^3) 1]
+                                elseif angle(z) ≈ 5π/6
+                                        [1 0; -im*exp(8im/3*z^3) 1]
+                                elseif angle(z) ≈ -π/6
+                                        [1 im*exp(-8im/3*z^3); 0 1]
+                                elseif angle(z) ≈ -5π/6
+                                        [1 -im*exp(-8im/3*z^3); 0 1]
+                                end, Γ)
+
+# Solve the Riemann–Hilbert problem. We transpose to recast a left
+# Riemann–Hilbert problem as a left one.
+Φ = transpose(rhsolve(transpose(G), 4*200)) # use 200 collocation points per ray
+z = Fun(ℂ) # The function z in the complex plane
+2(z*Φ[1,2])(Inf) # Evaluate 2lim_{z -> ∞} zΦ(z)_{1,2}
+</code></pre>
+<h1>References</h1>
+<ol>
+<li>T. Trogdon & S. Olver (2015), <a href="http://bookstore.siam.org/ot146/">Riemann–Hilbert Problems, Their Numerical Solution and the Computation of Nonlinear Special Functions</a>, SIAM.</li>
+<li>S. Olver (2012), <a href="https://link.springer.com/article/10.1007/s00211-012-0459-7">A general framework for solving Riemann–Hilbert problems numerically</a>, Numer. Math., 122: 305–340.</li>
+<li>S. Olver (2011), <a href="https://link.springer.com/article/10.1007/s10208-010-9079-8">Numerical solution of Riemann–Hilbert problems: Painlevé II</a>, Found. Comput. Maths, 11: 153–179.</li>
+</ol>
