Merge pull request #3 from TimG1964/Normalize_newlines

Undo normalize newlines

Author: TimG1964

src/raw.jl

@@ -70,7 +70,6 @@ end
 function Raw(data::Vector{UInt8})#, ctx::Vector{Bool}=Bool[false])
     needle = Vector{UInt8}("xml:space")
     has_xml_space = findfirst(needle, data) !== nothing
-    data=normalize_newlines(data)
     return Raw(RawDocument, 0, 0, 0, data, [false], has_xml_space)
 end
 function Raw(data::Vector{UInt8}, has_xml_space::Bool, ctx::Vector{Bool}=Bool[false])
@@ -101,40 +100,6 @@ Base.read(io::IO, ::Type{Raw}) = Raw(read(io))
 
 Base.parse(x::AbstractString, ::Type{Raw}) = Raw(Vector{UInt8}(x))
 
-
-"""
-    normalize_newlines(bytes::Vector{UInt8}) -> Vector{UInt8}
-
-Implements XML 1.1 §2.11 line-end normalization:
-- CR (0x0D) alone  → LF (0x0A)
-- CR LF pair       → LF
-- NEL (U+0085)     → LF
-- LS  (U+2028)     → LF
-"""
-function normalize_newlines(bytes::Vector{UInt8})
-    n = length(bytes)
-    out = Vector{UInt8}(undef, n)
-    outlen = 0
-    i = 1
-    while i <= n
-        @inbounds b = bytes[i]
-        if b == 0x0D
-            outlen += 1; out[outlen] = 0x0A
-            i += (i < n && (bytes[i+1] == 0x0A || bytes[i+1] == 0x85)) ? 2 : 1
-        elseif b == 0xC2 && i < n && bytes[i+1] == 0x85
-            outlen += 1; out[outlen] = 0x0A
-            i += 2
-        elseif b == 0xE2 && i+2 <= n && bytes[i+1] == 0x80 && bytes[i+2] == 0xA8
-            outlen += 1; out[outlen] = 0x0A
-            i += 3
-        else
-            outlen += 1; out[outlen] = b
-            i += 1
-        end
-    end
-    return resize!(out, outlen)
-end
-
 # Mostly for debugging
 Base.peek(o::Raw, n::Int) = String(view(o.data[o.pos+o.len+1:min(end, o.pos + o.len + n + 1)]))
 

test/runtests.jl

@@ -579,58 +579,6 @@ end
 
 end
 
-#-----------------------------------------------------------------------------# Normalize_newlines
-# Helper to make writing tests easier
-to_bytes(s) = Vector{UInt8}(s)
-from_bytes(b) = String(b)
-
-@testset "normalize_newlines" begin
-    # 1. Lone CR -> LF
-    @test XML.normalize_newlines(to_bytes("a\rb")) == to_bytes("a\nb")
-
-    # 2. CRLF -> LF
-    @test XML.normalize_newlines(to_bytes("a\r\nb")) == to_bytes("a\nb")
-
-    # 3. CR NEL (0x85) -> LF
-    @test XML.normalize_newlines(UInt8[0x61, 0x0D, 0x85, 0x62]) == to_bytes("a\nb")
-
-    # 4. NEL (U+0085) UTF-8 form 0xC2 0x85 -> LF
-    @test XML.normalize_newlines(UInt8[0x61, 0xC2, 0x85, 0x62]) == to_bytes("a\nb")
-
-    # 5. LINE SEPARATOR (U+2028) UTF-8 form 0xE2 0x80 0xA8 -> LF
-    @test XML.normalize_newlines(UInt8[0x61, 0xE2, 0x80, 0xA8, 0x62]) == to_bytes("a\nb")
-
-    # 6. Mixed newline types in one string
-    mixed = UInt8[0x61, 0x0D, 0x0A, 0x62, 0xC2, 0x85, 0x63, 0xE2, 0x80, 0xA8, 0x64, 0x0D, 0x65]
-    expected = to_bytes("a\nb\nc\nd\ne")
-    @test XML.normalize_newlines(mixed) == expected
-
-    # 7. Consecutive CRs
-    @test XML.normalize_newlines(to_bytes("a\r\rb")) == to_bytes("a\n\nb")
-
-    # 8. Leading/trailing newlines
-    @test XML.normalize_newlines(to_bytes("\rabc\r")) == to_bytes("\nabc\n")
-
-    # 9. Empty input
-    @test XML.normalize_newlines(UInt8[]) == UInt8[]
-
-    # 10. No newline characters
-    @test XML.normalize_newlines(to_bytes("abcdef")) == to_bytes("abcdef")
-
-    # 11. Unicode safety: multi-byte chars around newlines
-    s = "α\r\nβ"  # α = 0xCE 0xB1, β = 0xCE 0xB2
-    @test XML.normalize_newlines(to_bytes(s)) == to_bytes("α\nβ")
-
-    # 12. Boundary case: CR at end of buffer
-    @test XML.normalize_newlines(UInt8[0x61, 0x0D]) == to_bytes("a\n")
-
-    # 13. Boundary case: 0xC2 at end (incomplete UTF-8 NEL)
-    @test XML.normalize_newlines(UInt8[0x61, 0xC2]) == UInt8[0x61, 0xC2]
-
-    # 14. Boundary case: 0xE2 0x80 at end (incomplete LINE SEPARATOR)
-    @test XML.normalize_newlines(UInt8[0x61, 0xE2, 0x80]) == UInt8[0x61, 0xE2, 0x80]
-end
-
 #-----------------------------------------------------------------------------# roundtrip
 @testset "read/write/read roundtrip" begin
     for path in all_files