Introduce Encoding parametric singleton type

First step towards efficient encoders for common encodings,
as well as towards providing information about encodings.

This also allows adding convenience methods to base I/O functions taking
an additional encoding parameter without risking ambiguities.

Author: nalimilan

src/StringEncodings.jl

@@ -1,4 +1,4 @@
-# This file is a part of Julia. License is MIT: http://julialang.org/license
+# This file is a part of StringEncodings.jl. License is MIT: http://julialang.org/license
 
 module StringEncodings
 import Base: close, eof, flush, read, readall, write, show
@@ -8,6 +8,7 @@ export StringEncoder, StringDecoder, encode, decode, encodings
 export StringEncodingError, OutputBufferError, IConvError
 export InvalidEncodingError, InvalidSequenceError, IncompleteSequenceError
 
+include("encodings.jl")
 
 abstract StringEncodingError
 
@@ -62,7 +63,7 @@ function iconv_close(cd::Ptr{Void})
     end
 end
 
-function iconv_open(tocode, fromcode)
+function iconv_open(tocode::ASCIIString, fromcode::ASCIIString)
     p = ccall((:iconv_open, libiconv), Ptr{Void}, (Cstring, Cstring), tocode, fromcode)
     if p != Ptr{Void}(-1)
         return p
@@ -173,14 +174,16 @@ end
 ## StringEncoder
 
 """
-    StringEncoder(istream, to, from="UTF-8")
+    StringEncoder(istream, to, from=enc"UTF-8")
 
 Returns a new write-only I/O stream, which converts any text in the encoding `from`
 written to it into text in the encoding `to` written to ostream. Calling `close` on the
 stream is necessary to complete the encoding (but does not close `ostream`).
+
+`to` and `from` can be specified either as a string or as an `Encoding` object.
 """
-function StringEncoder(ostream::IO, to::ASCIIString, from::ASCIIString="UTF-8")
-    cd = iconv_open(to, from)
+function StringEncoder(ostream::IO, to::Encoding, from::Encoding=enc"UTF-8")
+    cd = iconv_open(ASCIIString(to), ASCIIString(from))
     inbuf = Vector{UInt8}(BUFSIZE)
     outbuf = Vector{UInt8}(BUFSIZE)
     s = StringEncoder(ostream, cd, inbuf, outbuf,
@@ -190,6 +193,11 @@ function StringEncoder(ostream::IO, to::ASCIIString, from::ASCIIString="UTF-8")
     s
 end
 
+StringEncoder(ostream::IO, to::AbstractString, from::Encoding=enc"UTF-8") =
+    StringEncoder(ostream, Encoding(to), from)
+StringEncoder(ostream::IO, to::AbstractString, from::AbstractString) =
+    StringEncoder(ostream, Encoding(to), Encoding(from))
+
 # Flush input buffer and convert it into output buffer
 # Returns the number of bytes written to output buffer
 function flush(s::StringEncoder)
@@ -226,16 +234,18 @@ end
 ## StringDecoder
 
 """
-    StringDecoder(istream, from, to="UTF-8")
+    StringDecoder(istream, from, to=enc"UTF-8")
 
 Returns a new read-only I/O stream, which converts text in the encoding `from`
 read from `istream` into text in the encoding `to`.
 
+`to` and `from` can be specified either as a string or as an `Encoding` object.
+
 Note that some implementations (notably the Windows one) may accept invalid sequences
 in the input data without raising an error.
 """
-function StringDecoder(istream::IO, from::ASCIIString, to::ASCIIString="UTF-8")
-    cd = iconv_open(to, from)
+function StringDecoder(istream::IO, from::Encoding, to::Encoding=enc"UTF-8")
+    cd = iconv_open(ASCIIString(to), ASCIIString(from))
     inbuf = Vector{UInt8}(BUFSIZE)
     outbuf = Vector{UInt8}(BUFSIZE)
     s = StringDecoder(istream, cd, inbuf, outbuf,
@@ -245,6 +255,11 @@ function StringDecoder(istream::IO, from::ASCIIString, to::ASCIIString="UTF-8")
     s
 end
 
+StringDecoder(istream::IO, from::AbstractString, to::Encoding=enc"UTF-8") =
+    StringDecoder(istream, Encoding(from), to)
+StringDecoder(istream::IO, from::AbstractString, to::AbstractString) =
+    StringDecoder(istream, Encoding(from), Encoding(to))
+
 # Fill input buffer and convert it into output buffer
 # Returns the number of bytes written to output buffer
 function fill_buffer!(s::StringDecoder)
@@ -289,68 +304,67 @@ end
 ## Convenience I/O functions
 if isdefined(Base, :readstring)
     @doc """
-        readstring(stream or filename, enc::ASCIIString)
+        readstring(stream or filename, enc::Encoding)
 
     Read the entire contents of an I/O stream or a file in encoding `enc` as a string.
     """ ->
-    Base.readstring(s::IO, enc::ASCIIString) = readstring(StringDecoder(s, enc))
-    Base.readstring(filename::AbstractString, enc::ASCIIString) = open(io->readstring(io, enc), filename)
+    Base.readstring(s::IO, enc::Encoding) = readstring(StringDecoder(s, enc))
+    Base.readstring(filename::AbstractString, enc::Encoding) = open(io->readstring(io, enc), filename)
 else # Compatibility with Julia 0.4
     @doc """
-        readall(stream or filename, enc::ASCIIString)
+        readall(stream or filename, enc::Encoding)
 
     Read the entire contents of an I/O stream or a file in encoding `enc` as a string.
     """ ->
-    Base.readall(s::IO, enc::ASCIIString) = readall(StringDecoder(s, enc))
-    Base.readall(filename::AbstractString, enc::ASCIIString) = open(io->readall(io, enc), filename)
+    Base.readall(s::IO, enc::Encoding) = readall(StringDecoder(s, enc))
+    Base.readall(filename::AbstractString, enc::Encoding) = open(io->readall(io, enc), filename)
 end
 
 
 ## Functions to encode/decode strings
 
-encoding_string(::Type{ASCIIString}) = "ASCII"
-encoding_string(::Type{UTF8String})  = "UTF-8"
-encoding_string(::Type{UTF16String}) = (ENDIAN_BOM == 0x04030201) ? "UTF-16LE" : "UTF-16BE"
-encoding_string(::Type{UTF32String}) = (ENDIAN_BOM == 0x04030201) ? "UTF-32LE" : "UTF-32BE"
-
 """
-    decode([T,] a::Vector{UInt8}, enc::ASCIIString)
+    decode([T,] a::Vector{UInt8}, enc)
 
 Convert an array of bytes `a` representing text in encoding `enc` to a string of type `T`.
 By default, a `UTF8String` is returned.
 
+`enc` can be specified either as a string or as an `Encoding` object.
+
 Note that some implementations (notably the Windows one) may accept invalid sequences
 in the input data without raising an error.
 """
-function decode{T<:AbstractString}(::Type{T}, a::Vector{UInt8}, enc::ASCIIString)
+function decode{T<:AbstractString}(::Type{T}, a::Vector{UInt8}, enc::Encoding)
     b = IOBuffer(a)
     try
-        T(readbytes(StringDecoder(b, enc, encoding_string(T))))
+        T(readbytes(StringDecoder(b, enc, encoding(T))))
     finally
         close(b)
     end
 end
 
-decode(a::Vector{UInt8}, enc::ASCIIString) = decode(UTF8String, a, enc)
+decode{T<:AbstractString}(::Type{T}, a::Vector{UInt8}, enc::AbstractString) = decode(T, a, Encoding(enc))
+
+decode(a::Vector{UInt8}, enc::AbstractString) = decode(UTF8String, a, Encoding(enc))
+decode(a::Vector{UInt8}, enc::Union{AbstractString, Encoding}) = decode(UTF8String, a, enc)
 
 """
-    encode(s::AbstractString, enc::ASCIIString)
+    encode(s::AbstractString, enc)
 
 Convert string `s` to an array of bytes representing text in encoding `enc`.
+`enc` can be specified either as a string or as an `Encoding` object.
 """
-function encode(s::AbstractString, enc::ASCIIString)
+function encode(s::AbstractString, enc::Encoding)
     b = IOBuffer()
-    p = StringEncoder(b, enc, encoding_string(typeof(s)))
+    p = StringEncoder(b, enc, encoding(typeof(s)))
     write(p, s)
     close(p)
     takebuf_array(b)
 end
 
+encode(s::AbstractString, enc::AbstractString) = encode(s, Encoding(enc))
 
-## Function to list supported encodings
-include("encodings.jl")
-
-function test_encoding(enc)
+function test_encoding(enc::ASCIIString)
     # We assume that an encoding is supported if it's possible to convert from it to UTF-8:
     cd = ccall((:iconv_open, libiconv), Ptr{Void}, (Cstring, Cstring), enc, "UTF-8")
     if cd == Ptr{Void}(-1)

src/encodings.jl

@@ -1,4 +1,35 @@
-import StringEncodings: libiconv, iconv_close
+# This file is a part of StringEncodings.jl. License is MIT: http://julialang.org/license
+
+# Parametric singleton type representing a given string encoding via its symbol parameter
+
+import Base: show, print, convert
+export Encoding, @enc_str
+
+immutable Encoding{enc} end
+
+Encoding(s) = Encoding{symbol(s)}()
+macro enc_str(s)
+    :(Encoding{$(Expr(:quote, symbol(s)))}())
+end
+
+convert{T<:AbstractString, enc}(::Type{T}, ::Encoding{enc}) = string(enc)
+
+show{enc}(io::IO, ::Encoding{enc}) = print(io, string(enc), " string encoding type")
+print{enc}(io::IO, ::Encoding{enc}) = print(io, enc)
+
+
+## Get the encoding used by a string type
+encoding(::Type{ASCIIString}) = enc"ASCII"
+encoding(::Type{UTF8String})  = enc"UTF-8"
+
+if ENDIAN_BOM == 0x04030201
+    encoding(::Type{UTF16String}) = enc"UTF-16LE"
+    encoding(::Type{UTF32String}) = enc"UTF-32LE"
+else
+    encoding(::Type{UTF16String}) = enc"UTF-16BE"
+    encoding(::Type{UTF32String}) = enc"UTF-32BE"
+end
+
 encodings_list = ["1026", "1046", "1047", "10646-1:1993", "10646-1:1993/UCS4",
                   "437", "500", "500V1", "850", "851", "852", "855", "856", "857",
                   "860", "861", "862", "863", "864", "865", "866", "866NAV", "869",

test/runtests.jl

@@ -7,7 +7,7 @@ for s in ("", "\0", "a", "café crème",
           "a string \0€ チャネルパ\0ー\0トナーの選択 with embedded and trailing nuls\0")
     # Test round-trip to Unicode formats, checking against pure-Julia implementation
     for (T, nullen) in ((UTF8String, 0), (UTF16String, 2), (UTF32String, 4))
-        enc = StringEncodings.encoding_string(T)
+        enc = StringEncodings.encoding(T)
         a = reinterpret(UInt8, T(s).data)
         # Adjust for explicit \0 only for .data on UTF16String/UTF32String
         a = a[1:end - nullen]
@@ -102,7 +102,7 @@ mktemp() do p, io
     s = "café crème"
     write(io, encode(s, "CP1252"))
     close(io)
-    @test readall(p, "CP1252") == s
+    @test readall(p, enc"CP1252") == s
 end
 
 @test_throws InvalidEncodingError p = StringEncoder(IOBuffer(), "nonexistent_encoding")
@@ -136,10 +136,13 @@ mktemp() do path, io
     write(io, encode(s, "ISO-2022-JP"))
     close(io)
 
-    @test readstring(path, "ISO-2022-JP") == s
-    @test open(io->readstring(io, "ISO-2022-JP"), path) == s
+    @test readstring(path, enc"ISO-2022-JP") == s
+    @test open(io->readstring(io, enc"ISO-2022-JP"), path) == s
 end
 
+
+## Test encodings support
+
 encodings_list = encodings()
 @test "ASCII" in encodings_list
 @test "UTF-8" in encodings_list