get rid of some unused code in tokenize (#15)

Author: KristofferC

Tokenize/src/_precompile.jl

@@ -9,8 +9,6 @@ function _precompile_()
     precompile(Tokenize.Tokens.Token, ())
     precompile(Tokenize.Tokens.kind, (Tokenize.Tokens.Token,))
 
-    precompile(Tokenize.Lexers.is_cat_id_start, (Char, Int32,))
-    precompile(Tokenize.Lexers.is_identifier_char, (Char,))
     precompile(Tokenize.Lexers.is_identifier_start_char, (Char,))
     precompile(Tokenize.Lexers.readchar, (Tokenize.Lexers.Lexer{GenericIOBuffer{Array{UInt8, 1}}},))
     precompile(Tokenize.Lexers.next_token, (Tokenize.Lexers.Lexer{GenericIOBuffer{Array{UInt8, 1}}},))
@@ -68,5 +66,5 @@ function _precompile_()
     precompile(Tokenize.Lexers.accept_batch, (Tokenize.Lexers.Lexer{GenericIOBuffer{Array{UInt8, 1}}}, typeof(Tokenize.Lexers.isdigit),))
     precompile(Tokenize.Lexers.accept, (Tokenize.Lexers.Lexer{GenericIOBuffer{Array{UInt8, 1}}}, Char,))
 
-    precompile(Tokenize.Lexers.readchar, (GenericIOBuffer{Array{UInt8, 1}},))    
+    precompile(Tokenize.Lexers.readchar, (GenericIOBuffer{Array{UInt8, 1}},))
 end

Tokenize/src/utilities.jl

@@ -1,113 +1,7 @@
-#=
-The code in here has been extracted from the JuliaParser.jl package
-with license:
-
-The JuliaParser.jl package is licensed under the MIT "Expat" License:
-
-> Copyright (c) 2014: Jake Bolewski.
->
-> Permission is hereby granted, free of charge, to any person obtaining
-> a copy of this software and associated documentation files (the
-> "Software"), to deal in the Software without restriction, including
-> without limitation the rights to use, copy, modify, merge, publish,
-> distribute, sublicense, and/or sell copies of the Software, and to
-> permit persons to whom the Software is furnished to do so, subject to
-> the following conditions:
->
-> The above copyright notice and this permission notice shall be
-> included in all copies or substantial portions of the Software.
->
-> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-> EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-> MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-> IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-> CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-> TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-> SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-=#
-
 import Base.Unicode
 
-
-@inline function utf8_trailing(i)
-    if i < 193
-        return 0
-    elseif i < 225
-        return 1
-    elseif i < 241
-        return 2
-    elseif i < 249
-        return 3
-    elseif i < 253
-        return 4
-    else
-        return 5
-    end
-end
-
-const utf8_offset = [0x00000000
-                    0x00003080
-                    0x000e2080
-                    0x03c82080
-                    0xfa082080
-                    0x82082080]
-# const EOF_CHAR = convert(Char,typemax(UInt32))
 const EOF_CHAR = typemax(Char)
 
-
-function is_cat_id_start(ch::Char, cat::Integer)
-    c = UInt32(ch)
-    return (cat == Unicode.UTF8PROC_CATEGORY_LU || cat == Unicode.UTF8PROC_CATEGORY_LL ||
-            cat == Unicode.UTF8PROC_CATEGORY_LT || cat == Unicode.UTF8PROC_CATEGORY_LM ||
-            cat == Unicode.UTF8PROC_CATEGORY_LO || cat == Unicode.UTF8PROC_CATEGORY_NL ||
-            cat == Unicode.UTF8PROC_CATEGORY_SC ||  # allow currency symbols
-            cat == Unicode.UTF8PROC_CATEGORY_SO ||  # other symbols
-
-            # math symbol (category Sm) whitelist
-            (c >= 0x2140 && c <= 0x2a1c &&
-             ((c >= 0x2140 && c <= 0x2144) || # ⅀, ⅁, ⅂, ⅃, ⅄
-              c == 0x223f || c == 0x22be || c == 0x22bf || # ∿, ⊾, ⊿
-              c == 0x22a4 || c == 0x22a5 || # ⊤ ⊥
-
-              (c >= 0x2202 && c <= 0x2233 &&
-               (c == 0x2202 || c == 0x2205 || c == 0x2206 || # ∂, ∅, ∆
-                c == 0x2207 || c == 0x220e || c == 0x220f || # ∇, ∎, ∏
-                c == 0x2210 || c == 0x2211 || # ∐, ∑
-                c == 0x221e || c == 0x221f || # ∞, ∟
-                c >= 0x222b)) || # ∫, ∬, ∭, ∮, ∯, ∰, ∱, ∲, ∳
-
-              (c >= 0x22c0 && c <= 0x22c3) ||  # N-ary big ops: ⋀, ⋁, ⋂, ⋃
-              (c >= 0x25F8 && c <= 0x25ff) ||  # ◸, ◹, ◺, ◻, ◼, ◽, ◾, ◿
-
-              (c >= 0x266f &&
-               (c == 0x266f || c == 0x27d8 || c == 0x27d9 || # ♯, ⟘, ⟙
-                (c >= 0x27c0 && c <= 0x27c1) ||  # ⟀, ⟁
-                (c >= 0x29b0 && c <= 0x29b4) ||  # ⦰, ⦱, ⦲, ⦳, ⦴
-                (c >= 0x2a00 && c <= 0x2a06) ||  # ⨀, ⨁, ⨂, ⨃, ⨄, ⨅, ⨆
-                (c >= 0x2a09 && c <= 0x2a16) ||  # ⨉, ⨊, ⨋, ⨌, ⨍, ⨎, ⨏, ⨐, ⨑, ⨒,
-                                                 # ⨓, ⨔, ⨕, ⨖
-                c == 0x2a1b || c == 0x2a1c)))) || # ⨛, ⨜
-
-            (c >= 0x1d6c1 && # variants of \nabla and \partial
-             (c == 0x1d6c1 || c == 0x1d6db ||
-              c == 0x1d6fb || c == 0x1d715 ||
-              c == 0x1d735 || c == 0x1d74f ||
-              c == 0x1d76f || c == 0x1d789 ||
-              c == 0x1d7a9 || c == 0x1d7c3)) ||
-
-            # super- and subscript +-=()
-            (c >= 0x207a && c <= 0x207e) ||
-            (c >= 0x208a && c <= 0x208e) ||
-
-            # angle symbols
-            (c >= 0x2220 && c <= 0x2222) || # ∠, ∡, ∢
-            (c >= 0x299b && c <= 0x29af) || # ⦛, ⦜, ⦝, ⦞, ⦟, ⦠, ⦡, ⦢, ⦣, ⦤, ⦥,
-                                            # ⦦, ⦧, ⦨, ⦩, ⦪, ⦫, ⦬, ⦭, ⦮, ⦯
-            # Other_ID_Start
-            c == 0x2118 || c == 0x212E || # ℘, ℮
-            (c >= 0x309B && c <= 0x309C)) # katakana-hiragana sound marks
-end
-
 function is_identifier_char(c::Char)
     c == EOF_CHAR && return false
     return Base.is_id_char(c)
@@ -145,55 +39,6 @@ function is_never_id_char(ch::Char)
     )
 end
 
-function peekchar(io::Base.GenericIOBuffer)
-    if !io.readable || io.ptr > io.size
-        return EOF_CHAR
-    end
-    ch, _ = readutf(io)
-    return ch
-end
-
-function readutf(io, offset = 0)
-    ch = convert(UInt8, io.data[io.ptr + offset])
-    if ch < 0x80
-        return convert(Char, ch), 0
-    end
-    trailing = utf8_trailing(ch + 1)
-    c::UInt32 = 0
-    for j = 1:trailing
-        c += ch
-        c <<= 6
-        ch = convert(UInt8, io.data[io.ptr + j + offset])
-    end
-    c += ch
-    c -= utf8_offset[trailing + 1]
-    return convert(Char, c), trailing
-end
-
-function dpeekchar(io::IOBuffer)
-    if !io.readable || io.ptr > io.size
-        return EOF_CHAR, EOF_CHAR
-    end
-    ch1, trailing = readutf(io)
-    offset = trailing + 1
-
-    if io.ptr + offset > io.size
-        return ch1, EOF_CHAR
-    end
-    ch2, _ = readutf(io, offset)
-
-    return ch1, ch2
-end
-
-# this implementation is copied from Base
-peekchar(s::IOStream) = begin
-    _CHTMP = Ref{Char}()
-    if ccall(:ios_peekutf8, Int32, (Ptr{Nothing}, Ptr{Char}), s, _CHTMP) < 0
-        return EOF_CHAR
-    end
-    return _CHTMP[]
-end
-
 eof(io::IO) = Base.eof(io)
 eof(c::Char) = c === EOF_CHAR