Consolidate source of Tokenize.jl into main src dir

This will help integrate the two codebases a bit more consistently.
Tokenize is still a separate module for now.

Author: c42f

LICENSE.md

@@ -20,7 +20,7 @@ The JuliaSyntax.jl package is licensed under the MIT "Expat" License:
 > OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 > SOFTWARE.
 
-The code in src/Tokenize and test/lexer.jl is derived from the Tokenize.jl
+The code in src/tokenize*.jl and test/tokenize.jl is derived from the Tokenize.jl
 package and is also licensed under the MIT "Expat" License:
 
 > Copyright (c) 2016: Kristoffer Carlsson.

src/JuliaSyntax.jl

@@ -8,7 +8,7 @@ include("utils.jl")
 include("kinds.jl")
 
 # Lexing uses a significantly modified version of Tokenize.jl
-include("Tokenize/Tokenize.jl")
+include("tokenize.jl")
 using .Tokenize: Token
 
 # Source and diagnostics

src/Tokenize/Tokenize.jl

@@ -9,7 +9,7 @@ import ..JuliaSyntax: kind,
 
 import Base.eof
 
-include("utilities.jl")
+include("tokenize_utils.jl")
 
 #-------------------------------------------------------------------------------
 # Tokens

src/Tokenize/token.jl

@@ -8,7 +8,7 @@ using JuliaSyntax: GreenNode, SyntaxNode,
     children, child, setchild!, SyntaxHead
 
 @testset "Tokenize" begin
-    include("lexer.jl")
+    include("tokenize.jl")
 end
 
 include("test_utils.jl")