Merge pull request #28 from JunoLab/julia-0.7

WIP: Julia 0.7

Author: pfitzseb

REQUIRE

@@ -1,4 +1,4 @@
-julia 0.6
+julia 0.7-beta
 Lazy 0.6
 LNR
 Tokenize 0.1

pkg.yml

@@ -4,12 +4,15 @@ module CodeTools
 
 using Lazy, LNR
 
-const AString = AbstractString
-
 include("utils.jl")
 include("eval.jl")
 include("module.jl")
 include("summaries.jl")
+
+# # Summaries shims
+# signature(_) = nothing
+# description(_) = nothing
+
 include("completions.jl")
 include("doc.jl")
 

src/CodeTools.jl

@@ -19,13 +19,15 @@ const builtin_completions =
 
 const identifier_pattern = r"^@?[_\p{L}][\p{Xwd}!]*+$"
 
+_names(mod; all = false, imported = false) = filter!(x -> !Base.isdeprecated(mod, Symbol(x)), names(mod, all=all, imported=imported))
+
 moduleusings(mod) = ccall(:jl_module_usings, Any, (Any,), mod)
 
-filtervalid(names) = @>> names map(string) filter(x->ismatch(identifier_pattern, x))
+filtervalid(names) = @>> names map(string) filter(x->occursin(identifier_pattern, x))
 
 accessible(mod::Module) =
-  [names(mod, true, true);
-   map(names, moduleusings(mod))...] |> unique |> filtervalid
+  [_names(mod, all=true, imported=true);
+   map(_names, moduleusings(mod))...] |> unique |> filtervalid
 
 Base.getindex(b::Binding) = isdefined(b.mod, b.var) ? getfield(b.mod, b.var) : nothing
 
@@ -37,7 +39,7 @@ completiontype(x) =
 
 const meta_cache = Dict{Any,Any}()
 
-function withmeta(completion::AString, mod::Module)
+function withmeta(completion::AbstractString, mod::Module)
   isdefined(mod, Symbol(completion)) || return completion
   b = Binding(mod, Symbol(completion))
   mod = b.mod
@@ -51,7 +53,7 @@ function withmeta(completion::AString, mod::Module)
   c
 end
 
-for name in map(string, names(Base))
+for name in map(string, _names(Base))
   meta_cache[(Base, name)] = withmeta(name, Base)
 end
 
@@ -60,9 +62,9 @@ withmeta(completions::Vector, mod::Module) =
 
 function namecompletions_(mod::Module, qualified = false)
   if !qualified
-    [withmeta(filter!(x->!Base.isdeprecated(mod, Symbol(x)), accessible(mod)), mod); builtin_completions]
+    [withmeta(accessible(mod), mod); builtin_completions]
   else
-    withmeta(filter!(x->!Base.isdeprecated(mod, Symbol(x)), filtervalid(names(mod, true))), mod)
+    withmeta(filtervalid(_names(mod, all=true)), mod)
   end
 end
 
@@ -92,11 +94,11 @@ function pathprefix(line)
 end
 
 function stringmeta(cs, prefix)
-  map(c -> d(:text => c, :_prefix => prefix, :type => :file), cs)
+  map(c -> Dict(:text => c, :_prefix => prefix, :type => :file), cs)
 end
 
 function pathmeta(cs, path, prefix)
-  stringmeta(map(c -> replace(joinpath(path, c), r"^\./", ""), cs), prefix)
+  stringmeta(map(c -> replace(joinpath(path, c), r"^\./" => ""), cs), prefix)
 end
 
 function children(dir, ext = ""; depth = 0, out = String[], prefix = "")
@@ -140,37 +142,29 @@ function pathcompletions(line)
     elseif func == "open"
       pathmeta(children(joinpath(pwd(), dir)), dir, path)
     end
+  catch e
+    []
   end
 end
 
 # Package Completions
 # –––––––––––––––––––
+# not sure why this errors with "ArgumentError: Module Pkg3 not found in current path."
 
-packages(dir = Pkg.dir()) =
-  @>> dir readdir filter(x->!ismatch(r"^\.|^METADATA$|^REQUIRE$", x))
-
-all_packages() = packages(Pkg.dir("METADATA"))
+import Pkg
 
-required_packages() =
-  @>> Pkg.dir("REQUIRE") readstring lines
+pkgmeta(xs) = [Dict(:text => x, :type => "package") for x in xs]
 
-unused_packages() = setdiff(all_packages(), required_packages())
-
-pkgmeta(xs) = map(x -> d(:text=>x, :type=>"package"), xs)
+function stdlibs()
+  srcdir = joinpath(Sys.BINDIR,"..","..","stdlib","v0.7")
+  releasedir = joinpath(Sys.BINDIR,"..","share","julia","stdlib","v0.7")
+  return ispath(srcdir) ? readdir(srcdir) :
+                          ispath(releasedir) ? readdir(releasedir) : []
+end
 
 function pkgcompletions(line)
-  if ismatch(r"^using", line)
-    return pkgmeta(packages())
-  end
-  m = funcprefix(line)
-  m == nothing && return
-  func, _ = m
-  if func in ["Pkg.add", "Pkg.clone", "Pkg.build", "Pkg.test"]
-    pkgmeta(all_packages())
-  elseif func in ["Pkg.pin", "Pkg.checkout"]
-    pkgmeta(packages())
-  elseif func in ["Pkg.rm"]
-    pkgmeta(required_packages())
+  if occursin(r"^using|^import", line)
+    return pkgmeta(sort!(append!(collect(keys(Pkg.API.installed())), stdlibs())))
   end
 end
 
@@ -179,7 +173,7 @@ end
 
 const providers = [pkgcompletions, pathcompletions]
 
-function completions(line::AString, mod::Module = Main; default = true)
+function completions(line::AbstractString, mod::Module = Main; default = true)
   for provider in providers
     cs = provider(line)
     cs ≠ nothing && return cs

src/completions.jl

@@ -39,6 +39,6 @@ function methodsorwith(code, cursor, mod = Main)
   thing = thingorfunc(code, cursor, mod)
   thing == nothing && return
   return thing == Module ? methodswith(Module) :
-         (isa(thing, Function) && isgeneric(thing)) || isleaftype(thing) ? methods(thing) :
+         (isa(thing, Function) && isgeneric(thing)) || isconcretetype(thing) ? methods(thing) :
          eval(Main, :(methodswith($(typeof(thing)), true))) # I have no idea why I thought this was necessary
 end

src/doc.jl

@@ -16,29 +16,70 @@ end
 getthing(mod::Module, name::AbstractString, default = nothing) =
   name == "" ?
     default :
-    @as x name split(x, ".", keep=false) map(Symbol, x) getthing(mod, x, default)
+    @as x name split(x, ".", keepempty=false) map(Symbol, x) getthing(mod, x, default)
 
-getthing(mod::Module, ::Void, default) = default
+getthing(mod::Module, ::Nothing, default) = default
 
 getthing(mod::Module, x, default) = error("can't get $x from a module")
 
 getthing(args...) = getthing(Main, args...)
 
 # include_string with line numbers
 
-function Base.include_string(s::AbstractString, fname::AbstractString, line::Integer)
-  include_string("\n"^(line-1)*s, fname)
+function Base.include_string(mod, s::AbstractString, fname::AbstractString, line::Integer)
+  include_string(mod, "\n"^(line-1)*s, fname)
 end
 
-function Base.include_string(mod::Module, args...)
-  eval(mod, :(include_string($(args...))))
+function getmodule(parent::Union{Nothing, Module}, mod::String)
+  parent ≠ nothing && (mod = string(parent, ".", mod))
+  getmodule(mod)
+end
+
+function getmodule(mod::String)
+  mods = split(mod, '.')
+  if length(mods) == 1
+    getpackage(mod)
+  else
+    getsubmod(mods)
+  end
+end
+
+function getpackage(mod)
+  inds = filter(x -> x.name==mod, collect(keys(Base.loaded_modules)))
+  if length(inds) == 1
+    return get(Base.loaded_modules, first(inds), Main)
+  elseif length(inds) == 0
+    return nothing
+  else
+    @warn "no support for multiple packages with the same name yet"
+    return get(Base.loaded_modules, first(inds), Main)
+  end
+end
+
+function getsubmod(mods)
+  mod = getpackage(popfirst!(mods))
+  mod == nothing && (return nothing)
+  for submod in mods
+    submod = Symbol(submod)
+    if isdefined(mod, submod) && !Base.isdeprecated(mod, submod)
+      s = getfield(mod, submod)
+      if s isa Module
+        mod = s
+      else
+        return nothing
+      end
+    else
+      return nothing
+    end
+  end
+  mod
 end
 
 # Get the current module for a file/pos
 
-function getmodule(code, pos; filemod = nothing)
+function getmodule(code::AbstractString, pos; filemod = nothing)
   codem = codemodule(code, pos)
   modstr = (codem != "" && filemod != nothing) ? "$filemod.$codem" :
            codem == "" ? filemod : codem
-  getthing(modstr, Main)
+  getmodule(modstr)
 end

src/eval.jl

@@ -31,10 +31,10 @@ function jl_files(set)
   return files
 end
 
+# Recursion + Mutable State = Job Security
 """
 Takes a start directory and returns a set of nearby directories.
 """
-# Recursion + Mutable State = Job Security
 function dirsnearby(dir; descend = 1, ascend = 1, set = Set{String}())
   push!(set, dir)
   if descend > 0
@@ -125,20 +125,32 @@ Takes a given Julia source file and another (absolute) path, gives the
 line on which the path is included in the file or 0.
 """
 function includeline(file::AbstractString, included_file::AbstractString)
-  # check for self-includes
+  # check for erroneous self includes, doesn't detect more complex cycles though
   file == included_file && return 0
 
-  i = 0
-  open(file) do io
-    for (index, line) in enumerate(eachline(io))
-      m = match(r"^[^#]*\binclude\(\"([a-zA-Z_\.\\/]*)\"\)", line)
-      if m != nothing && normpath(joinpath(dirname(file), m.captures[1])) == included_file
-        i = index
-        break
+  line = 1
+  tokens = Tokenize.tokenize(read(file, String))
+
+  t, state = iterate(tokens)
+  while true
+    if Tokens.kind(t) == Tokens.WHITESPACE
+      line += count(x -> x == '\n', t.val)
+    elseif Tokens.kind(t) == Tokens.IDENTIFIER && t.val == "include"
+      t, state = iterate(tokens, state)
+      if Tokens.kind(t) == Tokens.LPAREN
+        t, state = iterate(tokens, state)
+        if Tokens.kind(t) == Tokens.STRING
+          if normpath(joinpath(dirname(file), chop(t.val, head=1, tail=1))) == included_file
+            return line
+          end
+        end
       end
+    elseif Tokens.kind(t) == Tokens.ENDMARKER
+      break
     end
+    t, state = iterate(tokens, state)
   end
-  return i
+  return 0
 end
 
 """
@@ -160,7 +172,7 @@ function filemodule_(path::AbstractString)
   loc = find_include(path)
   if loc != nothing
     file, line = loc
-    mod = codemodule(readstring(file), line)
+    mod = codemodule(read(file, String), line)
     super = filemodule(file)
     if super != "" && mod != ""
       return "$super.$mod"
@@ -174,9 +186,9 @@ end
 const filemodule = memoize(filemodule_)
 
 # Get all modules
-
-children(m::Module) =
-  @>> names(m, true) map(x->getthing(m, [x])) filter(x->isa(x, Module) && x ≠ m)
+function children(m::Module)
+  return @>> [moduleusings(m); getmodule.(Ref(m), string.(_names(m, all=true, imported=true)))] filter(x->isa(x, Module) && x ≠ m) unique
+end
 
 function allchildren(m::Module, cs = Set{Module}())
   for c in children(m)

src/module.jl

@@ -1,5 +1,5 @@
 import Base.Docs: Binding, @var
-import Base.Markdown: MD, Code, Paragraph
+using Markdown: MD, Code, Paragraph, plain
 
 # flat_content(md) = md
 # flat_content(xs::Vector) = reduce((xs, x) -> vcat(xs,flat_content(x)), [], xs)
@@ -31,6 +31,8 @@ function hasdoc(b::Binding)
     false
 end
 
+hasdoc(b) = hasdoc(Docs.aliasof(b, typeof(b)))
+
 function trygetdoc(b)
   docs = try
     Docs.doc(b)
@@ -39,8 +41,6 @@ function trygetdoc(b)
   end
 end
 
-hasdoc(b) = hasdoc(Docs.aliasof(b, typeof(b)))
-
 function fullsignature(b::Binding)
   hasdoc(b) || return
   docs = trygetdoc(b)
@@ -60,14 +60,14 @@ end
 function signature(b::Binding)
   sig = fullsignature(b)
   sig == nothing && return
-  replace(sig, r" -> .*$", "")
+  replace(sig, r" -> .*$" => "")
 end
 
 function returns(b::Binding)
   r = r" -> (.*)"
   sig = fullsignature(b)
   sig == nothing && return
-  if ismatch(r, sig)
+  if occursin(r, sig)
     ret = match(r, sig).captures[1]
     if length(ret) < 10
       ret
@@ -87,9 +87,9 @@ function description(b::Binding)
     first = md[2]
   end
   if isa(first, Paragraph)
-    desc = Markdown.plain(first)
+    desc = plain(first)
     return length(desc) > 100 ?
-      desc[1:100]*"..." :
+      desc[1:nextind(desc, 0, 99)]*"..." :
       desc
   end
 end