Add pip install --editable functionality (#101)

* WIP: `pip --editable` support

* WIP: first complete try at `pip --editable` support

* WIP: Add tests to `pip install --editable` support

* WIP: small `pip install --editable` tweaks to `README.md`

* Bump version to v0.2.19

* Revert `specstr(::PipPkgSpec)` to old short forum

* Make local tests more Windows-friendly... maybe

* Handle `file://` URLs with local installs

* Add `editable` field to `CondaPkg.status` output

* Remove commented-out test

* less strict merging of editable flag

* always include editable flag in pixispec if it's set

* reuse existing tests for editable test

* remove unneeded test modules

* typo

* null backend doesn't install so no point checking editability

* keep readme brief

[skip ci]

* update changelog

[skip ci]

---------

Co-authored-by: Christopher Doris <github.com/cjdoris>

Author: dingraha

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## Unreleased
+* Add `editable` property to pip packages, specifying whether to install them in
+  editable mode.
+
 ## 0.2.31 (2025-08-28)
 * Bug fix: pip packages specified by file location are now correctly converted to "path"
   installs with Pixi.

README.md

@@ -57,7 +57,7 @@ These functions are intended to be used interactively when the Pkg REPL is not a
 - `rm(pkg)` removes a dependency or a vector of dependencies.
 - `add_channel(channel)` adds a channel.
 - `rm_channel(channel)` removes a channel.
-- `add_pip(pkg; version="")` adds/replaces a pip dependency.
+- `add_pip(pkg; version="", binary="", editable=false)` adds/replaces a pip dependency.
 - `rm_pip(pkg)` removes a pip dependency.
 
 ### CondaPkg.toml
@@ -90,6 +90,7 @@ some-local-package = "@ ./foo.zip"
 version = "~=2.1"
 extras = ["email", "timezone"]
 binary = "no"  # or "only"
+editable = true
 ```
 
 ## Access the Conda environment

src/PkgREPL.jl

@@ -26,7 +26,7 @@ $
     CondaPkg.PkgSpec(name, version = version, channel = channel, build = build)
 end
 
-function parse_pip_pkg(x::String; binary::String = "")
+function parse_pip_pkg(x::String; binary::String = "", editable=false)
     m = match(
         r"""
 ^
@@ -41,7 +41,7 @@ $
     name = m.captures[1]
     extras = split(something(m.captures[3], ""), ",", keepempty = false)
     version = something(m.captures[4], "")
-    CondaPkg.PipPkgSpec(name, version = version, binary = binary, extras = extras)
+    CondaPkg.PipPkgSpec(name, version = version, binary = binary, extras = extras, editable = editable)
 end
 
 function parse_channel(x::String)
@@ -62,6 +62,13 @@ const binary_opt = Pkg.REPLMode.OptionDeclaration([
     :api => :binary => identity,
 ])
 
+const editable_opt = Pkg.REPLMode.OptionDeclaration([
+    :name => "editable",
+    :short_name => "e",
+    :takes_arg => false,
+    :api => :editable => true,
+])
+
 ### status
 
 function status()
@@ -191,13 +198,13 @@ const channel_add_spec = Pkg.REPLMode.CommandSpec(
 
 ### pip_add
 
-function pip_add(args; binary = "")
-    CondaPkg.add([parse_pip_pkg(arg, binary = binary) for arg in args])
+function pip_add(args; binary = "", editable = false)
+    CondaPkg.add([parse_pip_pkg(arg, binary = binary, editable = editable) for arg in args])
 end
 
 const pip_add_help = Markdown.parse("""
 ```
-conda pip_add [--binary={only|no}] pkg ...
+conda pip_add [--binary={only|no}] [--editable] pkg ...
 ```
 
 Add Pip packages to the environment.
@@ -209,6 +216,7 @@ pkg> conda pip_add build
 pkg> conda pip_add build~=0.7                # version range
 pkg> conda pip_add pydantic[email,timezone]  # extras
 pkg> conda pip_add --binary=no nmslib        # always build from source
+pkg> conda pip_add --editable nmslib@/path/to/package_source  # install locally
 ```
 """)
 
@@ -219,7 +227,7 @@ const pip_add_spec = Pkg.REPLMode.CommandSpec(
     help = pip_add_help,
     description = "add Pip packages",
     arg_count = 0 => Inf,
-    option_spec = [binary_opt],
+    option_spec = [binary_opt, editable_opt],
 )
 
 ### rm

src/deps.jl

@@ -80,6 +80,7 @@ function parse_deps(toml)
                 version = ""
                 binary = ""
                 extras = String[]
+                editable = false
                 if dep isa AbstractString
                     version = _convert(String, dep)
                 elseif dep isa AbstractDict
@@ -90,16 +91,18 @@ function parse_deps(toml)
                             binary = _convert(String, v)
                         elseif k == "extras"
                             extras = _convert(Vector{String}, v)
+                        elseif k == "editable"
+                            editable = _convert(Bool, v)
                         else
                             error(
-                                "pip.deps keys must be 'version', 'extras' or 'binary', got '$k'",
+                                "pip.deps keys must be 'version', 'extras', 'binary' or 'editable', got '$k'",
                             )
                         end
                     end
                 else
                     error("pip.deps must be String or Dict, got $(typeof(dep))")
                 end
-                pkg = PipPkgSpec(name, version = version, binary = binary, extras = extras)
+                pkg = PipPkgSpec(name, version = version, binary = binary, extras = extras, editable = editable)
                 push!(pip_packages, pkg)
             end
         end
@@ -231,6 +234,9 @@ function status(; io::IO = stderr)
             if !isempty(pkg.extras)
                 push!(specparts, "[$(join(pkg.extras, ", "))]")
             end
+            if pkg.editable
+                push!(specparts, "editable")
+            end
             isempty(specparts) ||
                 printstyled(io, " (", join(specparts, ", "), ")", color = :light_black)
             println(io)
@@ -340,6 +346,9 @@ function add!(toml, pkg::PipPkgSpec)
     if !isempty(pkg.extras)
         dep["extras"] = pkg.extras
     end
+    if pkg.editable
+        dep["editable"] = pkg.editable
+    end
     if issubset(keys(dep), ["version"])
         deps[pkg.name] = pkg.version
     else
@@ -405,7 +414,7 @@ Removes a channel from the current environment.
 rm_channel(channel::AbstractString; kw...) = rm(ChannelSpec(channel); kw...)
 
 """
-    add_pip(pkg; version="", binary="", extras=[], resolve=true)
+    add_pip(pkg; version="", binary="", extras=[], resolve=true, editable=false)
 
 Adds a pip dependency to the current environment.
 
@@ -414,8 +423,8 @@ Adds a pip dependency to the current environment.
     Use conda dependencies instead if at all possible. Pip does not handle version
     conflicts gracefully, so it is possible to get incompatible versions.
 """
-add_pip(pkg::AbstractString; version = "", binary = "", extras = String[], kw...) =
-    add(PipPkgSpec(pkg, version = version, binary = binary, extras = extras); kw...)
+add_pip(pkg::AbstractString; version = "", binary = "", extras = String[], editable = false, kw...) =
+    add(PipPkgSpec(pkg, version = version, binary = binary, extras = extras, editable = editable); kw...)
 
 """
     rm_pip(pkg; resolve=true)

src/meta.jl

@@ -4,7 +4,7 @@ information about the most recent resolve.
 """
 
 # increment whenever the metadata format changes
-const META_VERSION = 14
+const META_VERSION = 15
 
 @kwdef mutable struct Meta
     timestamp::Float64
@@ -48,6 +48,9 @@ end
 function read_meta(io::IO, ::Type{Symbol})
     Symbol(read_meta(io, String))
 end
+function read_meta(io::IO, ::Type{Bool})
+    read(io, Bool)
+end
 function read_meta(io::IO, ::Type{Vector{T}}) where {T}
     len = read(io, Int)
     ans = Vector{T}()
@@ -76,7 +79,8 @@ function read_meta(io::IO, ::Type{PipPkgSpec})
     version = read_meta(io, String)
     binary = read_meta(io, String)
     extras = read_meta(io, Vector{String})
-    PipPkgSpec(name, version = version, binary = binary, extras = extras)
+    editable = read_meta(io, Bool)
+    PipPkgSpec(name, version = version, binary = binary, extras = extras, editable = editable)
 end
 
 function write_meta(io::IO, meta::Meta)
@@ -102,6 +106,9 @@ end
 function write_meta(io::IO, x::Symbol)
     write_meta(io, String(x))
 end
+function write_meta(io::IO, x::Bool)
+    write(io, x)
+end
 function write_meta(io::IO, x::Vector)
     write(io, convert(Int, length(x)))
     for item in x
@@ -125,4 +132,5 @@ function write_meta(io::IO, x::PipPkgSpec)
     write_meta(io, x.version)
     write_meta(io, x.binary)
     write_meta(io, x.extras)
+    write_meta(io, x.editable)
 end

src/resolve.jl

@@ -518,6 +518,7 @@ function _resolve_merge_pip_packages(packages)
         urls = String[]
         binary = ""
         extras = String[]
+        editable = false
         for (fn, pkg) in pkgs
             @assert pkg.name == name
             if startswith(pkg.version, "@")
@@ -539,6 +540,7 @@ function _resolve_merge_pip_packages(packages)
                 end
             end
             append!(extras, pkg.extras)
+            editable |= pkg.editable
         end
         sort!(unique!(urls))
         sort!(unique!(versions))
@@ -555,7 +557,16 @@ function _resolve_merge_pip_packages(packages)
                 "direct references ('@ ...') and version specifiers both given for pip package '$name'",
             )
         end
-        push!(specs, PipPkgSpec(name, version = version, binary = binary, extras = extras))
+        push!(
+            specs,
+            PipPkgSpec(
+                name,
+                version = version,
+                binary = binary,
+                extras = extras,
+                editable = editable,
+            ),
+        )
     end
     sort!(specs, by = x -> x.name)
 end
@@ -669,9 +680,16 @@ function _resolve_pip_install(io, pip_specs, load_path, backend)
         elseif spec.binary == "no"
             push!(args, "--no-binary", spec.name)
         end
+        if spec.editable
+            # remove the @ from the beginning of the path.
+            url = replace(spec.version, r"@\s*" => "")
+            push!(args, "--editable", url)
+        end
     end
     for spec in pip_specs
-        push!(args, specstr(spec))
+        if !spec.editable
+            push!(args, specstr(spec))
+        end
     end
     vrb = _verbosity_flags()
     flags = vrb

src/spec.jl

@@ -133,22 +133,31 @@ struct PipPkgSpec
     version::String
     binary::String
     extras::Vector{String}
-    function PipPkgSpec(name; version = "", binary = "", extras = String[])
+    editable::Bool
+    function PipPkgSpec(
+        name;
+        version = "",
+        binary = "",
+        extras = String[],
+        editable = false,
+    )
         name = validate_pip_pkg(name)
         version = validate_pip_version(version)
         binary = validate_pip_binary(binary)
         extras = validate_pip_extras(extras)
-        new(name, version, binary, extras)
+        validate_pip_editable(editable, version)
+        new(name, version, binary, extras, editable)
     end
 end
 
 Base.:(==)(x::PipPkgSpec, y::PipPkgSpec) =
     (x.name == y.name) &&
     (x.version == y.version) &&
     (x.binary == y.binary) &&
-    (x.extras == y.extras)
+    (x.extras == y.extras) &&
+    (x.editable == y.editable)
 Base.hash(x::PipPkgSpec, h::UInt) =
-    hash(x.extras, hash(x.binary, hash(x.version, hash(x.name, h))))
+    hash(x.editable, hash(x.extras, hash(x.binary, hash(x.version, hash(x.name, h)))))
 
 is_valid_pip_pkg(name) = occursin(r"^\s*[-_.A-Za-z0-9]+\s*$", name)
 
@@ -233,6 +242,9 @@ function pixispec(x::PipPkgSpec)
     else
         spec["version"] = x.version == "" ? "*" : x.version
     end
+    if x.editable
+        spec["editable"] = true
+    end
     if !isempty(x.extras)
         spec["extras"] = x.extras
     end
@@ -242,3 +254,10 @@ function pixispec(x::PipPkgSpec)
         spec
     end
 end
+
+validate_pip_editable(editable, version) =
+    if editable && !startswith(version, "@")
+        error(
+            "invalid pip version for editable install: must start with `@` but version is $(version)",
+        )
+    end

test/internals.jl

@@ -25,6 +25,8 @@ end
     @test_throws Exception CondaPkg.PipPkgSpec("")
     @test_throws Exception CondaPkg.PipPkgSpec("foo!")
     @test_throws Exception CondaPkg.PipPkgSpec("foo", version = "1.2")
+    @test_throws Exception CondaPkg.PipPkgSpec("foo", editable = true)
+    @test_throws Exception CondaPkg.PipPkgSpec("foo", version = "1.2", editable = true)
     spec = CondaPkg.PipPkgSpec("  F...OO_-0  ", version = "  @./SOME/Path  ")
     @test spec.name == "f-oo-0"
     @test spec.version == "@./SOME/Path"
@@ -36,6 +38,8 @@ end
         CondaPkg.PkgSpec("foo", version = "=1.2.3", channel = "bar"),
         CondaPkg.ChannelSpec("fooo"),
         CondaPkg.PipPkgSpec("foooo", version = "==2.3.4"),
+        CondaPkg.PipPkgSpec("bar", version = "@ /abs/path/somewhere", editable = true),
+        CondaPkg.PipPkgSpec("baz", version = "@ ./rel/path/somewhere", editable = true)
     ]
     for spec in specs
         io = IOBuffer()

test/main.jl

@@ -164,10 +164,11 @@ end
 end
 
 @testitem "pip install/remove local python package" begin
-    @testset "file $file" for file in [
-        "example-python-package",
-        "example_python_package-1.0.0-py3-none-any.whl",
-        "example_python_package-1.0.0.tar.gz",
+    @testset "file $file $kwargs" for (file, kwargs) in [
+        ("example-python-package", NamedTuple()),
+        ("example_python_package-1.0.0-py3-none-any.whl", NamedTuple()),
+        ("example_python_package-1.0.0.tar.gz", NamedTuple()),
+        ("example-python-package", (editable = true,)),
     ]
         include("setup.jl")
         CondaPkg.add("python", version = "==3.10.2")
@@ -180,14 +181,47 @@ end
 
         # install package
         path = "./test/data/$file"
-        CondaPkg.add_pip("example-python-package", version = "@$path")
+        fullpath = abspath(dirname(CondaPkg.cur_deps_file()), path)
+        @assert ispath(fullpath)
+        editable = get(kwargs, :editable, false)
+        CondaPkg.add_pip("example-python-package", version = "@$path"; kwargs...)
         @test occursin("example-python-package", status())
-        @test occursin("(@$path)", status())
+        if isempty(kwargs)
+            @test occursin("(@$path)", status())
+        else
+            @test occursin("(@$path,", status())
+        end
+        @test occursin("editable", status()) == editable
         CondaPkg.withenv() do
             isnull || run(`python -c "import example_python_package"`)
         end
         @test occursin("v1.0.0", status()) == !isnull
 
+        # check editability
+        if editable && !isnull
+            @assert isdir(fullpath)
+            added_path = joinpath(fullpath, "src", "example_python_package", "added.py")
+            # check a particular submodule does not exist
+            rm(added_path; force = true)
+            CondaPkg.withenv() do
+                @test_throws Exception run(
+                    `python -c "from example_python_package.added import foo"`,
+                )
+            end
+            # now add it and check we can import it
+            write(added_path, "foo = 12")
+            CondaPkg.withenv() do
+                run(`python -c "from example_python_package.added import foo"`)
+            end
+            # remove it again
+            rm(added_path; force = true)
+            CondaPkg.withenv() do
+                @test_throws Exception run(
+                    `python -c "from example_python_package.added import foo"`,
+                )
+            end
+        end
+
         # remove package
         CondaPkg.rm_pip("example-python-package")
         @test !occursin("example-python-package", status())