--outdated flag to status (#2284)

Allows filtering the output of `status` to only show packages that are "outdated" (not on their latest version).
This also shows what packages (or project compat) that are holding packages back from updating to their latest version.

Author: KristofferC

CHANGELOG.md

@@ -1,3 +1,9 @@
+Pkg v1.8 Release Notes
+======================
+
+- New `outdated::Bool` kwarg to `Pkg.status` (`--outdated` or `-o` in the REPL mode) to show
+  information about packages not at the latest version.
+
 Pkg v1.7 Release Notes
 ======================
 

src/API.jl

@@ -1522,9 +1522,8 @@ end
 
 @deprecate status(mode::PackageMode) status(mode=mode)
 
-function status(ctx::Context, pkgs::Vector{PackageSpec}; diff::Bool=false, mode=PKGMODE_PROJECT,
-                io::IO=stdout, kwargs...)
-    Operations.status(ctx.env, pkgs, mode=mode, git_diff=diff, io=io)
+function status(ctx::Context, pkgs::Vector{PackageSpec}; diff::Bool=false, mode=PKGMODE_PROJECT, outdated::Bool=false, io::IO=stdout, kwargs...)
+    Operations.status(ctx.env, ctx.registries, pkgs; mode, git_diff=diff, io, outdated)
     return nothing
 end
 
@@ -1664,7 +1663,7 @@ function redo_undo(ctx, mode::Symbol, direction::Int)
     snapshot = state.entries[state.idx]
     ctx.env.manifest, ctx.env.project = snapshot.manifest, snapshot.project
     write_env(ctx.env; update_undo=false)
-    Operations.show_update(ctx.env; io=ctx.io)
+    Operations.show_update(ctx.env, ctx.registries; io=ctx.io)
 end
 
 

src/Operations.jl

@@ -1052,7 +1052,7 @@ function rm(ctx::Context, pkgs::Vector{PackageSpec}; mode::PackageMode)
     prune_manifest(ctx.env)
     # update project & manifest
     write_env(ctx.env)
-    show_update(ctx.env; io=ctx.io)
+    show_update(ctx.env, ctx.registries; io=ctx.io)
 end
 
 update_package_add(ctx::Context, pkg::PackageSpec, ::Nothing, is_dep::Bool) = pkg
@@ -1196,7 +1196,7 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}, new_git=Set{UUID}();
     download_artifacts(ctx.env, platform=platform, julia_version=ctx.julia_version, io=ctx.io)
 
     write_env(ctx.env) # write env before building
-    show_update(ctx.env; io=ctx.io)
+    show_update(ctx.env, ctx.registries; io=ctx.io)
     build_versions(ctx, union(new_apply, new_git))
 end
 
@@ -1214,7 +1214,7 @@ function develop(ctx::Context, pkgs::Vector{PackageSpec}, new_git::Set{UUID};
     new_apply = download_source(ctx)
     download_artifacts(ctx.env; platform=platform, julia_version=ctx.julia_version, io=ctx.io)
     write_env(ctx.env) # write env before building
-    show_update(ctx.env; io=ctx.io)
+    show_update(ctx.env, ctx.registries; io=ctx.io)
     build_versions(ctx, union(new_apply, new_git))
 end
 
@@ -1280,7 +1280,7 @@ function up(ctx::Context, pkgs::Vector{PackageSpec}, level::UpgradeLevel;
     new_apply = download_source(ctx)
     download_artifacts(ctx.env, julia_version=ctx.julia_version, io=ctx.io)
     write_env(ctx.env; skip_writing_project) # write env before building
-    show_update(ctx.env; io=ctx.io)
+    show_update(ctx.env, ctx.registries; io=ctx.io)
     build_versions(ctx, union(new_apply, new_git))
 end
 
@@ -1322,7 +1322,7 @@ function pin(ctx::Context, pkgs::Vector{PackageSpec})
     new = download_source(ctx)
     download_artifacts(ctx.env; julia_version=ctx.julia_version, io=ctx.io)
     write_env(ctx.env) # write env before building
-    show_update(ctx.env; io=ctx.io)
+    show_update(ctx.env, ctx.registries; io=ctx.io)
     build_versions(ctx, new)
 end
 
@@ -1360,12 +1360,12 @@ function free(ctx::Context, pkgs::Vector{PackageSpec})
         new = download_source(ctx)
         download_artifacts(ctx.env, io=ctx.io)
         write_env(ctx.env) # write env before building
-        show_update(ctx.env; io=ctx.io)
+        show_update(ctx.env, ctx.registries; io=ctx.io)
         build_versions(ctx, new)
     else
         foreach(pkg -> manifest_info(ctx.env.manifest, pkg.uuid).pinned = false, pkgs)
         write_env(ctx.env)
-        show_update(ctx.env; io=ctx.io)
+        show_update(ctx.env, ctx.registries; io=ctx.io)
     end
 end
 
@@ -1636,7 +1636,7 @@ function test(ctx::Context, pkgs::Vector{PackageSpec};
         sandbox(ctx, pkg, source_path, testdir(source_path), test_project_override; force_latest_compatible_version, allow_earlier_backwards_compatible_versions, allow_reresolve) do
             test_fn !== nothing && test_fn()
             sandbox_ctx = Context(;io=ctx.io)
-            status(sandbox_ctx.env; mode=PKGMODE_COMBINED, io=sandbox_ctx.io)
+            status(sandbox_ctx.env, sandbox_ctx.registries; mode=PKGMODE_COMBINED, io=sandbox_ctx.io)
             Pkg._auto_precompile(sandbox_ctx)
             printpkgstyle(ctx.io, :Testing, "Running tests...")
             flush(stdout)
@@ -1731,6 +1731,79 @@ function print_diff(io::IO, old::Union{Nothing,PackageSpec}, new::Union{Nothing,
     end
 end
 
+function status_compat_info(pkg::PackageSpec, env::EnvCache, regs::Vector{Registry.RegistryInstance})
+    manifest, project = env.manifest, env.project
+    packages_holding_back = String[]
+    max_version, max_version_in_compat = v"0", v"0"
+    for reg in regs
+        reg_pkg = get(reg, pkg.uuid, nothing)
+        reg_pkg === nothing && continue
+        info = Registry.registry_info(reg_pkg)
+        reg_compat_info = Registry.compat_info(info)
+        max_version_reg = maximum(keys(reg_compat_info); init=v"0")
+        max_version = max(max_version, max_version_reg)
+        compat_spec = get_compat(env.project, pkg.name)
+        versions_in_compat = filter(in(compat_spec), keys(reg_compat_info))
+        max_version_in_compat = max(max_version_in_compat, maximum(versions_in_compat; init=v"0"))
+    end
+    max_version == v"0" && return nothing
+    pkg.version == max_version && return nothing
+
+    # Check compat of project
+    if pkg.version == max_version_in_compat && max_version_in_compat != max_version
+        return ["compat"], max_version, max_version_in_compat
+    end
+
+    manifest_info = get(manifest, pkg.uuid, nothing)
+    manifest_info === nothing && return nothing
+
+    # Check compat of dependees
+    for (uuid, dep_pkg) in manifest
+        is_stdlib(uuid) && continue
+        if !(pkg.uuid in values(dep_pkg.deps))
+            continue
+        end
+        dep_info = get(manifest, uuid, nothing)
+        dep_info === nothing && continue
+        for reg in regs
+            reg_pkg = get(reg, uuid, nothing)
+            reg_pkg === nothing && continue
+            info = Registry.registry_info(reg_pkg)
+            reg_compat_info = Registry.compat_info(info)
+            compat_info_v = get(reg_compat_info, dep_info.version, nothing)
+            compat_info_v === nothing && continue
+            compat_info_v_uuid = compat_info_v[pkg.uuid]
+            if !(max_version in compat_info_v_uuid)
+                push!(packages_holding_back, dep_pkg.name)
+            end
+        end
+    end
+
+    # Check compat with Julia itself
+    julia_compatible_versions = Set{VersionNumber}()
+    for reg in regs
+        reg_pkg = get(reg, pkg.uuid, nothing)
+        reg_pkg === nothing && continue
+        info = Registry.registry_info(reg_pkg)
+        reg_compat_info = Registry.compat_info(info)
+        compat_info_v = get(reg_compat_info, pkg.version, nothing)
+        versions = keys(reg_compat_info)
+        for v in versions
+            compat_info_v = get(reg_compat_info, v, nothing)
+            compat_info_v === nothing && continue
+            compat_info_v_uuid = compat_info_v[JULIA_UUID]
+            if VERSION in compat_info_v_uuid
+                push!(julia_compatible_versions, v)
+            end
+        end
+    end
+    if !(max_version in julia_compatible_versions)
+        push!(packages_holding_back, "julia")
+    end
+
+    return sort!(unique!(packages_holding_back)), max_version, max_version_in_compat
+end
+
 function diff_array(old_env::Union{EnvCache,Nothing}, new_env::EnvCache; manifest=true)
     function index_pkgs(pkgs, uuid)
         idx = findfirst(pkg -> pkg.uuid == uuid, pkgs)
@@ -1757,9 +1830,18 @@ function is_package_downloaded(project_file::String, pkg::PackageSpec)
     return true
 end
 
-function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, header::Symbol,
-                      uuids::Vector, names::Vector; manifest=true, diff=false, ignore_indent=false, io)
+struct PackageStatusData
+    uuid::UUID
+    old::Union{Nothing, PackageSpec}
+    new::Union{Nothing, PackageSpec}
+    downloaded::Bool
+    compat_data::Union{Nothing, Tuple{Vector{String}, VersionNumber, VersionNumber}}
+end
+
+function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, registries::Vector{Registry.RegistryInstance}, header::Symbol,
+                      uuids::Vector, names::Vector; manifest=true, diff=false, ignore_indent::Bool, outdated::Bool, io::IO)
     not_installed_indicator = sprint((io, args) -> printstyled(io, args...; color=:red), "→", context=io)
+    not_latest_version_indicator = sprint((io, args) -> printstyled(io, args...; color=:yellow), "↓", context=io)
     filter = !isempty(uuids) || !isempty(names)
     # setup
     xs = diff_array(old_env, env; manifest=manifest)
@@ -1785,20 +1867,62 @@ function print_status(env::EnvCache, old_env::Union{Nothing,EnvCache}, header::S
     # Sort stdlibs and _jlls towards the end in status output
     xs = sort!(xs, by = (x -> (is_stdlib(x[1]), endswith(something(x[3], x[2]).name, "_jll"), something(x[3], x[2]).name, x[1])))
     all_packages_downloaded = true
+
+    package_statuses = PackageStatusData[]
     for (uuid, old, new) in xs
         if Types.is_project_uuid(env, uuid)
             continue
         end
+
+        latest_version = true
+        # Compat info
+        cinfo = nothing
+        if outdated
+            if diff == false && !is_stdlib(new.uuid)
+                @assert old == nothing
+                cinfo = status_compat_info(new, env, registries)
+                if cinfo !== nothing
+                    latest_version = false
+                end
+            end
+        end
+        # if we are running with compat, only show packages that are upper bounded
+        if outdated && latest_version
+            continue
+        end
+
         pkg_downloaded = !is_instantiated(new) || is_package_downloaded(env.project_file, new)
+
         all_packages_downloaded &= pkg_downloaded
-        print(io, pkg_downloaded ? " " : not_installed_indicator)
-        printstyled(io, " [", string(uuid)[1:8], "] "; color = :light_black)
-        diff ? print_diff(io, old, new) : print_single(io, new)
+        push!(package_statuses, PackageStatusData(uuid, old, new, pkg_downloaded, cinfo))
+    end
+
+    for pkg in package_statuses
+        latest_version = pkg.compat_data === nothing
+        print(io, pkg.downloaded ? (all_packages_downloaded ? "" : " ") : not_installed_indicator)
+        printstyled(io, " [", string(pkg.uuid)[1:8], "] "; color = :light_black)
+        diff ? print_diff(io, pkg.old, pkg.new) : print_single(io, pkg.new)
+        if outdated && !diff && pkg.compat_data !== nothing
+            packages_holding_back, max_version, max_version_compat = pkg.compat_data
+            if pkg.new.version !== max_version_compat && max_version_compat != max_version
+                printstyled(io, " [<v", max_version_compat, "]", color=:light_magenta)
+                printstyled(io, ",")
+            end
+            printstyled(io, " (<v", max_version, ")"; color=Base.warn_color())
+            if packages_holding_back == ["compat"]
+                printstyled(io, " [compat]"; color=:light_magenta)
+            else
+                pkg_str = isempty(packages_holding_back) ? "" : string(": ", join(packages_holding_back, ", "))
+                printstyled(io, pkg_str; color=Base.warn_color())
+            end
+        end
         println(io)
     end
+
     if !all_packages_downloaded
         printpkgstyle(io, :Info, "packages marked with $not_installed_indicator not downloaded, use `instantiate` to download", ignore_indent)
     end
+
     return nothing
 end
 
@@ -1819,20 +1943,20 @@ function git_head_env(env, project_dir)
     end
 end
 
-function show_update(env::EnvCache; ignore_indent=false, io::IO)
+function show_update(env::EnvCache, registries::Vector{Registry.RegistryInstance}; io::IO)
     old_env = EnvCache()
     old_env.project = env.original_project
     old_env.manifest = env.original_manifest
-    status(env; header=:Updating, mode=PKGMODE_COMBINED, env_diff=old_env, ignore_indent=ignore_indent, io=io)
+    status(env, registries; header=:Updating, mode=PKGMODE_COMBINED, env_diff=old_env, ignore_indent=false, io=io)
     return nothing
 end
 
-function status(env::EnvCache, pkgs::Vector{PackageSpec}=PackageSpec[];
-                header=nothing, mode::PackageMode=PKGMODE_PROJECT, git_diff::Bool=false, env_diff=nothing, ignore_indent=false, io::IO)
+function status(env::EnvCache, registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec}=PackageSpec[];
+                header=nothing, mode::PackageMode=PKGMODE_PROJECT, git_diff::Bool=false, env_diff=nothing, ignore_indent=true, io::IO, outdated::Bool=false)
     io == Base.devnull && return
     # if a package, print header
     if header === nothing && env.pkg !== nothing
-       printpkgstyle(io, :Project, string(env.pkg.name, " v", env.pkg.version); color=Base.info_color())
+       printpkgstyle(io, :Project, string(env.pkg.name, " v", env.pkg.version), true; color=Base.info_color())
     end
     # load old env
     old_env = nothing
@@ -1855,10 +1979,10 @@ function status(env::EnvCache, pkgs::Vector{PackageSpec}=PackageSpec[];
     diff = old_env !== nothing
     header = something(header, diff ? :Diff : :Status)
     if mode == PKGMODE_PROJECT || mode == PKGMODE_COMBINED
-        print_status(env, old_env, header, filter_uuids, filter_names; manifest=false, diff=diff, ignore_indent=ignore_indent, io=io)
+        print_status(env, old_env, registries, header, filter_uuids, filter_names; manifest=false, diff, ignore_indent, io, outdated)
     end
     if mode == PKGMODE_MANIFEST || mode == PKGMODE_COMBINED
-        print_status(env, old_env, header, filter_uuids, filter_names; diff=diff, ignore_indent=ignore_indent, io=io)
+        print_status(env, old_env, registries, header, filter_uuids, filter_names; diff, ignore_indent, io, outdated)
     end
 end
 

src/Pkg.jl

@@ -384,25 +384,47 @@ from packages that are tracking a path.
 const resolve = API.resolve
 
 """
-    Pkg.status([pkgs...]; mode::PackageMode=PKGMODE_PROJECT, diff::Bool=false, io::IO=stdout)
+    Pkg.status([pkgs...]; mode::PackageMode=PKGMODE_PROJECT, diff::Bool=false, compat::Bool=false, io::IO=stdout)
 
 Print out the status of the project/manifest.
 If `mode` is `PKGMODE_PROJECT`, print out status only about the packages
 that are in the project (explicitly added). If `mode` is `PKGMODE_MANIFEST`,
 print status also about those in the manifest (recursive dependencies). If there are
 any packages listed as arguments, the output will be limited to those packages.
+
 Setting `diff=true` will, if the environment is in a git repository, limit
 the output to the difference as compared to the last git commit.
 
+Setting `outdated=true` will only show packages that are not on the latest version,
+their maximum version and why they are not on the latest version (either due to other
+packages holding them back due to compatibility constraints, or due to compatibility in the project file).
+As an example, a status output like:
+```
+pkg> Pkg.status(; outdated=true)
+Status `Manifest.toml`
+ [a8cc5b0e] Crayons v2.0.0 [<v3.0.0], (<v4.0.4)
+ [b8a86587] NearestNeighbors v0.4.8 (<v0.4.9) [compat]
+ [2ab3a3ac] LogExpFunctions v0.2.5 (<v0.3.0): SpecialFunctions
+```
+means that the latest version of Crayons is 4.0.4 but the latest version compatible
+with the `[compat]` section in the current project is 3.0.0.
+The latest version of NearestNeighbors is 0.4.9 but due to compat constrains in the project
+it is held back to 0.4.8.
+The latest version of LogExpFunctions is 0.3.0 but SpecialFunctions
+is holding it back to 0.2.5.
+
 See [`Pkg.project`](@ref) and [`Pkg.dependencies`](@ref) to get the project/manifest
 status as a Julia object instead of printing it.
 
 !!! compat "Julia 1.1"
     `Pkg.status` with package arguments requires at least Julia 1.1.
 
 !!! compat "Julia 1.3"
-    The `diff` keyword argument requires Julia 1.3. In earlier versions `diff=true`
+    The `diff` keyword argument requires at least Julia 1.3. In earlier versions `diff=true`
     is the default for environments in git repositories.
+
+!!! compat "Julia 1.8"
+    The `outdated` keyword argument reguires at least Julia 1.8
 """
 const status = API.status
 

src/REPLMode/REPLMode.jl

@@ -147,7 +147,7 @@ wrap_option(option::String)  = length(option) == 1 ? "-$option" : "--$option"
 is_opt(word::AbstractString) = first(word) == '-' && word != "-"
 
 function parse_option(word::AbstractString)::Option
-    m = match(r"^(?: -([a-z]) | --([a-z]{2,})(?:\s*=\s*(\S*))? )$"ix, word)
+    m = match(r"^(?: -([a-z]) | --((?:[a-z]{1,}-?)*)(?:\s*=\s*(\S*))? )$"ix, word)
     m === nothing && pkgerror("malformed option: ", repr(word))
     option_name = m.captures[1] !== nothing ? m.captures[1] : m.captures[2]
     option_arg  = m.captures[3] === nothing ? nothing : String(m.captures[3])