Merge pull request #259 from crystal-lang/feature/sat-solver

Solving dependencies now relies a SAT solver, capable to handle conflicts, and to choose the closest solution to the ideal one: update everything, update/add/remove some with minimal impact, ...

Author: ysbaddaden

man/shards.1

@@ -41,7 +41,7 @@ dependencies aren't satisfied.
 .PP
 \fBinit\fR
 .RS 4
-Initializes a shard folder and creates a default \fIshard.yml\fR.
+Initializes a default \fIshard.yml\fR in the current folder.
 .RE
 .PP
 \fBinstall\fR
@@ -56,37 +56,52 @@ a requirement, but may succeed if a new dependency was added, as long as it
 doesn't generate a conflict, thus generating a new \fIshard.lock\fR file.
 .RE
 .PP
-\fBlist\fR
+\fBlist [--tree]\fR
 .RS 4
-Lists the installed dependencies and their versions.
+Lists the installed dependencies and their versions. Specifying \fI--tree\fR
+will list nested dependencies in a tree manner, instead of a flattened list.
+.RE
+.PP
+\fBlock [--update [<shards>]]\fR
+.RS 4
+Resolves dependencies and creates or updates the \fIshard.lock\fR file as per
+the \fBinstall\fR command, but never installs the dependencies.
+.PP
+Specifying \fI--update\fR follows the same semantics as the \fBupdate\fR
+command.
 .RE
 .PP
 \fBprune\fR
 .RS 4
 Removes unused dependencies from \fIlib\fR folder.
 .RE
 .PP
-\fBupdate\fR
+\fBupdate [<shards>]\fR
 .RS 4
 Resolves and updates all dependencies into the \fIlib\fR folder again,
 whatever the locked versions and commits in the \fIshard.lock\fR file.
+.PP
+Specifying \fIshards\fR will update these dependencies only, trying to be as
+conservative as possible with other dependencies, respecting the locked versions
+and commits in the \fIshard.lock\fR file.
+.PP
 Eventually generates a new \fIshard.lock\fR file.
 .RE
 .PP
 \fBversion\fR [\fI<path>\fR]
 .RS 4
-Prints the current version of the shard.
+Print the current version of the shard located at \fIpath\fR.
 .RE
 .SH OPTIONS
 .PP
 \fB\-\-version\fR
 .RS 4
-Prints the \fIshards\fR version.
+Print the \fIshards\fR version.
 .RE
 .PP
 \fB\-h, \-\-help\fR
 .RS 4
-Prints usage synopsis.
+Print usage synopsis.
 .RE
 .PP
 \fB\-\-no-color\fR
@@ -103,12 +118,12 @@ locked dependencies will be installed. Commands will fail if dependencies in
 .PP
 \fB\-q, \-\-quiet\fR
 .RS 4
-Decreases the log verbosity, printing only warnings and errors.
+Decrease the log verbosity, printing only warnings and errors.
 .RE
 .PP
 \fB\-v, \-\-verbose\fR
 .RS 4
-Increases the log verbosity, printing all debug statements.
+Increase the log verbosity, printing all debug statements.
 .REAUTHOR
 Written by Julien Portalier.
 .SH "REPORTING BUGS"

src/cli.cr

@@ -7,15 +7,16 @@ module Shards
       shards [<options>...] [<command>]
 
       Commands:
-          build [<targets>] [<options>]  - Builds the specified <targets> in `bin` path.
-          check                          - Verifies all dependencies are installed.
-          init                           - Initializes a shard folder.
-          install                        - Installs dependencies from `shard.lock` file.
-          list [--tree]                  - Lists installed dependencies.
-          outdated [--pre]               - Lists dependencies that are outdated.
-          prune                          - Removes unused dependencies from `lib` folder.
-          update                         - Updates dependencies and `shards.lock`.
-          version [<path>]               - Prints the current version of the shard.
+          build [<targets>] [<options>]  - Build the specified <targets> in `bin` path.
+          check                          - Verify all dependencies are installed.
+          init                           - Initialize a `shard.yml` file.
+          install                        - Install dependencies, creating or using the `shard.lock` file.
+          list [--tree]                  - List installed dependencies.
+          lock [--update] [<shards>]     - Lock dependencies in `shard.lock` but doesn't install them.
+          outdated [--pre]               - List dependencies that are outdated.
+          prune                          - Remove unused dependencies from `lib` folder.
+          update [<shards>]              - Update dependencies and `shard.lock`.
+          version [<path>]               - Print the current version of the shard.
 
       Options:
       HELP
@@ -28,12 +29,12 @@ module Shards
       path = Dir.current
 
       opts.on("--no-color", "Disable colored output.") { self.colors = false }
-      opts.on("--version", "Prints the `shards` version.") { puts self.version_string; exit }
+      opts.on("--version", "Print the `shards` version.") { puts self.version_string; exit }
       opts.on("--production", "Run in release mode. No development dependencies and strict sync between shard.yml and shard.lock.") { self.production = true }
       opts.on("--local", "Don't update remote repositories, use the local cache only.") { self.local = true }
-      opts.on("-v", "--verbose", "Increases the log verbosity, printing all debug statements.") { self.logger.level = Logger::Severity::DEBUG }
-      opts.on("-q", "--quiet", "Decreases the log verbosity, printing only warnings and errors.") { self.logger.level = Logger::Severity::WARN }
-      opts.on("-h", "--help", "Prints usage synopsis.") { self.display_help_and_exit(opts) }
+      opts.on("-v", "--verbose", "Increase the log verbosity, printing all debug statements.") { self.logger.level = Logger::Severity::DEBUG }
+      opts.on("-q", "--quiet", "Decrease the log verbosity, printing only warnings and errors.") { self.logger.level = Logger::Severity::WARN }
+      opts.on("-h", "--help", "Print usage synopsis.") { self.display_help_and_exit(opts) }
 
       opts.unknown_args do |args, options|
         case args[0]? || DEFAULT_COMMAND
@@ -47,12 +48,22 @@ module Shards
           Commands::Install.run(path)
         when "list"
           Commands::List.run(path, tree: args.includes?("--tree"))
+        when "lock"
+          Commands::Lock.run(
+            path,
+            args[1..-1].reject(&.starts_with?("--")),
+            print: args.includes?("--print"),
+            update: args.includes?("--update")
+          )
         when "outdated"
           Commands::Outdated.run(path, prereleases: args.includes?("--pre"))
         when "prune"
           Commands::Prune.run(path)
         when "update"
-          Commands::Update.run(path)
+          Commands::Update.run(
+            path,
+            args[1..-1].reject(&.starts_with?("--"))
+          )
         when "version"
           Commands::Version.run(args[1]? || path)
         else

src/commands/build.cr

@@ -10,7 +10,7 @@ module Shards
         end
 
         if targets.empty?
-          targets = manager.spec.targets.map(&.name)
+          targets = spec.targets.map(&.name)
         end
 
         targets.each do |name|
@@ -23,15 +23,15 @@ module Shards
       end
 
       private def build(target, options)
-        Shards.logger.info "Building: #{target.name}"
+        Shards.logger.info { "Building: #{target.name}" }
 
         args = [
           "build",
           "-o", File.join(Shards.bin_path, target.name),
           target.main,
         ]
         options.each { |option| args << option }
-        Shards.logger.debug "crystal #{args.join(' ')}"
+        Shards.logger.debug { "crystal #{args.join(' ')}" }
 
         error = IO::Memory.new
         status = Process.run("crystal", args: args, output: Process::Redirect::Inherit, error: error)

src/commands/check.cr

@@ -4,7 +4,7 @@ require "../versions"
 module Shards
   module Commands
     class Check < Command
-      def run(*args)
+      def run
         if has_dependencies?
           locks # ensures that lockfile exists
           verify(spec.dependencies)

src/commands/command.cr

@@ -1,5 +1,4 @@
 require "../lock"
-require "../manager"
 require "../spec"
 
 module Shards
@@ -22,10 +21,10 @@ module Shards
       @lockfile_path = File.join(@path, LOCK_FILENAME)
     end
 
-    abstract def run(*args)
+    abstract def run(*args, **kwargs)
 
-    def self.run(path, *args)
-      new(path).run(*args)
+    def self.run(path, *args, **kwargs)
+      new(path).run(*args, **kwargs)
     end
 
     def spec
@@ -40,13 +39,9 @@ module Shards
       File.basename(spec_path)
     end
 
-    def manager
-      @manager ||= Manager.new(spec)
-    end
-
     def locks
       @locks ||= if lockfile?
-                   Lock.from_file(lockfile_path)
+                   Shards::Lock.from_file(lockfile_path)
                  else
                    raise Error.new("Missing #{LOCK_FILENAME}. Please run 'shards install'")
                  end
@@ -55,5 +50,10 @@ module Shards
     def lockfile?
       File.exists?(lockfile_path)
     end
+
+    def write_lockfile(packages)
+      Shards.logger.info { "Writing #{LOCK_FILENAME}" }
+      Shards::Lock.write(packages, LOCK_FILENAME)
+    end
   end
 end

src/commands/init.cr

@@ -4,7 +4,7 @@ require "ecr/macros"
 module Shards
   module Commands
     class Init < Command
-      def run(*args)
+      def run
         if File.exists?(shard_path)
           raise Error.new("#{SPEC_FILENAME} already exists")
         end

src/commands/install.cr

@@ -1,77 +1,100 @@
 require "./command"
+require "../solver"
 
 module Shards
   module Commands
-    # OPTIMIZE: avoid updating GIT caches until required
     class Install < Command
-      def run(*args)
+      def run
+        Shards.logger.info { "Resolving dependencies" }
+
+        solver = Solver.new(spec)
+
         if lockfile?
-          manager.locks = locks
-          manager.resolve
-          install(manager.packages, locks)
-        else
-          manager.resolve
-          install(manager.packages)
+          # install must be as conservative as possible:
+          solver.locks = locks
         end
 
-        if generate_lockfile?
-          manager.to_lock(lockfile_path)
+        solver.prepare(development: !Shards.production?)
+
+        if packages = solver.solve
+          return if packages.empty?
+
+          if lockfile?
+            validate(packages)
+          end
+
+          install(packages)
+
+          if generate_lockfile?(packages)
+            write_lockfile(packages)
+          end
+        else
+          solver.each_conflict do |message|
+            Shards.logger.warn { "Conflict #{message}" }
+          end
+          Shards.logger.error { "Failed to resolve dependencies" }
         end
       end
 
-      private def install(packages : Set, locks : Array(Dependency))
+      private def validate(packages)
         packages.each do |package|
-          version = nil
-
-          if lock = locks.find { |dependency| dependency.name == package.name }
-            if version = lock["version"]?
-              unless package.matching_versions.includes?(version)
-                raise LockConflict.new("#{package.name} requirements changed")
-              end
-            elsif version = lock["commit"]?
-              unless package.matches?(version)
-                raise LockConflict.new("#{package.name} requirements changed")
-              end
+          if lock = locks.find { |d| d.name == package.name }
+            if version = lock.version?
+              validate_locked_version(package, version)
+            elsif commit = lock["commit"]?
+              validate_locked_commit(package, commit)
             else
               raise InvalidLock.new # unknown lock resolver
             end
           elsif Shards.production?
             raise LockConflict.new("can't install new dependency #{package.name} in production")
           end
-
-          install(package, version)
         end
       end
 
-      private def install(packages : Set)
-        packages
-          .compact_map { |package| install(package) }
-          .each(&.postinstall)
+      private def validate_locked_version(package, version)
+        return if Shards.production? && package.version == version
+        return if Versions.matches?(version, package.spec.version)
+        raise LockConflict.new("#{package.name} requirements changed")
+      end
 
-        # always install executables because the path resolver never installs
-        # dependencies, but uses them as-is:
-        packages.each(&.install_executables)
+      private def validate_locked_commit(package, commit)
+        return if commit == package.commit
+        raise LockConflict.new("#{package.name} requirements changed")
       end
 
-      private def install(package : Package, version = nil)
-        version ||= package.version
+      private def install(packages : Array(Package))
+        # first install all dependencies:
+        installed = packages.compact_map { |package| install(package) }
+
+        # then execute the postinstall script of installed dependencies (with
+        # access to all transitive dependencies):
+        installed.each(&.postinstall)
+
+        # always install executables because the path resolver never actually
+        # installs dependencies:
+        packages.each(&.install_executables)
+      end
 
-        if package.installed?(version)
-          Shards.logger.info "Using #{package.name} (#{package.report_version})"
+      private def install(package : Package)
+        if package.installed?
+          Shards.logger.info { "Using #{package.name} (#{package.report_version})" }
           return
         end
 
-        Shards.logger.info "Installing #{package.name} (#{package.report_version})"
-        package.install(version)
+        Shards.logger.info { "Installing #{package.name} (#{package.report_version})" }
+        package.install
         package
       end
 
-      private def generate_lockfile?
-        !Shards.production? && manager.packages.any? && (!lockfile? || outdated_lockfile?)
+      private def generate_lockfile?(packages)
+        !Shards.production? && !packages.empty? && (!lockfile? || outdated_lockfile?(packages))
       end
 
-      private def outdated_lockfile?
-        locks.size != manager.packages.size
+      private def outdated_lockfile?(packages)
+        a = packages.map { |x| {x.name, x.version, x.commit} }
+        b = locks.map { |x| {x.name, x["version"]?, x["commit"]?} }
+        a != b
       end
     end
   end