Validate module dependencies

Author: bastelfreak

.github/workflows/tests.yaml

@@ -245,6 +245,28 @@ jobs:
         if: runner.os == 'Linux'
         run: bundle exec rake ci:boltspec:linux
 
+  check_dependencies:
+    name: Check mod deps
+    needs:
+      - cache_modules
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.4'
+          bundler-cache: true
+      - name: Cache modules
+        id: modules
+        uses: actions/cache@v4
+        with:
+          path: modules
+          key: ${{ runner.os }}-modules-${{ hashFiles('**/Puppetfile') }}
+      - name: Validate all module dependencies
+        run: bundle exec ./scripts/check_dependencies.rb
+
   tests:
     needs:
       - rubocop_and_matrix
@@ -255,6 +277,7 @@ jobs:
       - winrm_transport
       - apply
       - boltspec
+      - check_dependencies
     runs-on: ubuntu-24.04
     name: Test suite
     steps:

.rubocop_todo.yml

@@ -60,12 +60,13 @@ Lint/HashCompareByIdentity:
   Exclude:
     - 'lib/bolt/util.rb'
 
-# Offense count: 3
+# Offense count: 5
 # Configuration parameters: AllowedParentClasses.
 Lint/MissingSuper:
   Exclude:
     - 'lib/bolt/apply_result.rb'
     - 'lib/bolt/transport/podman/connection.rb'
+    - 'scripts/check_dependencies.rb'
     - 'spec/unit/logger_spec.rb'
 
 # Offense count: 2
@@ -89,6 +90,12 @@ Lint/RedundantCopDisableDirective:
   Exclude:
     - 'lib/bolt/apply_inventory.rb'
 
+# Offense count: 1
+# This cop supports unsafe autocorrection (--autocorrect-all).
+Lint/RedundantDirGlobSort:
+  Exclude:
+    - 'scripts/check_dependencies.rb'
+
 # Offense count: 7
 # This cop supports safe autocorrection (--autocorrect).
 Lint/RedundantRequireStatement:
@@ -241,7 +248,7 @@ Performance/RedundantStringChars:
   Exclude:
     - 'lib/bolt/config/transport/base.rb'
 
-# Offense count: 35
+# Offense count: 36
 # This cop supports safe autocorrection (--autocorrect).
 Performance/RegexpMatch:
   Enabled: false
@@ -388,7 +395,7 @@ RSpec/DescribeSymbol:
 RSpec/DescribedClass:
   Enabled: false
 
-# Offense count: 655
+# Offense count: 656
 # Configuration parameters: CountAsOne.
 RSpec/ExampleLength:
   Max: 42
@@ -728,7 +735,7 @@ Style/ConcatArrayLiterals:
     - 'spec/integration/cli/cli_spec.rb'
     - 'spec/unit/module_installer_spec.rb'
 
-# Offense count: 181
+# Offense count: 183
 # Configuration parameters: AllowedConstants.
 Style/Documentation:
   Enabled: false
@@ -804,6 +811,15 @@ Style/FileWrite:
   Exclude:
     - 'spec/lib/bolt_spec/pal.rb'
 
+# Offense count: 1
+# This cop supports unsafe autocorrection (--autocorrect-all).
+# Configuration parameters: EnforcedStyle.
+# SupportedStyles: always, always_true, never
+Style/FrozenStringLiteralComment:
+  Exclude:
+    - '**/*.arb'
+    - 'scripts/check_dependencies.rb'
+
 # Offense count: 97
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: MinBodyLength, AllowConsecutiveConditionals.
@@ -822,14 +838,15 @@ Style/HashAsLastArrayItem:
     - 'spec/unit/applicator_spec.rb'
     - 'spec/unit/pal/yaml_plan/evaluator_spec.rb'
 
-# Offense count: 2
+# Offense count: 3
 # This cop supports unsafe autocorrection (--autocorrect-all).
 # Configuration parameters: AllowSplatArgument.
 Style/HashConversion:
   Exclude:
     - 'lib/bolt/cli.rb'
+    - 'scripts/check_dependencies.rb'
 
-# Offense count: 4
+# Offense count: 5
 # This cop supports unsafe autocorrection (--autocorrect-all).
 # Configuration parameters: AllowedReceivers.
 # AllowedReceivers: Thread.current
@@ -838,6 +855,7 @@ Style/HashEachMethods:
     - 'acceptance/Rakefile'
     - 'lib/bolt/plugin/module.rb'
     - 'lib/bolt/shell/bash.rb'
+    - 'scripts/check_dependencies.rb'
     - 'spec/unit/executor_spec.rb'
 
 # Offense count: 5
@@ -848,6 +866,12 @@ Style/HashExcept:
     - 'lib/bolt/plugin/cache.rb'
     - 'lib/bolt/result.rb'
 
+# Offense count: 1
+# This cop supports unsafe autocorrection (--autocorrect-all).
+Style/HashTransformValues:
+  Exclude:
+    - 'scripts/check_dependencies.rb'
+
 # Offense count: 160
 # This cop supports safe autocorrection (--autocorrect).
 Style/IfUnlessModifier:
@@ -1054,7 +1078,7 @@ Style/StderrPuts:
 Style/StringConcatenation:
   Enabled: false
 
-# Offense count: 3271
+# Offense count: 3277
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: EnforcedStyle, ConsistentQuotesInMultiline.
 # SupportedStyles: single_quotes, double_quotes
@@ -1070,6 +1094,14 @@ Style/SuperArguments:
     - 'lib/bolt/error.rb'
     - 'lib/bolt_spec/plans/action_stubs/plan_stub.rb'
 
+# Offense count: 1
+# This cop supports unsafe autocorrection (--autocorrect-all).
+# Configuration parameters: AllowMethodsWithArguments, AllowedMethods, AllowedPatterns, AllowComments.
+# AllowedMethods: define_method
+Style/SymbolProc:
+  Exclude:
+    - 'scripts/check_dependencies.rb'
+
 # Offense count: 29
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: EnforcedStyle, AllowSafeAssignment.
@@ -1104,7 +1136,7 @@ Style/YAMLFileRead:
   Exclude:
     - 'spec/unit/module_installer_spec.rb'
 
-# Offense count: 2
+# Offense count: 3
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: AllowHeredoc, AllowURI, AllowQualifiedName, URISchemes, IgnoreCopDirectives, AllowedPatterns, SplitStrings.
 # URISchemes: http, https

Gemfile

@@ -36,3 +36,5 @@ end
 
 # https://github.com/OpenVoxProject/puppet/issues/90
 gem 'syslog', '~> 0.3' if RUBY_VERSION >= '3.4'
+
+gem 'puppet_metadata', '~> 5.3'

scripts/check_dependencies.rb

@@ -0,0 +1,126 @@
+#!/usr/bin/env ruby
+
+require 'puppet_metadata'
+require 'semantic_puppet'
+
+def normalize_name(name)
+  name.tr('-', '/')
+end
+
+class LocalModules < SemanticPuppet::Dependency::Source
+  attr_reader :modules
+
+  def initialize(module_dirs)
+    @modules = {}
+
+    module_dirs.each do |module_dir|
+      Dir[File.join(module_dir, '*', 'metadata.json')].sort.map do |metadata|
+        mod = PuppetMetadata.read(metadata)
+        dependencies = mod.dependencies.map { |name, requirement| [normalize_name(name), requirement.to_s] }
+        name = normalize_name(mod.name)
+        @modules[name] = create_release(name, mod.version, dependencies)
+      rescue PuppetMetadata::InvalidMetadataException => e
+        raise "#{metadata}: #{e}"
+      end
+    end
+
+    @modules.freeze
+  end
+
+  def fetch(name)
+    if @modules.key?(name)
+      [@modules[name]]
+    else
+      []
+    end
+  end
+end
+
+class ProvidedModules < SemanticPuppet::Dependency::Source
+  attr_reader :modules
+
+  def initialize(modules)
+    @modules = {}
+
+    modules.each do |name, version|
+      name = normalize_name(name)
+      @modules[name] = create_release(name, version)
+    end
+
+    @modules.freeze
+  end
+
+  def fetch(name)
+    if @modules.key?(name)
+      [@modules[name]]
+    else
+      []
+    end
+  end
+end
+
+# TODO: read this from environment.conf
+local_source = LocalModules.new(['modules'])
+SemanticPuppet::Dependency.add_source(local_source)
+
+modules = Hash[local_source.modules.map { |name, mod| [name, mod.version.to_s] }]
+
+graph = SemanticPuppet::Dependency.query(modules)
+begin
+  releases = SemanticPuppet::Dependency.resolve(graph)
+  puts "Satisfied all dependencies:"
+  releases.each do |release|
+    # TODO: print source
+    puts "  #{release.name} #{release.version}"
+  end
+rescue SemanticPuppet::Dependency::UnsatisfiableGraph => e
+  # There was some dependency that wasn't matched. We know its name and that
+  # it's a dependency we requested previously. Now to investigate why to
+  # provide useful hints
+
+  # TODO: it's *very* weird to store unsatisfiable in a module
+  # Strange API
+  unsatisfiable = SemanticPuppet::Dependency.unsatisfiable
+
+  unless unsatisfiable
+    # In older semantic_puppet unsatisfiable was unreliable
+    # Needs https://github.com/puppetlabs/semantic_puppet/pull/37
+    warn e
+    warn "semantic_puppet provides insufficient information to hint why"
+    exit 1
+  end
+
+  warn "Unable to satisfy #{unsatisfiable}"
+
+  releases = graph.dependencies[unsatisfiable]
+  if releases.empty?
+    # This should never happen
+    warn "No releases found for #{unsatisfiable}"
+    exit 2
+  end
+
+  releases.each do |release|
+    warn "Investigating version #{release.version}"
+    unsatisfied = release.dependencies.select { |_, candidates| candidates.empty? }
+    if unsatisfied.empty?
+      # Is this ever the case, did we miss anything else?
+      warn "  All dependencies satisfied"
+    else
+      unsatisfied.each do |dependency, _|
+        constraint = release.constraints_for(dependency).find { |constraint| constraint[:source] == 'initialize' }
+        warn "  Dependency #{dependency} (#{constraint[:description]}) can't be satified"
+
+        constraint_graph = SemanticPuppet::Dependency.query(dependency => '>= 0')
+        begin
+          releases = SemanticPuppet::Dependency.resolve(constraint_graph)
+          mod_releases = releases.select { |release| release.name == dependency }
+          warn "    Available versions: #{mod_releases.map { |r| r.version }.join(', ')}"
+        rescue SemanticPuppet::Dependency::UnsatisfiableGraph
+          warn "    Unable to find any release"
+        end
+      end
+    end
+  end
+
+  exit 3
+end