Merge pull request #12998 from dependabot/brrygrdn/dg-7658-establish-file-grapher

brrygrdn · web-flow · commit 5df39360cca2 · 2025-09-19T14:03:47.000+01:00
Structure Dependabot::DependencyGrapher as an ecosystem component with generic fallback
diff --git a/bundler/spec/dependabot/bundler/dependency_grapher_spec.rb b/bundler/spec/dependabot/bundler/dependency_grapher_spec.rb
@@ -0,0 +1,109 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "spec_helper"
+require "dependabot/bundler"
+require "dependabot/dependency_graphers"
+
+# TODO: Implement a concrete Bundler class
+RSpec.describe "Dependabot::DependencyGraphers::Generic" do
+  context "with a bundler project" do
+    subject(:grapher) do
+      Dependabot::DependencyGraphers.for_package_manager("bundler").new(
+        dependency_files:,
+        dependencies:
+      )
+    end
+
+    let(:parser) do
+      Dependabot::FileParsers.for_package_manager("bundler").new(
+        dependency_files:,
+        repo_contents_path: nil,
+        source: source,
+        credentials: [],
+        reject_external_code: false
+      )
+    end
+
+    let(:source) do
+      Dependabot::Source.new(
+        provider: "github",
+        repo: "dependabot-fixtures/dependabot-test-ruby-package",
+        directory: "/",
+        branch: "main"
+      )
+    end
+
+    let(:dependencies) { parser.parse }
+
+    # NOTE: This documents existing behaviour where Gemfile PURLs do not include a resolved version
+    #
+    # Package URLs deal in resolved versions, so for a Gemfile only project we only have a range
+    # which cannot currently be submitted to the Dependency Submission API.
+    #
+    # This is working as intended, but when we implement a concrete Bundler grapher we will want
+    # to execute a `bundle install` to resolve the file at runtime so we can submit the resolved
+    # dependencies.
+    context "with a Gemfile only" do
+      let(:gemfile) do
+        bundler_project_dependency_file("subdependency", filename: "Gemfile")
+      end
+
+      let(:dependency_files) do
+        [gemfile]
+      end
+
+      it "specifies the Gemfile as the relevant dependency file" do
+        expect(grapher.relevant_dependency_file).to eql(gemfile)
+      end
+
+      it "correctly serializes the resolved dependencies" do
+        expect(grapher.resolved_dependencies.count).to be(1)
+
+        ibandit = grapher.resolved_dependencies["ibandit"]
+        expect(ibandit[:package_url]).to eql("pkg:gem/ibandit")
+        expect(ibandit[:relationship]).to eql("direct")
+        expect(ibandit[:scope]).to eql("runtime")
+        expect(ibandit[:dependencies]).to be_empty
+      end
+    end
+
+    context "with a Gemfile and Gemfile.lock" do
+      let(:gemfile) do
+        bundler_project_dependency_file("subdependency", filename: "Gemfile")
+      end
+
+      let(:gemfile_lock) do
+        bundler_project_dependency_file("subdependency", filename: "Gemfile.lock")
+      end
+
+      let(:dependency_files) do
+        [gemfile, gemfile_lock]
+      end
+
+      it "specifies the Gemfile.lock as the relevant dependency file" do
+        expect(grapher.relevant_dependency_file).to eql(gemfile_lock)
+      end
+
+      it "correctly serializes the resolved dependencies" do
+        resolved_dependencies = grapher.resolved_dependencies
+
+        expect(resolved_dependencies.count).to be(2)
+
+        expect(resolved_dependencies.keys).to eql(%w(ibandit i18n))
+
+        ibandit = resolved_dependencies["ibandit"]
+        expect(ibandit[:package_url]).to eql("pkg:gem/ibandit@0.7.0")
+        expect(ibandit[:relationship]).to eql("direct")
+        expect(ibandit[:scope]).to eql("runtime")
+        expect(ibandit[:dependencies]).to be_empty # NYI: We don't set any subdependencies yet, this should contain i18n
+
+        i18n = resolved_dependencies["i18n"]
+        expect(i18n[:package_url]).to eql("pkg:gem/i18n@0.7.0.beta1")
+        expect(i18n[:relationship]).to eql("indirect")
+        expect(i18n[:scope]).to eql("runtime")
+        expect(i18n[:dependencies]).to be_empty
+      end
+    end
+  end
+end
diff --git a/common/lib/dependabot/dependency_graphers.rb b/common/lib/dependabot/dependency_graphers.rb
@@ -0,0 +1,33 @@
+# typed: strong
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "dependabot/dependency_graphers/base"
+require "dependabot/dependency_graphers/generic"
+
+module Dependabot
+  module DependencyGraphers
+    extend T::Sig
+
+    @graphers = T.let({}, T::Hash[String, T.class_of(Base)])
+
+    sig { params(package_manager: String).returns(T.class_of(Base)) }
+    def self.for_package_manager(package_manager)
+      grapher = @graphers[package_manager]
+      return grapher if grapher
+
+      # If an ecosystem has not defined its own graphing strategy, then we use
+      # a best-effort generic while we are rolling out graphing capabilities.
+      #
+      # This approach allows us to assess the quality of data from the ecosystem's
+      # parser and triage the scope of work to implement the non-generic class.
+      Generic
+    end
+
+    sig { params(package_manager: String, grapher: T.class_of(Base)).void }
+    def self.register(package_manager, grapher)
+      @graphers[package_manager] = grapher
+    end
+  end
+end
diff --git a/common/lib/dependabot/dependency_graphers/README.md b/common/lib/dependabot/dependency_graphers/README.md
@@ -0,0 +1,54 @@
+# Dependency Graphers
+
+Dependency graphers are used to convert a set of parsed dependencies into a data structure we can use to output the dependency graph of a project in a generic data structure based on GitHub's [Dependency submission API](https://docs.github.com/en/rest/dependency-graph/dependency-submission).
+
+We will expect each language Dependabot supports to implement a `Dependabot::DependencyGraphers` class in future, but for now any modules that do not implement a specific class fail over to a 'best effort' generic implementation that works in most cases.
+
+## Public API
+
+Each `Dependabot::DependencyGraphers` class implements the following methods:
+
+| Method                      | Description                                                                                   |
+|-----------------------------|-----------------------------------------------------------------------------------------------|
+| `.relevant_dependency_file` | Checks the list of `Dependabot::DependencyFile` objects assigned and determines which one the dependency list should be reported against. In most languages this will be the lockfile, if present, and the manifest otherwise. |
+| `.resolved_dependencies`    | Processes the assigned `Dependabot::Dependency` objects into an informational hash |
+
+An example of a `.resolved_dependencies` hash for a Bundler project:
+
+```ruby
+"addressable": {
+  "package_url": "pkg:gem/addressable@2.8.6",
+  "relationship": "indirect",
+  "scope": "runtime",
+  "dependencies": [],
+  "metadata": {}
+},
+"ast": {
+  "package_url": "pkg:gem/ast@2.4.2",
+  "relationship": "indirect",
+  "scope": "runtime",
+  "dependencies": [],
+  "metadata": {}
+},
+"aws-eventstream": {
+  "package_url": "pkg:gem/aws-eventstream@1.3.0",
+  "relationship": "indirect",
+  "scope": "runtime",
+  "dependencies": [],
+  "metadata": {}
+}
+```
+
+## Writing a file fetcher for a new language
+
+All new file fetchers should inherit from `Dependabot::DependencyGraphers::Base` and
+implement the following methods:
+
+| Method                           | Description                                                                                   |
+|----------------------------------|-----------------------------------------------------------------------------------------------|
+| `.relevant_dependency_file`      | See Public API section. |
+| `.fetch_subdependencies`         | Private method to fetch a list of package names, or PURLs, that are subdependencies of a given `Dependabot::Dependency`. It is expected that some languages will need to perform additional native commands to obtain this data. |
+| `.purl_pkg_for`                  | Private method to map the given `Dependabot::Dependency` to the correct [Package-URL type](https://github.com/package-url/purl-spec/blob/main/PURL-TYPES.rst) for the package manager involved. |
+
+> [!WARNING]
+> While PURLs are preferred in all languages for `.fetch_subdependencies`, for languages where multiple versions of a single dependency are permitted they _must_ be provided to be precise.
diff --git a/common/lib/dependabot/dependency_graphers/base.rb b/common/lib/dependabot/dependency_graphers/base.rb
@@ -0,0 +1,95 @@
+# typed: strong
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+module Dependabot
+  module DependencyGraphers
+    class Base
+      extend T::Sig
+      extend T::Helpers
+
+      abstract!
+
+      # TODO(brrygrdn): Inject the Dependency parser instead of pre-parsed `dependencies`
+      #
+      # Semantically it makes sense for the grapher to wrap the parser as a higher order function, but we already know
+      # that some package managers will require extra native commands before, after or during the parse - in extreme
+      # cases it may make sense to use an alternative parser that is more optimal.
+      #
+      # By injecting the parser, this allows the ecosystem to encapsulate the package manager specifics without the
+      # executor needing to manage parser modes / feature flags.
+      sig do
+        params(
+          dependency_files: T::Array[Dependabot::DependencyFile],
+          dependencies: T::Array[Dependabot::Dependency]
+        ).void
+      end
+      def initialize(dependency_files:, dependencies:)
+        @dependency_files = dependency_files
+        @dependencies = dependencies
+      end
+
+      # Each grapher must implement a heuristic to determine which dependency file should be used as the owner
+      # of the resolved_dependencies.
+      #
+      # Conventionally, this is the lockfile for the file set but some parses may only include the manifest
+      # so this method should take into account the correct priority based on which files were parsed.
+      sig { abstract.returns(Dependabot::DependencyFile) }
+      def relevant_dependency_file; end
+
+      sig { returns(T::Hash[Symbol, T.untyped]) }
+      def resolved_dependencies
+        @dependencies.each_with_object({}) do |dep, resolved|
+          resolved[dep.name] = {
+            package_url: build_purl(dep),
+            relationship: relationship_for(dep),
+            scope: scope_for(dep),
+            dependencies: fetch_subdependencies(dep),
+            metadata: {}
+          }
+        end
+      end
+
+      private
+
+      # Each grapher is expected to implement a method to look up the parents of a given dependency.
+      #
+      # The strategy that should be used is highly dependent on the ecosystem, in some cases the parser
+      # may be able to set this information in the dependency.metadata collection, in others the grapher
+      # will need to run additional native commands.
+      sig { abstract.params(dependency: Dependabot::Dependency).returns(T::Array[String]) }
+      def fetch_subdependencies(dependency); end
+
+      # Each grapher is expected to implement a method to map the various package managers it supports to
+      # the correct Package-URL type, see:
+      #   https://github.com/package-url/purl-spec/blob/main/PURL-TYPES.rst
+      sig { abstract.params(package_manager: String).returns(String) }
+      def purl_pkg_for(package_manager); end
+
+      # Generate a purl for the provided Dependency object
+      sig { params(dependency: Dependabot::Dependency).returns(String) }
+      def build_purl(dependency)
+        "pkg:#{purl_pkg_for(dependency.package_manager)}/#{dependency.name}@#{dependency.version}".chomp("@")
+      end
+
+      sig { params(dep: Dependabot::Dependency).returns(String) }
+      def relationship_for(dep)
+        if dep.top_level?
+          "direct"
+        else
+          "indirect"
+        end
+      end
+
+      sig { params(dependency: Dependabot::Dependency).returns(String) }
+      def scope_for(dependency)
+        if dependency.production?
+          "runtime"
+        else
+          "development"
+        end
+      end
+    end
+  end
+end
diff --git a/common/lib/dependabot/dependency_graphers/generic.rb b/common/lib/dependabot/dependency_graphers/generic.rb
@@ -0,0 +1,76 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "sorbet-runtime"
+
+require "dependabot/dependency_graphers/base"
+
+module Dependabot
+  module DependencyGraphers
+    class Generic < Base
+      extend T::Sig
+      extend T::Helpers
+
+      # Our generic strategy is to use the right-most file in the dependency file list on the
+      # assumption that this is normally the lockfile.
+      #
+      # This isn't a durable strategy but it's good enough to allow most ecosystems to 'just work'
+      # as we roll out ecosystem-specific graphers.
+      sig { override.returns(Dependabot::DependencyFile) }
+      def relevant_dependency_file
+        T.must(filtered_dependency_files.last)
+      end
+
+      private
+
+      sig { returns(T::Array[Dependabot::DependencyFile]) }
+      def filtered_dependency_files
+        @dependency_files.reject { |f| f.support_file? || f.vendored_file? }
+      end
+
+      # Our generic strategy is to check if the parser has attached a `depends_on` key to the Dependency's
+      # metadata, but in most cases this will be empty.
+      sig { override.params(dependency: Dependabot::Dependency).returns(T::Array[String]) }
+      def fetch_subdependencies(dependency)
+        dependency.metadata.fetch(:depends_on, [])
+      end
+
+      # TODO: Delegate this to ecosystem-specific base classes
+      sig { override.params(package_manager: String).returns(String) }
+      def purl_pkg_for(package_manager)
+        case package_manager
+        when "bundler"
+          "gem"
+        when "npm_and_yarn", "bun"
+          "npm"
+        when "maven", "gradle"
+          "maven"
+        when "pip", "uv"
+          "pypi"
+        when "cargo"
+          "cargo"
+        when "hex"
+          "hex"
+        when "composer"
+          "composer"
+        when "nuget"
+          "nuget"
+        when "go_modules"
+          "golang"
+        when "docker"
+          "docker"
+        when "github_actions"
+          "github"
+        when "terraform"
+          "terraform"
+        when "pub"
+          "pub"
+        when "elm"
+          "elm"
+        else
+          "generic"
+        end
+      end
+    end
+  end
+end
diff --git a/go_modules/lib/dependabot/go_modules.rb b/go_modules/lib/dependabot/go_modules.rb
@@ -3,6 +3,7 @@
 
 # These all need to be required so the various classes can be registered in a
 # lookup table of package manager names to concrete classes.
+require "dependabot/go_modules/dependency_grapher"
 require "dependabot/go_modules/file_fetcher"
 require "dependabot/go_modules/file_parser"
 require "dependabot/go_modules/update_checker"
diff --git a/go_modules/lib/dependabot/go_modules/dependency_grapher.rb b/go_modules/lib/dependabot/go_modules/dependency_grapher.rb
diff --git a/go_modules/spec/dependabot/go_modules/dependency_grapher_spec.rb b/go_modules/spec/dependabot/go_modules/dependency_grapher_spec.rb
diff --git a/updater/lib/github_api/dependency_submission.rb b/updater/lib/github_api/dependency_submission.rb
