Merge pull request glennsarti#33 from logicminds/git_spec_searcher

Adds git clone method to search for metadata file

Author: glennsarti

Gemfile

@@ -5,7 +5,7 @@ gemspec
 
 group :development do
   gem 'rspec', '>= 3.2', :require => false
-
+  gem 'pry'
   if RUBY_VERSION =~ /^2\.1\./
     gem "rubocop", "<= 0.57.2", :require => false, :platforms => [:ruby, :x64_mingw]
     gem 'rake', '~> 12.3',      :require => false

lib/puppetfile-resolver/spec_searchers/git.rb

@@ -5,6 +5,7 @@
 require 'puppetfile-resolver/spec_searchers/git_configuration'
 require 'puppetfile-resolver/spec_searchers/git/github'
 require 'puppetfile-resolver/spec_searchers/git/gitlab'
+require 'puppetfile-resolver/spec_searchers/git/gclone'
 
 module PuppetfileResolver
   module SpecSearchers
@@ -14,12 +15,13 @@ def self.find_all(puppetfile_module, dependency, cache, resolver_ui, config)
         # Has the information been cached?
         return cache.load(dep_id) if cache.exist?(dep_id)
 
-        # We _could_ git clone this, but it'll take too long. So for now, just
-        # try and resolve github based repositories by crafting a URL
-        metadata = GitHub.metadata(puppetfile_module, resolver_ui, config)
-        metadata = GitLab.metadata(puppetfile_module, resolver_ui, config) if metadata.nil?
-
-        # TODO: Once we've exhausted using alternate methods lets just `git clone` it
+        # The git clone method takes around (1s) depending on repo size.  Not sure if the
+        # other methods take longer or shorter but I preserved the legacy code for now.
+        # Technically, the gclone class could replace the other classes and speed up
+        # this process here.
+        metadata = GitHub.metadata(puppetfile_module, resolver_ui, config) ||
+                   GitLab.metadata(puppetfile_module, resolver_ui, config) ||
+                   GClone.metadata(puppetfile_module, resolver_ui, config)
 
         if metadata.nil? || metadata.empty?
           # Cache that we couldn't find the metadata

lib/puppetfile-resolver/spec_searchers/git/gclone.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require 'tempfile'
+require 'English'
+require 'puppetfile-resolver/util'
+require 'puppetfile-resolver/spec_searchers/common'
+require 'puppetfile-resolver/spec_searchers/git_configuration'
+require 'puppetfile-resolver/util'
+require 'uri'
+
+module PuppetfileResolver
+  module SpecSearchers
+    module Git
+      module GClone
+        # @summary clones the remote url and reads the metadata file
+        # @returns [String] the content of the metadata file
+        def self.metadata(puppetfile_module, resolver_ui, config)
+          repo_url = puppetfile_module.remote
+
+          unless PuppetfileResolver::Util.git?
+            resolver_ui.debug { 'Git executible not found, unable to use git clone resolution' }
+
+            return nil
+          end
+          return nil if repo_url.nil?
+          return nil unless valid_http_url?(repo_url)
+
+          metadata_file = 'metadata.json'
+
+          ref = puppetfile_module.ref ||
+                puppetfile_module.tag ||
+                puppetfile_module.commit ||
+                puppetfile_module.branch ||
+                'HEAD'
+
+          resolver_ui.debug { "Querying git repository #{repo_url}" }
+
+          clone_and_read_file(repo_url, ref, metadata_file, config)
+        end
+
+        # @summary clones the git url and reads the file at the given ref
+        #          a temp directory will be created and then destroyed during
+        #          the cloning and reading process
+        # @param ref [String] the git ref, branch, commit, tag
+        # @param file [String] the file you wish to read
+        # @returns [String] the content of the file
+        def self.clone_and_read_file(url, ref, file, config)
+          clone_cmd = ['git', 'clone', '--bare', '--depth=1', '--single-branch']
+          err_msg = ''
+          if config.git.proxy
+            err_msg += " with proxy #{config.git.proxy}: "
+            proxy = "--config \"http.proxy=#{config.git.proxy}\" --config \"https.proxy=#{config.proxy}\""
+            clone_cmd.push(proxy)
+          end
+
+          Dir.mktmpdir(nil, config.git.clone_dir) do |dir|
+            clone_cmd.push("--branch=#{ref}") if ref != 'HEAD'
+            clone_cmd.push(url, dir)
+            out, err_out, process = ::PuppetfileResolver::Util.run_command(clone_cmd)
+            err_msg += out
+            raise err_msg unless process.success?
+            Dir.chdir(dir) do
+              content, err_out, process = ::PuppetfileResolver::Util.run_command(['git', 'show', "#{ref}:#{file}"])
+              raise 'InvalidContent' unless process.success? && content.length > 2
+              return content
+            end
+          end
+        end
+
+        def self.valid_http_url?(url)
+          # uri does not work with git urls, return true
+          return true if url.start_with?('git@')
+
+          uri = URI.parse(url)
+          uri.is_a?(URI::HTTP) && !uri.host.nil?
+        rescue URI::InvalidURIError
+          false
+        end
+      end
+    end
+  end
+end

lib/puppetfile-resolver/spec_searchers/git_configuration.rb

@@ -3,7 +3,7 @@
 module PuppetfileResolver
   module SpecSearchers
     class GitConfiguration
-      attr_accessor :proxy
+      attr_accessor :proxy, :clone_dir
     end
   end
 end

lib/puppetfile-resolver/util.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'open3'
+
 module PuppetfileResolver
   module Util
     def self.symbolise_object(object)
@@ -43,5 +45,22 @@ def self.net_http_get(uri, proxy = nil)
       Net::HTTP.start(*start_args, http_options) { |http| return http.request(Net::HTTP::Get.new(uri)) }
       nil
     end
+
+    # @summary runs the command on the shell
+    # @param cmd [Array] an array of command and args
+    # @returns [Array] the result of running the comand and the process
+    # @example run_command(['git', '--version'])
+    def self.run_command(cmd)
+      Open3.capture3(*cmd)
+    end
+
+    # @summary checks if git is installed and on the path
+    # @returns [Boolean] true if git is found in the path
+    def self.git?
+      Open3.capture3('git', '--version')
+      true
+    rescue Errno::ENOENT
+      false
+    end
   end
 end

spec/unit/puppetfile-resolver/spec_searchers/git/gclone_spec.rb

@@ -0,0 +1,50 @@
+require 'spec_helper'
+require 'puppetfile-resolver/spec_searchers/git/gclone'
+require 'puppetfile-resolver/spec_searchers/git_configuration'
+require 'logger'
+require 'json'
+
+describe PuppetfileResolver::SpecSearchers::Git::GClone do
+  PuppetfileModule = Struct.new(:remote, :ref, :branch, :commit, :tag, keyword_init: true)
+  config = PuppetfileResolver::SpecSearchers::Configuration.new
+  config.local.puppet_module_paths = [File.join(FIXTURES_DIR, 'modulepath')]
+
+  let(:url) do
+    'https://github.com/puppetlabs/puppetlabs-powershell'
+  end
+
+  let(:puppetfile_module) do
+    PuppetfileModule.new(remote: url)
+  end
+
+
+  context 'valid url' do
+    it 'reads metadata' do
+      content = subject.metadata(puppetfile_module, Logger.new(STDERR), config)
+      expect(JSON.parse(content)['name']).to eq('puppetlabs-powershell')
+    end
+
+    context 'with ref' do
+
+      let(:puppetfile_module) do
+        PuppetfileModule.new(remote: url, ref: '2.1.2')
+      end
+
+      it 'reads metadata' do
+        content = subject.metadata(puppetfile_module, Logger.new(STDERR), config)
+        expect(JSON.parse(content)['name']).to eq('puppetlabs-powershell')
+      end
+    end
+  end
+
+  context 'invalid url' do
+    let(:url) do
+      'https://github.com/puppetlabs/puppetlabs-powershellbad'
+    end
+
+    it 'throws exception' do
+      expect{subject.metadata(puppetfile_module, Logger.new(STDERR), config)}
+      .to raise_exception(RuntimeError)
+    end
+  end
+end