Add Puppetserver v4 catalog API support

Author: seanmil

lib/octocatalog-diff/catalog.rb

@@ -191,6 +191,8 @@ def resources
       build
       raise OctocatalogDiff::Errors::CatalogError, 'Catalog does not appear to have been built' if !valid? && error_message.nil?
       raise OctocatalogDiff::Errors::CatalogError, error_message unless valid?
+      # Handle the structure returned by the /puppet/v4/catalog Puppetserver endpoint:
+      return @catalog['catalog']['resources'] if @catalog['catalog'].is_a?(Hash) && @catalog['catalog']['resources'].is_a?(Array)
       return @catalog['data']['resources'] if @catalog['data'].is_a?(Hash) && @catalog['data']['resources'].is_a?(Array)
       return @catalog['resources'] if @catalog['resources'].is_a?(Array)
       # This is a bug condition

lib/octocatalog-diff/catalog/puppetmaster.rb

@@ -62,16 +62,19 @@ def build_catalog(logger = Logger.new(StringIO.new))
         fetch_catalog(logger)
       end
 
-      # Returns a hash of parameters for each supported version of the Puppet Server Catalog API.
+      # Returns a hash of parameters for the requested version of the Puppet Server Catalog API.
       # @return [Hash] Hash of parameters
       #
       # Note: The double escaping of the facts here is implemented to correspond to a long standing
       # bug in the Puppet code. See https://github.com/puppetlabs/puppet/pull/1818 and
       # https://docs.puppet.com/puppet/latest/http_api/http_catalog.html#parameters for explanation.
-      def puppet_catalog_api
-        {
+      def puppet_catalog_api(version)
+        api_style = {
           2 => {
             url: "https://#{@options[:puppet_master]}/#{@options[:branch]}/catalog/#{@node}",
+            headers: {
+              'Accept' => 'text/pson'
+            },
             parameters: {
               'facts_format' => 'pson',
               'facts' => CGI.escape(@facts.fudge_timestamp.without('trusted').to_pson),
@@ -80,24 +83,59 @@ def puppet_catalog_api
           },
           3 => {
             url: "https://#{@options[:puppet_master]}/puppet/v3/catalog/#{@node}",
+            headers: {
+              'Accept' => 'text/pson'
+            },
             parameters: {
               'environment' => @options[:branch],
               'facts_format' => 'pson',
               'facts' => CGI.escape(@facts.fudge_timestamp.without('trusted').to_pson),
               'transaction_uuid' => SecureRandom.uuid
             }
+          },
+          4 => {
+            url: "https://#{@options[:puppet_master]}/puppet/v4/catalog",
+            headers: {
+              'Content-Type' => 'application/json'
+            },
+            parameters: {
+              'certname' => @node,
+              'persistence' => {
+                'facts' => @options[:puppet_master_update_facts] || false,
+                'catalog' => @options[:puppet_master_update_catalog] || false
+              },
+              'environment' => @options[:branch],
+              'facts' => { 'values' => @facts.facts['values'] },
+              'options' => {
+                'prefer_requested_environment' => true,
+                'capture_logs' => false,
+                'log_level' => 'warning'
+              },
+              'transaction_uuid' => SecureRandom.uuid
+            }
           }
         }
+
+        params = api_style[version]
+        return nil if params.nil?
+
+        unless @options[:puppet_master_token].nil?
+          params[:headers]['X-Authentication'] = @options[:puppet_master_token]
+        end
+
+        params[:parameters] = params[:parameters].to_json if version >= 4
+
+        params
       end
 
       # Fetch catalog by contacting the Puppet master, sending the facts, and asking for the catalog. When the
       # catalog is returned in PSON format, parse it to JSON and then set appropriate variables.
       def fetch_catalog(logger)
         api_version = @options[:puppet_master_api_version] || DEFAULT_PUPPET_SERVER_API
-        api = puppet_catalog_api[api_version]
+        api = puppet_catalog_api(api_version)
         raise ArgumentError, "Unsupported or invalid API version #{api_version}" unless api.is_a?(Hash)
 
-        more_options = { headers: { 'Accept' => 'text/pson' }, timeout: @timeout }
+        more_options = { headers: api[:headers], timeout: @timeout }
         post_hash = api[:parameters]
 
         response = nil

lib/octocatalog-diff/cli/options/puppet_master_api_version.rb

@@ -14,8 +14,8 @@ def parse(parser, options)
       options: options,
       cli_name: 'puppet-master-api-version',
       option_name: 'puppet_master_api_version',
-      desc: 'Puppet Master API version (2 for Puppet 3.x, 3 for Puppet 4.x)',
-      validator: ->(x) { x =~ /^[23]$/ || raise(ArgumentError, 'Only API versions 2 and 3 are supported') },
+      desc: 'Puppet Master API version (2 for Puppet 3.x, 3 for Puppet 4.x, 4 for Puppet Server >= 6.3.0)',
+      validator: ->(x) { x =~ /^[234]$/ || raise(ArgumentError, 'Only API versions 2, 3, and 4 are supported') },
       translator: ->(x) { x.to_i }
     )
   end

lib/octocatalog-diff/cli/options/puppet_master_token.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Specify a PE RBAC token used to authenticate to Puppetserver for v4
+# catalog API calls.
+# @param parser [OptionParser object] The OptionParser argument
+# @param options [Hash] Options hash being constructed; this is modified in this method.
+OctocatalogDiff::Cli::Options::Option.newoption(:puppet_master_token) do
+  has_weight 310
+
+  def parse(parser, options)
+    OctocatalogDiff::Cli::Options.option_globally_or_per_branch(
+      parser: parser,
+      options: options,
+      datatype: '',
+      cli_name: 'puppet-master-token',
+      option_name: 'puppet_master_token',
+      desc: 'PE RBAC token to authenticate to the Puppetserver API v4'
+    )
+  end
+end

lib/octocatalog-diff/cli/options/puppet_master_token_file.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# Specify a path to a file containing a PE RBAC token used to authenticate to the
+# Puppetserver for a v4 catalog API call.
+# @param parser [OptionParser object] The OptionParser argument
+# @param options [Hash] Options hash being constructed; this is modified in this method.
+OctocatalogDiff::Cli::Options::Option.newoption(:puppet_master_token_file) do
+  has_weight 300
+
+  def parse(parser, options)
+    OctocatalogDiff::Cli::Options.option_globally_or_per_branch(
+      parser: parser,
+      options: options,
+      datatype: '',
+      cli_name: 'puppet-master-token-file',
+      option_name: 'puppet_master_token_file',
+      desc: 'File containing PE RBAC token to authenticate to the Puppetserver API v4',
+      translator: ->(x) { x.start_with?('/', '~') ? x : File.join(options[:basedir], x) },
+      post_process: lambda do |opts|
+        %w(to from).each do |prefix|
+          fileopt = "#{prefix}_puppet_master_token_file".to_sym
+          tokenopt = "#{prefix}_puppet_master_token".to_sym
+
+          tokenfile = opts[fileopt]
+          next if tokenfile.nil?
+
+          raise(Errno::ENOENT, "Token file #{tokenfile} is not readable") unless File.readable?(tokenfile)
+
+          token = File.read(tokenfile).strip
+          opts[tokenopt] ||= token
+        end
+      end
+    )
+  end
+end

lib/octocatalog-diff/cli/options/puppet_master_update_catalog.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Specify if, when using the Puppetserver v4 catalog API, the Puppetserver should
+# update the catalog in PuppetDB.
+# @param parser [OptionParser object] The OptionParser argument
+# @param options [Hash] Options hash being constructed; this is modified in this method.
+OctocatalogDiff::Cli::Options::Option.newoption(:puppet_master_update_catalog) do
+  has_weight 320
+
+  def parse(parser, options)
+    OctocatalogDiff::Cli::Options.option_globally_or_per_branch(
+      parser: parser,
+      options: options,
+      datatype: false,
+      cli_name: 'puppet-master-update-catalog',
+      option_name: 'puppet_master_update_catalog',
+      desc: 'Update catalog in PuppetDB when using Puppetmaster API version 4'
+    )
+  end
+end

lib/octocatalog-diff/cli/options/puppet_master_update_facts.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Specify if, when using the Puppetserver v4 catalog API, the Puppetserver should
+# update the facts in PuppetDB.
+# @param parser [OptionParser object] The OptionParser argument
+# @param options [Hash] Options hash being constructed; this is modified in this method.
+OctocatalogDiff::Cli::Options::Option.newoption(:puppet_master_update_facts) do
+  has_weight 320
+
+  def parse(parser, options)
+    OctocatalogDiff::Cli::Options.option_globally_or_per_branch(
+      parser: parser,
+      options: options,
+      datatype: false,
+      cli_name: 'puppet-master-update-facts',
+      option_name: 'puppet_master_update_facts',
+      desc: 'Update facts in PuppetDB when using Puppetmaster API version 4'
+    )
+  end
+end

spec/octocatalog-diff/fixtures/catalogs/tiny-catalog-v4-api.json

@@ -0,0 +1,41 @@
+{
+  "catalog": {
+    "tags": ["settings"],
+    "name": "my.rspec.node",
+    "version": "production",
+    "code_id": null,
+    "catalog_uuid": "89869359-db50-472f-b435-1d37c22be9eb",
+    "catalog_format": 1,
+    "environment": "production",
+    "resources": [
+      {
+        "type": "Stage",
+        "title": "main",
+        "tags": ["stage"],
+        "exported": false,
+        "parameters": {
+          "name": "main"
+        }
+      },
+      {
+        "type": "Class",
+        "title": "Settings",
+        "tags": ["class","settings"],
+        "exported": false
+      }
+    ],
+    "edges": [
+      {
+        "source": "Stage[main]",
+        "target": "Class[Settings]"
+      },
+      {
+        "source": "Stage[main]",
+        "target": "Class[main]"
+      }
+    ],
+    "classes": [
+      "settings"
+    ]
+  }
+}

spec/octocatalog-diff/fixtures/configs/puppet-master-token.txt

@@ -0,0 +1 @@
+secretpuppetmastertoken

spec/octocatalog-diff/tests/catalog/puppetmaster_spec.rb

@@ -73,19 +73,21 @@
           @context = context
           {
             code: 200,
-            parsed: JSON.parse(File.read(OctocatalogDiff::Spec.fixture_path('catalogs/tiny-catalog.json')))
+            parsed: JSON.parse(File.read(OctocatalogDiff::Spec.fixture_path(fixture_catalog)))
           }
         end
       end
 
       let(:api_url) do
         {
           2 => 'https://fake-puppetmaster.non-existent-domain.com:8140/foobranch/catalog/foo',
-          3 => 'https://fake-puppetmaster.non-existent-domain.com:8140/puppet/v3/catalog/foo'
+          3 => 'https://fake-puppetmaster.non-existent-domain.com:8140/puppet/v3/catalog/foo',
+          4 => 'https://fake-puppetmaster.non-existent-domain.com:8140/puppet/v4/catalog'
         }
       end
 
       let(:api_sets_environment) { { 2 => false, 3 => true } }
+      let(:fixture_catalog) { 'catalogs/tiny-catalog.json' }
 
       [2, 3].each do |api_version|
         context "api v#{api_version}" do
@@ -100,6 +102,10 @@
             expect(@url).to eq(api_url[api_version])
           end
 
+          it 'should set the Accept header' do
+            expect(@opts[:headers]['Accept']).to eq('text/pson')
+          end
+
           it 'should post the correct facts to HTTParty' do
             answer = JSON.parse(File.read(OctocatalogDiff::Spec.fixture_path('facts/facts_esc.json')))
             answer.delete('_timestamp')
@@ -136,6 +142,90 @@
         end
       end
 
+      context 'api v4' do
+        let(:fixture_catalog) { 'catalogs/tiny-catalog-v4-api.json' }
+        let(:extra_opts) { {} }
+
+        before(:each) do
+          opts = { puppet_master_api_version: 4 }
+          @obj = OctocatalogDiff::Catalog::PuppetMaster.new(valid_options.merge(opts).merge(extra_opts))
+          @logger, @logger_str = OctocatalogDiff::Spec.setup_logger
+          @obj.build(@logger)
+          @parsed_data = JSON.parse(@post_data)
+        end
+
+        it 'should post to the correct URL' do
+          expect(@url).to eq(api_url[4])
+        end
+
+        it 'should set the Content-Type header correctly' do
+          expect(@opts[:headers]['Content-Type']).to eq('application/json')
+        end
+
+        it 'should not set the X-Authentication header when no token is provided' do
+          expect(@opts[:headers].key?('X-Authentication')).to eq false
+        end
+
+        it 'should post the correct facts to HTTParty' do
+          answer = JSON.parse(File.read(OctocatalogDiff::Spec.fixture_path('facts/facts_esc.json')))
+          answer.delete('_timestamp')
+          result = @parsed_data['facts']['values']
+          expect(result).to eq(answer)
+        end
+
+        it 'should set the environment in the parameters correctly for the API' do
+          expect(@parsed_data['environment']).to eq('foobranch')
+        end
+
+        it 'should default to false for persistence' do
+          expect(@parsed_data['persistence']['facts']).to eq false
+          expect(@parsed_data['persistence']['catalog']).to eq false
+        end
+
+        it 'should parse the response and set instance variables correctly' do
+          expect(@obj.catalog).to be_a_kind_of(Hash)
+          expect(@obj.catalog_json).to be_a_kind_of(String)
+          expect(@obj.error_message).to eq(nil)
+        end
+
+        it 'should log correctly' do
+          logs = @logger_str.string
+          expect(logs).to match(/Start retrieving facts for foo from OctocatalogDiff::Catalog::PuppetMaster/)
+          expect(logs).to match(%r{Retrieving facts from.*fixtures/facts/facts_esc.yaml})
+          expect(logs).to match(%r{Retrieving facts from.*fixtures/facts/facts_esc.yaml})
+
+          answer = Regexp.new("Retrieve catalog from #{api_url[4]} environment foobranch")
+          expect(logs).to match(answer)
+
+          answer2 = Regexp.new("Response from #{api_url[4]} environment foobranch was 200")
+          expect(logs).to match(answer2)
+        end
+
+        context 'when a RBAC token is passed' do
+          let(:extra_opts) { { puppet_master_token: 'mytoken' } }
+
+          it 'should set the token in the headers' do
+            expect(@opts[:headers]['X-Authentication']).to eq 'mytoken'
+          end
+        end
+
+        context 'when facts persistence is requested' do
+          let(:extra_opts) { { puppet_master_update_facts: true } }
+
+          it 'should set the request in the parameters' do
+            expect(@parsed_data['persistence']['facts']).to eq true
+          end
+        end
+
+        context 'when catalog persistence is requested' do
+          let(:extra_opts) { { puppet_master_update_catalog: true } }
+
+          it 'should set the request in the parameters' do
+            expect(@parsed_data['persistence']['catalog']).to eq true
+          end
+        end
+      end
+
       context 'response is not 200' do
         before(:each) do
           allow(OctocatalogDiff::Util::HTTParty).to receive(:post) do |_url, _opts, _post_data, _context|