(PUP-9323) Allow agent and apply to lazily resolve deferred values

If `Puppet[:preprocess_deferrred]` is true (the default), then the `binary_file`
deferred function will fail to read the file, since it hasn't been created yet.

If the value is false, then the file is created before `binary_file` is called.

Author: joshcooper

lib/puppet/application/apply.rb

@@ -241,7 +241,7 @@ def main
         end
 
         # Resolve all deferred values and replace them / mutate the catalog
-        Puppet::Pops::Evaluator::DeferredResolver.resolve_and_replace(node.facts, catalog, apply_environment)
+        Puppet::Pops::Evaluator::DeferredResolver.resolve_and_replace(node.facts, catalog, apply_environment, Puppet[:preprocess_deferred])
 
         # Translate it to a RAL catalog
         catalog = catalog.to_ral
@@ -350,7 +350,7 @@ def read_catalog(text)
         raise Puppet::Error, _("Could not deserialize catalog from %{format}: %{detail}") % { format: format, detail: detail }, detail.backtrace
       end
       # Resolve all deferred values and replace them / mutate the catalog
-      Puppet::Pops::Evaluator::DeferredResolver.resolve_and_replace(node.facts, catalog, configured_environment)
+      Puppet::Pops::Evaluator::DeferredResolver.resolve_and_replace(node.facts, catalog, configured_environment, Puppet[:preprocess_deferred])
 
       catalog.to_ral
     end

lib/puppet/configurer.rb

@@ -112,7 +112,7 @@ def convert_catalog(result, duration, facts, options = {})
     catalog_conversion_time = thinmark do
       # Will mutate the result and replace all Deferred values with resolved values
       if facts
-        Puppet::Pops::Evaluator::DeferredResolver.resolve_and_replace(facts, result, Puppet.lookup(:current_environment))
+        Puppet::Pops::Evaluator::DeferredResolver.resolve_and_replace(facts, result, Puppet.lookup(:current_environment), Puppet[:preprocess_deferred])
       end
 
       catalog = result.to_ral

lib/puppet/defaults.rb

@@ -2021,6 +2021,16 @@ def self.initialize_default_settings!(settings)
         being evaluated.  This allows you to interactively see exactly
         what is being done.",
     },
+    :preprocess_deferred => {
+      :default => true,
+      :type => :boolean,
+      :desc => "Whether puppet should call deferred functions before applying
+        the catalog. If set to `true`, then all prerequisites needed for the
+        deferred function must be satified prior to puppet running. If set to
+        `false`, then deferred functions will follow puppet relationships and
+        ordering. This allows puppet to install prerequisites needed for a
+        deferred function and call the deferred function in the same run."
+    },
     :summarize => {
         :default  => false,
         :type     => :boolean,

spec/integration/application/agent_spec.rb

@@ -98,6 +98,18 @@
   end
 
   context 'rich data' do
+    let(:deferred_file) { tmpfile('deferred') }
+    let(:deferred_manifest) do <<~END
+      file { '#{deferred_file}':
+        ensure => file,
+        content => '123',
+      } ->
+      notify { 'deferred':
+        message => Deferred('binary_file', ['#{deferred_file}'])
+      }
+      END
+    end
+
     it "calls a deferred 4x function" do
       catalog_handler = -> (req, res) {
         catalog = compile_to_catalog(<<-MANIFEST, node)
@@ -142,6 +154,43 @@
       end
     end
 
+    it "fails to apply a deferred function with an unsatified prerequisite" do
+      catalog_handler = -> (req, res) {
+        catalog = compile_to_catalog(deferred_manifest, node)
+        res.body = formatter.render(catalog)
+        res['Content-Type'] = formatter.mime
+      }
+
+      server.start_server(mounts: {catalog: catalog_handler}) do |port|
+        Puppet[:serverport] = port
+        expect {
+          agent.command_line.args << '--test'
+          agent.run
+        }.to exit_with(1)
+         .and output(%r{Using environment}).to_stdout
+         .and output(%r{The given file '#{deferred_file}' does not exist}).to_stderr
+      end
+    end
+
+    it "applies a deferred function and its prerequisite in the same run" do
+      Puppet[:preprocess_deferred] = false
+
+      catalog_handler = -> (req, res) {
+        catalog = compile_to_catalog(deferred_manifest, node)
+        res.body = formatter.render(catalog)
+        res['Content-Type'] = formatter.mime
+      }
+
+      server.start_server(mounts: {catalog: catalog_handler}) do |port|
+        Puppet[:serverport] = port
+        expect {
+          agent.command_line.args << '--test'
+          agent.run
+        }.to exit_with(2)
+         .and output(%r{defined 'message' as Binary\("MTIz"\)}).to_stdout
+      end
+    end
+
     it "re-evaluates a deferred function in a cached catalog" do
       Puppet[:report] = false
       Puppet[:use_cached_catalog] = true

spec/integration/application/apply_spec.rb

@@ -665,6 +665,18 @@ def bogus()
   end
 
   context 'rich data' do
+    let(:deferred_file) { tmpfile('deferred') }
+    let(:deferred_manifest) do <<~END
+      file { '#{deferred_file}':
+        ensure => file,
+        content => '123',
+      } ->
+      notify { 'deferred':
+        message => Deferred('binary_file', ['#{deferred_file}'])
+      }
+      END
+    end
+
     it "calls a deferred 4x function" do
       apply.command_line.args = ['-e', 'notify { "deferred3x": message => Deferred("join", [[1,2,3], ":"]) }']
 
@@ -681,5 +693,25 @@ def bogus()
       }.to exit_with(0) # for some reason apply returns 0 instead of 2
        .and output(%r{Notice: /Stage\[main\]/Main/Notify\[deferred4x\]/message: defined 'message' as 'I am deferred'}).to_stdout
     end
+
+    it "fails to apply a deferred function with an unsatified prerequisite" do
+      apply.command_line.args = ['-e', deferred_manifest]
+      expect {
+        apply.run
+      }.to exit_with(1) # for some reason apply returns 0 instead of 2
+       .and output(/Compiled catalog/).to_stdout
+       .and output(%r{The given file '#{deferred_file}' does not exist}).to_stderr
+    end
+
+    it "applies a deferred function and its prerequisite in the same run" do
+      Puppet[:preprocess_deferred] = false
+
+      apply.command_line.args = ['-e', deferred_manifest]
+      expect {
+        apply.run
+      }.to exit_with(0) # for some reason apply returns 0 instead of 2
+        .and output(%r{defined 'message' as Binary\("MTIz"\)}).to_stdout
+    end
+
   end
 end