(PUP-10946) add max-files parameter for file and tidy resources

Adds a new parameter, `max-files`, to limit the number of resources
that may be created by types supporting recursion(file, tidy).

Default value for `max-files` is 0. In this case a warning is logged
in case the number of files exceeds hardcoded soft limit of 1000.

In case `max-files` is set to a value different that 0 and the number
of files is greater than `max-files`, a Puppet::Error is raised and the
resource is marked as failed.

Author: Ciprian Badescu

lib/puppet/file_serving/fileset.rb

@@ -5,7 +5,7 @@
 # Operate recursively on a path, returning a set of file paths.
 class Puppet::FileServing::Fileset
   attr_reader :path, :ignore, :links
-  attr_accessor :recurse, :recurselimit, :checksum_type
+  attr_accessor :recurse, :recurselimit, :max_files, :checksum_type
 
   # Produce a hash of files, with merged so that earlier files
   # with the same postfix win.  E.g., /dir1/subfile beats /dir2/subfile.
@@ -40,6 +40,7 @@ def initialize(path, options = {})
     self.links = :manage
     @recurse = false
     @recurselimit = :infinite
+    @max_files = 0
 
     if options.is_a?(Puppet::Indirector::Request)
       initialize_from_request(options)
@@ -58,6 +59,13 @@ def initialize(path, options = {})
   # level deep, which Find doesn't do.
   def files
     files = perform_recursion
+    soft_max_files = 1000
+
+    if max_files != 0  && files.size > max_files
+      raise Puppet::Error.new _("The directory '%{path}' contains %{entries} entries, which exceeds the limit of %{max_files} specified by the max_files parameter for this resource. The limit may be increased, but be aware that large number of file resources can result in excessive resource consumption and degraded performance. Consider using an alternate method to manage large directory trees") % { path: path, entries: files.size, max_files: max_files }
+    elsif max_files == 0 && files.size > soft_max_files
+      Puppet.warning _("The directory '%{path}' contains %{entries} entries, which exceeds the default soft limit %{max_files} and may cause excessive resource consumption and degraded performance. To remove this warning set a value for `max_files` parameter or consider using an alternate method to manage large directory trees") % { path: path, entries: files.size, max_files: soft_max_files }
+    end
 
     # Now strip off the leading path, so each file becomes relative, and remove
     # any slashes that might end up at the beginning of the path.
@@ -96,7 +104,7 @@ def initialize_from_hash(options)
   end
 
   def initialize_from_request(request)
-    [:links, :ignore, :recurse, :recurselimit, :checksum_type].each do |param|
+    [:links, :ignore, :recurse, :recurselimit, :max_files, :checksum_type].each do |param|
       if request.options.include?(param) # use 'include?' so the values can be false
         value = request.options[param]
       elsif request.options.include?(param.to_s)

lib/puppet/http/service/file_server.rb

@@ -106,7 +106,7 @@ def get_file_metadata(path:, environment:, links: :manage, checksum_type: Puppet
   #   An array with the request response and an array of the deserialized
   #   metadata for each file returned from the server
   #
-  def get_file_metadatas(path: nil, environment:, recurse: :false, recurselimit: nil, ignore: nil, links: :manage, checksum_type: Puppet[:digest_algorithm], source_permissions: :ignore)
+  def get_file_metadatas(path: nil, environment:, recurse: :false, recurselimit: nil, max_files: nil, ignore: nil, links: :manage, checksum_type: Puppet[:digest_algorithm], source_permissions: :ignore)
     validate_path(path)
 
     headers = add_puppet_headers('Accept' => get_mime_types(Puppet::FileServing::Metadata).join(', '))
@@ -117,6 +117,7 @@ def get_file_metadatas(path: nil, environment:, recurse: :false, recurselimit: n
       params: {
         recurse: recurse,
         recurselimit: recurselimit,
+        max_files: max_files,
         ignore: ignore,
         links: links,
         checksum_type: checksum_type,

lib/puppet/indirector/catalog/compiler.rb

@@ -194,6 +194,7 @@ def inline_metadata(catalog, checksum_type)
           :source_permissions => resource[:source_permissions] ? resource[:source_permissions].to_sym : :ignore,
           :recurse            => true,
           :recurselimit       => resource[:recurselimit],
+          :max_files          => resource[:max_files],
           :ignore             => resource[:ignore],
         }
 

lib/puppet/indirector/file_metadata/rest.rb

@@ -46,6 +46,7 @@ def search(request)
       environment: request.environment.to_s,
       recurse: request.options[:recurse],
       recurselimit: request.options[:recurselimit],
+      max_files: request.options[:max_files],
       ignore: request.options[:ignore],
       links: request.options[:links],
       checksum_type: request.options[:checksum_type],

lib/puppet/type/file.rb

@@ -220,6 +220,22 @@ def self.title_patterns
     end
   end
 
+  newparam(:max_files) do
+    desc "In case the resource is a directory and the recursion is enabled, puppet will
+      generate a new resource for each file file found, possible leading to
+      an excessive number of resources generated without any control.
+
+      Setting `max_files` will check the number of file resources that
+      will eventually be created and will raise a resource argument error if the
+      limit will be exceeded.
+
+      Use value `0` to disable the check. In this case, a warning is logged if
+      the number of files exceeds 1000."
+
+    defaultto 0
+    newvalues(/^[0-9]+$/)
+  end
+
   newparam(:replace, :boolean => true, :parent => Puppet::Parameter::Boolean) do
     desc "Whether to replace a file or symlink that already exists on the local system but
       whose content doesn't match what the `source` or `content` attribute
@@ -576,7 +592,7 @@ def newchild(path)
     options = @original_parameters.merge(:path => full_path).reject { |param, value| value.nil? }
 
     # These should never be passed to our children.
-    [:parent, :ensure, :recurse, :recurselimit, :target, :alias, :source].each do |param|
+    [:parent, :ensure, :recurse, :recurselimit, :max_files, :target, :alias, :source].each do |param|
       options.delete(param) if options.include?(param)
     end
 
@@ -753,6 +769,7 @@ def perform_recursion(path)
       :links => self[:links],
       :recurse => (self[:recurse] == :remote ? true : self[:recurse]),
       :recurselimit => self[:recurselimit],
+      :max_files => self[:max_files],
       :source_permissions => self[:source_permissions],
       :ignore => self[:ignore],
       :checksum_type => (self[:source] || self[:content]) ? self[:checksum] : :none,

lib/puppet/type/tidy.rb

@@ -50,6 +50,22 @@
     end
   end
 
+  newparam(:max_files) do
+    desc "In case the resource is a directory and the recursion is enabled, puppet will
+      generate a new resource for each file file found, possible leading to
+      an excessive number of resources generated without any control.
+
+      Setting `max_files` will check the number of file resources that
+      will eventually be created and will raise a resource argument error if the
+      limit will be exceeded.
+
+      Use value `0` to disable the check. In this case, a warning is logged if
+      the number of files exceeds 1000."
+
+    defaultto 0
+    newvalues(/^[0-9]+$/)
+  end
+
   newparam(:matches) do
     desc <<-'EOT'
       One or more (shell type) file glob patterns, which restrict
@@ -256,9 +272,12 @@ def generate
 
     case self[:recurse]
     when Integer, /^\d+$/
-      parameter = { :recurse => true, :recurselimit => self[:recurse] }
+      parameter = { :max_files => self[:max_files],
+                    :recurse => true,
+                    :recurselimit => self[:recurse] }
     when true, :true, :inf
-      parameter = { :recurse => true }
+      parameter = { :max_files => self[:max_files],
+                    :recurse => true }
     end
 
     if parameter

spec/unit/file_serving/fileset_spec.rb

@@ -46,6 +46,13 @@
       expect(set.recurselimit).to eq(3)
     end
 
+    it "accepts a 'max_files' option" do
+      expect(Puppet::FileSystem).to receive(:lstat).with(somefile).and_return(double('stat'))
+      set = Puppet::FileServing::Fileset.new(somefile, :recurselimit => 3, :max_files => 100)
+      expect(set.recurselimit).to eq(3)
+      expect(set.max_files).to eq(100)
+    end
+
     it "accepts an 'ignore' option" do
       expect(Puppet::FileSystem).to receive(:lstat).with(somefile).and_return(double('stat'))
       set = Puppet::FileServing::Fileset.new(somefile, :ignore => ".svn")
@@ -160,6 +167,29 @@ def mock_dir_structure(path, stat_method = :lstat)
       end
     end
 
+    def mock_big_dir_structure(path, stat_method = :lstat)
+      allow(Puppet::FileSystem).to receive(stat_method).with(path).and_return(@dirstat)
+
+      # Keep track of the files we're stubbing.
+      @files = %w{.}
+
+      top_names = (1..10).map {|i| "dir_#{i}" }
+      sub_names = (1..100).map {|i| "file__#{i}" }
+
+      allow(Dir).to receive(:entries).with(path, encoding: Encoding::UTF_8).and_return(top_names)
+      top_names.each do |subdir|
+        @files << subdir # relative path
+        subpath = File.join(path, subdir)
+        allow(Puppet::FileSystem).to receive(stat_method).with(subpath).and_return(@dirstat)
+        allow(Dir).to receive(:entries).with(subpath, encoding: Encoding::UTF_8).and_return(sub_names)
+        sub_names.each do |file|
+          @files << File.join(subdir, file) # relative path
+          subfile_path = File.join(subpath, file)
+          allow(Puppet::FileSystem).to receive(stat_method).with(subfile_path).and_return(@filestat)
+        end
+      end
+    end
+
     def setup_mocks_for_dir(mock_dir, base_path)
       path = File.join(base_path, mock_dir.name)
       allow(Puppet::FileSystem).to receive(:lstat).with(path).and_return(MockStat.new(path, true))
@@ -258,6 +288,20 @@ def directory?
       expect(@fileset.files.find { |file| file.include?("0") }).to be_nil
     end
 
+    it "raises exception if number of files is greater than :max_files" do
+      mock_dir_structure(@path)
+      @fileset.recurse = true
+      @fileset.max_files = 22
+      expect { @fileset.files }.to raise_error(Puppet::Error, "The directory '#{@path}' contains 28 entries, which exceeds the limit of 22 specified by the max_files parameter for this resource. The limit may be increased, but be aware that large number of file resources can result in excessive resource consumption and degraded performance. Consider using an alternate method to manage large directory trees")
+    end
+
+    it "logs a warning if number of files is greater than soft max_files limit of 1000" do
+      mock_big_dir_structure(@path)
+      @fileset.recurse = true
+      expect(Puppet).to receive(:warning).with("The directory '#{@path}' contains 1010 entries, which exceeds the default soft limit 1000 and may cause excessive resource consumption and degraded performance. To remove this warning set a value for `max_files` parameter or consider using an alternate method to manage large directory trees")
+      expect { @fileset.files }.to_not raise_error
+    end
+
     it "ignores files that match a pattern given as a boolean" do
       mock_dir_structure(@path)
       @fileset.recurse = true

spec/unit/indirector/catalog/compiler_spec.rb

@@ -909,9 +909,10 @@ def stubs_directory_metadata(relative_path)
         it "inlines child metadata" do
           catalog = compile_to_catalog(<<-MANIFEST, node)
             file { '#{path}':
-              ensure  => directory,
-              recurse => true,
-              source  => '#{source_dir}'
+              ensure    => directory,
+              recurse   => true,
+              source    => '#{source_dir}',
+              max_files => 1234,
             }
           MANIFEST
 
@@ -925,6 +926,7 @@ def stubs_directory_metadata(relative_path)
             :source_permissions => :ignore,
             :recurse            => true,
             :recurselimit       => nil,
+            :max_files          => 1234,
             :ignore             => nil,
           }
           expect(Puppet::FileServing::Metadata.indirection).to receive(:search).with(source_dir, options).and_return([metadata, child_metadata])
@@ -938,14 +940,15 @@ def stubs_directory_metadata(relative_path)
         it "uses resource parameters when inlining metadata" do
           catalog = compile_to_catalog(<<-MANIFEST, node)
             file { '#{path}':
-              ensure  => directory,
-              recurse => true,
-              source  => '#{source_dir}',
-              checksum => sha256,
+              ensure             => directory,
+              recurse            => true,
+              source             => '#{source_dir}',
+              checksum           => sha256,
               source_permissions => use_when_creating,
-              recurselimit => 2,
-              ignore => 'foo.+',
-              links => follow,
+              recurselimit       => 2,
+              max_files          => 4321,
+              ignore             => 'foo.+',
+              links              => follow,
             }
           MANIFEST
 
@@ -956,6 +959,7 @@ def stubs_directory_metadata(relative_path)
             :source_permissions => :use_when_creating,
             :recurse            => true,
             :recurselimit       => 2,
+            :max_files          => 4321,
             :ignore             => 'foo.+',
           }
           expect(Puppet::FileServing::Metadata.indirection).to receive(:search).with(source_dir, options).and_return([metadata, child_metadata])

spec/unit/type/tidy_spec.rb

@@ -195,17 +195,27 @@
         allow(Puppet::FileServing::Fileset).to receive(:new).and_return(@fileset)
       end
 
-      it "should use a Fileset for infinite recursion" do
-        expect(Puppet::FileServing::Fileset).to receive(:new).with(@basepath, :recurse => true).and_return(@fileset)
+      it "should use a Fileset with default max_files for infinite recursion" do
+        expect(Puppet::FileServing::Fileset).to receive(:new).with(@basepath, :recurse => true, :max_files=>0).and_return(@fileset)
         expect(@fileset).to receive(:files).and_return(%w{. one two})
         allow(@tidy).to receive(:tidy?).and_return(false)
 
         @tidy.generate
       end
 
-      it "should use a Fileset for limited recursion" do
+      it "should use a Fileset with default max_files for limited recursion" do
         @tidy[:recurse] = 42
-        expect(Puppet::FileServing::Fileset).to receive(:new).with(@basepath, :recurse => true, :recurselimit => 42).and_return(@fileset)
+        expect(Puppet::FileServing::Fileset).to receive(:new).with(@basepath, :recurse => true, :recurselimit => 42, :max_files=>0).and_return(@fileset)
+        expect(@fileset).to receive(:files).and_return(%w{. one two})
+        allow(@tidy).to receive(:tidy?).and_return(false)
+
+        @tidy.generate
+      end
+
+      it "should use a Fileset with max_files for limited recursion" do
+        @tidy[:recurse] = 42
+        @tidy[:max_files] = 9876
+        expect(Puppet::FileServing::Fileset).to receive(:new).with(@basepath, :recurse => true, :recurselimit => 42, :max_files=>9876).and_return(@fileset)
         expect(@fileset).to receive(:files).and_return(%w{. one two})
         allow(@tidy).to receive(:tidy?).and_return(false)
 
@@ -411,7 +421,7 @@
       @tidy[:recurse] = true
       @tidy[:rmdirs] = true
       fileset = double('fileset')
-      expect(Puppet::FileServing::Fileset).to receive(:new).with(@basepath, :recurse => true).and_return(fileset)
+      expect(Puppet::FileServing::Fileset).to receive(:new).with(@basepath, :recurse => true, :max_files=>0).and_return(fileset)
       expect(fileset).to receive(:files).and_return(%w{. one two one/subone two/subtwo one/subone/ssone})
       allow(@tidy).to receive(:tidy?).and_return(true)
 
@@ -433,7 +443,7 @@
       @tidy[:recurse] = true
       @tidy[:rmdirs] = true
       fileset = double('fileset')
-      expect(Puppet::FileServing::Fileset).to receive(:new).with(@basepath, :recurse => true).and_return(fileset)
+      expect(Puppet::FileServing::Fileset).to receive(:new).with(@basepath, :recurse => true, :max_files=>0).and_return(fileset)
       expect(fileset).to receive(:files).and_return(%w{. a a/2 a/1 a/3})
       allow(@tidy).to receive(:tidy?).and_return(true)
 
@@ -446,7 +456,7 @@
       @tidy[:noop] = true
 
       fileset = double('fileset')
-      expect(Puppet::FileServing::Fileset).to receive(:new).with(@basepath, :recurse => true).and_return(fileset)
+      expect(Puppet::FileServing::Fileset).to receive(:new).with(@basepath, :recurse => true, :max_files=>0).and_return(fileset)
       expect(fileset).to receive(:files).and_return(%w{. a a/2 a/1 a/3})
       allow(@tidy).to receive(:tidy?).and_return(true)