Use Dir.scan if available

Author: fxn

lib/zeitwerk/loader.rb

@@ -5,6 +5,7 @@
 
 module Zeitwerk
   class Loader
+    require_relative "loader/scanner"
     require_relative "loader/helpers"
     require_relative "loader/callbacks"
     require_relative "loader/config"
@@ -13,8 +14,9 @@ class Loader
     extend Internal
 
     include RealModName
-    include Callbacks
+    include Scanner
     include Helpers
+    include Callbacks
     include Config
     include EagerLoad
 
@@ -168,7 +170,7 @@ def unload
             # and the constant path would escape unloadable_cpath? This is just
             # defensive code to clean things up as much as we are able to.
             unload_cref(cref)
-            unloaded_files.add(abspath) if ruby?(abspath)
+            unloaded_files.add(abspath) if rb_extension?(abspath)
           end
         end
 
@@ -186,7 +188,7 @@ def unload
           end
 
           unload_cref(cref)
-          unloaded_files.add(abspath) if ruby?(abspath)
+          unloaded_files.add(abspath) if rb_extension?(abspath)
         end
 
         unless unloaded_files.empty?
@@ -482,7 +484,7 @@ def all_dirs
     #: (Zeitwerk::Cref, String) -> void
     private def autoload_subdir(cref, subdir)
       if autoload_path = autoload_path_set_by_me_for?(cref)
-        if ruby?(autoload_path)
+        if rb_extension?(autoload_path)
           # Scanning visited a Ruby file first, and now a directory for the same
           # constant has been found. This means we are dealing with an explicit
           # namespace whose definition was seen first.
@@ -512,7 +514,7 @@ def all_dirs
     private def autoload_file(cref, file)
       if autoload_path = cref.autoload? || Registry.inceptions.registered?(cref)
         # First autoload for a Ruby file wins, just ignore subsequent ones.
-        if ruby?(autoload_path)
+        if rb_extension?(autoload_path)
           shadowed_files << file
           log("file #{file} is ignored because #{autoload_path} has precedence") if logger
         else
@@ -547,7 +549,7 @@ def all_dirs
       cref.autoload(abspath)
 
       if logger
-        if ruby?(abspath)
+        if rb_extension?(abspath)
           log("autoload set for #{cref}, to be loaded from #{abspath}")
         else
           log("autoload set for #{cref}, to be autovivified from #{abspath}")

lib/zeitwerk/loader/eager_load.rb

@@ -116,7 +116,7 @@ def load_file(path)
     abspath = File.expand_path(path)
 
     raise Zeitwerk::Error.new("#{abspath} does not exist") unless File.exist?(abspath)
-    raise Zeitwerk::Error.new("#{abspath} is not a Ruby file") if !ruby?(abspath)
+    raise Zeitwerk::Error.new("#{abspath} is not a Ruby file") if !rb_extension?(abspath)
     raise Zeitwerk::Error.new("#{abspath} is ignored") if ignored_path?(abspath)
 
     file_basename = File.basename(abspath, ".rb")

lib/zeitwerk/loader/helpers.rb

@@ -1,130 +1,15 @@
 # frozen_string_literal: true
 
 module Zeitwerk::Loader::Helpers
-  # --- Logging -----------------------------------------------------------------------------------
+  CNAME_VALIDATOR = Module.new #: Module
+  private_constant :CNAME_VALIDATOR
 
   #: (to_s() -> String) -> void
   private def log(message)
     method_name = logger.respond_to?(:debug) ? :debug : :call
     logger.send(method_name, "Zeitwerk@#{tag}: #{message}")
   end
 
-  # --- Files and directories ---------------------------------------------------------------------
-
-  #: (String) { (String, String, Symbol) -> void } -> void
-  private def ls(dir)
-    children = scan_dir(dir)
-
-    # The order in which a directory is listed depends on the file system.
-    #
-    # Since client code may run on different platforms, it seems convenient to
-    # sort directory entries. This provides more deterministic behavior, with
-    # consistent eager loading in particular.
-    children.sort_by!(&:first)
-
-    children.each do |basename, abspath, ftype|
-      if ftype == :directory && !has_at_least_one_ruby_file?(abspath)
-        log("directory #{abspath} is ignored because it has no Ruby files") if logger
-        next
-      end
-
-      yield basename, abspath, ftype
-    end
-  end
-
-  # Looks for a Ruby file using breadth-first search. This type of search is
-  # important to list as less directories as possible and return fast in the
-  # common case in which there are Ruby files.
-  #
-  #: (String) -> bool
-  private def has_at_least_one_ruby_file?(dir)
-    to_visit = [dir]
-
-    while (dir = to_visit.shift)
-      scan_dir(dir) do |_, abspath, ftype|
-        return true if ftype == :file
-        to_visit << abspath
-      end
-    end
-
-    false
-  end
-
-  # This is a low-level method to scan directories. It filters out some stuff
-  # the loader is never interested in, and passes the ftype up. The rest of the
-  # library should generally use `ls`.
-  #
-  # Keep an eye on https://bugs.ruby-lang.org/issues/21800.
-  #
-  #: (String) { (String, String, Symbol) -> void } -> void
-  #: (String) -> [[String, String, Symbol]]
-  private def scan_dir(dir)
-    children = [] unless block_given?
-
-    Dir.each_child(dir) do |basename|
-      next if hidden?(basename)
-
-      abspath = File.join(dir, basename)
-      next if ignored_path?(abspath)
-
-      ftype = supported_ftype?(abspath)
-      next unless ftype
-
-      # Conceptually, root directories start separate trees.
-      next if :directory == ftype && root_dir?(abspath)
-
-      # We freeze abspath because that saves allocations when passed later to
-      # File methods. See https://github.com/fxn/zeitwerk/pull/125.
-      if block_given?
-        yield basename, abspath.freeze, ftype
-      else
-        children << [basename, abspath.freeze, ftype]
-      end
-    end
-
-    children unless block_given?
-  end
-
-  # Encodes the documented conventions.
-  #
-  #: (String) -> Symbol?
-  private def supported_ftype?(abspath)
-    if ruby?(abspath)
-      :file # By convention, we can avoid a syscall here.
-    elsif dir?(abspath)
-      :directory
-    end
-  end
-
-  #: (String) -> bool
-  private def ruby?(path)
-    path.end_with?(".rb")
-  end
-
-  #: (String) -> bool
-  private def dir?(path)
-    File.directory?(path)
-  end
-
-  #: (String) -> bool
-  private def hidden?(basename)
-    basename.start_with?(".")
-  end
-
-  #: (String) { (String) -> void } -> void
-  private def walk_up(abspath)
-    loop do
-      yield abspath
-      abspath, basename = File.split(abspath)
-      break if basename == "/"
-    end
-  end
-
-  # --- Inflection --------------------------------------------------------------------------------
-
-  CNAME_VALIDATOR = Module.new #: Module
-  private_constant :CNAME_VALIDATOR
-
   #: (String, String) -> Symbol ! Zeitwerk::NameError
   private def cname_for(basename, abspath)
     cname = inflector.camelize(basename, abspath)
@@ -146,7 +31,7 @@ module Zeitwerk::Loader::Helpers
     begin
       CNAME_VALIDATOR.const_defined?(cname, false)
     rescue ::NameError => error
-      path_type = ruby?(abspath) ? "file" : "directory"
+      path_type = rb_extension?(abspath) ? "file" : "directory"
 
       raise Zeitwerk::NameError.new(<<~MESSAGE, error.name)
         #{error.message} inferred by #{inflector.class} from #{path_type}

lib/zeitwerk/loader/scanner.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+module Zeitwerk::Loader::Scanner
+  #: (String) { (String, String, Symbol) -> void } -> void
+  private def ls(dir)
+    children = relevant_dir_entries(dir)
+
+    # The order in which a directory is listed depends on the file system.
+    #
+    # Since client code may run on different platforms, it seems convenient to
+    # sort directory entries. This provides more deterministic behavior, with
+    # consistent eager loading in particular.
+    children.sort_by!(&:first)
+
+    children.each do |basename, abspath, ftype|
+      if :directory == ftype && !has_at_least_one_ruby_file?(abspath)
+        log("directory #{abspath} is ignored because it has no Ruby files") if logger
+        next
+      end
+
+      yield basename, abspath, ftype
+    end
+  end
+
+  # Looks for a Ruby file using breadth-first search. This type of search is
+  # important to list as less directories as possible and return fast in the
+  # common case in which there are Ruby files in the passed directory.
+  #
+  #: (String) -> bool
+  private def has_at_least_one_ruby_file?(dir)
+    to_visit = [dir]
+
+    while (dir = to_visit.shift)
+      relevant_dir_entries(dir) do |_, abspath, ftype|
+        return true if :file == ftype
+        to_visit << abspath
+      end
+    end
+
+    false
+  end
+
+  #: (String) { (String, String, Symbol) -> void } -> void
+  #: (String) -> [[String, String, Symbol]]
+  private def relevant_dir_entries(dir)
+    return enum_for(__method__, dir).to_a unless block_given?
+
+    each_ruby_file_or_directory(dir) do |basename, abspath, ftype|
+      next if ignored_path?(abspath)
+
+      if :link == ftype
+        begin
+          ftype = File.stat(abspath).ftype.to_sym
+        rescue Errno::ENOENT
+          warn "ignoring broken symlink #{abspath}"
+          next
+        end
+      end
+
+      if :file == ftype
+        yield basename, abspath, ftype if rb_extension?(basename)
+      elsif :directory == ftype
+        # Conceptually, root directories represent a separate project tree.
+        yield basename, abspath, ftype unless root_dir?(abspath)
+      end
+    end
+  end
+
+  # Dir.scan is more efficient in common platforms, but it is going to take a
+  # while for it to be available.
+  #
+  # The following compatibility methods have the same semantics but are written
+  # to favor the performance of the Ruby fallback, which can save syscalls.
+  #
+  # In particular, by convention, any directory entry with a .rb extension is
+  # assumed to be a file or a symlink to a file.
+  #
+  # These methods also freeze abspaths because that saves allocations when
+  # passed later to File methods. See https://github.com/fxn/zeitwerk/pull/125.
+
+  if Dir.respond_to?(:scan) # Available in Ruby 4.1.
+    #: (String) { (String, String, Symbol) -> void } -> void
+    private def each_ruby_file_or_directory(dir)
+      Dir.scan(dir) do |basename, ftype|
+        next if hidden?(basename)
+
+        if rb_extension?(basename)
+          abspath = File.join(dir, basename).freeze
+          yield basename, abspath, :file # By convention.
+        elsif :directory == ftype
+          abspath = File.join(dir, basename).freeze
+          yield basename, abspath, :directory
+        elsif :link == ftype
+          abspath = File.join(dir, basename).freeze
+          yield basename, abspath, :directory if dir?(abspath)
+        end
+      end
+    end
+  else
+    #: (String) { (String, String, Symbol) -> void } -> void
+    private def each_ruby_file_or_directory(dir)
+      Dir.each_child(dir) do |basename|
+        next if hidden?(basename)
+
+        if rb_extension?(basename)
+          abspath = File.join(dir, basename).freeze
+          yield basename, abspath, :file # By convention.
+        else
+          abspath = File.join(dir, basename).freeze
+          if dir?(abspath)
+            yield basename, abspath, :directory
+          end
+        end
+      end
+    end
+  end
+
+  # Encodes the documented conventions.
+  #
+  #: (String) -> Symbol?
+  private def supported_ftype?(abspath)
+    if rb_extension?(abspath)
+      :file # By convention, we can avoid a syscall here.
+    elsif dir?(abspath)
+      :directory
+    end
+  end
+
+  #: (String) -> bool
+  private def rb_extension?(path)
+    path.end_with?(".rb")
+  end
+
+  #: (String) -> bool
+  private def dir?(path)
+    File.directory?(path)
+  end
+
+  #: (String) -> bool
+  private def hidden?(basename)
+    basename.start_with?(".")
+  end
+
+  #: (String) { (String) -> void } -> void
+  private def walk_up(abspath)
+    loop do
+      yield abspath
+      abspath, basename = File.split(abspath)
+      break if basename == "/"
+    end
+  end
+end