Merge pull request #20297 from Homebrew/test_bot_download_concurrency

workflows/tests: set HOMEBREW_DOWNLOAD_CONCURRENCY for test-bot tests.

Author: MikeMcQuaid

.github/workflows/tests.yml

@@ -368,6 +368,7 @@ jobs:
     env:
       HOMEBREW_TEST_BOT_ANALYTICS: 1
       HOMEBREW_ENFORCE_SBOM: 1
+      HOMEBREW_DOWNLOAD_CONCURRENCY: 4
     steps:
       - name: Install Homebrew and Homebrew's dependencies
         # All other images are built from our Homebrew Dockerfile.

Library/Homebrew/download_queue.rb

@@ -38,6 +38,8 @@ def fetch
         rescue ChecksumMismatchError => e
           opoo "#{downloadable.download_type} reports different checksum: #{e.expected}"
           Homebrew.failed = true if downloadable.is_a?(Resource::Patch)
+        rescue => e
+          raise e unless bottle_manifest_error?(downloadable, e)
         end
       else
         spinner = Spinner.new
@@ -68,6 +70,9 @@ def fetch
               raise future.state.to_s
             end
 
+            exception = future.reason if future.rejected?
+            next 1 if bottle_manifest_error?(downloadable, exception)
+
             message = "#{downloadable.download_type} #{downloadable.name}"
             if tty
               stdout_print_and_flush "#{status} #{message}#{"\n" unless last}"
@@ -76,14 +81,13 @@ def fetch
             end
 
             if future.rejected?
-              if (e = future.reason).is_a?(ChecksumMismatchError)
-                opoo "#{downloadable.download_type} reports different checksum: #{e.expected}"
+              if exception.is_a?(ChecksumMismatchError)
+                opoo "#{downloadable.download_type} reports different checksum: #{exception.expected}"
                 Homebrew.failed = true if downloadable.is_a?(Resource::Patch)
                 next 2
               else
                 message = future.reason.to_s
-                onoe message
-                Homebrew.failed = true
+                ofail message
                 next message.count("\n")
               end
             end
@@ -165,6 +169,13 @@ def shutdown
 
     private
 
+    sig { params(downloadable: Downloadable, exception: T.nilable(Exception)).returns(T::Boolean) }
+    def bottle_manifest_error?(downloadable, exception)
+      return false if exception.nil?
+
+      downloadable.is_a?(Resource::BottleManifest) || exception.is_a?(Resource::BottleManifest::Error)
+    end
+
     sig { void }
     def cancel
       # FIXME: Implement graceful cancellation of running downloads based on

Library/Homebrew/formula_installer.rb

@@ -320,6 +320,8 @@ def prelude_fetch
     # Needs to be done before expand_dependencies for compute_dependencies
     fetch_bottle_tab if pour_bottle?
 
+    fetch_fetch_deps unless ignore_deps?
+
     @ran_prelude_fetch = true
   end
 
@@ -346,7 +348,9 @@ def prelude
     forbidden_formula_check
 
     check_install_sanity
-    install_fetch_deps unless ignore_deps?
+
+    # with the download queue: these should have already been installed
+    install_fetch_deps if !ignore_deps? && download_queue.nil?
   end
 
   sig { void }
@@ -475,6 +479,18 @@ def check_install_sanity
   sig { params(_formula: Formula).returns(T.nilable(T::Boolean)) }
   def fresh_install?(_formula) = false
 
+  sig { void }
+  def fetch_fetch_deps
+    return if @compute_dependencies.blank?
+
+    compute_dependencies(use_cache: false) if @compute_dependencies.any? do |dep,|
+      next false unless dep.implicit?
+
+      fetch_dependencies
+      true
+    end
+  end
+
   sig { void }
   def install_fetch_deps
     return if @compute_dependencies.blank?
@@ -1455,8 +1471,6 @@ def fetch
     # We also skip bottle installs from local bottle paths, as these are done in CI
     # as part of the build lifecycle before attestations are produced.
     if check_attestation &&
-       # TODO: support this for download queues at some point
-       download_queue.nil? &&
        Homebrew::Attestation.enabled? &&
        formula.tap&.core_tap? &&
        formula.name != "gh"
@@ -1545,7 +1559,7 @@ def pour
       # download queue has already done the actual staging but we'll lie about
       # pouring now for nicer output
       ohai "Pouring #{downloadable.downloader.basename}"
-      downloadable.downloader.stage unless download_queue
+      downloadable.downloader.stage if download_queue.nil? || !formula.prefix.exist?
     end
 
     Tab.clear_cache

Library/Homebrew/retryable_download.rb

@@ -73,6 +73,7 @@ def fetch(verify_download_integrity: true, timeout: nil, quiet: false)
       downloadable.verify_download_integrity(download) if verify_download_integrity && !json_download
 
       if pour && downloadable.is_a?(Bottle)
+        HOMEBREW_CELLAR.mkpath
         UnpackStrategy.detect(download, prioritize_extension: true)
                       .extract_nestedly(to: HOMEBREW_CELLAR)
       elsif json_download