[DevSetting] Add install_proxy_url which will allow ParallelCluster to set Proxy environment for Build Image installation

* Adding a recipe to setup proxy env as part of cookbook to avoid Sizing issue in parallelcluster.yaml
* Add s3 global endpoint to overcome cfn_bootstrap script installation
* Add s3 enpoint for no proxy list so it goes to VPC endpoint
* addding old style s3 naming convention bucket-name.s3-eu-west-1.amazonaws.com to handle S3 redirecting (307 Temporary Redirect) when we use non-us-east-1 region
* Add cdn for amazonlinux in no_proxy list to re-direct to VPC endpoint
* add unit test

Author: Himani Anil Deshpande

cookbooks/aws-parallelcluster-entrypoints/recipes/install.rb

@@ -17,6 +17,7 @@
 return if node['conditions']['ami_bootstrapped']
 
 include_recipe "aws-parallelcluster-shared::setup_envars"
+include_recipe "aws-parallelcluster-shared::detect_proxy" if node['cluster']['install_proxy_url']
 
 include_recipe 'aws-parallelcluster-platform::install'
 include_recipe 'aws-parallelcluster-environment::install'

cookbooks/aws-parallelcluster-entrypoints/spec/unit/recipes/install_spec.rb

@@ -23,9 +23,11 @@
     aws-parallelcluster-awsbatch::install
   )
 
+  detect_proxy_recipe = 'aws-parallelcluster-shared::detect_proxy'
+
   before do
     @included_recipes = []
-    all_recipes.each do |recipe_name|
+    (all_recipes + [detect_proxy_recipe]).each do |recipe_name|
       allow_any_instance_of(Chef::Recipe).to receive(:include_recipe).with(recipe_name) do
         @included_recipes << recipe_name
       end
@@ -61,6 +63,7 @@
           it "includes all recipes in the right order" do
             chef_run
             expect(@included_recipes).to eq(all_recipes)
+            expect(@included_recipes).not_to include(detect_proxy_recipe)
           end
         end
 
@@ -78,6 +81,22 @@
             expect(@included_recipes).to eq(all_recipes - %w(aws-parallelcluster-awsbatch::install))
           end
         end
+
+        context "when install_proxy_url is set" do
+          cached(:chef_run) do
+            runner = runner(platform: platform, version: version) do |node|
+              node.override['conditions']['ami_bootstrapped'] = false
+              node.override['cluster']['skip_awsbatch_cli_install'] = false
+              node.override['cluster']['install_proxy_url'] = 'http://10.0.0.109:8888'
+            end
+            runner.converge(described_recipe)
+          end
+
+          it "includes detect_proxy recipe" do
+            chef_run
+            expect(@included_recipes).to include(detect_proxy_recipe)
+          end
+        end
       end
     end
   end

cookbooks/aws-parallelcluster-shared/recipes/detect_proxy.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+#
+# Cookbook:: aws-parallelcluster
+# Recipe:: detect_proxy
+#
+# Copyright:: 2026 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the
+# License. A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This recipe configures proxy environment variables for build-image in isolated networks.
+#
+# It reads the proxy URL from node['cluster']['install_proxy_url'] (set via ExtraChefAttributes)
+# and configures http_proxy/https_proxy ENV vars for the Chef run. This makes all subsequent
+# Chef resources (remote_file, bash, execute, etc.) use the explicit proxy for HTTPS traffic
+# instead of trying direct connections that would fail in an isolated network.
+#
+# The no_proxy list excludes S3 endpoints so downloads from S3 go through the VPC Gateway
+# Endpoint directly, not through the proxy.
+# S3 endpoints are excluded so cookbook/dependency downloads from S3 go through
+# the S3 VPC Gateway Endpoint directly, not through the proxy.
+# Both regional (s3.{region}.amazonaws.com) and global (s3.amazonaws.com) endpoints
+# are included because some resources use the global endpoint (e.g., cloudformation-examples
+# bucket uses https://s3.amazonaws.com/cloudformation-examples/...).
+# Note: only the regional S3 endpoint is in no_proxy because the S3 VPC Gateway Endpoint
+# handles regional endpoints correctly. The global s3.amazonaws.com endpoint does NOT work
+# through the VPC Gateway Endpoint (SSL errors), so it is intentionally left out of no_proxy
+# and instead goes through the proxy which has internet access. The proxy allowlist in
+# proxy_stack.yaml must include s3.amazonaws.com for this to work.
+# IMDS (169.254.169.254) and ECS task metadata (169.254.170.2) are also excluded.
+#
+# This recipe only runs when install_proxy_url is set — normal builds are unaffected.
+
+ruby_block 'configure proxy from install_proxy_url' do
+  block do
+    proxy_url = node['cluster']['install_proxy_url']
+
+    if proxy_url && !proxy_url.empty?
+      # Validate proxy URL format: must be http://host:port
+      unless proxy_url.match?(%r{^https?://[^/:]+:\d+/?$})
+        raise "Invalid install_proxy_url '#{proxy_url}'. Expected format: http://host:port"
+      end
+
+      region = node['cluster']['region']
+
+      # S3 endpoints bypass the proxy and use the VPC Gateway Endpoint.
+      # Includes regional (s3.{region}), dash-style (s3-{region}), global (s3.amazonaws.com),
+      # and dualstack (s3.dualstack.{region}) variants used by different AWS services and repos.
+      no_proxy = [
+        "localhost",
+        "127.0.0.1",
+        "169.254.169.254",
+        "169.254.170.2",
+        ".s3.#{region}.amazonaws.com",
+        "s3.#{region}.amazonaws.com",
+        ".s3-#{region}.amazonaws.com",
+        "s3-#{region}.amazonaws.com",
+        ".s3.amazonaws.com",
+        ".s3.dualstack.#{region}.amazonaws.com",
+        "s3.dualstack.#{region}.amazonaws.com",
+      ].join(",")
+
+      Chef::Log.info("Configuring proxy: #{proxy_url}")
+
+      ENV['http_proxy'] = proxy_url
+      ENV['https_proxy'] = proxy_url
+      ENV['HTTP_PROXY'] = proxy_url
+      ENV['HTTPS_PROXY'] = proxy_url
+      ENV['no_proxy'] = no_proxy
+      ENV['NO_PROXY'] = no_proxy
+    else
+      Chef::Log.info("No install_proxy_url set, skipping proxy configuration")
+    end
+  end
+end

cookbooks/aws-parallelcluster-shared/spec/unit/recipes/detect_proxy_spec.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe 'aws-parallelcluster-shared::detect_proxy' do
+  for_all_oses do |platform, version|
+    context "on #{platform}#{version}" do
+      before(:each) do
+        # Clean proxy ENV vars between tests to prevent leakage
+        %w(http_proxy https_proxy HTTP_PROXY HTTPS_PROXY no_proxy NO_PROXY).each do |var|
+          ENV.delete(var)
+        end
+      end
+
+      context 'when install_proxy_url is set with valid URL' do
+        cached(:test_region) { 'test-region-1' }
+        cached(:chef_run) do
+          runner(platform: platform, version: version) do |node|
+            node.override['cluster'] = { 'install_proxy_url' => 'http://10.0.0.109:8888', 'region' => test_region }
+          end.converge(described_recipe)
+        end
+
+        it 'configures proxy environment variables' do
+          expect(chef_run).to run_ruby_block('configure proxy from install_proxy_url')
+        end
+
+        it 'sets proxy env vars in the ruby block' do
+          chef_run
+          block = chef_run.ruby_block('configure proxy from install_proxy_url')
+          block.block.call
+
+          expect(ENV['http_proxy']).to eq('http://10.0.0.109:8888')
+          expect(ENV['https_proxy']).to eq('http://10.0.0.109:8888')
+          expect(ENV['HTTP_PROXY']).to eq('http://10.0.0.109:8888')
+          expect(ENV['HTTPS_PROXY']).to eq('http://10.0.0.109:8888')
+          expect(ENV['no_proxy']).to include(".s3.#{test_region}.amazonaws.com")
+          expect(ENV['no_proxy']).to include("s3.#{test_region}.amazonaws.com")
+          expect(ENV['no_proxy']).to include(".s3-#{test_region}.amazonaws.com")
+          expect(ENV['no_proxy']).to include('.s3.amazonaws.com')
+          expect(ENV['no_proxy']).to include(".s3.dualstack.#{test_region}.amazonaws.com")
+          expect(ENV['no_proxy']).to include('169.254.169.254')
+          expect(ENV['no_proxy']).to include('localhost')
+        end
+      end
+
+      [nil, ''].each do |proxy_value|
+        context "when install_proxy_url is #{proxy_value.nil? ? 'not set' : 'empty string'}" do
+          cached(:chef_run) do
+            runner(platform: platform, version: version) do |node|
+              attrs = { 'region' => 'test-region-1' }
+              attrs['install_proxy_url'] = proxy_value unless proxy_value.nil?
+              node.override['cluster'] = attrs
+            end.converge(described_recipe)
+          end
+
+          it 'does not configure proxy' do
+            chef_run
+            block = chef_run.ruby_block('configure proxy from install_proxy_url')
+            block.block.call
+
+            expect(ENV['http_proxy']).to be_nil
+          end
+        end
+      end
+
+      { 'not-a-valid-url' => 'invalid format', 'http://10.0.0.109' => 'missing port' }.each do |url, description|
+        context "when install_proxy_url has #{description}" do
+          cached(:chef_run) do
+            runner(platform: platform, version: version) do |node|
+              node.override['cluster'] = { 'install_proxy_url' => url, 'region' => 'test-region-1' }
+            end.converge(described_recipe)
+          end
+
+          it 'raises an error' do
+            chef_run
+            block = chef_run.ruby_block('configure proxy from install_proxy_url')
+            expect { block.block.call }.to raise_error(RuntimeError, /Invalid install_proxy_url/)
+          end
+        end
+      end
+    end
+  end
+end