[release-2.11] fix(pure): Make Pure FlashArray HTTP client timeout configurable (#5615)

**Backport:** #5551

**Make Pure FlashArray HTTP client timeout configurable**

**Problem:**

During migrations of VMs with many disks, simultaneous `CopyVolume`
requests to Pure FlashArray were timing out, leaving PVCs stuck in
`Pending`. In one observed case, 15 disks were migrated but only 7
reached `Bound` status — the remaining 8 populator pods failed with:

```
failed to copy VMDK using VVol storage API: copy operation failed: Pure FlashArray CopyVolume failed:
failed to send copy volume request: Post "https://<array>/api/2.46/volumes?overwrite=true":
context deadline exceeded (Client.Timeout exceeded while awaiting headers)
```

The root cause is that the HTTP client timeout was hardcoded to 30
seconds with no way to extend it, making it impossible to accommodate
slower or heavily-loaded arrays.

**Changes:**

- `NewRestClient` now accepts an `httpTimeoutSeconds int` parameter
instead of a hardcoded value. A value of `<= 0` falls back to the 30s
default.
- `NewFlashArrayClonner` threads the parameter through to
`NewRestClient`.
- A `--storage-api-timeout-seconds` CLI flag (default: `30`) is added to
the `vsphere-xcopy-volume-populator` binary.

**How to configure:**

Pass `--storage-api-timeout-seconds=<value>` to the populator binary.
Full operator-side wiring (CRD field → `VSphereXcopyPluginConfig` →
`VSphereXcopyVolumePopulatorSpec` → populator-controller pod args) is a
follow-up.

**Default behaviour is unchanged** — the timeout remains 30 seconds
unless explicitly overridden.

---------

Signed-off-by: Michael Jons <Michael.Jons@tre.se>
Co-authored-by: Michael Jons <Michael.Jons@tre.se>

Author: github-actions[bot]

cmd/vsphere-xcopy-volume-populator/README.md

@@ -158,6 +158,7 @@ to have a secret with the following fields:
 | STORAGE_PASSWORD | string | y* | |
 | STORAGE_TOKEN | string | n** | |
 | STORAGE_SKIP_SSL_VERIFICATION | true/false | n | false |
+| STORAGE_HTTP_TIMEOUT_SECONDS | integer | n | 30 |
 
 \* For most storage vendors, `STORAGE_USERNAME` and `STORAGE_PASSWORD` are required. Pure FlashArray is an exception - see below.
 

cmd/vsphere-xcopy-volume-populator/internal/pure/flashArray.go

@@ -37,13 +37,13 @@ printf "px_%s" $(oc get storagecluster -A -o=jsonpath='{.items[0].status.cluster
 // Authentication is mutually exclusive:
 // - If apiToken is provided (non-empty), it will be used for authentication (username/password ignored)
 // - If apiToken is empty, username and password will be used for authentication
-func NewFlashArrayClonner(hostname, username, password, apiToken string, skipSSLVerification bool, clusterPrefix string) (FlashArrayClonner, error) {
+func NewFlashArrayClonner(hostname, username, password, apiToken string, skipSSLVerification bool, clusterPrefix string, httpTimeoutSeconds int) (FlashArrayClonner, error) {
 	if clusterPrefix == "" {
 		return FlashArrayClonner{}, errors.New(helpMessage)
 	}
 
 	// Create the REST client for all operations
-	restClient, err := NewRestClient(hostname, username, password, apiToken, skipSSLVerification)
+	restClient, err := NewRestClient(hostname, username, password, apiToken, skipSSLVerification, httpTimeoutSeconds)
 	if err != nil {
 		return FlashArrayClonner{}, fmt.Errorf("failed to create REST client: %w", err)
 	}

cmd/vsphere-xcopy-volume-populator/internal/pure/flashArray_test.go

@@ -310,7 +310,7 @@ func TestAuthenticationMethods(t *testing.T) {
 			hostname := strings.TrimPrefix(server.URL, "https://")
 
 			// Create REST client with test parameters
-			client, err := NewRestClient(hostname, tc.username, tc.password, tc.token, true)
+			client, err := NewRestClient(hostname, tc.username, tc.password, tc.token, true, 30)
 
 			if tc.expectError {
 				if err == nil {

cmd/vsphere-xcopy-volume-populator/internal/pure/rest_client.go

@@ -119,7 +119,11 @@ type HostConnectionRequest struct {
 // NewRestClient creates a new REST client for Pure FlashArray
 // If apiToken is provided (non-empty), it will be used directly, skipping username/password authentication
 // If apiToken is empty, username and password will be used to obtain an API token
-func NewRestClient(hostname, username, password, apiToken string, skipSSLVerify bool) (*RestClient, error) {
+// httpTimeoutSeconds controls the HTTP client timeout; pass 0 to use the default of 30 seconds
+func NewRestClient(hostname, username, password, apiToken string, skipSSLVerify bool, httpTimeoutSeconds int) (*RestClient, error) {
+	if httpTimeoutSeconds <= 0 {
+		httpTimeoutSeconds = 30
+	}
 	// Create base transport with TLS config
 	baseTransport := &http.Transport{
 		TLSClientConfig: &tls.Config{
@@ -135,7 +139,7 @@ func NewRestClient(hostname, username, password, apiToken string, skipSSLVerify
 	client := &RestClient{
 		hostname: hostname,
 		httpClient: &http.Client{
-			Timeout:   30 * time.Second,
+			Timeout:   time.Duration(httpTimeoutSeconds) * time.Second,
 			Transport: transport,
 		},
 	}

cmd/vsphere-xcopy-volume-populator/vsphere-xcopy-volume-populator.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"os"
 	"path"
+	"strconv"
 	"strings"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -59,6 +60,7 @@ var (
 	vspherePassword            string
 	esxiCloneMethod            string
 	sshTimeoutSeconds          int
+	storageAPITimeoutSeconds   string
 
 	// kube args
 	httpEndpoint string
@@ -104,8 +106,12 @@ func main() {
 		}
 		storageApi = &sm
 	case forklift.StorageVendorProductPureFlashArray:
+		apiTimeout, err := strconv.Atoi(storageAPITimeoutSeconds)
+		if err != nil && storageAPITimeoutSeconds != "" {
+			klog.Warningf("invalid value %q for storage-http-timeout-seconds, using default (30s): %v", storageAPITimeoutSeconds, err)
+		}
 		sm, err := pure.NewFlashArrayClonner(
-			storageHostname, storageUsername, storagePassword, storageToken, storageSkipSSLVerification == "true", os.Getenv(pure.ClusterPrefixEnv))
+			storageHostname, storageUsername, storagePassword, storageToken, storageSkipSSLVerification == "true", os.Getenv(pure.ClusterPrefixEnv), apiTimeout)
 		if err != nil {
 			klog.Fatalf("failed to initialize Pure FlashArray clonner with %s", err)
 		}
@@ -306,6 +312,7 @@ func handleArgs() {
 	flag.StringVar(&vspherePassword, "vsphere-password", os.Getenv("GOVMOMI_PASSWORD"), "vSphere's API password")
 	flag.StringVar(&esxiCloneMethod, "esxi-clone-method", os.Getenv("ESXI_CLONE_METHOD"), "ESXi clone method: 'vib' (default) or 'ssh'")
 	flag.IntVar(&sshTimeoutSeconds, "ssh-timeout-seconds", 30, "SSH timeout in seconds for ESXi operations (default: 30)")
+	flag.StringVar(&storageAPITimeoutSeconds, "storage-http-timeout-seconds", os.Getenv("STORAGE_HTTP_TIMEOUT_SECONDS"), "HTTP client timeout in seconds for storage API requests (default: 30)")
 	flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.")
 	flag.StringVar(&masterURL, "master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.")
 	// Metrics args