diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index cca4559cc..797fbb1cf 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -34,6 +34,12 @@ jobs: - name: Run e2e tests run: ARTIFACT_PATH=/tmp/artifacts make test-e2e + +# To not store pprof +# - uses: actions/upload-artifact@v4 +# with: +# name: upgrade-e2e-artifacts +# path: test/e2e/results/** - uses: actions/upload-artifact@v4 if: failure() diff --git a/cmd/operator-controller/main.go b/cmd/operator-controller/main.go index 56949ffd7..80ab0e4e6 100644 --- a/cmd/operator-controller/main.go +++ b/cmd/operator-controller/main.go @@ -83,6 +83,7 @@ var ( type config struct { metricsAddr string + pprofAddr string certFile string keyFile string enableLeaderElection bool @@ -131,6 +132,7 @@ func init() { //create flagset, the collection of flags for this command flags := operatorControllerCmd.Flags() flags.StringVar(&cfg.metricsAddr, "metrics-bind-address", "", "The address for the metrics endpoint. Requires tls-cert and tls-key. (Default: ':8443')") + flags.StringVar(&cfg.pprofAddr, "pprof-bind-address", "0", "The address the pprof endpoint binds to. an empty string or 0 disables pprof") flags.StringVar(&cfg.probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flags.StringVar(&cfg.catalogdCasDir, "catalogd-cas-dir", "", "The directory of TLS certificate authorities to use for verifying HTTPS connections to the Catalogd web service.") flags.StringVar(&cfg.pullCasDir, "pull-cas-dir", "", "The directory of TLS certificate authorities to use for verifying HTTPS connections to image registries.") @@ -265,6 +267,7 @@ func run() error { mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme.Scheme, Metrics: metricsServerOptions, + PprofBindAddress: cfg.pprofAddr, HealthProbeBindAddress: cfg.probeAddr, LeaderElection: cfg.enableLeaderElection, LeaderElectionID: "9c4404e7.operatorframework.io", diff --git a/config/base/catalogd/manager/catalogd_service.yaml b/config/base/catalogd/manager/catalogd_service.yaml index 693b687f3..6ca796bbf 100644 --- a/config/base/catalogd/manager/catalogd_service.yaml +++ b/config/base/catalogd/manager/catalogd_service.yaml @@ -14,6 +14,9 @@ spec: protocol: TCP port: 80 targetPort: 8443 + - name: pprof + port: 8083 + targetPort: 8083 - name: webhook protocol: TCP port: 9443 diff --git a/config/base/catalogd/manager/manager.yaml b/config/base/catalogd/manager/manager.yaml index 5c52165ec..12e04544d 100644 --- a/config/base/catalogd/manager/manager.yaml +++ b/config/base/catalogd/manager/manager.yaml @@ -45,6 +45,7 @@ spec: - ./catalogd args: - --leader-elect + - --pprof-bind-address=:8083 - --metrics-bind-address=:7443 - --external-address=catalogd-service.olmv1-system.svc image: controller:latest diff --git a/config/base/operator-controller/manager/manager.yaml b/config/base/operator-controller/manager/manager.yaml index db34940c3..979d52b32 100644 --- a/config/base/operator-controller/manager/manager.yaml +++ b/config/base/operator-controller/manager/manager.yaml @@ -44,6 +44,7 @@ spec: - /operator-controller args: - "--health-probe-bind-address=:8081" + - "--pprof-bind-address=:8082" - "--metrics-bind-address=:8443" - "--leader-elect" image: controller:latest diff --git a/internal/catalogd/controllers/core/clustercatalog_controller.go b/internal/catalogd/controllers/core/clustercatalog_controller.go index be9d816fd..d0597d3ee 100644 --- a/internal/catalogd/controllers/core/clustercatalog_controller.go +++ b/internal/catalogd/controllers/core/clustercatalog_controller.go @@ -152,6 +152,7 @@ func (r *ClusterCatalogReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&ocv1.ClusterCatalog{}). + Named("catalogd-clustercatalog-controller"). Complete(r) } diff --git a/internal/catalogd/controllers/core/pull_secret_controller.go b/internal/catalogd/controllers/core/pull_secret_controller.go index 0255309ca..810581047 100644 --- a/internal/catalogd/controllers/core/pull_secret_controller.go +++ b/internal/catalogd/controllers/core/pull_secret_controller.go @@ -64,6 +64,7 @@ func (r *PullSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request) func (r *PullSecretReconciler) SetupWithManager(mgr ctrl.Manager) error { _, err := ctrl.NewControllerManagedBy(mgr). For(&corev1.Secret{}). + Named("catalogd-pull-secret-controller"). WithEventFilter(newSecretPredicate(r.SecretKey)). Build(r) diff --git a/internal/operator-controller/controllers/clustercatalog_controller.go b/internal/operator-controller/controllers/clustercatalog_controller.go index c7e7edb03..17a6ccae2 100644 --- a/internal/operator-controller/controllers/clustercatalog_controller.go +++ b/internal/operator-controller/controllers/clustercatalog_controller.go @@ -93,6 +93,7 @@ func (r *ClusterCatalogReconciler) Reconcile(ctx context.Context, req ctrl.Reque // SetupWithManager sets up the controller with the Manager. func (r *ClusterCatalogReconciler) SetupWithManager(mgr ctrl.Manager) error { _, err := ctrl.NewControllerManagedBy(mgr). + Named("con-oper-clustercatalog-controller"). For(&ocv1.ClusterCatalog{}). Build(r) diff --git a/internal/operator-controller/controllers/clusterextension_controller.go b/internal/operator-controller/controllers/clusterextension_controller.go index d914b831b..d19f55639 100644 --- a/internal/operator-controller/controllers/clusterextension_controller.go +++ b/internal/operator-controller/controllers/clusterextension_controller.go @@ -407,6 +407,7 @@ func SetDeprecationStatus(ext *ocv1.ClusterExtension, bundleName string, depreca func (r *ClusterExtensionReconciler) SetupWithManager(mgr ctrl.Manager) error { controller, err := ctrl.NewControllerManagedBy(mgr). For(&ocv1.ClusterExtension{}). + Named("con-oper-cluster-extension-controller"). Watches(&ocv1.ClusterCatalog{}, crhandler.EnqueueRequestsFromMapFunc(clusterExtensionRequestsForCatalog(mgr.GetClient(), mgr.GetLogger())), builder.WithPredicates(predicate.Funcs{ diff --git a/internal/operator-controller/controllers/pull_secret_controller.go b/internal/operator-controller/controllers/pull_secret_controller.go index 6db1ae564..8ad7a50b6 100644 --- a/internal/operator-controller/controllers/pull_secret_controller.go +++ b/internal/operator-controller/controllers/pull_secret_controller.go @@ -63,6 +63,7 @@ func (r *PullSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request) // SetupWithManager sets up the controller with the Manager. func (r *PullSecretReconciler) SetupWithManager(mgr ctrl.Manager) error { _, err := ctrl.NewControllerManagedBy(mgr). + Named("con-oper-pull-secret-controller"). For(&corev1.Secret{}). WithEventFilter(newSecretPredicate(r.SecretKey)). Build(r) diff --git a/test/e2e/cluster_extension_install_test.go b/test/e2e/cluster_extension_install_test.go index 7c57a078c..f5332de81 100644 --- a/test/e2e/cluster_extension_install_test.go +++ b/test/e2e/cluster_extension_install_test.go @@ -377,6 +377,10 @@ func TestClusterExtensionInstallRegistry(t *testing.T) { assert.NotEmpty(ct, clusterExtension.Status.Install.Bundle) } }, pollDuration, pollInterval) + + // For this case we cannot get the metrics + //FetchCatalogdMetricsExportedEndpoint(t) + //FetchOperatorControllerMetricsExportedEndpoint(t) }) } } @@ -455,6 +459,9 @@ location = "docker-registry.operator-controller-e2e.svc.cluster.local:5000"`, assert.NotEmpty(ct, clusterExtension.Status.Install.Bundle) } }, pollDuration, pollInterval) + + FetchCatalogdMetricsExportedEndpoint(t) + FetchOperatorControllerMetricsExportedEndpoint(t) } func TestClusterExtensionInstallRegistryMultipleBundles(t *testing.T) { @@ -505,6 +512,9 @@ func TestClusterExtensionInstallRegistryMultipleBundles(t *testing.T) { assert.Contains(ct, cond.Message, "in multiple catalogs with the same priority [extra-test-catalog test-catalog]") } }, pollDuration, pollInterval) + + FetchCatalogdMetricsExportedEndpoint(t) + FetchOperatorControllerMetricsExportedEndpoint(t) } func TestClusterExtensionBlockInstallNonSuccessorVersion(t *testing.T) { @@ -568,6 +578,9 @@ func TestClusterExtensionBlockInstallNonSuccessorVersion(t *testing.T) { assert.Equal(ct, "error upgrading from currently installed version \"1.0.0\": no bundles found for package \"test\" matching version \"1.2.0\"", cond.Message) } }, pollDuration, pollInterval) + + FetchCatalogdMetricsExportedEndpoint(t) + FetchOperatorControllerMetricsExportedEndpoint(t) } func TestClusterExtensionForceInstallNonSuccessorVersion(t *testing.T) { @@ -618,6 +631,9 @@ func TestClusterExtensionForceInstallNonSuccessorVersion(t *testing.T) { assert.Equal(ct, ocv1.ReasonSucceeded, cond.Reason) } }, pollDuration, pollInterval) + + FetchCatalogdMetricsExportedEndpoint(t) + FetchOperatorControllerMetricsExportedEndpoint(t) } func TestClusterExtensionInstallSuccessorVersion(t *testing.T) { @@ -666,6 +682,9 @@ func TestClusterExtensionInstallSuccessorVersion(t *testing.T) { assert.Equal(ct, ocv1.ReasonSucceeded, cond.Reason) } }, pollDuration, pollInterval) + + FetchCatalogdMetricsExportedEndpoint(t) + FetchOperatorControllerMetricsExportedEndpoint(t) } func TestClusterExtensionInstallReResolvesWhenCatalogIsPatched(t *testing.T) { @@ -733,6 +752,9 @@ func TestClusterExtensionInstallReResolvesWhenCatalogIsPatched(t *testing.T) { assert.Equal(ct, ocv1.ReasonSucceeded, cond.Reason) } }, pollDuration, pollInterval) + + FetchCatalogdMetricsExportedEndpoint(t) + FetchOperatorControllerMetricsExportedEndpoint(t) } func TestClusterExtensionInstallReResolvesWhenNewCatalog(t *testing.T) { @@ -814,6 +836,9 @@ func TestClusterExtensionInstallReResolvesWhenNewCatalog(t *testing.T) { assert.Equal(ct, ocv1.ReasonSucceeded, cond.Reason) } }, pollDuration, pollInterval) + + FetchCatalogdMetricsExportedEndpoint(t) + FetchOperatorControllerMetricsExportedEndpoint(t) } func TestClusterExtensionInstallReResolvesWhenManagedContentChanged(t *testing.T) { @@ -866,6 +891,9 @@ func TestClusterExtensionInstallReResolvesWhenManagedContentChanged(t *testing.T require.EventuallyWithT(t, func(ct *assert.CollectT) { assert.NoError(ct, globalClient.Get(context.Background(), types.NamespacedName{Name: testConfigMap.Name, Namespace: testConfigMap.Namespace}, testConfigMap)) }, pollDuration, pollInterval) + + FetchCatalogdMetricsExportedEndpoint(t) + FetchOperatorControllerMetricsExportedEndpoint(t) } func TestClusterExtensionRecoversFromInitialInstallFailedWhenFailureFixed(t *testing.T) { @@ -959,4 +987,7 @@ func TestClusterExtensionRecoversFromInitialInstallFailedWhenFailureFixed(t *tes assert.Equal(ct, ocv1.ReasonSucceeded, cond.Reason) } }, pollDuration, pollInterval) + + FetchCatalogdMetricsExportedEndpoint(t) + FetchOperatorControllerMetricsExportedEndpoint(t) } diff --git a/test/e2e/metrics_test.go b/test/e2e/metrics_test.go index 3d15035b8..a39989214 100644 --- a/test/e2e/metrics_test.go +++ b/test/e2e/metrics_test.go @@ -16,7 +16,13 @@ package e2e import ( "bytes" "context" + "encoding/json" "errors" + "fmt" + "os" + "os/exec" + "regexp" + "strconv" "strings" "testing" "time" @@ -36,8 +42,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/config" ) -// TestOperatorControllerMetricsExportedEndpoint verifies that the metrics endpoint for the operator controller -func TestOperatorControllerMetricsExportedEndpoint(t *testing.T) { +// FetchOperatorControllerMetricsExportedEndpoint verifies that the metrics endpoint for the operator controller +func FetchOperatorControllerMetricsExportedEndpoint(t *testing.T) { kubeClient, restConfig := findK8sClient(t) mtc := NewMetricsTestConfig( t, @@ -49,13 +55,14 @@ func TestOperatorControllerMetricsExportedEndpoint(t *testing.T) { "operator-controller-controller-manager", "oper-curl-metrics", "https://operator-controller-service.NAMESPACE.svc.cluster.local:8443/metrics", + "oper-con", ) mtc.run() } -// TestCatalogdMetricsExportedEndpoint verifies that the metrics endpoint for catalogd -func TestCatalogdMetricsExportedEndpoint(t *testing.T) { +// FetchCatalogdMetricsExportedEndpoint verifies that the metrics endpoint for catalogd +func FetchCatalogdMetricsExportedEndpoint(t *testing.T) { kubeClient, restConfig := findK8sClient(t) mtc := NewMetricsTestConfig( t, @@ -67,11 +74,173 @@ func TestCatalogdMetricsExportedEndpoint(t *testing.T) { "catalogd-controller-manager", "catalogd-curl-metrics", "https://catalogd-service.NAMESPACE.svc.cluster.local:7443/metrics", + "catalogd", ) mtc.run() } +// fetchMetrics retrieves Prometheus metrics from the endpoint +func (c *MetricsTestConfig) fetchMetrics(ctx context.Context, token string) map[string]float64 { + c.t.Log("Fetching Prometheus metrics after test execution") + + cmd := []string{ + "curl", "-s", "-k", + "-H", "Authorization: Bearer " + token, + c.metricsURL, + } + + req := c.kubeClient.CoreV1().RESTClient(). + Post(). + Resource("pods"). + Namespace(c.namespace). + Name(c.curlPodName). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: "curl", + Command: cmd, + Stdin: false, + Stdout: true, + Stderr: true, + TTY: false, + }, scheme.ParameterCodec) + + executor, err := remotecommand.NewSPDYExecutor(c.restConfig, "POST", req.URL()) + require.NoError(c.t, err, "Error creating SPDY executor") + + var stdout, stderr bytes.Buffer + streamOpts := remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + } + + err = executor.StreamWithContext(ctx, streamOpts) + require.NoError(c.t, err, "Error streaming exec request: %v", stderr.String()) + + return parseMetrics(stdout.String()) +} + +// parseMetrics extracts only the required metrics from Prometheus response +func parseMetrics(metricsText string) map[string]float64 { + relevantMetrics := []string{ + "controller_runtime_reconcile_time_seconds_sum", + "http_request_duration_seconds_sum", + "controller_runtime_reconcile_total", + "go_cpu_classes_gc_total_cpu_seconds_total", + "go_cpu_classes_idle_cpu_seconds_total", + "controller_runtime_reconcile_errors_total", + "controller_runtime_webhook_latency_seconds", + } + + mapMetrics := make(map[string]float64) + lines := strings.Split(metricsText, "\n") + for _, line := range lines { + // Check for partial matches + for _, partial := range relevantMetrics { + if strings.Contains(line, partial) { + re := regexp.MustCompile(`(\S+)\s+([0-9.]+)`) + match := re.FindStringSubmatch(line) + if len(match) == 3 { + metricName := match[1] + value, err := strconv.ParseFloat(match[2], 64) + if err == nil { + mapMetrics[metricName] = value + } + } + } + } + } + + return mapMetrics +} + +func (c *MetricsTestConfig) fetchPprofAndStore(ctx context.Context, token string, profileType string) { + c.t.Logf("fetching pprof profiling data for profile: %s", profileType) + + var url string + if strings.Contains(c.metricsURL, "catalogd-service") { + url = fmt.Sprintf("http://catalogd-service.olmv1-system.svc.cluster.local:8083/debug/pprof/%s", profileType) + } else if strings.Contains(c.metricsURL, "operator-controller-service") { + url = fmt.Sprintf("http://operator-controller-service.olmv1-system.svc.cluster.local:8082/debug/pprof/%s", profileType) + } else { + c.t.Fatalf("unknown service in metricsURL: %s", c.metricsURL) + } + + savePath := fmt.Sprintf("/tmp/%s.pprof", profileType) + cmd := []string{ + "sh", "-c", + fmt.Sprintf(`curl -s -k -H "Authorization: Bearer %s" %s > %s`, token, url, savePath), + } + + req := c.kubeClient.CoreV1().RESTClient(). + Post(). + Resource("pods"). + Namespace(c.namespace). + Name(c.curlPodName). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: "curl", + Command: cmd, + Stdin: false, + Stdout: true, + Stderr: true, + TTY: false, + }, scheme.ParameterCodec) + + executor, err := remotecommand.NewSPDYExecutor(c.restConfig, "POST", req.URL()) + require.NoError(c.t, err, "rrror creating SPDY executor for pod exec") + + var stdout, stderr bytes.Buffer + streamOpts := remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + } + + err = executor.StreamWithContext(ctx, streamOpts) + require.NoError(c.t, err, "rrror executing curl in pod: %v", stderr.String()) + + tt := strings.ReplaceAll(c.t.Name(), "/", "_") + localP := fmt.Sprintf("results/%s_%s_%s.pprof", tt, profileType, time.Now().Format("20060102_150405")) + copyCmd := fmt.Sprintf("kubectl cp %s/%s:%s %s", c.namespace, c.curlPodName, savePath, localP) + + output, copyErr := exec.Command("sh", "-c", copyCmd).CombinedOutput() + require.NoError(c.t, copyErr, "failed copy pprof: %s", string(output)) + c.t.Logf("Pprof data successfully saved to: %s", localP) + + textP := strings.Replace(localP, ".pprof", ".txt", 1) + textCmd := fmt.Sprintf("go tool pprof -text %s > %s", localP, textP) + textOutput, textErr := exec.Command("sh", "-c", textCmd).CombinedOutput() + require.NoError(c.t, textErr, "failed to convert pprof: %s", string(textOutput)) + + c.t.Logf("Pprof text saved to: %s", textP) +} + +func (c *MetricsTestConfig) fetchAndStorePprof(ctx context.Context, token string) { + profiles := []string{"profile", "heap", "goroutine"} + for _, profileType := range profiles { + c.fetchPprofAndStore(ctx, token, profileType) + } +} + +// storeMetricsResult writes only relevant metrics to a JSON file +func (c *MetricsTestConfig) storeMetricsResult(metrics map[string]float64) { + dir := "results" + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + c.t.Fatalf("Failed to create directory %s: %v", dir, err) + } + + filePath := fmt.Sprintf("%s/metrics_%s_%s_%s.txt", dir, c.name, c.t.Name(), time.Now().Format("20060102_150405")) + jsonData, err := json.MarshalIndent(metrics, "", " ") + if err != nil { + c.t.Fatalf("Failed to encode metrics as JSON: %v", err) + } + + err = os.WriteFile(filePath, jsonData, 0644) + require.NoError(c.t, err, "failed to save metrics") + + c.t.Logf("filtered metrics saved to: %s", filePath) +} + func findK8sClient(t *testing.T) (kubernetes.Interface, *rest.Config) { cfg, err := config.GetConfig() require.NoError(t, err, "Failed to get Kubernetes config") @@ -94,6 +263,7 @@ type MetricsTestConfig struct { serviceAccount string curlPodName string metricsURL string + name string } // NewMetricsTestConfig initializes a new MetricsTestConfig. @@ -107,6 +277,7 @@ func NewMetricsTestConfig( serviceAccount string, curlPodName string, metricsURL string, + name string, ) *MetricsTestConfig { // Discover which namespace the relevant Pod is running in namespace := getComponentNamespace(t, kubeClient, selector) @@ -124,24 +295,41 @@ func NewMetricsTestConfig( serviceAccount: serviceAccount, curlPodName: curlPodName, metricsURL: metricsURL, + name: name, } } // run executes the entire test flow func (c *MetricsTestConfig) run() { ctx := context.Background() - defer c.cleanup(ctx) + //defer c.cleanup(ctx) c.createMetricsClusterRoleBinding(ctx) token := c.getServiceAccountToken(ctx) c.createCurlMetricsPod(ctx) c.waitForPodReady(ctx) // Exec `curl` in the Pod to validate the metrics c.validateMetricsEndpoint(ctx, token) + + metrics := c.fetchMetrics(ctx, token) + c.storeMetricsResult(metrics) + + c.fetchAndStorePprof(ctx, token) } // createMetricsClusterRoleBinding to bind the cluster role so metrics are accessible func (c *MetricsTestConfig) createMetricsClusterRoleBinding(ctx context.Context) { - c.t.Logf("Creating ClusterRoleBinding %q in namespace %q", c.clusterBinding, c.namespace) + c.t.Logf("Ensuring ClusterRoleBinding %q exists in namespace %q", c.clusterBinding, c.namespace) + + _, err := c.kubeClient.RbacV1().ClusterRoleBindings().Get(ctx, c.clusterBinding, metav1.GetOptions{}) + if err == nil { + c.t.Logf("ClusterRoleBinding %q already exists, skipping creation", c.clusterBinding) + return + } + + if !apierrors.IsNotFound(err) { + require.NoError(c.t, err, "Error checking for existing ClusterRoleBinding") + return + } crb := &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ @@ -161,8 +349,9 @@ func (c *MetricsTestConfig) createMetricsClusterRoleBinding(ctx context.Context) }, } - _, err := c.kubeClient.RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{}) + _, err = c.kubeClient.RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{}) require.NoError(c.t, err, "Error creating ClusterRoleBinding") + c.t.Logf("Successfully created ClusterRoleBinding %q", c.clusterBinding) } // getServiceAccountToken creates a TokenRequest for the service account @@ -188,7 +377,18 @@ func (c *MetricsTestConfig) getServiceAccountToken(ctx context.Context) string { // createCurlMetricsPod spawns a pod running `curlimages/curl` to check metrics func (c *MetricsTestConfig) createCurlMetricsPod(ctx context.Context) { - c.t.Logf("Creating curl pod (%s/%s) to validate the metrics endpoint", c.namespace, c.curlPodName) + c.t.Logf("Ensuring curl pod (%s/%s) exists to validate the metrics endpoint", c.namespace, c.curlPodName) + + _, err := c.kubeClient.CoreV1().Pods(c.namespace).Get(ctx, c.curlPodName, metav1.GetOptions{}) + if err == nil { + c.t.Logf("Curl pod %q already exists, skipping creation", c.curlPodName) + return + } + + if !apierrors.IsNotFound(err) { + require.NoError(c.t, err, "Error checking for existing curl pod") + return + } pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -220,7 +420,7 @@ func (c *MetricsTestConfig) createCurlMetricsPod(ctx context.Context) { }, } - _, err := c.kubeClient.CoreV1().Pods(c.namespace).Create(ctx, pod, metav1.CreateOptions{}) + _, err = c.kubeClient.CoreV1().Pods(c.namespace).Create(ctx, pod, metav1.CreateOptions{}) require.NoError(c.t, err, "Error creating curl pod") }