Capture process information in support export archive

This update adds the capture of running processes for each Pod
of the selected PostgresCluster. The 'ps' command is run on any
possible container in the cluster, but only successful results
are stored in the archive.

Issue: [sc-18017]

Author: tjmoore4

internal/cmd/exec.go

@@ -69,3 +69,13 @@ func (exec Executor) patronictl(cmd string) (string, string, error) {
 
 	return stdout.String(), stderr.String(), err
 }
+
+// processes returns the output of a ps command
+func (exec Executor) processes() (string, string, error) {
+	var stdout, stderr bytes.Buffer
+
+	command := "ps aux --width 500"
+	err := exec(nil, &stdout, &stderr, "bash", "-ceu", "--", command)
+
+	return stdout.String(), stderr.String(), err
+}

internal/cmd/exec_test.go

@@ -126,3 +126,22 @@ func TestPatronictl(t *testing.T) {
 	})
 
 }
+
+func TestProcesses(t *testing.T) {
+
+	t.Run("default", func(t *testing.T) {
+		expected := errors.New("pass-through")
+		exec := func(
+			stdin io.Reader, stdout, stderr io.Writer, command ...string,
+		) error {
+			assert.DeepEqual(t, command, []string{"bash", "-ceu", "--", "ps aux --width 500"})
+			assert.Assert(t, stdout != nil, "should capture stdout")
+			assert.Assert(t, stderr != nil, "should capture stderr")
+			return expected
+		}
+		_, _, err := Executor(exec).processes()
+		assert.ErrorContains(t, err, "pass-through")
+
+	})
+
+}

internal/cmd/export.go

@@ -360,6 +360,11 @@ kubectl pgo support export daisy --monitoring-namespace another-namespace --outp
 			err = gatherPatroniInfo(ctx, clientset, restConfig, namespace, clusterName, tw, cmd)
 		}
 
+		// Exec to get Container processes
+		if err == nil {
+			err = gatherProcessInfo(ctx, clientset, restConfig, namespace, clusterName, tw, cmd)
+		}
+
 		// Print cli output
 		path := clusterName + "/logs/cli"
 		if logErr := writeTar(tw, cliOutput.Bytes(), path, cmd); logErr != nil {
@@ -805,7 +810,6 @@ func gatherPodLogs(ctx context.Context,
 	tw *tar.Writer,
 	cmd *cobra.Command,
 ) error {
-	// TODO: update to use specific client after SSA change
 	// Get all Pods that match the given Label
 	pods, err := clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
 		LabelSelector: labelSelector,
@@ -863,7 +867,6 @@ func gatherPatroniInfo(ctx context.Context,
 	tw *tar.Writer,
 	cmd *cobra.Command,
 ) error {
-	// TODO: update to use specific client after SSA change
 	// Get the primary instance Pod by its labels
 	pods, err := clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
 		LabelSelector: util.PrimaryInstanceLabels(clusterName),
@@ -931,6 +934,79 @@ func gatherPatroniInfo(ctx context.Context,
 	return nil
 }
 
+// gatherProcessInfo takes a client and buffer execs into relevant pods to grab
+// running process information for each Pod.
+func gatherProcessInfo(ctx context.Context,
+	clientset *kubernetes.Clientset,
+	config *rest.Config,
+	namespace string,
+	clusterName string,
+	tw *tar.Writer,
+	cmd *cobra.Command,
+) error {
+	// Get the cluster Pods by label
+	pods, err := clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
+		LabelSelector: "postgres-operator.crunchydata.com/cluster=" + clusterName,
+	})
+	if err != nil {
+		if apierrors.IsForbidden(err) {
+			cmd.Println(err.Error())
+			return nil
+		}
+		return err
+	}
+
+	if len(pods.Items) == 0 {
+		// If we didn't find any resources, skip
+		cmd.Println("PostgresCluster Pods not found when gathering process information, skipping")
+		return nil
+	}
+
+	podExec, err := util.NewPodExecutor(config)
+	if err != nil {
+		return err
+	}
+
+	for _, pod := range pods.Items {
+		for _, container := range pod.Spec.Containers {
+			// Attempt to exec in and run 'ps' command in all available containers,
+			// regardless of state, etc. Many of the resulting process lists will
+			// be nearly identical because certain Pods use a shared process
+			// namespace, but this function aims to gather as much detail as possible.
+			// - https://kubernetes.io/docs/tasks/configure-pod-container/share-process-namespace/
+			exec := func(stdin io.Reader, stdout, stderr io.Writer, command ...string,
+			) error {
+				return podExec(namespace, pod.GetName(), container.Name,
+					stdin, stdout, stderr, command...)
+			}
+
+			stdout, stderr, err := Executor(exec).processes()
+			if err != nil {
+				// If we get an RBAC error, let the user know. Otherwise, just
+				// try the next container.
+				if apierrors.IsForbidden(err) {
+					cmd.Printf("Failed to get processes for Container \"%s\" in Pod \"%s\". Error: \"%s\"\n",
+						container.Name, pod.GetName(), err.Error())
+				}
+				continue
+			}
+
+			var buf bytes.Buffer
+			buf.Write([]byte(stdout))
+			if stderr != "" {
+				buf.Write([]byte(stderr))
+			}
+
+			path := clusterName + "/" + "processes" + "/" + pod.GetName() + "/" + container.Name
+			if err := writeTar(tw, buf.Bytes(), path, cmd); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
 // translateTimestampSince returns the elapsed time since timestamp in
 // human-readable approximation.
 func translateTimestampSince(timestamp metav1.Time) string {

testing/kuttl/e2e/support-export/01--support_export.yaml

@@ -69,4 +69,28 @@ commands:
       exit 1
     fi
 
+    PROCESSES_DIR="./kuttl-support-cluster/processes/"
+
+    # Check for the files that contain an expected pgBackRest server process.
+    # Expected to be found in the Postgres instance Pod's 'database',
+    # 'replication-cert-copy', 'pgbackrest', and 'pgbackrest-config' containers
+    # and the pgBackRest repo Pod's 'pgbackrest' and 'pgbackrest-config'
+    # containers, i.e. 6 files total.
+    found=$(grep -lR "pgbackrest server" ${PROCESSES_DIR} | wc -l)
+    if [ "${found}" -ne 6 ]; then
+      echo "Expected to find 6 pgBackRest processes, got ${found}"
+      eval "$CLEANUP"
+      exit 1
+    fi
+
+    # Check for the files that contain an expected Postgres process. Expected
+    # to be found in the Postgres instance Pod's 'database', 'replication-cert-copy',
+    # 'pgbackrest', and 'pgbackrest-config' containers, i.e. 4 files total.
+    found=$(grep -lR "postgres -D /pgdata/pg" ${PROCESSES_DIR} | wc -l)
+    if [ "${found}" -ne 4 ]; then
+      echo "Expected to find 4 Postgres processes, got ${found}"
+      eval "$CLEANUP"
+      exit 1
+    fi
+
 - script: rm -r ./kuttl-support-cluster ./crunchy_k8s_support_export_*.tar.gz

testing/kuttl/e2e/support-export/README

@@ -22,6 +22,7 @@ in this test. Over time, we should build on these tests to improve coverage.
     - compression level
     - Node "list" and YAML files
     - Event file
+    - Pod Processes
     The support export archive is deleted.
 
 #### Invalid Cluster