must-gather: collect selinux info #2509
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| . namespace | ||
| NRO_NAMESPACE=$(nro_namespace) | ||
|
|
||
| SELINUX_INFO_DIR="/must-gather/selinux_info" | ||
| mkdir -p "$SELINUX_INFO_DIR" | ||
|
|
||
| # common variables | ||
| OC_GET_PODS="oc get pods -n $NRO_NAMESPACE" | ||
|
|
||
| function replace_dot_with_underscore() { | ||
| echo $1 | sed 's/\./_/g' | ||
| } | ||
|
|
||
| function gather_selinux_data() { | ||
| local rte_pods=$($OC_GET_PODS -l name='resource-topology' -o jsonpath='{.items[*].metadata.name}') | ||
|
|
||
| for pod in $rte_pods; do | ||
| local original_node_name=$($OC_GET_PODS $pod -o jsonpath='{.spec.nodeName}') | ||
| local node_name=$(replace_dot_with_underscore $original_node_name) | ||
|
|
||
| echo "Gathering SELinux data from node: $node_name" | ||
|
|
||
| local node_dir="$SELINUX_INFO_DIR/$node_name" | ||
| mkdir -p "$node_dir" | ||
|
|
||
| oc debug node/$original_node_name -- bash -c " | ||
| chroot /host bash -c ' | ||
| mkdir -p /tmp/selinux_data | ||
|
|
||
| echo \"=== SELinux context for /var/lib/kubelet ===\" > /tmp/selinux_data/contexts | ||
| ls -Z /var/lib/kubelet >> /tmp/selinux_data/contexts 2>&1 | ||
| echo \"\" >> /tmp/selinux_data/contexts | ||
| echo \"=== SELinux context for kubelet.sock ===\" >> /tmp/selinux_data/contexts | ||
| ls -Z /var/lib/kubelet/pod-resources/kubelet.sock >> /tmp/selinux_data/contexts 2>&1 | ||
|
|
||
| systemctl show kubelet.service > /tmp/selinux_data/kubelet_systemctl_show 2>&1 | ||
|
|
||
| systemctl cat kubelet.service > /tmp/selinux_data/kubelet_systemctl_cat 2>&1 | ||
|
|
||
| echo \"=== SELinux audit logs ===\" > /tmp/selinux_data/audit_selinux.log | ||
| grep -i selinux /var/log/audit/audit.log >> /tmp/selinux_data/audit_selinux.log 2>&1 | ||
|
|
||
| echo \"=== Pod-resources related audit logs ===\" > /tmp/selinux_data/audit_podresources.log | ||
| grep \"kubelet.*pod-resources\" /var/log/audit/audit.log >> /tmp/selinux_data/audit_podresources.log 2>&1 | ||
|
|
||
| tar czf - -C /tmp selinux_data | ||
shajmakh marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| ' | ||
| " 2>/dev/null | tar xzf - -C "$node_dir" --strip-components=1 | ||
|
|
||
| if [ $? -eq 0 ]; then | ||
| echo "Successfully collected SELinux data from node: $node_name" | ||
| else | ||
| echo "Failed to collect SELinux data from node: $node_name" | ||
| fi | ||
| done | ||
| } | ||
|
|
||
| if [ -z "${NRO_NAMESPACE}" ]; then | ||
| echo "NUMAResources Operator namespace not detected. Skipping SELinux data gathering" | ||
| else | ||
| gather_selinux_data | ||
| fi | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -24,8 +24,11 @@ import ( | |
| "os/exec" | ||
| "path/filepath" | ||
| "regexp" | ||
| "strings" | ||
| "time" | ||
|
|
||
| "github.com/opencontainers/selinux/go-selinux" | ||
|
|
||
| corev1 "k8s.io/api/core/v1" | ||
| "k8s.io/klog/v2" | ||
|
|
||
|
|
@@ -81,8 +84,9 @@ var _ = Describe("[must-gather] NRO data collected", func() { | |
| cmd := exec.Command(cmdline[0], cmdline[1:]...) | ||
| cmd.Stderr = GinkgoWriter | ||
|
|
||
| output, err := cmd.Output() | ||
| Expect(err).ToNot(HaveOccurred()) | ||
| fmt.Fprintf(GinkgoWriter, "output: %s\n", string(output)) | ||
| }) | ||
|
|
||
| AfterEach(func() { | ||
|
|
@@ -184,6 +188,109 @@ var _ = Describe("[must-gather] NRO data collected", func() { | |
| Expect(podFolderNames).To(ContainElement(MatchRegexp("^secondary-scheduler*"))) | ||
| } | ||
| }) | ||
|
|
||
| It("check SELinux data files have been collected", func(ctx context.Context) { | ||
| minVersion, err := platform.ParseVersion("4.22") | ||
| Expect(err).ToNot(HaveOccurred(), "failed to parse minimum version") | ||
| supported, err := configuration.PlatVersion.AtLeast(minVersion) | ||
| Expect(err).ToNot(HaveOccurred(), "failed to compare versions: %v vs %v", configuration.PlatVersion, minVersion) | ||
| if !supported { | ||
| Skip(fmt.Sprintf("SELinux data collection is only supported on cluster version %s or newer, got %s", minVersion, configuration.PlatVersion)) | ||
| } | ||
|
|
||
| destDirContent, err := os.ReadDir(destDir) | ||
| Expect(err).NotTo(HaveOccurred(), "unable to read contents from destDir:%s. error: %w", destDir, err) | ||
|
|
||
| for _, content := range destDirContent { | ||
| if !content.IsDir() { | ||
| continue | ||
| } | ||
| mgContentFolder := filepath.Join(destDir, content.Name()) | ||
|
|
||
| By(fmt.Sprintf("Checking Folder: %q", mgContentFolder)) | ||
| By("Looking for SELinux info directory") | ||
|
|
||
| selinuxInfoFolder := filepath.Join(mgContentFolder, "selinux_info") | ||
| _, err = os.Stat(selinuxInfoFolder) | ||
| Expect(err).ToNot(HaveOccurred(), "selinux_info directory should exist") | ||
|
|
||
| selinuxDirContent, err := os.ReadDir(selinuxInfoFolder) | ||
| Expect(err).NotTo(HaveOccurred()) | ||
| Expect(selinuxDirContent).ToNot(BeEmpty(), "selinux_info directory should contain node directories") | ||
|
|
||
| // Check each node directory | ||
| for _, nodeDir := range selinuxDirContent { | ||
| if !nodeDir.IsDir() { | ||
| // We expect only directories in selinux_info, but check defensively | ||
| By(fmt.Sprintf("Warning: unexpected non-directory item in selinux_info: %s", nodeDir.Name())) | ||
| continue | ||
| } | ||
|
|
||
| nodeName := nodeDir.Name() | ||
| nodeSelinuxFolder := filepath.Join(selinuxInfoFolder, nodeName) | ||
| By(fmt.Sprintf("Checking SELinux data for node: %s", nodeName)) | ||
|
|
||
| // Check required files exist | ||
| requiredFiles := []string{ | ||
| "contexts", | ||
| "kubelet_systemctl_show", | ||
| "kubelet_systemctl_cat", | ||
| "audit_selinux.log", | ||
| "audit_podresources.log", | ||
| } | ||
| err = checkfilesExist(requiredFiles, nodeSelinuxFolder) | ||
| Expect(err).ToNot(HaveOccurred(), "required SELinux files should exist for node %s", nodeName) | ||
|
|
||
| // Verify contexts file contains kubelet.sock SELinux context | ||
| By(fmt.Sprintf("Verifying kubelet.sock SELinux context for node: %s", nodeName)) | ||
| contextsFile := filepath.Join(nodeSelinuxFolder, "contexts") | ||
| contextsData, err := os.ReadFile(contextsFile) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
|
|
||
| contextsContent := string(contextsData) | ||
|
|
||
| // Parse SELinux context from the kubelet.sock line using official selinux package | ||
| By(fmt.Sprintf("Parsing SELinux context for kubelet.sock on node: %s", nodeName)) | ||
| found := false | ||
| for _, line := range strings.Split(contextsContent, "\n") { | ||
| if strings.Contains(line, "/var/lib/kubelet/pod-resources/kubelet.sock") { | ||
| // Extract SELinux context from ls -Z output (format: "context filename") | ||
| parts := strings.Fields(line) | ||
| if len(parts) >= 2 { | ||
| selinuxLabel := parts[0] | ||
| context, err := selinux.NewContext(selinuxLabel) | ||
| Expect(err).ToNot(HaveOccurred(), "should parse SELinux context: %s", selinuxLabel) | ||
|
|
||
| // Check that the type field contains kubelet_var_lib_t | ||
| contextType := context["type"] | ||
| Expect(contextType).To(Equal("kubelet_var_lib_t"), "kubelet.sock should have kubelet_var_lib_t SELinux context type, got: %s", contextType) | ||
|
Member
There was a problem hiding this comment. this is for 4.18+ (right?), the test like this will run d/s on all versions, and will fail. we can either mark this with a new label and d/s will adapt accordingly and run it only where supported (see https://github.com/openshift-kni/numaresources-operator/blob/main/internal/api/features/_topics.json)
Collaborator
Author
There was a problem hiding this comment. Good catch I'll update the test accordingly
Collaborator
Author
There was a problem hiding this comment. @shajmakh Thinking about it again, are must gather test are part of the serial suite where same tests are running against all releases? If so, and we want this test to run on 4.18+ we need to backport from 4.22->4.18, do we want that?
Member
There was a problem hiding this comment. these tests runs also in d/s, so yes, we use the same test image for all releases. But also these tests run u/s via prow, so if we want to backport the data collection support to older versions, we need to have the test changes also backported so we can have coverage u/s.
Collaborator
Author
There was a problem hiding this comment. For now I added skip if version < 4.22, for every backport I'll decrease the minimum version. |
||
|
|
||
| By(fmt.Sprintf("Found valid SELinux context for kubelet.sock: %s (type: %s)", selinuxLabel, contextType)) | ||
| found = true | ||
| break | ||
| } | ||
| } | ||
| } | ||
| Expect(found).To(BeTrue(), "should find kubelet.sock with valid SELinux context in contexts file") | ||
|
|
||
| // Verify kubelet systemctl show has content | ||
| By(fmt.Sprintf("Verifying kubelet_systemctl_show content for node: %s", nodeName)) | ||
| kubeletShowFile := filepath.Join(nodeSelinuxFolder, "kubelet_systemctl_show") | ||
| kubeletShowData, err := os.ReadFile(kubeletShowFile) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
| kubeletShowContent := string(kubeletShowData) | ||
| Expect(kubeletShowContent).To(ContainSubstring("ExecStart="), "kubelet_systemctl_show should contain ExecStart property") | ||
|
|
||
| // Verify kubelet systemctl cat has content | ||
| By(fmt.Sprintf("Verifying kubelet_systemctl_cat content for node: %s", nodeName)) | ||
| kubeletCatFile := filepath.Join(nodeSelinuxFolder, "kubelet_systemctl_cat") | ||
| kubeletCatData, err := os.ReadFile(kubeletCatFile) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
| kubeletCatContent := string(kubeletCatData) | ||
| Expect(kubeletCatContent).To(ContainSubstring("/usr/sbin/restorecon"), "kubelet_systemctl_cat should contain restorecon command in ExecStartPre") | ||
| } | ||
| } | ||
| }) | ||
| }) | ||
| }) | ||
|
|
||
|
|
||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.