Skip to content

Commit a86d943

Browse files
authored
feat(collector): Add host collector to copy files to a bundle (#1068)
* Boiler plate for host copy collector * feat: Add copy host collector * Add tests * No need to handle symlinks in a special way System libraries (os.ReadAll, os.ReadDir) already handle symlinks
1 parent ff48706 commit a86d943

13 files changed

+468
-4
lines changed

Makefile

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ endef
3636

3737
BUILDFLAGS = -tags "netgo containers_image_ostree_stub exclude_graphdriver_devicemapper exclude_graphdriver_btrfs containers_image_openpgp" -installsuffix netgo
3838
BUILDPATHS = ./pkg/... ./cmd/... ./internal/...
39+
TESTFLAGS ?= -v -coverprofile cover.out
3940

4041
all: test support-bundle preflight collect analyze
4142

@@ -46,9 +47,9 @@ ffi: fmt vet
4647
.PHONY: test
4748
test: generate fmt vet
4849
if [ -n $(RUN) ]; then \
49-
go test ${BUILDFLAGS} ${BUILDPATHS} -coverprofile cover.out -run $(RUN); \
50+
go test ${BUILDFLAGS} ${BUILDPATHS} ${TESTFLAGS} -run $(RUN); \
5051
else \
51-
go test ${BUILDFLAGS} ${BUILDPATHS} -coverprofile cover.out; \
52+
go test ${BUILDFLAGS} ${BUILDPATHS} ${TESTFLAGS}; \
5253
fi
5354

5455
# Go tests that require a K8s instance

config/crds/troubleshoot.sh_hostcollectors.yaml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1002,6 +1002,17 @@ spec:
10021002
- certificatePath
10031003
- keyPath
10041004
type: object
1005+
copy:
1006+
properties:
1007+
collectorName:
1008+
type: string
1009+
exclude:
1010+
type: BoolString
1011+
path:
1012+
type: string
1013+
required:
1014+
- path
1015+
type: object
10051016
cpu:
10061017
properties:
10071018
collectorName:

config/crds/troubleshoot.sh_hostpreflights.yaml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1002,6 +1002,17 @@ spec:
10021002
- certificatePath
10031003
- keyPath
10041004
type: object
1005+
copy:
1006+
properties:
1007+
collectorName:
1008+
type: string
1009+
exclude:
1010+
type: BoolString
1011+
path:
1012+
type: string
1013+
required:
1014+
- path
1015+
type: object
10051016
cpu:
10061017
properties:
10071018
collectorName:

config/crds/troubleshoot.sh_supportbundles.yaml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11026,6 +11026,17 @@ spec:
1102611026
- certificatePath
1102711027
- keyPath
1102811028
type: object
11029+
copy:
11030+
properties:
11031+
collectorName:
11032+
type: string
11033+
exclude:
11034+
type: BoolString
11035+
path:
11036+
type: string
11037+
required:
11038+
- path
11039+
type: object
1102911040
cpu:
1103011041
properties:
1103111042
collectorName:

internal/testutils/utils.go

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package testutils
33
import (
44
"crypto/rand"
55
"encoding/hex"
6+
"encoding/json"
67
"fmt"
78
"os"
89
"path/filepath"
@@ -32,3 +33,30 @@ func TempFilename(prefix string) string {
3233
rand.Read(randBytes)
3334
return filepath.Join(os.TempDir(), fmt.Sprintf("%s_%s", prefix, hex.EncodeToString(randBytes)))
3435
}
36+
37+
func CreateTestFile(t *testing.T, path string) {
38+
t.Helper()
39+
40+
CreateTestFileWithData(t, path, "Garbage for "+path)
41+
}
42+
43+
func CreateTestFileWithData(t *testing.T, path, data string) {
44+
t.Helper()
45+
46+
dir := filepath.Dir(path)
47+
err := os.MkdirAll(dir, 0755)
48+
require.NoError(t, err)
49+
err = os.WriteFile(path, []byte(data), 0644)
50+
require.NoError(t, err)
51+
}
52+
53+
func LogJSON(t *testing.T, v interface{}) {
54+
t.Helper()
55+
56+
b, err := json.MarshalIndent(v, "", " ")
57+
if err != nil {
58+
t.Log(v)
59+
} else {
60+
t.Log(string(b))
61+
}
62+
}

pkg/apis/troubleshoot/v1beta2/collector_shared.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -450,6 +450,7 @@ func (c *Collect) AccessReviewSpecs(overrideNS string) []authorizationv1.SelfSub
450450
}
451451

452452
func (c *Collect) GetName() string {
453+
// TODO: Is this used anywhere? Should we just remove it?
453454
var collector, name, selector string
454455
if c.ClusterInfo != nil {
455456
collector = "cluster-info"

pkg/apis/troubleshoot/v1beta2/hostcollector_shared.go

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ type SubnetAvailable struct {
6161

6262
type DiskUsage struct {
6363
HostCollectorMeta `json:",inline" yaml:",inline"`
64-
Path string `json:"path"`
64+
Path string `json:"path" yaml:"path"`
6565
}
6666

6767
type HostHTTP struct {
@@ -71,6 +71,11 @@ type HostHTTP struct {
7171
Put *Put `json:"put,omitempty" yaml:"put,omitempty"`
7272
}
7373

74+
type HostCopy struct {
75+
HostCollectorMeta `json:",inline" yaml:",inline"`
76+
Path string `json:"path" yaml:"path"`
77+
}
78+
7479
type HostTime struct {
7580
HostCollectorMeta `json:",inline" yaml:",inline"`
7681
}
@@ -196,9 +201,11 @@ type HostCollect struct {
196201
HostServices *HostServices `json:"hostServices,omitempty" yaml:"hostServices,omitempty"`
197202
HostOS *HostOS `json:"hostOS,omitempty" yaml:"hostOS,omitempty"`
198203
HostRun *HostRun `json:"run,omitempty" yaml:"run,omitempty"`
204+
HostCopy *HostCopy `json:"copy,omitempty" yaml:"copy,omitempty"`
199205
}
200206

201207
func (c *HostCollect) GetName() string {
208+
// TODO: Is this used anywhere? Should we just remove it?
202209
var collector string
203210
if c.CPU != nil {
204211
collector = "cpu"

pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go

Lines changed: 21 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

pkg/collect/host_collector.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ func GetHostCollector(collector *troubleshootv1beta2.HostCollect, bundlePath str
5757
return &CollectHostOS{collector.HostOS, bundlePath}, true
5858
case collector.HostRun != nil:
5959
return &CollectHostRun{collector.HostRun, bundlePath}, true
60+
case collector.HostCopy != nil:
61+
return &CollectHostCopy{collector.HostCopy, bundlePath}, true
6062
default:
6163
return nil, false
6264
}

pkg/collect/host_copy.go

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
package collect
2+
3+
import (
4+
"fmt"
5+
"os"
6+
"path/filepath"
7+
"strings"
8+
9+
troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
10+
"k8s.io/klog/v2"
11+
)
12+
13+
type CollectHostCopy struct {
14+
hostCollector *troubleshootv1beta2.HostCopy
15+
BundlePath string
16+
}
17+
18+
func (c *CollectHostCopy) Title() string {
19+
return hostCollectorTitleOrDefault(c.hostCollector.HostCollectorMeta, "copy")
20+
}
21+
22+
func (c *CollectHostCopy) IsExcluded() (bool, error) {
23+
return isExcluded(c.hostCollector.Exclude)
24+
}
25+
26+
func (c *CollectHostCopy) Collect(progressChan chan<- interface{}) (map[string][]byte, error) {
27+
// 1. Construct subdirectory path in the bundle path to copy files into
28+
// output.SaveResult() will create the directory if it doesn't exist
29+
bundleRelPath := filepath.Join("host-collectors", c.Title())
30+
bundlePathDest := filepath.Join(c.BundlePath, bundleRelPath)
31+
32+
// 2. Enumerate all files that match the glob pattern
33+
paths, err := filepath.Glob(c.hostCollector.Path)
34+
if err != nil {
35+
klog.Errorf("Invalid glob pattern %q: %v", c.hostCollector.Path, err)
36+
return nil, err
37+
}
38+
39+
if len(paths) == 0 {
40+
klog.V(1).Info("No files found to copy")
41+
return NewResult(), nil
42+
}
43+
44+
// 3. Copy content in found host paths to the subdirectory
45+
klog.V(1).Infof("Copy files from %q to %q", c.hostCollector.Path, bundleRelPath)
46+
result, err := c.copyFilesToBundle(paths, bundlePathDest)
47+
if err != nil {
48+
klog.Errorf("Failed to copy files from %q to %q: %v", c.hostCollector.Path, "<bundle>/"+bundleRelPath, err)
49+
fileName := fmt.Sprintf("%s/errors.json", c.relBundlePath(bundlePathDest))
50+
output := NewResult()
51+
err := output.SaveResult(c.BundlePath, fileName, marshalErrors([]string{err.Error()}))
52+
if err != nil {
53+
return nil, err
54+
}
55+
return output, nil
56+
}
57+
58+
return result, nil
59+
}
60+
61+
func (c *CollectHostCopy) relBundlePath(path string) string {
62+
s := strings.ReplaceAll(path, c.BundlePath, "")
63+
return strings.TrimPrefix(s, "/")
64+
}
65+
66+
// copyFilesToBundle copies files from the host to the bundle.
67+
func (c *CollectHostCopy) copyFilesToBundle(paths []string, dstDir string) (CollectorResult, error) {
68+
result := NewResult()
69+
70+
for _, path := range paths {
71+
dst := filepath.Join(dstDir, filepath.Base(path))
72+
err := c.doCopy(path, dst, result)
73+
if err != nil {
74+
return nil, err
75+
}
76+
}
77+
78+
return result, nil
79+
}
80+
81+
func (c *CollectHostCopy) doCopy(src, dst string, result CollectorResult) error {
82+
srcInfo, err := os.Stat(src)
83+
if err != nil {
84+
return err
85+
}
86+
87+
if srcInfo.Mode().IsRegular() {
88+
err := c.copyFile(src, dst, result)
89+
if err != nil {
90+
return err
91+
}
92+
} else if srcInfo.IsDir() {
93+
err := c.copyDir(src, dst, result)
94+
if err != nil {
95+
return err
96+
}
97+
} else {
98+
klog.V(2).Infof("Skipping non-file, non-directory path: %q", src)
99+
}
100+
101+
return nil
102+
}
103+
104+
// copyFile copies a file from src to the bundle
105+
func (c *CollectHostCopy) copyFile(src, dst string, result CollectorResult) error {
106+
in, err := os.Open(src)
107+
if err != nil {
108+
return err
109+
}
110+
defer in.Close()
111+
112+
relDest := c.relBundlePath(dst)
113+
err = result.SaveResult(c.BundlePath, relDest, in)
114+
if err != nil {
115+
return err
116+
}
117+
return nil
118+
}
119+
120+
// copyDir recursively copies a directory tree to the bundle
121+
func (c *CollectHostCopy) copyDir(src, dst string, result CollectorResult) error {
122+
// Enunmerate all entries in the source dir
123+
entries, err := os.ReadDir(src)
124+
if err != nil {
125+
return err
126+
}
127+
128+
for _, entry := range entries {
129+
srcPointer := filepath.Join(src, entry.Name())
130+
dstPointer := filepath.Join(dst, entry.Name())
131+
132+
err = c.doCopy(srcPointer, dstPointer, result)
133+
if err != nil {
134+
return err
135+
}
136+
}
137+
138+
return nil
139+
}

0 commit comments

Comments
 (0)