Skip to content

Commit 8dcfa98

Browse files
authored
Copy from host collector (#391)
* Copy from host collector * namespace improvements * better support for multiple nodes
1 parent 6007f15 commit 8dcfa98

File tree

16 files changed

+361
-228
lines changed

16 files changed

+361
-228
lines changed

cmd/preflight/cli/run.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -203,8 +203,14 @@ func collectInCluster(preflightSpec *troubleshootv1beta2.Preflight, finishedCh c
203203
return nil, errors.Wrap(err, "failed to convert kube flags to rest config")
204204
}
205205

206+
namespace := v.GetString("namespace")
207+
if namespace == "" {
208+
kubeconfig := k8sutil.GetKubeconfig()
209+
namespace, _, _ = kubeconfig.Namespace()
210+
}
211+
206212
collectOpts := preflight.CollectOpts{
207-
Namespace: v.GetString("namespace"),
213+
Namespace: namespace,
208214
IgnorePermissionErrors: v.GetBool("collect-without-permissions"),
209215
ProgressChan: progressCh,
210216
KubernetesRestConfig: restConfig,

cmd/troubleshoot/cli/run.go

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,17 @@ func runTroubleshoot(v *viper.Viper, arg string) error {
4242
os.Exit(0)
4343
}()
4444

45-
k8sConfig, err := k8sutil.GetRESTConfig()
45+
restConfig, err := k8sutil.GetRESTConfig()
4646
if err != nil {
4747
return errors.Wrap(err, "failed to convert kube flags to rest config")
4848
}
4949

50+
namespace := v.GetString("namespace")
51+
if namespace == "" {
52+
kubeconfig := k8sutil.GetKubeconfig()
53+
namespace, _, _ = kubeconfig.Namespace()
54+
}
55+
5056
var sinceTime *time.Time
5157
if v.GetString("since-time") != "" || v.GetString("since") != "" {
5258
sinceTime, err = parseTimeFlags(v)
@@ -149,8 +155,8 @@ func runTroubleshoot(v *viper.Viper, arg string) error {
149155
createOpts := supportbundle.SupportBundleCreateOpts{
150156
CollectorProgressCallback: collectorCB,
151157
CollectWithoutPermissions: v.GetBool("collect-without-permissions"),
152-
KubernetesRestConfig: k8sConfig,
153-
Namespace: v.GetString("namespace"),
158+
KubernetesRestConfig: restConfig,
159+
Namespace: namespace,
154160
ProgressChan: progressChan,
155161
SinceTime: sinceTime,
156162
}

pkg/analyze/download.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ func AnalyzeLocal(localBundlePath string, analyzers []*troubleshootv1beta2.Analy
3434
for _, analyzer := range analyzers {
3535
analyzeResult, err := Analyze(analyzer, fcp.getFileContents, fcp.getChildFileContents)
3636
if err != nil {
37-
logger.Printf("an analyzer failed to run: %v\n", err)
37+
logger.Printf("An analyzer failed to run: %v", err)
3838
continue
3939
}
4040

pkg/apis/troubleshoot/v1beta2/collector_shared.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,17 @@ type Copy struct {
100100
ContainerName string `json:"containerName,omitempty" yaml:"containerName,omitempty"`
101101
}
102102

103+
type CopyFromHost struct {
104+
CollectorMeta `json:",inline" yaml:",inline"`
105+
Name string `json:"name,omitempty" yaml:"name,omitempty"`
106+
Namespace string `json:"namespace" yaml:"namespace"`
107+
Image string `json:"image" yaml:"image"`
108+
ImagePullPolicy string `json:"imagePullPolicy,omitempty" yaml:"imagePullPolicy,omitempty"`
109+
ImagePullSecret *ImagePullSecrets `json:"imagePullSecret,omitempty" yaml:"imagePullSecret,omitempty"`
110+
Timeout string `json:"timeout,omitempty" yaml:"timeout,omitempty"`
111+
HostPath string `json:"hostPath" yaml:"hostPath"`
112+
}
113+
103114
type HTTP struct {
104115
CollectorMeta `json:",inline" yaml:",inline"`
105116
Name string `json:"name,omitempty" yaml:"name,omitempty"`
@@ -172,6 +183,7 @@ type Collect struct {
172183
Exec *Exec `json:"exec,omitempty" yaml:"exec,omitempty"`
173184
Data *Data `json:"data,omitempty" yaml:"data,omitempty"`
174185
Copy *Copy `json:"copy,omitempty" yaml:"copy,omitempty"`
186+
CopyFromHost *CopyFromHost `json:"copyFromHost,omitempty" yaml:"copyFromHost,omitempty"`
175187
HTTP *HTTP `json:"http,omitempty" yaml:"http,omitempty"`
176188
Postgres *Database `json:"postgres,omitempty" yaml:"postgres,omitempty"`
177189
Mysql *Database `json:"mysql,omitempty" yaml:"mysql,omitempty"`
@@ -350,6 +362,10 @@ func (c *Collect) AccessReviewSpecs(overrideNS string) []authorizationv1.SelfSub
350362
},
351363
NonResourceAttributes: nil,
352364
})
365+
} else if c.CopyFromHost != nil {
366+
// TODO
367+
} else if c.Collectd != nil {
368+
// TODO
353369
} else if c.HTTP != nil {
354370
// NOOP
355371
} else if c.RegistryImages != nil &&
@@ -409,6 +425,10 @@ func (c *Collect) GetName() string {
409425
name = c.Copy.CollectorName
410426
selector = strings.Join(c.Copy.Selector, ",")
411427
}
428+
if c.CopyFromHost != nil {
429+
collector = "copy-from-host"
430+
name = c.CopyFromHost.CollectorName
431+
}
412432
if c.HTTP != nil {
413433
collector = "http"
414434
name = c.HTTP.CollectorName

pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go

Lines changed: 26 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

pkg/collect/collectd.go

Lines changed: 12 additions & 206 deletions
Original file line numberDiff line numberDiff line change
@@ -2,215 +2,21 @@ package collect
22

33
import (
44
"context"
5-
"path"
6-
"path/filepath"
7-
"time"
85

9-
"github.com/pkg/errors"
106
troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
11-
"github.com/replicatedhq/troubleshoot/pkg/logger"
12-
"github.com/segmentio/ksuid"
13-
appsv1 "k8s.io/api/apps/v1"
14-
corev1 "k8s.io/api/core/v1"
15-
kuberneteserrors "k8s.io/apimachinery/pkg/api/errors"
16-
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
17-
"k8s.io/apimachinery/pkg/labels"
187
"k8s.io/client-go/kubernetes"
8+
restclient "k8s.io/client-go/rest"
199
)
2010

21-
func Collectd(c *Collector, collectdCollector *troubleshootv1beta2.Collectd) (map[string][]byte, error) {
22-
ctx := context.Background()
23-
label := ksuid.New().String()
24-
namespace := collectdCollector.Namespace
25-
26-
client, err := kubernetes.NewForConfig(c.ClientConfig)
27-
if err != nil {
28-
return nil, errors.Wrap(err, "failed to create client from config")
29-
}
30-
31-
dsName, err := createDaemonSet(ctx, client, collectdCollector, namespace, label)
32-
if dsName != "" {
33-
defer func() {
34-
if err := client.AppsV1().DaemonSets(namespace).Delete(ctx, dsName, metav1.DeleteOptions{}); err != nil {
35-
logger.Printf("Failed to delete daemonset %s: %v\n", dsName, err)
36-
}
37-
}()
38-
39-
if collectdCollector.ImagePullSecret != nil && collectdCollector.ImagePullSecret.Data != nil {
40-
defer func() {
41-
err := client.CoreV1().Secrets(namespace).Delete(ctx, collectdCollector.ImagePullSecret.Name, metav1.DeleteOptions{})
42-
if err != nil && !kuberneteserrors.IsNotFound(err) {
43-
logger.Printf("Failed to delete secret %s: %v\n", collectdCollector.ImagePullSecret.Name, err)
44-
}
45-
}()
46-
}
47-
}
48-
if err != nil {
49-
return nil, errors.Wrap(err, "failed to create daemonset")
50-
}
51-
52-
if collectdCollector.Timeout == "" {
53-
return collectRRDFiles(ctx, client, c, collectdCollector, label, namespace)
54-
}
55-
56-
timeout, err := time.ParseDuration(collectdCollector.Timeout)
57-
if err != nil {
58-
return nil, errors.Wrap(err, "failed to parse timeout")
59-
}
60-
61-
childCtx, cancel := context.WithCancel(ctx)
62-
defer cancel()
63-
64-
errCh := make(chan error, 1)
65-
resultCh := make(chan map[string][]byte, 1)
66-
go func() {
67-
b, err := collectRRDFiles(childCtx, client, c, collectdCollector, label, namespace)
68-
if err != nil {
69-
errCh <- err
70-
} else {
71-
resultCh <- b
72-
}
73-
}()
74-
75-
select {
76-
case <-time.After(timeout):
77-
return nil, errors.New("timeout")
78-
case result := <-resultCh:
79-
return result, nil
80-
case err := <-errCh:
81-
return nil, err
82-
}
83-
}
84-
85-
func createDaemonSet(ctx context.Context, client *kubernetes.Clientset, rrdCollector *troubleshootv1beta2.Collectd, namespace string, label string) (string, error) {
86-
pullPolicy := corev1.PullIfNotPresent
87-
volumeType := corev1.HostPathDirectory
88-
if rrdCollector.ImagePullPolicy != "" {
89-
pullPolicy = corev1.PullPolicy(rrdCollector.ImagePullPolicy)
90-
}
91-
dsLabels := map[string]string{
92-
"rrd-collector": label,
93-
}
94-
95-
ds := appsv1.DaemonSet{
96-
ObjectMeta: metav1.ObjectMeta{
97-
GenerateName: "troubleshoot",
98-
Namespace: namespace,
99-
Labels: dsLabels,
100-
},
101-
Spec: appsv1.DaemonSetSpec{
102-
Selector: &metav1.LabelSelector{
103-
MatchLabels: dsLabels,
104-
},
105-
Template: corev1.PodTemplateSpec{
106-
ObjectMeta: metav1.ObjectMeta{
107-
Labels: dsLabels,
108-
},
109-
Spec: corev1.PodSpec{
110-
RestartPolicy: corev1.RestartPolicyAlways,
111-
Containers: []corev1.Container{
112-
{
113-
Image: rrdCollector.Image,
114-
ImagePullPolicy: pullPolicy,
115-
Name: "collector",
116-
Command: []string{"sleep"},
117-
Args: []string{"1000000"},
118-
VolumeMounts: []corev1.VolumeMount{
119-
{
120-
Name: "rrd",
121-
MountPath: "/rrd",
122-
},
123-
},
124-
},
125-
},
126-
Volumes: []corev1.Volume{
127-
{
128-
Name: "rrd",
129-
VolumeSource: corev1.VolumeSource{
130-
HostPath: &corev1.HostPathVolumeSource{
131-
Path: rrdCollector.HostPath,
132-
Type: &volumeType,
133-
},
134-
},
135-
},
136-
},
137-
},
138-
},
139-
},
140-
}
141-
142-
if rrdCollector.ImagePullSecret != nil && rrdCollector.ImagePullSecret.Data != nil {
143-
secretName, err := createSecret(ctx, client, namespace, rrdCollector.ImagePullSecret)
144-
if err != nil {
145-
return "", errors.Wrap(err, "failed to create secret")
146-
}
147-
ds.Spec.Template.Spec.ImagePullSecrets = append(ds.Spec.Template.Spec.ImagePullSecrets, corev1.LocalObjectReference{Name: secretName})
148-
}
149-
150-
createdDS, err := client.AppsV1().DaemonSets(namespace).Create(ctx, &ds, metav1.CreateOptions{})
151-
if err != nil {
152-
return "", errors.Wrap(err, "failed to create daemonset")
153-
}
154-
155-
// This timeout is different from collector timeout.
156-
// Time it takes to pull images should not count towards collector timeout.
157-
childCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
158-
defer cancel()
159-
for {
160-
select {
161-
case <-time.After(1 * time.Second):
162-
case <-childCtx.Done():
163-
return createdDS.Name, errors.Wrap(ctx.Err(), "failed to wait for daemonset")
164-
}
165-
166-
ds, err := client.AppsV1().DaemonSets(namespace).Get(ctx, createdDS.Name, metav1.GetOptions{})
167-
if err != nil {
168-
if !kuberneteserrors.IsNotFound(err) {
169-
continue
170-
}
171-
return createdDS.Name, errors.Wrap(err, "failed to get daemonset")
172-
}
173-
174-
if ds.Status.DesiredNumberScheduled != ds.Status.NumberReady {
175-
continue
176-
}
177-
178-
break
179-
}
180-
181-
return createdDS.Name, nil
182-
}
183-
184-
func collectRRDFiles(ctx context.Context, client *kubernetes.Clientset, c *Collector, rrdCollector *troubleshootv1beta2.Collectd, label string, namespace string) (map[string][]byte, error) {
185-
labelSelector := map[string]string{
186-
"rrd-collector": label,
187-
}
188-
opts := metav1.ListOptions{
189-
LabelSelector: labels.SelectorFromSet(labelSelector).String(),
190-
}
191-
192-
pods, err := client.CoreV1().Pods(namespace).List(ctx, opts)
193-
if err != nil {
194-
return nil, errors.Wrap(err, "list rrd collector pods")
195-
}
196-
197-
pathPrefix := path.Join("collectd", "rrd")
198-
runOutput := map[string][]byte{}
199-
for _, pod := range pods.Items {
200-
stdout, stderr, err := getFilesFromPod(ctx, client, c, pod.Name, "", namespace, "/rrd")
201-
if err != nil {
202-
runOutput[path.Join(pathPrefix, pod.Spec.NodeName)+".error"] = []byte(err.Error())
203-
if len(stdout) > 0 {
204-
runOutput[filepath.Join(pathPrefix, pod.Spec.NodeName)+".stdout"] = stdout
205-
}
206-
if len(stderr) > 0 {
207-
runOutput[filepath.Join(pathPrefix, pod.Spec.NodeName)+".stderr"] = stderr
208-
}
209-
continue
210-
}
211-
212-
runOutput[path.Join(pathPrefix, pod.Spec.NodeName)+".tar"] = stdout
213-
}
214-
215-
return runOutput, nil
11+
func Collectd(ctx context.Context, namespace string, clientConfig *restclient.Config, client kubernetes.Interface, collector *troubleshootv1beta2.Collectd) (map[string][]byte, error) {
12+
return CopyFromHost(ctx, namespace, clientConfig, client, &troubleshootv1beta2.CopyFromHost{
13+
CollectorMeta: collector.CollectorMeta,
14+
Name: "collectd/rrd",
15+
Namespace: collector.Namespace,
16+
Image: collector.Image,
17+
ImagePullPolicy: collector.ImagePullPolicy,
18+
ImagePullSecret: collector.ImagePullSecret,
19+
Timeout: collector.Timeout,
20+
HostPath: collector.HostPath,
21+
})
21622
}

0 commit comments

Comments
 (0)