|
| 1 | +/* |
| 2 | +Copyright 2020 The Kubernetes Authors. |
| 3 | +
|
| 4 | +Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | +you may not use this file except in compliance with the License. |
| 6 | +You may obtain a copy of the License at |
| 7 | +
|
| 8 | + http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | +
|
| 10 | +Unless required by applicable law or agreed to in writing, software |
| 11 | +distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | +See the License for the specific language governing permissions and |
| 14 | +limitations under the License. |
| 15 | +*/ |
| 16 | + |
| 17 | +package tracing |
| 18 | + |
| 19 | +import ( |
| 20 | + "context" |
| 21 | + "fmt" |
| 22 | + "strings" |
| 23 | + "time" |
| 24 | + |
| 25 | + "github.com/onsi/ginkgo" |
| 26 | + "github.com/onsi/gomega" |
| 27 | + "go.opentelemetry.io/otel/api/global" |
| 28 | + sdktrace "go.opentelemetry.io/otel/sdk/trace" |
| 29 | + v1 "k8s.io/api/core/v1" |
| 30 | + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 31 | + clientset "k8s.io/client-go/kubernetes" |
| 32 | + "k8s.io/kubernetes/test/e2e/framework" |
| 33 | + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" |
| 34 | + e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" |
| 35 | + e2essh "k8s.io/kubernetes/test/e2e/framework/ssh" |
| 36 | + instrumentation "k8s.io/kubernetes/test/e2e/instrumentation/common" |
| 37 | +) |
| 38 | + |
| 39 | +const ( |
| 40 | + podName = "otel-collector" |
| 41 | + containerName = "collector" |
| 42 | +) |
| 43 | + |
| 44 | +// The API Server Tracing test ensures that an opentelemetry collector can |
| 45 | +// collect traces from the API Server, and that context is correctly propagated |
| 46 | +var _ = instrumentation.SIGDescribe("[Feature:APIServerTracing]", func() { |
| 47 | + f := framework.NewDefaultFramework("apiserver-tracing") |
| 48 | + var c clientset.Interface |
| 49 | + var otelPod *v1.Pod |
| 50 | + |
| 51 | + ginkgo.BeforeEach(func() { |
| 52 | + config, err := framework.LoadConfig() |
| 53 | + framework.ExpectNoError(err) |
| 54 | + c, err = clientset.NewForConfig(config) |
| 55 | + framework.ExpectNoError(err) |
| 56 | + |
| 57 | + ginkgo.By("Creating an opentelemetry collector on the master node, which logs spans to stdout.") |
| 58 | + masterNode, err := masterNodeName(c) |
| 59 | + framework.ExpectNoError(err) |
| 60 | + otelPod, err = c.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), opentelemetryCollectorPod(masterNode), metav1.CreateOptions{}) |
| 61 | + framework.ExpectNoError(err) |
| 62 | + |
| 63 | + _, err = c.CoreV1().ConfigMaps(f.Namespace.Name).Create(context.TODO(), opentelemetryConfigmap(), metav1.CreateOptions{}) |
| 64 | + framework.ExpectNoError(err) |
| 65 | + |
| 66 | + ready := e2epod.CheckPodsRunningReady(f.ClientSet, f.Namespace.Name, []string{podName}, 20*time.Second) |
| 67 | + framework.ExpectEqual(ready, true) |
| 68 | + }) |
| 69 | + |
| 70 | + ginkgo.It("should send a request with a sampled trace context, and observe child spans from the collector pod", func() { |
| 71 | + e2eskipper.SkipUnlessSSHKeyPresent() |
| 72 | + |
| 73 | + ginkgo.By("Setting up OpenTelemetry to sample all requests") |
| 74 | + tp := sdktrace.NewTracerProvider( |
| 75 | + sdktrace.WithConfig(sdktrace.Config{ |
| 76 | + DefaultSampler: sdktrace.AlwaysSample()}, |
| 77 | + )) |
| 78 | + // This is needed because the no-op tracer doesn't propagate the SpanContext |
| 79 | + // https://github.com/open-telemetry/opentelemetry-go/issues/877#issuecomment-651398357 |
| 80 | + global.SetTracerProvider(tp) |
| 81 | + |
| 82 | + ginkgo.By("Creating a context with a sampled parent span") |
| 83 | + ctx, span := tp.Tracer("apiservertest").Start(context.Background(), "OpenTelemetrySpan") |
| 84 | + |
| 85 | + traceID := span.SpanContext().TraceID |
| 86 | + ginkgo.By(fmt.Sprintf("Checking for Trace ID: %v in logs.", span.SpanContext().TraceID)) |
| 87 | + gomega.Eventually(func() error { |
| 88 | + // Send any request using the context with a sampled span |
| 89 | + _, err := c.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) |
| 90 | + if err != nil { |
| 91 | + return err |
| 92 | + } |
| 93 | + |
| 94 | + // Get logs from the opentelemetry collector pod on the master. |
| 95 | + // We must use SSH because we can't fetch logs from master pods |
| 96 | + // using the pod logs subresource. |
| 97 | + result, err := e2essh.SSH( |
| 98 | + fmt.Sprintf("sudo cat /var/log/pods/%v_%v_%v/%v/*", f.Namespace.Name, podName, otelPod.UID, containerName), |
| 99 | + framework.APIAddress()+":22", |
| 100 | + framework.TestContext.Provider, |
| 101 | + ) |
| 102 | + logs := result.Stdout |
| 103 | + if err != nil { |
| 104 | + return err |
| 105 | + } |
| 106 | + if result.Stderr != "" { |
| 107 | + return fmt.Errorf("Non-empty stderr when querying for logs on the master: %v", result.Stderr) |
| 108 | + } |
| 109 | + // Check the opentelemetry collector logs to see if they contain our trace ID |
| 110 | + if strings.Contains(logs, traceID.String()) { |
| 111 | + return nil |
| 112 | + } |
| 113 | + return fmt.Errorf("Failed to find trace ID %v in log: \n%v", traceID.String(), logs) |
| 114 | + }, 2*time.Minute, 10*time.Second).Should(gomega.BeNil()) |
| 115 | + |
| 116 | + }) |
| 117 | +}) |
| 118 | + |
| 119 | +func masterNodeName(c clientset.Interface) (string, error) { |
| 120 | + nodes, err := c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) |
| 121 | + framework.ExpectNoError(err) |
| 122 | + for _, node := range nodes.Items { |
| 123 | + if strings.HasSuffix(node.Name, "master") { |
| 124 | + return node.Name, nil |
| 125 | + } |
| 126 | + } |
| 127 | + return "", fmt.Errorf("Didn't find master node in list of nodes") |
| 128 | +} |
| 129 | + |
| 130 | +func opentelemetryCollectorPod(masterNode string) *v1.Pod { |
| 131 | + return &v1.Pod{ |
| 132 | + ObjectMeta: metav1.ObjectMeta{ |
| 133 | + Name: podName, |
| 134 | + }, |
| 135 | + Spec: v1.PodSpec{ |
| 136 | + NodeName: masterNode, |
| 137 | + Containers: []v1.Container{{ |
| 138 | + Name: containerName, |
| 139 | + Image: "otel/opentelemetry-collector-dev:latest", |
| 140 | + Args: []string{ |
| 141 | + "--config=/conf/otel-collector-config.yaml", |
| 142 | + }, |
| 143 | + Ports: []v1.ContainerPort{{ |
| 144 | + ContainerPort: 55680, |
| 145 | + HostPort: 55680, |
| 146 | + }}, |
| 147 | + VolumeMounts: []v1.VolumeMount{{ |
| 148 | + Name: "otel-collector-config-vol", |
| 149 | + MountPath: "/conf", |
| 150 | + }}, |
| 151 | + }}, |
| 152 | + Volumes: []v1.Volume{{ |
| 153 | + Name: "otel-collector-config-vol", |
| 154 | + VolumeSource: v1.VolumeSource{ |
| 155 | + ConfigMap: &v1.ConfigMapVolumeSource{ |
| 156 | + LocalObjectReference: v1.LocalObjectReference{ |
| 157 | + Name: "otel-collector-conf", |
| 158 | + }, |
| 159 | + }, |
| 160 | + }, |
| 161 | + }}, |
| 162 | + }, |
| 163 | + } |
| 164 | +} |
| 165 | + |
| 166 | +func opentelemetryConfigmap() *v1.ConfigMap { |
| 167 | + return &v1.ConfigMap{ |
| 168 | + ObjectMeta: metav1.ObjectMeta{ |
| 169 | + Name: "otel-collector-conf", |
| 170 | + }, |
| 171 | + Data: map[string]string{ |
| 172 | + "otel-collector-config.yaml": `receivers: |
| 173 | + otlp: |
| 174 | + protocols: |
| 175 | + grpc: |
| 176 | + http: |
| 177 | +exporters: |
| 178 | + logging: |
| 179 | + logLevel: debug |
| 180 | +service: |
| 181 | + pipelines: |
| 182 | + traces: |
| 183 | + receivers: [otlp] |
| 184 | + exporters: [logging]`, |
| 185 | + }, |
| 186 | + } |
| 187 | +} |
0 commit comments