Skip to content

Commit 0f11516

Browse files
authored
Add quickstarttest library (#58)
Which is used to test the instrumentation quickstarts. See the included README.md file.
1 parent cb0c648 commit 0f11516

File tree

3 files changed

+337
-0
lines changed

3 files changed

+337
-0
lines changed

quickstarttest/README.md

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
This go module is used to test the docker compose based OpenTelemetry instrumentation quickstarts:
2+
3+
- https://cloud.google.com/stackdriver/docs/instrumentation/setup/go
4+
- https://cloud.google.com/stackdriver/docs/instrumentation/setup/java
5+
- https://cloud.google.com/stackdriver/docs/instrumentation/setup/nodejs
6+
- https://cloud.google.com/stackdriver/docs/instrumentation/setup/python
7+
8+
9+
The repos hosting the quickstart code use this lib by writing a test that calls
10+
`InstrumentationQuickstartTest()`. The tests run in a Cloud Build trigger so they can access
11+
the Cloud Observability APIs.
12+
13+
`InstrumentationQuickstartTest()` runs the instrumentation quickstart docker compose setup in
14+
the cwd and verifies that metrics, logs, and traces are successfully sent from the collector to
15+
GCP. It checks the collector's self observability prometheus metrics to verify that the
16+
exporters were successful.
17+
18+
The `COMPOSE_OVERRIDE_FILE` environment variable can be set to a comma-separated list of paths
19+
to additional compose files to pass to docker compose (see
20+
https://docs.docker.com/compose/multiple-compose-files/merge/). This is used in the Cloud Build
21+
triggers to connect the docker containers to the [`cloudbuild` docker
22+
network](https://cloud.google.com/build/docs/build-config-file-schema#network) for ADC. It can
23+
also be used to run the test with `GOOGLE_APPLICATION_CREDENTIALS` for example.

quickstarttest/testcases.go

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
// Copyright 2024 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package quickstarttest
16+
17+
import (
18+
"context"
19+
"fmt"
20+
"net/http"
21+
"os"
22+
"path/filepath"
23+
"strings"
24+
"testing"
25+
"time"
26+
27+
dto "github.com/prometheus/client_model/go"
28+
"github.com/prometheus/common/expfmt"
29+
"github.com/stretchr/testify/assert"
30+
"github.com/stretchr/testify/require"
31+
"github.com/testcontainers/testcontainers-go/modules/compose"
32+
"github.com/testcontainers/testcontainers-go/wait"
33+
)
34+
35+
const (
36+
sentItemsThreshold = 50.0
37+
)
38+
39+
type testCase struct {
40+
metricName string
41+
exporter string
42+
threshold float64
43+
}
44+
45+
var testCases = []testCase{
46+
{metricName: "otelcol_exporter_sent_spans", exporter: "googlecloud", threshold: sentItemsThreshold},
47+
{metricName: "otelcol_exporter_sent_log_records", exporter: "googlecloud", threshold: sentItemsThreshold},
48+
{metricName: "otelcol_exporter_sent_metric_points", exporter: "googlemanagedprometheus", threshold: sentItemsThreshold},
49+
}
50+
51+
// InstrumentationQuickstartTest runs the instrumentation quickstart docker compose setup in
52+
// the quickstartRoot directory and verifies that metrics, logs, and traces are successfully
53+
// sent from the collector to GCP.
54+
//
55+
// Respects a COMPOSE_OVERRIDE_FILE environment variable set to a comma-separated list of paths
56+
// to additional compose files to include.
57+
func InstrumentationQuickstartTest(t *testing.T, quickstartRoot string) {
58+
ctx := context.Background()
59+
composeStack := composeUp(ctx, t, quickstartRoot)
60+
61+
// Let the docker compose app run until some spans/logs/metrics are sent to GCP
62+
t.Logf("Compose stack is up, waiting for prometheus metrics indicating successful export")
63+
64+
// Check the collector's self-observability prometheus metrics to see that exports to GCP were successful.
65+
for _, tc := range testCases {
66+
t.Run(tc.metricName, func(t *testing.T) {
67+
require.EventuallyWithT(
68+
t,
69+
func(collect *assert.CollectT) {
70+
promMetrics, err := getPromMetrics(ctx, composeStack)
71+
if !assert.NoError(collect, err) {
72+
return
73+
}
74+
verifyPromMetric(collect, promMetrics, tc)
75+
},
76+
time.Minute*2, // wait for up to
77+
time.Second, // check at interval
78+
)
79+
})
80+
}
81+
}
82+
83+
func composeUp(ctx context.Context, t *testing.T, quickstartRoot string) compose.ComposeStack {
84+
composeFiles := []string{filepath.Join(quickstartRoot, "docker-compose.yaml")}
85+
if composeOverrideFile := os.Getenv("COMPOSE_OVERRIDE_FILE"); composeOverrideFile != "" {
86+
composeFiles = append(composeFiles, strings.Split(composeOverrideFile, ",")...)
87+
}
88+
89+
var (
90+
composeStack compose.ComposeStack
91+
err error
92+
)
93+
composeStack, err = compose.NewDockerCompose(composeFiles...)
94+
require.NoError(t, err)
95+
96+
require.NoError(t, err)
97+
composeStack = composeStack.WithOsEnv().
98+
WaitForService("app", wait.ForHTTP("/single").WithPort("8080")).
99+
WaitForService("otelcol", wait.ForHTTP("/metrics").WithPort("8888"))
100+
101+
t.Cleanup(func() {
102+
require.NoError(t, composeStack.Down(ctx, compose.RemoveOrphans(true)))
103+
})
104+
require.NoError(t, composeStack.Up(ctx))
105+
return composeStack
106+
}
107+
108+
func getPromMetrics(ctx context.Context, composeStack compose.ComposeStack) (map[string]*dto.MetricFamily, error) {
109+
promUri, err := getPromEndpoint(ctx, composeStack)
110+
if err != nil {
111+
return nil, err
112+
}
113+
114+
resp, err := http.Get(promUri)
115+
if err != nil {
116+
return nil, err
117+
}
118+
defer resp.Body.Close()
119+
120+
var parser expfmt.TextParser
121+
parsed, err := parser.TextToMetricFamilies(resp.Body)
122+
if err != nil {
123+
return nil, err
124+
}
125+
return parsed, nil
126+
}
127+
128+
func getPromEndpoint(ctx context.Context, composeStack compose.ComposeStack) (string, error) {
129+
collectorContainer, err := composeStack.ServiceContainer(ctx, "otelcol")
130+
if err != nil {
131+
return "", err
132+
}
133+
collectorHost, err := collectorContainer.Host(ctx)
134+
if err != nil {
135+
return "", err
136+
}
137+
collectorPort, err := collectorContainer.MappedPort(ctx, "8888")
138+
if err != nil {
139+
return "", err
140+
}
141+
return fmt.Sprintf("http://%s:%s/metrics", collectorHost, collectorPort.Port()), nil
142+
}
143+
144+
func verifyPromMetric(t assert.TestingT, promMetrics map[string]*dto.MetricFamily, tc testCase) {
145+
if !assert.Contains(t, promMetrics, tc.metricName, "prometheus metrics do not contain %v:\n%v", tc.metricName, promMetrics) {
146+
return
147+
}
148+
mf := promMetrics[tc.metricName]
149+
150+
for _, metric := range mf.Metric {
151+
for _, labelPair := range metric.GetLabel() {
152+
if labelPair.GetName() == "exporter" && labelPair.GetValue() == tc.exporter {
153+
value := metric.GetCounter().GetValue()
154+
assert.Greater(t, value, tc.threshold, "Metric %v was expected to have value > %v, got %v", metric, sentItemsThreshold, value)
155+
return
156+
}
157+
}
158+
}
159+
assert.Fail(t, "Could not find a metric sample for exporter=%v, got metrics %v", tc.exporter, mf)
160+
}

quickstarttest/testcases_test.go

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
// Copyright 2024 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package quickstarttest
16+
17+
import (
18+
"fmt"
19+
"strings"
20+
"testing"
21+
22+
"github.com/prometheus/common/expfmt"
23+
"github.com/stretchr/testify/require"
24+
)
25+
26+
type MockT struct {
27+
Failed bool
28+
Message string
29+
}
30+
31+
func (t *MockT) Errorf(format string, args ...any) {
32+
t.Failed = true
33+
t.Message = fmt.Sprintf(format, args...)
34+
}
35+
36+
func TestVerifyPromMetric(t *testing.T) {
37+
tcs := []struct {
38+
name string
39+
textFormat string
40+
testCase testCase
41+
expectFail bool
42+
}{
43+
{
44+
name: "metric is above threshold pass",
45+
textFormat: `
46+
# HELP otelcol_exporter_sent_log_records Number of log record successfully sent to destination.
47+
# TYPE otelcol_exporter_sent_log_records counter
48+
otelcol_exporter_sent_log_records{exporter="googlecloud"} 631
49+
`,
50+
testCase: testCase{
51+
exporter: "googlecloud",
52+
metricName: "otelcol_exporter_sent_log_records",
53+
threshold: 100,
54+
},
55+
},
56+
{
57+
name: "metric is below threshold fail",
58+
textFormat: `
59+
# HELP otelcol_exporter_sent_log_records Number of log record successfully sent to destination.
60+
# TYPE otelcol_exporter_sent_log_records counter
61+
otelcol_exporter_sent_log_records{exporter="googlecloud"} 1
62+
`,
63+
expectFail: true,
64+
testCase: testCase{
65+
exporter: "googlecloud",
66+
metricName: "otelcol_exporter_sent_log_records",
67+
threshold: 100,
68+
},
69+
},
70+
{
71+
name: "metric is not present fail",
72+
textFormat: ``,
73+
testCase: testCase{
74+
exporter: "googlecloud",
75+
metricName: "otelcol_exporter_sent_log_records",
76+
threshold: 100,
77+
},
78+
expectFail: true,
79+
},
80+
{
81+
name: "exporter is not present fail",
82+
textFormat: `
83+
# HELP otelcol_exporter_sent_log_records Number of log record successfully sent to destination.
84+
# TYPE otelcol_exporter_sent_log_records counter
85+
otelcol_exporter_sent_log_records{exporter="googlecloud"} 631
86+
`,
87+
expectFail: true,
88+
testCase: testCase{
89+
exporter: "fooexporter",
90+
metricName: "otelcol_exporter_sent_log_records",
91+
threshold: 100,
92+
},
93+
},
94+
}
95+
for _, tc := range tcs {
96+
t.Run(tc.name, func(t *testing.T) {
97+
mockT := &MockT{}
98+
var parser expfmt.TextParser
99+
actual, err := parser.TextToMetricFamilies(strings.NewReader(tc.textFormat))
100+
require.NoError(t, err)
101+
verifyPromMetric(mockT, actual, tc.testCase)
102+
103+
if tc.expectFail {
104+
require.True(t, mockT.Failed, "Expected test case to fail but passed")
105+
} else {
106+
require.Falsef(t, mockT.Failed, "Expected test case to pass but failed with: %v", mockT.Message)
107+
}
108+
})
109+
}
110+
}
111+
112+
func TestVerifyPromRealTestCasesSuccess(t *testing.T) {
113+
var parser expfmt.TextParser
114+
// Taken from a real run of the quickstart
115+
actual, err := parser.TextToMetricFamilies(strings.NewReader(`
116+
# HELP otelcol_exporter_sent_log_records Number of log record successfully sent to destination.
117+
# TYPE otelcol_exporter_sent_log_records counter
118+
otelcol_exporter_sent_log_records{exporter="googlecloud",service_instance_id="cc2396b4-e313-4c5a-8c35-0cb221a02fa8",service_name="otelcol-contrib",service_version="0.107.0"} 437
119+
# HELP otelcol_exporter_sent_metric_points Number of metric points successfully sent to destination.
120+
# TYPE otelcol_exporter_sent_metric_points counter
121+
otelcol_exporter_sent_metric_points{exporter="googlemanagedprometheus",service_instance_id="cc2396b4-e313-4c5a-8c35-0cb221a02fa8",service_name="otelcol-contrib",service_version="0.107.0"} 333
122+
# HELP otelcol_exporter_sent_spans Number of spans successfully sent to destination.
123+
# TYPE otelcol_exporter_sent_spans counter
124+
otelcol_exporter_sent_spans{exporter="googlecloud",service_instance_id="cc2396b4-e313-4c5a-8c35-0cb221a02fa8",service_name="otelcol-contrib",service_version="0.107.0"} 499
125+
`))
126+
require.NoError(t, err)
127+
128+
for _, tc := range testCases {
129+
verifyPromMetric(t, actual, tc)
130+
}
131+
}
132+
133+
func TestVerifyPromRealTestCasesFails(t *testing.T) {
134+
var parser expfmt.TextParser
135+
// Taken from a real run of the quickstart
136+
actual, err := parser.TextToMetricFamilies(strings.NewReader(`
137+
# HELP otelcol_exporter_sent_log_records Number of log record successfully sent to destination.
138+
# TYPE otelcol_exporter_sent_log_records counter
139+
otelcol_exporter_sent_log_records{exporter="googlecloud",service_instance_id="cc2396b4-e313-4c5a-8c35-0cb221a02fa8",service_name="otelcol-contrib",service_version="0.107.0"} 0
140+
# HELP otelcol_exporter_sent_metric_points Number of metric points successfully sent to destination.
141+
# TYPE otelcol_exporter_sent_metric_points counter
142+
otelcol_exporter_sent_metric_points{exporter="googlemanagedprometheus",service_instance_id="cc2396b4-e313-4c5a-8c35-0cb221a02fa8",service_name="otelcol-contrib",service_version="0.107.0"} 0
143+
# HELP otelcol_exporter_sent_spans Number of spans successfully sent to destination.
144+
# TYPE otelcol_exporter_sent_spans counter
145+
otelcol_exporter_sent_spans{exporter="googlecloud",service_instance_id="cc2396b4-e313-4c5a-8c35-0cb221a02fa8",service_name="otelcol-contrib",service_version="0.107.0"} 0
146+
`))
147+
require.NoError(t, err)
148+
149+
mockT := &MockT{}
150+
for _, tc := range testCases {
151+
verifyPromMetric(mockT, actual, tc)
152+
}
153+
require.True(t, mockT.Failed, "Expected test case to fail but passed")
154+
}

0 commit comments

Comments
 (0)