diff --git a/api/grpc/mpi/v1/command.pb.go b/api/grpc/mpi/v1/command.pb.go index 7c137c5e9..1c4fa9b0c 100644 --- a/api/grpc/mpi/v1/command.pb.go +++ b/api/grpc/mpi/v1/command.pb.go @@ -8,7 +8,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.6 +// protoc-gen-go v1.36.7 // protoc (unknown) // source: mpi/v1/command.proto diff --git a/api/grpc/mpi/v1/common.pb.go b/api/grpc/mpi/v1/common.pb.go index 9ba42f536..26b95bf14 100644 --- a/api/grpc/mpi/v1/common.pb.go +++ b/api/grpc/mpi/v1/common.pb.go @@ -5,7 +5,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.6 +// protoc-gen-go v1.36.7 // protoc (unknown) // source: mpi/v1/common.proto diff --git a/api/grpc/mpi/v1/files.pb.go b/api/grpc/mpi/v1/files.pb.go index 9ebb60f91..c68585426 100644 --- a/api/grpc/mpi/v1/files.pb.go +++ b/api/grpc/mpi/v1/files.pb.go @@ -5,7 +5,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.6 +// protoc-gen-go v1.36.7 // protoc (unknown) // source: mpi/v1/files.proto diff --git a/go.mod b/go.mod index 1c1886634..3111055dc 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,8 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver v0.124.1 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/tcplogreceiver v0.124.1 github.com/open-telemetry/opentelemetry-collector-contrib/testbed v0.124.1 + github.com/prometheus/client_model v0.6.1 + github.com/prometheus/common v0.62.0 github.com/shirou/gopsutil/v4 v4.25.3 github.com/spf13/pflag v1.0.6 github.com/stretchr/testify v1.10.0 @@ -202,8 +204,6 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.16.0 // indirect github.com/rs/cors v1.11.1 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect diff --git a/test/helpers/test_containers_utils.go b/test/helpers/test_containers_utils.go index baee47d40..bf8ae6890 100644 --- a/test/helpers/test_containers_utils.go +++ b/test/helpers/test_containers_utils.go @@ -25,6 +25,13 @@ type Parameters struct { LogMessage string } +type MockCollectorContainers struct { + // AgentPlus testcontainers.Container + AgentOSS testcontainers.Container + Otel testcontainers.Container + Prometheus testcontainers.Container +} + func StartContainer( ctx context.Context, tb testing.TB, @@ -296,6 +303,161 @@ func StartAuxiliaryMockManagementPlaneGrpcContainer(ctx context.Context, tb test return container } +func StartMockCollectorStack(ctx context.Context, tb testing.TB, + containerNetwork *testcontainers.DockerNetwork, agentConfig string, +) *MockCollectorContainers { + tb.Helper() + + packageName := Env(tb, "PACKAGE_NAME") + packageRepo := Env(tb, "PACKAGES_REPO") + baseImage := Env(tb, "BASE_IMAGE") + // osRelease := Env(tb, "OS_RELEASE") + // osVersion := Env(tb, "OS_VERSION") + buildTarget := Env(tb, "BUILD_TARGET") + dockerfilePath := Env(tb, "DOCKERFILE_PATH") + // containerRegistry := Env(tb, "CONTAINER_NGINX_IMAGE_REGISTRY") + // tag := Env(tb, "TAG") + // imagePath := Env(tb, "IMAGE_PATH") + + // agentPlus, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + // ContainerRequest: testcontainers.ContainerRequest{ + // FromDockerfile: testcontainers.FromDockerfile{ + // Context: "../../../", + // Dockerfile: "./test/docker/nginx-plus/deb/Dockerfile", + // KeepImage: false, + // PrintBuildLog: true, + // BuildArgs: map[string]*string{ + // "PACKAGE_NAME": ToPtr(packageName), + // "PACKAGES_REPO": ToPtr(packageRepo), + // "BASE_IMAGE": ToPtr(baseImage), + // "OS_RELEASE": ToPtr(osRelease), + // "OS_VERSION": ToPtr(osVersion), + // "ENTRY_POINT": ToPtr("./test/docker/entrypoint.sh"), + // "CONTAINER_NGINX_IMAGE_REGISTRY": ToPtr(containerRegistry), + // "IMAGE_PATH": ToPtr(imagePath), + // "TAG": ToPtr(tag), + // }, + // BuildOptionsModifier: func(buildOptions *types.ImageBuildOptions) { + // buildOptions.Target = "install-nginx" + // }, + // }, + // Name: "agent-with-nginx-plus", + // Networks: []string{containerNetwork.Name}, + // Files: []testcontainers.ContainerFile{ + // { + // HostFilePath: agentConfig, + // ContainerFilePath: "/etc/nginx-agent/nginx-agent.conf", + // FileMode: configFilePermissions, + // }, + // { + // HostFilePath: "../../mock/collector/nginx-plus/nginx.conf", + // ContainerFilePath: "/etc/nginx/nginx.conf", + // FileMode: configFilePermissions, + // }, + // { + // HostFilePath: "../../mock/collector/nginx-plus/conf.d/default.conf", + // ContainerFilePath: "/etc/nginx/conf.d/default.conf", + // FileMode: configFilePermissions, + // }, + // }, + // }, + // Started: true, + // }) + // require.NoError(tb, err) + + agentOSS, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + FromDockerfile: testcontainers.FromDockerfile{ + Context: "../../../", + Dockerfile: dockerfilePath, + KeepImage: false, + PrintBuildLog: true, + BuildArgs: map[string]*string{ + "PACKAGE_NAME": ToPtr(packageName), + "PACKAGES_REPO": ToPtr(packageRepo), + "BASE_IMAGE": ToPtr(baseImage), + "ENTRY_POINT": ToPtr("./test/docker/entrypoint.sh"), + }, + BuildOptionsModifier: func(buildOptions *types.ImageBuildOptions) { + buildOptions.Target = buildTarget + }, + }, + Name: "agent-with-nginx-oss", + Networks: []string{containerNetwork.Name}, + Files: []testcontainers.ContainerFile{ + { + HostFilePath: agentConfig, + ContainerFilePath: "/etc/nginx-agent/nginx-agent.conf", + FileMode: configFilePermissions, + }, + { + HostFilePath: "../../mock/collector/nginx-oss/nginx.conf", + ContainerFilePath: "/etc/nginx/nginx.conf", + FileMode: configFilePermissions, + }, + { + HostFilePath: "../../mock/collector/nginx-oss/conf.d/default.conf", + ContainerFilePath: "/etc/nginx/conf.d/default.conf", + FileMode: configFilePermissions, + }, + }, + }, + Started: true, + }) + require.NoError(tb, err) + + otel, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + FromDockerfile: testcontainers.FromDockerfile{ + Context: "../../../", + Dockerfile: "./test/mock/collector/mock-collector/Dockerfile", + KeepImage: false, + PrintBuildLog: true, + }, + Name: "otel-collector", + ExposedPorts: []string{"4317/tcp", "9090/tcp", "9775/tcp"}, + Networks: []string{containerNetwork.Name}, + Files: []testcontainers.ContainerFile{ + { + HostFilePath: "../../mock/collector/otel-collector.yaml", + ContainerFilePath: "/etc/otel-collector.yaml", + FileMode: configFilePermissions, + }, + }, + WaitingFor: wait.ForLog("Everything is ready. Begin running and processing data."), + }, + Started: true, + }) + require.NoError(tb, err) + + prometheus, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: "prom/prometheus:latest", + Name: "prometheus", + ExposedPorts: []string{"9090/tcp"}, + Networks: []string{containerNetwork.Name}, + Files: []testcontainers.ContainerFile{ + { + HostFilePath: "../../mock/collector/prometheus.yaml", + ContainerFilePath: "/etc/prometheus/prometheus.yaml", + FileMode: configFilePermissions, + }, + }, + Cmd: []string{"--config.file=/etc/prometheus/prometheus.yml"}, + WaitingFor: wait.ForLog("Server is ready to receive web requests."), + }, + Started: true, + }) + require.NoError(tb, err) + + return &MockCollectorContainers{ + // AgentPlus: agentPlus, + AgentOSS: agentOSS, + Otel: otel, + Prometheus: prometheus, + } +} + func ToPtr[T any](value T) *T { return &value } @@ -359,3 +521,34 @@ func LogAndTerminateContainers( require.NoError(tb, err) } } + +func LogAndTerminateStack(ctx context.Context, tb testing.TB, + containers *MockCollectorContainers, +) { + tb.Helper() + + logAndTerminate := func(name string, container testcontainers.Container) { + if container == nil { + tb.Logf("Skipping log collection for %s: container is nil", name) + return + } + + tb.Logf("======================== Logging %s Container Logs ========================", name) + logReader, err := container.Logs(ctx) + require.NoError(tb, err) + + buf, err := io.ReadAll(logReader) + require.NoError(tb, err) + logs := string(buf) + + tb.Log(logs) + + err = container.Terminate(ctx) + require.NoError(tb, err) + } + + // logAndTerminate("agent-plus", containers.AgentPlus) + logAndTerminate("agent-oss", containers.AgentOSS) + logAndTerminate("otel", containers.Otel) + logAndTerminate("prometheus", containers.Prometheus) +} diff --git a/test/integration/metrics/metrics_test.go b/test/integration/metrics/metrics_test.go new file mode 100644 index 000000000..16136e6be --- /dev/null +++ b/test/integration/metrics/metrics_test.go @@ -0,0 +1,112 @@ +// Copyright (c) F5, Inc. +// +// This source code is licensed under the Apache License, Version 2.0 license found in the +// LICENSE file in the root directory of this source tree. + +package metrics + +import ( + "context" + "testing" + "time" + + "github.com/nginx/agent/v3/test/integration/utils" + + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/suite" +) + +type MetricsTestSuite struct { + suite.Suite + ctx context.Context + teardownTest func(testing.TB) + metricFamilies map[string]*dto.MetricFamily +} + +func (s *MetricsTestSuite) SetupSuite() { + s.ctx = context.Background() + s.teardownTest = utils.SetupMetricsTest(s.T()) + time.Sleep(30 * time.Second) + s.metricFamilies = utils.ScrapeCollectorMetricFamilies(s.T(), s.ctx, utils.MockCollectorStack.Otel) +} + +func (s *MetricsTestSuite) TearDownSuite() { + s.teardownTest(s.T()) +} + +func (s *MetricsTestSuite) TestNginxOSS_Test1_TestRequestCount() { + family := s.metricFamilies["nginx_http_request_count"] + s.T().Logf("nginx_http_request_count metric family: %v", family) + s.Require().NotNil(family) + + baselineMetric := utils.SumMetricFamily(family) + s.T().Logf("NGINX HTTP request count total: %v", baselineMetric) + + requestCount := 5 + for range requestCount { + url := "http://127.0.0.1/" + _, _, err := utils.MockCollectorStack.AgentOSS.Exec( + s.ctx, + []string{"curl", "-s", url}, + ) + s.Require().NoError(err) + } + + time.Sleep(65 * time.Second) + + s.metricFamilies = utils.ScrapeCollectorMetricFamilies(s.T(), s.ctx, utils.MockCollectorStack.Otel) + family = s.metricFamilies["nginx_http_request_count"] + s.T().Logf("nginx_http_request_count metric family: %v", family) + s.Require().NotNil(family) + + got := utils.SumMetricFamily(family) + + s.T().Logf("NGINX HTTP request count total: %v", got) + s.Require().GreaterOrEqual(got, baselineMetric+float64(requestCount)) +} + +func (s *MetricsTestSuite) TestNginxOSS_Test2_TestResponseCode() { + family := s.metricFamilies["nginx_http_response_count"] + s.T().Logf("nginx_http_response_count family: %v", family) + s.Require().NotNil(family) + + responseCodes := []string{"1xx", "2xx", "3xx", "4xx"} + codeRes := make([]float64, 0, len(responseCodes)) + for code := range responseCodes { + codeRes = append(codeRes, utils.SumMetricFamilyLabel(family, "nginx_status_range", responseCodes[code])) + s.T().Logf("NGINX HTTP response code %s total: %v", responseCodes[code], codeRes[code]) + s.Require().NotNil(codeRes[code]) + } +} + +func (s *MetricsTestSuite) TestHostMetrics_Test1_TestSystemCPUUtilization() { + family := s.metricFamilies["system_cpu_utilization"] + s.T().Logf("system_cpu_utilization metric family: %v", family) + s.Require().NotNil(family) + + cpuUtilizationSystem := utils.SumMetricFamilyLabel(family, "state", "system") + cpuUtilizationUser := utils.SumMetricFamilyLabel(family, "state", "user") + + s.T().Logf("System cpu utilization: %v", cpuUtilizationSystem) + s.T().Logf("System cpu utilization: %v", cpuUtilizationUser) + s.Require().NotNil(cpuUtilizationSystem) + s.Require().NotNil(cpuUtilizationUser) +} + +func (s *MetricsTestSuite) TestHostMetrics_Test2_TestSystemMemoryUsage() { + family := s.metricFamilies["system_memory_usage"] + s.T().Logf("system_memory_usage metric family: %v", family) + s.Require().NotNil(family) + + memoryUsageFree := utils.SumMetricFamilyLabel(family, "state", "free") + memoryUsageUsed := utils.SumMetricFamilyLabel(family, "state", "used") + + s.T().Logf("System memory usage: %v", memoryUsageFree) + s.T().Logf("System memory usage: %v", memoryUsageUsed) + s.Require().NotNil(memoryUsageFree) + s.Require().NotNil(memoryUsageUsed) +} + +func TestMetricsTestSuite(t *testing.T) { + suite.Run(t, new(MetricsTestSuite)) +} diff --git a/test/integration/utils/mock_collector_utils.go b/test/integration/utils/mock_collector_utils.go new file mode 100644 index 000000000..9e1694d11 --- /dev/null +++ b/test/integration/utils/mock_collector_utils.go @@ -0,0 +1,186 @@ +// Copyright (c) F5, Inc. +// +// This source code is licensed under the Apache License, Version 2.0 license found in the +// LICENSE file in the root directory of this source tree. + +package utils + +import ( + "bytes" + "context" + "fmt" + "net" + "net/http" + "os" + "testing" + + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + + "github.com/go-resty/resty/v2" + "github.com/testcontainers/testcontainers-go" + + "github.com/nginx/agent/v3/test/helpers" +) + +var MockCollectorStack *helpers.MockCollectorContainers + +const envContainer = "Container" + +func SetupMetricsTest(tb testing.TB) func(testing.TB) { + tb.Helper() + ctx := context.Background() + + if os.Getenv("TEST_ENV") == envContainer { + setupStackEnvironment(ctx, tb) + } + + return func(tb testing.TB) { + tb.Helper() + + if os.Getenv("TEST_ENV") == envContainer { + helpers.LogAndTerminateStack( + ctx, + tb, + MockCollectorStack, + ) + } + } +} + +func setupStackEnvironment(ctx context.Context, tb testing.TB) { + tb.Helper() + tb.Log("Running tests in a container environment") + + containerNetwork := createContainerNetwork(ctx, tb) + setupMockCollectorStack(ctx, tb, containerNetwork) +} + +func setupMockCollectorStack(ctx context.Context, tb testing.TB, containerNetwork *testcontainers.DockerNetwork) { + tb.Helper() + + tb.Log("Starting mock collector stack") + + agentConfig := "../../mock/collector/nginx-agent.conf" + MockCollectorStack = helpers.StartMockCollectorStack(ctx, tb, containerNetwork, agentConfig) +} + +func ScrapeCollectorMetricFamilies(t *testing.T, ctx context.Context, + otelContainer testcontainers.Container, +) map[string]*dto.MetricFamily { + t.Helper() + + host, _ := otelContainer.Host(ctx) + port, _ := otelContainer.MappedPort(ctx, "9775") + + address := net.JoinHostPort(host, port.Port()) + url := fmt.Sprintf("http://%s/metrics", address) + + client := resty.New() + resp, err := client.R().EnableTrace().Get(url) + if err != nil { + t.Fatalf("failed to get response from Otel Collector: %v", err) + } + if resp.StatusCode() != http.StatusOK { + t.Fatalf("Unexpected status code: %d", resp.StatusCode()) + } + + parser := expfmt.TextParser{} + metricFamilies, err := parser.TextToMetricFamilies(bytes.NewReader(resp.Body())) + if err != nil { + t.Fatalf("failed to parse metrics: %v", err) + } + + return metricFamilies +} + +func SumMetricFamily(metricFamily *dto.MetricFamily) float64 { + var total float64 + for _, metric := range metricFamily.GetMetric() { + if value := metricValue(metricFamily, metric); value != nil { + total += *value + } + } + + return total +} + +func SumMetricFamilyLabel(metricFamily *dto.MetricFamily, key, val string) float64 { + var total float64 + for _, metric := range metricFamily.GetMetric() { + labels := make(map[string]string) + for _, labelPair := range metric.GetLabel() { + labels[labelPair.GetName()] = labelPair.GetValue() + } + if labels[key] != val { + continue + } + if value := metricValue(metricFamily, metric); value != nil { + total += *value + } + } + + return total +} + +func metricValue(metricFamily *dto.MetricFamily, metric *dto.Metric) *float64 { + switch metricFamily.GetType() { + case dto.MetricType_COUNTER: + return getCounterValue(metric) + case dto.MetricType_GAUGE: + return getGaugeValue(metric) + case dto.MetricType_SUMMARY: + return getSummaryValue(metric) + case dto.MetricType_UNTYPED: + return getUntypedValue(metric) + case dto.MetricType_HISTOGRAM, dto.MetricType_GAUGE_HISTOGRAM: + return getHistogramValue(metric) + } + + return nil +} + +func getCounterValue(metric *dto.Metric) *float64 { + if counter := metric.GetCounter(); counter != nil { + val := counter.GetValue() + return &val + } + + return nil +} + +func getGaugeValue(metric *dto.Metric) *float64 { + if gauge := metric.GetGauge(); gauge != nil { + val := gauge.GetValue() + return &val + } + + return nil +} + +func getSummaryValue(metric *dto.Metric) *float64 { + if summary := metric.GetSummary(); summary != nil { + val := summary.GetSampleSum() + return &val + } + + return nil +} + +func getUntypedValue(metric *dto.Metric) *float64 { + if untyped := metric.GetUntyped(); untyped != nil { + val := untyped.GetValue() + return &val + } + + return nil +} + +func getHistogramValue(metric *dto.Metric) *float64 { + if histogram := metric.GetHistogram(); histogram != nil { + val := histogram.GetSampleSum() + return &val + } + + return nil +} diff --git a/test/mock/collector/README.md b/test/mock/collector/README.md index 938fab5f7..c06cfdd43 100644 --- a/test/mock/collector/README.md +++ b/test/mock/collector/README.md @@ -24,6 +24,11 @@ To start run everything run the following make run-mock-management-otel-collector ``` +To start everything except the NGINX Plus & NGINX App Protect run the following +``` +make run-mock-otel-collector-without-nap +``` + Once everything is started there should be 7 containers running ``` CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES @@ -43,3 +48,8 @@ To stop everything run the following ``` make stop-mock-management-otel-collector ``` + +Or run the following if you started everything except the NGINX Plus & NGINX App Protect +``` +make stop-mock-otel-collector-without-nap +```