From 7135204304af22de461b1cd2b9dcc98044b763d2 Mon Sep 17 00:00:00 2001 From: Paramadon Date: Tue, 8 Oct 2024 18:04:05 -0400 Subject: [PATCH 01/19] Adding CI JMX pipeline --- cmd/config-translator/translator_test.go | 6 + .../invalidContainerInsightsJmx.json | 14 ++ .../validContainerInsightsJmx.json | 16 +++ .../sampleConfig/container_insights_jmx.conf | 0 .../sampleConfig/container_insights_jmx.json | 16 +++ .../sampleConfig/container_insights_jmx.yaml | 0 translator/tocwconfig/tocwconfig_test.go | 9 ++ translator/translate/otel/common/common.go | 6 +- .../exporter/awsemf/awsemfjmx_config.yaml | 37 +++++ .../otel/exporter/awsemf/translator.go | 10 +- .../containerinsightsjmx/translator.go | 77 ++++++++++ .../containerinsightsjmx/translator_test.go | 72 ++++++++++ .../jmxfilterprocessor/testdata/config.json | 20 +++ .../jmxfilterprocessor/testdata/config.yaml | 25 ++++ .../jmxfilterprocessor/translator.go | 82 +++++++++++ .../jmxfilterprocessor/translator_test.go | 81 +++++++++++ .../testdata/config.json | 17 +++ .../testdata/config.yaml | 28 ++++ .../jmxtransformprocessor/translate.go | 92 ++++++++++++ .../jmxtransformprocessor/translator_test.go | 66 +++++++++ .../configs/config.yaml | 134 ++++++++++++++++++ .../metrictransformprocessorjmx/translator.go | 37 +++++ .../resourcedetectionjmx/configs/config.yaml | 9 ++ .../resourcedetectionjmx/translator.go | 61 ++++++++ .../processor/resourceprocessor/translator.go | 48 +++++++ .../receiver/awscontainerinsight/utils.go | 6 + translator/translate/otel/translate_otel.go | 2 + 27 files changed, 968 insertions(+), 3 deletions(-) create mode 100644 translator/config/sampleSchema/invalidContainerInsightsJmx.json create mode 100644 translator/config/sampleSchema/validContainerInsightsJmx.json create mode 100644 translator/tocwconfig/sampleConfig/container_insights_jmx.conf create mode 100644 translator/tocwconfig/sampleConfig/container_insights_jmx.json create mode 100644 translator/tocwconfig/sampleConfig/container_insights_jmx.yaml create mode 100644 translator/translate/otel/exporter/awsemf/awsemfjmx_config.yaml create mode 100644 translator/translate/otel/pipeline/containerinsightsjmx/translator.go create mode 100644 translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go create mode 100644 translator/translate/otel/processor/jmxfilterprocessor/testdata/config.json create mode 100644 translator/translate/otel/processor/jmxfilterprocessor/testdata/config.yaml create mode 100644 translator/translate/otel/processor/jmxfilterprocessor/translator.go create mode 100644 translator/translate/otel/processor/jmxfilterprocessor/translator_test.go create mode 100644 translator/translate/otel/processor/jmxtransformprocessor/testdata/config.json create mode 100644 translator/translate/otel/processor/jmxtransformprocessor/testdata/config.yaml create mode 100644 translator/translate/otel/processor/jmxtransformprocessor/translate.go create mode 100644 translator/translate/otel/processor/jmxtransformprocessor/translator_test.go create mode 100644 translator/translate/otel/processor/metrictransformprocessorjmx/configs/config.yaml create mode 100644 translator/translate/otel/processor/metrictransformprocessorjmx/translator.go create mode 100644 translator/translate/otel/processor/resourcedetectionjmx/configs/config.yaml create mode 100644 translator/translate/otel/processor/resourcedetectionjmx/translator.go diff --git a/cmd/config-translator/translator_test.go b/cmd/config-translator/translator_test.go index e9298c3807..5220075e77 100644 --- a/cmd/config-translator/translator_test.go +++ b/cmd/config-translator/translator_test.go @@ -187,6 +187,12 @@ func TestMetricsDestinationsConfig(t *testing.T) { expectedErrorMap["required"] = 1 checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidMetricsDestinations.json", false, expectedErrorMap) } +func TestJmxConfig(t *testing.T) { + checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/validContainerInsightsJmx.json", true, map[string]int{}) + expectedErrorMap := map[string]int{} + expectedErrorMap["required"] = 1 + //checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidContainerInsightsJmx.json", false, expectedErrorMap) +} // Validate all sampleConfig files schema func TestSampleConfigSchema(t *testing.T) { diff --git a/translator/config/sampleSchema/invalidContainerInsightsJmx.json b/translator/config/sampleSchema/invalidContainerInsightsJmx.json new file mode 100644 index 0000000000..d7ff78abfe --- /dev/null +++ b/translator/config/sampleSchema/invalidContainerInsightsJmx.json @@ -0,0 +1,14 @@ +{ + "agent": { + "region": "us-west-2" + }, + "logs": { + "metrics_collected": { + "emf": { + }, + "kubernete": { + } + }, + "force_flush_interval": 5 + } +} \ No newline at end of file diff --git a/translator/config/sampleSchema/validContainerInsightsJmx.json b/translator/config/sampleSchema/validContainerInsightsJmx.json new file mode 100644 index 0000000000..29668bf013 --- /dev/null +++ b/translator/config/sampleSchema/validContainerInsightsJmx.json @@ -0,0 +1,16 @@ +{ + "agent": { + "region": "us-west-2" + }, + "logs": { + "metrics_collected": { + "emf": { + }, + "kubernetes": { + "cluster_name": "jvmCluster", + "namespace":"siprmp-jmx" + } + }, + "force_flush_interval": 5 + } +} \ No newline at end of file diff --git a/translator/tocwconfig/sampleConfig/container_insights_jmx.conf b/translator/tocwconfig/sampleConfig/container_insights_jmx.conf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/translator/tocwconfig/sampleConfig/container_insights_jmx.json b/translator/tocwconfig/sampleConfig/container_insights_jmx.json new file mode 100644 index 0000000000..29668bf013 --- /dev/null +++ b/translator/tocwconfig/sampleConfig/container_insights_jmx.json @@ -0,0 +1,16 @@ +{ + "agent": { + "region": "us-west-2" + }, + "logs": { + "metrics_collected": { + "emf": { + }, + "kubernetes": { + "cluster_name": "jvmCluster", + "namespace":"siprmp-jmx" + } + }, + "force_flush_interval": 5 + } +} \ No newline at end of file diff --git a/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml b/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/translator/tocwconfig/tocwconfig_test.go b/translator/tocwconfig/tocwconfig_test.go index 52755f6e93..5a82921043 100644 --- a/translator/tocwconfig/tocwconfig_test.go +++ b/translator/tocwconfig/tocwconfig_test.go @@ -81,7 +81,16 @@ func TestGenericAppSignalsConfig(t *testing.T) { checkTranslation(t, "base_appsignals_config", "linux", expectedEnvVars, "") checkTranslation(t, "base_appsignals_config", "windows", expectedEnvVars, "") } +func TestContØainerInsightsJMX(t *testing.T) { + resetContext(t) + context.CurrentContext().SetRunInContainer(false) + context.CurrentContext().SetMode(config.ModeOnPremise) + t.Setenv(config.HOST_NAME, "host_name_from_env") + t.Setenv(config.HOST_IP, "127.0.0.1") + expectedEnvVars := map[string]string{"CWAGENT_LOG_LEVEL": "DEBUG"} + checkTranslation(t, "container_insights_jmx", "linux", expectedEnvVars, "") +} func TestGenericAppSignalsFallbackConfig(t *testing.T) { resetContext(t) context.CurrentContext().SetRunInContainer(false) diff --git a/translator/translate/otel/common/common.go b/translator/translate/otel/common/common.go index 51dab5bcda..a27b16c14b 100644 --- a/translator/translate/otel/common/common.go +++ b/translator/translate/otel/common/common.go @@ -95,8 +95,10 @@ var ( component.DataTypeTraces: {AppSignalsTraces, AppSignalsTracesFallback}, component.DataTypeMetrics: {AppSignalsMetrics, AppSignalsMetricsFallback}, } - JmxConfigKey = ConfigKey(MetricsKey, MetricsCollectedKey, JmxKey) - JmxTargets = []string{"activemq", "cassandra", "hbase", "hadoop", "jetty", "jvm", "kafka", "kafka-consumer", "kafka-producer", "solr", "tomcat", "wildfly"} + JmxConfigKey = ConfigKey(MetricsKey, MetricsCollectedKey, JmxKey) + ContainerInsightsConfigKey = ConfigKey(LogsKey, MetricsCollectedKey, KubernetesKey) + + JmxTargets = []string{"activemq", "cassandra", "hbase", "hadoop", "jetty", "jvm", "kafka", "kafka-consumer", "kafka-producer", "solr", "tomcat", "wildfly"} AgentDebugConfigKey = ConfigKey(AgentKey, DebugKey) MetricsAggregationDimensionsKey = ConfigKey(MetricsKey, AggregationDimensionsKey) diff --git a/translator/translate/otel/exporter/awsemf/awsemfjmx_config.yaml b/translator/translate/otel/exporter/awsemf/awsemfjmx_config.yaml new file mode 100644 index 0000000000..1fc49e024c --- /dev/null +++ b/translator/translate/otel/exporter/awsemf/awsemfjmx_config.yaml @@ -0,0 +1,37 @@ +namespace: ContainerInsights/Prometheus +log_group_name: '/aws/containerinsights/{ClusterName}/jmx' +log_stream_name: '{NodeName}' +detailed_metrics: false +dimension_rollup_option: NoDimensionRollup +version: "0" +retain_initial_value_of_delta_metric: false +resource_to_telemetry_conversion: + enabled: true +metric_declarations: + - dimensions: [[ClusterName,Namespace]] + metric_name_selectors: + - java_lang_operatingsystem_freeswapspacesize + - java_lang_operatingsystem_availableprocessors + - catalina_manager_rejectedsessions + - catalina_globalrequestprocessor_bytesreceived + - catalina_globalrequestprocessor_processingtime + - jvm_memory_pool_bytes_used + - java_lang_operatingsystem_systemcpuload + - java_lang_operatingsystem_totalphysicalmemorysize + - java_lang_operatingsystem_freephysicalmemorysize + - java_lang_operatingsystem_openfiledescriptorcount + - catalina_manager_activesessions + - java_lang_operatingsystem_totalswapspacesize + - java_lang_operatingsystem_processcpuload + - catalina_globalrequestprocessor_requestcount + - catalina_globalrequestprocessor_errorcount + - jvm_threads_daemon + - catalina_globalrequestprocessor_bytessent + - jvm_classes_loaded + - jvm_threads_current + - dimensions: [[ClusterName, Namespace, area]] + metric_name_selectors: + - jvm_memory_bytes_used + - dimensions: [[ClusterName,Namespace, pool]] + metric_name_selectors: + - jvm_memory_pool_bytes_used \ No newline at end of file diff --git a/translator/translate/otel/exporter/awsemf/translator.go b/translator/translate/otel/exporter/awsemf/translator.go index 764fa3532d..a20282c196 100644 --- a/translator/translate/otel/exporter/awsemf/translator.go +++ b/translator/translate/otel/exporter/awsemf/translator.go @@ -46,6 +46,9 @@ var appSignalsConfigK8s string //go:embed appsignals_config_generic.yaml var appSignalsConfigGeneric string +//go:embed awsemfjmx_config.yaml +var awsEmfJmxDefaultConfig string + var ( ecsBasePathKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.ECSKey) kubernetesBasePathKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.KubernetesKey) @@ -82,6 +85,8 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { defaultConfig := defaultGenericConfig if t.isAppSignals(c) { defaultConfig = getAppSignalsConfig() + } else if t.isCiJMX(c) { + defaultConfig = awsEmfJmxDefaultConfig } else if isEcs(c) { defaultConfig = defaultEcsConfig } else if isKubernetes(c) { @@ -128,7 +133,7 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { if err := setEcsFields(c, cfg); err != nil { return nil, err } - } else if isKubernetes(c) { + } else if isKubernetes(c) && !t.isCiJMX(c) { if err := setKubernetesFields(c, cfg); err != nil { return nil, err } @@ -168,6 +173,9 @@ func getAppSignalsConfig() string { func (t *translator) isAppSignals(conf *confmap.Conf) bool { return (t.name == common.AppSignals || t.name == common.AppSignalsFallback) && (conf.IsSet(common.AppSignalsMetrics) || conf.IsSet(common.AppSignalsTraces) || conf.IsSet(common.AppSignalsMetricsFallback) || conf.IsSet(common.AppSignalsTracesFallback)) } +func (t *translator) isCiJMX(conf *confmap.Conf) bool { + return (t.name == "containerinsightsjmx") && (conf.IsSet(common.ContainerInsightsConfigKey)) +} func isEcs(conf *confmap.Conf) bool { return conf.IsSet(ecsBasePathKey) diff --git a/translator/translate/otel/pipeline/containerinsightsjmx/translator.go b/translator/translate/otel/pipeline/containerinsightsjmx/translator.go new file mode 100644 index 0000000000..2a3e6e65a3 --- /dev/null +++ b/translator/translate/otel/pipeline/containerinsightsjmx/translator.go @@ -0,0 +1,77 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package containerinsightsjmx + +import ( + "fmt" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/awsemf" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/debug" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/cumulativetodeltaprocessor" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/jmxfilterprocessor" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/jmxtransformprocessor" + metricstransformprocessorjmx "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/metrictransformprocessorjmx" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/resourcedetectionjmx" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/resourceprocessor" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/otlp" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" +) + +const ( + pipelineName = "containerinsightsjmx" + clusterName = "cluster_name" +) + +var ( + baseKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey) + eksKey = common.ConfigKey(baseKey, common.KubernetesKey) + jmxKey = common.ConfigKey(eksKey, "jmx_container_insights") +) + +type translator struct { +} + +var _ common.Translator[*common.ComponentTranslators] = (*translator)(nil) + +func NewTranslator() common.Translator[*common.ComponentTranslators] { + return &translator{} +} + +func (t *translator) ID() component.ID { + return component.NewIDWithName(component.DataTypeMetrics, pipelineName) +} + +// Translate creates a pipeline for container insights if the logs.metrics_collected.ecs or logs.metrics_collected.kubernetes +// section is present. +func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators, error) { + if conf == nil || (!conf.IsSet(jmxKey)) { + return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: fmt.Sprint(jmxKey)} + } + if val, _ := common.GetBool(conf, jmxKey); !val { + return nil, nil + } + + translators := common.ComponentTranslators{ + Receivers: common.NewTranslatorMap[component.Config](), + Processors: common.NewTranslatorMap[component.Config](), + Exporters: common.NewTranslatorMap[component.Config](), + Extensions: common.NewTranslatorMap[component.Config](), + } + + translators.Receivers.Set(otlp.NewTranslatorWithName(common.JmxKey)) + translators.Processors.Set(jmxfilterprocessor.NewTranslatorWithName(pipelineName)) //Filter metrics + translators.Processors.Set(resourcedetectionjmx.NewTranslator()) //Adds k8s cluster/nodename name + translators.Processors.Set(resourceprocessor.NewTranslator(resourceprocessor.WithName("jmxResource"))) //Change resource attribute names + translators.Processors.Set(jmxtransformprocessor.NewTranslatorWithName(pipelineName)) //Removes attributes that are not of [ClusterName, Namespace] + translators.Processors.Set(metricstransformprocessorjmx.NewTranslatorWithName(pipelineName)) //Renames metrics and adds pool and area dimensions + translators.Processors.Set(cumulativetodeltaprocessor.NewTranslator(common.WithName(pipelineName), cumulativetodeltaprocessor.WithConfigKeys(jmxKey))) + translators.Exporters.Set(debug.NewTranslator()) //Provides debug info for metrics + translators.Exporters.Set(awsemf.NewTranslatorWithName(pipelineName)) //Sends metrics to cloudwatch console (also adds resource attributes to metrics) + + return &translators, nil + +} diff --git a/translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go b/translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go new file mode 100644 index 0000000000..8e6d5027c0 --- /dev/null +++ b/translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go @@ -0,0 +1,72 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package containerinsightsjmx + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + + "github.com/aws/amazon-cloudwatch-agent/internal/util/collections" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" +) + +func TestTranslator(t *testing.T) { + type want struct { + pipelineType string + receivers []string + processors []string + exporters []string + extensions []string + } + cit := NewTranslator() + require.EqualValues(t, "metrics/containerinsightsjmx", cit.ID().String()) + testCases := map[string]struct { + input map[string]interface{} + want *want + wantErr error + }{ + "WithoutECSOrKubernetesKey": { + input: map[string]interface{}{}, + wantErr: &common.MissingKeyError{ID: cit.ID(), JsonKey: fmt.Sprint(eksKey)}, + }, + "WithKubernetesKey": { + input: map[string]interface{}{ + "logs": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "kubernetes": nil, + }, + }, + }, + want: &want{ + pipelineType: "metrics/containerinsightsjmx", + receivers: []string{"otlp/jmx"}, + processors: []string{"filter/containerinsightsjmx", "resourcedetection", "resource/jmxResource", "transform/containerinsightsjmx", "metricstransform/containerinsightsjmx"}, + exporters: []string{"debug", "awsemf/jmx"}, + extensions: []string{}, + }, + }, + } + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + conf := confmap.NewFromStringMap(testCase.input) + got, err := cit.Translate(conf) + require.Equal(t, testCase.wantErr, err) + if testCase.want == nil { + require.Nil(t, got) + } else { + require.NotNil(t, got) + assert.Equal(t, testCase.want.receivers, collections.MapSlice(got.Receivers.Keys(), component.ID.String)) + assert.Equal(t, testCase.want.processors, collections.MapSlice(got.Processors.Keys(), component.ID.String)) + assert.Equal(t, testCase.want.exporters, collections.MapSlice(got.Exporters.Keys(), component.ID.String)) + assert.Equal(t, testCase.want.extensions, collections.MapSlice(got.Extensions.Keys(), component.ID.String)) + } + }) + } +} diff --git a/translator/translate/otel/processor/jmxfilterprocessor/testdata/config.json b/translator/translate/otel/processor/jmxfilterprocessor/testdata/config.json new file mode 100644 index 0000000000..510cb41463 --- /dev/null +++ b/translator/translate/otel/processor/jmxfilterprocessor/testdata/config.json @@ -0,0 +1,20 @@ +{ + "agent": { + "region": "us-east-1" + }, + "logs": { + "metrics_collected": { + "emf": { + }, + "kubernetes": { + "cluster_name": "TestCluster", + "metrics_collection_interval": 30, + "disable_metric_extraction": true, + "prefer_full_pod_name": true, + "accelerated_compute_metrics": false + } + }, + "force_flush_interval": 5, + "endpoint_override":"https://fake_endpoint" + } +} diff --git a/translator/translate/otel/processor/jmxfilterprocessor/testdata/config.yaml b/translator/translate/otel/processor/jmxfilterprocessor/testdata/config.yaml new file mode 100644 index 0000000000..d0e19d1d1a --- /dev/null +++ b/translator/translate/otel/processor/jmxfilterprocessor/testdata/config.yaml @@ -0,0 +1,25 @@ +metrics: + include: + match_type: strict + metric_names: + - jvm.classes.loaded + - jvm.memory.heap.used + - jvm.memory.nonheap.used + - jvm.memory.pool.used + - jvm.operating.system.total.swap.space.size + - jvm.operating.system.system.cpu.load + - jvm.operating.system.process.cpu.load + - jvm.operating.system.free.swap.space.size + - jvm.operating.system.total.physical.memory.size + - jvm.operating.system.free.physical.memory.size + - jvm.operating.system.open.file.descriptor.count + - jvm.operating.system.available.processors + - jvm.threads.count + - jvm.threads.daemon + - tomcat.sessions + - tomcat.rejected_sessions + - tomcat.traffic.received + - tomcat.traffic.sent + - tomcat.request_count + - tomcat.errors + - tomcat.processing_time \ No newline at end of file diff --git a/translator/translate/otel/processor/jmxfilterprocessor/translator.go b/translator/translate/otel/processor/jmxfilterprocessor/translator.go new file mode 100644 index 0000000000..146634af0c --- /dev/null +++ b/translator/translate/otel/processor/jmxfilterprocessor/translator.go @@ -0,0 +1,82 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package jmxfilterprocessor + +import ( + "fmt" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + "go.opentelemetry.io/collector/processor" + + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" +) + +const ( + matchTypeStrict = "strict" +) + +type translator struct { + name string + factory processor.Factory +} + +type Option func(any) + +var _ common.Translator[component.Config] = (*translator)(nil) + +func NewTranslatorWithName(name string) common.Translator[component.Config] { + return &translator{name: name, factory: filterprocessor.NewFactory()} +} + +func (t *translator) ID() component.ID { + return component.NewIDWithName(t.factory.Type(), t.name) +} + +func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { + if conf == nil || !conf.IsSet(common.ContainerInsightsConfigKey) { + return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: common.ContainerInsightsConfigKey} + } + + cfg := t.factory.CreateDefaultConfig().(*filterprocessor.Config) + + includeMetricNames := []string{ + "jvm.classes.loaded", + "jvm.memory.heap.used", + "jvm.memory.nonheap.used", + "jvm.memory.pool.used", + "jvm.operating.system.total.swap.space.size", + "jvm.operating.system.system.cpu.load", + "jvm.operating.system.process.cpu.load", + "jvm.operating.system.free.swap.space.size", + "jvm.operating.system.total.physical.memory.size", + "jvm.operating.system.free.physical.memory.size", + "jvm.operating.system.open.file.descriptor.count", + "jvm.operating.system.available.processors", + "jvm.threads.count", + "jvm.threads.daemon", + "tomcat.sessions", + "tomcat.rejected_sessions", + "tomcat.traffic.received", + "tomcat.traffic.sent", + "tomcat.request_count", + "tomcat.errors", + "tomcat.processing_time", + } + + c := confmap.NewFromStringMap(map[string]interface{}{ + "metrics": map[string]any{ + "include": map[string]any{ + "match_type": matchTypeStrict, + "metric_names": includeMetricNames, + }, + }, + }) + + if err := c.Unmarshal(&cfg); err != nil { + return nil, fmt.Errorf("unable to unmarshal jmx filter processor (%s): %w", t.ID(), err) + } + + return cfg, nil +} diff --git a/translator/translate/otel/processor/jmxfilterprocessor/translator_test.go b/translator/translate/otel/processor/jmxfilterprocessor/translator_test.go new file mode 100644 index 0000000000..ac9bbc6aa7 --- /dev/null +++ b/translator/translate/otel/processor/jmxfilterprocessor/translator_test.go @@ -0,0 +1,81 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package jmxfilterprocessor + +import ( + "path/filepath" + "testing" + + "github.com/aws/amazon-cloudwatch-agent/internal/util/testutil" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" + "gopkg.in/yaml.v2" +) + +func TestTranslator(t *testing.T) { + factory := filterprocessor.NewFactory() + + testCases := map[string]struct { + input map[string]any + index int + wantID string + want *confmap.Conf + wantErr error + }{ + "ConfigWithNoJmxSet": { + input: map[string]any{ + "metrics": map[string]any{ + "metrics_collected": map[string]any{ + "cpu": map[string]any{}, + }, + }, + }, + index: 0, + wantID: "filter/jmx", + wantErr: &common.MissingKeyError{ + ID: component.NewIDWithName(factory.Type(), "jmx"), + JsonKey: common.ContainerInsightsConfigKey, + }, + }, + "WithCompleteConfig": { + input: testutil.GetJson(t, filepath.Join("testdata", "config.json")), + index: 0, + wantID: "filter/jmx", + want: testutil.GetConf(t, filepath.Join("testdata", "config.yaml")), + }, + } + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + tt := NewTranslatorWithName("jmx") + require.EqualValues(t, testCase.wantID, tt.ID().String()) + + conf := confmap.NewFromStringMap(testCase.input) + got, err := tt.Translate(conf) + require.Equal(t, testCase.wantErr, err) + + if err == nil { + require.NotNil(t, got) + + gotCfg, ok := got.(*filterprocessor.Config) + require.True(t, ok) + + wantCfg := factory.CreateDefaultConfig() + require.NoError(t, testCase.want.Unmarshal(wantCfg)) + + gotYAML, err := yaml.Marshal(gotCfg) + require.NoError(t, err) + + wantYAML, err := yaml.Marshal(wantCfg) + require.NoError(t, err) + + require.Equal(t, wantCfg, gotCfg, "Expected:\n%s\nGot:\n%s", wantYAML, gotYAML) + } + }) + } +} diff --git a/translator/translate/otel/processor/jmxtransformprocessor/testdata/config.json b/translator/translate/otel/processor/jmxtransformprocessor/testdata/config.json new file mode 100644 index 0000000000..760aff7ca9 --- /dev/null +++ b/translator/translate/otel/processor/jmxtransformprocessor/testdata/config.json @@ -0,0 +1,17 @@ +{ + "agent": { + "region": "us-east-1" + }, + "logs": { + "metrics_collected": { + "emf": { + }, + "kubernetes": { + "cluster_name": "TestCluster", + "metrics_collection_interval": 30 + } + }, + "force_flush_interval": 5, + "endpoint_override":"https://fake_endpoint" + } +} \ No newline at end of file diff --git a/translator/translate/otel/processor/jmxtransformprocessor/testdata/config.yaml b/translator/translate/otel/processor/jmxtransformprocessor/testdata/config.yaml new file mode 100644 index 0000000000..a27b7a606a --- /dev/null +++ b/translator/translate/otel/processor/jmxtransformprocessor/testdata/config.yaml @@ -0,0 +1,28 @@ +metric_statements: + - context: resource + statements: + - keep_keys(attributes, ["ClusterName", "Namespace", "NodeName"]) + - context: metric + statements: + - set(unit, "Bytes") where name == "jvm.memory.heap.used" + - set(unit, "Bytes") where name == "jvm.memory.nonheap.used" + - set(unit, "Bytes") where name == "jvm.memory.pool.used" + - set(unit, "Bytes") where name == "tomcat.traffic.received" + - set(unit, "Bytes") where name == "tomcat.traffic.sent" + - set(unit, "Bytes") where name == "jvm.operating.system.total.swap.space.size" + - set(unit, "Bytes") where name == "jvm.operating.system.free.swap.space.size" + - set(unit, "Bytes") where name == "jvm.operating.system.total.physical.memory.size" + - set(unit, "Bytes") where name == "jvm.operating.system.free.physical.memory.size" + - set(unit, "Count") where name == "tomcat.sessions" + - set(unit, "Count") where name == "tomcat.rejected_sessions" + - set(unit, "Count") where name == "jvm.threads.count" + - set(unit, "Count") where name == "jvm.threads.daemon" + - set(unit, "Count") where name == "jvm.operating.system.open.file.descriptor.count" + - set(unit, "Count") where name == "jvm.operating.system.available.processors" + - set(unit, "Count") where name == "tomcat.request_count" + - set(unit, "Count") where name == "tomcat.errors" + - set(unit, "Count") where name == "jvm.classes.loaded" + - set(unit, "Count") where name == "jvm.operating.system.system.cpu.load" + - set(unit, "Count") where name == "jvm.operating.system.process.cpu.load" + - set(unit, "Milliseconds") where name == "tomcat.processing_time" + diff --git a/translator/translate/otel/processor/jmxtransformprocessor/translate.go b/translator/translate/otel/processor/jmxtransformprocessor/translate.go new file mode 100644 index 0000000000..da86988bbd --- /dev/null +++ b/translator/translate/otel/processor/jmxtransformprocessor/translate.go @@ -0,0 +1,92 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package jmxtransformprocessor + +import ( + _ "embed" + "fmt" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + "go.opentelemetry.io/collector/processor" + + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" +) + +//go:embed testdata/config.yaml +var transformJmxConfig string + +type translator struct { + name string + factory processor.Factory +} +type Context struct { + name string + statements []string +} + +var _ common.Translator[component.Config] = (*translator)(nil) + +func NewTranslatorWithName(name string) common.Translator[component.Config] { + return &translator{name, transformprocessor.NewFactory()} +} + +func (t *translator) ID() component.ID { + return component.NewIDWithName(t.factory.Type(), t.name) +} + +func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { + if !(conf != nil && conf.IsSet(common.ContainerInsightsConfigKey)) { + return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: common.ContainerInsightsConfigKey} + } + cfg := t.factory.CreateDefaultConfig().(*transformprocessor.Config) + clusterName := conf.Get(common.ConfigKey(common.ContainerInsightsConfigKey, "cluster_name")) + + if clusterName == nil { + return common.GetYamlFileToYamlConfig(cfg, transformJmxConfig) + } + c := confmap.NewFromStringMap(map[string]interface{}{ + "metric_statements": []map[string]interface{}{ + { + "context": "resource", + "statements": []string{ + "keep_keys(attributes, [\"ClusterName\", \"Namespace\",\"NodeName\"])", + }, + }, + { + "context": "metric", + "statements": []string{ + "set(unit, \"Bytes\") where name == \"jvm.memory.heap.used\"", + "set(unit, \"Bytes\") where name == \"jvm.memory.nonheap.used\"", + "set(unit, \"Bytes\") where name == \"jvm.memory.pool.used\"", + "set(unit, \"Bytes\") where name == \"tomcat.traffic.received\"", + "set(unit, \"Bytes\") where name == \"tomcat.traffic.sent\"", + "set(unit, \"Bytes\") where name == \"jvm.operating.system.total.swap.space.size\"", + "set(unit, \"Bytes\") where name == \"jvm.operating.system.free.swap.space.size\"", + "set(unit, \"Bytes\") where name == \"jvm.operating.system.total.physical.memory.size\"", + "set(unit, \"Bytes\") where name == \"jvm.operating.system.free.physical.memory.size\"", + "set(unit, \"Count\") where name == \"tomcat.sessions\"", + "set(unit, \"Count\") where name == \"tomcat.rejected_sessions\"", + "set(unit, \"Count\") where name == \"jvm.threads.count\"", + "set(unit, \"Count\") where name == \"jvm.threads.daemon\"", + "set(unit, \"Count\") where name == \"jvm.operating.system.open.file.descriptor.count\"", + "set(unit, \"Count\") where name == \"jvm.operating.system.available.processors\"", + "set(unit, \"Count\") where name == \"tomcat.request_count\"", + "set(unit, \"Count\") where name == \"tomcat.errors\"", + "set(unit, \"Count\") where name == \"jvm.classes.loaded\"", + "set(unit, \"Count\") where name == \"jvm.operating.system.system.cpu.load\"", + "set(unit, \"Count\") where name == \"jvm.operating.system.process.cpu.load\"", + "set(unit, \"Milliseconds\") where name == \"tomcat.processing_time\"", + }, + }, + }, + }) + + if err := c.Unmarshal(&cfg); err != nil { + return nil, fmt.Errorf("unable to unmarshal jmx filter processor (%s): %w", t.ID(), err) + } + + return cfg, nil +} diff --git a/translator/translate/otel/processor/jmxtransformprocessor/translator_test.go b/translator/translate/otel/processor/jmxtransformprocessor/translator_test.go new file mode 100644 index 0000000000..1a43e066f1 --- /dev/null +++ b/translator/translate/otel/processor/jmxtransformprocessor/translator_test.go @@ -0,0 +1,66 @@ +package jmxtransformprocessor + +import ( + _ "embed" + "github.com/aws/amazon-cloudwatch-agent/internal/util/testutil" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor" + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/component" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/confmap" +) + +func TestTranslator(t *testing.T) { + factory := transformprocessor.NewFactory() + + testCases := map[string]struct { + translator common.Translator[component.Config] + input map[string]any + index int + wantID string + want string + wantErr error + }{ + "NoContainerInsights": { + input: map[string]any{}, + wantErr: &common.MissingKeyError{ + ID: component.NewIDWithName(factory.Type(), "jmx"), + JsonKey: common.ContainerInsightsConfigKey, + }, + }, + "WithContainerInsights": { + input: testutil.GetJson(t, filepath.Join("testdata", "config.json")), + index: 0, + wantID: "filter/jmx", + want: filepath.Join("testdata", "config.json"), + }, + } + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + tt := NewTranslatorWithName("jmx") + + conf := confmap.NewFromStringMap(testCase.input) + got, err := tt.Translate(conf) + require.Equal(t, testCase.wantErr, err) + + if err == nil { + require.NotNil(t, got) + gotCfg, ok := got.(*transformprocessor.Config) + + require.True(t, ok) + wantCfg := factory.CreateDefaultConfig() + yamlConfig, err := common.GetYamlFileToYamlConfig(wantCfg, testCase.want) + require.NoError(t, err) + assert.Equal(t, yamlConfig.(*transformprocessor.Config), gotCfg) + + assert.Equal(t, gotCfg, wantCfg) + + } + }) + } +} diff --git a/translator/translate/otel/processor/metrictransformprocessorjmx/configs/config.yaml b/translator/translate/otel/processor/metrictransformprocessorjmx/configs/config.yaml new file mode 100644 index 0000000000..00dc4bb660 --- /dev/null +++ b/translator/translate/otel/processor/metrictransformprocessorjmx/configs/config.yaml @@ -0,0 +1,134 @@ +transforms: + - include: jvm.classes.loaded + match_type: strict + action: update + new_name: jvm_classes_loaded + + + - include: jvm.memory.heap.used + match_type: strict + action: update + new_name: jvm_memory_bytes_used + operations: + - action: add_label + new_label: area + new_value: "heap" + + - include: jvm.memory.nonheap.used + match_type: strict + action: update + new_name: jvm_memory_bytes_used + operations: + - action: add_label + new_label: area + new_value: "nonheap" + + - include: jvm.memory.pool.used + match_type: strict + action: update + new_name: jvm_memory_pool_bytes_used + operations: + - action: update_label + label: name + new_label: pool + + - include: jvm.operating.system.total.swap.space.size + match_type: strict + action: update + new_name: java_lang_operatingsystem_totalswapspacesize + + + - include: jvm.operating.system.system.cpu.load + match_type: strict + action: update + new_name: java_lang_operatingsystem_systemcpuload + + - include: jvm.operating.system.process.cpu.load + match_type: strict + action: update + new_name: java_lang_operatingsystem_processcpuload + + + - include: jvm.operating.system.free.swap.space.size + match_type: strict + action: update + new_name: java_lang_operatingsystem_freeswapspacesize + + + - include: jvm.operating.system.total.physical.memory.size + match_type: strict + action: update + new_name: java_lang_operatingsystem_totalphysicalmemorysize + + + - include: jvm.operating.system.free.physical.memory.size + match_type: strict + action: update + new_name: java_lang_operatingsystem_freephysicalmemorysize + + + - include: jvm.operating.system.open.file.descriptor.count + match_type: strict + action: update + new_name: java_lang_operatingsystem_openfiledescriptorcount + + + - include: jvm.operating.system.available.processors + match_type: strict + action: update + new_name: java_lang_operatingsystem_availableprocessors + + + - include: jvm.threads.count + match_type: strict + action: update + new_name: jvm_threads_current + + + - include: jvm.threads.daemon + match_type: strict + action: update + new_name: jvm_threads_daemon + + + - include: tomcat.sessions + match_type: strict + action: update + new_name: catalina_manager_activesessions + + + - include: tomcat.rejected_sessions + match_type: strict + action: update + new_name: catalina_manager_rejectedsessions + + + - include: tomcat.traffic.received + match_type: strict + action: update + new_name: catalina_globalrequestprocessor_bytesreceived + + + - include: tomcat.traffic.sent + match_type: strict + action: update + new_name: catalina_globalrequestprocessor_bytessent + + - include: tomcat.request_count + match_type: strict + action: update + new_name: catalina_globalrequestprocessor_requestcount + + + - include: tomcat.errors + match_type: strict + action: update + new_name: catalina_globalrequestprocessor_errorcount + + + - include: tomcat.processing_time + match_type: strict + action: update + new_name: catalina_globalrequestprocessor_processingtime + + diff --git a/translator/translate/otel/processor/metrictransformprocessorjmx/translator.go b/translator/translate/otel/processor/metrictransformprocessorjmx/translator.go new file mode 100644 index 0000000000..521ba0b519 --- /dev/null +++ b/translator/translate/otel/processor/metrictransformprocessorjmx/translator.go @@ -0,0 +1,37 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package metricstransformprocessorjmx + +import ( + _ "embed" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + "go.opentelemetry.io/collector/processor" + + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" +) + +//go:embed configs/config.yaml +var metricTransformJMXConfig string + +type translator struct { + name string + factory processor.Factory +} + +var _ common.Translator[component.Config] = (*translator)(nil) + +func NewTranslatorWithName(name string) common.Translator[component.Config] { + return &translator{name, metricstransformprocessor.NewFactory()} +} + +func (t *translator) ID() component.ID { + return component.NewIDWithName(t.factory.Type(), t.name) +} + +func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { + cfg := t.factory.CreateDefaultConfig().(*metricstransformprocessor.Config) + return common.GetYamlFileToYamlConfig(cfg, metricTransformJMXConfig) +} diff --git a/translator/translate/otel/processor/resourcedetectionjmx/configs/config.yaml b/translator/translate/otel/processor/resourcedetectionjmx/configs/config.yaml new file mode 100644 index 0000000000..116bcb65d6 --- /dev/null +++ b/translator/translate/otel/processor/resourcedetectionjmx/configs/config.yaml @@ -0,0 +1,9 @@ +detectors: [env, eks, system] +timeout: 15s +override: false +eks: + resource_attributes: + k8s.cluster.name: + enabled: true +system: + hostname_sources: ["os"] \ No newline at end of file diff --git a/translator/translate/otel/processor/resourcedetectionjmx/translator.go b/translator/translate/otel/processor/resourcedetectionjmx/translator.go new file mode 100644 index 0000000000..99a059dfa7 --- /dev/null +++ b/translator/translate/otel/processor/resourcedetectionjmx/translator.go @@ -0,0 +1,61 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package resourcedetectionjmx + +import ( + _ "embed" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + "go.opentelemetry.io/collector/processor" + + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" +) + +//go:embed configs/config.yaml +var resourceDetectionJMX string + +type translator struct { + name string + dataType component.DataType + factory processor.Factory +} + +type Option interface { + apply(t *translator) +} + +type optionFunc func(t *translator) + +func (o optionFunc) apply(t *translator) { + o(t) +} + +// WithDataType determines where the translator should look to find +// the configuration. +func WithDataType(dataType component.DataType) Option { + return optionFunc(func(t *translator) { + t.dataType = dataType + }) +} + +var _ common.Translator[component.Config] = (*translator)(nil) + +func NewTranslator(opts ...Option) common.Translator[component.Config] { + t := &translator{factory: resourcedetectionprocessor.NewFactory()} + for _, opt := range opts { + opt.apply(t) + } + return t +} + +func (t *translator) ID() component.ID { + return component.NewIDWithName(t.factory.Type(), t.name) +} + +func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { + cfg := t.factory.CreateDefaultConfig().(*resourcedetectionprocessor.Config) + return common.GetYamlFileToYamlConfig(cfg, resourceDetectionJMX) +} diff --git a/translator/translate/otel/processor/resourceprocessor/translator.go b/translator/translate/otel/processor/resourceprocessor/translator.go index d8b77a5c37..8a029b1371 100644 --- a/translator/translate/otel/processor/resourceprocessor/translator.go +++ b/translator/translate/otel/processor/resourceprocessor/translator.go @@ -65,6 +65,54 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { c := confmap.NewFromStringMap(map[string]any{ "attributes": attributes, }) + if t.name == "jmxResource" { + clusterName, ok := common.GetString(conf, common.ConfigKey(eksKey, "cluster_name")) + + if ok { + c = confmap.NewFromStringMap(map[string]any{ + //from config + "attributes": []any{ + map[string]any{ + "key": "Namespace", + "from_attribute": "k8s.namespace.name", + "action": "insert", + }, + map[string]any{ + "key": "ClusterName", + "value": clusterName, + "action": "upsert", + }, + map[string]any{ + "key": "NodeName", + "from_attribute": "host.name", + "action": "insert", + }, + }, + }) + } else { + c = confmap.NewFromStringMap(map[string]any{ + //from resource detection processor + "attributes": []any{ + map[string]any{ + "key": "ClusterName", + "from_attribute": "k8s.cluster.name", + "action": "insert", + }, + map[string]any{ + "key": "Namespace", + "from_attribute": "k8s.namespace.name", + "action": "insert", + }, + map[string]any{ + "key": "NodeName", + "from_attribute": "host.name", + "action": "insert", + }, + }, + }) + } + + } if err := c.Unmarshal(&cfg); err != nil { return nil, fmt.Errorf("unable to unmarshal resource processor: %w", err) } diff --git a/translator/translate/otel/receiver/awscontainerinsight/utils.go b/translator/translate/otel/receiver/awscontainerinsight/utils.go index 721951b056..36a31fd209 100644 --- a/translator/translate/otel/receiver/awscontainerinsight/utils.go +++ b/translator/translate/otel/receiver/awscontainerinsight/utils.go @@ -24,6 +24,12 @@ func EnhancedContainerInsightsEnabled(conf *confmap.Conf) bool { return isSet } +func JmxEnabled(conf *confmap.Conf) bool { + isEnabled, _ := common.GetBool(conf, common.ConfigKey(common.MetricsKey, common.MetricsCollectedKey, common.JmxKey)) + return isEnabled + +} + func AcceleratedComputeMetricsEnabled(conf *confmap.Conf) bool { return common.GetOrDefaultBool(conf, common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.KubernetesKey, common.EnableAcceleratedComputeMetric), true) } diff --git a/translator/translate/otel/translate_otel.go b/translator/translate/otel/translate_otel.go index 4fa223394e..18e65400cf 100644 --- a/translator/translate/otel/translate_otel.go +++ b/translator/translate/otel/translate_otel.go @@ -6,6 +6,7 @@ package otel import ( "errors" "fmt" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/containerinsightsjmx" "log" "time" @@ -67,6 +68,7 @@ func Translate(jsonConfig interface{}, os string) (*otelcol.Config, error) { translators.Set(prometheus.NewTranslator()) translators.Set(emf_logs.NewTranslator()) translators.Set(xray.NewTranslator()) + translators.Set(containerinsightsjmx.NewTranslator()) translators.Merge(jmx.NewTranslators(conf)) translators.Merge(registry) pipelines, err := pipeline.NewTranslator(translators).Translate(conf) From b57dcda49717b73b918eae0c117430abb79b15ab Mon Sep 17 00:00:00 2001 From: Paramadon Date: Thu, 10 Oct 2024 00:18:10 -0400 Subject: [PATCH 02/19] we are getting metrics --- Makefile | 4 +- .../sampleConfig/jmx_eks_config_linux.json | 73 +++-- .../containerinsightsjmx/translator.go | 12 +- .../processor/resourceprocessor/translator.go | 12 +- .../translate/otel/translate_otel_test.go | 299 +++++++++--------- 5 files changed, 205 insertions(+), 195 deletions(-) diff --git a/Makefile b/Makefile index 21b2fe7d88..4f6451939b 100644 --- a/Makefile +++ b/Makefile @@ -25,9 +25,9 @@ WIN_BUILD = GOOS=windows GOARCH=amd64 go build -trimpath -buildmode=${CWAGENT_BU DARWIN_BUILD_AMD64 = CGO_ENABLED=1 GO111MODULE=on GOOS=darwin GOARCH=amd64 go build -trimpath -ldflags="${LDFLAGS}" -o $(BUILD_SPACE)/bin/darwin_amd64 DARWIN_BUILD_ARM64 = CGO_ENABLED=1 GO111MODULE=on GOOS=darwin GOARCH=arm64 go build -trimpath -ldflags="${LDFLAGS}" -o $(BUILD_SPACE)/bin/darwin_arm64 -IMAGE_REGISTRY = amazon +IMAGE_REGISTRY = 506463145083.dkr.ecr.us-west-2.amazonaws.com IMAGE_REPO = cloudwatch-agent -IMAGE_TAG = $(VERSION) +IMAGE_TAG = latest IMAGE = $(IMAGE_REGISTRY)/$(IMAGE_REPO):$(IMAGE_TAG) DOCKER_BUILD_FROM_SOURCE = docker build -t $(IMAGE) -f ./amazon-cloudwatch-container-insights/cloudwatch-agent-dockerfile/source/Dockerfile DOCKER_WINDOWS_BUILD_FROM_SOURCE = docker build -t $(IMAGE) -f ./amazon-cloudwatch-container-insights/cloudwatch-agent-dockerfile/source/Dockerfile.Windows diff --git a/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.json b/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.json index 06c225e7a5..67f5952d66 100644 --- a/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.json +++ b/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.json @@ -1,43 +1,42 @@ { + "agent": { + "debug": true + }, + "logs": { + "metrics_collected": { + "emf": { }, + "kubernetes": { + "cluster_name": "TestCluster", + "jmx_container_insights": true + } + } + }, "metrics": { - "metrics_destinations": { - "amp": { - "workspace_id": "ws-12345" - }, - "cloudwatch": {} - }, "metrics_collected": { - "jmx": [ - { - "jvm": { - "measurement": [ - "jvm.memory.heap.init", - { - "name": "jvm.memory.heap.used", - "rename": "JVM_MEM_HEAP_USED", - "unit": "unit" - }, - "jvm.memory.nonheap.init" - ] - }, - "append_dimensions": { - "k1": "v1" - } - }, - { - "kafka-consumer": { - "measurement": [ - { - "name": "kafka.consumer.fetch-rate", - "rename": "kafka.fetch-rate" - } - ] - }, - "append_dimensions": { - "k2": "v2" - } + "jmx": { + "jvm": { + "measurement": [ + "jvm.classes.loaded", + "jvm.memory.pool.used", + "jvm.operating.system.total.swap.space.size", + "jvm.operating.system.system.cpu.load", + "jvm.operating.system.process.cpu.load", + "jvm.operating.system.free.swap.space.size", + "jvm.operating.system.total.physical.memory.size", + "jvm.operating.system.free.physical.memory.size", + "jvm.operating.system.open.file.descriptor.count", + "jvm.operating.system.available.processors", + "jvm.threads.count", + "jvm.memory.heap.used", + "jvm.memory.nonheap.init", + "jvm.threads.daemon" + ] } - ] - } + } + }, + "metrics_destinations": { + "cloudwatch": { } + }, + "namespace": "jmx-test-141" } } \ No newline at end of file diff --git a/translator/translate/otel/pipeline/containerinsightsjmx/translator.go b/translator/translate/otel/pipeline/containerinsightsjmx/translator.go index 2a3e6e65a3..72e18612d8 100644 --- a/translator/translate/otel/pipeline/containerinsightsjmx/translator.go +++ b/translator/translate/otel/pipeline/containerinsightsjmx/translator.go @@ -62,12 +62,12 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators Extensions: common.NewTranslatorMap[component.Config](), } - translators.Receivers.Set(otlp.NewTranslatorWithName(common.JmxKey)) - translators.Processors.Set(jmxfilterprocessor.NewTranslatorWithName(pipelineName)) //Filter metrics - translators.Processors.Set(resourcedetectionjmx.NewTranslator()) //Adds k8s cluster/nodename name - translators.Processors.Set(resourceprocessor.NewTranslator(resourceprocessor.WithName("jmxResource"))) //Change resource attribute names - translators.Processors.Set(jmxtransformprocessor.NewTranslatorWithName(pipelineName)) //Removes attributes that are not of [ClusterName, Namespace] - translators.Processors.Set(metricstransformprocessorjmx.NewTranslatorWithName(pipelineName)) //Renames metrics and adds pool and area dimensions + translators.Receivers.Set(otlp.NewTranslator(common.WithName(common.PipelineNameJmx))) + translators.Processors.Set(jmxfilterprocessor.NewTranslatorWithName(pipelineName)) //Filter metrics + translators.Processors.Set(resourcedetectionjmx.NewTranslator()) //Adds k8s cluster/nodename name + translators.Processors.Set(resourceprocessor.NewTranslator(common.WithName("jmxResource"))) //Change resource attribute names + translators.Processors.Set(jmxtransformprocessor.NewTranslatorWithName(pipelineName)) //Removes attributes that are not of [ClusterName, Namespace] + translators.Processors.Set(metricstransformprocessorjmx.NewTranslatorWithName(pipelineName)) //Renames metrics and adds pool and area dimensions translators.Processors.Set(cumulativetodeltaprocessor.NewTranslator(common.WithName(pipelineName), cumulativetodeltaprocessor.WithConfigKeys(jmxKey))) translators.Exporters.Set(debug.NewTranslator()) //Provides debug info for metrics translators.Exporters.Set(awsemf.NewTranslatorWithName(pipelineName)) //Sends metrics to cloudwatch console (also adds resource attributes to metrics) diff --git a/translator/translate/otel/processor/resourceprocessor/translator.go b/translator/translate/otel/processor/resourceprocessor/translator.go index 8a029b1371..ad41e2443a 100644 --- a/translator/translate/otel/processor/resourceprocessor/translator.go +++ b/translator/translate/otel/processor/resourceprocessor/translator.go @@ -23,7 +23,11 @@ type translator struct { factory processor.Factory } -var _ common.Translator[component.Config] = (*translator)(nil) +var ( + baseKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey) + eksKey = common.ConfigKey(baseKey, common.KubernetesKey) + _ common.Translator[component.Config] = (*translator)(nil) +) func NewTranslator(opts ...common.TranslatorOption) common.Translator[component.Config] { t := &translator{factory: resourceprocessor.NewFactory()} @@ -46,7 +50,7 @@ func (t *translator) ID() component.ID { // Translate creates a processor config based on the fields in the // Metrics section of the JSON config. func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { - if conf == nil || !conf.IsSet(common.JmxConfigKey) { + if conf == nil || (!conf.IsSet(common.JmxConfigKey) && t.Name() != "jmxResource") { return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: common.JmxConfigKey} } @@ -55,7 +59,7 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { if strings.HasPrefix(t.Name(), common.PipelineNameJmx) { attributes = t.getJMXAttributes(conf) } - if len(attributes) == 0 { + if len(attributes) == 0 && t.Name() != "jmxResource" { baseKey := common.JmxConfigKey if t.Index() != -1 { baseKey = fmt.Sprintf("%s[%d]", baseKey, t.Index()) @@ -65,7 +69,7 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { c := confmap.NewFromStringMap(map[string]any{ "attributes": attributes, }) - if t.name == "jmxResource" { + if t.Name() == "jmxResource" { clusterName, ok := common.GetString(conf, common.ConfigKey(eksKey, "cluster_name")) if ok { diff --git a/translator/translate/otel/translate_otel_test.go b/translator/translate/otel/translate_otel_test.go index da34cf5c60..de5f387193 100644 --- a/translator/translate/otel/translate_otel_test.go +++ b/translator/translate/otel/translate_otel_test.go @@ -11,174 +11,181 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" - "github.com/aws/amazon-cloudwatch-agent/tool/testutil" "github.com/aws/amazon-cloudwatch-agent/translator" _ "github.com/aws/amazon-cloudwatch-agent/translator/registerrules" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/agent" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "github.com/aws/amazon-cloudwatch-agent/translator/util/eksdetector" ) func TestTranslator(t *testing.T) { - agent.Global_Config.Region = "us-east-1" - testutil.SetPrometheusRemoteWriteTestingEnv(t) testCases := map[string]struct { input interface{} wantErrContains string detector func() (eksdetector.Detector, error) isEKSDataStore func() eksdetector.IsEKSCache }{ - "WithInvalidConfig": { - input: "", - wantErrContains: "invalid json config", - }, - "WithEmptyConfig": { - input: map[string]interface{}{}, - wantErrContains: "no valid pipelines", - }, - "WithoutReceivers": { - input: map[string]interface{}{ - "metrics": map[string]interface{}{}, - }, - wantErrContains: "no valid pipelines", - }, - "WithMinimalConfig": { - input: map[string]interface{}{ - "metrics": map[string]interface{}{ - "metrics_collected": map[string]interface{}{ - "cpu": map[string]interface{}{}, - }, - }, - }, - }, - "WithAppSignalsMetricsEnabled": { - input: map[string]interface{}{ - "logs": map[string]interface{}{ - "metrics_collected": map[string]interface{}{ - "application_signals": map[string]interface{}{}, - }, - }, - }, - detector: eksdetector.TestEKSDetector, - isEKSDataStore: eksdetector.TestIsEKSCacheEKS, - }, - "WithAppSignalsTracesEnabled": { - input: map[string]interface{}{ - "traces": map[string]interface{}{ - "traces_collected": map[string]interface{}{ - "application_signals": map[string]interface{}{}, - }, - }, - }, - detector: eksdetector.TestEKSDetector, - isEKSDataStore: eksdetector.TestIsEKSCacheEKS, - }, - "WithAppSignalsMetricsAndTracesEnabled": { - input: map[string]interface{}{ - "logs": map[string]interface{}{ - "metrics_collected": map[string]interface{}{ - "application_signals": map[string]interface{}{}, - }, - }, - "traces": map[string]interface{}{ - "traces_collected": map[string]interface{}{ - "application_signals": map[string]interface{}{}, - }, - }, - }, - detector: eksdetector.TestEKSDetector, - isEKSDataStore: eksdetector.TestIsEKSCacheEKS, - }, - "WithAppSignalsMultipleMetricsReceiversConfig": { - input: map[string]interface{}{ - "logs": map[string]interface{}{ - "metrics_collected": map[string]interface{}{ - "application_signals": map[string]interface{}{}, - "cpu": map[string]interface{}{}, - }, - }, - "traces": map[string]interface{}{ - "traces_collected": map[string]interface{}{ - "application_signals": map[string]interface{}{}, - "otlp": map[string]interface{}{}, - "otlp2": map[string]interface{}{}, - }, - }, - }, - detector: eksdetector.TestEKSDetector, - isEKSDataStore: eksdetector.TestIsEKSCacheEKS, - }, - "WithAppSignalsFallbackMetricsEnabled": { - input: map[string]interface{}{ - "logs": map[string]interface{}{ - "metrics_collected": map[string]interface{}{ - "app_signals": map[string]interface{}{}, - }, - }, - }, - detector: eksdetector.TestEKSDetector, - isEKSDataStore: eksdetector.TestIsEKSCacheEKS, - }, - "WithAppSignalsFallbackTracesEnabled": { + "WithValidConfig": { input: map[string]interface{}{ - "traces": map[string]interface{}{ - "traces_collected": map[string]interface{}{ - "app_signals": map[string]interface{}{}, - }, + "agent": map[string]interface{}{ + "debug": true, }, - }, - detector: eksdetector.TestEKSDetector, - isEKSDataStore: eksdetector.TestIsEKSCacheEKS, - }, - "WithAppSignalsFallbackMetricsAndTracesEnabled": { - input: map[string]interface{}{ "logs": map[string]interface{}{ "metrics_collected": map[string]interface{}{ - "app_signals": map[string]interface{}{}, - }, - }, - "traces": map[string]interface{}{ - "traces_collected": map[string]interface{}{ - "app_signals": map[string]interface{}{}, - }, - }, - }, - detector: eksdetector.TestEKSDetector, - isEKSDataStore: eksdetector.TestIsEKSCacheEKS, - }, - "WithAppSignalsFallbackMultipleMetricsReceiversConfig": { - input: map[string]interface{}{ - "logs": map[string]interface{}{ - "metrics_collected": map[string]interface{}{ - "app_signals": map[string]interface{}{}, - "cpu": map[string]interface{}{}, - }, - }, - "traces": map[string]interface{}{ - "traces_collected": map[string]interface{}{ - "app_signals": map[string]interface{}{}, - "otlp": map[string]interface{}{}, - "otlp2": map[string]interface{}{}, - }, - }, - }, - detector: eksdetector.TestEKSDetector, - isEKSDataStore: eksdetector.TestIsEKSCacheEKS, - }, - "WithAMPDestinationConfig": { - input: map[string]interface{}{ - "metrics": map[string]interface{}{ - "metrics_destinations": map[string]interface{}{ - "amp": map[string]interface{}{ - "workspace_id": "ws-12345", + "kubernetes": map[string]interface{}{ + "cluster_name": "TestCluster", + "jmx_container_insights": true, }, }, - "metrics_collected": map[string]interface{}{ - "cpu": map[string]interface{}{}, - }, }, }, }, + //"WithEmptyConfig": { + // input: map[string]interface{}{}, + // wantErrContains: "no valid pipelines", + //}, + //"WithoutReceivers": { + // input: map[string]interface{}{ + // "metrics": map[string]interface{}{}, + // }, + // wantErrContains: "no valid pipelines", + //}, + //"WithMinimalConfig": { + // input: map[string]interface{}{ + // "metrics": map[string]interface{}{ + // "metrics_collected": map[string]interface{}{ + // "cpu": map[string]interface{}{}, + // }, + // }, + // }, + //}, + //"WithAppSignalsMetricsEnabled": { + // input: map[string]interface{}{ + // "logs": map[string]interface{}{ + // "metrics_collected": map[string]interface{}{ + // "application_signals": map[string]interface{}{}, + // }, + // }, + // }, + // detector: eksdetector.TestEKSDetector, + // isEKSDataStore: eksdetector.TestIsEKSCacheEKS, + //}, + //"WithAppSignalsTracesEnabled": { + // input: map[string]interface{}{ + // "traces": map[string]interface{}{ + // "traces_collected": map[string]interface{}{ + // "application_signals": map[string]interface{}{}, + // }, + // }, + // }, + // detector: eksdetector.TestEKSDetector, + // isEKSDataStore: eksdetector.TestIsEKSCacheEKS, + //}, + //"WithAppSignalsMetricsAndTracesEnabled": { + // input: map[string]interface{}{ + // "logs": map[string]interface{}{ + // "metrics_collected": map[string]interface{}{ + // "application_signals": map[string]interface{}{}, + // }, + // }, + // "traces": map[string]interface{}{ + // "traces_collected": map[string]interface{}{ + // "application_signals": map[string]interface{}{}, + // }, + // }, + // }, + // detector: eksdetector.TestEKSDetector, + // isEKSDataStore: eksdetector.TestIsEKSCacheEKS, + //}, + //"WithAppSignalsMultipleMetricsReceiversConfig": { + // input: map[string]interface{}{ + // "logs": map[string]interface{}{ + // "metrics_collected": map[string]interface{}{ + // "application_signals": map[string]interface{}{}, + // "cpu": map[string]interface{}{}, + // }, + // }, + // "traces": map[string]interface{}{ + // "traces_collected": map[string]interface{}{ + // "application_signals": map[string]interface{}{}, + // "otlp": map[string]interface{}{}, + // "otlp2": map[string]interface{}{}, + // }, + // }, + // }, + // detector: eksdetector.TestEKSDetector, + // isEKSDataStore: eksdetector.TestIsEKSCacheEKS, + //}, + //"WithAppSignalsFallbackMetricsEnabled": { + // input: map[string]interface{}{ + // "logs": map[string]interface{}{ + // "metrics_collected": map[string]interface{}{ + // "app_signals": map[string]interface{}{}, + // }, + // }, + // }, + // detector: eksdetector.TestEKSDetector, + // isEKSDataStore: eksdetector.TestIsEKSCacheEKS, + //}, + //"WithAppSignalsFallbackTracesEnabled": { + // input: map[string]interface{}{ + // "traces": map[string]interface{}{ + // "traces_collected": map[string]interface{}{ + // "app_signals": map[string]interface{}{}, + // }, + // }, + // }, + // detector: eksdetector.TestEKSDetector, + // isEKSDataStore: eksdetector.TestIsEKSCacheEKS, + //}, + //"WithAppSignalsFallbackMetricsAndTracesEnabled": { + // input: map[string]interface{}{ + // "logs": map[string]interface{}{ + // "metrics_collected": map[string]interface{}{ + // "app_signals": map[string]interface{}{}, + // }, + // }, + // "traces": map[string]interface{}{ + // "traces_collected": map[string]interface{}{ + // "app_signals": map[string]interface{}{}, + // }, + // }, + // }, + // detector: eksdetector.TestEKSDetector, + // isEKSDataStore: eksdetector.TestIsEKSCacheEKS, + //}, + //"WithAppSignalsFallbackMultipleMetricsReceiversConfig": { + // input: map[string]interface{}{ + // "logs": map[string]interface{}{ + // "metrics_collected": map[string]interface{}{ + // "app_signals": map[string]interface{}{}, + // "cpu": map[string]interface{}{}, + // }, + // }, + // "traces": map[string]interface{}{ + // "traces_collected": map[string]interface{}{ + // "app_signals": map[string]interface{}{}, + // "otlp": map[string]interface{}{}, + // "otlp2": map[string]interface{}{}, + // }, + // }, + // }, + // detector: eksdetector.TestEKSDetector, + // isEKSDataStore: eksdetector.TestIsEKSCacheEKS, + //}, + //"WithAMPDestinationConfig": { + // input: map[string]interface{}{ + // "metrics": map[string]interface{}{ + // "metrics_destinations": map[string]interface{}{ + // "amp": map[string]interface{}{ + // "workspace_id": "ws-12345", + // }, + // }, + // "metrics_collected": map[string]interface{}{ + // "cpu": map[string]interface{}{}, + // }, + // }, + // }, + //}, } for name, testCase := range testCases { t.Run(name, func(t *testing.T) { From 5d73e3349221948b68dfb78f534dd2cba7c46478 Mon Sep 17 00:00:00 2001 From: Paramadon Date: Thu, 10 Oct 2024 00:56:37 -0400 Subject: [PATCH 03/19] merging processors --- extension/agenthealth/config_test.go | 2 +- processor/rollupprocessor/config_test.go | 2 +- translator/translate/otel/common/common.go | 15 ++--- .../otel/exporter/awsemf/translator.go | 2 +- .../otel/exporter/awsxray/translator_test.go | 2 +- .../prometheusremotewrite/translator_test.go | 2 +- .../containerinsightsjmx/translator.go | 29 ++++----- .../testdata/ContainerInsightsJmxConfig.yaml} | 0 .../processor/filterprocessor/translator.go | 9 ++- .../filterprocessor/translator_test.go | 2 +- .../jmxfilterprocessor/translator.go | 44 +------------ .../jmxfilterprocessor/translator_test.go | 2 +- .../jmxtransformprocessor/translate.go | 45 +------------- .../metricsdecorator/translator_unix_test.go | 4 +- .../translator_windows_test.go | 4 +- .../configs/config.yaml | 1 - .../metricstransformprocessor/translator.go | 8 +++ .../metrictransformprocessorjmx/translator.go | 37 ----------- .../configs/jmx_config.yaml} | 0 .../processor/resourcedetection/translator.go | 14 +++++ .../resourcedetectionjmx/translator.go | 61 ------------------- .../otel/receiver/awsxray/translator_test.go | 2 +- .../otel/receiver/jmx/translator_test.go | 2 +- .../otel/receiver/otlp/translator_test.go | 6 +- 24 files changed, 70 insertions(+), 225 deletions(-) rename translator/translate/otel/processor/{jmxfilterprocessor/testdata/config.yaml => filterprocessor/testdata/ContainerInsightsJmxConfig.yaml} (100%) rename translator/translate/otel/processor/{metrictransformprocessorjmx => metricstransformprocessor}/configs/config.yaml (99%) delete mode 100644 translator/translate/otel/processor/metrictransformprocessorjmx/translator.go rename translator/translate/otel/processor/{resourcedetectionjmx/configs/config.yaml => resourcedetection/configs/jmx_config.yaml} (100%) delete mode 100644 translator/translate/otel/processor/resourcedetectionjmx/translator.go diff --git a/extension/agenthealth/config_test.go b/extension/agenthealth/config_test.go index ee0ef301f6..01666f534c 100644 --- a/extension/agenthealth/config_test.go +++ b/extension/agenthealth/config_test.go @@ -34,7 +34,7 @@ func TestLoadConfig(t *testing.T) { }, } for _, testCase := range testCases { - conf, err := confmaptest.LoadConf(filepath.Join("testdata", "config.yaml")) + conf, err := confmaptest.LoadConf(filepath.Join("testdata", "ContainerInsightsJmxConfig.yaml")) require.NoError(t, err) cfg := NewFactory().CreateDefaultConfig() sub, err := conf.Sub(testCase.id.String()) diff --git a/processor/rollupprocessor/config_test.go b/processor/rollupprocessor/config_test.go index f1965a425c..136b2a1a3a 100644 --- a/processor/rollupprocessor/config_test.go +++ b/processor/rollupprocessor/config_test.go @@ -40,7 +40,7 @@ func TestLoadConfig(t *testing.T) { }, } for _, testCase := range testCases { - conf, err := confmaptest.LoadConf(filepath.Join("testdata", "config.yaml")) + conf, err := confmaptest.LoadConf(filepath.Join("testdata", "ContainerInsightsJmxConfig.yaml")) require.NoError(t, err) cfg := NewFactory().CreateDefaultConfig() sub, err := conf.Sub(testCase.id.String()) diff --git a/translator/translate/otel/common/common.go b/translator/translate/otel/common/common.go index a27b16c14b..0b468ceffb 100644 --- a/translator/translate/otel/common/common.go +++ b/translator/translate/otel/common/common.go @@ -76,13 +76,14 @@ const ( ) const ( - PipelineNameHost = "host" - PipelineNameHostDeltaMetrics = "hostDeltaMetrics" - PipelineNameJmx = "jmx" - PipelineNameEmfLogs = "emf_logs" - AppSignals = "application_signals" - AppSignalsFallback = "app_signals" - AppSignalsRules = "rules" + PipelineNameHost = "host" + PipelineNameHostDeltaMetrics = "hostDeltaMetrics" + PipelineNameJmx = "jmx" + PipelineNameContainerInsightsJmx = "containerinsightsjmx" + PipelineNameEmfLogs = "emf_logs" + AppSignals = "application_signals" + AppSignalsFallback = "app_signals" + AppSignalsRules = "rules" ) var ( diff --git a/translator/translate/otel/exporter/awsemf/translator.go b/translator/translate/otel/exporter/awsemf/translator.go index a20282c196..a850bd63ec 100644 --- a/translator/translate/otel/exporter/awsemf/translator.go +++ b/translator/translate/otel/exporter/awsemf/translator.go @@ -174,7 +174,7 @@ func (t *translator) isAppSignals(conf *confmap.Conf) bool { return (t.name == common.AppSignals || t.name == common.AppSignalsFallback) && (conf.IsSet(common.AppSignalsMetrics) || conf.IsSet(common.AppSignalsTraces) || conf.IsSet(common.AppSignalsMetricsFallback) || conf.IsSet(common.AppSignalsTracesFallback)) } func (t *translator) isCiJMX(conf *confmap.Conf) bool { - return (t.name == "containerinsightsjmx") && (conf.IsSet(common.ContainerInsightsConfigKey)) + return (t.name == common.PipelineNameContainerInsightsJmx) && (conf.IsSet(common.ContainerInsightsConfigKey)) } func isEcs(conf *confmap.Conf) bool { diff --git a/translator/translate/otel/exporter/awsxray/translator_test.go b/translator/translate/otel/exporter/awsxray/translator_test.go index c3609d6c26..b0221fd1b2 100644 --- a/translator/translate/otel/exporter/awsxray/translator_test.go +++ b/translator/translate/otel/exporter/awsxray/translator_test.go @@ -59,7 +59,7 @@ func TestTranslator(t *testing.T) { }, "WithCompleteConfig": { input: testutil.GetJson(t, filepath.Join("testdata", "config.json")), - want: testutil.GetConf(t, filepath.Join("testdata", "config.yaml")), + want: testutil.GetConf(t, filepath.Join("testdata", "ContainerInsightsJmxConfig.yaml")), mode: config.ModeOnPrem, }, "WithAppSignalsEnabledEKS": { diff --git a/translator/translate/otel/exporter/prometheusremotewrite/translator_test.go b/translator/translate/otel/exporter/prometheusremotewrite/translator_test.go index a5b9a146d1..635db8ebc4 100644 --- a/translator/translate/otel/exporter/prometheusremotewrite/translator_test.go +++ b/translator/translate/otel/exporter/prometheusremotewrite/translator_test.go @@ -37,7 +37,7 @@ func TestTranslator(t *testing.T) { }, "WithAMPDestination": { input: testutil.GetJson(t, filepath.Join("testdata", "config.json")), - want: testutil.GetConf(t, filepath.Join("testdata", "config.yaml")), + want: testutil.GetConf(t, filepath.Join("testdata", "ContainerInsightsJmxConfig.yaml")), }, } for name, testCase := range testCases { diff --git a/translator/translate/otel/pipeline/containerinsightsjmx/translator.go b/translator/translate/otel/pipeline/containerinsightsjmx/translator.go index 72e18612d8..bd42f13726 100644 --- a/translator/translate/otel/pipeline/containerinsightsjmx/translator.go +++ b/translator/translate/otel/pipeline/containerinsightsjmx/translator.go @@ -8,10 +8,10 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/awsemf" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/debug" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/cumulativetodeltaprocessor" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/jmxfilterprocessor" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/filterprocessor" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/jmxtransformprocessor" - metricstransformprocessorjmx "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/metrictransformprocessorjmx" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/resourcedetectionjmx" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/metricstransformprocessor" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/resourcedetection" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/resourceprocessor" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/otlp" @@ -21,11 +21,6 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) -const ( - pipelineName = "containerinsightsjmx" - clusterName = "cluster_name" -) - var ( baseKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey) eksKey = common.ConfigKey(baseKey, common.KubernetesKey) @@ -42,7 +37,7 @@ func NewTranslator() common.Translator[*common.ComponentTranslators] { } func (t *translator) ID() component.ID { - return component.NewIDWithName(component.DataTypeMetrics, pipelineName) + return component.NewIDWithName(component.DataTypeMetrics, common.PipelineNameContainerInsightsJmx) } // Translate creates a pipeline for container insights if the logs.metrics_collected.ecs or logs.metrics_collected.kubernetes @@ -63,14 +58,14 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators } translators.Receivers.Set(otlp.NewTranslator(common.WithName(common.PipelineNameJmx))) - translators.Processors.Set(jmxfilterprocessor.NewTranslatorWithName(pipelineName)) //Filter metrics - translators.Processors.Set(resourcedetectionjmx.NewTranslator()) //Adds k8s cluster/nodename name - translators.Processors.Set(resourceprocessor.NewTranslator(common.WithName("jmxResource"))) //Change resource attribute names - translators.Processors.Set(jmxtransformprocessor.NewTranslatorWithName(pipelineName)) //Removes attributes that are not of [ClusterName, Namespace] - translators.Processors.Set(metricstransformprocessorjmx.NewTranslatorWithName(pipelineName)) //Renames metrics and adds pool and area dimensions - translators.Processors.Set(cumulativetodeltaprocessor.NewTranslator(common.WithName(pipelineName), cumulativetodeltaprocessor.WithConfigKeys(jmxKey))) - translators.Exporters.Set(debug.NewTranslator()) //Provides debug info for metrics - translators.Exporters.Set(awsemf.NewTranslatorWithName(pipelineName)) //Sends metrics to cloudwatch console (also adds resource attributes to metrics) + translators.Processors.Set(filterprocessor.NewTranslator(common.WithName(common.PipelineNameContainerInsightsJmx))) //Filter metrics + translators.Processors.Set(resourcedetection.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx)) //Adds k8s cluster/nodename name + translators.Processors.Set(resourceprocessor.NewTranslator(common.WithName("jmxResource"))) //Change resource attribute names + translators.Processors.Set(jmxtransformprocessor.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx)) //Removes attributes that are not of [ClusterName, Namespace] + translators.Processors.Set(metricstransformprocessor.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx)) //Renames metrics and adds pool and area dimensions + translators.Processors.Set(cumulativetodeltaprocessor.NewTranslator(common.WithName(common.PipelineNameContainerInsightsJmx), cumulativetodeltaprocessor.WithConfigKeys(jmxKey))) + translators.Exporters.Set(debug.NewTranslator()) //Provides debug info for metrics + translators.Exporters.Set(awsemf.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx)) //Sends metrics to cloudwatch console (also adds resource attributes to metrics) return &translators, nil diff --git a/translator/translate/otel/processor/jmxfilterprocessor/testdata/config.yaml b/translator/translate/otel/processor/filterprocessor/testdata/ContainerInsightsJmxConfig.yaml similarity index 100% rename from translator/translate/otel/processor/jmxfilterprocessor/testdata/config.yaml rename to translator/translate/otel/processor/filterprocessor/testdata/ContainerInsightsJmxConfig.yaml diff --git a/translator/translate/otel/processor/filterprocessor/translator.go b/translator/translate/otel/processor/filterprocessor/translator.go index f5bf2d8707..b56bf3468c 100644 --- a/translator/translate/otel/processor/filterprocessor/translator.go +++ b/translator/translate/otel/processor/filterprocessor/translator.go @@ -4,6 +4,7 @@ package filterprocessor import ( + _ "embed" "fmt" "strconv" @@ -19,6 +20,9 @@ const ( matchTypeStrict = "strict" ) +//go:embed testdata/ContainerInsightsJmxConfig.yaml +var ContainerInsightsJmxConfig string + type translator struct { common.NameProvider common.IndexProvider @@ -48,11 +52,14 @@ func (t *translator) ID() component.ID { // Translate creates a processor config based on the fields in the // Metrics section of the JSON config. func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { - if conf == nil || !conf.IsSet(common.JmxConfigKey) { + if conf == nil || (!conf.IsSet(common.JmxConfigKey) && t.Name() != common.PipelineNameContainerInsightsJmx) { return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: common.JmxConfigKey} } cfg := t.factory.CreateDefaultConfig().(*filterprocessor.Config) + if t.Name() == common.PipelineNameContainerInsightsJmx { + return common.GetYamlFileToYamlConfig(cfg, ContainerInsightsJmxConfig) + } jmxMap := common.GetIndexedMap(conf, common.JmxConfigKey, t.Index()) diff --git a/translator/translate/otel/processor/filterprocessor/translator_test.go b/translator/translate/otel/processor/filterprocessor/translator_test.go index f069782296..c5d71c5a3e 100644 --- a/translator/translate/otel/processor/filterprocessor/translator_test.go +++ b/translator/translate/otel/processor/filterprocessor/translator_test.go @@ -114,7 +114,7 @@ func TestTranslator(t *testing.T) { input: testutil.GetJson(t, filepath.Join("testdata", "config.json")), index: -1, wantID: "filter/jmx", - want: testutil.GetConf(t, filepath.Join("testdata", "config.yaml")), + want: testutil.GetConf(t, filepath.Join("testdata", "ContainerInsightsJmxConfig.yaml")), }, } for name, testCase := range testCases { diff --git a/translator/translate/otel/processor/jmxfilterprocessor/translator.go b/translator/translate/otel/processor/jmxfilterprocessor/translator.go index 146634af0c..6a2f114f8d 100644 --- a/translator/translate/otel/processor/jmxfilterprocessor/translator.go +++ b/translator/translate/otel/processor/jmxfilterprocessor/translator.go @@ -4,7 +4,7 @@ package jmxfilterprocessor import ( - "fmt" + _ "embed" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" @@ -13,10 +13,6 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) -const ( - matchTypeStrict = "strict" -) - type translator struct { name string factory processor.Factory @@ -40,42 +36,8 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { } cfg := t.factory.CreateDefaultConfig().(*filterprocessor.Config) - - includeMetricNames := []string{ - "jvm.classes.loaded", - "jvm.memory.heap.used", - "jvm.memory.nonheap.used", - "jvm.memory.pool.used", - "jvm.operating.system.total.swap.space.size", - "jvm.operating.system.system.cpu.load", - "jvm.operating.system.process.cpu.load", - "jvm.operating.system.free.swap.space.size", - "jvm.operating.system.total.physical.memory.size", - "jvm.operating.system.free.physical.memory.size", - "jvm.operating.system.open.file.descriptor.count", - "jvm.operating.system.available.processors", - "jvm.threads.count", - "jvm.threads.daemon", - "tomcat.sessions", - "tomcat.rejected_sessions", - "tomcat.traffic.received", - "tomcat.traffic.sent", - "tomcat.request_count", - "tomcat.errors", - "tomcat.processing_time", - } - - c := confmap.NewFromStringMap(map[string]interface{}{ - "metrics": map[string]any{ - "include": map[string]any{ - "match_type": matchTypeStrict, - "metric_names": includeMetricNames, - }, - }, - }) - - if err := c.Unmarshal(&cfg); err != nil { - return nil, fmt.Errorf("unable to unmarshal jmx filter processor (%s): %w", t.ID(), err) + if t.name == common.PipelineNameContainerInsightsJmx { + return common.GetYamlFileToYamlConfig(cfg, metricTransformJmxConfig) } return cfg, nil diff --git a/translator/translate/otel/processor/jmxfilterprocessor/translator_test.go b/translator/translate/otel/processor/jmxfilterprocessor/translator_test.go index ac9bbc6aa7..fdac2243a5 100644 --- a/translator/translate/otel/processor/jmxfilterprocessor/translator_test.go +++ b/translator/translate/otel/processor/jmxfilterprocessor/translator_test.go @@ -46,7 +46,7 @@ func TestTranslator(t *testing.T) { input: testutil.GetJson(t, filepath.Join("testdata", "config.json")), index: 0, wantID: "filter/jmx", - want: testutil.GetConf(t, filepath.Join("testdata", "config.yaml")), + want: testutil.GetConf(t, filepath.Join("testdata", "ContainerInsightsJmxConfig.yaml")), }, } diff --git a/translator/translate/otel/processor/jmxtransformprocessor/translate.go b/translator/translate/otel/processor/jmxtransformprocessor/translate.go index da86988bbd..cf7c386aed 100644 --- a/translator/translate/otel/processor/jmxtransformprocessor/translate.go +++ b/translator/translate/otel/processor/jmxtransformprocessor/translate.go @@ -5,7 +5,6 @@ package jmxtransformprocessor import ( _ "embed" - "fmt" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor" "go.opentelemetry.io/collector/component" @@ -42,51 +41,9 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: common.ContainerInsightsConfigKey} } cfg := t.factory.CreateDefaultConfig().(*transformprocessor.Config) - clusterName := conf.Get(common.ConfigKey(common.ContainerInsightsConfigKey, "cluster_name")) - - if clusterName == nil { + if t.name == common.PipelineNameContainerInsightsJmx { return common.GetYamlFileToYamlConfig(cfg, transformJmxConfig) } - c := confmap.NewFromStringMap(map[string]interface{}{ - "metric_statements": []map[string]interface{}{ - { - "context": "resource", - "statements": []string{ - "keep_keys(attributes, [\"ClusterName\", \"Namespace\",\"NodeName\"])", - }, - }, - { - "context": "metric", - "statements": []string{ - "set(unit, \"Bytes\") where name == \"jvm.memory.heap.used\"", - "set(unit, \"Bytes\") where name == \"jvm.memory.nonheap.used\"", - "set(unit, \"Bytes\") where name == \"jvm.memory.pool.used\"", - "set(unit, \"Bytes\") where name == \"tomcat.traffic.received\"", - "set(unit, \"Bytes\") where name == \"tomcat.traffic.sent\"", - "set(unit, \"Bytes\") where name == \"jvm.operating.system.total.swap.space.size\"", - "set(unit, \"Bytes\") where name == \"jvm.operating.system.free.swap.space.size\"", - "set(unit, \"Bytes\") where name == \"jvm.operating.system.total.physical.memory.size\"", - "set(unit, \"Bytes\") where name == \"jvm.operating.system.free.physical.memory.size\"", - "set(unit, \"Count\") where name == \"tomcat.sessions\"", - "set(unit, \"Count\") where name == \"tomcat.rejected_sessions\"", - "set(unit, \"Count\") where name == \"jvm.threads.count\"", - "set(unit, \"Count\") where name == \"jvm.threads.daemon\"", - "set(unit, \"Count\") where name == \"jvm.operating.system.open.file.descriptor.count\"", - "set(unit, \"Count\") where name == \"jvm.operating.system.available.processors\"", - "set(unit, \"Count\") where name == \"tomcat.request_count\"", - "set(unit, \"Count\") where name == \"tomcat.errors\"", - "set(unit, \"Count\") where name == \"jvm.classes.loaded\"", - "set(unit, \"Count\") where name == \"jvm.operating.system.system.cpu.load\"", - "set(unit, \"Count\") where name == \"jvm.operating.system.process.cpu.load\"", - "set(unit, \"Milliseconds\") where name == \"tomcat.processing_time\"", - }, - }, - }, - }) - - if err := c.Unmarshal(&cfg); err != nil { - return nil, fmt.Errorf("unable to unmarshal jmx filter processor (%s): %w", t.ID(), err) - } return cfg, nil } diff --git a/translator/translate/otel/processor/metricsdecorator/translator_unix_test.go b/translator/translate/otel/processor/metricsdecorator/translator_unix_test.go index d00e60d538..6b05733488 100644 --- a/translator/translate/otel/processor/metricsdecorator/translator_unix_test.go +++ b/translator/translate/otel/processor/metricsdecorator/translator_unix_test.go @@ -30,7 +30,7 @@ func TestTranslate(t *testing.T) { translatorcontext.CurrentContext().SetOs(translatorconfig.OS_TYPE_LINUX) transl := NewTranslator().(*translator) expectedCfg := transl.factory.CreateDefaultConfig().(*transformprocessor.Config) - c := testutil.GetConf(t, filepath.Join("testdata", "unix", "config.yaml")) + c := testutil.GetConf(t, filepath.Join("testdata", "unix", "ContainerInsightsJmxConfig.yaml")) require.NoError(t, c.Unmarshal(&expectedCfg)) conf := confmap.NewFromStringMap(testutil.GetJson(t, filepath.Join("testdata", "unix", "config.json"))) @@ -53,7 +53,7 @@ func TestMetricDecoration(t *testing.T) { translatorcontext.CurrentContext().SetOs(translatorconfig.OS_TYPE_LINUX) transl := NewTranslator().(*translator) cfg := transl.factory.CreateDefaultConfig().(*transformprocessor.Config) - conf := testutil.GetConf(t, filepath.Join("testdata", "unix", "config.yaml")) + conf := testutil.GetConf(t, filepath.Join("testdata", "unix", "ContainerInsightsJmxConfig.yaml")) require.NoError(t, conf.Unmarshal(&cfg)) sink := new(consumertest.MetricsSink) diff --git a/translator/translate/otel/processor/metricsdecorator/translator_windows_test.go b/translator/translate/otel/processor/metricsdecorator/translator_windows_test.go index 4b0a8c9593..b9f620ba10 100644 --- a/translator/translate/otel/processor/metricsdecorator/translator_windows_test.go +++ b/translator/translate/otel/processor/metricsdecorator/translator_windows_test.go @@ -31,7 +31,7 @@ func TestTranslate(t *testing.T) { translatorcontext.CurrentContext().SetOs(translatorconfig.OS_TYPE_WINDOWS) transl := NewTranslator().(*translator) expectedCfg := transl.factory.CreateDefaultConfig().(*transformprocessor.Config) - c := testutil.GetConf(t, filepath.Join("testdata", "windows", "config.yaml")) + c := testutil.GetConf(t, filepath.Join("testdata", "windows", "ContainerInsightsJmxConfig.yaml")) err := c.Unmarshal(&expectedCfg) assert.NoError(t, err) @@ -54,7 +54,7 @@ func TestMetricDecoration(t *testing.T) { translatorcontext.CurrentContext().SetOs(translatorconfig.OS_TYPE_WINDOWS) transl := NewTranslator().(*translator) cfg := transl.factory.CreateDefaultConfig().(*transformprocessor.Config) - conf := testutil.GetConf(t, filepath.Join("testdata", "windows", "config.yaml")) + conf := testutil.GetConf(t, filepath.Join("testdata", "windows", "ContainerInsightsJmxConfig.yaml")) err := conf.Unmarshal(&cfg) assert.NoError(t, err) sink := new(consumertest.MetricsSink) diff --git a/translator/translate/otel/processor/metrictransformprocessorjmx/configs/config.yaml b/translator/translate/otel/processor/metricstransformprocessor/configs/config.yaml similarity index 99% rename from translator/translate/otel/processor/metrictransformprocessorjmx/configs/config.yaml rename to translator/translate/otel/processor/metricstransformprocessor/configs/config.yaml index 00dc4bb660..3ae4860d50 100644 --- a/translator/translate/otel/processor/metrictransformprocessorjmx/configs/config.yaml +++ b/translator/translate/otel/processor/metricstransformprocessor/configs/config.yaml @@ -131,4 +131,3 @@ transforms: action: update new_name: catalina_globalrequestprocessor_processingtime - diff --git a/translator/translate/otel/processor/metricstransformprocessor/translator.go b/translator/translate/otel/processor/metricstransformprocessor/translator.go index 4b83e190fc..fc6a843472 100644 --- a/translator/translate/otel/processor/metricstransformprocessor/translator.go +++ b/translator/translate/otel/processor/metricstransformprocessor/translator.go @@ -4,6 +4,7 @@ package metricstransformprocessor import ( + _ "embed" "fmt" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor" @@ -16,6 +17,9 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/awscontainerinsight" ) +//go:embed configs/config.yaml +var metricTransformJmxConfig string + var metricDuplicateTypes = []string{ containerinsightscommon.TypeGpuContainer, containerinsightscommon.TypeGpuPod, @@ -64,6 +68,10 @@ func (t *translator) ID() component.ID { func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { cfg := t.factory.CreateDefaultConfig().(*metricstransformprocessor.Config) + if t.name == common.PipelineNameContainerInsightsJmx { + return common.GetYamlFileToYamlConfig(cfg, metricTransformJmxConfig) + } + transformRules := []map[string]interface{}{ { "include": "apiserver_request_total", diff --git a/translator/translate/otel/processor/metrictransformprocessorjmx/translator.go b/translator/translate/otel/processor/metrictransformprocessorjmx/translator.go deleted file mode 100644 index 521ba0b519..0000000000 --- a/translator/translate/otel/processor/metrictransformprocessorjmx/translator.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package metricstransformprocessorjmx - -import ( - _ "embed" - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor" - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/confmap" - "go.opentelemetry.io/collector/processor" - - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" -) - -//go:embed configs/config.yaml -var metricTransformJMXConfig string - -type translator struct { - name string - factory processor.Factory -} - -var _ common.Translator[component.Config] = (*translator)(nil) - -func NewTranslatorWithName(name string) common.Translator[component.Config] { - return &translator{name, metricstransformprocessor.NewFactory()} -} - -func (t *translator) ID() component.ID { - return component.NewIDWithName(t.factory.Type(), t.name) -} - -func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { - cfg := t.factory.CreateDefaultConfig().(*metricstransformprocessor.Config) - return common.GetYamlFileToYamlConfig(cfg, metricTransformJMXConfig) -} diff --git a/translator/translate/otel/processor/resourcedetectionjmx/configs/config.yaml b/translator/translate/otel/processor/resourcedetection/configs/jmx_config.yaml similarity index 100% rename from translator/translate/otel/processor/resourcedetectionjmx/configs/config.yaml rename to translator/translate/otel/processor/resourcedetection/configs/jmx_config.yaml diff --git a/translator/translate/otel/processor/resourcedetection/translator.go b/translator/translate/otel/processor/resourcedetection/translator.go index 454e59ed79..d658ffe080 100644 --- a/translator/translate/otel/processor/resourcedetection/translator.go +++ b/translator/translate/otel/processor/resourcedetection/translator.go @@ -23,6 +23,9 @@ var appSignalsDefaultResourceDetectionConfig string //go:embed configs/ecs_config.yaml var appSignalsECSResourceDetectionConfig string +//go:embed configs/jmx_config.yaml +var resourceDetectionJMX string + type translator struct { name string dataType component.DataType @@ -49,6 +52,14 @@ func WithDataType(dataType component.DataType) Option { var _ common.Translator[component.Config] = (*translator)(nil) +func NewTranslatorWithName(name string, opts ...Option) common.Translator[component.Config] { + t := &translator{name: name, factory: resourcedetectionprocessor.NewFactory()} + for _, opt := range opts { + opt.apply(t) + } + return t +} + func NewTranslator(opts ...Option) common.Translator[component.Config] { t := &translator{factory: resourcedetectionprocessor.NewFactory()} for _, opt := range opts { @@ -64,6 +75,9 @@ func (t *translator) ID() component.ID { func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { cfg := t.factory.CreateDefaultConfig().(*resourcedetectionprocessor.Config) + if t.name == common.PipelineNameContainerInsightsJmx { + return common.GetYamlFileToYamlConfig(cfg, resourceDetectionJMX) + } mode := context.CurrentContext().KubernetesMode() if mode == "" { mode = context.CurrentContext().Mode() diff --git a/translator/translate/otel/processor/resourcedetectionjmx/translator.go b/translator/translate/otel/processor/resourcedetectionjmx/translator.go deleted file mode 100644 index 99a059dfa7..0000000000 --- a/translator/translate/otel/processor/resourcedetectionjmx/translator.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package resourcedetectionjmx - -import ( - _ "embed" - - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor" - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/confmap" - "go.opentelemetry.io/collector/processor" - - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" -) - -//go:embed configs/config.yaml -var resourceDetectionJMX string - -type translator struct { - name string - dataType component.DataType - factory processor.Factory -} - -type Option interface { - apply(t *translator) -} - -type optionFunc func(t *translator) - -func (o optionFunc) apply(t *translator) { - o(t) -} - -// WithDataType determines where the translator should look to find -// the configuration. -func WithDataType(dataType component.DataType) Option { - return optionFunc(func(t *translator) { - t.dataType = dataType - }) -} - -var _ common.Translator[component.Config] = (*translator)(nil) - -func NewTranslator(opts ...Option) common.Translator[component.Config] { - t := &translator{factory: resourcedetectionprocessor.NewFactory()} - for _, opt := range opts { - opt.apply(t) - } - return t -} - -func (t *translator) ID() component.ID { - return component.NewIDWithName(t.factory.Type(), t.name) -} - -func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { - cfg := t.factory.CreateDefaultConfig().(*resourcedetectionprocessor.Config) - return common.GetYamlFileToYamlConfig(cfg, resourceDetectionJMX) -} diff --git a/translator/translate/otel/receiver/awsxray/translator_test.go b/translator/translate/otel/receiver/awsxray/translator_test.go index 5a385fe942..e1d5650b79 100644 --- a/translator/translate/otel/receiver/awsxray/translator_test.go +++ b/translator/translate/otel/receiver/awsxray/translator_test.go @@ -49,7 +49,7 @@ func TestTranslator(t *testing.T) { }, "WithCompleteConfig": { input: testutil.GetJson(t, filepath.Join("testdata", "config.json")), - want: testutil.GetConf(t, filepath.Join("testdata", "config.yaml")), + want: testutil.GetConf(t, filepath.Join("testdata", "ContainerInsightsJmxConfig.yaml")), }, } factory := awsxrayreceiver.NewFactory() diff --git a/translator/translate/otel/receiver/jmx/translator_test.go b/translator/translate/otel/receiver/jmx/translator_test.go index 5db211ad03..90f098018a 100644 --- a/translator/translate/otel/receiver/jmx/translator_test.go +++ b/translator/translate/otel/receiver/jmx/translator_test.go @@ -100,7 +100,7 @@ func TestTranslator(t *testing.T) { input: testutil.GetJson(t, filepath.Join("testdata", "config.json")), index: -1, wantID: "jmx", - want: testutil.GetConf(t, filepath.Join("testdata", "config.yaml")), + want: testutil.GetConf(t, filepath.Join("testdata", "ContainerInsightsJmxConfig.yaml")), }, } for name, testCase := range testCases { diff --git a/translator/translate/otel/receiver/otlp/translator_test.go b/translator/translate/otel/receiver/otlp/translator_test.go index e45435f018..824c5b523a 100644 --- a/translator/translate/otel/receiver/otlp/translator_test.go +++ b/translator/translate/otel/receiver/otlp/translator_test.go @@ -73,7 +73,7 @@ func TestTracesTranslator(t *testing.T) { }, "WithCompleteConfig": { input: testutil.GetJson(t, filepath.Join("testdata", "traces", "config.json")), - want: testutil.GetConf(t, filepath.Join("testdata", "traces", "config.yaml")), + want: testutil.GetConf(t, filepath.Join("testdata", "traces", "ContainerInsightsJmxConfig.yaml")), }, } factory := otlpreceiver.NewFactory() @@ -166,7 +166,7 @@ func TestMetricsTranslator(t *testing.T) { "WithCompleteConfig": { input: testutil.GetJson(t, filepath.Join("testdata", "metrics", "config.json")), index: -1, - want: testutil.GetConf(t, filepath.Join("testdata", "metrics", "config.yaml")), + want: testutil.GetConf(t, filepath.Join("testdata", "metrics", "ContainerInsightsJmxConfig.yaml")), }, } factory := otlpreceiver.NewFactory() @@ -263,7 +263,7 @@ func TestMetricsEmfTranslator(t *testing.T) { "WithCompleteConfig": { input: testutil.GetJson(t, filepath.Join("testdata", "metrics_emf", "config.json")), index: -1, - want: testutil.GetConf(t, filepath.Join("testdata", "metrics_emf", "config.yaml")), + want: testutil.GetConf(t, filepath.Join("testdata", "metrics_emf", "ContainerInsightsJmxConfig.yaml")), }, } factory := otlpreceiver.NewFactory() From 146ff79e1e406f484fed16e737a73aad27461ecb Mon Sep 17 00:00:00 2001 From: Paramadon Date: Thu, 10 Oct 2024 01:39:36 -0400 Subject: [PATCH 04/19] adding test/merging processors --- .../containerinsightsjmx/translator.go | 10 +- .../filterprocessor/translator_test.go | 15 + .../jmxfilterprocessor/testdata/config.json | 20 -- .../jmxfilterprocessor/translator.go | 44 --- .../jmxfilterprocessor/translator_test.go | 81 ----- .../jmxtransformprocessor/translator_test.go | 14 + .../configs/config.json | 13 + .../translator_test.go | 18 ++ .../resourcedetection/configs/config.json | 13 + .../resourcedetection/translator_test.go | 18 ++ .../resourceprocessor/testdata/config.json | 13 + .../resourceprocessor/testdata/config.yaml | 10 + .../processor/resourceprocessor/translator.go | 6 +- .../resourceprocessor/translator_test.go | 18 ++ .../translate/otel/translate_otel_test.go | 294 +++++++++--------- 15 files changed, 287 insertions(+), 300 deletions(-) delete mode 100644 translator/translate/otel/processor/jmxfilterprocessor/testdata/config.json delete mode 100644 translator/translate/otel/processor/jmxfilterprocessor/translator.go delete mode 100644 translator/translate/otel/processor/jmxfilterprocessor/translator_test.go create mode 100644 translator/translate/otel/processor/metricstransformprocessor/configs/config.json create mode 100644 translator/translate/otel/processor/resourcedetection/configs/config.json create mode 100644 translator/translate/otel/processor/resourceprocessor/testdata/config.json create mode 100644 translator/translate/otel/processor/resourceprocessor/testdata/config.yaml diff --git a/translator/translate/otel/pipeline/containerinsightsjmx/translator.go b/translator/translate/otel/pipeline/containerinsightsjmx/translator.go index bd42f13726..fba46d2e72 100644 --- a/translator/translate/otel/pipeline/containerinsightsjmx/translator.go +++ b/translator/translate/otel/pipeline/containerinsightsjmx/translator.go @@ -58,11 +58,11 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators } translators.Receivers.Set(otlp.NewTranslator(common.WithName(common.PipelineNameJmx))) - translators.Processors.Set(filterprocessor.NewTranslator(common.WithName(common.PipelineNameContainerInsightsJmx))) //Filter metrics - translators.Processors.Set(resourcedetection.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx)) //Adds k8s cluster/nodename name - translators.Processors.Set(resourceprocessor.NewTranslator(common.WithName("jmxResource"))) //Change resource attribute names - translators.Processors.Set(jmxtransformprocessor.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx)) //Removes attributes that are not of [ClusterName, Namespace] - translators.Processors.Set(metricstransformprocessor.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx)) //Renames metrics and adds pool and area dimensions + translators.Processors.Set(filterprocessor.NewTranslator(common.WithName(common.PipelineNameContainerInsightsJmx))) //Filter metrics + translators.Processors.Set(resourcedetection.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx)) //Adds k8s cluster/nodename name + translators.Processors.Set(resourceprocessor.NewTranslator(common.WithName(common.PipelineNameContainerInsightsJmx))) //Change resource attribute names + translators.Processors.Set(jmxtransformprocessor.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx)) //Removes attributes that are not of [ClusterName, Namespace] + translators.Processors.Set(metricstransformprocessor.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx)) //Renames metrics and adds pool and area dimensions translators.Processors.Set(cumulativetodeltaprocessor.NewTranslator(common.WithName(common.PipelineNameContainerInsightsJmx), cumulativetodeltaprocessor.WithConfigKeys(jmxKey))) translators.Exporters.Set(debug.NewTranslator()) //Provides debug info for metrics translators.Exporters.Set(awsemf.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx)) //Sends metrics to cloudwatch console (also adds resource attributes to metrics) diff --git a/translator/translate/otel/processor/filterprocessor/translator_test.go b/translator/translate/otel/processor/filterprocessor/translator_test.go index c5d71c5a3e..a2820290f4 100644 --- a/translator/translate/otel/processor/filterprocessor/translator_test.go +++ b/translator/translate/otel/processor/filterprocessor/translator_test.go @@ -4,6 +4,7 @@ package filterprocessor import ( + "github.com/stretchr/testify/assert" "path/filepath" "testing" @@ -135,3 +136,17 @@ func TestTranslator(t *testing.T) { }) } } + +func TestContainerInsightsJmx(t *testing.T) { + transl := NewTranslator(common.WithName(common.PipelineNameContainerInsightsJmx)).(*translator) + expectedCfg := transl.factory.CreateDefaultConfig().(*filterprocessor.Config) + c := testutil.GetConf(t, filepath.Join("testdata", "ContainerInsightsJmxConfig.yaml")) + require.NoError(t, c.Unmarshal(&expectedCfg)) + + conf := confmap.NewFromStringMap(testutil.GetJson(t, filepath.Join("testdata", "config.json"))) + translatedCfg, err := transl.Translate(conf) + assert.NoError(t, err) + actualCfg, ok := translatedCfg.(*filterprocessor.Config) + assert.True(t, ok) + assert.Equal(t, len(expectedCfg.Metrics.Include.MetricNames), len(actualCfg.Metrics.Include.MetricNames)) +} diff --git a/translator/translate/otel/processor/jmxfilterprocessor/testdata/config.json b/translator/translate/otel/processor/jmxfilterprocessor/testdata/config.json deleted file mode 100644 index 510cb41463..0000000000 --- a/translator/translate/otel/processor/jmxfilterprocessor/testdata/config.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "agent": { - "region": "us-east-1" - }, - "logs": { - "metrics_collected": { - "emf": { - }, - "kubernetes": { - "cluster_name": "TestCluster", - "metrics_collection_interval": 30, - "disable_metric_extraction": true, - "prefer_full_pod_name": true, - "accelerated_compute_metrics": false - } - }, - "force_flush_interval": 5, - "endpoint_override":"https://fake_endpoint" - } -} diff --git a/translator/translate/otel/processor/jmxfilterprocessor/translator.go b/translator/translate/otel/processor/jmxfilterprocessor/translator.go deleted file mode 100644 index 6a2f114f8d..0000000000 --- a/translator/translate/otel/processor/jmxfilterprocessor/translator.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package jmxfilterprocessor - -import ( - _ "embed" - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor" - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/confmap" - "go.opentelemetry.io/collector/processor" - - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" -) - -type translator struct { - name string - factory processor.Factory -} - -type Option func(any) - -var _ common.Translator[component.Config] = (*translator)(nil) - -func NewTranslatorWithName(name string) common.Translator[component.Config] { - return &translator{name: name, factory: filterprocessor.NewFactory()} -} - -func (t *translator) ID() component.ID { - return component.NewIDWithName(t.factory.Type(), t.name) -} - -func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { - if conf == nil || !conf.IsSet(common.ContainerInsightsConfigKey) { - return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: common.ContainerInsightsConfigKey} - } - - cfg := t.factory.CreateDefaultConfig().(*filterprocessor.Config) - if t.name == common.PipelineNameContainerInsightsJmx { - return common.GetYamlFileToYamlConfig(cfg, metricTransformJmxConfig) - } - - return cfg, nil -} diff --git a/translator/translate/otel/processor/jmxfilterprocessor/translator_test.go b/translator/translate/otel/processor/jmxfilterprocessor/translator_test.go deleted file mode 100644 index fdac2243a5..0000000000 --- a/translator/translate/otel/processor/jmxfilterprocessor/translator_test.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: MIT - -package jmxfilterprocessor - -import ( - "path/filepath" - "testing" - - "github.com/aws/amazon-cloudwatch-agent/internal/util/testutil" - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor" - "github.com/stretchr/testify/require" - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/confmap" - - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" - "gopkg.in/yaml.v2" -) - -func TestTranslator(t *testing.T) { - factory := filterprocessor.NewFactory() - - testCases := map[string]struct { - input map[string]any - index int - wantID string - want *confmap.Conf - wantErr error - }{ - "ConfigWithNoJmxSet": { - input: map[string]any{ - "metrics": map[string]any{ - "metrics_collected": map[string]any{ - "cpu": map[string]any{}, - }, - }, - }, - index: 0, - wantID: "filter/jmx", - wantErr: &common.MissingKeyError{ - ID: component.NewIDWithName(factory.Type(), "jmx"), - JsonKey: common.ContainerInsightsConfigKey, - }, - }, - "WithCompleteConfig": { - input: testutil.GetJson(t, filepath.Join("testdata", "config.json")), - index: 0, - wantID: "filter/jmx", - want: testutil.GetConf(t, filepath.Join("testdata", "ContainerInsightsJmxConfig.yaml")), - }, - } - - for name, testCase := range testCases { - t.Run(name, func(t *testing.T) { - tt := NewTranslatorWithName("jmx") - require.EqualValues(t, testCase.wantID, tt.ID().String()) - - conf := confmap.NewFromStringMap(testCase.input) - got, err := tt.Translate(conf) - require.Equal(t, testCase.wantErr, err) - - if err == nil { - require.NotNil(t, got) - - gotCfg, ok := got.(*filterprocessor.Config) - require.True(t, ok) - - wantCfg := factory.CreateDefaultConfig() - require.NoError(t, testCase.want.Unmarshal(wantCfg)) - - gotYAML, err := yaml.Marshal(gotCfg) - require.NoError(t, err) - - wantYAML, err := yaml.Marshal(wantCfg) - require.NoError(t, err) - - require.Equal(t, wantCfg, gotCfg, "Expected:\n%s\nGot:\n%s", wantYAML, gotYAML) - } - }) - } -} diff --git a/translator/translate/otel/processor/jmxtransformprocessor/translator_test.go b/translator/translate/otel/processor/jmxtransformprocessor/translator_test.go index 1a43e066f1..12376adc50 100644 --- a/translator/translate/otel/processor/jmxtransformprocessor/translator_test.go +++ b/translator/translate/otel/processor/jmxtransformprocessor/translator_test.go @@ -64,3 +64,17 @@ func TestTranslator(t *testing.T) { }) } } + +func TestContainerInsightsJmx(t *testing.T) { + transl := NewTranslatorWithName(common.PipelineNameContainerInsightsJmx).(*translator) + expectedCfg := transl.factory.CreateDefaultConfig().(*transformprocessor.Config) + c := testutil.GetConf(t, filepath.Join("testdata", "config.yaml")) + require.NoError(t, c.Unmarshal(&expectedCfg)) + + conf := confmap.NewFromStringMap(testutil.GetJson(t, filepath.Join("testdata", "config.json"))) + translatedCfg, err := transl.Translate(conf) + assert.NoError(t, err) + actualCfg, ok := translatedCfg.(*transformprocessor.Config) + assert.True(t, ok) + assert.Equal(t, len(expectedCfg.MetricStatements), len(actualCfg.MetricStatements)) +} diff --git a/translator/translate/otel/processor/metricstransformprocessor/configs/config.json b/translator/translate/otel/processor/metricstransformprocessor/configs/config.json new file mode 100644 index 0000000000..46cae7bee5 --- /dev/null +++ b/translator/translate/otel/processor/metricstransformprocessor/configs/config.json @@ -0,0 +1,13 @@ +{ + "agent": { + "debug": true + }, + "logs": { + "metrics_collected": { + "kubernetes": { + "cluster_name": "TestCluster", + "metrics_collection_interval": 30 + } + } + } +} \ No newline at end of file diff --git a/translator/translate/otel/processor/metricstransformprocessor/translator_test.go b/translator/translate/otel/processor/metricstransformprocessor/translator_test.go index c22498c789..3cc7f72019 100644 --- a/translator/translate/otel/processor/metricstransformprocessor/translator_test.go +++ b/translator/translate/otel/processor/metricstransformprocessor/translator_test.go @@ -5,6 +5,9 @@ package metricstransformprocessor import ( "fmt" + "github.com/aws/amazon-cloudwatch-agent/internal/util/testutil" + "github.com/stretchr/testify/assert" + "path/filepath" "testing" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor" @@ -56,3 +59,18 @@ func TestTranslator(t *testing.T) { }) } } + +func TestContainerInsightsJmx(t *testing.T) { + transl := NewTranslatorWithName(common.PipelineNameContainerInsightsJmx).(*translator) + expectedCfg := transl.factory.CreateDefaultConfig().(*metricstransformprocessor.Config) + c := testutil.GetConf(t, filepath.Join("configs", "config.yaml")) + require.NoError(t, c.Unmarshal(&expectedCfg)) + + conf := confmap.NewFromStringMap(testutil.GetJson(t, filepath.Join("configs", "config.json"))) + translatedCfg, err := transl.Translate(conf) + assert.NoError(t, err) + actualCfg, ok := translatedCfg.(*metricstransformprocessor.Config) + assert.True(t, ok) + assert.Equal(t, len(expectedCfg.Transforms), len(actualCfg.Transforms)) + +} diff --git a/translator/translate/otel/processor/resourcedetection/configs/config.json b/translator/translate/otel/processor/resourcedetection/configs/config.json new file mode 100644 index 0000000000..46cae7bee5 --- /dev/null +++ b/translator/translate/otel/processor/resourcedetection/configs/config.json @@ -0,0 +1,13 @@ +{ + "agent": { + "debug": true + }, + "logs": { + "metrics_collected": { + "kubernetes": { + "cluster_name": "TestCluster", + "metrics_collection_interval": 30 + } + } + } +} \ No newline at end of file diff --git a/translator/translate/otel/processor/resourcedetection/translator_test.go b/translator/translate/otel/processor/resourcedetection/translator_test.go index 6de59538b0..b9cb046e0e 100644 --- a/translator/translate/otel/processor/resourcedetection/translator_test.go +++ b/translator/translate/otel/processor/resourcedetection/translator_test.go @@ -4,6 +4,9 @@ package resourcedetection import ( + "github.com/aws/amazon-cloudwatch-agent/internal/util/testutil" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" + "path/filepath" "testing" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor" @@ -142,3 +145,18 @@ func TestTranslate(t *testing.T) { }) } } + +func TestContainerInsightsJmx(t *testing.T) { + transl := NewTranslatorWithName(common.PipelineNameContainerInsightsJmx).(*translator) + expectedCfg := transl.factory.CreateDefaultConfig().(*resourcedetectionprocessor.Config) + c := testutil.GetConf(t, filepath.Join("configs", "jmx_config.yaml")) + require.NoError(t, c.Unmarshal(&expectedCfg)) + + conf := confmap.NewFromStringMap(testutil.GetJson(t, filepath.Join("configs", "config.json"))) + translatedCfg, err := transl.Translate(conf) + assert.NoError(t, err) + actualCfg, ok := translatedCfg.(*resourcedetectionprocessor.Config) + assert.True(t, ok) + assert.Equal(t, len(actualCfg.Detectors), len(expectedCfg.Detectors)) + +} diff --git a/translator/translate/otel/processor/resourceprocessor/testdata/config.json b/translator/translate/otel/processor/resourceprocessor/testdata/config.json new file mode 100644 index 0000000000..46cae7bee5 --- /dev/null +++ b/translator/translate/otel/processor/resourceprocessor/testdata/config.json @@ -0,0 +1,13 @@ +{ + "agent": { + "debug": true + }, + "logs": { + "metrics_collected": { + "kubernetes": { + "cluster_name": "TestCluster", + "metrics_collection_interval": 30 + } + } + } +} \ No newline at end of file diff --git a/translator/translate/otel/processor/resourceprocessor/testdata/config.yaml b/translator/translate/otel/processor/resourceprocessor/testdata/config.yaml new file mode 100644 index 0000000000..5a2083d167 --- /dev/null +++ b/translator/translate/otel/processor/resourceprocessor/testdata/config.yaml @@ -0,0 +1,10 @@ +attributes: + - key: "Namespace" + from_attribute: "k8s.namespace.name" + action: "insert" + - key: "ClusterName" + value: "TestCluster" + action: "upsert" + - key: "NodeName" + from_attribute: "host.name" + action: "insert" diff --git a/translator/translate/otel/processor/resourceprocessor/translator.go b/translator/translate/otel/processor/resourceprocessor/translator.go index ad41e2443a..998e500c13 100644 --- a/translator/translate/otel/processor/resourceprocessor/translator.go +++ b/translator/translate/otel/processor/resourceprocessor/translator.go @@ -50,7 +50,7 @@ func (t *translator) ID() component.ID { // Translate creates a processor config based on the fields in the // Metrics section of the JSON config. func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { - if conf == nil || (!conf.IsSet(common.JmxConfigKey) && t.Name() != "jmxResource") { + if conf == nil || (!conf.IsSet(common.JmxConfigKey) && t.Name() != common.PipelineNameContainerInsightsJmx) { return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: common.JmxConfigKey} } @@ -59,7 +59,7 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { if strings.HasPrefix(t.Name(), common.PipelineNameJmx) { attributes = t.getJMXAttributes(conf) } - if len(attributes) == 0 && t.Name() != "jmxResource" { + if len(attributes) == 0 && t.Name() != common.PipelineNameContainerInsightsJmx { baseKey := common.JmxConfigKey if t.Index() != -1 { baseKey = fmt.Sprintf("%s[%d]", baseKey, t.Index()) @@ -69,7 +69,7 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { c := confmap.NewFromStringMap(map[string]any{ "attributes": attributes, }) - if t.Name() == "jmxResource" { + if t.Name() == common.PipelineNameContainerInsightsJmx { clusterName, ok := common.GetString(conf, common.ConfigKey(eksKey, "cluster_name")) if ok { diff --git a/translator/translate/otel/processor/resourceprocessor/translator_test.go b/translator/translate/otel/processor/resourceprocessor/translator_test.go index 5a76b2feb8..a60ea8eb42 100644 --- a/translator/translate/otel/processor/resourceprocessor/translator_test.go +++ b/translator/translate/otel/processor/resourceprocessor/translator_test.go @@ -4,6 +4,9 @@ package resourceprocessor import ( + "github.com/aws/amazon-cloudwatch-agent/internal/util/testutil" + "github.com/stretchr/testify/require" + "path/filepath" "testing" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor" @@ -187,3 +190,18 @@ func TestTranslator(t *testing.T) { }) } } + +func TestContainerInsightsJmx(t *testing.T) { + transl := NewTranslator(common.WithName(common.PipelineNameContainerInsightsJmx)).(*translator) + expectedCfg := transl.factory.CreateDefaultConfig().(*resourceprocessor.Config) + c := testutil.GetConf(t, filepath.Join("testdata", "config.yaml")) + require.NoError(t, c.Unmarshal(&expectedCfg)) + + conf := confmap.NewFromStringMap(testutil.GetJson(t, filepath.Join("testdata", "config.json"))) + translatedCfg, err := transl.Translate(conf) + assert.NoError(t, err) + actualCfg, ok := translatedCfg.(*resourceprocessor.Config) + assert.True(t, ok) + assert.Equal(t, len(actualCfg.AttributesActions), len(expectedCfg.AttributesActions)) + +} diff --git a/translator/translate/otel/translate_otel_test.go b/translator/translate/otel/translate_otel_test.go index de5f387193..f17e8fe1aa 100644 --- a/translator/translate/otel/translate_otel_test.go +++ b/translator/translate/otel/translate_otel_test.go @@ -39,153 +39,153 @@ func TestTranslator(t *testing.T) { }, }, }, - //"WithEmptyConfig": { - // input: map[string]interface{}{}, - // wantErrContains: "no valid pipelines", - //}, - //"WithoutReceivers": { - // input: map[string]interface{}{ - // "metrics": map[string]interface{}{}, - // }, - // wantErrContains: "no valid pipelines", - //}, - //"WithMinimalConfig": { - // input: map[string]interface{}{ - // "metrics": map[string]interface{}{ - // "metrics_collected": map[string]interface{}{ - // "cpu": map[string]interface{}{}, - // }, - // }, - // }, - //}, - //"WithAppSignalsMetricsEnabled": { - // input: map[string]interface{}{ - // "logs": map[string]interface{}{ - // "metrics_collected": map[string]interface{}{ - // "application_signals": map[string]interface{}{}, - // }, - // }, - // }, - // detector: eksdetector.TestEKSDetector, - // isEKSDataStore: eksdetector.TestIsEKSCacheEKS, - //}, - //"WithAppSignalsTracesEnabled": { - // input: map[string]interface{}{ - // "traces": map[string]interface{}{ - // "traces_collected": map[string]interface{}{ - // "application_signals": map[string]interface{}{}, - // }, - // }, - // }, - // detector: eksdetector.TestEKSDetector, - // isEKSDataStore: eksdetector.TestIsEKSCacheEKS, - //}, - //"WithAppSignalsMetricsAndTracesEnabled": { - // input: map[string]interface{}{ - // "logs": map[string]interface{}{ - // "metrics_collected": map[string]interface{}{ - // "application_signals": map[string]interface{}{}, - // }, - // }, - // "traces": map[string]interface{}{ - // "traces_collected": map[string]interface{}{ - // "application_signals": map[string]interface{}{}, - // }, - // }, - // }, - // detector: eksdetector.TestEKSDetector, - // isEKSDataStore: eksdetector.TestIsEKSCacheEKS, - //}, - //"WithAppSignalsMultipleMetricsReceiversConfig": { - // input: map[string]interface{}{ - // "logs": map[string]interface{}{ - // "metrics_collected": map[string]interface{}{ - // "application_signals": map[string]interface{}{}, - // "cpu": map[string]interface{}{}, - // }, - // }, - // "traces": map[string]interface{}{ - // "traces_collected": map[string]interface{}{ - // "application_signals": map[string]interface{}{}, - // "otlp": map[string]interface{}{}, - // "otlp2": map[string]interface{}{}, - // }, - // }, - // }, - // detector: eksdetector.TestEKSDetector, - // isEKSDataStore: eksdetector.TestIsEKSCacheEKS, - //}, - //"WithAppSignalsFallbackMetricsEnabled": { - // input: map[string]interface{}{ - // "logs": map[string]interface{}{ - // "metrics_collected": map[string]interface{}{ - // "app_signals": map[string]interface{}{}, - // }, - // }, - // }, - // detector: eksdetector.TestEKSDetector, - // isEKSDataStore: eksdetector.TestIsEKSCacheEKS, - //}, - //"WithAppSignalsFallbackTracesEnabled": { - // input: map[string]interface{}{ - // "traces": map[string]interface{}{ - // "traces_collected": map[string]interface{}{ - // "app_signals": map[string]interface{}{}, - // }, - // }, - // }, - // detector: eksdetector.TestEKSDetector, - // isEKSDataStore: eksdetector.TestIsEKSCacheEKS, - //}, - //"WithAppSignalsFallbackMetricsAndTracesEnabled": { - // input: map[string]interface{}{ - // "logs": map[string]interface{}{ - // "metrics_collected": map[string]interface{}{ - // "app_signals": map[string]interface{}{}, - // }, - // }, - // "traces": map[string]interface{}{ - // "traces_collected": map[string]interface{}{ - // "app_signals": map[string]interface{}{}, - // }, - // }, - // }, - // detector: eksdetector.TestEKSDetector, - // isEKSDataStore: eksdetector.TestIsEKSCacheEKS, - //}, - //"WithAppSignalsFallbackMultipleMetricsReceiversConfig": { - // input: map[string]interface{}{ - // "logs": map[string]interface{}{ - // "metrics_collected": map[string]interface{}{ - // "app_signals": map[string]interface{}{}, - // "cpu": map[string]interface{}{}, - // }, - // }, - // "traces": map[string]interface{}{ - // "traces_collected": map[string]interface{}{ - // "app_signals": map[string]interface{}{}, - // "otlp": map[string]interface{}{}, - // "otlp2": map[string]interface{}{}, - // }, - // }, - // }, - // detector: eksdetector.TestEKSDetector, - // isEKSDataStore: eksdetector.TestIsEKSCacheEKS, - //}, - //"WithAMPDestinationConfig": { - // input: map[string]interface{}{ - // "metrics": map[string]interface{}{ - // "metrics_destinations": map[string]interface{}{ - // "amp": map[string]interface{}{ - // "workspace_id": "ws-12345", - // }, - // }, - // "metrics_collected": map[string]interface{}{ - // "cpu": map[string]interface{}{}, - // }, - // }, - // }, - //}, + "WithEmptyConfig": { + input: map[string]interface{}{}, + wantErrContains: "no valid pipelines", + }, + "WithoutReceivers": { + input: map[string]interface{}{ + "metrics": map[string]interface{}{}, + }, + wantErrContains: "no valid pipelines", + }, + "WithMinimalConfig": { + input: map[string]interface{}{ + "metrics": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "cpu": map[string]interface{}{}, + }, + }, + }, + }, + "WithAppSignalsMetricsEnabled": { + input: map[string]interface{}{ + "logs": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "application_signals": map[string]interface{}{}, + }, + }, + }, + detector: eksdetector.TestEKSDetector, + isEKSDataStore: eksdetector.TestIsEKSCacheEKS, + }, + "WithAppSignalsTracesEnabled": { + input: map[string]interface{}{ + "traces": map[string]interface{}{ + "traces_collected": map[string]interface{}{ + "application_signals": map[string]interface{}{}, + }, + }, + }, + detector: eksdetector.TestEKSDetector, + isEKSDataStore: eksdetector.TestIsEKSCacheEKS, + }, + "WithAppSignalsMetricsAndTracesEnabled": { + input: map[string]interface{}{ + "logs": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "application_signals": map[string]interface{}{}, + }, + }, + "traces": map[string]interface{}{ + "traces_collected": map[string]interface{}{ + "application_signals": map[string]interface{}{}, + }, + }, + }, + detector: eksdetector.TestEKSDetector, + isEKSDataStore: eksdetector.TestIsEKSCacheEKS, + }, + "WithAppSignalsMultipleMetricsReceiversConfig": { + input: map[string]interface{}{ + "logs": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "application_signals": map[string]interface{}{}, + "cpu": map[string]interface{}{}, + }, + }, + "traces": map[string]interface{}{ + "traces_collected": map[string]interface{}{ + "application_signals": map[string]interface{}{}, + "otlp": map[string]interface{}{}, + "otlp2": map[string]interface{}{}, + }, + }, + }, + detector: eksdetector.TestEKSDetector, + isEKSDataStore: eksdetector.TestIsEKSCacheEKS, + }, + "WithAppSignalsFallbackMetricsEnabled": { + input: map[string]interface{}{ + "logs": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "app_signals": map[string]interface{}{}, + }, + }, + }, + detector: eksdetector.TestEKSDetector, + isEKSDataStore: eksdetector.TestIsEKSCacheEKS, + }, + "WithAppSignalsFallbackTracesEnabled": { + input: map[string]interface{}{ + "traces": map[string]interface{}{ + "traces_collected": map[string]interface{}{ + "app_signals": map[string]interface{}{}, + }, + }, + }, + detector: eksdetector.TestEKSDetector, + isEKSDataStore: eksdetector.TestIsEKSCacheEKS, + }, + "WithAppSignalsFallbackMetricsAndTracesEnabled": { + input: map[string]interface{}{ + "logs": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "app_signals": map[string]interface{}{}, + }, + }, + "traces": map[string]interface{}{ + "traces_collected": map[string]interface{}{ + "app_signals": map[string]interface{}{}, + }, + }, + }, + detector: eksdetector.TestEKSDetector, + isEKSDataStore: eksdetector.TestIsEKSCacheEKS, + }, + "WithAppSignalsFallbackMultipleMetricsReceiversConfig": { + input: map[string]interface{}{ + "logs": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "app_signals": map[string]interface{}{}, + "cpu": map[string]interface{}{}, + }, + }, + "traces": map[string]interface{}{ + "traces_collected": map[string]interface{}{ + "app_signals": map[string]interface{}{}, + "otlp": map[string]interface{}{}, + "otlp2": map[string]interface{}{}, + }, + }, + }, + detector: eksdetector.TestEKSDetector, + isEKSDataStore: eksdetector.TestIsEKSCacheEKS, + }, + "WithAMPDestinationConfig": { + input: map[string]interface{}{ + "metrics": map[string]interface{}{ + "metrics_destinations": map[string]interface{}{ + "amp": map[string]interface{}{ + "workspace_id": "ws-12345", + }, + }, + "metrics_collected": map[string]interface{}{ + "cpu": map[string]interface{}{}, + }, + }, + }, + }, } for name, testCase := range testCases { t.Run(name, func(t *testing.T) { From ac1b5faf2bc5b87c42e55855aa26c2659115a24f Mon Sep 17 00:00:00 2001 From: Paramadon Date: Thu, 10 Oct 2024 01:50:22 -0400 Subject: [PATCH 05/19] fixed otel_test.go tests(missing region) --- translator/translate/otel/translate_otel_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/translator/translate/otel/translate_otel_test.go b/translator/translate/otel/translate_otel_test.go index f17e8fe1aa..2eb6f3c428 100644 --- a/translator/translate/otel/translate_otel_test.go +++ b/translator/translate/otel/translate_otel_test.go @@ -4,6 +4,8 @@ package otel import ( + "github.com/aws/amazon-cloudwatch-agent/tool/testutil" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/agent" "testing" "github.com/stretchr/testify/assert" @@ -18,6 +20,8 @@ import ( ) func TestTranslator(t *testing.T) { + agent.Global_Config.Region = "us-east-1" + testutil.SetPrometheusRemoteWriteTestingEnv(t) testCases := map[string]struct { input interface{} wantErrContains string From 03aa3fc6a1184165d279cab172f7171e2e84641f Mon Sep 17 00:00:00 2001 From: Paramadon Date: Thu, 10 Oct 2024 02:12:53 -0400 Subject: [PATCH 06/19] reverting jmx_eks_config --- .../sampleConfig/container_insights_jmx.conf | 26 + .../sampleConfig/container_insights_jmx.json | 37 +- .../sampleConfig/container_insights_jmx.yaml | 927 ++++++++++++++++++ .../sampleConfig/jmx_eks_config_linux.conf | 2 +- .../sampleConfig/jmx_eks_config_linux.json | 73 +- .../sampleConfig/jmx_eks_config_linux.yaml | 8 +- translator/tocwconfig/tocwconfig_test.go | 11 +- 7 files changed, 1031 insertions(+), 53 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/container_insights_jmx.conf b/translator/tocwconfig/sampleConfig/container_insights_jmx.conf index e69de29bb2..21716e9140 100644 --- a/translator/tocwconfig/sampleConfig/container_insights_jmx.conf +++ b/translator/tocwconfig/sampleConfig/container_insights_jmx.conf @@ -0,0 +1,26 @@ +[agent] + collection_jitter = "0s" + debug = true + flush_interval = "1s" + flush_jitter = "0s" + hostname = "" + interval = "60s" + logfile = "" + logtarget = "lumberjack" + metric_batch_size = 1000 + metric_buffer_limit = 10000 + omit_hostname = false + precision = "" + quiet = false + round_interval = false + +[outputs] + + [[outputs.cloudwatch]] + + [[outputs.cloudwatchlogs]] + force_flush_interval = "5s" + log_stream_name = "bcd0745edc19" + mode = "EC2" + region = "us-west-2" + region_type = "ACJ" diff --git a/translator/tocwconfig/sampleConfig/container_insights_jmx.json b/translator/tocwconfig/sampleConfig/container_insights_jmx.json index 29668bf013..eaaf509b86 100644 --- a/translator/tocwconfig/sampleConfig/container_insights_jmx.json +++ b/translator/tocwconfig/sampleConfig/container_insights_jmx.json @@ -1,16 +1,41 @@ { "agent": { - "region": "us-west-2" + "debug": true }, "logs": { "metrics_collected": { - "emf": { - }, + "emf": { }, "kubernetes": { - "cluster_name": "jvmCluster", - "namespace":"siprmp-jmx" + "cluster_name": "TestCluster", + "jmx_container_insights": true + } + } + }, + "metrics": { + "metrics_collected": { + "jmx": { + "jvm": { + "measurement": [ + "jvm.classes.loaded", + "jvm.memory.pool.used", + "jvm.operating.system.total.swap.space.size", + "jvm.operating.system.system.cpu.load", + "jvm.operating.system.process.cpu.load", + "jvm.operating.system.free.swap.space.size", + "jvm.operating.system.total.physical.memory.size", + "jvm.operating.system.free.physical.memory.size", + "jvm.operating.system.open.file.descriptor.count", + "jvm.operating.system.available.processors", + "jvm.threads.count", + "jvm.memory.heap.used", + "jvm.memory.nonheap.init", + "jvm.threads.daemon" + ] + } } }, - "force_flush_interval": 5 + "metrics_destinations": { + "cloudwatch": { } + } } } \ No newline at end of file diff --git a/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml b/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml index e69de29bb2..637376042e 100644 --- a/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml +++ b/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml @@ -0,0 +1,927 @@ +exporters: + awscloudwatch: + force_flush_interval: 1m0s + max_datums_per_call: 1000 + max_values_per_datum: 150 + middleware: agenthealth/metrics + namespace: CWAgent + region: us-west-2 + resource_to_telemetry_conversion: + enabled: true + awscloudwatchlogs/emf_logs: + certificate_file_path: "" + emf_only: true + endpoint: "" + imds_retries: 1 + local_mode: false + log_group_name: emf/logs/default + log_retention: 0 + log_stream_name: bcd0745edc19 + max_retries: 2 + middleware: agenthealth/logs + no_verify_ssl: false + num_workers: 8 + profile: "" + proxy_address: "" + raw_log: true + region: us-west-2 + request_timeout_seconds: 30 + resource_arn: "" + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 5m0s + max_interval: 30s + multiplier: 1.5 + randomization_factor: 0.5 + role_arn: "" + sending_queue: + enabled: true + num_consumers: 1 + queue_size: 1000 + awsemf/containerinsights: + certificate_file_path: "" + detailed_metrics: false + dimension_rollup_option: NoDimensionRollup + disable_metric_extraction: false + eks_fargate_container_insights_enabled: false + endpoint: "" + enhanced_container_insights: false + imds_retries: 1 + local_mode: false + log_group_name: /aws/containerinsights/{ClusterName}/performance + log_retention: 0 + log_stream_name: '{NodeName}' + max_retries: 2 + metric_declarations: + - dimensions: + - - ClusterName + - Namespace + - PodName + - - ClusterName + - - ClusterName + - Namespace + - Service + - - ClusterName + - Namespace + metric_name_selectors: + - pod_cpu_utilization + - pod_memory_utilization + - pod_network_rx_bytes + - pod_network_tx_bytes + - pod_cpu_utilization_over_pod_limit + - pod_memory_utilization_over_pod_limit + - dimensions: + - - ClusterName + - Namespace + - PodName + metric_name_selectors: + - pod_number_of_container_restarts + - dimensions: + - - ClusterName + - Namespace + - PodName + - - ClusterName + metric_name_selectors: + - pod_cpu_reserved_capacity + - pod_memory_reserved_capacity + - dimensions: + - - ClusterName + - InstanceId + - NodeName + - - ClusterName + metric_name_selectors: + - node_cpu_utilization + - node_memory_utilization + - node_network_total_bytes + - node_cpu_reserved_capacity + - node_memory_reserved_capacity + - node_number_of_running_pods + - node_number_of_running_containers + - dimensions: + - - ClusterName + metric_name_selectors: + - node_cpu_usage_total + - node_cpu_limit + - node_memory_working_set + - node_memory_limit + - dimensions: + - - ClusterName + - InstanceId + - NodeName + - - ClusterName + metric_name_selectors: + - node_filesystem_utilization + - dimensions: + - - ClusterName + - Namespace + - Service + - - ClusterName + metric_name_selectors: + - service_number_of_running_pods + - dimensions: + - - ClusterName + - Namespace + - - ClusterName + metric_name_selectors: + - namespace_number_of_running_pods + - dimensions: + - - ClusterName + metric_name_selectors: + - cluster_node_count + - cluster_failed_node_count + middleware: agenthealth/logs + namespace: ContainerInsights + no_verify_ssl: false + num_workers: 8 + output_destination: cloudwatch + parse_json_encoded_attr_values: + - Sources + - kubernetes + profile: "" + proxy_address: "" + region: us-west-2 + request_timeout_seconds: 30 + resource_arn: "" + resource_to_telemetry_conversion: + enabled: true + retain_initial_value_of_delta_metric: false + role_arn: "" + version: "0" + awsemf/containerinsightsjmx: + certificate_file_path: "" + detailed_metrics: false + dimension_rollup_option: NoDimensionRollup + disable_metric_extraction: false + eks_fargate_container_insights_enabled: false + endpoint: "" + enhanced_container_insights: false + imds_retries: 1 + local_mode: false + log_group_name: /aws/containerinsights/{ClusterName}/jmx + log_retention: 0 + log_stream_name: '{NodeName}' + max_retries: 2 + metric_declarations: + - dimensions: + - - ClusterName + - Namespace + metric_name_selectors: + - java_lang_operatingsystem_freeswapspacesize + - java_lang_operatingsystem_availableprocessors + - catalina_manager_rejectedsessions + - catalina_globalrequestprocessor_bytesreceived + - catalina_globalrequestprocessor_processingtime + - jvm_memory_pool_bytes_used + - java_lang_operatingsystem_systemcpuload + - java_lang_operatingsystem_totalphysicalmemorysize + - java_lang_operatingsystem_freephysicalmemorysize + - java_lang_operatingsystem_openfiledescriptorcount + - catalina_manager_activesessions + - java_lang_operatingsystem_totalswapspacesize + - java_lang_operatingsystem_processcpuload + - catalina_globalrequestprocessor_requestcount + - catalina_globalrequestprocessor_errorcount + - jvm_threads_daemon + - catalina_globalrequestprocessor_bytessent + - jvm_classes_loaded + - jvm_threads_current + - dimensions: + - - ClusterName + - Namespace + - area + metric_name_selectors: + - jvm_memory_bytes_used + - dimensions: + - - ClusterName + - Namespace + - pool + metric_name_selectors: + - jvm_memory_pool_bytes_used + middleware: agenthealth/logs + namespace: ContainerInsights/Prometheus + no_verify_ssl: false + num_workers: 8 + output_destination: cloudwatch + profile: "" + proxy_address: "" + region: us-west-2 + request_timeout_seconds: 30 + resource_arn: "" + resource_to_telemetry_conversion: + enabled: true + retain_initial_value_of_delta_metric: false + role_arn: "" + version: "0" + debug: + sampling_initial: 2 + sampling_thereafter: 500 + verbosity: Detailed +extensions: + agenthealth/logs: + is_usage_data_enabled: true + stats: + operations: + - PutLogEvents + usage_flags: + mode: EC2 + region_type: ACJ + agenthealth/metrics: + is_usage_data_enabled: true + stats: + operations: + - PutMetricData + usage_flags: + mode: EC2 + region_type: ACJ +processors: + batch/containerinsights: + metadata_cardinality_limit: 1000 + send_batch_max_size: 0 + send_batch_size: 8192 + timeout: 5s + batch/emf_logs: + metadata_cardinality_limit: 1000 + send_batch_max_size: 0 + send_batch_size: 8192 + timeout: 5s + cumulativetodelta/containerinsightsjmx: + exclude: + match_type: "" + include: + match_type: "" + initial_value: 2 + max_staleness: 0s + cumulativetodelta/jmx: + exclude: + match_type: "" + include: + match_type: "" + initial_value: 2 + max_staleness: 0s + filter/containerinsightsjmx: + error_mode: propagate + logs: {} + metrics: + include: + match_type: strict + metric_names: + - jvm.classes.loaded + - jvm.memory.heap.used + - jvm.memory.nonheap.used + - jvm.memory.pool.used + - jvm.operating.system.total.swap.space.size + - jvm.operating.system.system.cpu.load + - jvm.operating.system.process.cpu.load + - jvm.operating.system.free.swap.space.size + - jvm.operating.system.total.physical.memory.size + - jvm.operating.system.free.physical.memory.size + - jvm.operating.system.open.file.descriptor.count + - jvm.operating.system.available.processors + - jvm.threads.count + - jvm.threads.daemon + - tomcat.sessions + - tomcat.rejected_sessions + - tomcat.traffic.received + - tomcat.traffic.sent + - tomcat.request_count + - tomcat.errors + - tomcat.processing_time + spans: {} + traces: {} + filter/jmx: + error_mode: propagate + logs: {} + metrics: + include: + match_type: strict + metric_names: + - jvm.classes.loaded + - jvm.memory.pool.used + - jvm.operating.system.total.swap.space.size + - jvm.operating.system.system.cpu.load + - jvm.operating.system.process.cpu.load + - jvm.operating.system.free.swap.space.size + - jvm.operating.system.total.physical.memory.size + - jvm.operating.system.free.physical.memory.size + - jvm.operating.system.open.file.descriptor.count + - jvm.operating.system.available.processors + - jvm.threads.count + - jvm.memory.heap.used + - jvm.memory.nonheap.init + - jvm.threads.daemon + spans: {} + traces: {} + metricstransform/containerinsightsjmx: + transforms: + - action: update + aggregation_type: "" + include: jvm.classes.loaded + match_type: strict + new_name: jvm_classes_loaded + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.memory.heap.used + match_type: strict + new_name: jvm_memory_bytes_used + operations: + - action: add_label + aggregation_type: "" + experimental_scale: 0 + label: "" + label_value: "" + new_label: area + new_value: heap + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.memory.nonheap.used + match_type: strict + new_name: jvm_memory_bytes_used + operations: + - action: add_label + aggregation_type: "" + experimental_scale: 0 + label: "" + label_value: "" + new_label: area + new_value: nonheap + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.memory.pool.used + match_type: strict + new_name: jvm_memory_pool_bytes_used + operations: + - action: update_label + aggregation_type: "" + experimental_scale: 0 + label: name + label_value: "" + new_label: pool + new_value: "" + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.operating.system.total.swap.space.size + match_type: strict + new_name: java_lang_operatingsystem_totalswapspacesize + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.operating.system.system.cpu.load + match_type: strict + new_name: java_lang_operatingsystem_systemcpuload + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.operating.system.process.cpu.load + match_type: strict + new_name: java_lang_operatingsystem_processcpuload + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.operating.system.free.swap.space.size + match_type: strict + new_name: java_lang_operatingsystem_freeswapspacesize + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.operating.system.total.physical.memory.size + match_type: strict + new_name: java_lang_operatingsystem_totalphysicalmemorysize + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.operating.system.free.physical.memory.size + match_type: strict + new_name: java_lang_operatingsystem_freephysicalmemorysize + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.operating.system.open.file.descriptor.count + match_type: strict + new_name: java_lang_operatingsystem_openfiledescriptorcount + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.operating.system.available.processors + match_type: strict + new_name: java_lang_operatingsystem_availableprocessors + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.threads.count + match_type: strict + new_name: jvm_threads_current + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.threads.daemon + match_type: strict + new_name: jvm_threads_daemon + submatch_case: "" + - action: update + aggregation_type: "" + include: tomcat.sessions + match_type: strict + new_name: catalina_manager_activesessions + submatch_case: "" + - action: update + aggregation_type: "" + include: tomcat.rejected_sessions + match_type: strict + new_name: catalina_manager_rejectedsessions + submatch_case: "" + - action: update + aggregation_type: "" + include: tomcat.traffic.received + match_type: strict + new_name: catalina_globalrequestprocessor_bytesreceived + submatch_case: "" + - action: update + aggregation_type: "" + include: tomcat.traffic.sent + match_type: strict + new_name: catalina_globalrequestprocessor_bytessent + submatch_case: "" + - action: update + aggregation_type: "" + include: tomcat.request_count + match_type: strict + new_name: catalina_globalrequestprocessor_requestcount + submatch_case: "" + - action: update + aggregation_type: "" + include: tomcat.errors + match_type: strict + new_name: catalina_globalrequestprocessor_errorcount + submatch_case: "" + - action: update + aggregation_type: "" + include: tomcat.processing_time + match_type: strict + new_name: catalina_globalrequestprocessor_processingtime + submatch_case: "" + resource/containerinsightsjmx: + attributes: + - action: insert + converted_type: "" + from_attribute: k8s.namespace.name + from_context: "" + key: Namespace + pattern: "" + - action: upsert + converted_type: "" + from_attribute: "" + from_context: "" + key: ClusterName + pattern: "" + value: TestCluster + - action: insert + converted_type: "" + from_attribute: host.name + from_context: "" + key: NodeName + pattern: "" + resourcedetection/containerinsightsjmx: + aks: + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + k8s.cluster.name: + enabled: false + azure: + resource_attributes: + azure.resourcegroup.name: + enabled: true + azure.vm.name: + enabled: true + azure.vm.scaleset.name: + enabled: true + azure.vm.size: + enabled: true + cloud.account.id: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + host.id: + enabled: true + host.name: + enabled: true + tags: [] + compression: "" + consul: + address: "" + datacenter: "" + namespace: "" + resource_attributes: + cloud.region: + enabled: true + host.id: + enabled: true + host.name: + enabled: true + token_file: "" + detectors: + - env + - eks + - system + disable_keep_alives: false + docker: + resource_attributes: + host.name: + enabled: true + os.type: + enabled: true + ec2: + resource_attributes: + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + host.id: + enabled: true + host.image.id: + enabled: true + host.name: + enabled: true + host.type: + enabled: true + tags: [] + ecs: + resource_attributes: + aws.ecs.cluster.arn: + enabled: true + aws.ecs.launchtype: + enabled: true + aws.ecs.task.arn: + enabled: true + aws.ecs.task.family: + enabled: true + aws.ecs.task.id: + enabled: true + aws.ecs.task.revision: + enabled: true + aws.log.group.arns: + enabled: true + aws.log.group.names: + enabled: true + aws.log.stream.arns: + enabled: true + aws.log.stream.names: + enabled: true + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + eks: + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + k8s.cluster.name: + enabled: true + elasticbeanstalk: + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + deployment.environment: + enabled: true + service.instance.id: + enabled: true + service.version: + enabled: true + endpoint: "" + gcp: + resource_attributes: + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + faas.id: + enabled: true + faas.instance: + enabled: true + faas.name: + enabled: true + faas.version: + enabled: true + gcp.cloud_run.job.execution: + enabled: true + gcp.cloud_run.job.task_index: + enabled: true + gcp.gce.instance.hostname: + enabled: false + gcp.gce.instance.name: + enabled: false + host.id: + enabled: true + host.name: + enabled: true + host.type: + enabled: true + k8s.cluster.name: + enabled: true + heroku: + resource_attributes: + cloud.provider: + enabled: true + heroku.app.id: + enabled: true + heroku.dyno.id: + enabled: true + heroku.release.commit: + enabled: true + heroku.release.creation_timestamp: + enabled: true + service.instance.id: + enabled: true + service.name: + enabled: true + service.version: + enabled: true + http2_ping_timeout: 0s + http2_read_idle_timeout: 0s + idle_conn_timeout: 1m30s + k8snode: + auth_type: serviceAccount + context: "" + kube_config_path: "" + node_from_env_var: "" + resource_attributes: + k8s.node.name: + enabled: true + k8s.node.uid: + enabled: true + lambda: + resource_attributes: + aws.log.group.names: + enabled: true + aws.log.stream.names: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + faas.instance: + enabled: true + faas.max_memory: + enabled: true + faas.name: + enabled: true + faas.version: + enabled: true + max_idle_conns: 100 + openshift: + address: "" + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + k8s.cluster.name: + enabled: true + tls: + ca_file: "" + cert_file: "" + include_system_ca_certs_pool: false + insecure: false + insecure_skip_verify: false + key_file: "" + max_version: "" + min_version: "" + reload_interval: 0s + server_name_override: "" + token: "" + override: false + proxy_url: "" + read_buffer_size: 0 + system: + hostname_sources: + - os + resource_attributes: + host.arch: + enabled: false + host.cpu.cache.l2.size: + enabled: false + host.cpu.family: + enabled: false + host.cpu.model.id: + enabled: false + host.cpu.model.name: + enabled: false + host.cpu.stepping: + enabled: false + host.cpu.vendor.id: + enabled: false + host.id: + enabled: false + host.ip: + enabled: false + host.mac: + enabled: false + host.name: + enabled: true + os.description: + enabled: false + os.type: + enabled: true + timeout: 15s + tls: + ca_file: "" + cert_file: "" + include_system_ca_certs_pool: false + insecure: false + insecure_skip_verify: false + key_file: "" + max_version: "" + min_version: "" + reload_interval: 0s + server_name_override: "" + write_buffer_size: 0 + transform/containerinsightsjmx: + error_mode: propagate + flatten_data: false + log_statements: [] + metric_statements: + - context: resource + statements: + - keep_keys(attributes, ["ClusterName", "Namespace", "NodeName"]) + - context: metric + statements: + - set(unit, "Bytes") where name == "jvm.memory.heap.used" + - set(unit, "Bytes") where name == "jvm.memory.nonheap.used" + - set(unit, "Bytes") where name == "jvm.memory.pool.used" + - set(unit, "Bytes") where name == "tomcat.traffic.received" + - set(unit, "Bytes") where name == "tomcat.traffic.sent" + - set(unit, "Bytes") where name == "jvm.operating.system.total.swap.space.size" + - set(unit, "Bytes") where name == "jvm.operating.system.free.swap.space.size" + - set(unit, "Bytes") where name == "jvm.operating.system.total.physical.memory.size" + - set(unit, "Bytes") where name == "jvm.operating.system.free.physical.memory.size" + - set(unit, "Count") where name == "tomcat.sessions" + - set(unit, "Count") where name == "tomcat.rejected_sessions" + - set(unit, "Count") where name == "jvm.threads.count" + - set(unit, "Count") where name == "jvm.threads.daemon" + - set(unit, "Count") where name == "jvm.operating.system.open.file.descriptor.count" + - set(unit, "Count") where name == "jvm.operating.system.available.processors" + - set(unit, "Count") where name == "tomcat.request_count" + - set(unit, "Count") where name == "tomcat.errors" + - set(unit, "Count") where name == "jvm.classes.loaded" + - set(unit, "Count") where name == "jvm.operating.system.system.cpu.load" + - set(unit, "Count") where name == "jvm.operating.system.process.cpu.load" + - set(unit, "Milliseconds") where name == "tomcat.processing_time" + trace_statements: [] +receivers: + awscontainerinsightreceiver: + accelerated_compute_metrics: true + add_container_name_metric_label: false + add_full_pod_name_metric_label: false + add_service_as_attribute: true + certificate_file_path: "" + cluster_name: TestCluster + collection_interval: 1m0s + container_orchestrator: eks + enable_control_plane_metrics: false + endpoint: "" + host_ip: "" + host_name: "" + imds_retries: 1 + kube_config_path: "" + leader_lock_name: cwagent-clusterleader + leader_lock_using_config_map_only: true + local_mode: false + max_retries: 0 + no_verify_ssl: false + num_workers: 0 + prefer_full_pod_name: false + profile: "" + proxy_address: "" + region: us-west-2 + request_timeout_seconds: 0 + resource_arn: "" + role_arn: "" + otlp/jmx: + protocols: + http: + endpoint: 0.0.0.0:4314 + include_metadata: false + logs_url_path: /v1/logs + max_request_body_size: 0 + metrics_url_path: /v1/metrics + traces_url_path: /v1/traces + tcplog/emf_logs: + encoding: utf-8 + id: tcp_input + listen_address: 0.0.0.0:25888 + operators: [] + retry_on_failure: + enabled: false + initial_interval: 0s + max_elapsed_time: 0s + max_interval: 0s + type: tcp_input + udplog/emf_logs: + encoding: utf-8 + id: udp_input + listen_address: 0.0.0.0:25888 + multiline: + line_end_pattern: .^ + line_start_pattern: "" + omit_pattern: false + operators: [] + retry_on_failure: + enabled: false + initial_interval: 0s + max_elapsed_time: 0s + max_interval: 0s + type: udp_input +service: + extensions: + - agenthealth/logs + - agenthealth/metrics + pipelines: + logs/emf_logs: + exporters: + - awscloudwatchlogs/emf_logs + processors: + - batch/emf_logs + receivers: + - tcplog/emf_logs + - udplog/emf_logs + metrics/containerinsights: + exporters: + - awsemf/containerinsights + processors: + - batch/containerinsights + receivers: + - awscontainerinsightreceiver + metrics/containerinsightsjmx: + exporters: + - debug + - awsemf/containerinsightsjmx + processors: + - filter/containerinsightsjmx + - resourcedetection/containerinsightsjmx + - resource/containerinsightsjmx + - transform/containerinsightsjmx + - metricstransform/containerinsightsjmx + - cumulativetodelta/containerinsightsjmx + receivers: + - otlp/jmx + metrics/jmx/cloudwatch: + exporters: + - awscloudwatch + processors: + - filter/jmx + - cumulativetodelta/jmx + receivers: + - otlp/jmx + telemetry: + logs: + development: false + disable_caller: false + disable_stacktrace: false + encoding: console + level: debug + sampling: + enabled: true + initial: 2 + thereafter: 500 + tick: 10s + metrics: + address: "" + level: None + traces: {} diff --git a/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.conf b/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.conf index 9c37f8b52d..8263cf1b4d 100644 --- a/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.conf +++ b/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.conf @@ -16,4 +16,4 @@ [outputs] - [[outputs.cloudwatch]] + [[outputs.cloudwatch]] \ No newline at end of file diff --git a/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.json b/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.json index 67f5952d66..06c225e7a5 100644 --- a/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.json +++ b/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.json @@ -1,42 +1,43 @@ { - "agent": { - "debug": true - }, - "logs": { - "metrics_collected": { - "emf": { }, - "kubernetes": { - "cluster_name": "TestCluster", - "jmx_container_insights": true - } - } - }, "metrics": { - "metrics_collected": { - "jmx": { - "jvm": { - "measurement": [ - "jvm.classes.loaded", - "jvm.memory.pool.used", - "jvm.operating.system.total.swap.space.size", - "jvm.operating.system.system.cpu.load", - "jvm.operating.system.process.cpu.load", - "jvm.operating.system.free.swap.space.size", - "jvm.operating.system.total.physical.memory.size", - "jvm.operating.system.free.physical.memory.size", - "jvm.operating.system.open.file.descriptor.count", - "jvm.operating.system.available.processors", - "jvm.threads.count", - "jvm.memory.heap.used", - "jvm.memory.nonheap.init", - "jvm.threads.daemon" - ] - } - } - }, "metrics_destinations": { - "cloudwatch": { } + "amp": { + "workspace_id": "ws-12345" + }, + "cloudwatch": {} }, - "namespace": "jmx-test-141" + "metrics_collected": { + "jmx": [ + { + "jvm": { + "measurement": [ + "jvm.memory.heap.init", + { + "name": "jvm.memory.heap.used", + "rename": "JVM_MEM_HEAP_USED", + "unit": "unit" + }, + "jvm.memory.nonheap.init" + ] + }, + "append_dimensions": { + "k1": "v1" + } + }, + { + "kafka-consumer": { + "measurement": [ + { + "name": "kafka.consumer.fetch-rate", + "rename": "kafka.fetch-rate" + } + ] + }, + "append_dimensions": { + "k2": "v2" + } + } + ] + } } } \ No newline at end of file diff --git a/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.yaml b/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.yaml index 66759372ba..4c514f4901 100644 --- a/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.yaml @@ -131,8 +131,8 @@ processors: metric_statements: - context: metric statements: - - set(unit, "unit") where name == "jvm.memory.heap.used" - - set(name, "JVM_MEM_HEAP_USED") where name == "jvm.memory.heap.used" + - set(unit, "unit") where name == "jvm.memory.heap.used" + - set(name, "JVM_MEM_HEAP_USED") where name == "jvm.memory.heap.used" trace_statements: [] transform/jmx/1: error_mode: propagate @@ -141,7 +141,7 @@ processors: metric_statements: - context: metric statements: - - set(name, "kafka.fetch-rate") where name == "kafka.consumer.fetch-rate" + - set(name, "kafka.fetch-rate") where name == "kafka.consumer.fetch-rate" trace_statements: [] receivers: otlp/jmx: @@ -213,4 +213,4 @@ service: metrics: address: "" level: None - traces: {} + traces: {} \ No newline at end of file diff --git a/translator/tocwconfig/tocwconfig_test.go b/translator/tocwconfig/tocwconfig_test.go index 5a82921043..a147399262 100644 --- a/translator/tocwconfig/tocwconfig_test.go +++ b/translator/tocwconfig/tocwconfig_test.go @@ -9,6 +9,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/aws/amazon-cloudwatch-agent/tool/testutil" "io/fs" "os" "path/filepath" @@ -81,13 +82,11 @@ func TestGenericAppSignalsConfig(t *testing.T) { checkTranslation(t, "base_appsignals_config", "linux", expectedEnvVars, "") checkTranslation(t, "base_appsignals_config", "windows", expectedEnvVars, "") } -func TestContØainerInsightsJMX(t *testing.T) { - +func TestContainerInsightsJMX(t *testing.T) { resetContext(t) - context.CurrentContext().SetRunInContainer(false) - context.CurrentContext().SetMode(config.ModeOnPremise) - t.Setenv(config.HOST_NAME, "host_name_from_env") - t.Setenv(config.HOST_IP, "127.0.0.1") + testutil.SetPrometheusRemoteWriteTestingEnv(t) + context.CurrentContext().SetMode(config.ModeEC2) + context.CurrentContext().SetRunInContainer(true) expectedEnvVars := map[string]string{"CWAGENT_LOG_LEVEL": "DEBUG"} checkTranslation(t, "container_insights_jmx", "linux", expectedEnvVars, "") } From 2feb6e61ef489ee6f8e82dc8ded6f81b66bb9012 Mon Sep 17 00:00:00 2001 From: Paramadon Date: Thu, 10 Oct 2024 09:47:49 -0400 Subject: [PATCH 07/19] fixing lint --- translator/tocwconfig/tocwconfig_test.go | 2 +- .../pipeline/containerinsightsjmx/translator.go | 10 +++++----- .../processor/filterprocessor/translator_test.go | 2 +- .../processor/jmxtransformprocessor/translate.go | 2 +- .../jmxtransformprocessor/translator_test.go | 14 +++++++++----- .../metricstransformprocessor/translator_test.go | 4 ++-- .../processor/resourcedetection/translator_test.go | 4 ++-- .../processor/resourceprocessor/translator_test.go | 4 ++-- translator/translate/otel/translate_otel.go | 2 +- translator/translate/otel/translate_otel_test.go | 4 ++-- 10 files changed, 26 insertions(+), 22 deletions(-) diff --git a/translator/tocwconfig/tocwconfig_test.go b/translator/tocwconfig/tocwconfig_test.go index a147399262..7c101d7fc7 100644 --- a/translator/tocwconfig/tocwconfig_test.go +++ b/translator/tocwconfig/tocwconfig_test.go @@ -9,7 +9,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/aws/amazon-cloudwatch-agent/tool/testutil" "io/fs" "os" "path/filepath" @@ -28,6 +27,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/cfg/commonconfig" "github.com/aws/amazon-cloudwatch-agent/cfg/envconfig" "github.com/aws/amazon-cloudwatch-agent/internal/retryer" + "github.com/aws/amazon-cloudwatch-agent/tool/testutil" "github.com/aws/amazon-cloudwatch-agent/translator" "github.com/aws/amazon-cloudwatch-agent/translator/cmdutil" "github.com/aws/amazon-cloudwatch-agent/translator/config" diff --git a/translator/translate/otel/pipeline/containerinsightsjmx/translator.go b/translator/translate/otel/pipeline/containerinsightsjmx/translator.go index fba46d2e72..576d942a61 100644 --- a/translator/translate/otel/pipeline/containerinsightsjmx/translator.go +++ b/translator/translate/otel/pipeline/containerinsightsjmx/translator.go @@ -5,6 +5,11 @@ package containerinsightsjmx import ( "fmt" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/awsemf" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/debug" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/cumulativetodeltaprocessor" @@ -14,11 +19,6 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/resourcedetection" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/resourceprocessor" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/otlp" - - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/confmap" - - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) var ( diff --git a/translator/translate/otel/processor/filterprocessor/translator_test.go b/translator/translate/otel/processor/filterprocessor/translator_test.go index a2820290f4..55e78558d1 100644 --- a/translator/translate/otel/processor/filterprocessor/translator_test.go +++ b/translator/translate/otel/processor/filterprocessor/translator_test.go @@ -4,11 +4,11 @@ package filterprocessor import ( - "github.com/stretchr/testify/assert" "path/filepath" "testing" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" diff --git a/translator/translate/otel/processor/jmxtransformprocessor/translate.go b/translator/translate/otel/processor/jmxtransformprocessor/translate.go index cf7c386aed..a90651ba16 100644 --- a/translator/translate/otel/processor/jmxtransformprocessor/translate.go +++ b/translator/translate/otel/processor/jmxtransformprocessor/translate.go @@ -5,8 +5,8 @@ package jmxtransformprocessor import ( _ "embed" - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" "go.opentelemetry.io/collector/processor" diff --git a/translator/translate/otel/processor/jmxtransformprocessor/translator_test.go b/translator/translate/otel/processor/jmxtransformprocessor/translator_test.go index 12376adc50..db9a2b3ae5 100644 --- a/translator/translate/otel/processor/jmxtransformprocessor/translator_test.go +++ b/translator/translate/otel/processor/jmxtransformprocessor/translator_test.go @@ -1,17 +1,21 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + package jmxtransformprocessor import ( _ "embed" - "github.com/aws/amazon-cloudwatch-agent/internal/util/testutil" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor" - "github.com/stretchr/testify/assert" - "go.opentelemetry.io/collector/component" "path/filepath" "testing" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" + + "github.com/aws/amazon-cloudwatch-agent/internal/util/testutil" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) func TestTranslator(t *testing.T) { diff --git a/translator/translate/otel/processor/metricstransformprocessor/translator_test.go b/translator/translate/otel/processor/metricstransformprocessor/translator_test.go index 3cc7f72019..4a395687d9 100644 --- a/translator/translate/otel/processor/metricstransformprocessor/translator_test.go +++ b/translator/translate/otel/processor/metricstransformprocessor/translator_test.go @@ -5,16 +5,16 @@ package metricstransformprocessor import ( "fmt" - "github.com/aws/amazon-cloudwatch-agent/internal/util/testutil" - "github.com/stretchr/testify/assert" "path/filepath" "testing" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" + "github.com/aws/amazon-cloudwatch-agent/internal/util/testutil" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) diff --git a/translator/translate/otel/processor/resourcedetection/translator_test.go b/translator/translate/otel/processor/resourcedetection/translator_test.go index b9cb046e0e..23fa4599f5 100644 --- a/translator/translate/otel/processor/resourcedetection/translator_test.go +++ b/translator/translate/otel/processor/resourcedetection/translator_test.go @@ -4,8 +4,6 @@ package resourcedetection import ( - "github.com/aws/amazon-cloudwatch-agent/internal/util/testutil" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "path/filepath" "testing" @@ -15,8 +13,10 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" + "github.com/aws/amazon-cloudwatch-agent/internal/util/testutil" translatorconfig "github.com/aws/amazon-cloudwatch-agent/translator/config" "github.com/aws/amazon-cloudwatch-agent/translator/context" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "github.com/aws/amazon-cloudwatch-agent/translator/util/ecsutil" ) diff --git a/translator/translate/otel/processor/resourceprocessor/translator_test.go b/translator/translate/otel/processor/resourceprocessor/translator_test.go index a60ea8eb42..d80f54286c 100644 --- a/translator/translate/otel/processor/resourceprocessor/translator_test.go +++ b/translator/translate/otel/processor/resourceprocessor/translator_test.go @@ -4,16 +4,16 @@ package resourceprocessor import ( - "github.com/aws/amazon-cloudwatch-agent/internal/util/testutil" - "github.com/stretchr/testify/require" "path/filepath" "testing" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" + "github.com/aws/amazon-cloudwatch-agent/internal/util/testutil" "github.com/aws/amazon-cloudwatch-agent/translator/context" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) diff --git a/translator/translate/otel/translate_otel.go b/translator/translate/otel/translate_otel.go index 18e65400cf..aa38c00ef7 100644 --- a/translator/translate/otel/translate_otel.go +++ b/translator/translate/otel/translate_otel.go @@ -6,7 +6,6 @@ package otel import ( "errors" "fmt" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/containerinsightsjmx" "log" "time" @@ -24,6 +23,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/applicationsignals" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/containerinsights" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/containerinsightsjmx" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/emf_logs" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/host" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/jmx" diff --git a/translator/translate/otel/translate_otel_test.go b/translator/translate/otel/translate_otel_test.go index 2eb6f3c428..565c781d2d 100644 --- a/translator/translate/otel/translate_otel_test.go +++ b/translator/translate/otel/translate_otel_test.go @@ -4,8 +4,6 @@ package otel import ( - "github.com/aws/amazon-cloudwatch-agent/tool/testutil" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/agent" "testing" "github.com/stretchr/testify/assert" @@ -13,8 +11,10 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" + "github.com/aws/amazon-cloudwatch-agent/tool/testutil" "github.com/aws/amazon-cloudwatch-agent/translator" _ "github.com/aws/amazon-cloudwatch-agent/translator/registerrules" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/agent" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "github.com/aws/amazon-cloudwatch-agent/translator/util/eksdetector" ) From fae0e7bf604c9dfaacbb2ea348971ef0524d8a15 Mon Sep 17 00:00:00 2001 From: Paramadon Date: Thu, 10 Oct 2024 09:58:59 -0400 Subject: [PATCH 08/19] fixing ContainerInsightsJmxConfig.yaml names --- Makefile | 4 ++-- extension/agenthealth/config_test.go | 2 +- processor/rollupprocessor/config_test.go | 2 +- .../translate/otel/exporter/awsxray/translator_test.go | 2 +- .../otel/exporter/prometheusremotewrite/translator_test.go | 2 +- .../otel/processor/metricsdecorator/translator_unix_test.go | 4 ++-- .../processor/metricsdecorator/translator_windows_test.go | 4 ++-- .../translate/otel/receiver/awsxray/translator_test.go | 2 +- translator/translate/otel/receiver/jmx/translator_test.go | 2 +- translator/translate/otel/receiver/otlp/translator_test.go | 6 +++--- 10 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 4f6451939b..21b2fe7d88 100644 --- a/Makefile +++ b/Makefile @@ -25,9 +25,9 @@ WIN_BUILD = GOOS=windows GOARCH=amd64 go build -trimpath -buildmode=${CWAGENT_BU DARWIN_BUILD_AMD64 = CGO_ENABLED=1 GO111MODULE=on GOOS=darwin GOARCH=amd64 go build -trimpath -ldflags="${LDFLAGS}" -o $(BUILD_SPACE)/bin/darwin_amd64 DARWIN_BUILD_ARM64 = CGO_ENABLED=1 GO111MODULE=on GOOS=darwin GOARCH=arm64 go build -trimpath -ldflags="${LDFLAGS}" -o $(BUILD_SPACE)/bin/darwin_arm64 -IMAGE_REGISTRY = 506463145083.dkr.ecr.us-west-2.amazonaws.com +IMAGE_REGISTRY = amazon IMAGE_REPO = cloudwatch-agent -IMAGE_TAG = latest +IMAGE_TAG = $(VERSION) IMAGE = $(IMAGE_REGISTRY)/$(IMAGE_REPO):$(IMAGE_TAG) DOCKER_BUILD_FROM_SOURCE = docker build -t $(IMAGE) -f ./amazon-cloudwatch-container-insights/cloudwatch-agent-dockerfile/source/Dockerfile DOCKER_WINDOWS_BUILD_FROM_SOURCE = docker build -t $(IMAGE) -f ./amazon-cloudwatch-container-insights/cloudwatch-agent-dockerfile/source/Dockerfile.Windows diff --git a/extension/agenthealth/config_test.go b/extension/agenthealth/config_test.go index 01666f534c..ee0ef301f6 100644 --- a/extension/agenthealth/config_test.go +++ b/extension/agenthealth/config_test.go @@ -34,7 +34,7 @@ func TestLoadConfig(t *testing.T) { }, } for _, testCase := range testCases { - conf, err := confmaptest.LoadConf(filepath.Join("testdata", "ContainerInsightsJmxConfig.yaml")) + conf, err := confmaptest.LoadConf(filepath.Join("testdata", "config.yaml")) require.NoError(t, err) cfg := NewFactory().CreateDefaultConfig() sub, err := conf.Sub(testCase.id.String()) diff --git a/processor/rollupprocessor/config_test.go b/processor/rollupprocessor/config_test.go index 136b2a1a3a..f1965a425c 100644 --- a/processor/rollupprocessor/config_test.go +++ b/processor/rollupprocessor/config_test.go @@ -40,7 +40,7 @@ func TestLoadConfig(t *testing.T) { }, } for _, testCase := range testCases { - conf, err := confmaptest.LoadConf(filepath.Join("testdata", "ContainerInsightsJmxConfig.yaml")) + conf, err := confmaptest.LoadConf(filepath.Join("testdata", "config.yaml")) require.NoError(t, err) cfg := NewFactory().CreateDefaultConfig() sub, err := conf.Sub(testCase.id.String()) diff --git a/translator/translate/otel/exporter/awsxray/translator_test.go b/translator/translate/otel/exporter/awsxray/translator_test.go index b0221fd1b2..c3609d6c26 100644 --- a/translator/translate/otel/exporter/awsxray/translator_test.go +++ b/translator/translate/otel/exporter/awsxray/translator_test.go @@ -59,7 +59,7 @@ func TestTranslator(t *testing.T) { }, "WithCompleteConfig": { input: testutil.GetJson(t, filepath.Join("testdata", "config.json")), - want: testutil.GetConf(t, filepath.Join("testdata", "ContainerInsightsJmxConfig.yaml")), + want: testutil.GetConf(t, filepath.Join("testdata", "config.yaml")), mode: config.ModeOnPrem, }, "WithAppSignalsEnabledEKS": { diff --git a/translator/translate/otel/exporter/prometheusremotewrite/translator_test.go b/translator/translate/otel/exporter/prometheusremotewrite/translator_test.go index 635db8ebc4..a5b9a146d1 100644 --- a/translator/translate/otel/exporter/prometheusremotewrite/translator_test.go +++ b/translator/translate/otel/exporter/prometheusremotewrite/translator_test.go @@ -37,7 +37,7 @@ func TestTranslator(t *testing.T) { }, "WithAMPDestination": { input: testutil.GetJson(t, filepath.Join("testdata", "config.json")), - want: testutil.GetConf(t, filepath.Join("testdata", "ContainerInsightsJmxConfig.yaml")), + want: testutil.GetConf(t, filepath.Join("testdata", "config.yaml")), }, } for name, testCase := range testCases { diff --git a/translator/translate/otel/processor/metricsdecorator/translator_unix_test.go b/translator/translate/otel/processor/metricsdecorator/translator_unix_test.go index 6b05733488..d00e60d538 100644 --- a/translator/translate/otel/processor/metricsdecorator/translator_unix_test.go +++ b/translator/translate/otel/processor/metricsdecorator/translator_unix_test.go @@ -30,7 +30,7 @@ func TestTranslate(t *testing.T) { translatorcontext.CurrentContext().SetOs(translatorconfig.OS_TYPE_LINUX) transl := NewTranslator().(*translator) expectedCfg := transl.factory.CreateDefaultConfig().(*transformprocessor.Config) - c := testutil.GetConf(t, filepath.Join("testdata", "unix", "ContainerInsightsJmxConfig.yaml")) + c := testutil.GetConf(t, filepath.Join("testdata", "unix", "config.yaml")) require.NoError(t, c.Unmarshal(&expectedCfg)) conf := confmap.NewFromStringMap(testutil.GetJson(t, filepath.Join("testdata", "unix", "config.json"))) @@ -53,7 +53,7 @@ func TestMetricDecoration(t *testing.T) { translatorcontext.CurrentContext().SetOs(translatorconfig.OS_TYPE_LINUX) transl := NewTranslator().(*translator) cfg := transl.factory.CreateDefaultConfig().(*transformprocessor.Config) - conf := testutil.GetConf(t, filepath.Join("testdata", "unix", "ContainerInsightsJmxConfig.yaml")) + conf := testutil.GetConf(t, filepath.Join("testdata", "unix", "config.yaml")) require.NoError(t, conf.Unmarshal(&cfg)) sink := new(consumertest.MetricsSink) diff --git a/translator/translate/otel/processor/metricsdecorator/translator_windows_test.go b/translator/translate/otel/processor/metricsdecorator/translator_windows_test.go index b9f620ba10..4b0a8c9593 100644 --- a/translator/translate/otel/processor/metricsdecorator/translator_windows_test.go +++ b/translator/translate/otel/processor/metricsdecorator/translator_windows_test.go @@ -31,7 +31,7 @@ func TestTranslate(t *testing.T) { translatorcontext.CurrentContext().SetOs(translatorconfig.OS_TYPE_WINDOWS) transl := NewTranslator().(*translator) expectedCfg := transl.factory.CreateDefaultConfig().(*transformprocessor.Config) - c := testutil.GetConf(t, filepath.Join("testdata", "windows", "ContainerInsightsJmxConfig.yaml")) + c := testutil.GetConf(t, filepath.Join("testdata", "windows", "config.yaml")) err := c.Unmarshal(&expectedCfg) assert.NoError(t, err) @@ -54,7 +54,7 @@ func TestMetricDecoration(t *testing.T) { translatorcontext.CurrentContext().SetOs(translatorconfig.OS_TYPE_WINDOWS) transl := NewTranslator().(*translator) cfg := transl.factory.CreateDefaultConfig().(*transformprocessor.Config) - conf := testutil.GetConf(t, filepath.Join("testdata", "windows", "ContainerInsightsJmxConfig.yaml")) + conf := testutil.GetConf(t, filepath.Join("testdata", "windows", "config.yaml")) err := conf.Unmarshal(&cfg) assert.NoError(t, err) sink := new(consumertest.MetricsSink) diff --git a/translator/translate/otel/receiver/awsxray/translator_test.go b/translator/translate/otel/receiver/awsxray/translator_test.go index e1d5650b79..5a385fe942 100644 --- a/translator/translate/otel/receiver/awsxray/translator_test.go +++ b/translator/translate/otel/receiver/awsxray/translator_test.go @@ -49,7 +49,7 @@ func TestTranslator(t *testing.T) { }, "WithCompleteConfig": { input: testutil.GetJson(t, filepath.Join("testdata", "config.json")), - want: testutil.GetConf(t, filepath.Join("testdata", "ContainerInsightsJmxConfig.yaml")), + want: testutil.GetConf(t, filepath.Join("testdata", "config.yaml")), }, } factory := awsxrayreceiver.NewFactory() diff --git a/translator/translate/otel/receiver/jmx/translator_test.go b/translator/translate/otel/receiver/jmx/translator_test.go index 90f098018a..5db211ad03 100644 --- a/translator/translate/otel/receiver/jmx/translator_test.go +++ b/translator/translate/otel/receiver/jmx/translator_test.go @@ -100,7 +100,7 @@ func TestTranslator(t *testing.T) { input: testutil.GetJson(t, filepath.Join("testdata", "config.json")), index: -1, wantID: "jmx", - want: testutil.GetConf(t, filepath.Join("testdata", "ContainerInsightsJmxConfig.yaml")), + want: testutil.GetConf(t, filepath.Join("testdata", "config.yaml")), }, } for name, testCase := range testCases { diff --git a/translator/translate/otel/receiver/otlp/translator_test.go b/translator/translate/otel/receiver/otlp/translator_test.go index 824c5b523a..e45435f018 100644 --- a/translator/translate/otel/receiver/otlp/translator_test.go +++ b/translator/translate/otel/receiver/otlp/translator_test.go @@ -73,7 +73,7 @@ func TestTracesTranslator(t *testing.T) { }, "WithCompleteConfig": { input: testutil.GetJson(t, filepath.Join("testdata", "traces", "config.json")), - want: testutil.GetConf(t, filepath.Join("testdata", "traces", "ContainerInsightsJmxConfig.yaml")), + want: testutil.GetConf(t, filepath.Join("testdata", "traces", "config.yaml")), }, } factory := otlpreceiver.NewFactory() @@ -166,7 +166,7 @@ func TestMetricsTranslator(t *testing.T) { "WithCompleteConfig": { input: testutil.GetJson(t, filepath.Join("testdata", "metrics", "config.json")), index: -1, - want: testutil.GetConf(t, filepath.Join("testdata", "metrics", "ContainerInsightsJmxConfig.yaml")), + want: testutil.GetConf(t, filepath.Join("testdata", "metrics", "config.yaml")), }, } factory := otlpreceiver.NewFactory() @@ -263,7 +263,7 @@ func TestMetricsEmfTranslator(t *testing.T) { "WithCompleteConfig": { input: testutil.GetJson(t, filepath.Join("testdata", "metrics_emf", "config.json")), index: -1, - want: testutil.GetConf(t, filepath.Join("testdata", "metrics_emf", "ContainerInsightsJmxConfig.yaml")), + want: testutil.GetConf(t, filepath.Join("testdata", "metrics_emf", "config.yaml")), }, } factory := otlpreceiver.NewFactory() From d214af3fa9e4f50d6f8930098a34ad76def4c4fb Mon Sep 17 00:00:00 2001 From: Paramadon Date: Thu, 10 Oct 2024 10:12:13 -0400 Subject: [PATCH 09/19] fixed unit tests --- .../pipeline/containerinsightsjmx/translator_test.go | 10 ++++++---- .../processor/filterprocessor/testdata/config.yaml | 10 ---------- .../otel/processor/filterprocessor/translator_test.go | 6 ------ .../processor/jmxtransformprocessor/translator_test.go | 6 ------ 4 files changed, 6 insertions(+), 26 deletions(-) delete mode 100644 translator/translate/otel/processor/filterprocessor/testdata/config.yaml diff --git a/translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go b/translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go index 8e6d5027c0..02a1e11c53 100644 --- a/translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go +++ b/translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go @@ -33,21 +33,23 @@ func TestTranslator(t *testing.T) { }{ "WithoutECSOrKubernetesKey": { input: map[string]interface{}{}, - wantErr: &common.MissingKeyError{ID: cit.ID(), JsonKey: fmt.Sprint(eksKey)}, + wantErr: &common.MissingKeyError{ID: cit.ID(), JsonKey: fmt.Sprint(jmxKey)}, }, "WithKubernetesKey": { input: map[string]interface{}{ "logs": map[string]interface{}{ "metrics_collected": map[string]interface{}{ - "kubernetes": nil, + "kubernetes": map[string]interface{}{ + "jmx_container_insights": true, + }, }, }, }, want: &want{ pipelineType: "metrics/containerinsightsjmx", receivers: []string{"otlp/jmx"}, - processors: []string{"filter/containerinsightsjmx", "resourcedetection", "resource/jmxResource", "transform/containerinsightsjmx", "metricstransform/containerinsightsjmx"}, - exporters: []string{"debug", "awsemf/jmx"}, + processors: []string{"filter/containerinsightsjmx", "resourcedetection/containerinsightsjmx", "resource/containerinsightsjmx", "transform/containerinsightsjmx", "metricstransform/containerinsightsjmx", "cumulativetodelta/containerinsightsjmx"}, + exporters: []string{"debug", "awsemf/containerinsightsjmx"}, extensions: []string{}, }, }, diff --git a/translator/translate/otel/processor/filterprocessor/testdata/config.yaml b/translator/translate/otel/processor/filterprocessor/testdata/config.yaml deleted file mode 100644 index 88d3883f15..0000000000 --- a/translator/translate/otel/processor/filterprocessor/testdata/config.yaml +++ /dev/null @@ -1,10 +0,0 @@ -metrics: - include: - match_type: strict - metric_names: - - jvm.memory.heap.init - - jvm.memory.heap.used - - jvm.memory.nonheap.init - - jvm.memory.nonheap.used - - jvm.threads.count - - tomcat.sessions diff --git a/translator/translate/otel/processor/filterprocessor/translator_test.go b/translator/translate/otel/processor/filterprocessor/translator_test.go index 55e78558d1..7ecbadc507 100644 --- a/translator/translate/otel/processor/filterprocessor/translator_test.go +++ b/translator/translate/otel/processor/filterprocessor/translator_test.go @@ -111,12 +111,6 @@ func TestTranslator(t *testing.T) { }, }), }, - "WithCompleteConfig": { - input: testutil.GetJson(t, filepath.Join("testdata", "config.json")), - index: -1, - wantID: "filter/jmx", - want: testutil.GetConf(t, filepath.Join("testdata", "ContainerInsightsJmxConfig.yaml")), - }, } for name, testCase := range testCases { t.Run(name, func(t *testing.T) { diff --git a/translator/translate/otel/processor/jmxtransformprocessor/translator_test.go b/translator/translate/otel/processor/jmxtransformprocessor/translator_test.go index db9a2b3ae5..482ec01396 100644 --- a/translator/translate/otel/processor/jmxtransformprocessor/translator_test.go +++ b/translator/translate/otel/processor/jmxtransformprocessor/translator_test.go @@ -36,12 +36,6 @@ func TestTranslator(t *testing.T) { JsonKey: common.ContainerInsightsConfigKey, }, }, - "WithContainerInsights": { - input: testutil.GetJson(t, filepath.Join("testdata", "config.json")), - index: 0, - wantID: "filter/jmx", - want: filepath.Join("testdata", "config.json"), - }, } for name, testCase := range testCases { From 8b51ee18ea631799c42018bdd609c45541e37c59 Mon Sep 17 00:00:00 2001 From: Paramadon Date: Thu, 10 Oct 2024 10:47:45 -0400 Subject: [PATCH 10/19] fixing dynamic log stream name in unit test --- .../sampleConfig/container_insights_jmx.conf | 8 +++-- .../sampleConfig/container_insights_jmx.yaml | 32 ++++++++++++------- translator/tocwconfig/tocwconfig_test.go | 13 +++++--- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/translator/tocwconfig/sampleConfig/container_insights_jmx.conf b/translator/tocwconfig/sampleConfig/container_insights_jmx.conf index 21716e9140..b081c028f6 100644 --- a/translator/tocwconfig/sampleConfig/container_insights_jmx.conf +++ b/translator/tocwconfig/sampleConfig/container_insights_jmx.conf @@ -3,7 +3,7 @@ debug = true flush_interval = "1s" flush_jitter = "0s" - hostname = "" + hostname = "host_name_from_env" interval = "60s" logfile = "" logtarget = "lumberjack" @@ -20,7 +20,9 @@ [[outputs.cloudwatchlogs]] force_flush_interval = "5s" - log_stream_name = "bcd0745edc19" - mode = "EC2" + log_stream_name = "host_name_from_env" + mode = "OP" + profile = "AmazonCloudWatchAgent" region = "us-west-2" region_type = "ACJ" + shared_credential_file = "fake-path" \ No newline at end of file diff --git a/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml b/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml index 637376042e..a424e933b2 100644 --- a/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml +++ b/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml @@ -5,23 +5,25 @@ exporters: max_values_per_datum: 150 middleware: agenthealth/metrics namespace: CWAgent + profile: AmazonCloudWatchAgent region: us-west-2 resource_to_telemetry_conversion: enabled: true + shared_credential_file: fake-path awscloudwatchlogs/emf_logs: certificate_file_path: "" emf_only: true endpoint: "" imds_retries: 1 - local_mode: false + local_mode: true log_group_name: emf/logs/default log_retention: 0 - log_stream_name: bcd0745edc19 + log_stream_name: host_name_from_env max_retries: 2 middleware: agenthealth/logs no_verify_ssl: false num_workers: 8 - profile: "" + profile: AmazonCloudWatchAgent proxy_address: "" raw_log: true region: us-west-2 @@ -39,6 +41,8 @@ exporters: enabled: true num_consumers: 1 queue_size: 1000 + shared_credentials_file: + - fake-path awsemf/containerinsights: certificate_file_path: "" detailed_metrics: false @@ -48,7 +52,7 @@ exporters: endpoint: "" enhanced_container_insights: false imds_retries: 1 - local_mode: false + local_mode: true log_group_name: /aws/containerinsights/{ClusterName}/performance log_retention: 0 log_stream_name: '{NodeName}' @@ -138,7 +142,7 @@ exporters: parse_json_encoded_attr_values: - Sources - kubernetes - profile: "" + profile: AmazonCloudWatchAgent proxy_address: "" region: us-west-2 request_timeout_seconds: 30 @@ -147,6 +151,8 @@ exporters: enabled: true retain_initial_value_of_delta_metric: false role_arn: "" + shared_credentials_file: + - fake-path version: "0" awsemf/containerinsightsjmx: certificate_file_path: "" @@ -157,7 +163,7 @@ exporters: endpoint: "" enhanced_container_insights: false imds_retries: 1 - local_mode: false + local_mode: true log_group_name: /aws/containerinsights/{ClusterName}/jmx log_retention: 0 log_stream_name: '{NodeName}' @@ -203,7 +209,7 @@ exporters: no_verify_ssl: false num_workers: 8 output_destination: cloudwatch - profile: "" + profile: AmazonCloudWatchAgent proxy_address: "" region: us-west-2 request_timeout_seconds: 30 @@ -212,6 +218,8 @@ exporters: enabled: true retain_initial_value_of_delta_metric: false role_arn: "" + shared_credentials_file: + - fake-path version: "0" debug: sampling_initial: 2 @@ -224,7 +232,7 @@ extensions: operations: - PutLogEvents usage_flags: - mode: EC2 + mode: OP region_type: ACJ agenthealth/metrics: is_usage_data_enabled: true @@ -232,7 +240,7 @@ extensions: operations: - PutMetricData usage_flags: - mode: EC2 + mode: OP region_type: ACJ processors: batch/containerinsights: @@ -822,17 +830,19 @@ receivers: kube_config_path: "" leader_lock_name: cwagent-clusterleader leader_lock_using_config_map_only: true - local_mode: false + local_mode: true max_retries: 0 no_verify_ssl: false num_workers: 0 prefer_full_pod_name: false - profile: "" + profile: AmazonCloudWatchAgent proxy_address: "" region: us-west-2 request_timeout_seconds: 0 resource_arn: "" role_arn: "" + shared_credentials_file: + - fake-path otlp/jmx: protocols: http: diff --git a/translator/tocwconfig/tocwconfig_test.go b/translator/tocwconfig/tocwconfig_test.go index 7c101d7fc7..b631187bbe 100644 --- a/translator/tocwconfig/tocwconfig_test.go +++ b/translator/tocwconfig/tocwconfig_test.go @@ -27,7 +27,6 @@ import ( "github.com/aws/amazon-cloudwatch-agent/cfg/commonconfig" "github.com/aws/amazon-cloudwatch-agent/cfg/envconfig" "github.com/aws/amazon-cloudwatch-agent/internal/retryer" - "github.com/aws/amazon-cloudwatch-agent/tool/testutil" "github.com/aws/amazon-cloudwatch-agent/translator" "github.com/aws/amazon-cloudwatch-agent/translator/cmdutil" "github.com/aws/amazon-cloudwatch-agent/translator/config" @@ -84,12 +83,18 @@ func TestGenericAppSignalsConfig(t *testing.T) { } func TestContainerInsightsJMX(t *testing.T) { resetContext(t) - testutil.SetPrometheusRemoteWriteTestingEnv(t) - context.CurrentContext().SetMode(config.ModeEC2) context.CurrentContext().SetRunInContainer(true) - expectedEnvVars := map[string]string{"CWAGENT_LOG_LEVEL": "DEBUG"} + context.CurrentContext().SetMode(config.ModeOnPremise) + t.Setenv(config.HOST_NAME, "host_name_from_env") + t.Setenv(config.HOST_IP, "127.0.0.1") + + expectedEnvVars := map[string]string{ + "CWAGENT_LOG_LEVEL": "DEBUG"} + checkTranslation(t, "container_insights_jmx", "linux", expectedEnvVars, "") + } + func TestGenericAppSignalsFallbackConfig(t *testing.T) { resetContext(t) context.CurrentContext().SetRunInContainer(false) From cbbe22c54ea36e2ffa2f10218bdd3267ccb97ac2 Mon Sep 17 00:00:00 2001 From: Paramadon Date: Thu, 10 Oct 2024 12:09:04 -0400 Subject: [PATCH 11/19] resolving comments --- cmd/config-translator/translator_test.go | 5 +- .../invalidContainerInsightsJmx.json | 4 +- .../validContainerInsightsJmx.json | 4 +- .../sampleConfig/container_insights_jmx.conf | 2 +- .../sampleConfig/container_insights_jmx.json | 16 ++--- .../sampleConfig/container_insights_jmx.yaml | 69 +++++++++---------- .../sampleConfig/jmx_eks_config_linux.conf | 2 +- .../sampleConfig/jmx_eks_config_linux.yaml | 8 +-- .../containerinsightsjmx/translator.go | 39 ++++++----- .../containerinsightsjmx/translator_test.go | 4 +- .../testdata/ContainerInsightsJmxConfig.yaml | 16 ++--- .../testdata/config.yaml | 16 ++--- .../configs/config.yaml | 16 ++--- .../processor/resourceprocessor/translator.go | 5 +- 14 files changed, 104 insertions(+), 102 deletions(-) diff --git a/cmd/config-translator/translator_test.go b/cmd/config-translator/translator_test.go index 5220075e77..d176a70692 100644 --- a/cmd/config-translator/translator_test.go +++ b/cmd/config-translator/translator_test.go @@ -189,9 +189,8 @@ func TestMetricsDestinationsConfig(t *testing.T) { } func TestJmxConfig(t *testing.T) { checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/validContainerInsightsJmx.json", true, map[string]int{}) - expectedErrorMap := map[string]int{} - expectedErrorMap["required"] = 1 - //checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidContainerInsightsJmx.json", false, expectedErrorMap) + expectedErrorMap := map[string]int{"additional_property_not_allowed": 1, "number_any_of": 1, "required": 1} + checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidContainerInsightsJmx.json", false, expectedErrorMap) } // Validate all sampleConfig files schema diff --git a/translator/config/sampleSchema/invalidContainerInsightsJmx.json b/translator/config/sampleSchema/invalidContainerInsightsJmx.json index d7ff78abfe..214fd5d7a1 100644 --- a/translator/config/sampleSchema/invalidContainerInsightsJmx.json +++ b/translator/config/sampleSchema/invalidContainerInsightsJmx.json @@ -3,10 +3,10 @@ "region": "us-west-2" }, "logs": { - "metrics_collected": { + "metrics": { "emf": { }, - "kubernete": { + "kubernetes": { } }, "force_flush_interval": 5 diff --git a/translator/config/sampleSchema/validContainerInsightsJmx.json b/translator/config/sampleSchema/validContainerInsightsJmx.json index 29668bf013..5d07791f50 100644 --- a/translator/config/sampleSchema/validContainerInsightsJmx.json +++ b/translator/config/sampleSchema/validContainerInsightsJmx.json @@ -7,8 +7,8 @@ "emf": { }, "kubernetes": { - "cluster_name": "jvmCluster", - "namespace":"siprmp-jmx" + "cluster_name": "TestCluster", + "jmx_container_insights": true } }, "force_flush_interval": 5 diff --git a/translator/tocwconfig/sampleConfig/container_insights_jmx.conf b/translator/tocwconfig/sampleConfig/container_insights_jmx.conf index b081c028f6..e415aab31b 100644 --- a/translator/tocwconfig/sampleConfig/container_insights_jmx.conf +++ b/translator/tocwconfig/sampleConfig/container_insights_jmx.conf @@ -25,4 +25,4 @@ profile = "AmazonCloudWatchAgent" region = "us-west-2" region_type = "ACJ" - shared_credential_file = "fake-path" \ No newline at end of file + shared_credential_file = "fake-path" diff --git a/translator/tocwconfig/sampleConfig/container_insights_jmx.json b/translator/tocwconfig/sampleConfig/container_insights_jmx.json index eaaf509b86..c73c60a7cd 100644 --- a/translator/tocwconfig/sampleConfig/container_insights_jmx.json +++ b/translator/tocwconfig/sampleConfig/container_insights_jmx.json @@ -18,14 +18,14 @@ "measurement": [ "jvm.classes.loaded", "jvm.memory.pool.used", - "jvm.operating.system.total.swap.space.size", - "jvm.operating.system.system.cpu.load", - "jvm.operating.system.process.cpu.load", - "jvm.operating.system.free.swap.space.size", - "jvm.operating.system.total.physical.memory.size", - "jvm.operating.system.free.physical.memory.size", - "jvm.operating.system.open.file.descriptor.count", - "jvm.operating.system.available.processors", + "jvm.system.swap.space.total", + "jvm.system.cpu.utilization", + "jvm.cpu.recent_utilization", + "jvm.system.swap.space.free", + "jvm.system.physical.memory.total", + "jvm.system.physical.memory.free", + "jvm.open_file_descriptor.count", + "jvm.system.available.processors", "jvm.threads.count", "jvm.memory.heap.used", "jvm.memory.nonheap.init", diff --git a/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml b/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml index a424e933b2..630ea083e9 100644 --- a/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml +++ b/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml @@ -221,10 +221,6 @@ exporters: shared_credentials_file: - fake-path version: "0" - debug: - sampling_initial: 2 - sampling_thereafter: 500 - verbosity: Detailed extensions: agenthealth/logs: is_usage_data_enabled: true @@ -278,14 +274,14 @@ processors: - jvm.memory.heap.used - jvm.memory.nonheap.used - jvm.memory.pool.used - - jvm.operating.system.total.swap.space.size - - jvm.operating.system.system.cpu.load - - jvm.operating.system.process.cpu.load - - jvm.operating.system.free.swap.space.size - - jvm.operating.system.total.physical.memory.size - - jvm.operating.system.free.physical.memory.size - - jvm.operating.system.open.file.descriptor.count - - jvm.operating.system.available.processors + - jvm.system.swap.space.total + - jvm.system.cpu.utilization + - jvm.cpu.recent_utilization + - jvm.system.swap.space.free + - jvm.system.physical.memory.total + - jvm.system.physical.memory.free + - jvm.open_file_descriptor.count + - jvm.system.available.processors - jvm.threads.count - jvm.threads.daemon - tomcat.sessions @@ -306,14 +302,14 @@ processors: metric_names: - jvm.classes.loaded - jvm.memory.pool.used - - jvm.operating.system.total.swap.space.size - - jvm.operating.system.system.cpu.load - - jvm.operating.system.process.cpu.load - - jvm.operating.system.free.swap.space.size - - jvm.operating.system.total.physical.memory.size - - jvm.operating.system.free.physical.memory.size - - jvm.operating.system.open.file.descriptor.count - - jvm.operating.system.available.processors + - jvm.system.swap.space.total + - jvm.system.cpu.utilization + - jvm.cpu.recent_utilization + - jvm.system.swap.space.free + - jvm.system.physical.memory.total + - jvm.system.physical.memory.free + - jvm.open_file_descriptor.count + - jvm.system.available.processors - jvm.threads.count - jvm.memory.heap.used - jvm.memory.nonheap.init @@ -372,49 +368,49 @@ processors: submatch_case: "" - action: update aggregation_type: "" - include: jvm.operating.system.total.swap.space.size + include: jvm.system.swap.space.total match_type: strict new_name: java_lang_operatingsystem_totalswapspacesize submatch_case: "" - action: update aggregation_type: "" - include: jvm.operating.system.system.cpu.load + include: jvm.system.cpu.utilization match_type: strict new_name: java_lang_operatingsystem_systemcpuload submatch_case: "" - action: update aggregation_type: "" - include: jvm.operating.system.process.cpu.load + include: jvm.cpu.recent_utilization match_type: strict new_name: java_lang_operatingsystem_processcpuload submatch_case: "" - action: update aggregation_type: "" - include: jvm.operating.system.free.swap.space.size + include: jvm.system.swap.space.free match_type: strict new_name: java_lang_operatingsystem_freeswapspacesize submatch_case: "" - action: update aggregation_type: "" - include: jvm.operating.system.total.physical.memory.size + include: jvm.system.physical.memory.total match_type: strict new_name: java_lang_operatingsystem_totalphysicalmemorysize submatch_case: "" - action: update aggregation_type: "" - include: jvm.operating.system.free.physical.memory.size + include: jvm.system.physical.memory.free match_type: strict new_name: java_lang_operatingsystem_freephysicalmemorysize submatch_case: "" - action: update aggregation_type: "" - include: jvm.operating.system.open.file.descriptor.count + include: jvm.open_file_descriptor.count match_type: strict new_name: java_lang_operatingsystem_openfiledescriptorcount submatch_case: "" - action: update aggregation_type: "" - include: jvm.operating.system.available.processors + include: jvm.system.available.processors match_type: strict new_name: java_lang_operatingsystem_availableprocessors submatch_case: "" @@ -795,21 +791,21 @@ processors: - set(unit, "Bytes") where name == "jvm.memory.pool.used" - set(unit, "Bytes") where name == "tomcat.traffic.received" - set(unit, "Bytes") where name == "tomcat.traffic.sent" - - set(unit, "Bytes") where name == "jvm.operating.system.total.swap.space.size" - - set(unit, "Bytes") where name == "jvm.operating.system.free.swap.space.size" - - set(unit, "Bytes") where name == "jvm.operating.system.total.physical.memory.size" - - set(unit, "Bytes") where name == "jvm.operating.system.free.physical.memory.size" + - set(unit, "Bytes") where name == "jvm.system.swap.space.total" + - set(unit, "Bytes") where name == "jvm.system.swap.space.free" + - set(unit, "Bytes") where name == "jvm.system.physical.memory.total" + - set(unit, "Bytes") where name == "jvm.system.physical.memory.free" - set(unit, "Count") where name == "tomcat.sessions" - set(unit, "Count") where name == "tomcat.rejected_sessions" - set(unit, "Count") where name == "jvm.threads.count" - set(unit, "Count") where name == "jvm.threads.daemon" - - set(unit, "Count") where name == "jvm.operating.system.open.file.descriptor.count" - - set(unit, "Count") where name == "jvm.operating.system.available.processors" + - set(unit, "Count") where name == "jvm.open_file_descriptor.count" + - set(unit, "Count") where name == "jvm.system.available.processors" - set(unit, "Count") where name == "tomcat.request_count" - set(unit, "Count") where name == "tomcat.errors" - set(unit, "Count") where name == "jvm.classes.loaded" - - set(unit, "Count") where name == "jvm.operating.system.system.cpu.load" - - set(unit, "Count") where name == "jvm.operating.system.process.cpu.load" + - set(unit, "Count") where name == "jvm.system.cpu.utilization" + - set(unit, "Count") where name == "jvm.cpu.recent_utilization" - set(unit, "Milliseconds") where name == "tomcat.processing_time" trace_statements: [] receivers: @@ -900,7 +896,6 @@ service: - awscontainerinsightreceiver metrics/containerinsightsjmx: exporters: - - debug - awsemf/containerinsightsjmx processors: - filter/containerinsightsjmx diff --git a/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.conf b/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.conf index 8263cf1b4d..9c37f8b52d 100644 --- a/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.conf +++ b/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.conf @@ -16,4 +16,4 @@ [outputs] - [[outputs.cloudwatch]] \ No newline at end of file + [[outputs.cloudwatch]] diff --git a/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.yaml b/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.yaml index 4c514f4901..66759372ba 100644 --- a/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/jmx_eks_config_linux.yaml @@ -131,8 +131,8 @@ processors: metric_statements: - context: metric statements: - - set(unit, "unit") where name == "jvm.memory.heap.used" - - set(name, "JVM_MEM_HEAP_USED") where name == "jvm.memory.heap.used" + - set(unit, "unit") where name == "jvm.memory.heap.used" + - set(name, "JVM_MEM_HEAP_USED") where name == "jvm.memory.heap.used" trace_statements: [] transform/jmx/1: error_mode: propagate @@ -141,7 +141,7 @@ processors: metric_statements: - context: metric statements: - - set(name, "kafka.fetch-rate") where name == "kafka.consumer.fetch-rate" + - set(name, "kafka.fetch-rate") where name == "kafka.consumer.fetch-rate" trace_statements: [] receivers: otlp/jmx: @@ -213,4 +213,4 @@ service: metrics: address: "" level: None - traces: {} \ No newline at end of file + traces: {} diff --git a/translator/translate/otel/pipeline/containerinsightsjmx/translator.go b/translator/translate/otel/pipeline/containerinsightsjmx/translator.go index 576d942a61..2184e684a7 100644 --- a/translator/translate/otel/pipeline/containerinsightsjmx/translator.go +++ b/translator/translate/otel/pipeline/containerinsightsjmx/translator.go @@ -5,13 +5,13 @@ package containerinsightsjmx import ( "fmt" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/extension/agenthealth" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/awsemf" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/debug" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/cumulativetodeltaprocessor" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/filterprocessor" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/jmxtransformprocessor" @@ -40,7 +40,7 @@ func (t *translator) ID() component.ID { return component.NewIDWithName(component.DataTypeMetrics, common.PipelineNameContainerInsightsJmx) } -// Translate creates a pipeline for container insights if the logs.metrics_collected.ecs or logs.metrics_collected.kubernetes +// Translate creates a pipeline for container insights jmx if the logs.metrics_collected.kubernetes // section is present. func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators, error) { if conf == nil || (!conf.IsSet(jmxKey)) { @@ -49,24 +49,29 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators if val, _ := common.GetBool(conf, jmxKey); !val { return nil, nil } - translators := common.ComponentTranslators{ - Receivers: common.NewTranslatorMap[component.Config](), - Processors: common.NewTranslatorMap[component.Config](), - Exporters: common.NewTranslatorMap[component.Config](), - Extensions: common.NewTranslatorMap[component.Config](), + Receivers: common.NewTranslatorMap( + otlp.NewTranslator(common.WithName(common.PipelineNameJmx)), + ), + Processors: common.NewTranslatorMap( + filterprocessor.NewTranslator(common.WithName(common.PipelineNameContainerInsightsJmx)), // Filter metrics + resourcedetection.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx), // Adds k8s cluster/nodename name + resourceprocessor.NewTranslator(common.WithName(common.PipelineNameContainerInsightsJmx)), // Change resource attribute names + jmxtransformprocessor.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx), // Removes attributes that are not of [ClusterName, Namespace] + metricstransformprocessor.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx), // Renames metrics and adds pool and area dimensions + cumulativetodeltaprocessor.NewTranslator( + common.WithName(common.PipelineNameContainerInsightsJmx), + cumulativetodeltaprocessor.WithConfigKeys(jmxKey), + ), + ), + Exporters: common.NewTranslatorMap( + awsemf.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx), + ), + Extensions: common.NewTranslatorMap( + agenthealth.NewTranslator(component.DataTypeLogs, []string{agenthealth.OperationPutLogEvents}), + ), } - translators.Receivers.Set(otlp.NewTranslator(common.WithName(common.PipelineNameJmx))) - translators.Processors.Set(filterprocessor.NewTranslator(common.WithName(common.PipelineNameContainerInsightsJmx))) //Filter metrics - translators.Processors.Set(resourcedetection.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx)) //Adds k8s cluster/nodename name - translators.Processors.Set(resourceprocessor.NewTranslator(common.WithName(common.PipelineNameContainerInsightsJmx))) //Change resource attribute names - translators.Processors.Set(jmxtransformprocessor.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx)) //Removes attributes that are not of [ClusterName, Namespace] - translators.Processors.Set(metricstransformprocessor.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx)) //Renames metrics and adds pool and area dimensions - translators.Processors.Set(cumulativetodeltaprocessor.NewTranslator(common.WithName(common.PipelineNameContainerInsightsJmx), cumulativetodeltaprocessor.WithConfigKeys(jmxKey))) - translators.Exporters.Set(debug.NewTranslator()) //Provides debug info for metrics - translators.Exporters.Set(awsemf.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx)) //Sends metrics to cloudwatch console (also adds resource attributes to metrics) - return &translators, nil } diff --git a/translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go b/translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go index 02a1e11c53..44ef5e9ff4 100644 --- a/translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go +++ b/translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go @@ -49,8 +49,8 @@ func TestTranslator(t *testing.T) { pipelineType: "metrics/containerinsightsjmx", receivers: []string{"otlp/jmx"}, processors: []string{"filter/containerinsightsjmx", "resourcedetection/containerinsightsjmx", "resource/containerinsightsjmx", "transform/containerinsightsjmx", "metricstransform/containerinsightsjmx", "cumulativetodelta/containerinsightsjmx"}, - exporters: []string{"debug", "awsemf/containerinsightsjmx"}, - extensions: []string{}, + exporters: []string{"awsemf/containerinsightsjmx"}, + extensions: []string{"agenthealth/logs"}, }, }, } diff --git a/translator/translate/otel/processor/filterprocessor/testdata/ContainerInsightsJmxConfig.yaml b/translator/translate/otel/processor/filterprocessor/testdata/ContainerInsightsJmxConfig.yaml index d0e19d1d1a..9a79d6a3c1 100644 --- a/translator/translate/otel/processor/filterprocessor/testdata/ContainerInsightsJmxConfig.yaml +++ b/translator/translate/otel/processor/filterprocessor/testdata/ContainerInsightsJmxConfig.yaml @@ -6,14 +6,14 @@ metrics: - jvm.memory.heap.used - jvm.memory.nonheap.used - jvm.memory.pool.used - - jvm.operating.system.total.swap.space.size - - jvm.operating.system.system.cpu.load - - jvm.operating.system.process.cpu.load - - jvm.operating.system.free.swap.space.size - - jvm.operating.system.total.physical.memory.size - - jvm.operating.system.free.physical.memory.size - - jvm.operating.system.open.file.descriptor.count - - jvm.operating.system.available.processors + - jvm.system.swap.space.total + - jvm.system.cpu.utilization + - jvm.cpu.recent_utilization + - jvm.system.swap.space.free + - jvm.system.physical.memory.total + - jvm.system.physical.memory.free + - jvm.open_file_descriptor.count + - jvm.system.available.processors - jvm.threads.count - jvm.threads.daemon - tomcat.sessions diff --git a/translator/translate/otel/processor/jmxtransformprocessor/testdata/config.yaml b/translator/translate/otel/processor/jmxtransformprocessor/testdata/config.yaml index a27b7a606a..484fa6ca3c 100644 --- a/translator/translate/otel/processor/jmxtransformprocessor/testdata/config.yaml +++ b/translator/translate/otel/processor/jmxtransformprocessor/testdata/config.yaml @@ -9,20 +9,20 @@ metric_statements: - set(unit, "Bytes") where name == "jvm.memory.pool.used" - set(unit, "Bytes") where name == "tomcat.traffic.received" - set(unit, "Bytes") where name == "tomcat.traffic.sent" - - set(unit, "Bytes") where name == "jvm.operating.system.total.swap.space.size" - - set(unit, "Bytes") where name == "jvm.operating.system.free.swap.space.size" - - set(unit, "Bytes") where name == "jvm.operating.system.total.physical.memory.size" - - set(unit, "Bytes") where name == "jvm.operating.system.free.physical.memory.size" + - set(unit, "Bytes") where name == "jvm.system.swap.space.total" + - set(unit, "Bytes") where name == "jvm.system.swap.space.free" + - set(unit, "Bytes") where name == "jvm.system.physical.memory.total" + - set(unit, "Bytes") where name == "jvm.system.physical.memory.free" - set(unit, "Count") where name == "tomcat.sessions" - set(unit, "Count") where name == "tomcat.rejected_sessions" - set(unit, "Count") where name == "jvm.threads.count" - set(unit, "Count") where name == "jvm.threads.daemon" - - set(unit, "Count") where name == "jvm.operating.system.open.file.descriptor.count" - - set(unit, "Count") where name == "jvm.operating.system.available.processors" + - set(unit, "Count") where name == "jvm.open_file_descriptor.count" + - set(unit, "Count") where name == "jvm.system.available.processors" - set(unit, "Count") where name == "tomcat.request_count" - set(unit, "Count") where name == "tomcat.errors" - set(unit, "Count") where name == "jvm.classes.loaded" - - set(unit, "Count") where name == "jvm.operating.system.system.cpu.load" - - set(unit, "Count") where name == "jvm.operating.system.process.cpu.load" + - set(unit, "Count") where name == "jvm.system.cpu.utilization" + - set(unit, "Count") where name == "jvm.cpu.recent_utilization" - set(unit, "Milliseconds") where name == "tomcat.processing_time" diff --git a/translator/translate/otel/processor/metricstransformprocessor/configs/config.yaml b/translator/translate/otel/processor/metricstransformprocessor/configs/config.yaml index 3ae4860d50..4034ad0a05 100644 --- a/translator/translate/otel/processor/metricstransformprocessor/configs/config.yaml +++ b/translator/translate/otel/processor/metricstransformprocessor/configs/config.yaml @@ -32,48 +32,48 @@ transforms: label: name new_label: pool - - include: jvm.operating.system.total.swap.space.size + - include: jvm.system.swap.space.total match_type: strict action: update new_name: java_lang_operatingsystem_totalswapspacesize - - include: jvm.operating.system.system.cpu.load + - include: jvm.system.cpu.utilization match_type: strict action: update new_name: java_lang_operatingsystem_systemcpuload - - include: jvm.operating.system.process.cpu.load + - include: jvm.cpu.recent_utilization match_type: strict action: update new_name: java_lang_operatingsystem_processcpuload - - include: jvm.operating.system.free.swap.space.size + - include: jvm.system.swap.space.free match_type: strict action: update new_name: java_lang_operatingsystem_freeswapspacesize - - include: jvm.operating.system.total.physical.memory.size + - include: jvm.system.physical.memory.total match_type: strict action: update new_name: java_lang_operatingsystem_totalphysicalmemorysize - - include: jvm.operating.system.free.physical.memory.size + - include: jvm.system.physical.memory.free match_type: strict action: update new_name: java_lang_operatingsystem_freephysicalmemorysize - - include: jvm.operating.system.open.file.descriptor.count + - include: jvm.open_file_descriptor.count match_type: strict action: update new_name: java_lang_operatingsystem_openfiledescriptorcount - - include: jvm.operating.system.available.processors + - include: jvm.system.available.processors match_type: strict action: update new_name: java_lang_operatingsystem_availableprocessors diff --git a/translator/translate/otel/processor/resourceprocessor/translator.go b/translator/translate/otel/processor/resourceprocessor/translator.go index 998e500c13..b3fa22fe29 100644 --- a/translator/translate/otel/processor/resourceprocessor/translator.go +++ b/translator/translate/otel/processor/resourceprocessor/translator.go @@ -69,7 +69,10 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { c := confmap.NewFromStringMap(map[string]any{ "attributes": attributes, }) - if t.Name() == common.PipelineNameContainerInsightsJmx { + + if strings.HasPrefix(t.Name(), common.PipelineNameJmx) { + attributes = t.getJMXAttributes(conf) + } else if t.Name() == common.PipelineNameContainerInsightsJmx { clusterName, ok := common.GetString(conf, common.ConfigKey(eksKey, "cluster_name")) if ok { From fcc323505a314d45a458d72229acebdf193f1794 Mon Sep 17 00:00:00 2001 From: Paramadon Date: Thu, 10 Oct 2024 12:44:30 -0400 Subject: [PATCH 12/19] fixing issues --- .../containerinsightsjmx/translator.go | 2 +- .../processor/resourceprocessor/translator.go | 86 +++++++++---------- 2 files changed, 42 insertions(+), 46 deletions(-) diff --git a/translator/translate/otel/pipeline/containerinsightsjmx/translator.go b/translator/translate/otel/pipeline/containerinsightsjmx/translator.go index 2184e684a7..c2cf8ca96d 100644 --- a/translator/translate/otel/pipeline/containerinsightsjmx/translator.go +++ b/translator/translate/otel/pipeline/containerinsightsjmx/translator.go @@ -5,13 +5,13 @@ package containerinsightsjmx import ( "fmt" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/extension/agenthealth" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/awsemf" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/extension/agenthealth" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/cumulativetodeltaprocessor" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/filterprocessor" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/jmxtransformprocessor" diff --git a/translator/translate/otel/processor/resourceprocessor/translator.go b/translator/translate/otel/processor/resourceprocessor/translator.go index b3fa22fe29..7928475ff7 100644 --- a/translator/translate/otel/processor/resourceprocessor/translator.go +++ b/translator/translate/otel/processor/resourceprocessor/translator.go @@ -56,9 +56,11 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { cfg := t.factory.CreateDefaultConfig().(*resourceprocessor.Config) var attributes []any + if strings.HasPrefix(t.Name(), common.PipelineNameJmx) { attributes = t.getJMXAttributes(conf) } + if len(attributes) == 0 && t.Name() != common.PipelineNameContainerInsightsJmx { baseKey := common.JmxConfigKey if t.Index() != -1 { @@ -66,65 +68,59 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { } return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: common.ConfigKey(baseKey, common.AppendDimensionsKey)} } - c := confmap.NewFromStringMap(map[string]any{ - "attributes": attributes, - }) - if strings.HasPrefix(t.Name(), common.PipelineNameJmx) { - attributes = t.getJMXAttributes(conf) - } else if t.Name() == common.PipelineNameContainerInsightsJmx { + if t.Name() == common.PipelineNameContainerInsightsJmx { clusterName, ok := common.GetString(conf, common.ConfigKey(eksKey, "cluster_name")) if ok { - c = confmap.NewFromStringMap(map[string]any{ - //from config - "attributes": []any{ - map[string]any{ - "key": "Namespace", - "from_attribute": "k8s.namespace.name", - "action": "insert", - }, - map[string]any{ - "key": "ClusterName", - "value": clusterName, - "action": "upsert", - }, - map[string]any{ - "key": "NodeName", - "from_attribute": "host.name", - "action": "insert", - }, + attributes = []any{ + map[string]any{ + "key": "Namespace", + "from_attribute": "k8s.namespace.name", + "action": "insert", + }, + map[string]any{ + "key": "ClusterName", + "value": clusterName, + "action": "upsert", + }, + map[string]any{ + "key": "NodeName", + "from_attribute": "host.name", + "action": "insert", }, - }) + } } else { - c = confmap.NewFromStringMap(map[string]any{ - //from resource detection processor - "attributes": []any{ - map[string]any{ - "key": "ClusterName", - "from_attribute": "k8s.cluster.name", - "action": "insert", - }, - map[string]any{ - "key": "Namespace", - "from_attribute": "k8s.namespace.name", - "action": "insert", - }, - map[string]any{ - "key": "NodeName", - "from_attribute": "host.name", - "action": "insert", - }, + attributes = []any{ + map[string]any{ + "key": "ClusterName", + "from_attribute": "k8s.cluster.name", + "action": "insert", }, - }) + map[string]any{ + "key": "Namespace", + "from_attribute": "k8s.namespace.name", + "action": "insert", + }, + map[string]any{ + "key": "NodeName", + "from_attribute": "host.name", + "action": "insert", + }, + } } - } + + c := confmap.NewFromStringMap(map[string]any{ + "attributes": attributes, + }) + if err := c.Unmarshal(&cfg); err != nil { return nil, fmt.Errorf("unable to unmarshal resource processor: %w", err) } return cfg, nil + } func (t *translator) getJMXAttributes(conf *confmap.Conf) []any { From 0688b6829d63bb33916cd1d0478af402ce5ec3df Mon Sep 17 00:00:00 2001 From: Paramadon Date: Thu, 10 Oct 2024 19:07:23 -0400 Subject: [PATCH 13/19] Resolving comments --- .../sampleConfig/container_insights_jmx.conf | 6 +- .../sampleConfig/container_insights_jmx.json | 28 - .../sampleConfig/container_insights_jmx.yaml | 1718 ++++++++--------- translator/tocwconfig/tocwconfig_test.go | 2 +- ...jmx_config.yaml => awsemf_jmx_config.yaml} | 0 .../otel/exporter/awsemf/translator.go | 2 +- .../containerinsightsjmx/translator.go | 11 +- .../containerinsightsjmx/translator_test.go | 2 + .../resourceprocessor/testdata/config.json | 3 +- .../processor/resourceprocessor/translator.go | 44 +- .../testdata/config.json | 0 .../testdata/config.yaml | 0 .../translate.go | 2 +- .../translator_test.go | 2 +- 14 files changed, 837 insertions(+), 983 deletions(-) rename translator/translate/otel/exporter/awsemf/{awsemfjmx_config.yaml => awsemf_jmx_config.yaml} (100%) rename translator/translate/otel/processor/{jmxtransformprocessor => transformprocessor}/testdata/config.json (100%) rename translator/translate/otel/processor/{jmxtransformprocessor => transformprocessor}/testdata/config.yaml (100%) rename translator/translate/otel/processor/{jmxtransformprocessor => transformprocessor}/translate.go (97%) rename translator/translate/otel/processor/{jmxtransformprocessor => transformprocessor}/translator_test.go (98%) diff --git a/translator/tocwconfig/sampleConfig/container_insights_jmx.conf b/translator/tocwconfig/sampleConfig/container_insights_jmx.conf index e415aab31b..66b62800a8 100644 --- a/translator/tocwconfig/sampleConfig/container_insights_jmx.conf +++ b/translator/tocwconfig/sampleConfig/container_insights_jmx.conf @@ -16,13 +16,9 @@ [outputs] - [[outputs.cloudwatch]] - [[outputs.cloudwatchlogs]] force_flush_interval = "5s" log_stream_name = "host_name_from_env" - mode = "OP" - profile = "AmazonCloudWatchAgent" + mode = "EC2" region = "us-west-2" region_type = "ACJ" - shared_credential_file = "fake-path" diff --git a/translator/tocwconfig/sampleConfig/container_insights_jmx.json b/translator/tocwconfig/sampleConfig/container_insights_jmx.json index c73c60a7cd..72d7adf71e 100644 --- a/translator/tocwconfig/sampleConfig/container_insights_jmx.json +++ b/translator/tocwconfig/sampleConfig/container_insights_jmx.json @@ -4,38 +4,10 @@ }, "logs": { "metrics_collected": { - "emf": { }, "kubernetes": { "cluster_name": "TestCluster", "jmx_container_insights": true } } - }, - "metrics": { - "metrics_collected": { - "jmx": { - "jvm": { - "measurement": [ - "jvm.classes.loaded", - "jvm.memory.pool.used", - "jvm.system.swap.space.total", - "jvm.system.cpu.utilization", - "jvm.cpu.recent_utilization", - "jvm.system.swap.space.free", - "jvm.system.physical.memory.total", - "jvm.system.physical.memory.free", - "jvm.open_file_descriptor.count", - "jvm.system.available.processors", - "jvm.threads.count", - "jvm.memory.heap.used", - "jvm.memory.nonheap.init", - "jvm.threads.daemon" - ] - } - } - }, - "metrics_destinations": { - "cloudwatch": { } - } } } \ No newline at end of file diff --git a/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml b/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml index 630ea083e9..a01450fc5d 100644 --- a/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml +++ b/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml @@ -1,932 +1,796 @@ exporters: - awscloudwatch: - force_flush_interval: 1m0s - max_datums_per_call: 1000 - max_values_per_datum: 150 - middleware: agenthealth/metrics - namespace: CWAgent - profile: AmazonCloudWatchAgent - region: us-west-2 - resource_to_telemetry_conversion: - enabled: true - shared_credential_file: fake-path - awscloudwatchlogs/emf_logs: - certificate_file_path: "" - emf_only: true - endpoint: "" - imds_retries: 1 - local_mode: true - log_group_name: emf/logs/default - log_retention: 0 - log_stream_name: host_name_from_env - max_retries: 2 - middleware: agenthealth/logs - no_verify_ssl: false - num_workers: 8 - profile: AmazonCloudWatchAgent - proxy_address: "" - raw_log: true - region: us-west-2 - request_timeout_seconds: 30 - resource_arn: "" - retry_on_failure: - enabled: true - initial_interval: 5s - max_elapsed_time: 5m0s - max_interval: 30s - multiplier: 1.5 - randomization_factor: 0.5 - role_arn: "" - sending_queue: - enabled: true - num_consumers: 1 - queue_size: 1000 - shared_credentials_file: - - fake-path - awsemf/containerinsights: - certificate_file_path: "" - detailed_metrics: false - dimension_rollup_option: NoDimensionRollup - disable_metric_extraction: false - eks_fargate_container_insights_enabled: false - endpoint: "" - enhanced_container_insights: false - imds_retries: 1 - local_mode: true - log_group_name: /aws/containerinsights/{ClusterName}/performance - log_retention: 0 - log_stream_name: '{NodeName}' - max_retries: 2 - metric_declarations: - - dimensions: - - - ClusterName - - Namespace - - PodName - - - ClusterName - - - ClusterName - - Namespace - - Service - - - ClusterName - - Namespace - metric_name_selectors: - - pod_cpu_utilization - - pod_memory_utilization - - pod_network_rx_bytes - - pod_network_tx_bytes - - pod_cpu_utilization_over_pod_limit - - pod_memory_utilization_over_pod_limit - - dimensions: - - - ClusterName - - Namespace - - PodName - metric_name_selectors: - - pod_number_of_container_restarts - - dimensions: - - - ClusterName - - Namespace - - PodName - - - ClusterName - metric_name_selectors: - - pod_cpu_reserved_capacity - - pod_memory_reserved_capacity - - dimensions: - - - ClusterName - - InstanceId - - NodeName - - - ClusterName - metric_name_selectors: - - node_cpu_utilization - - node_memory_utilization - - node_network_total_bytes - - node_cpu_reserved_capacity - - node_memory_reserved_capacity - - node_number_of_running_pods - - node_number_of_running_containers - - dimensions: - - - ClusterName - metric_name_selectors: - - node_cpu_usage_total - - node_cpu_limit - - node_memory_working_set - - node_memory_limit - - dimensions: - - - ClusterName - - InstanceId - - NodeName - - - ClusterName - metric_name_selectors: - - node_filesystem_utilization - - dimensions: - - - ClusterName - - Namespace - - Service - - - ClusterName - metric_name_selectors: - - service_number_of_running_pods - - dimensions: - - - ClusterName - - Namespace - - - ClusterName - metric_name_selectors: - - namespace_number_of_running_pods - - dimensions: - - - ClusterName - metric_name_selectors: - - cluster_node_count - - cluster_failed_node_count - middleware: agenthealth/logs - namespace: ContainerInsights - no_verify_ssl: false - num_workers: 8 - output_destination: cloudwatch - parse_json_encoded_attr_values: - - Sources - - kubernetes - profile: AmazonCloudWatchAgent - proxy_address: "" - region: us-west-2 - request_timeout_seconds: 30 - resource_arn: "" - resource_to_telemetry_conversion: - enabled: true - retain_initial_value_of_delta_metric: false - role_arn: "" - shared_credentials_file: - - fake-path - version: "0" - awsemf/containerinsightsjmx: - certificate_file_path: "" - detailed_metrics: false - dimension_rollup_option: NoDimensionRollup - disable_metric_extraction: false - eks_fargate_container_insights_enabled: false - endpoint: "" - enhanced_container_insights: false - imds_retries: 1 - local_mode: true - log_group_name: /aws/containerinsights/{ClusterName}/jmx - log_retention: 0 - log_stream_name: '{NodeName}' - max_retries: 2 - metric_declarations: - - dimensions: - - - ClusterName - - Namespace - metric_name_selectors: - - java_lang_operatingsystem_freeswapspacesize - - java_lang_operatingsystem_availableprocessors - - catalina_manager_rejectedsessions - - catalina_globalrequestprocessor_bytesreceived - - catalina_globalrequestprocessor_processingtime - - jvm_memory_pool_bytes_used - - java_lang_operatingsystem_systemcpuload - - java_lang_operatingsystem_totalphysicalmemorysize - - java_lang_operatingsystem_freephysicalmemorysize - - java_lang_operatingsystem_openfiledescriptorcount - - catalina_manager_activesessions - - java_lang_operatingsystem_totalswapspacesize - - java_lang_operatingsystem_processcpuload - - catalina_globalrequestprocessor_requestcount - - catalina_globalrequestprocessor_errorcount - - jvm_threads_daemon - - catalina_globalrequestprocessor_bytessent - - jvm_classes_loaded - - jvm_threads_current - - dimensions: - - - ClusterName - - Namespace - - area - metric_name_selectors: - - jvm_memory_bytes_used - - dimensions: - - - ClusterName - - Namespace - - pool - metric_name_selectors: - - jvm_memory_pool_bytes_used - middleware: agenthealth/logs - namespace: ContainerInsights/Prometheus - no_verify_ssl: false - num_workers: 8 - output_destination: cloudwatch - profile: AmazonCloudWatchAgent - proxy_address: "" - region: us-west-2 - request_timeout_seconds: 30 - resource_arn: "" - resource_to_telemetry_conversion: - enabled: true - retain_initial_value_of_delta_metric: false - role_arn: "" - shared_credentials_file: - - fake-path - version: "0" + awsemf/containerinsights: + certificate_file_path: "" + detailed_metrics: false + dimension_rollup_option: NoDimensionRollup + disable_metric_extraction: false + eks_fargate_container_insights_enabled: false + endpoint: "" + enhanced_container_insights: false + imds_retries: 1 + local_mode: false + log_group_name: /aws/containerinsights/{ClusterName}/performance + log_retention: 0 + log_stream_name: '{NodeName}' + max_retries: 2 + metric_declarations: + - dimensions: + - - ClusterName + - Namespace + - PodName + - - ClusterName + - - ClusterName + - Namespace + - Service + - - ClusterName + - Namespace + metric_name_selectors: + - pod_cpu_utilization + - pod_memory_utilization + - pod_network_rx_bytes + - pod_network_tx_bytes + - pod_cpu_utilization_over_pod_limit + - pod_memory_utilization_over_pod_limit + - dimensions: + - - ClusterName + - Namespace + - PodName + metric_name_selectors: + - pod_number_of_container_restarts + - dimensions: + - - ClusterName + - Namespace + - PodName + - - ClusterName + metric_name_selectors: + - pod_cpu_reserved_capacity + - pod_memory_reserved_capacity + - dimensions: + - - ClusterName + - InstanceId + - NodeName + - - ClusterName + metric_name_selectors: + - node_cpu_utilization + - node_memory_utilization + - node_network_total_bytes + - node_cpu_reserved_capacity + - node_memory_reserved_capacity + - node_number_of_running_pods + - node_number_of_running_containers + - dimensions: + - - ClusterName + metric_name_selectors: + - node_cpu_usage_total + - node_cpu_limit + - node_memory_working_set + - node_memory_limit + - dimensions: + - - ClusterName + - InstanceId + - NodeName + - - ClusterName + metric_name_selectors: + - node_filesystem_utilization + - dimensions: + - - ClusterName + - Namespace + - Service + - - ClusterName + metric_name_selectors: + - service_number_of_running_pods + - dimensions: + - - ClusterName + - Namespace + - - ClusterName + metric_name_selectors: + - namespace_number_of_running_pods + - dimensions: + - - ClusterName + metric_name_selectors: + - cluster_node_count + - cluster_failed_node_count + middleware: agenthealth/logs + namespace: ContainerInsights + no_verify_ssl: false + num_workers: 8 + output_destination: cloudwatch + parse_json_encoded_attr_values: + - Sources + - kubernetes + profile: "" + proxy_address: "" + region: us-west-2 + request_timeout_seconds: 30 + resource_arn: "" + resource_to_telemetry_conversion: + enabled: true + retain_initial_value_of_delta_metric: false + role_arn: "" + version: "0" + awsemf/containerinsightsjmx: + certificate_file_path: "" + detailed_metrics: false + dimension_rollup_option: NoDimensionRollup + disable_metric_extraction: false + eks_fargate_container_insights_enabled: false + endpoint: "" + enhanced_container_insights: false + imds_retries: 1 + local_mode: false + log_group_name: /aws/containerinsights/{ClusterName}/jmx + log_retention: 0 + log_stream_name: '{NodeName}' + max_retries: 2 + metric_declarations: + - dimensions: + - - ClusterName + - Namespace + metric_name_selectors: + - java_lang_operatingsystem_freeswapspacesize + - java_lang_operatingsystem_availableprocessors + - catalina_manager_rejectedsessions + - catalina_globalrequestprocessor_bytesreceived + - catalina_globalrequestprocessor_processingtime + - jvm_memory_pool_bytes_used + - java_lang_operatingsystem_systemcpuload + - java_lang_operatingsystem_totalphysicalmemorysize + - java_lang_operatingsystem_freephysicalmemorysize + - java_lang_operatingsystem_openfiledescriptorcount + - catalina_manager_activesessions + - java_lang_operatingsystem_totalswapspacesize + - java_lang_operatingsystem_processcpuload + - catalina_globalrequestprocessor_requestcount + - catalina_globalrequestprocessor_errorcount + - jvm_threads_daemon + - catalina_globalrequestprocessor_bytessent + - jvm_classes_loaded + - jvm_threads_current + - dimensions: + - - ClusterName + - Namespace + - area + metric_name_selectors: + - jvm_memory_bytes_used + - dimensions: + - - ClusterName + - Namespace + - pool + metric_name_selectors: + - jvm_memory_pool_bytes_used + middleware: agenthealth/logs + namespace: ContainerInsights/Prometheus + no_verify_ssl: false + num_workers: 8 + output_destination: cloudwatch + profile: "" + proxy_address: "" + region: us-west-2 + request_timeout_seconds: 30 + resource_arn: "" + resource_to_telemetry_conversion: + enabled: true + retain_initial_value_of_delta_metric: false + role_arn: "" + version: "0" extensions: - agenthealth/logs: - is_usage_data_enabled: true - stats: - operations: - - PutLogEvents - usage_flags: - mode: OP - region_type: ACJ - agenthealth/metrics: - is_usage_data_enabled: true - stats: - operations: - - PutMetricData - usage_flags: - mode: OP - region_type: ACJ + agenthealth/logs: + is_usage_data_enabled: true + stats: + operations: + - PutLogEvents + usage_flags: + mode: EC2 + region_type: ACJ processors: - batch/containerinsights: - metadata_cardinality_limit: 1000 - send_batch_max_size: 0 - send_batch_size: 8192 - timeout: 5s - batch/emf_logs: - metadata_cardinality_limit: 1000 - send_batch_max_size: 0 - send_batch_size: 8192 - timeout: 5s - cumulativetodelta/containerinsightsjmx: - exclude: - match_type: "" - include: - match_type: "" - initial_value: 2 - max_staleness: 0s - cumulativetodelta/jmx: - exclude: - match_type: "" - include: - match_type: "" - initial_value: 2 - max_staleness: 0s - filter/containerinsightsjmx: - error_mode: propagate - logs: {} - metrics: - include: - match_type: strict - metric_names: - - jvm.classes.loaded - - jvm.memory.heap.used - - jvm.memory.nonheap.used - - jvm.memory.pool.used - - jvm.system.swap.space.total - - jvm.system.cpu.utilization - - jvm.cpu.recent_utilization - - jvm.system.swap.space.free - - jvm.system.physical.memory.total - - jvm.system.physical.memory.free - - jvm.open_file_descriptor.count - - jvm.system.available.processors - - jvm.threads.count - - jvm.threads.daemon - - tomcat.sessions - - tomcat.rejected_sessions - - tomcat.traffic.received - - tomcat.traffic.sent - - tomcat.request_count - - tomcat.errors - - tomcat.processing_time - spans: {} - traces: {} - filter/jmx: - error_mode: propagate - logs: {} - metrics: - include: - match_type: strict - metric_names: - - jvm.classes.loaded - - jvm.memory.pool.used - - jvm.system.swap.space.total - - jvm.system.cpu.utilization - - jvm.cpu.recent_utilization - - jvm.system.swap.space.free - - jvm.system.physical.memory.total - - jvm.system.physical.memory.free - - jvm.open_file_descriptor.count - - jvm.system.available.processors - - jvm.threads.count - - jvm.memory.heap.used - - jvm.memory.nonheap.init - - jvm.threads.daemon - spans: {} - traces: {} - metricstransform/containerinsightsjmx: - transforms: - - action: update - aggregation_type: "" - include: jvm.classes.loaded - match_type: strict - new_name: jvm_classes_loaded - submatch_case: "" - - action: update - aggregation_type: "" - include: jvm.memory.heap.used - match_type: strict - new_name: jvm_memory_bytes_used - operations: - - action: add_label - aggregation_type: "" - experimental_scale: 0 - label: "" - label_value: "" - new_label: area - new_value: heap - submatch_case: "" - - action: update - aggregation_type: "" - include: jvm.memory.nonheap.used - match_type: strict - new_name: jvm_memory_bytes_used - operations: - - action: add_label - aggregation_type: "" - experimental_scale: 0 - label: "" - label_value: "" - new_label: area - new_value: nonheap - submatch_case: "" - - action: update - aggregation_type: "" - include: jvm.memory.pool.used - match_type: strict - new_name: jvm_memory_pool_bytes_used - operations: - - action: update_label - aggregation_type: "" - experimental_scale: 0 - label: name - label_value: "" - new_label: pool - new_value: "" - submatch_case: "" - - action: update - aggregation_type: "" - include: jvm.system.swap.space.total - match_type: strict - new_name: java_lang_operatingsystem_totalswapspacesize - submatch_case: "" - - action: update - aggregation_type: "" - include: jvm.system.cpu.utilization - match_type: strict - new_name: java_lang_operatingsystem_systemcpuload - submatch_case: "" - - action: update - aggregation_type: "" - include: jvm.cpu.recent_utilization - match_type: strict - new_name: java_lang_operatingsystem_processcpuload - submatch_case: "" - - action: update - aggregation_type: "" - include: jvm.system.swap.space.free - match_type: strict - new_name: java_lang_operatingsystem_freeswapspacesize - submatch_case: "" - - action: update - aggregation_type: "" - include: jvm.system.physical.memory.total - match_type: strict - new_name: java_lang_operatingsystem_totalphysicalmemorysize - submatch_case: "" - - action: update - aggregation_type: "" - include: jvm.system.physical.memory.free - match_type: strict - new_name: java_lang_operatingsystem_freephysicalmemorysize - submatch_case: "" - - action: update - aggregation_type: "" - include: jvm.open_file_descriptor.count - match_type: strict - new_name: java_lang_operatingsystem_openfiledescriptorcount - submatch_case: "" - - action: update - aggregation_type: "" - include: jvm.system.available.processors - match_type: strict - new_name: java_lang_operatingsystem_availableprocessors - submatch_case: "" - - action: update - aggregation_type: "" - include: jvm.threads.count - match_type: strict - new_name: jvm_threads_current - submatch_case: "" - - action: update - aggregation_type: "" - include: jvm.threads.daemon - match_type: strict - new_name: jvm_threads_daemon - submatch_case: "" - - action: update - aggregation_type: "" - include: tomcat.sessions - match_type: strict - new_name: catalina_manager_activesessions - submatch_case: "" - - action: update - aggregation_type: "" - include: tomcat.rejected_sessions - match_type: strict - new_name: catalina_manager_rejectedsessions - submatch_case: "" - - action: update - aggregation_type: "" - include: tomcat.traffic.received - match_type: strict - new_name: catalina_globalrequestprocessor_bytesreceived - submatch_case: "" - - action: update - aggregation_type: "" - include: tomcat.traffic.sent - match_type: strict - new_name: catalina_globalrequestprocessor_bytessent - submatch_case: "" - - action: update - aggregation_type: "" - include: tomcat.request_count - match_type: strict - new_name: catalina_globalrequestprocessor_requestcount - submatch_case: "" - - action: update - aggregation_type: "" - include: tomcat.errors - match_type: strict - new_name: catalina_globalrequestprocessor_errorcount - submatch_case: "" - - action: update - aggregation_type: "" - include: tomcat.processing_time - match_type: strict - new_name: catalina_globalrequestprocessor_processingtime - submatch_case: "" - resource/containerinsightsjmx: - attributes: - - action: insert - converted_type: "" - from_attribute: k8s.namespace.name - from_context: "" - key: Namespace - pattern: "" - - action: upsert - converted_type: "" - from_attribute: "" - from_context: "" - key: ClusterName - pattern: "" - value: TestCluster - - action: insert - converted_type: "" - from_attribute: host.name - from_context: "" - key: NodeName - pattern: "" - resourcedetection/containerinsightsjmx: - aks: - resource_attributes: - cloud.platform: - enabled: true - cloud.provider: - enabled: true - k8s.cluster.name: - enabled: false - azure: - resource_attributes: - azure.resourcegroup.name: - enabled: true - azure.vm.name: - enabled: true - azure.vm.scaleset.name: - enabled: true - azure.vm.size: - enabled: true - cloud.account.id: - enabled: true - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - host.id: - enabled: true - host.name: - enabled: true - tags: [] - compression: "" - consul: - address: "" - datacenter: "" - namespace: "" - resource_attributes: - cloud.region: - enabled: true - host.id: - enabled: true - host.name: - enabled: true - token_file: "" - detectors: - - env - - eks - - system - disable_keep_alives: false - docker: - resource_attributes: - host.name: - enabled: true - os.type: - enabled: true - ec2: - resource_attributes: - cloud.account.id: - enabled: true - cloud.availability_zone: - enabled: true - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - host.id: - enabled: true - host.image.id: - enabled: true - host.name: - enabled: true - host.type: - enabled: true - tags: [] - ecs: - resource_attributes: - aws.ecs.cluster.arn: - enabled: true - aws.ecs.launchtype: - enabled: true - aws.ecs.task.arn: - enabled: true - aws.ecs.task.family: - enabled: true - aws.ecs.task.id: - enabled: true - aws.ecs.task.revision: - enabled: true - aws.log.group.arns: - enabled: true - aws.log.group.names: - enabled: true - aws.log.stream.arns: - enabled: true - aws.log.stream.names: - enabled: true - cloud.account.id: - enabled: true - cloud.availability_zone: - enabled: true - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - eks: - resource_attributes: - cloud.platform: - enabled: true - cloud.provider: - enabled: true - k8s.cluster.name: - enabled: true - elasticbeanstalk: - resource_attributes: - cloud.platform: - enabled: true - cloud.provider: - enabled: true - deployment.environment: - enabled: true - service.instance.id: - enabled: true - service.version: - enabled: true - endpoint: "" - gcp: - resource_attributes: - cloud.account.id: - enabled: true - cloud.availability_zone: - enabled: true - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - faas.id: - enabled: true - faas.instance: - enabled: true - faas.name: - enabled: true - faas.version: - enabled: true - gcp.cloud_run.job.execution: - enabled: true - gcp.cloud_run.job.task_index: - enabled: true - gcp.gce.instance.hostname: - enabled: false - gcp.gce.instance.name: - enabled: false - host.id: - enabled: true - host.name: - enabled: true - host.type: - enabled: true - k8s.cluster.name: - enabled: true - heroku: - resource_attributes: - cloud.provider: - enabled: true - heroku.app.id: - enabled: true - heroku.dyno.id: - enabled: true - heroku.release.commit: - enabled: true - heroku.release.creation_timestamp: - enabled: true - service.instance.id: - enabled: true - service.name: - enabled: true - service.version: - enabled: true - http2_ping_timeout: 0s - http2_read_idle_timeout: 0s - idle_conn_timeout: 1m30s - k8snode: - auth_type: serviceAccount - context: "" - kube_config_path: "" - node_from_env_var: "" - resource_attributes: - k8s.node.name: - enabled: true - k8s.node.uid: - enabled: true - lambda: - resource_attributes: - aws.log.group.names: - enabled: true - aws.log.stream.names: - enabled: true - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - faas.instance: - enabled: true - faas.max_memory: - enabled: true - faas.name: - enabled: true - faas.version: - enabled: true - max_idle_conns: 100 - openshift: - address: "" - resource_attributes: - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - k8s.cluster.name: - enabled: true - tls: - ca_file: "" - cert_file: "" - include_system_ca_certs_pool: false - insecure: false - insecure_skip_verify: false - key_file: "" - max_version: "" - min_version: "" - reload_interval: 0s - server_name_override: "" - token: "" - override: false - proxy_url: "" - read_buffer_size: 0 - system: - hostname_sources: - - os - resource_attributes: - host.arch: - enabled: false - host.cpu.cache.l2.size: - enabled: false - host.cpu.family: - enabled: false - host.cpu.model.id: - enabled: false - host.cpu.model.name: - enabled: false - host.cpu.stepping: - enabled: false - host.cpu.vendor.id: - enabled: false - host.id: - enabled: false - host.ip: - enabled: false - host.mac: - enabled: false - host.name: - enabled: true - os.description: - enabled: false - os.type: - enabled: true - timeout: 15s - tls: - ca_file: "" - cert_file: "" - include_system_ca_certs_pool: false - insecure: false - insecure_skip_verify: false - key_file: "" - max_version: "" - min_version: "" - reload_interval: 0s - server_name_override: "" - write_buffer_size: 0 - transform/containerinsightsjmx: - error_mode: propagate - flatten_data: false - log_statements: [] - metric_statements: - - context: resource - statements: - - keep_keys(attributes, ["ClusterName", "Namespace", "NodeName"]) - - context: metric - statements: - - set(unit, "Bytes") where name == "jvm.memory.heap.used" - - set(unit, "Bytes") where name == "jvm.memory.nonheap.used" - - set(unit, "Bytes") where name == "jvm.memory.pool.used" - - set(unit, "Bytes") where name == "tomcat.traffic.received" - - set(unit, "Bytes") where name == "tomcat.traffic.sent" - - set(unit, "Bytes") where name == "jvm.system.swap.space.total" - - set(unit, "Bytes") where name == "jvm.system.swap.space.free" - - set(unit, "Bytes") where name == "jvm.system.physical.memory.total" - - set(unit, "Bytes") where name == "jvm.system.physical.memory.free" - - set(unit, "Count") where name == "tomcat.sessions" - - set(unit, "Count") where name == "tomcat.rejected_sessions" - - set(unit, "Count") where name == "jvm.threads.count" - - set(unit, "Count") where name == "jvm.threads.daemon" - - set(unit, "Count") where name == "jvm.open_file_descriptor.count" - - set(unit, "Count") where name == "jvm.system.available.processors" - - set(unit, "Count") where name == "tomcat.request_count" - - set(unit, "Count") where name == "tomcat.errors" - - set(unit, "Count") where name == "jvm.classes.loaded" - - set(unit, "Count") where name == "jvm.system.cpu.utilization" - - set(unit, "Count") where name == "jvm.cpu.recent_utilization" - - set(unit, "Milliseconds") where name == "tomcat.processing_time" - trace_statements: [] + batch/containerinsights: + metadata_cardinality_limit: 1000 + send_batch_max_size: 0 + send_batch_size: 8192 + timeout: 5s + cumulativetodelta/containerinsightsjmx: + exclude: + match_type: "" + include: + match_type: "" + initial_value: 2 + max_staleness: 0s + filter/containerinsightsjmx: + error_mode: propagate + logs: {} + metrics: + include: + match_type: strict + metric_names: + - jvm.classes.loaded + - jvm.memory.heap.used + - jvm.memory.nonheap.used + - jvm.memory.pool.used + - jvm.system.swap.space.total + - jvm.system.cpu.utilization + - jvm.cpu.recent_utilization + - jvm.system.swap.space.free + - jvm.system.physical.memory.total + - jvm.system.physical.memory.free + - jvm.open_file_descriptor.count + - jvm.system.available.processors + - jvm.threads.count + - jvm.threads.daemon + - tomcat.sessions + - tomcat.rejected_sessions + - tomcat.traffic.received + - tomcat.traffic.sent + - tomcat.request_count + - tomcat.errors + - tomcat.processing_time + spans: {} + traces: {} + metricstransform/containerinsightsjmx: + transforms: + - action: update + aggregation_type: "" + include: jvm.classes.loaded + match_type: strict + new_name: jvm_classes_loaded + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.memory.heap.used + match_type: strict + new_name: jvm_memory_bytes_used + operations: + - action: add_label + aggregation_type: "" + experimental_scale: 0 + label: "" + label_value: "" + new_label: area + new_value: heap + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.memory.nonheap.used + match_type: strict + new_name: jvm_memory_bytes_used + operations: + - action: add_label + aggregation_type: "" + experimental_scale: 0 + label: "" + label_value: "" + new_label: area + new_value: nonheap + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.memory.pool.used + match_type: strict + new_name: jvm_memory_pool_bytes_used + operations: + - action: update_label + aggregation_type: "" + experimental_scale: 0 + label: name + label_value: "" + new_label: pool + new_value: "" + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.system.swap.space.total + match_type: strict + new_name: java_lang_operatingsystem_totalswapspacesize + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.system.cpu.utilization + match_type: strict + new_name: java_lang_operatingsystem_systemcpuload + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.cpu.recent_utilization + match_type: strict + new_name: java_lang_operatingsystem_processcpuload + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.system.swap.space.free + match_type: strict + new_name: java_lang_operatingsystem_freeswapspacesize + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.system.physical.memory.total + match_type: strict + new_name: java_lang_operatingsystem_totalphysicalmemorysize + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.system.physical.memory.free + match_type: strict + new_name: java_lang_operatingsystem_freephysicalmemorysize + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.open_file_descriptor.count + match_type: strict + new_name: java_lang_operatingsystem_openfiledescriptorcount + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.system.available.processors + match_type: strict + new_name: java_lang_operatingsystem_availableprocessors + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.threads.count + match_type: strict + new_name: jvm_threads_current + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.threads.daemon + match_type: strict + new_name: jvm_threads_daemon + submatch_case: "" + - action: update + aggregation_type: "" + include: tomcat.sessions + match_type: strict + new_name: catalina_manager_activesessions + submatch_case: "" + - action: update + aggregation_type: "" + include: tomcat.rejected_sessions + match_type: strict + new_name: catalina_manager_rejectedsessions + submatch_case: "" + - action: update + aggregation_type: "" + include: tomcat.traffic.received + match_type: strict + new_name: catalina_globalrequestprocessor_bytesreceived + submatch_case: "" + - action: update + aggregation_type: "" + include: tomcat.traffic.sent + match_type: strict + new_name: catalina_globalrequestprocessor_bytessent + submatch_case: "" + - action: update + aggregation_type: "" + include: tomcat.request_count + match_type: strict + new_name: catalina_globalrequestprocessor_requestcount + submatch_case: "" + - action: update + aggregation_type: "" + include: tomcat.errors + match_type: strict + new_name: catalina_globalrequestprocessor_errorcount + submatch_case: "" + - action: update + aggregation_type: "" + include: tomcat.processing_time + match_type: strict + new_name: catalina_globalrequestprocessor_processingtime + submatch_case: "" + resource/containerinsightsjmx: + attributes: + - action: insert + converted_type: "" + from_attribute: k8s.namespace.name + from_context: "" + key: Namespace + pattern: "" + - action: upsert + converted_type: "" + from_attribute: "" + from_context: "" + key: ClusterName + pattern: "" + value: TestCluster + - action: insert + converted_type: "" + from_attribute: host.name + from_context: "" + key: NodeName + pattern: "" + resourcedetection/containerinsightsjmx: + aks: + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + k8s.cluster.name: + enabled: false + azure: + resource_attributes: + azure.resourcegroup.name: + enabled: true + azure.vm.name: + enabled: true + azure.vm.scaleset.name: + enabled: true + azure.vm.size: + enabled: true + cloud.account.id: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + host.id: + enabled: true + host.name: + enabled: true + tags: [] + compression: "" + consul: + address: "" + datacenter: "" + namespace: "" + resource_attributes: + cloud.region: + enabled: true + host.id: + enabled: true + host.name: + enabled: true + token_file: "" + detectors: + - env + - eks + - system + disable_keep_alives: false + docker: + resource_attributes: + host.name: + enabled: true + os.type: + enabled: true + ec2: + resource_attributes: + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + host.id: + enabled: true + host.image.id: + enabled: true + host.name: + enabled: true + host.type: + enabled: true + tags: [] + ecs: + resource_attributes: + aws.ecs.cluster.arn: + enabled: true + aws.ecs.launchtype: + enabled: true + aws.ecs.task.arn: + enabled: true + aws.ecs.task.family: + enabled: true + aws.ecs.task.id: + enabled: true + aws.ecs.task.revision: + enabled: true + aws.log.group.arns: + enabled: true + aws.log.group.names: + enabled: true + aws.log.stream.arns: + enabled: true + aws.log.stream.names: + enabled: true + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + eks: + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + k8s.cluster.name: + enabled: true + elasticbeanstalk: + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + deployment.environment: + enabled: true + service.instance.id: + enabled: true + service.version: + enabled: true + endpoint: "" + gcp: + resource_attributes: + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + faas.id: + enabled: true + faas.instance: + enabled: true + faas.name: + enabled: true + faas.version: + enabled: true + gcp.cloud_run.job.execution: + enabled: true + gcp.cloud_run.job.task_index: + enabled: true + gcp.gce.instance.hostname: + enabled: false + gcp.gce.instance.name: + enabled: false + host.id: + enabled: true + host.name: + enabled: true + host.type: + enabled: true + k8s.cluster.name: + enabled: true + heroku: + resource_attributes: + cloud.provider: + enabled: true + heroku.app.id: + enabled: true + heroku.dyno.id: + enabled: true + heroku.release.commit: + enabled: true + heroku.release.creation_timestamp: + enabled: true + service.instance.id: + enabled: true + service.name: + enabled: true + service.version: + enabled: true + http2_ping_timeout: 0s + http2_read_idle_timeout: 0s + idle_conn_timeout: 1m30s + k8snode: + auth_type: serviceAccount + context: "" + kube_config_path: "" + node_from_env_var: "" + resource_attributes: + k8s.node.name: + enabled: true + k8s.node.uid: + enabled: true + lambda: + resource_attributes: + aws.log.group.names: + enabled: true + aws.log.stream.names: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + faas.instance: + enabled: true + faas.max_memory: + enabled: true + faas.name: + enabled: true + faas.version: + enabled: true + max_idle_conns: 100 + openshift: + address: "" + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + k8s.cluster.name: + enabled: true + tls: + ca_file: "" + cert_file: "" + include_system_ca_certs_pool: false + insecure: false + insecure_skip_verify: false + key_file: "" + max_version: "" + min_version: "" + reload_interval: 0s + server_name_override: "" + token: "" + override: false + proxy_url: "" + read_buffer_size: 0 + system: + hostname_sources: + - os + resource_attributes: + host.arch: + enabled: false + host.cpu.cache.l2.size: + enabled: false + host.cpu.family: + enabled: false + host.cpu.model.id: + enabled: false + host.cpu.model.name: + enabled: false + host.cpu.stepping: + enabled: false + host.cpu.vendor.id: + enabled: false + host.id: + enabled: false + host.ip: + enabled: false + host.mac: + enabled: false + host.name: + enabled: true + os.description: + enabled: false + os.type: + enabled: true + timeout: 15s + tls: + ca_file: "" + cert_file: "" + include_system_ca_certs_pool: false + insecure: false + insecure_skip_verify: false + key_file: "" + max_version: "" + min_version: "" + reload_interval: 0s + server_name_override: "" + write_buffer_size: 0 + transform/containerinsightsjmx: + error_mode: propagate + flatten_data: false + log_statements: [] + metric_statements: + - context: resource + statements: + - keep_keys(attributes, ["ClusterName", "Namespace", "NodeName"]) + - context: metric + statements: + - set(unit, "Bytes") where name == "jvm.memory.heap.used" + - set(unit, "Bytes") where name == "jvm.memory.nonheap.used" + - set(unit, "Bytes") where name == "jvm.memory.pool.used" + - set(unit, "Bytes") where name == "tomcat.traffic.received" + - set(unit, "Bytes") where name == "tomcat.traffic.sent" + - set(unit, "Bytes") where name == "jvm.system.swap.space.total" + - set(unit, "Bytes") where name == "jvm.system.swap.space.free" + - set(unit, "Bytes") where name == "jvm.system.physical.memory.total" + - set(unit, "Bytes") where name == "jvm.system.physical.memory.free" + - set(unit, "Count") where name == "tomcat.sessions" + - set(unit, "Count") where name == "tomcat.rejected_sessions" + - set(unit, "Count") where name == "jvm.threads.count" + - set(unit, "Count") where name == "jvm.threads.daemon" + - set(unit, "Count") where name == "jvm.open_file_descriptor.count" + - set(unit, "Count") where name == "jvm.system.available.processors" + - set(unit, "Count") where name == "tomcat.request_count" + - set(unit, "Count") where name == "tomcat.errors" + - set(unit, "Count") where name == "jvm.classes.loaded" + - set(unit, "Count") where name == "jvm.system.cpu.utilization" + - set(unit, "Count") where name == "jvm.cpu.recent_utilization" + - set(unit, "Milliseconds") where name == "tomcat.processing_time" + trace_statements: [] receivers: - awscontainerinsightreceiver: - accelerated_compute_metrics: true - add_container_name_metric_label: false - add_full_pod_name_metric_label: false - add_service_as_attribute: true - certificate_file_path: "" - cluster_name: TestCluster - collection_interval: 1m0s - container_orchestrator: eks - enable_control_plane_metrics: false - endpoint: "" - host_ip: "" - host_name: "" - imds_retries: 1 - kube_config_path: "" - leader_lock_name: cwagent-clusterleader - leader_lock_using_config_map_only: true - local_mode: true - max_retries: 0 - no_verify_ssl: false - num_workers: 0 - prefer_full_pod_name: false - profile: AmazonCloudWatchAgent - proxy_address: "" - region: us-west-2 - request_timeout_seconds: 0 - resource_arn: "" - role_arn: "" - shared_credentials_file: - - fake-path - otlp/jmx: - protocols: - http: - endpoint: 0.0.0.0:4314 - include_metadata: false - logs_url_path: /v1/logs - max_request_body_size: 0 - metrics_url_path: /v1/metrics - traces_url_path: /v1/traces - tcplog/emf_logs: - encoding: utf-8 - id: tcp_input - listen_address: 0.0.0.0:25888 - operators: [] - retry_on_failure: - enabled: false - initial_interval: 0s - max_elapsed_time: 0s - max_interval: 0s - type: tcp_input - udplog/emf_logs: - encoding: utf-8 - id: udp_input - listen_address: 0.0.0.0:25888 - multiline: - line_end_pattern: .^ - line_start_pattern: "" - omit_pattern: false - operators: [] - retry_on_failure: - enabled: false - initial_interval: 0s - max_elapsed_time: 0s - max_interval: 0s - type: udp_input + awscontainerinsightreceiver: + accelerated_compute_metrics: true + add_container_name_metric_label: false + add_full_pod_name_metric_label: false + add_service_as_attribute: true + certificate_file_path: "" + cluster_name: TestCluster + collection_interval: 1m0s + container_orchestrator: eks + enable_control_plane_metrics: false + endpoint: "" + host_ip: "" + host_name: "" + imds_retries: 1 + kube_config_path: "" + leader_lock_name: cwagent-clusterleader + leader_lock_using_config_map_only: true + local_mode: false + max_retries: 0 + no_verify_ssl: false + num_workers: 0 + prefer_full_pod_name: false + profile: "" + proxy_address: "" + region: us-west-2 + request_timeout_seconds: 0 + resource_arn: "" + role_arn: "" + otlp/jmx: + protocols: + http: + endpoint: 0.0.0.0:4314 + include_metadata: false + logs_url_path: /v1/logs + max_request_body_size: 0 + metrics_url_path: /v1/metrics + traces_url_path: /v1/traces service: - extensions: - - agenthealth/logs - - agenthealth/metrics - pipelines: - logs/emf_logs: - exporters: - - awscloudwatchlogs/emf_logs - processors: - - batch/emf_logs - receivers: - - tcplog/emf_logs - - udplog/emf_logs - metrics/containerinsights: - exporters: - - awsemf/containerinsights - processors: - - batch/containerinsights - receivers: - - awscontainerinsightreceiver - metrics/containerinsightsjmx: - exporters: - - awsemf/containerinsightsjmx - processors: - - filter/containerinsightsjmx - - resourcedetection/containerinsightsjmx - - resource/containerinsightsjmx - - transform/containerinsightsjmx - - metricstransform/containerinsightsjmx - - cumulativetodelta/containerinsightsjmx - receivers: - - otlp/jmx - metrics/jmx/cloudwatch: - exporters: - - awscloudwatch - processors: - - filter/jmx - - cumulativetodelta/jmx - receivers: - - otlp/jmx - telemetry: - logs: - development: false - disable_caller: false - disable_stacktrace: false - encoding: console - level: debug - sampling: - enabled: true - initial: 2 - thereafter: 500 - tick: 10s - metrics: - address: "" - level: None - traces: {} + extensions: + - agenthealth/logs + pipelines: + metrics/containerinsights: + exporters: + - awsemf/containerinsights + processors: + - batch/containerinsights + receivers: + - awscontainerinsightreceiver + metrics/containerinsightsjmx: + exporters: + - awsemf/containerinsightsjmx + processors: + - filter/containerinsightsjmx + - resourcedetection/containerinsightsjmx + - resource/containerinsightsjmx + - transform/containerinsightsjmx + - metricstransform/containerinsightsjmx + - cumulativetodelta/containerinsightsjmx + receivers: + - otlp/jmx + telemetry: + logs: + development: false + disable_caller: false + disable_stacktrace: false + encoding: console + level: debug + sampling: + enabled: true + initial: 2 + thereafter: 500 + tick: 10s + metrics: + address: "" + level: None + traces: {} diff --git a/translator/tocwconfig/tocwconfig_test.go b/translator/tocwconfig/tocwconfig_test.go index b631187bbe..ea3841cf15 100644 --- a/translator/tocwconfig/tocwconfig_test.go +++ b/translator/tocwconfig/tocwconfig_test.go @@ -84,7 +84,7 @@ func TestGenericAppSignalsConfig(t *testing.T) { func TestContainerInsightsJMX(t *testing.T) { resetContext(t) context.CurrentContext().SetRunInContainer(true) - context.CurrentContext().SetMode(config.ModeOnPremise) + context.CurrentContext().SetMode(config.ModeEC2) t.Setenv(config.HOST_NAME, "host_name_from_env") t.Setenv(config.HOST_IP, "127.0.0.1") diff --git a/translator/translate/otel/exporter/awsemf/awsemfjmx_config.yaml b/translator/translate/otel/exporter/awsemf/awsemf_jmx_config.yaml similarity index 100% rename from translator/translate/otel/exporter/awsemf/awsemfjmx_config.yaml rename to translator/translate/otel/exporter/awsemf/awsemf_jmx_config.yaml diff --git a/translator/translate/otel/exporter/awsemf/translator.go b/translator/translate/otel/exporter/awsemf/translator.go index a850bd63ec..4fbb296bad 100644 --- a/translator/translate/otel/exporter/awsemf/translator.go +++ b/translator/translate/otel/exporter/awsemf/translator.go @@ -46,7 +46,7 @@ var appSignalsConfigK8s string //go:embed appsignals_config_generic.yaml var appSignalsConfigGeneric string -//go:embed awsemfjmx_config.yaml +//go:embed awsemf_jmx_config.yaml var awsEmfJmxDefaultConfig string var ( diff --git a/translator/translate/otel/pipeline/containerinsightsjmx/translator.go b/translator/translate/otel/pipeline/containerinsightsjmx/translator.go index c2cf8ca96d..e8bdab12f6 100644 --- a/translator/translate/otel/pipeline/containerinsightsjmx/translator.go +++ b/translator/translate/otel/pipeline/containerinsightsjmx/translator.go @@ -9,15 +9,16 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" + "github.com/aws/amazon-cloudwatch-agent/translator/context" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/awsemf" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/extension/agenthealth" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/cumulativetodeltaprocessor" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/filterprocessor" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/jmxtransformprocessor" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/metricstransformprocessor" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/resourcedetection" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/resourceprocessor" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/transformprocessor" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/otlp" ) @@ -43,9 +44,13 @@ func (t *translator) ID() component.ID { // Translate creates a pipeline for container insights jmx if the logs.metrics_collected.kubernetes // section is present. func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators, error) { - if conf == nil || (!conf.IsSet(jmxKey)) { + if conf == nil || !conf.IsSet(jmxKey) { return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: fmt.Sprint(jmxKey)} } + if !context.CurrentContext().RunInContainer() { + return nil, nil + } + if val, _ := common.GetBool(conf, jmxKey); !val { return nil, nil } @@ -57,7 +62,7 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators filterprocessor.NewTranslator(common.WithName(common.PipelineNameContainerInsightsJmx)), // Filter metrics resourcedetection.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx), // Adds k8s cluster/nodename name resourceprocessor.NewTranslator(common.WithName(common.PipelineNameContainerInsightsJmx)), // Change resource attribute names - jmxtransformprocessor.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx), // Removes attributes that are not of [ClusterName, Namespace] + transformprocessor.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx), // Removes attributes that are not of [ClusterName, Namespace] metricstransformprocessor.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx), // Renames metrics and adds pool and area dimensions cumulativetodeltaprocessor.NewTranslator( common.WithName(common.PipelineNameContainerInsightsJmx), diff --git a/translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go b/translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go index 44ef5e9ff4..16c9d8f0d5 100644 --- a/translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go +++ b/translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go @@ -13,10 +13,12 @@ import ( "go.opentelemetry.io/collector/confmap" "github.com/aws/amazon-cloudwatch-agent/internal/util/collections" + "github.com/aws/amazon-cloudwatch-agent/translator/context" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) func TestTranslator(t *testing.T) { + context.CurrentContext().SetRunInContainer(true) type want struct { pipelineType string receivers []string diff --git a/translator/translate/otel/processor/resourceprocessor/testdata/config.json b/translator/translate/otel/processor/resourceprocessor/testdata/config.json index 46cae7bee5..91cabc00b8 100644 --- a/translator/translate/otel/processor/resourceprocessor/testdata/config.json +++ b/translator/translate/otel/processor/resourceprocessor/testdata/config.json @@ -5,9 +5,10 @@ "logs": { "metrics_collected": { "kubernetes": { - "cluster_name": "TestCluster", + "cluster_name": "TestCluster23", "metrics_collection_interval": 30 } } } + } \ No newline at end of file diff --git a/translator/translate/otel/processor/resourceprocessor/translator.go b/translator/translate/otel/processor/resourceprocessor/translator.go index 7928475ff7..e071fd09d9 100644 --- a/translator/translate/otel/processor/resourceprocessor/translator.go +++ b/translator/translate/otel/processor/resourceprocessor/translator.go @@ -56,20 +56,9 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { cfg := t.factory.CreateDefaultConfig().(*resourceprocessor.Config) var attributes []any - if strings.HasPrefix(t.Name(), common.PipelineNameJmx) { attributes = t.getJMXAttributes(conf) - } - - if len(attributes) == 0 && t.Name() != common.PipelineNameContainerInsightsJmx { - baseKey := common.JmxConfigKey - if t.Index() != -1 { - baseKey = fmt.Sprintf("%s[%d]", baseKey, t.Index()) - } - return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: common.ConfigKey(baseKey, common.AppendDimensionsKey)} - } - - if t.Name() == common.PipelineNameContainerInsightsJmx { + } else if t.Name() == common.PipelineNameContainerInsightsJmx { clusterName, ok := common.GetString(conf, common.ConfigKey(eksKey, "cluster_name")) if ok { @@ -81,7 +70,7 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { }, map[string]any{ "key": "ClusterName", - "value": clusterName, + "value": clusterName, // Ensure 'clusterName' is defined earlier "action": "upsert", }, map[string]any{ @@ -90,7 +79,26 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { "action": "insert", }, } + } else { + + attributes = []any{ + map[string]any{ + "key": "Namespace", + "from_attribute": "k8s.namespace.name", + "action": "insert", + }, + map[string]any{ + "key": "ClusterName", + "value": clusterName, + "action": "upsert", + }, + map[string]any{ + "key": "NodeName", + "from_attribute": "host.name", + "action": "insert", + }, + } attributes = []any{ map[string]any{ "key": "ClusterName", @@ -109,8 +117,15 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { }, } } - } + } + if len(attributes) == 0 { + baseKey := common.JmxConfigKey + if t.Index() != -1 { + baseKey = fmt.Sprintf("%s[%d]", baseKey, t.Index()) + } + return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: common.ConfigKey(baseKey, common.AppendDimensionsKey)} + } c := confmap.NewFromStringMap(map[string]any{ "attributes": attributes, }) @@ -120,7 +135,6 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { } return cfg, nil - } func (t *translator) getJMXAttributes(conf *confmap.Conf) []any { diff --git a/translator/translate/otel/processor/jmxtransformprocessor/testdata/config.json b/translator/translate/otel/processor/transformprocessor/testdata/config.json similarity index 100% rename from translator/translate/otel/processor/jmxtransformprocessor/testdata/config.json rename to translator/translate/otel/processor/transformprocessor/testdata/config.json diff --git a/translator/translate/otel/processor/jmxtransformprocessor/testdata/config.yaml b/translator/translate/otel/processor/transformprocessor/testdata/config.yaml similarity index 100% rename from translator/translate/otel/processor/jmxtransformprocessor/testdata/config.yaml rename to translator/translate/otel/processor/transformprocessor/testdata/config.yaml diff --git a/translator/translate/otel/processor/jmxtransformprocessor/translate.go b/translator/translate/otel/processor/transformprocessor/translate.go similarity index 97% rename from translator/translate/otel/processor/jmxtransformprocessor/translate.go rename to translator/translate/otel/processor/transformprocessor/translate.go index a90651ba16..6f1d9f2740 100644 --- a/translator/translate/otel/processor/jmxtransformprocessor/translate.go +++ b/translator/translate/otel/processor/transformprocessor/translate.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package jmxtransformprocessor +package transformprocessor import ( _ "embed" diff --git a/translator/translate/otel/processor/jmxtransformprocessor/translator_test.go b/translator/translate/otel/processor/transformprocessor/translator_test.go similarity index 98% rename from translator/translate/otel/processor/jmxtransformprocessor/translator_test.go rename to translator/translate/otel/processor/transformprocessor/translator_test.go index 482ec01396..cfd8f196d5 100644 --- a/translator/translate/otel/processor/jmxtransformprocessor/translator_test.go +++ b/translator/translate/otel/processor/transformprocessor/translator_test.go @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: MIT -package jmxtransformprocessor +package transformprocessor import ( _ "embed" From 185c0823968a7ab76a610bc611027d0dfc2d8e0c Mon Sep 17 00:00:00 2001 From: Paramadon Date: Thu, 10 Oct 2024 19:30:17 -0400 Subject: [PATCH 14/19] removing resource detection processor --- .../sampleConfig/container_insights_jmx.yaml | 291 +----------------- .../containerinsightsjmx/translator.go | 2 - .../containerinsightsjmx/translator_test.go | 2 +- .../resourcedetection/configs/config.json | 13 - .../resourcedetection/configs/jmx_config.yaml | 9 - .../processor/resourcedetection/translator.go | 14 - .../resourcedetection/translator_test.go | 18 -- .../processor/resourceprocessor/translator.go | 79 ++--- 8 files changed, 27 insertions(+), 401 deletions(-) delete mode 100644 translator/translate/otel/processor/resourcedetection/configs/config.json delete mode 100644 translator/translate/otel/processor/resourcedetection/configs/jmx_config.yaml diff --git a/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml b/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml index a01450fc5d..e3921a646d 100644 --- a/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml +++ b/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml @@ -394,297 +394,11 @@ processors: value: TestCluster - action: insert converted_type: "" - from_attribute: host.name + from_attribute: "" from_context: "" key: NodeName pattern: "" - resourcedetection/containerinsightsjmx: - aks: - resource_attributes: - cloud.platform: - enabled: true - cloud.provider: - enabled: true - k8s.cluster.name: - enabled: false - azure: - resource_attributes: - azure.resourcegroup.name: - enabled: true - azure.vm.name: - enabled: true - azure.vm.scaleset.name: - enabled: true - azure.vm.size: - enabled: true - cloud.account.id: - enabled: true - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - host.id: - enabled: true - host.name: - enabled: true - tags: [] - compression: "" - consul: - address: "" - datacenter: "" - namespace: "" - resource_attributes: - cloud.region: - enabled: true - host.id: - enabled: true - host.name: - enabled: true - token_file: "" - detectors: - - env - - eks - - system - disable_keep_alives: false - docker: - resource_attributes: - host.name: - enabled: true - os.type: - enabled: true - ec2: - resource_attributes: - cloud.account.id: - enabled: true - cloud.availability_zone: - enabled: true - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - host.id: - enabled: true - host.image.id: - enabled: true - host.name: - enabled: true - host.type: - enabled: true - tags: [] - ecs: - resource_attributes: - aws.ecs.cluster.arn: - enabled: true - aws.ecs.launchtype: - enabled: true - aws.ecs.task.arn: - enabled: true - aws.ecs.task.family: - enabled: true - aws.ecs.task.id: - enabled: true - aws.ecs.task.revision: - enabled: true - aws.log.group.arns: - enabled: true - aws.log.group.names: - enabled: true - aws.log.stream.arns: - enabled: true - aws.log.stream.names: - enabled: true - cloud.account.id: - enabled: true - cloud.availability_zone: - enabled: true - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - eks: - resource_attributes: - cloud.platform: - enabled: true - cloud.provider: - enabled: true - k8s.cluster.name: - enabled: true - elasticbeanstalk: - resource_attributes: - cloud.platform: - enabled: true - cloud.provider: - enabled: true - deployment.environment: - enabled: true - service.instance.id: - enabled: true - service.version: - enabled: true - endpoint: "" - gcp: - resource_attributes: - cloud.account.id: - enabled: true - cloud.availability_zone: - enabled: true - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - faas.id: - enabled: true - faas.instance: - enabled: true - faas.name: - enabled: true - faas.version: - enabled: true - gcp.cloud_run.job.execution: - enabled: true - gcp.cloud_run.job.task_index: - enabled: true - gcp.gce.instance.hostname: - enabled: false - gcp.gce.instance.name: - enabled: false - host.id: - enabled: true - host.name: - enabled: true - host.type: - enabled: true - k8s.cluster.name: - enabled: true - heroku: - resource_attributes: - cloud.provider: - enabled: true - heroku.app.id: - enabled: true - heroku.dyno.id: - enabled: true - heroku.release.commit: - enabled: true - heroku.release.creation_timestamp: - enabled: true - service.instance.id: - enabled: true - service.name: - enabled: true - service.version: - enabled: true - http2_ping_timeout: 0s - http2_read_idle_timeout: 0s - idle_conn_timeout: 1m30s - k8snode: - auth_type: serviceAccount - context: "" - kube_config_path: "" - node_from_env_var: "" - resource_attributes: - k8s.node.name: - enabled: true - k8s.node.uid: - enabled: true - lambda: - resource_attributes: - aws.log.group.names: - enabled: true - aws.log.stream.names: - enabled: true - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - faas.instance: - enabled: true - faas.max_memory: - enabled: true - faas.name: - enabled: true - faas.version: - enabled: true - max_idle_conns: 100 - openshift: - address: "" - resource_attributes: - cloud.platform: - enabled: true - cloud.provider: - enabled: true - cloud.region: - enabled: true - k8s.cluster.name: - enabled: true - tls: - ca_file: "" - cert_file: "" - include_system_ca_certs_pool: false - insecure: false - insecure_skip_verify: false - key_file: "" - max_version: "" - min_version: "" - reload_interval: 0s - server_name_override: "" - token: "" - override: false - proxy_url: "" - read_buffer_size: 0 - system: - hostname_sources: - - os - resource_attributes: - host.arch: - enabled: false - host.cpu.cache.l2.size: - enabled: false - host.cpu.family: - enabled: false - host.cpu.model.id: - enabled: false - host.cpu.model.name: - enabled: false - host.cpu.stepping: - enabled: false - host.cpu.vendor.id: - enabled: false - host.id: - enabled: false - host.ip: - enabled: false - host.mac: - enabled: false - host.name: - enabled: true - os.description: - enabled: false - os.type: - enabled: true - timeout: 15s - tls: - ca_file: "" - cert_file: "" - include_system_ca_certs_pool: false - insecure: false - insecure_skip_verify: false - key_file: "" - max_version: "" - min_version: "" - reload_interval: 0s - server_name_override: "" - write_buffer_size: 0 + value: "host_name_from_env" transform/containerinsightsjmx: error_mode: propagate flatten_data: false @@ -771,7 +485,6 @@ service: - awsemf/containerinsightsjmx processors: - filter/containerinsightsjmx - - resourcedetection/containerinsightsjmx - resource/containerinsightsjmx - transform/containerinsightsjmx - metricstransform/containerinsightsjmx diff --git a/translator/translate/otel/pipeline/containerinsightsjmx/translator.go b/translator/translate/otel/pipeline/containerinsightsjmx/translator.go index e8bdab12f6..8390ead747 100644 --- a/translator/translate/otel/pipeline/containerinsightsjmx/translator.go +++ b/translator/translate/otel/pipeline/containerinsightsjmx/translator.go @@ -16,7 +16,6 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/cumulativetodeltaprocessor" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/filterprocessor" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/metricstransformprocessor" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/resourcedetection" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/resourceprocessor" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/transformprocessor" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/otlp" @@ -60,7 +59,6 @@ func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators ), Processors: common.NewTranslatorMap( filterprocessor.NewTranslator(common.WithName(common.PipelineNameContainerInsightsJmx)), // Filter metrics - resourcedetection.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx), // Adds k8s cluster/nodename name resourceprocessor.NewTranslator(common.WithName(common.PipelineNameContainerInsightsJmx)), // Change resource attribute names transformprocessor.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx), // Removes attributes that are not of [ClusterName, Namespace] metricstransformprocessor.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx), // Renames metrics and adds pool and area dimensions diff --git a/translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go b/translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go index 16c9d8f0d5..72ad074f49 100644 --- a/translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go +++ b/translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go @@ -50,7 +50,7 @@ func TestTranslator(t *testing.T) { want: &want{ pipelineType: "metrics/containerinsightsjmx", receivers: []string{"otlp/jmx"}, - processors: []string{"filter/containerinsightsjmx", "resourcedetection/containerinsightsjmx", "resource/containerinsightsjmx", "transform/containerinsightsjmx", "metricstransform/containerinsightsjmx", "cumulativetodelta/containerinsightsjmx"}, + processors: []string{"filter/containerinsightsjmx", "resource/containerinsightsjmx", "transform/containerinsightsjmx", "metricstransform/containerinsightsjmx", "cumulativetodelta/containerinsightsjmx"}, exporters: []string{"awsemf/containerinsightsjmx"}, extensions: []string{"agenthealth/logs"}, }, diff --git a/translator/translate/otel/processor/resourcedetection/configs/config.json b/translator/translate/otel/processor/resourcedetection/configs/config.json deleted file mode 100644 index 46cae7bee5..0000000000 --- a/translator/translate/otel/processor/resourcedetection/configs/config.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "agent": { - "debug": true - }, - "logs": { - "metrics_collected": { - "kubernetes": { - "cluster_name": "TestCluster", - "metrics_collection_interval": 30 - } - } - } -} \ No newline at end of file diff --git a/translator/translate/otel/processor/resourcedetection/configs/jmx_config.yaml b/translator/translate/otel/processor/resourcedetection/configs/jmx_config.yaml deleted file mode 100644 index 116bcb65d6..0000000000 --- a/translator/translate/otel/processor/resourcedetection/configs/jmx_config.yaml +++ /dev/null @@ -1,9 +0,0 @@ -detectors: [env, eks, system] -timeout: 15s -override: false -eks: - resource_attributes: - k8s.cluster.name: - enabled: true -system: - hostname_sources: ["os"] \ No newline at end of file diff --git a/translator/translate/otel/processor/resourcedetection/translator.go b/translator/translate/otel/processor/resourcedetection/translator.go index d658ffe080..454e59ed79 100644 --- a/translator/translate/otel/processor/resourcedetection/translator.go +++ b/translator/translate/otel/processor/resourcedetection/translator.go @@ -23,9 +23,6 @@ var appSignalsDefaultResourceDetectionConfig string //go:embed configs/ecs_config.yaml var appSignalsECSResourceDetectionConfig string -//go:embed configs/jmx_config.yaml -var resourceDetectionJMX string - type translator struct { name string dataType component.DataType @@ -52,14 +49,6 @@ func WithDataType(dataType component.DataType) Option { var _ common.Translator[component.Config] = (*translator)(nil) -func NewTranslatorWithName(name string, opts ...Option) common.Translator[component.Config] { - t := &translator{name: name, factory: resourcedetectionprocessor.NewFactory()} - for _, opt := range opts { - opt.apply(t) - } - return t -} - func NewTranslator(opts ...Option) common.Translator[component.Config] { t := &translator{factory: resourcedetectionprocessor.NewFactory()} for _, opt := range opts { @@ -75,9 +64,6 @@ func (t *translator) ID() component.ID { func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { cfg := t.factory.CreateDefaultConfig().(*resourcedetectionprocessor.Config) - if t.name == common.PipelineNameContainerInsightsJmx { - return common.GetYamlFileToYamlConfig(cfg, resourceDetectionJMX) - } mode := context.CurrentContext().KubernetesMode() if mode == "" { mode = context.CurrentContext().Mode() diff --git a/translator/translate/otel/processor/resourcedetection/translator_test.go b/translator/translate/otel/processor/resourcedetection/translator_test.go index 23fa4599f5..6de59538b0 100644 --- a/translator/translate/otel/processor/resourcedetection/translator_test.go +++ b/translator/translate/otel/processor/resourcedetection/translator_test.go @@ -4,7 +4,6 @@ package resourcedetection import ( - "path/filepath" "testing" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor" @@ -13,10 +12,8 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" - "github.com/aws/amazon-cloudwatch-agent/internal/util/testutil" translatorconfig "github.com/aws/amazon-cloudwatch-agent/translator/config" "github.com/aws/amazon-cloudwatch-agent/translator/context" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" "github.com/aws/amazon-cloudwatch-agent/translator/util/ecsutil" ) @@ -145,18 +142,3 @@ func TestTranslate(t *testing.T) { }) } } - -func TestContainerInsightsJmx(t *testing.T) { - transl := NewTranslatorWithName(common.PipelineNameContainerInsightsJmx).(*translator) - expectedCfg := transl.factory.CreateDefaultConfig().(*resourcedetectionprocessor.Config) - c := testutil.GetConf(t, filepath.Join("configs", "jmx_config.yaml")) - require.NoError(t, c.Unmarshal(&expectedCfg)) - - conf := confmap.NewFromStringMap(testutil.GetJson(t, filepath.Join("configs", "config.json"))) - translatedCfg, err := transl.Translate(conf) - assert.NoError(t, err) - actualCfg, ok := translatedCfg.(*resourcedetectionprocessor.Config) - assert.True(t, ok) - assert.Equal(t, len(actualCfg.Detectors), len(expectedCfg.Detectors)) - -} diff --git a/translator/translate/otel/processor/resourceprocessor/translator.go b/translator/translate/otel/processor/resourceprocessor/translator.go index e071fd09d9..efe25f2712 100644 --- a/translator/translate/otel/processor/resourceprocessor/translator.go +++ b/translator/translate/otel/processor/resourceprocessor/translator.go @@ -5,9 +5,13 @@ package resourceprocessor import ( "fmt" + "os" "strconv" "strings" + "github.com/aws/amazon-cloudwatch-agent/translator/config" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/logs/util" + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" @@ -61,61 +65,26 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { } else if t.Name() == common.PipelineNameContainerInsightsJmx { clusterName, ok := common.GetString(conf, common.ConfigKey(eksKey, "cluster_name")) - if ok { - attributes = []any{ - map[string]any{ - "key": "Namespace", - "from_attribute": "k8s.namespace.name", - "action": "insert", - }, - map[string]any{ - "key": "ClusterName", - "value": clusterName, // Ensure 'clusterName' is defined earlier - "action": "upsert", - }, - map[string]any{ - "key": "NodeName", - "from_attribute": "host.name", - "action": "insert", - }, - } - - } else { - - attributes = []any{ - map[string]any{ - "key": "Namespace", - "from_attribute": "k8s.namespace.name", - "action": "insert", - }, - map[string]any{ - "key": "ClusterName", - "value": clusterName, - "action": "upsert", - }, - map[string]any{ - "key": "NodeName", - "from_attribute": "host.name", - "action": "insert", - }, - } - attributes = []any{ - map[string]any{ - "key": "ClusterName", - "from_attribute": "k8s.cluster.name", - "action": "insert", - }, - map[string]any{ - "key": "Namespace", - "from_attribute": "k8s.namespace.name", - "action": "insert", - }, - map[string]any{ - "key": "NodeName", - "from_attribute": "host.name", - "action": "insert", - }, - } + if !ok { + clusterName = util.GetClusterNameFromEc2Tagger() + } + nodeName := os.Getenv(config.HOST_NAME) + attributes = []any{ + map[string]any{ + "key": "Namespace", + "from_attribute": "k8s.namespace.name", + "action": "insert", + }, + map[string]any{ + "key": "ClusterName", + "value": clusterName, // Ensure 'clusterName' is defined earlier + "action": "upsert", + }, + map[string]any{ + "key": "NodeName", + "value": nodeName, + "action": "insert", + }, } } From e773dfb0475ee74e599da8c70a2858276fce64aa Mon Sep 17 00:00:00 2001 From: Paramadon Date: Thu, 10 Oct 2024 19:35:05 -0400 Subject: [PATCH 15/19] fixing awsemf translator syntax --- translator/translate/otel/exporter/awsemf/translator.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/translator/translate/otel/exporter/awsemf/translator.go b/translator/translate/otel/exporter/awsemf/translator.go index 4fbb296bad..1f3ad3b749 100644 --- a/translator/translate/otel/exporter/awsemf/translator.go +++ b/translator/translate/otel/exporter/awsemf/translator.go @@ -129,11 +129,15 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { if err := setAppSignalsFields(c, cfg); err != nil { return nil, err } + } else if t.isCiJMX(c) { + if err := setCiJmx(); err != nil { + return nil, err + } } else if isEcs(c) { if err := setEcsFields(c, cfg); err != nil { return nil, err } - } else if isKubernetes(c) && !t.isCiJMX(c) { + } else if isKubernetes(c) { if err := setKubernetesFields(c, cfg); err != nil { return nil, err } @@ -212,6 +216,9 @@ func setKubernetesFields(conf *confmap.Conf, cfg *awsemfexporter.Config) error { return nil } +func setCiJmx() error { + return nil +} func setPrometheusFields(conf *confmap.Conf, cfg *awsemfexporter.Config) error { setDisableMetricExtraction(prometheusBasePathKey, conf, cfg) From 48212f884c60bb7e758f3b23e450f6df1650591d Mon Sep 17 00:00:00 2001 From: Paramadon Date: Thu, 10 Oct 2024 19:42:13 -0400 Subject: [PATCH 16/19] fixing lint --- .../translate/otel/processor/resourceprocessor/translator.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/translator/translate/otel/processor/resourceprocessor/translator.go b/translator/translate/otel/processor/resourceprocessor/translator.go index efe25f2712..83ff61af83 100644 --- a/translator/translate/otel/processor/resourceprocessor/translator.go +++ b/translator/translate/otel/processor/resourceprocessor/translator.go @@ -9,15 +9,14 @@ import ( "strconv" "strings" - "github.com/aws/amazon-cloudwatch-agent/translator/config" - "github.com/aws/amazon-cloudwatch-agent/translator/translate/logs/util" - "github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" "go.opentelemetry.io/collector/processor" + "github.com/aws/amazon-cloudwatch-agent/translator/config" "github.com/aws/amazon-cloudwatch-agent/translator/context" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/logs/util" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) From da172b8fe6791a86e2ab319582eb34ad41cde4f3 Mon Sep 17 00:00:00 2001 From: Paramadon Date: Thu, 10 Oct 2024 19:50:15 -0400 Subject: [PATCH 17/19] reverting util.go --- .../translate/otel/receiver/awscontainerinsight/utils.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/translator/translate/otel/receiver/awscontainerinsight/utils.go b/translator/translate/otel/receiver/awscontainerinsight/utils.go index 36a31fd209..721951b056 100644 --- a/translator/translate/otel/receiver/awscontainerinsight/utils.go +++ b/translator/translate/otel/receiver/awscontainerinsight/utils.go @@ -24,12 +24,6 @@ func EnhancedContainerInsightsEnabled(conf *confmap.Conf) bool { return isSet } -func JmxEnabled(conf *confmap.Conf) bool { - isEnabled, _ := common.GetBool(conf, common.ConfigKey(common.MetricsKey, common.MetricsCollectedKey, common.JmxKey)) - return isEnabled - -} - func AcceleratedComputeMetricsEnabled(conf *confmap.Conf) bool { return common.GetOrDefaultBool(conf, common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.KubernetesKey, common.EnableAcceleratedComputeMetric), true) } From 806f9841f34acbcc8fd084222ac98616b7b76ed1 Mon Sep 17 00:00:00 2001 From: Paramadon Date: Thu, 10 Oct 2024 20:03:43 -0400 Subject: [PATCH 18/19] fixing formats of files --- .../{testdata => }/ContainerInsightsJmxConfig.yaml | 0 .../translate/otel/processor/filterprocessor/translator.go | 2 +- .../{configs => testdata}/config.json | 0 .../processor/metricstransformprocessor/translator_test.go | 2 +- .../processor/transformprocessor/{testdata => }/config.yaml | 0 .../otel/processor/transformprocessor/translate.go | 6 +----- .../otel/processor/transformprocessor/translator_test.go | 2 +- 7 files changed, 4 insertions(+), 8 deletions(-) rename translator/translate/otel/processor/filterprocessor/{testdata => }/ContainerInsightsJmxConfig.yaml (100%) rename translator/translate/otel/processor/metricstransformprocessor/{configs => testdata}/config.json (100%) rename translator/translate/otel/processor/transformprocessor/{testdata => }/config.yaml (100%) diff --git a/translator/translate/otel/processor/filterprocessor/testdata/ContainerInsightsJmxConfig.yaml b/translator/translate/otel/processor/filterprocessor/ContainerInsightsJmxConfig.yaml similarity index 100% rename from translator/translate/otel/processor/filterprocessor/testdata/ContainerInsightsJmxConfig.yaml rename to translator/translate/otel/processor/filterprocessor/ContainerInsightsJmxConfig.yaml diff --git a/translator/translate/otel/processor/filterprocessor/translator.go b/translator/translate/otel/processor/filterprocessor/translator.go index b56bf3468c..c322ac49a1 100644 --- a/translator/translate/otel/processor/filterprocessor/translator.go +++ b/translator/translate/otel/processor/filterprocessor/translator.go @@ -20,7 +20,7 @@ const ( matchTypeStrict = "strict" ) -//go:embed testdata/ContainerInsightsJmxConfig.yaml +//go:embed ContainerInsightsJmxConfig.yaml var ContainerInsightsJmxConfig string type translator struct { diff --git a/translator/translate/otel/processor/metricstransformprocessor/configs/config.json b/translator/translate/otel/processor/metricstransformprocessor/testdata/config.json similarity index 100% rename from translator/translate/otel/processor/metricstransformprocessor/configs/config.json rename to translator/translate/otel/processor/metricstransformprocessor/testdata/config.json diff --git a/translator/translate/otel/processor/metricstransformprocessor/translator_test.go b/translator/translate/otel/processor/metricstransformprocessor/translator_test.go index 4a395687d9..a4c11b4ca9 100644 --- a/translator/translate/otel/processor/metricstransformprocessor/translator_test.go +++ b/translator/translate/otel/processor/metricstransformprocessor/translator_test.go @@ -66,7 +66,7 @@ func TestContainerInsightsJmx(t *testing.T) { c := testutil.GetConf(t, filepath.Join("configs", "config.yaml")) require.NoError(t, c.Unmarshal(&expectedCfg)) - conf := confmap.NewFromStringMap(testutil.GetJson(t, filepath.Join("configs", "config.json"))) + conf := confmap.NewFromStringMap(testutil.GetJson(t, filepath.Join("testdata", "config.json"))) translatedCfg, err := transl.Translate(conf) assert.NoError(t, err) actualCfg, ok := translatedCfg.(*metricstransformprocessor.Config) diff --git a/translator/translate/otel/processor/transformprocessor/testdata/config.yaml b/translator/translate/otel/processor/transformprocessor/config.yaml similarity index 100% rename from translator/translate/otel/processor/transformprocessor/testdata/config.yaml rename to translator/translate/otel/processor/transformprocessor/config.yaml diff --git a/translator/translate/otel/processor/transformprocessor/translate.go b/translator/translate/otel/processor/transformprocessor/translate.go index 6f1d9f2740..1910f26899 100644 --- a/translator/translate/otel/processor/transformprocessor/translate.go +++ b/translator/translate/otel/processor/transformprocessor/translate.go @@ -14,17 +14,13 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) -//go:embed testdata/config.yaml +//go:embed config.yaml var transformJmxConfig string type translator struct { name string factory processor.Factory } -type Context struct { - name string - statements []string -} var _ common.Translator[component.Config] = (*translator)(nil) diff --git a/translator/translate/otel/processor/transformprocessor/translator_test.go b/translator/translate/otel/processor/transformprocessor/translator_test.go index cfd8f196d5..b3fe7e9f67 100644 --- a/translator/translate/otel/processor/transformprocessor/translator_test.go +++ b/translator/translate/otel/processor/transformprocessor/translator_test.go @@ -66,7 +66,7 @@ func TestTranslator(t *testing.T) { func TestContainerInsightsJmx(t *testing.T) { transl := NewTranslatorWithName(common.PipelineNameContainerInsightsJmx).(*translator) expectedCfg := transl.factory.CreateDefaultConfig().(*transformprocessor.Config) - c := testutil.GetConf(t, filepath.Join("testdata", "config.yaml")) + c := testutil.GetConf(t, "config.yaml") require.NoError(t, c.Unmarshal(&expectedCfg)) conf := confmap.NewFromStringMap(testutil.GetJson(t, filepath.Join("testdata", "config.json"))) From 8f7fcf342b8e80557dca6a1a78bce9b07abe1fec Mon Sep 17 00:00:00 2001 From: Paramadon Date: Thu, 10 Oct 2024 20:34:55 -0400 Subject: [PATCH 19/19] cleaning code --- cmd/config-translator/translator_test.go | 4 +- .../invalidContainerInsightsJmx.json | 14 ----- translator/config/schema.json | 4 ++ .../sampleConfig/container_insights_jmx.yaml | 6 +-- .../otel/exporter/awsemf/translator.go | 8 +-- .../containerinsightsjmx/translator.go | 4 +- ...sJmxConfig.yaml => filter_jmx_config.yaml} | 2 +- .../processor/filterprocessor/translator.go | 6 +-- .../filterprocessor/translator_test.go | 2 +- ....yaml => metricstransform_jmx_config.yaml} | 2 +- .../metricstransformprocessor/translator.go | 2 +- .../translator_test.go | 2 +- .../processor/resourceprocessor/translator.go | 53 ++++++++++--------- ...{config.yaml => transform_jmx_config.yaml} | 16 +++--- .../processor/transformprocessor/translate.go | 2 +- .../transformprocessor/translator_test.go | 2 +- 16 files changed, 59 insertions(+), 70 deletions(-) delete mode 100644 translator/config/sampleSchema/invalidContainerInsightsJmx.json rename translator/translate/otel/processor/filterprocessor/{ContainerInsightsJmxConfig.yaml => filter_jmx_config.yaml} (95%) rename translator/translate/otel/processor/metricstransformprocessor/{configs/config.yaml => metricstransform_jmx_config.yaml} (98%) rename translator/translate/otel/processor/transformprocessor/{config.yaml => transform_jmx_config.yaml} (95%) diff --git a/cmd/config-translator/translator_test.go b/cmd/config-translator/translator_test.go index d176a70692..b276fc4632 100644 --- a/cmd/config-translator/translator_test.go +++ b/cmd/config-translator/translator_test.go @@ -187,10 +187,8 @@ func TestMetricsDestinationsConfig(t *testing.T) { expectedErrorMap["required"] = 1 checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidMetricsDestinations.json", false, expectedErrorMap) } -func TestJmxConfig(t *testing.T) { +func TestContainerInsightsJmxConfig(t *testing.T) { checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/validContainerInsightsJmx.json", true, map[string]int{}) - expectedErrorMap := map[string]int{"additional_property_not_allowed": 1, "number_any_of": 1, "required": 1} - checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidContainerInsightsJmx.json", false, expectedErrorMap) } // Validate all sampleConfig files schema diff --git a/translator/config/sampleSchema/invalidContainerInsightsJmx.json b/translator/config/sampleSchema/invalidContainerInsightsJmx.json deleted file mode 100644 index 214fd5d7a1..0000000000 --- a/translator/config/sampleSchema/invalidContainerInsightsJmx.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "agent": { - "region": "us-west-2" - }, - "logs": { - "metrics": { - "emf": { - }, - "kubernetes": { - } - }, - "force_flush_interval": 5 - } -} \ No newline at end of file diff --git a/translator/config/schema.json b/translator/config/schema.json index f69c05be63..0d091cb8a7 100644 --- a/translator/config/schema.json +++ b/translator/config/schema.json @@ -768,6 +768,10 @@ "minLength": 1, "maxLength": 512 }, + "jmx_container_insights": { + "description": "Enable JMX Container Insights metrics", + "type": "boolean" + }, "metrics_collection_interval": { "$ref": "#/definitions/timeIntervalDefinition" }, diff --git a/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml b/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml index e3921a646d..f9dc971879 100644 --- a/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml +++ b/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml @@ -215,7 +215,7 @@ processors: - jvm.open_file_descriptor.count - jvm.system.available.processors - jvm.threads.count - - jvm.threads.daemon + - jvm.daemon_threads.count - tomcat.sessions - tomcat.rejected_sessions - tomcat.traffic.received @@ -331,7 +331,7 @@ processors: submatch_case: "" - action: update aggregation_type: "" - include: jvm.threads.daemon + include: jvm.daemon_threads.count match_type: strict new_name: jvm_threads_daemon submatch_case: "" @@ -421,7 +421,7 @@ processors: - set(unit, "Count") where name == "tomcat.sessions" - set(unit, "Count") where name == "tomcat.rejected_sessions" - set(unit, "Count") where name == "jvm.threads.count" - - set(unit, "Count") where name == "jvm.threads.daemon" + - set(unit, "Count") where name == "jvm.daemon_threads.count" - set(unit, "Count") where name == "jvm.open_file_descriptor.count" - set(unit, "Count") where name == "jvm.system.available.processors" - set(unit, "Count") where name == "tomcat.request_count" diff --git a/translator/translate/otel/exporter/awsemf/translator.go b/translator/translate/otel/exporter/awsemf/translator.go index 1f3ad3b749..7ce0af0472 100644 --- a/translator/translate/otel/exporter/awsemf/translator.go +++ b/translator/translate/otel/exporter/awsemf/translator.go @@ -47,7 +47,7 @@ var appSignalsConfigK8s string var appSignalsConfigGeneric string //go:embed awsemf_jmx_config.yaml -var awsEmfJmxDefaultConfig string +var defaultJmxConfig string var ( ecsBasePathKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.ECSKey) @@ -86,7 +86,7 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { if t.isAppSignals(c) { defaultConfig = getAppSignalsConfig() } else if t.isCiJMX(c) { - defaultConfig = awsEmfJmxDefaultConfig + defaultConfig = defaultJmxConfig } else if isEcs(c) { defaultConfig = defaultEcsConfig } else if isKubernetes(c) { @@ -130,7 +130,7 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { return nil, err } } else if t.isCiJMX(c) { - if err := setCiJmx(); err != nil { + if err := setCiJmxFields(); err != nil { return nil, err } } else if isEcs(c) { @@ -216,7 +216,7 @@ func setKubernetesFields(conf *confmap.Conf, cfg *awsemfexporter.Config) error { return nil } -func setCiJmx() error { +func setCiJmxFields() error { return nil } func setPrometheusFields(conf *confmap.Conf, cfg *awsemfexporter.Config) error { diff --git a/translator/translate/otel/pipeline/containerinsightsjmx/translator.go b/translator/translate/otel/pipeline/containerinsightsjmx/translator.go index 8390ead747..e34e30c92e 100644 --- a/translator/translate/otel/pipeline/containerinsightsjmx/translator.go +++ b/translator/translate/otel/pipeline/containerinsightsjmx/translator.go @@ -4,8 +4,6 @@ package containerinsightsjmx import ( - "fmt" - "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" @@ -44,7 +42,7 @@ func (t *translator) ID() component.ID { // section is present. func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators, error) { if conf == nil || !conf.IsSet(jmxKey) { - return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: fmt.Sprint(jmxKey)} + return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: jmxKey} } if !context.CurrentContext().RunInContainer() { return nil, nil diff --git a/translator/translate/otel/processor/filterprocessor/ContainerInsightsJmxConfig.yaml b/translator/translate/otel/processor/filterprocessor/filter_jmx_config.yaml similarity index 95% rename from translator/translate/otel/processor/filterprocessor/ContainerInsightsJmxConfig.yaml rename to translator/translate/otel/processor/filterprocessor/filter_jmx_config.yaml index 9a79d6a3c1..34e982d508 100644 --- a/translator/translate/otel/processor/filterprocessor/ContainerInsightsJmxConfig.yaml +++ b/translator/translate/otel/processor/filterprocessor/filter_jmx_config.yaml @@ -15,7 +15,7 @@ metrics: - jvm.open_file_descriptor.count - jvm.system.available.processors - jvm.threads.count - - jvm.threads.daemon + - jvm.daemon_threads.count - tomcat.sessions - tomcat.rejected_sessions - tomcat.traffic.received diff --git a/translator/translate/otel/processor/filterprocessor/translator.go b/translator/translate/otel/processor/filterprocessor/translator.go index c322ac49a1..b6faafa234 100644 --- a/translator/translate/otel/processor/filterprocessor/translator.go +++ b/translator/translate/otel/processor/filterprocessor/translator.go @@ -20,8 +20,8 @@ const ( matchTypeStrict = "strict" ) -//go:embed ContainerInsightsJmxConfig.yaml -var ContainerInsightsJmxConfig string +//go:embed filter_jmx_config.yaml +var containerInsightsJmxConfig string type translator struct { common.NameProvider @@ -58,7 +58,7 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { cfg := t.factory.CreateDefaultConfig().(*filterprocessor.Config) if t.Name() == common.PipelineNameContainerInsightsJmx { - return common.GetYamlFileToYamlConfig(cfg, ContainerInsightsJmxConfig) + return common.GetYamlFileToYamlConfig(cfg, containerInsightsJmxConfig) } jmxMap := common.GetIndexedMap(conf, common.JmxConfigKey, t.Index()) diff --git a/translator/translate/otel/processor/filterprocessor/translator_test.go b/translator/translate/otel/processor/filterprocessor/translator_test.go index 7ecbadc507..d87fff55ff 100644 --- a/translator/translate/otel/processor/filterprocessor/translator_test.go +++ b/translator/translate/otel/processor/filterprocessor/translator_test.go @@ -134,7 +134,7 @@ func TestTranslator(t *testing.T) { func TestContainerInsightsJmx(t *testing.T) { transl := NewTranslator(common.WithName(common.PipelineNameContainerInsightsJmx)).(*translator) expectedCfg := transl.factory.CreateDefaultConfig().(*filterprocessor.Config) - c := testutil.GetConf(t, filepath.Join("testdata", "ContainerInsightsJmxConfig.yaml")) + c := testutil.GetConf(t, "filter_jmx_config.yaml") require.NoError(t, c.Unmarshal(&expectedCfg)) conf := confmap.NewFromStringMap(testutil.GetJson(t, filepath.Join("testdata", "config.json"))) diff --git a/translator/translate/otel/processor/metricstransformprocessor/configs/config.yaml b/translator/translate/otel/processor/metricstransformprocessor/metricstransform_jmx_config.yaml similarity index 98% rename from translator/translate/otel/processor/metricstransformprocessor/configs/config.yaml rename to translator/translate/otel/processor/metricstransformprocessor/metricstransform_jmx_config.yaml index 4034ad0a05..caeb6bb9c9 100644 --- a/translator/translate/otel/processor/metricstransformprocessor/configs/config.yaml +++ b/translator/translate/otel/processor/metricstransformprocessor/metricstransform_jmx_config.yaml @@ -85,7 +85,7 @@ transforms: new_name: jvm_threads_current - - include: jvm.threads.daemon + - include: jvm.daemon_threads.count match_type: strict action: update new_name: jvm_threads_daemon diff --git a/translator/translate/otel/processor/metricstransformprocessor/translator.go b/translator/translate/otel/processor/metricstransformprocessor/translator.go index fc6a843472..dfbd37f788 100644 --- a/translator/translate/otel/processor/metricstransformprocessor/translator.go +++ b/translator/translate/otel/processor/metricstransformprocessor/translator.go @@ -17,7 +17,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/awscontainerinsight" ) -//go:embed configs/config.yaml +//go:embed metricstransform_jmx_config.yaml var metricTransformJmxConfig string var metricDuplicateTypes = []string{ diff --git a/translator/translate/otel/processor/metricstransformprocessor/translator_test.go b/translator/translate/otel/processor/metricstransformprocessor/translator_test.go index a4c11b4ca9..f6357d3af9 100644 --- a/translator/translate/otel/processor/metricstransformprocessor/translator_test.go +++ b/translator/translate/otel/processor/metricstransformprocessor/translator_test.go @@ -63,7 +63,7 @@ func TestTranslator(t *testing.T) { func TestContainerInsightsJmx(t *testing.T) { transl := NewTranslatorWithName(common.PipelineNameContainerInsightsJmx).(*translator) expectedCfg := transl.factory.CreateDefaultConfig().(*metricstransformprocessor.Config) - c := testutil.GetConf(t, filepath.Join("configs", "config.yaml")) + c := testutil.GetConf(t, "metricstransform_jmx_config.yaml") require.NoError(t, c.Unmarshal(&expectedCfg)) conf := confmap.NewFromStringMap(testutil.GetJson(t, filepath.Join("testdata", "config.json"))) diff --git a/translator/translate/otel/processor/resourceprocessor/translator.go b/translator/translate/otel/processor/resourceprocessor/translator.go index 83ff61af83..7b39f3de9d 100644 --- a/translator/translate/otel/processor/resourceprocessor/translator.go +++ b/translator/translate/otel/processor/resourceprocessor/translator.go @@ -28,7 +28,7 @@ type translator struct { var ( baseKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey) - eksKey = common.ConfigKey(baseKey, common.KubernetesKey) + k8sKey = common.ConfigKey(baseKey, common.KubernetesKey) _ common.Translator[component.Config] = (*translator)(nil) ) @@ -62,30 +62,7 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { if strings.HasPrefix(t.Name(), common.PipelineNameJmx) { attributes = t.getJMXAttributes(conf) } else if t.Name() == common.PipelineNameContainerInsightsJmx { - clusterName, ok := common.GetString(conf, common.ConfigKey(eksKey, "cluster_name")) - - if !ok { - clusterName = util.GetClusterNameFromEc2Tagger() - } - nodeName := os.Getenv(config.HOST_NAME) - attributes = []any{ - map[string]any{ - "key": "Namespace", - "from_attribute": "k8s.namespace.name", - "action": "insert", - }, - map[string]any{ - "key": "ClusterName", - "value": clusterName, // Ensure 'clusterName' is defined earlier - "action": "upsert", - }, - map[string]any{ - "key": "NodeName", - "value": nodeName, - "action": "insert", - }, - } - + attributes = t.getContainerInsightsJMXAttributes(conf) } if len(attributes) == 0 { baseKey := common.JmxConfigKey @@ -134,3 +111,29 @@ func (t *translator) getJMXAttributes(conf *confmap.Conf) []any { } return attributes } + +func (t *translator) getContainerInsightsJMXAttributes(conf *confmap.Conf) []any { + clusterName, ok := common.GetString(conf, common.ConfigKey(k8sKey, "cluster_name")) + + if !ok { + clusterName = util.GetClusterNameFromEc2Tagger() + } + nodeName := os.Getenv(config.HOST_NAME) + return []any{ + map[string]any{ + "key": "Namespace", + "from_attribute": "k8s.namespace.name", + "action": "insert", + }, + map[string]any{ + "key": "ClusterName", + "value": clusterName, // Ensure 'clusterName' is defined earlier + "action": "upsert", + }, + map[string]any{ + "key": "NodeName", + "value": nodeName, + "action": "insert", + }, + } +} diff --git a/translator/translate/otel/processor/transformprocessor/config.yaml b/translator/translate/otel/processor/transformprocessor/transform_jmx_config.yaml similarity index 95% rename from translator/translate/otel/processor/transformprocessor/config.yaml rename to translator/translate/otel/processor/transformprocessor/transform_jmx_config.yaml index 484fa6ca3c..32b6d8c32b 100644 --- a/translator/translate/otel/processor/transformprocessor/config.yaml +++ b/translator/translate/otel/processor/transformprocessor/transform_jmx_config.yaml @@ -7,22 +7,22 @@ metric_statements: - set(unit, "Bytes") where name == "jvm.memory.heap.used" - set(unit, "Bytes") where name == "jvm.memory.nonheap.used" - set(unit, "Bytes") where name == "jvm.memory.pool.used" - - set(unit, "Bytes") where name == "tomcat.traffic.received" - - set(unit, "Bytes") where name == "tomcat.traffic.sent" - set(unit, "Bytes") where name == "jvm.system.swap.space.total" - set(unit, "Bytes") where name == "jvm.system.swap.space.free" - set(unit, "Bytes") where name == "jvm.system.physical.memory.total" - set(unit, "Bytes") where name == "jvm.system.physical.memory.free" - - set(unit, "Count") where name == "tomcat.sessions" - - set(unit, "Count") where name == "tomcat.rejected_sessions" - set(unit, "Count") where name == "jvm.threads.count" - - set(unit, "Count") where name == "jvm.threads.daemon" + - set(unit, "Count") where name == "jvm.daemon_threads.count" + - set(unit, "Count") where name == "jvm.classes.loaded" + - set(unit, "Count") where name == "jvm.system.cpu.utilization" + - set(unit, "Count") where name == "jvm.cpu.recent_utilization" - set(unit, "Count") where name == "jvm.open_file_descriptor.count" - set(unit, "Count") where name == "jvm.system.available.processors" + - set(unit, "Bytes") where name == "tomcat.traffic.received" + - set(unit, "Bytes") where name == "tomcat.traffic.sent" + - set(unit, "Count") where name == "tomcat.sessions" + - set(unit, "Count") where name == "tomcat.rejected_sessions" - set(unit, "Count") where name == "tomcat.request_count" - set(unit, "Count") where name == "tomcat.errors" - - set(unit, "Count") where name == "jvm.classes.loaded" - - set(unit, "Count") where name == "jvm.system.cpu.utilization" - - set(unit, "Count") where name == "jvm.cpu.recent_utilization" - set(unit, "Milliseconds") where name == "tomcat.processing_time" diff --git a/translator/translate/otel/processor/transformprocessor/translate.go b/translator/translate/otel/processor/transformprocessor/translate.go index 1910f26899..52aaddcb76 100644 --- a/translator/translate/otel/processor/transformprocessor/translate.go +++ b/translator/translate/otel/processor/transformprocessor/translate.go @@ -14,7 +14,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) -//go:embed config.yaml +//go:embed transform_jmx_config.yaml var transformJmxConfig string type translator struct { diff --git a/translator/translate/otel/processor/transformprocessor/translator_test.go b/translator/translate/otel/processor/transformprocessor/translator_test.go index b3fe7e9f67..6741c47f13 100644 --- a/translator/translate/otel/processor/transformprocessor/translator_test.go +++ b/translator/translate/otel/processor/transformprocessor/translator_test.go @@ -66,7 +66,7 @@ func TestTranslator(t *testing.T) { func TestContainerInsightsJmx(t *testing.T) { transl := NewTranslatorWithName(common.PipelineNameContainerInsightsJmx).(*translator) expectedCfg := transl.factory.CreateDefaultConfig().(*transformprocessor.Config) - c := testutil.GetConf(t, "config.yaml") + c := testutil.GetConf(t, "transform_jmx_config.yaml") require.NoError(t, c.Unmarshal(&expectedCfg)) conf := confmap.NewFromStringMap(testutil.GetJson(t, filepath.Join("testdata", "config.json")))