diff --git a/cmd/config-translator/translator_test.go b/cmd/config-translator/translator_test.go index e9298c3807..b276fc4632 100644 --- a/cmd/config-translator/translator_test.go +++ b/cmd/config-translator/translator_test.go @@ -187,6 +187,9 @@ func TestMetricsDestinationsConfig(t *testing.T) { expectedErrorMap["required"] = 1 checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/invalidMetricsDestinations.json", false, expectedErrorMap) } +func TestContainerInsightsJmxConfig(t *testing.T) { + checkIfSchemaValidateAsExpected(t, "../../translator/config/sampleSchema/validContainerInsightsJmx.json", true, map[string]int{}) +} // Validate all sampleConfig files schema func TestSampleConfigSchema(t *testing.T) { diff --git a/translator/config/sampleSchema/validContainerInsightsJmx.json b/translator/config/sampleSchema/validContainerInsightsJmx.json new file mode 100644 index 0000000000..5d07791f50 --- /dev/null +++ b/translator/config/sampleSchema/validContainerInsightsJmx.json @@ -0,0 +1,16 @@ +{ + "agent": { + "region": "us-west-2" + }, + "logs": { + "metrics_collected": { + "emf": { + }, + "kubernetes": { + "cluster_name": "TestCluster", + "jmx_container_insights": true + } + }, + "force_flush_interval": 5 + } +} \ No newline at end of file diff --git a/translator/config/schema.json b/translator/config/schema.json index f69c05be63..0d091cb8a7 100644 --- a/translator/config/schema.json +++ b/translator/config/schema.json @@ -768,6 +768,10 @@ "minLength": 1, "maxLength": 512 }, + "jmx_container_insights": { + "description": "Enable JMX Container Insights metrics", + "type": "boolean" + }, "metrics_collection_interval": { "$ref": "#/definitions/timeIntervalDefinition" }, diff --git a/translator/tocwconfig/sampleConfig/container_insights_jmx.conf b/translator/tocwconfig/sampleConfig/container_insights_jmx.conf new file mode 100644 index 0000000000..66b62800a8 --- /dev/null +++ b/translator/tocwconfig/sampleConfig/container_insights_jmx.conf @@ -0,0 +1,24 @@ +[agent] + collection_jitter = "0s" + debug = true + flush_interval = "1s" + flush_jitter = "0s" + hostname = "host_name_from_env" + interval = "60s" + logfile = "" + logtarget = "lumberjack" + metric_batch_size = 1000 + metric_buffer_limit = 10000 + omit_hostname = false + precision = "" + quiet = false + round_interval = false + +[outputs] + + [[outputs.cloudwatchlogs]] + force_flush_interval = "5s" + log_stream_name = "host_name_from_env" + mode = "EC2" + region = "us-west-2" + region_type = "ACJ" diff --git a/translator/tocwconfig/sampleConfig/container_insights_jmx.json b/translator/tocwconfig/sampleConfig/container_insights_jmx.json new file mode 100644 index 0000000000..72d7adf71e --- /dev/null +++ b/translator/tocwconfig/sampleConfig/container_insights_jmx.json @@ -0,0 +1,13 @@ +{ + "agent": { + "debug": true + }, + "logs": { + "metrics_collected": { + "kubernetes": { + "cluster_name": "TestCluster", + "jmx_container_insights": true + } + } + } +} \ No newline at end of file diff --git a/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml b/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml new file mode 100644 index 0000000000..f9dc971879 --- /dev/null +++ b/translator/tocwconfig/sampleConfig/container_insights_jmx.yaml @@ -0,0 +1,509 @@ +exporters: + awsemf/containerinsights: + certificate_file_path: "" + detailed_metrics: false + dimension_rollup_option: NoDimensionRollup + disable_metric_extraction: false + eks_fargate_container_insights_enabled: false + endpoint: "" + enhanced_container_insights: false + imds_retries: 1 + local_mode: false + log_group_name: /aws/containerinsights/{ClusterName}/performance + log_retention: 0 + log_stream_name: '{NodeName}' + max_retries: 2 + metric_declarations: + - dimensions: + - - ClusterName + - Namespace + - PodName + - - ClusterName + - - ClusterName + - Namespace + - Service + - - ClusterName + - Namespace + metric_name_selectors: + - pod_cpu_utilization + - pod_memory_utilization + - pod_network_rx_bytes + - pod_network_tx_bytes + - pod_cpu_utilization_over_pod_limit + - pod_memory_utilization_over_pod_limit + - dimensions: + - - ClusterName + - Namespace + - PodName + metric_name_selectors: + - pod_number_of_container_restarts + - dimensions: + - - ClusterName + - Namespace + - PodName + - - ClusterName + metric_name_selectors: + - pod_cpu_reserved_capacity + - pod_memory_reserved_capacity + - dimensions: + - - ClusterName + - InstanceId + - NodeName + - - ClusterName + metric_name_selectors: + - node_cpu_utilization + - node_memory_utilization + - node_network_total_bytes + - node_cpu_reserved_capacity + - node_memory_reserved_capacity + - node_number_of_running_pods + - node_number_of_running_containers + - dimensions: + - - ClusterName + metric_name_selectors: + - node_cpu_usage_total + - node_cpu_limit + - node_memory_working_set + - node_memory_limit + - dimensions: + - - ClusterName + - InstanceId + - NodeName + - - ClusterName + metric_name_selectors: + - node_filesystem_utilization + - dimensions: + - - ClusterName + - Namespace + - Service + - - ClusterName + metric_name_selectors: + - service_number_of_running_pods + - dimensions: + - - ClusterName + - Namespace + - - ClusterName + metric_name_selectors: + - namespace_number_of_running_pods + - dimensions: + - - ClusterName + metric_name_selectors: + - cluster_node_count + - cluster_failed_node_count + middleware: agenthealth/logs + namespace: ContainerInsights + no_verify_ssl: false + num_workers: 8 + output_destination: cloudwatch + parse_json_encoded_attr_values: + - Sources + - kubernetes + profile: "" + proxy_address: "" + region: us-west-2 + request_timeout_seconds: 30 + resource_arn: "" + resource_to_telemetry_conversion: + enabled: true + retain_initial_value_of_delta_metric: false + role_arn: "" + version: "0" + awsemf/containerinsightsjmx: + certificate_file_path: "" + detailed_metrics: false + dimension_rollup_option: NoDimensionRollup + disable_metric_extraction: false + eks_fargate_container_insights_enabled: false + endpoint: "" + enhanced_container_insights: false + imds_retries: 1 + local_mode: false + log_group_name: /aws/containerinsights/{ClusterName}/jmx + log_retention: 0 + log_stream_name: '{NodeName}' + max_retries: 2 + metric_declarations: + - dimensions: + - - ClusterName + - Namespace + metric_name_selectors: + - java_lang_operatingsystem_freeswapspacesize + - java_lang_operatingsystem_availableprocessors + - catalina_manager_rejectedsessions + - catalina_globalrequestprocessor_bytesreceived + - catalina_globalrequestprocessor_processingtime + - jvm_memory_pool_bytes_used + - java_lang_operatingsystem_systemcpuload + - java_lang_operatingsystem_totalphysicalmemorysize + - java_lang_operatingsystem_freephysicalmemorysize + - java_lang_operatingsystem_openfiledescriptorcount + - catalina_manager_activesessions + - java_lang_operatingsystem_totalswapspacesize + - java_lang_operatingsystem_processcpuload + - catalina_globalrequestprocessor_requestcount + - catalina_globalrequestprocessor_errorcount + - jvm_threads_daemon + - catalina_globalrequestprocessor_bytessent + - jvm_classes_loaded + - jvm_threads_current + - dimensions: + - - ClusterName + - Namespace + - area + metric_name_selectors: + - jvm_memory_bytes_used + - dimensions: + - - ClusterName + - Namespace + - pool + metric_name_selectors: + - jvm_memory_pool_bytes_used + middleware: agenthealth/logs + namespace: ContainerInsights/Prometheus + no_verify_ssl: false + num_workers: 8 + output_destination: cloudwatch + profile: "" + proxy_address: "" + region: us-west-2 + request_timeout_seconds: 30 + resource_arn: "" + resource_to_telemetry_conversion: + enabled: true + retain_initial_value_of_delta_metric: false + role_arn: "" + version: "0" +extensions: + agenthealth/logs: + is_usage_data_enabled: true + stats: + operations: + - PutLogEvents + usage_flags: + mode: EC2 + region_type: ACJ +processors: + batch/containerinsights: + metadata_cardinality_limit: 1000 + send_batch_max_size: 0 + send_batch_size: 8192 + timeout: 5s + cumulativetodelta/containerinsightsjmx: + exclude: + match_type: "" + include: + match_type: "" + initial_value: 2 + max_staleness: 0s + filter/containerinsightsjmx: + error_mode: propagate + logs: {} + metrics: + include: + match_type: strict + metric_names: + - jvm.classes.loaded + - jvm.memory.heap.used + - jvm.memory.nonheap.used + - jvm.memory.pool.used + - jvm.system.swap.space.total + - jvm.system.cpu.utilization + - jvm.cpu.recent_utilization + - jvm.system.swap.space.free + - jvm.system.physical.memory.total + - jvm.system.physical.memory.free + - jvm.open_file_descriptor.count + - jvm.system.available.processors + - jvm.threads.count + - jvm.daemon_threads.count + - tomcat.sessions + - tomcat.rejected_sessions + - tomcat.traffic.received + - tomcat.traffic.sent + - tomcat.request_count + - tomcat.errors + - tomcat.processing_time + spans: {} + traces: {} + metricstransform/containerinsightsjmx: + transforms: + - action: update + aggregation_type: "" + include: jvm.classes.loaded + match_type: strict + new_name: jvm_classes_loaded + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.memory.heap.used + match_type: strict + new_name: jvm_memory_bytes_used + operations: + - action: add_label + aggregation_type: "" + experimental_scale: 0 + label: "" + label_value: "" + new_label: area + new_value: heap + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.memory.nonheap.used + match_type: strict + new_name: jvm_memory_bytes_used + operations: + - action: add_label + aggregation_type: "" + experimental_scale: 0 + label: "" + label_value: "" + new_label: area + new_value: nonheap + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.memory.pool.used + match_type: strict + new_name: jvm_memory_pool_bytes_used + operations: + - action: update_label + aggregation_type: "" + experimental_scale: 0 + label: name + label_value: "" + new_label: pool + new_value: "" + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.system.swap.space.total + match_type: strict + new_name: java_lang_operatingsystem_totalswapspacesize + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.system.cpu.utilization + match_type: strict + new_name: java_lang_operatingsystem_systemcpuload + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.cpu.recent_utilization + match_type: strict + new_name: java_lang_operatingsystem_processcpuload + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.system.swap.space.free + match_type: strict + new_name: java_lang_operatingsystem_freeswapspacesize + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.system.physical.memory.total + match_type: strict + new_name: java_lang_operatingsystem_totalphysicalmemorysize + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.system.physical.memory.free + match_type: strict + new_name: java_lang_operatingsystem_freephysicalmemorysize + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.open_file_descriptor.count + match_type: strict + new_name: java_lang_operatingsystem_openfiledescriptorcount + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.system.available.processors + match_type: strict + new_name: java_lang_operatingsystem_availableprocessors + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.threads.count + match_type: strict + new_name: jvm_threads_current + submatch_case: "" + - action: update + aggregation_type: "" + include: jvm.daemon_threads.count + match_type: strict + new_name: jvm_threads_daemon + submatch_case: "" + - action: update + aggregation_type: "" + include: tomcat.sessions + match_type: strict + new_name: catalina_manager_activesessions + submatch_case: "" + - action: update + aggregation_type: "" + include: tomcat.rejected_sessions + match_type: strict + new_name: catalina_manager_rejectedsessions + submatch_case: "" + - action: update + aggregation_type: "" + include: tomcat.traffic.received + match_type: strict + new_name: catalina_globalrequestprocessor_bytesreceived + submatch_case: "" + - action: update + aggregation_type: "" + include: tomcat.traffic.sent + match_type: strict + new_name: catalina_globalrequestprocessor_bytessent + submatch_case: "" + - action: update + aggregation_type: "" + include: tomcat.request_count + match_type: strict + new_name: catalina_globalrequestprocessor_requestcount + submatch_case: "" + - action: update + aggregation_type: "" + include: tomcat.errors + match_type: strict + new_name: catalina_globalrequestprocessor_errorcount + submatch_case: "" + - action: update + aggregation_type: "" + include: tomcat.processing_time + match_type: strict + new_name: catalina_globalrequestprocessor_processingtime + submatch_case: "" + resource/containerinsightsjmx: + attributes: + - action: insert + converted_type: "" + from_attribute: k8s.namespace.name + from_context: "" + key: Namespace + pattern: "" + - action: upsert + converted_type: "" + from_attribute: "" + from_context: "" + key: ClusterName + pattern: "" + value: TestCluster + - action: insert + converted_type: "" + from_attribute: "" + from_context: "" + key: NodeName + pattern: "" + value: "host_name_from_env" + transform/containerinsightsjmx: + error_mode: propagate + flatten_data: false + log_statements: [] + metric_statements: + - context: resource + statements: + - keep_keys(attributes, ["ClusterName", "Namespace", "NodeName"]) + - context: metric + statements: + - set(unit, "Bytes") where name == "jvm.memory.heap.used" + - set(unit, "Bytes") where name == "jvm.memory.nonheap.used" + - set(unit, "Bytes") where name == "jvm.memory.pool.used" + - set(unit, "Bytes") where name == "tomcat.traffic.received" + - set(unit, "Bytes") where name == "tomcat.traffic.sent" + - set(unit, "Bytes") where name == "jvm.system.swap.space.total" + - set(unit, "Bytes") where name == "jvm.system.swap.space.free" + - set(unit, "Bytes") where name == "jvm.system.physical.memory.total" + - set(unit, "Bytes") where name == "jvm.system.physical.memory.free" + - set(unit, "Count") where name == "tomcat.sessions" + - set(unit, "Count") where name == "tomcat.rejected_sessions" + - set(unit, "Count") where name == "jvm.threads.count" + - set(unit, "Count") where name == "jvm.daemon_threads.count" + - set(unit, "Count") where name == "jvm.open_file_descriptor.count" + - set(unit, "Count") where name == "jvm.system.available.processors" + - set(unit, "Count") where name == "tomcat.request_count" + - set(unit, "Count") where name == "tomcat.errors" + - set(unit, "Count") where name == "jvm.classes.loaded" + - set(unit, "Count") where name == "jvm.system.cpu.utilization" + - set(unit, "Count") where name == "jvm.cpu.recent_utilization" + - set(unit, "Milliseconds") where name == "tomcat.processing_time" + trace_statements: [] +receivers: + awscontainerinsightreceiver: + accelerated_compute_metrics: true + add_container_name_metric_label: false + add_full_pod_name_metric_label: false + add_service_as_attribute: true + certificate_file_path: "" + cluster_name: TestCluster + collection_interval: 1m0s + container_orchestrator: eks + enable_control_plane_metrics: false + endpoint: "" + host_ip: "" + host_name: "" + imds_retries: 1 + kube_config_path: "" + leader_lock_name: cwagent-clusterleader + leader_lock_using_config_map_only: true + local_mode: false + max_retries: 0 + no_verify_ssl: false + num_workers: 0 + prefer_full_pod_name: false + profile: "" + proxy_address: "" + region: us-west-2 + request_timeout_seconds: 0 + resource_arn: "" + role_arn: "" + otlp/jmx: + protocols: + http: + endpoint: 0.0.0.0:4314 + include_metadata: false + logs_url_path: /v1/logs + max_request_body_size: 0 + metrics_url_path: /v1/metrics + traces_url_path: /v1/traces +service: + extensions: + - agenthealth/logs + pipelines: + metrics/containerinsights: + exporters: + - awsemf/containerinsights + processors: + - batch/containerinsights + receivers: + - awscontainerinsightreceiver + metrics/containerinsightsjmx: + exporters: + - awsemf/containerinsightsjmx + processors: + - filter/containerinsightsjmx + - resource/containerinsightsjmx + - transform/containerinsightsjmx + - metricstransform/containerinsightsjmx + - cumulativetodelta/containerinsightsjmx + receivers: + - otlp/jmx + telemetry: + logs: + development: false + disable_caller: false + disable_stacktrace: false + encoding: console + level: debug + sampling: + enabled: true + initial: 2 + thereafter: 500 + tick: 10s + metrics: + address: "" + level: None + traces: {} diff --git a/translator/tocwconfig/tocwconfig_test.go b/translator/tocwconfig/tocwconfig_test.go index 52755f6e93..ea3841cf15 100644 --- a/translator/tocwconfig/tocwconfig_test.go +++ b/translator/tocwconfig/tocwconfig_test.go @@ -81,6 +81,19 @@ func TestGenericAppSignalsConfig(t *testing.T) { checkTranslation(t, "base_appsignals_config", "linux", expectedEnvVars, "") checkTranslation(t, "base_appsignals_config", "windows", expectedEnvVars, "") } +func TestContainerInsightsJMX(t *testing.T) { + resetContext(t) + context.CurrentContext().SetRunInContainer(true) + context.CurrentContext().SetMode(config.ModeEC2) + t.Setenv(config.HOST_NAME, "host_name_from_env") + t.Setenv(config.HOST_IP, "127.0.0.1") + + expectedEnvVars := map[string]string{ + "CWAGENT_LOG_LEVEL": "DEBUG"} + + checkTranslation(t, "container_insights_jmx", "linux", expectedEnvVars, "") + +} func TestGenericAppSignalsFallbackConfig(t *testing.T) { resetContext(t) diff --git a/translator/translate/otel/common/common.go b/translator/translate/otel/common/common.go index 51dab5bcda..0b468ceffb 100644 --- a/translator/translate/otel/common/common.go +++ b/translator/translate/otel/common/common.go @@ -76,13 +76,14 @@ const ( ) const ( - PipelineNameHost = "host" - PipelineNameHostDeltaMetrics = "hostDeltaMetrics" - PipelineNameJmx = "jmx" - PipelineNameEmfLogs = "emf_logs" - AppSignals = "application_signals" - AppSignalsFallback = "app_signals" - AppSignalsRules = "rules" + PipelineNameHost = "host" + PipelineNameHostDeltaMetrics = "hostDeltaMetrics" + PipelineNameJmx = "jmx" + PipelineNameContainerInsightsJmx = "containerinsightsjmx" + PipelineNameEmfLogs = "emf_logs" + AppSignals = "application_signals" + AppSignalsFallback = "app_signals" + AppSignalsRules = "rules" ) var ( @@ -95,8 +96,10 @@ var ( component.DataTypeTraces: {AppSignalsTraces, AppSignalsTracesFallback}, component.DataTypeMetrics: {AppSignalsMetrics, AppSignalsMetricsFallback}, } - JmxConfigKey = ConfigKey(MetricsKey, MetricsCollectedKey, JmxKey) - JmxTargets = []string{"activemq", "cassandra", "hbase", "hadoop", "jetty", "jvm", "kafka", "kafka-consumer", "kafka-producer", "solr", "tomcat", "wildfly"} + JmxConfigKey = ConfigKey(MetricsKey, MetricsCollectedKey, JmxKey) + ContainerInsightsConfigKey = ConfigKey(LogsKey, MetricsCollectedKey, KubernetesKey) + + JmxTargets = []string{"activemq", "cassandra", "hbase", "hadoop", "jetty", "jvm", "kafka", "kafka-consumer", "kafka-producer", "solr", "tomcat", "wildfly"} AgentDebugConfigKey = ConfigKey(AgentKey, DebugKey) MetricsAggregationDimensionsKey = ConfigKey(MetricsKey, AggregationDimensionsKey) diff --git a/translator/translate/otel/exporter/awsemf/awsemf_jmx_config.yaml b/translator/translate/otel/exporter/awsemf/awsemf_jmx_config.yaml new file mode 100644 index 0000000000..1fc49e024c --- /dev/null +++ b/translator/translate/otel/exporter/awsemf/awsemf_jmx_config.yaml @@ -0,0 +1,37 @@ +namespace: ContainerInsights/Prometheus +log_group_name: '/aws/containerinsights/{ClusterName}/jmx' +log_stream_name: '{NodeName}' +detailed_metrics: false +dimension_rollup_option: NoDimensionRollup +version: "0" +retain_initial_value_of_delta_metric: false +resource_to_telemetry_conversion: + enabled: true +metric_declarations: + - dimensions: [[ClusterName,Namespace]] + metric_name_selectors: + - java_lang_operatingsystem_freeswapspacesize + - java_lang_operatingsystem_availableprocessors + - catalina_manager_rejectedsessions + - catalina_globalrequestprocessor_bytesreceived + - catalina_globalrequestprocessor_processingtime + - jvm_memory_pool_bytes_used + - java_lang_operatingsystem_systemcpuload + - java_lang_operatingsystem_totalphysicalmemorysize + - java_lang_operatingsystem_freephysicalmemorysize + - java_lang_operatingsystem_openfiledescriptorcount + - catalina_manager_activesessions + - java_lang_operatingsystem_totalswapspacesize + - java_lang_operatingsystem_processcpuload + - catalina_globalrequestprocessor_requestcount + - catalina_globalrequestprocessor_errorcount + - jvm_threads_daemon + - catalina_globalrequestprocessor_bytessent + - jvm_classes_loaded + - jvm_threads_current + - dimensions: [[ClusterName, Namespace, area]] + metric_name_selectors: + - jvm_memory_bytes_used + - dimensions: [[ClusterName,Namespace, pool]] + metric_name_selectors: + - jvm_memory_pool_bytes_used \ No newline at end of file diff --git a/translator/translate/otel/exporter/awsemf/translator.go b/translator/translate/otel/exporter/awsemf/translator.go index 764fa3532d..7ce0af0472 100644 --- a/translator/translate/otel/exporter/awsemf/translator.go +++ b/translator/translate/otel/exporter/awsemf/translator.go @@ -46,6 +46,9 @@ var appSignalsConfigK8s string //go:embed appsignals_config_generic.yaml var appSignalsConfigGeneric string +//go:embed awsemf_jmx_config.yaml +var defaultJmxConfig string + var ( ecsBasePathKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.ECSKey) kubernetesBasePathKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey, common.KubernetesKey) @@ -82,6 +85,8 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { defaultConfig := defaultGenericConfig if t.isAppSignals(c) { defaultConfig = getAppSignalsConfig() + } else if t.isCiJMX(c) { + defaultConfig = defaultJmxConfig } else if isEcs(c) { defaultConfig = defaultEcsConfig } else if isKubernetes(c) { @@ -124,6 +129,10 @@ func (t *translator) Translate(c *confmap.Conf) (component.Config, error) { if err := setAppSignalsFields(c, cfg); err != nil { return nil, err } + } else if t.isCiJMX(c) { + if err := setCiJmxFields(); err != nil { + return nil, err + } } else if isEcs(c) { if err := setEcsFields(c, cfg); err != nil { return nil, err @@ -168,6 +177,9 @@ func getAppSignalsConfig() string { func (t *translator) isAppSignals(conf *confmap.Conf) bool { return (t.name == common.AppSignals || t.name == common.AppSignalsFallback) && (conf.IsSet(common.AppSignalsMetrics) || conf.IsSet(common.AppSignalsTraces) || conf.IsSet(common.AppSignalsMetricsFallback) || conf.IsSet(common.AppSignalsTracesFallback)) } +func (t *translator) isCiJMX(conf *confmap.Conf) bool { + return (t.name == common.PipelineNameContainerInsightsJmx) && (conf.IsSet(common.ContainerInsightsConfigKey)) +} func isEcs(conf *confmap.Conf) bool { return conf.IsSet(ecsBasePathKey) @@ -204,6 +216,9 @@ func setKubernetesFields(conf *confmap.Conf, cfg *awsemfexporter.Config) error { return nil } +func setCiJmxFields() error { + return nil +} func setPrometheusFields(conf *confmap.Conf, cfg *awsemfexporter.Config) error { setDisableMetricExtraction(prometheusBasePathKey, conf, cfg) diff --git a/translator/translate/otel/pipeline/containerinsightsjmx/translator.go b/translator/translate/otel/pipeline/containerinsightsjmx/translator.go new file mode 100644 index 0000000000..e34e30c92e --- /dev/null +++ b/translator/translate/otel/pipeline/containerinsightsjmx/translator.go @@ -0,0 +1,78 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package containerinsightsjmx + +import ( + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + + "github.com/aws/amazon-cloudwatch-agent/translator/context" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/exporter/awsemf" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/extension/agenthealth" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/cumulativetodeltaprocessor" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/filterprocessor" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/metricstransformprocessor" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/resourceprocessor" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/transformprocessor" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/otlp" +) + +var ( + baseKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey) + eksKey = common.ConfigKey(baseKey, common.KubernetesKey) + jmxKey = common.ConfigKey(eksKey, "jmx_container_insights") +) + +type translator struct { +} + +var _ common.Translator[*common.ComponentTranslators] = (*translator)(nil) + +func NewTranslator() common.Translator[*common.ComponentTranslators] { + return &translator{} +} + +func (t *translator) ID() component.ID { + return component.NewIDWithName(component.DataTypeMetrics, common.PipelineNameContainerInsightsJmx) +} + +// Translate creates a pipeline for container insights jmx if the logs.metrics_collected.kubernetes +// section is present. +func (t *translator) Translate(conf *confmap.Conf) (*common.ComponentTranslators, error) { + if conf == nil || !conf.IsSet(jmxKey) { + return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: jmxKey} + } + if !context.CurrentContext().RunInContainer() { + return nil, nil + } + + if val, _ := common.GetBool(conf, jmxKey); !val { + return nil, nil + } + translators := common.ComponentTranslators{ + Receivers: common.NewTranslatorMap( + otlp.NewTranslator(common.WithName(common.PipelineNameJmx)), + ), + Processors: common.NewTranslatorMap( + filterprocessor.NewTranslator(common.WithName(common.PipelineNameContainerInsightsJmx)), // Filter metrics + resourceprocessor.NewTranslator(common.WithName(common.PipelineNameContainerInsightsJmx)), // Change resource attribute names + transformprocessor.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx), // Removes attributes that are not of [ClusterName, Namespace] + metricstransformprocessor.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx), // Renames metrics and adds pool and area dimensions + cumulativetodeltaprocessor.NewTranslator( + common.WithName(common.PipelineNameContainerInsightsJmx), + cumulativetodeltaprocessor.WithConfigKeys(jmxKey), + ), + ), + Exporters: common.NewTranslatorMap( + awsemf.NewTranslatorWithName(common.PipelineNameContainerInsightsJmx), + ), + Extensions: common.NewTranslatorMap( + agenthealth.NewTranslator(component.DataTypeLogs, []string{agenthealth.OperationPutLogEvents}), + ), + } + + return &translators, nil + +} diff --git a/translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go b/translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go new file mode 100644 index 0000000000..72ad074f49 --- /dev/null +++ b/translator/translate/otel/pipeline/containerinsightsjmx/translator_test.go @@ -0,0 +1,76 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package containerinsightsjmx + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + + "github.com/aws/amazon-cloudwatch-agent/internal/util/collections" + "github.com/aws/amazon-cloudwatch-agent/translator/context" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" +) + +func TestTranslator(t *testing.T) { + context.CurrentContext().SetRunInContainer(true) + type want struct { + pipelineType string + receivers []string + processors []string + exporters []string + extensions []string + } + cit := NewTranslator() + require.EqualValues(t, "metrics/containerinsightsjmx", cit.ID().String()) + testCases := map[string]struct { + input map[string]interface{} + want *want + wantErr error + }{ + "WithoutECSOrKubernetesKey": { + input: map[string]interface{}{}, + wantErr: &common.MissingKeyError{ID: cit.ID(), JsonKey: fmt.Sprint(jmxKey)}, + }, + "WithKubernetesKey": { + input: map[string]interface{}{ + "logs": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "kubernetes": map[string]interface{}{ + "jmx_container_insights": true, + }, + }, + }, + }, + want: &want{ + pipelineType: "metrics/containerinsightsjmx", + receivers: []string{"otlp/jmx"}, + processors: []string{"filter/containerinsightsjmx", "resource/containerinsightsjmx", "transform/containerinsightsjmx", "metricstransform/containerinsightsjmx", "cumulativetodelta/containerinsightsjmx"}, + exporters: []string{"awsemf/containerinsightsjmx"}, + extensions: []string{"agenthealth/logs"}, + }, + }, + } + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + conf := confmap.NewFromStringMap(testCase.input) + got, err := cit.Translate(conf) + require.Equal(t, testCase.wantErr, err) + if testCase.want == nil { + require.Nil(t, got) + } else { + require.NotNil(t, got) + assert.Equal(t, testCase.want.receivers, collections.MapSlice(got.Receivers.Keys(), component.ID.String)) + assert.Equal(t, testCase.want.processors, collections.MapSlice(got.Processors.Keys(), component.ID.String)) + assert.Equal(t, testCase.want.exporters, collections.MapSlice(got.Exporters.Keys(), component.ID.String)) + assert.Equal(t, testCase.want.extensions, collections.MapSlice(got.Extensions.Keys(), component.ID.String)) + } + }) + } +} diff --git a/translator/translate/otel/processor/filterprocessor/filter_jmx_config.yaml b/translator/translate/otel/processor/filterprocessor/filter_jmx_config.yaml new file mode 100644 index 0000000000..34e982d508 --- /dev/null +++ b/translator/translate/otel/processor/filterprocessor/filter_jmx_config.yaml @@ -0,0 +1,25 @@ +metrics: + include: + match_type: strict + metric_names: + - jvm.classes.loaded + - jvm.memory.heap.used + - jvm.memory.nonheap.used + - jvm.memory.pool.used + - jvm.system.swap.space.total + - jvm.system.cpu.utilization + - jvm.cpu.recent_utilization + - jvm.system.swap.space.free + - jvm.system.physical.memory.total + - jvm.system.physical.memory.free + - jvm.open_file_descriptor.count + - jvm.system.available.processors + - jvm.threads.count + - jvm.daemon_threads.count + - tomcat.sessions + - tomcat.rejected_sessions + - tomcat.traffic.received + - tomcat.traffic.sent + - tomcat.request_count + - tomcat.errors + - tomcat.processing_time \ No newline at end of file diff --git a/translator/translate/otel/processor/filterprocessor/testdata/config.yaml b/translator/translate/otel/processor/filterprocessor/testdata/config.yaml deleted file mode 100644 index 88d3883f15..0000000000 --- a/translator/translate/otel/processor/filterprocessor/testdata/config.yaml +++ /dev/null @@ -1,10 +0,0 @@ -metrics: - include: - match_type: strict - metric_names: - - jvm.memory.heap.init - - jvm.memory.heap.used - - jvm.memory.nonheap.init - - jvm.memory.nonheap.used - - jvm.threads.count - - tomcat.sessions diff --git a/translator/translate/otel/processor/filterprocessor/translator.go b/translator/translate/otel/processor/filterprocessor/translator.go index f5bf2d8707..b6faafa234 100644 --- a/translator/translate/otel/processor/filterprocessor/translator.go +++ b/translator/translate/otel/processor/filterprocessor/translator.go @@ -4,6 +4,7 @@ package filterprocessor import ( + _ "embed" "fmt" "strconv" @@ -19,6 +20,9 @@ const ( matchTypeStrict = "strict" ) +//go:embed filter_jmx_config.yaml +var containerInsightsJmxConfig string + type translator struct { common.NameProvider common.IndexProvider @@ -48,11 +52,14 @@ func (t *translator) ID() component.ID { // Translate creates a processor config based on the fields in the // Metrics section of the JSON config. func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { - if conf == nil || !conf.IsSet(common.JmxConfigKey) { + if conf == nil || (!conf.IsSet(common.JmxConfigKey) && t.Name() != common.PipelineNameContainerInsightsJmx) { return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: common.JmxConfigKey} } cfg := t.factory.CreateDefaultConfig().(*filterprocessor.Config) + if t.Name() == common.PipelineNameContainerInsightsJmx { + return common.GetYamlFileToYamlConfig(cfg, containerInsightsJmxConfig) + } jmxMap := common.GetIndexedMap(conf, common.JmxConfigKey, t.Index()) diff --git a/translator/translate/otel/processor/filterprocessor/translator_test.go b/translator/translate/otel/processor/filterprocessor/translator_test.go index f069782296..d87fff55ff 100644 --- a/translator/translate/otel/processor/filterprocessor/translator_test.go +++ b/translator/translate/otel/processor/filterprocessor/translator_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" @@ -110,12 +111,6 @@ func TestTranslator(t *testing.T) { }, }), }, - "WithCompleteConfig": { - input: testutil.GetJson(t, filepath.Join("testdata", "config.json")), - index: -1, - wantID: "filter/jmx", - want: testutil.GetConf(t, filepath.Join("testdata", "config.yaml")), - }, } for name, testCase := range testCases { t.Run(name, func(t *testing.T) { @@ -135,3 +130,17 @@ func TestTranslator(t *testing.T) { }) } } + +func TestContainerInsightsJmx(t *testing.T) { + transl := NewTranslator(common.WithName(common.PipelineNameContainerInsightsJmx)).(*translator) + expectedCfg := transl.factory.CreateDefaultConfig().(*filterprocessor.Config) + c := testutil.GetConf(t, "filter_jmx_config.yaml") + require.NoError(t, c.Unmarshal(&expectedCfg)) + + conf := confmap.NewFromStringMap(testutil.GetJson(t, filepath.Join("testdata", "config.json"))) + translatedCfg, err := transl.Translate(conf) + assert.NoError(t, err) + actualCfg, ok := translatedCfg.(*filterprocessor.Config) + assert.True(t, ok) + assert.Equal(t, len(expectedCfg.Metrics.Include.MetricNames), len(actualCfg.Metrics.Include.MetricNames)) +} diff --git a/translator/translate/otel/processor/metricstransformprocessor/metricstransform_jmx_config.yaml b/translator/translate/otel/processor/metricstransformprocessor/metricstransform_jmx_config.yaml new file mode 100644 index 0000000000..caeb6bb9c9 --- /dev/null +++ b/translator/translate/otel/processor/metricstransformprocessor/metricstransform_jmx_config.yaml @@ -0,0 +1,133 @@ +transforms: + - include: jvm.classes.loaded + match_type: strict + action: update + new_name: jvm_classes_loaded + + + - include: jvm.memory.heap.used + match_type: strict + action: update + new_name: jvm_memory_bytes_used + operations: + - action: add_label + new_label: area + new_value: "heap" + + - include: jvm.memory.nonheap.used + match_type: strict + action: update + new_name: jvm_memory_bytes_used + operations: + - action: add_label + new_label: area + new_value: "nonheap" + + - include: jvm.memory.pool.used + match_type: strict + action: update + new_name: jvm_memory_pool_bytes_used + operations: + - action: update_label + label: name + new_label: pool + + - include: jvm.system.swap.space.total + match_type: strict + action: update + new_name: java_lang_operatingsystem_totalswapspacesize + + + - include: jvm.system.cpu.utilization + match_type: strict + action: update + new_name: java_lang_operatingsystem_systemcpuload + + - include: jvm.cpu.recent_utilization + match_type: strict + action: update + new_name: java_lang_operatingsystem_processcpuload + + + - include: jvm.system.swap.space.free + match_type: strict + action: update + new_name: java_lang_operatingsystem_freeswapspacesize + + + - include: jvm.system.physical.memory.total + match_type: strict + action: update + new_name: java_lang_operatingsystem_totalphysicalmemorysize + + + - include: jvm.system.physical.memory.free + match_type: strict + action: update + new_name: java_lang_operatingsystem_freephysicalmemorysize + + + - include: jvm.open_file_descriptor.count + match_type: strict + action: update + new_name: java_lang_operatingsystem_openfiledescriptorcount + + + - include: jvm.system.available.processors + match_type: strict + action: update + new_name: java_lang_operatingsystem_availableprocessors + + + - include: jvm.threads.count + match_type: strict + action: update + new_name: jvm_threads_current + + + - include: jvm.daemon_threads.count + match_type: strict + action: update + new_name: jvm_threads_daemon + + + - include: tomcat.sessions + match_type: strict + action: update + new_name: catalina_manager_activesessions + + + - include: tomcat.rejected_sessions + match_type: strict + action: update + new_name: catalina_manager_rejectedsessions + + + - include: tomcat.traffic.received + match_type: strict + action: update + new_name: catalina_globalrequestprocessor_bytesreceived + + + - include: tomcat.traffic.sent + match_type: strict + action: update + new_name: catalina_globalrequestprocessor_bytessent + + - include: tomcat.request_count + match_type: strict + action: update + new_name: catalina_globalrequestprocessor_requestcount + + + - include: tomcat.errors + match_type: strict + action: update + new_name: catalina_globalrequestprocessor_errorcount + + + - include: tomcat.processing_time + match_type: strict + action: update + new_name: catalina_globalrequestprocessor_processingtime + diff --git a/translator/translate/otel/processor/metricstransformprocessor/testdata/config.json b/translator/translate/otel/processor/metricstransformprocessor/testdata/config.json new file mode 100644 index 0000000000..46cae7bee5 --- /dev/null +++ b/translator/translate/otel/processor/metricstransformprocessor/testdata/config.json @@ -0,0 +1,13 @@ +{ + "agent": { + "debug": true + }, + "logs": { + "metrics_collected": { + "kubernetes": { + "cluster_name": "TestCluster", + "metrics_collection_interval": 30 + } + } + } +} \ No newline at end of file diff --git a/translator/translate/otel/processor/metricstransformprocessor/translator.go b/translator/translate/otel/processor/metricstransformprocessor/translator.go index 4b83e190fc..dfbd37f788 100644 --- a/translator/translate/otel/processor/metricstransformprocessor/translator.go +++ b/translator/translate/otel/processor/metricstransformprocessor/translator.go @@ -4,6 +4,7 @@ package metricstransformprocessor import ( + _ "embed" "fmt" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor" @@ -16,6 +17,9 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/awscontainerinsight" ) +//go:embed metricstransform_jmx_config.yaml +var metricTransformJmxConfig string + var metricDuplicateTypes = []string{ containerinsightscommon.TypeGpuContainer, containerinsightscommon.TypeGpuPod, @@ -64,6 +68,10 @@ func (t *translator) ID() component.ID { func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { cfg := t.factory.CreateDefaultConfig().(*metricstransformprocessor.Config) + if t.name == common.PipelineNameContainerInsightsJmx { + return common.GetYamlFileToYamlConfig(cfg, metricTransformJmxConfig) + } + transformRules := []map[string]interface{}{ { "include": "apiserver_request_total", diff --git a/translator/translate/otel/processor/metricstransformprocessor/translator_test.go b/translator/translate/otel/processor/metricstransformprocessor/translator_test.go index c22498c789..f6357d3af9 100644 --- a/translator/translate/otel/processor/metricstransformprocessor/translator_test.go +++ b/translator/translate/otel/processor/metricstransformprocessor/translator_test.go @@ -5,13 +5,16 @@ package metricstransformprocessor import ( "fmt" + "path/filepath" "testing" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" + "github.com/aws/amazon-cloudwatch-agent/internal/util/testutil" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) @@ -56,3 +59,18 @@ func TestTranslator(t *testing.T) { }) } } + +func TestContainerInsightsJmx(t *testing.T) { + transl := NewTranslatorWithName(common.PipelineNameContainerInsightsJmx).(*translator) + expectedCfg := transl.factory.CreateDefaultConfig().(*metricstransformprocessor.Config) + c := testutil.GetConf(t, "metricstransform_jmx_config.yaml") + require.NoError(t, c.Unmarshal(&expectedCfg)) + + conf := confmap.NewFromStringMap(testutil.GetJson(t, filepath.Join("testdata", "config.json"))) + translatedCfg, err := transl.Translate(conf) + assert.NoError(t, err) + actualCfg, ok := translatedCfg.(*metricstransformprocessor.Config) + assert.True(t, ok) + assert.Equal(t, len(expectedCfg.Transforms), len(actualCfg.Transforms)) + +} diff --git a/translator/translate/otel/processor/resourceprocessor/testdata/config.json b/translator/translate/otel/processor/resourceprocessor/testdata/config.json new file mode 100644 index 0000000000..91cabc00b8 --- /dev/null +++ b/translator/translate/otel/processor/resourceprocessor/testdata/config.json @@ -0,0 +1,14 @@ +{ + "agent": { + "debug": true + }, + "logs": { + "metrics_collected": { + "kubernetes": { + "cluster_name": "TestCluster23", + "metrics_collection_interval": 30 + } + } + } + +} \ No newline at end of file diff --git a/translator/translate/otel/processor/resourceprocessor/testdata/config.yaml b/translator/translate/otel/processor/resourceprocessor/testdata/config.yaml new file mode 100644 index 0000000000..5a2083d167 --- /dev/null +++ b/translator/translate/otel/processor/resourceprocessor/testdata/config.yaml @@ -0,0 +1,10 @@ +attributes: + - key: "Namespace" + from_attribute: "k8s.namespace.name" + action: "insert" + - key: "ClusterName" + value: "TestCluster" + action: "upsert" + - key: "NodeName" + from_attribute: "host.name" + action: "insert" diff --git a/translator/translate/otel/processor/resourceprocessor/translator.go b/translator/translate/otel/processor/resourceprocessor/translator.go index d8b77a5c37..7b39f3de9d 100644 --- a/translator/translate/otel/processor/resourceprocessor/translator.go +++ b/translator/translate/otel/processor/resourceprocessor/translator.go @@ -5,6 +5,7 @@ package resourceprocessor import ( "fmt" + "os" "strconv" "strings" @@ -13,7 +14,9 @@ import ( "go.opentelemetry.io/collector/confmap" "go.opentelemetry.io/collector/processor" + "github.com/aws/amazon-cloudwatch-agent/translator/config" "github.com/aws/amazon-cloudwatch-agent/translator/context" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/logs/util" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) @@ -23,7 +26,11 @@ type translator struct { factory processor.Factory } -var _ common.Translator[component.Config] = (*translator)(nil) +var ( + baseKey = common.ConfigKey(common.LogsKey, common.MetricsCollectedKey) + k8sKey = common.ConfigKey(baseKey, common.KubernetesKey) + _ common.Translator[component.Config] = (*translator)(nil) +) func NewTranslator(opts ...common.TranslatorOption) common.Translator[component.Config] { t := &translator{factory: resourceprocessor.NewFactory()} @@ -46,7 +53,7 @@ func (t *translator) ID() component.ID { // Translate creates a processor config based on the fields in the // Metrics section of the JSON config. func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { - if conf == nil || !conf.IsSet(common.JmxConfigKey) { + if conf == nil || (!conf.IsSet(common.JmxConfigKey) && t.Name() != common.PipelineNameContainerInsightsJmx) { return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: common.JmxConfigKey} } @@ -54,6 +61,8 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { var attributes []any if strings.HasPrefix(t.Name(), common.PipelineNameJmx) { attributes = t.getJMXAttributes(conf) + } else if t.Name() == common.PipelineNameContainerInsightsJmx { + attributes = t.getContainerInsightsJMXAttributes(conf) } if len(attributes) == 0 { baseKey := common.JmxConfigKey @@ -65,6 +74,7 @@ func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { c := confmap.NewFromStringMap(map[string]any{ "attributes": attributes, }) + if err := c.Unmarshal(&cfg); err != nil { return nil, fmt.Errorf("unable to unmarshal resource processor: %w", err) } @@ -101,3 +111,29 @@ func (t *translator) getJMXAttributes(conf *confmap.Conf) []any { } return attributes } + +func (t *translator) getContainerInsightsJMXAttributes(conf *confmap.Conf) []any { + clusterName, ok := common.GetString(conf, common.ConfigKey(k8sKey, "cluster_name")) + + if !ok { + clusterName = util.GetClusterNameFromEc2Tagger() + } + nodeName := os.Getenv(config.HOST_NAME) + return []any{ + map[string]any{ + "key": "Namespace", + "from_attribute": "k8s.namespace.name", + "action": "insert", + }, + map[string]any{ + "key": "ClusterName", + "value": clusterName, // Ensure 'clusterName' is defined earlier + "action": "upsert", + }, + map[string]any{ + "key": "NodeName", + "value": nodeName, + "action": "insert", + }, + } +} diff --git a/translator/translate/otel/processor/resourceprocessor/translator_test.go b/translator/translate/otel/processor/resourceprocessor/translator_test.go index 5a76b2feb8..d80f54286c 100644 --- a/translator/translate/otel/processor/resourceprocessor/translator_test.go +++ b/translator/translate/otel/processor/resourceprocessor/translator_test.go @@ -4,13 +4,16 @@ package resourceprocessor import ( + "path/filepath" "testing" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" + "github.com/aws/amazon-cloudwatch-agent/internal/util/testutil" "github.com/aws/amazon-cloudwatch-agent/translator/context" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) @@ -187,3 +190,18 @@ func TestTranslator(t *testing.T) { }) } } + +func TestContainerInsightsJmx(t *testing.T) { + transl := NewTranslator(common.WithName(common.PipelineNameContainerInsightsJmx)).(*translator) + expectedCfg := transl.factory.CreateDefaultConfig().(*resourceprocessor.Config) + c := testutil.GetConf(t, filepath.Join("testdata", "config.yaml")) + require.NoError(t, c.Unmarshal(&expectedCfg)) + + conf := confmap.NewFromStringMap(testutil.GetJson(t, filepath.Join("testdata", "config.json"))) + translatedCfg, err := transl.Translate(conf) + assert.NoError(t, err) + actualCfg, ok := translatedCfg.(*resourceprocessor.Config) + assert.True(t, ok) + assert.Equal(t, len(actualCfg.AttributesActions), len(expectedCfg.AttributesActions)) + +} diff --git a/translator/translate/otel/processor/transformprocessor/testdata/config.json b/translator/translate/otel/processor/transformprocessor/testdata/config.json new file mode 100644 index 0000000000..760aff7ca9 --- /dev/null +++ b/translator/translate/otel/processor/transformprocessor/testdata/config.json @@ -0,0 +1,17 @@ +{ + "agent": { + "region": "us-east-1" + }, + "logs": { + "metrics_collected": { + "emf": { + }, + "kubernetes": { + "cluster_name": "TestCluster", + "metrics_collection_interval": 30 + } + }, + "force_flush_interval": 5, + "endpoint_override":"https://fake_endpoint" + } +} \ No newline at end of file diff --git a/translator/translate/otel/processor/transformprocessor/transform_jmx_config.yaml b/translator/translate/otel/processor/transformprocessor/transform_jmx_config.yaml new file mode 100644 index 0000000000..32b6d8c32b --- /dev/null +++ b/translator/translate/otel/processor/transformprocessor/transform_jmx_config.yaml @@ -0,0 +1,28 @@ +metric_statements: + - context: resource + statements: + - keep_keys(attributes, ["ClusterName", "Namespace", "NodeName"]) + - context: metric + statements: + - set(unit, "Bytes") where name == "jvm.memory.heap.used" + - set(unit, "Bytes") where name == "jvm.memory.nonheap.used" + - set(unit, "Bytes") where name == "jvm.memory.pool.used" + - set(unit, "Bytes") where name == "jvm.system.swap.space.total" + - set(unit, "Bytes") where name == "jvm.system.swap.space.free" + - set(unit, "Bytes") where name == "jvm.system.physical.memory.total" + - set(unit, "Bytes") where name == "jvm.system.physical.memory.free" + - set(unit, "Count") where name == "jvm.threads.count" + - set(unit, "Count") where name == "jvm.daemon_threads.count" + - set(unit, "Count") where name == "jvm.classes.loaded" + - set(unit, "Count") where name == "jvm.system.cpu.utilization" + - set(unit, "Count") where name == "jvm.cpu.recent_utilization" + - set(unit, "Count") where name == "jvm.open_file_descriptor.count" + - set(unit, "Count") where name == "jvm.system.available.processors" + - set(unit, "Bytes") where name == "tomcat.traffic.received" + - set(unit, "Bytes") where name == "tomcat.traffic.sent" + - set(unit, "Count") where name == "tomcat.sessions" + - set(unit, "Count") where name == "tomcat.rejected_sessions" + - set(unit, "Count") where name == "tomcat.request_count" + - set(unit, "Count") where name == "tomcat.errors" + - set(unit, "Milliseconds") where name == "tomcat.processing_time" + diff --git a/translator/translate/otel/processor/transformprocessor/translate.go b/translator/translate/otel/processor/transformprocessor/translate.go new file mode 100644 index 0000000000..52aaddcb76 --- /dev/null +++ b/translator/translate/otel/processor/transformprocessor/translate.go @@ -0,0 +1,45 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package transformprocessor + +import ( + _ "embed" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + "go.opentelemetry.io/collector/processor" + + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" +) + +//go:embed transform_jmx_config.yaml +var transformJmxConfig string + +type translator struct { + name string + factory processor.Factory +} + +var _ common.Translator[component.Config] = (*translator)(nil) + +func NewTranslatorWithName(name string) common.Translator[component.Config] { + return &translator{name, transformprocessor.NewFactory()} +} + +func (t *translator) ID() component.ID { + return component.NewIDWithName(t.factory.Type(), t.name) +} + +func (t *translator) Translate(conf *confmap.Conf) (component.Config, error) { + if !(conf != nil && conf.IsSet(common.ContainerInsightsConfigKey)) { + return nil, &common.MissingKeyError{ID: t.ID(), JsonKey: common.ContainerInsightsConfigKey} + } + cfg := t.factory.CreateDefaultConfig().(*transformprocessor.Config) + if t.name == common.PipelineNameContainerInsightsJmx { + return common.GetYamlFileToYamlConfig(cfg, transformJmxConfig) + } + + return cfg, nil +} diff --git a/translator/translate/otel/processor/transformprocessor/translator_test.go b/translator/translate/otel/processor/transformprocessor/translator_test.go new file mode 100644 index 0000000000..6741c47f13 --- /dev/null +++ b/translator/translate/otel/processor/transformprocessor/translator_test.go @@ -0,0 +1,78 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package transformprocessor + +import ( + _ "embed" + "path/filepath" + "testing" + + "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + + "github.com/aws/amazon-cloudwatch-agent/internal/util/testutil" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" +) + +func TestTranslator(t *testing.T) { + factory := transformprocessor.NewFactory() + + testCases := map[string]struct { + translator common.Translator[component.Config] + input map[string]any + index int + wantID string + want string + wantErr error + }{ + "NoContainerInsights": { + input: map[string]any{}, + wantErr: &common.MissingKeyError{ + ID: component.NewIDWithName(factory.Type(), "jmx"), + JsonKey: common.ContainerInsightsConfigKey, + }, + }, + } + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + tt := NewTranslatorWithName("jmx") + + conf := confmap.NewFromStringMap(testCase.input) + got, err := tt.Translate(conf) + require.Equal(t, testCase.wantErr, err) + + if err == nil { + require.NotNil(t, got) + gotCfg, ok := got.(*transformprocessor.Config) + + require.True(t, ok) + wantCfg := factory.CreateDefaultConfig() + yamlConfig, err := common.GetYamlFileToYamlConfig(wantCfg, testCase.want) + require.NoError(t, err) + assert.Equal(t, yamlConfig.(*transformprocessor.Config), gotCfg) + + assert.Equal(t, gotCfg, wantCfg) + + } + }) + } +} + +func TestContainerInsightsJmx(t *testing.T) { + transl := NewTranslatorWithName(common.PipelineNameContainerInsightsJmx).(*translator) + expectedCfg := transl.factory.CreateDefaultConfig().(*transformprocessor.Config) + c := testutil.GetConf(t, "transform_jmx_config.yaml") + require.NoError(t, c.Unmarshal(&expectedCfg)) + + conf := confmap.NewFromStringMap(testutil.GetJson(t, filepath.Join("testdata", "config.json"))) + translatedCfg, err := transl.Translate(conf) + assert.NoError(t, err) + actualCfg, ok := translatedCfg.(*transformprocessor.Config) + assert.True(t, ok) + assert.Equal(t, len(expectedCfg.MetricStatements), len(actualCfg.MetricStatements)) +} diff --git a/translator/translate/otel/translate_otel.go b/translator/translate/otel/translate_otel.go index 4fa223394e..aa38c00ef7 100644 --- a/translator/translate/otel/translate_otel.go +++ b/translator/translate/otel/translate_otel.go @@ -23,6 +23,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/applicationsignals" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/containerinsights" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/containerinsightsjmx" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/emf_logs" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/host" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/pipeline/jmx" @@ -67,6 +68,7 @@ func Translate(jsonConfig interface{}, os string) (*otelcol.Config, error) { translators.Set(prometheus.NewTranslator()) translators.Set(emf_logs.NewTranslator()) translators.Set(xray.NewTranslator()) + translators.Set(containerinsightsjmx.NewTranslator()) translators.Merge(jmx.NewTranslators(conf)) translators.Merge(registry) pipelines, err := pipeline.NewTranslator(translators).Translate(conf) diff --git a/translator/translate/otel/translate_otel_test.go b/translator/translate/otel/translate_otel_test.go index da34cf5c60..565c781d2d 100644 --- a/translator/translate/otel/translate_otel_test.go +++ b/translator/translate/otel/translate_otel_test.go @@ -28,9 +28,20 @@ func TestTranslator(t *testing.T) { detector func() (eksdetector.Detector, error) isEKSDataStore func() eksdetector.IsEKSCache }{ - "WithInvalidConfig": { - input: "", - wantErrContains: "invalid json config", + "WithValidConfig": { + input: map[string]interface{}{ + "agent": map[string]interface{}{ + "debug": true, + }, + "logs": map[string]interface{}{ + "metrics_collected": map[string]interface{}{ + "kubernetes": map[string]interface{}{ + "cluster_name": "TestCluster", + "jmx_container_insights": true, + }, + }, + }, + }, }, "WithEmptyConfig": { input: map[string]interface{}{},