Add Nvme scrape configs for LIS CSI (#382)

Author: agarakan

exporter/awsemfexporter/metric_translator.go

@@ -147,7 +147,8 @@ func (mt metricTranslator) translateOTelToGroupedMetric(rm pmetric.ResourceMetri
 			strings.HasPrefix(serviceName.Str(), "containerInsightsDCGMExporterScraper") ||
 			strings.HasPrefix(serviceName.Str(), "containerInsightsNeuronMonitorScraper") ||
 			strings.HasPrefix(serviceName.Str(), "containerInsightsKueueMetricsScraper") ||
-			strings.HasPrefix(serviceName.Str(), "containerInsightsNVMeEBSScraper") {
+			strings.HasPrefix(serviceName.Str(), "containerInsightsNVMeEBSScraper") ||
+			strings.HasPrefix(serviceName.Str(), "containerInsightsNVMeLISScraper") {
 			// the prometheus metrics that come from the container insight receiver need to be clearly tagged as coming from container insights
 			metricReceiver = containerInsightsReceiver
 		}

exporter/awsemfexporter/metric_translator_test.go

@@ -280,6 +280,8 @@ func TestTranslateOtToGroupedMetric(t *testing.T) {
 	kueueMetric.Resource().Attributes().PutStr(conventions.AttributeServiceName, "containerInsightsKueueMetricsScraper")
 	nvmeMetric := createTestResourceMetricsHelper(defaultNumberOfTestMetrics + 1)
 	nvmeMetric.Resource().Attributes().PutStr(conventions.AttributeServiceName, "containerInsightsNVMeEBSScraper")
+	nvmeLisMetric := createTestResourceMetricsHelper(defaultNumberOfTestMetrics + 1)
+	nvmeLisMetric.Resource().Attributes().PutStr(conventions.AttributeServiceName, "containerInsightsNVMeLISScraper")
 
 	counterSumMetrics := map[string]*metricInfo{
 		"spanCounter": {
@@ -410,6 +412,19 @@ func TestTranslateOtToGroupedMetric(t *testing.T) {
 			"myServiceNS/containerInsightsNVMeEBSScraper",
 			containerInsightsReceiver,
 		},
+		{
+			"nvme lis receiver",
+			nvmeLisMetric,
+			map[string]string{
+				"isItAnError": "false",
+				"spanName":    "testSpan",
+			},
+			map[string]string{
+				"spanName": "testSpan",
+			},
+			"myServiceNS/containerInsightsNVMeLISScraper",
+			containerInsightsReceiver,
+		},
 	}
 
 	for _, tc := range testCases {

receiver/awscontainerinsightreceiver/internal/nvme/metric_unit.go

@@ -19,9 +19,21 @@ const (
 	ebsExceededEC2IOPSTime = "aws_ebs_csi_ec2_exceeded_iops_seconds_total"
 	ebsExceededEC2TPTime   = "aws_ebs_csi_ec2_exceeded_tp_seconds_total"
 	ebsVolumeQueueLength   = "aws_ebs_csi_volume_queue_length"
+
+	// LIS Original Metric Names
+	lisReadOpsTotal      = "aws_ec2_instance_store_csi_read_ops_total"
+	lisWriteOpsTotal     = "aws_ec2_instance_store_csi_write_ops_total"
+	lisReadBytesTotal    = "aws_ec2_instance_store_csi_read_bytes_total"
+	lisWriteBytesTotal   = "aws_ec2_instance_store_csi_write_bytes_total"
+	lisReadTime          = "aws_ec2_instance_store_csi_read_seconds_total"
+	lisWriteTime         = "aws_ec2_instance_store_csi_write_seconds_total"
+	lisExceededIOPSTime  = "aws_ec2_instance_store_csi_ec2_exceeded_iops_seconds_total"
+	lisExceededTPTime    = "aws_ec2_instance_store_csi_ec2_exceeded_tp_seconds_total"
+	lisVolumeQueueLength = "aws_ec2_instance_store_csi_volume_queue_length"
 )
 
 var MetricToUnit = map[string]string{
+	// EBS metrics
 	ebsReadOpsTotal:        containerinsight.UnitCount,
 	ebsWriteOpsTotal:       containerinsight.UnitCount,
 	ebsReadBytesTotal:      containerinsight.UnitBytes,
@@ -33,4 +45,15 @@ var MetricToUnit = map[string]string{
 	ebsExceededEC2IOPSTime: containerinsight.UnitSecond,
 	ebsExceededEC2TPTime:   containerinsight.UnitSecond,
 	ebsVolumeQueueLength:   containerinsight.UnitCount,
+
+	// LIS metrics
+	lisReadOpsTotal:      containerinsight.UnitCount,
+	lisWriteOpsTotal:     containerinsight.UnitCount,
+	lisReadBytesTotal:    containerinsight.UnitBytes,
+	lisWriteBytesTotal:   containerinsight.UnitBytes,
+	lisReadTime:          containerinsight.UnitSecond,
+	lisWriteTime:         containerinsight.UnitSecond,
+	lisExceededIOPSTime:  containerinsight.UnitSecond,
+	lisExceededTPTime:    containerinsight.UnitSecond,
+	lisVolumeQueueLength: containerinsight.UnitCount,
 }

receiver/awscontainerinsightreceiver/internal/nvme/nvme_ebs_scraper_config.go

@@ -29,7 +29,7 @@ type hostInfoProvider interface {
 	GetInstanceType() string
 }
 
-func GetScraperConfig(hostInfoProvider hostInfoProvider) *config.ScrapeConfig {
+func GetEbsScraperConfig(hostInfoProvider hostInfoProvider) *config.ScrapeConfig {
 	return &config.ScrapeConfig{
 		ScrapeInterval:         model.Duration(collectionInterval),
 		ScrapeTimeout:          model.Duration(collectionInterval),

receiver/awscontainerinsightreceiver/internal/nvme/nvme_ebs_scraper_test.go

@@ -141,7 +141,7 @@ func TestNewNVMEScraperEndToEnd(t *testing.T) {
 		Consumer:          mConsumer,
 		Host:              componenttest.NewNopHost(),
 		HostInfoProvider:  mockHostInfoProvider{},
-		ScraperConfigs:    GetScraperConfig(mockHostInfoProvider{}),
+		ScraperConfigs:    GetEbsScraperConfig(mockHostInfoProvider{}),
 		Logger:            settings.Logger,
 	})
 	assert.NoError(t, err)

receiver/awscontainerinsightreceiver/internal/nvme/nvme_lis_scraper_config.go

@@ -0,0 +1,104 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package nvme // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscontainerinsightreceiver/internal/nvme"
+
+import (
+	"os"
+	"time"
+
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/prometheus/config"
+	"github.com/prometheus/prometheus/discovery"
+	"github.com/prometheus/prometheus/discovery/kubernetes"
+	"github.com/prometheus/prometheus/model/relabel"
+
+	ci "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/containerinsight"
+)
+
+const (
+	lisCollectionInterval        = 60 * time.Second
+	lisJobName                   = "containerInsightsNVMeLISScraper"
+	lisScraperMetricsPath        = "/metrics"
+	lisScraperK8sServiceSelector = "app=nvme-csi-plugin"
+	lisNamespaceDiscoveryName    = "kube-system"
+)
+
+func GetLisScraperConfig(hostInfoProvider hostInfoProvider) *config.ScrapeConfig {
+	return &config.ScrapeConfig{
+		ScrapeInterval:         model.Duration(lisCollectionInterval),
+		ScrapeTimeout:          model.Duration(lisCollectionInterval),
+		ScrapeProtocols:        config.DefaultScrapeProtocols,
+		JobName:                lisJobName,
+		Scheme:                 "http",
+		MetricsPath:            lisScraperMetricsPath,
+		ScrapeFallbackProtocol: config.PrometheusText0_0_4,
+		ServiceDiscoveryConfigs: discovery.Configs{
+			&kubernetes.SDConfig{
+				Role: kubernetes.RoleService,
+				NamespaceDiscovery: kubernetes.NamespaceDiscovery{
+					Names: []string{lisNamespaceDiscoveryName},
+				},
+				Selectors: []kubernetes.SelectorConfig{
+					{
+						Role:  kubernetes.RoleService,
+						Label: lisScraperK8sServiceSelector,
+					},
+				},
+			},
+		},
+		MetricRelabelConfigs: getLisMetricRelabelConfig(hostInfoProvider),
+	}
+}
+
+func getLisMetricRelabelConfig(hostInfoProvider hostInfoProvider) []*relabel.Config {
+	return []*relabel.Config{
+		{
+			SourceLabels: model.LabelNames{"__name__"},
+			Regex:        relabel.MustNewRegexp("aws_ec2_instance_store_csi_.*"),
+			Action:       relabel.Keep,
+		},
+
+		// Below metrics are histogram type which are not supported for container insights yet
+		{
+			SourceLabels: model.LabelNames{"__name__"},
+			Regex:        relabel.MustNewRegexp(".*_bucket|.*_sum|.*_count.*"),
+			Action:       relabel.Drop,
+		},
+		// Below metrics are NVMe data collection metrics which are not supported to maintain parity with EBS NVMe metrics
+		{
+			SourceLabels: model.LabelNames{"__name__"},
+			Regex:        relabel.MustNewRegexp(".*_nvme_collector_.*"),
+			Action:       relabel.Drop,
+		},
+		// Inject static values (clusterName/instanceId/nodeName/volumeID)
+		{
+			SourceLabels: model.LabelNames{"instance_id"},
+			TargetLabel:  ci.NodeNameKey,
+			Regex:        relabel.MustNewRegexp(".*"),
+			Replacement:  os.Getenv("HOST_NAME"),
+			Action:       relabel.Replace,
+		},
+		{
+			SourceLabels: model.LabelNames{"instance_id"},
+			TargetLabel:  ci.ClusterNameKey,
+			Regex:        relabel.MustNewRegexp(".*"),
+			Replacement:  hostInfoProvider.GetClusterName(),
+			Action:       relabel.Replace,
+		},
+		{
+			SourceLabels: model.LabelNames{"instance_id"},
+			TargetLabel:  ci.InstanceID,
+			Regex:        relabel.MustNewRegexp("(.*)"),
+			Replacement:  "${1}",
+			Action:       relabel.Replace,
+		},
+		{
+			SourceLabels: model.LabelNames{"volume_id"},
+			TargetLabel:  ci.VolumeID,
+			Regex:        relabel.MustNewRegexp("(.*)"),
+			Replacement:  "${1}",
+			Action:       relabel.Replace,
+		},
+	}
+}