[mdatagen] Add semconv reference for metrics in metadata schema (#13920)

<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
#### Description

This PR adds support in mdatagen for defining SemConv reference for
metrics.

Enhancing your metrics/signals in `metadata.yaml` like bellow:

```yaml
metrics:
  system.cpu.time:
    enabled: true
    stability:
      level: beta
    description: Monotonic cumulative sum int metric enabled by default.
    unit: s
    sum:
      value_type: int
      monotonic: true
      aggregation_temporality: cumulative
    semantic_convention:
      semconv_ref: https://github.com/open-telemetry/semantic-conventions/blob/v1.37.0/docs/system/system-metrics.md#metric-systemcputime
```

Will provide the following generated docs:

### system.cpu.time

Monotonic cumulative sum int metric enabled by default.

| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic
| Stability | Semantic Convention |
| ---- | ----------- | ---------- | ----------------------- | ---------
| --------- | ------------------- |
| s | Sum | Int | Cumulative | true | beta |
[system.cpu.time](https://github.com/open-telemetry/semantic-conventions/blob/v1.37.0/docs/system/system-metrics.md#metric-systemcputime)
|

Validation is added so as to ensure that provided SemConv urls match the
top_level semconv version as well as the metric's name.

<!-- Issue number if applicable -->
#### Link to tracking issue
Related to
#13910
and
#13297.

<!--Describe what testing was performed and which tests were added.-->
#### Testing
Tuned

<!--Describe the documentation added.-->
#### Documentation
Tuned

<!--Please delete paragraphs that you did not use before submitting.-->

---------

Signed-off-by: ChrsMark <[email protected]>

Author: ChrsMark

.chloggen/mdatagen_add_semconv_ref.yaml

@@ -0,0 +1,25 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: cmd/mdatagen
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add semconv reference for metrics
+
+# One or more tracking issues or pull requests related to the change
+issues: [13920]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

cmd/mdatagen/internal/loader_test.go

@@ -47,7 +47,7 @@ func TestLoadMetadata(t *testing.T) {
 				GithubProject:        "open-telemetry/opentelemetry-collector",
 				GeneratedPackageName: "metadata",
 				Type:                 "sample",
-				SemConvVersion:       "1.9.0",
+				SemConvVersion:       "1.37.0",
 				PackageName:          "go.opentelemetry.io/collector/cmd/mdatagen/internal/samplereceiver",
 				Status: &Status{
 					DisableCodeCov: true,
@@ -261,6 +261,21 @@ func TestLoadMetadata(t *testing.T) {
 							Mono:                   Mono{Monotonic: true},
 						},
 					},
+					"system.cpu.time": {
+						Signal: Signal{
+							Enabled:               true,
+							Stability:             Stability{Level: component.StabilityLevelBeta},
+							SemanticConvention:    &SemanticConvention{SemanticConventionRef: "https://github.com/open-telemetry/semantic-conventions/blob/v1.37.0/docs/system/system-metrics.md#metric-systemcputime"},
+							Description:           "Monotonic cumulative sum int metric enabled by default.",
+							ExtendedDocumentation: "The metric will be become optional soon.",
+						},
+						Unit: strPtr("s"),
+						Sum: &Sum{
+							MetricValueType:        MetricValueType{pmetric.NumberDataPointValueTypeInt},
+							AggregationTemporality: AggregationTemporality{Aggregation: pmetric.AggregationTemporalityCumulative},
+							Mono:                   Mono{Monotonic: true},
+						},
+					},
 					"optional.metric": {
 						Signal: Signal{
 							Enabled:     false,
@@ -524,6 +539,11 @@ func TestLoadMetadata(t *testing.T) {
 			want:    Metadata{},
 			wantErr: "decoding failed due to the following error(s):\n\n'metrics[default.metric]' decoding failed due to the following error(s):\n\n'stability' decoding failed due to the following error(s):\n\n'level' unsupported stability level: \"development42\"",
 		},
+		{
+			name:    "testdata/invalid_metric_semconvref.yaml",
+			want:    Metadata{},
+			wantErr: "metric \"default.metric\": invalid semantic-conventions URL: want https://github.com/open-telemetry/semantic-conventions/blob/v1.37.2/*#metric-defaultmetric, got \"https://github.com/open-telemetry/semantic-conventions/blob/v1.38.0/docs/system/system-metrics.md#metric-systemcputime\"",
+		},
 		{
 			name:    "testdata/no_metric_stability.yaml",
 			want:    Metadata{},

cmd/mdatagen/internal/metadata.go

@@ -181,8 +181,8 @@ func (md *Metadata) validateMetricsAndEvents() error {
 	var errs error
 	usedAttrs := map[AttributeName]bool{}
 	errs = errors.Join(errs,
-		validateMetrics(md.Metrics, md.Attributes, usedAttrs),
-		validateMetrics(md.Telemetry.Metrics, md.Attributes, usedAttrs),
+		validateMetrics(md.Metrics, md.Attributes, usedAttrs, md.SemConvVersion),
+		validateMetrics(md.Telemetry.Metrics, md.Attributes, usedAttrs, md.SemConvVersion),
 		validateEvents(md.Events, md.Attributes, usedAttrs),
 		md.validateAttributes(usedAttrs))
 	return errs
@@ -226,10 +226,10 @@ func (md *Metadata) supportsSignal(signal string) bool {
 	return false
 }
 
-func validateMetrics(metrics map[MetricName]Metric, attributes map[AttributeName]Attribute, usedAttrs map[AttributeName]bool) error {
+func validateMetrics(metrics map[MetricName]Metric, attributes map[AttributeName]Attribute, usedAttrs map[AttributeName]bool, semConvVersion string) error {
 	var errs error
 	for mn, m := range metrics {
-		if err := m.validate(); err != nil {
+		if err := m.validate(mn, semConvVersion); err != nil {
 			errs = errors.Join(errs, fmt.Errorf(`metric "%v": %w`, mn, err))
 			continue
 		}
@@ -367,6 +367,10 @@ func (mvt ValueType) Primitive() string {
 	}
 }
 
+type SemanticConvention struct {
+	SemanticConventionRef string `mapstructure:"ref"`
+}
+
 type Warnings struct {
 	// A warning that will be displayed if the field is enabled in user config.
 	IfEnabled string `mapstructure:"if_enabled"`
@@ -478,6 +482,9 @@ type Signal struct {
 	// Description of the signal.
 	Description string `mapstructure:"description"`
 
+	// The semantic convention reference of the signal.
+	SemanticConvention *SemanticConvention `mapstructure:"semantic_convention"`
+
 	// The stability level of the signal.
 	Stability Stability `mapstructure:"stability"`
 

cmd/mdatagen/internal/metric.go

@@ -6,6 +6,8 @@ package internal // import "go.opentelemetry.io/collector/cmd/mdatagen/internal"
 import (
 	"errors"
 	"fmt"
+	"regexp"
+	"strings"
 
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
@@ -15,6 +17,8 @@ import (
 	"go.opentelemetry.io/collector/pdata/pmetric"
 )
 
+var reNonAlnum = regexp.MustCompile(`[^a-z0-9]+`)
+
 type MetricName string
 
 func (mn MetricName) Render() (string, error) {
@@ -69,7 +73,7 @@ func (s *Stability) Unmarshal(parser *confmap.Conf) error {
 	return parser.Unmarshal(s)
 }
 
-func (m *Metric) validate() error {
+func (m *Metric) validate(metricName MetricName, semConvVersion string) error {
 	var errs error
 	if m.Sum == nil && m.Gauge == nil && m.Histogram == nil {
 		errs = errors.Join(errs, errors.New("missing metric type key, "+
@@ -91,9 +95,49 @@ func (m *Metric) validate() error {
 	if m.Gauge != nil {
 		errs = errors.Join(errs, m.Gauge.Validate())
 	}
+	if m.SemanticConvention != nil {
+		if err := validateSemConvMetricURL(m.SemanticConvention.SemanticConventionRef, semConvVersion, string(metricName)); err != nil {
+			errs = errors.Join(errs, err)
+		}
+	}
 	return errs
 }
 
+func metricAnchor(metricName string) string {
+	m := strings.ToLower(strings.TrimSpace(metricName))
+	m = reNonAlnum.ReplaceAllString(m, "")
+	return "metric-" + m
+}
+
+// validateSemConvMetricURL verifies the URL matches exactly:
+// https://github.com/open-telemetry/semantic-conventions/blob/<semConvVersion>/*#metric-<metricName>
+func validateSemConvMetricURL(rawURL, semConvVersion, metricName string) error {
+	if strings.TrimSpace(rawURL) == "" {
+		return errors.New("url is empty")
+	}
+	if strings.TrimSpace(semConvVersion) == "" {
+		return errors.New("semConvVersion is empty")
+	}
+	if strings.TrimSpace(metricName) == "" {
+		return errors.New("metricName is empty")
+	}
+	semConvVersion = "v" + semConvVersion
+
+	anchor := metricAnchor(metricName)
+	// Build a strict regex that enforces https, repo, blob, given version, any doc path, and exact anchor.
+	pattern := fmt.Sprintf(`^https://github\.com/open-telemetry/semantic-conventions/blob/%s/[^#\s]+#%s$`,
+		semConvVersion,
+		anchor,
+	)
+	re := regexp.MustCompile(pattern)
+	if !re.MatchString(rawURL) {
+		return fmt.Errorf(
+			"invalid semantic-conventions URL: want https://github.com/open-telemetry/semantic-conventions/blob/%s/*#%s, got %q",
+			semConvVersion, anchor, rawURL)
+	}
+	return nil
+}
+
 func (m *Metric) Unmarshal(parser *confmap.Conf) error {
 	if !parser.IsSet("enabled") {
 		return errors.New("missing required field: `enabled`")

cmd/mdatagen/internal/samplereceiver/documentation.md

@@ -63,6 +63,16 @@ Monotonic cumulative sum int metric with string input_type enabled by default.
 | slice_attr | Attribute with a slice value. | Any Slice | Recommended |
 | map_attr | Attribute with a map value. | Any Map | Recommended |
 
+### system.cpu.time
+
+Monotonic cumulative sum int metric enabled by default.
+
+The metric will be become optional soon.
+
+| Unit | Metric Type | Value Type | Aggregation Temporality | Monotonic | Stability | Semantic Convention |
+| ---- | ----------- | ---------- | ----------------------- | --------- | --------- | ------------------- |
+| s | Sum | Int | Cumulative | true | Beta | [system.cpu.time](https://github.com/open-telemetry/semantic-conventions/blob/v1.37.0/docs/system/system-metrics.md#metric-systemcputime) |
+
 ## Optional Metrics
 
 The following metrics are not emitted by default. Each of them can be enabled by applying the following configuration: