[cmd/mdatagen] Add basic support for entities to metadata.yaml schema (#14053)

When entities are defined, mdatagen generates
`AssociateWith{EntityType}()` methods on ResourceBuilder that associate
resources with entity types using the entity refs API. The `entities`
section is backward compatible - existing metadata.yaml files without
entities continue to work as before. This change is fully additive and
conditionally executed only if metadata.yaml has `entities` defined.

The generated Go API is experimental and will change to entities
builder. However, for now, it allows introducing concept of entities in
mdatagen and emit entities attached to the resource.

The new metadata.yaml syntax for defining entities adopts [Weaver
v2](https://github.com/open-telemetry/weaver/blob/main/schemas/semconv-syntax.v2.md#entities-definition)

Updates
#14051

Author: dmitryax

.chloggen/mdatagen-entities-support.yaml

@@ -0,0 +1,28 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: cmd/mdatagen
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "`metadata.yaml` now supports an optional `entities` section to organize resource attributes into logical entities with identity and description attributes"
+
+# One or more tracking issues or pull requests related to the change
+issues: [14051]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  When entities are defined, mdatagen generates `AssociateWith{EntityType}()` methods on ResourceBuilder
+  that associate resources with entity types using the entity refs API. The entities section is backward
+  compatible - existing metadata.yaml files without entities continue to work as before.
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

cmd/mdatagen/go.mod

@@ -16,6 +16,7 @@ require (
 	go.opentelemetry.io/collector/consumer/consumertest v0.139.0
 	go.opentelemetry.io/collector/filter v0.139.0
 	go.opentelemetry.io/collector/pdata v1.45.0
+	go.opentelemetry.io/collector/pdata/xpdata v0.139.0
 	go.opentelemetry.io/collector/pipeline v1.45.0
 	go.opentelemetry.io/collector/processor v1.45.0
 	go.opentelemetry.io/collector/processor/processortest v0.139.0

cmd/mdatagen/internal/command.go

@@ -462,6 +462,7 @@ func validateYAMLKeyOrder(raw []byte) error {
 	}
 	for _, p := range [][]string{
 		{"resource_attributes"},
+		{"entities"},
 		{"attributes"},
 		{"metrics"},
 		{"events"},

cmd/mdatagen/internal/metadata.go

@@ -30,6 +30,8 @@ type Metadata struct {
 	SemConvVersion string `mapstructure:"sem_conv_version"`
 	// ResourceAttributes that can be emitted by the component.
 	ResourceAttributes map[AttributeName]Attribute `mapstructure:"resource_attributes"`
+	// Entities organizes resource attributes into logical entities.
+	Entities []Entity `mapstructure:"entities"`
 	// Attributes emitted by one or more metrics.
 	Attributes map[AttributeName]Attribute `mapstructure:"attributes"`
 	// Metrics that can be emitted by the component.
@@ -56,6 +58,10 @@ func (md Metadata) GetCodeCovComponentID() string {
 	return strings.ReplaceAll(md.Status.Class+"_"+md.Type, "/", "_")
 }
 
+func (md Metadata) HasEntities() bool {
+	return len(md.Entities) > 0
+}
+
 func (md *Metadata) Validate() error {
 	var errs error
 	if err := md.validateType(); err != nil {
@@ -75,6 +81,10 @@ func (md *Metadata) Validate() error {
 		errs = errors.Join(errs, err)
 	}
 
+	if err := md.validateEntities(); err != nil {
+		errs = errors.Join(errs, err)
+	}
+
 	if err := md.validateMetricsAndEvents(); err != nil {
 		errs = errors.Join(errs, err)
 	}
@@ -122,6 +132,51 @@ func (md *Metadata) validateResourceAttributes() error {
 	return errs
 }
 
+func (md *Metadata) validateEntities() error {
+	var errs error
+	usedAttrs := make(map[AttributeName]string)
+	seenTypes := make(map[string]bool)
+
+	for _, entity := range md.Entities {
+		if entity.Type == "" {
+			errs = errors.Join(errs, errors.New("entity type cannot be empty"))
+			continue
+		}
+		if seenTypes[entity.Type] {
+			errs = errors.Join(errs, fmt.Errorf(`duplicate entity type: %v`, entity.Type))
+		}
+		seenTypes[entity.Type] = true
+
+		if entity.Brief == "" {
+			errs = errors.Join(errs, fmt.Errorf(`entity "%v": brief is required`, entity.Type))
+		}
+		if len(entity.Identity) == 0 {
+			errs = errors.Join(errs, fmt.Errorf(`entity "%v": identity is required`, entity.Type))
+		}
+		for _, ref := range entity.Identity {
+			if _, ok := md.ResourceAttributes[ref.Ref]; !ok {
+				errs = errors.Join(errs, fmt.Errorf(`entity "%v": identity refers to undefined resource attribute: %v`, entity.Type, ref.Ref))
+			}
+			if otherEntity, used := usedAttrs[ref.Ref]; used {
+				errs = errors.Join(errs, fmt.Errorf(`entity "%v": attribute %v is already used by entity "%v"`, entity.Type, ref.Ref, otherEntity))
+			} else {
+				usedAttrs[ref.Ref] = entity.Type
+			}
+		}
+		for _, ref := range entity.Description {
+			if _, ok := md.ResourceAttributes[ref.Ref]; !ok {
+				errs = errors.Join(errs, fmt.Errorf(`entity "%v": description refers to undefined resource attribute: %v`, entity.Type, ref.Ref))
+			}
+			if otherEntity, used := usedAttrs[ref.Ref]; used {
+				errs = errors.Join(errs, fmt.Errorf(`entity "%v": attribute %v is already used by entity "%v"`, entity.Type, ref.Ref, otherEntity))
+			} else {
+				usedAttrs[ref.Ref] = entity.Type
+			}
+		}
+	}
+	return errs
+}
+
 func (md *Metadata) validateMetricsAndEvents() error {
 	var errs error
 	usedAttrs := map[AttributeName]bool{}
@@ -441,3 +496,21 @@ func (s Signal) HasConditionalAttributes(attrs map[AttributeName]Attribute) bool
 	}
 	return false
 }
+
+type Entity struct {
+	// Type is the type of the entity.
+	Type string `mapstructure:"type"`
+	// Brief is a brief description of the entity.
+	Brief string `mapstructure:"brief"`
+	// Stability is the stability level of the entity.
+	Stability string `mapstructure:"stability"`
+	// Identity contains references to resource attributes that uniquely identify the entity.
+	Identity []EntityAttributeRef `mapstructure:"identity"`
+	// Description contains references to resource attributes that describe the entity.
+	Description []EntityAttributeRef `mapstructure:"description"`
+}
+
+type EntityAttributeRef struct {
+	// Ref is the reference to a resource attribute.
+	Ref AttributeName `mapstructure:"ref"`
+}

cmd/mdatagen/internal/metadata_test.go

@@ -120,12 +120,32 @@ func TestValidate(t *testing.T) {
 			name:    "testdata/no_type_attr.yaml",
 			wantErr: "empty type for attribute: used_attr",
 		},
+		{
+			name:    "testdata/entity_undefined_id_attribute.yaml",
+			wantErr: `entity "host": identity refers to undefined resource attribute: host.missing`,
+		},
+		{
+			name:    "testdata/entity_undefined_description_attribute.yaml",
+			wantErr: `entity "host": description refers to undefined resource attribute: host.missing`,
+		},
+		{
+			name:    "testdata/entity_empty_id_attributes.yaml",
+			wantErr: `entity "host": identity is required`,
+		},
+		{
+			name:    "testdata/entity_duplicate_attributes.yaml",
+			wantErr: `attribute host.name is already used by entity`,
+		},
+		{
+			name:    "testdata/entity_duplicate_types.yaml",
+			wantErr: `duplicate entity type: host`,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			_, err := LoadMetadata(tt.name)
 			require.Error(t, err)
-			require.EqualError(t, err, tt.wantErr)
+			require.ErrorContains(t, err, tt.wantErr)
 		})
 	}
 }

cmd/mdatagen/internal/sampleconnector/documentation.md

@@ -113,3 +113,19 @@ metrics:
 | string.resource.attr_disable_warning | Resource attribute with any string value. | Any Str | true |
 | string.resource.attr_remove_warning | Resource attribute with any string value. | Any Str | false |
 | string.resource.attr_to_be_removed | Resource attribute with any string value. | Any Str | true |
+
+## Entities
+
+The following entities are defined for this component:
+
+### test.entity
+
+A test entity.
+
+**Stability:** stable
+
+**Identity Attributes:**
+- `string.resource.attr`
+
+**Description Attributes:**
+- `map.resource.attr`

cmd/mdatagen/internal/sampleconnector/internal/metadata/generated_resource.go

@@ -65,6 +65,15 @@ resource_attributes:
     warnings:
       if_enabled: This resource_attribute is deprecated and will be removed soon.
 
+entities:
+  - type: test.entity
+    brief: A test entity.
+    stability: stable
+    identity:
+      - ref: string.resource.attr
+    description:
+      - ref: map.resource.attr
+
 attributes:
   boolean_attr:
     description: Attribute with a boolean value.

cmd/mdatagen/internal/sampleconnector/metadata.yaml

@@ -212,6 +212,43 @@ events:
 
 {{- end }}
 
+{{- if .Entities }}
+
+## Entities
+
+The following entities are defined for this component:
+
+{{- range $entity := .Entities }}
+
+### {{ $entity.Type }}
+
+{{ $entity.Brief }}
+
+{{- if $entity.Stability }}
+
+**Stability:** {{ $entity.Stability }}
+{{- end }}
+
+{{- if $entity.Identity }}
+
+**Identity Attributes:**
+{{- range $entity.Identity }}
+- `{{ .Ref }}`
+{{- end }}
+{{- end }}
+
+{{- if $entity.Description }}
+
+**Description Attributes:**
+{{- range $entity.Description }}
+- `{{ .Ref }}`
+{{- end }}
+{{- end }}
+
+{{- end }}
+
+{{- end }}
+
 {{- if .Telemetry.Metrics }}
 
 ## Internal Telemetry

cmd/mdatagen/internal/templates/documentation.md.tmpl

@@ -4,6 +4,9 @@ package {{ .Package }}
 
 import (
 	"go.opentelemetry.io/collector/pdata/pcommon"
+{{- if .HasEntities }}
+	"go.opentelemetry.io/collector/pdata/xpdata/entity"
+{{- end }}
 )
 
 // ResourceBuilder is a helper struct to build resources predefined in metadata.yaml.
@@ -43,6 +46,31 @@ func (rb *ResourceBuilder) Set{{ $name.Render }}(val {{ $attr.Type.Primitive }})
 {{- end }}
 {{ end }}
 
+{{- if .HasEntities }}
+
+{{ range $entity := .Entities }}
+// AssociateWith{{ $entity.Type | publicVar }} associates the resource with entity type "{{ $entity.Type }}".
+// This method is experimental and will be replaced with an entity builder pattern in the future.
+// However, for now, it allows associating resources with entities and producing correct entity references.
+func (rb *ResourceBuilder) AssociateWith{{ $entity.Type | publicVar }}() {
+	entityRef := entity.ResourceEntityRefs(rb.res).AppendEmpty()
+	entityRef.SetType("{{ $entity.Type }}")
+	{{- if $entity.Identity }}
+	idKeys := entityRef.IdKeys()
+	{{- range $entity.Identity }}
+	idKeys.Append("{{ .Ref }}")
+	{{- end }}
+	{{- end }}
+	{{- if $entity.Description }}
+	descKeys := entityRef.DescriptionKeys()
+	{{- range $entity.Description }}
+	descKeys.Append("{{ .Ref }}")
+	{{- end }}
+	{{- end }}
+}
+{{ end }}
+{{- end }}
+
 // Emit returns the built resource and resets the internal builder state.
 func (rb *ResourceBuilder) Emit() pcommon.Resource {
 	r := rb.res