feat(metadata): vendor YAML schemas and add CLI validation command

- Vendor metadata schema YAML files from monorepo into CDK resources directory
- Include fixes for SecretStore required fields and ConnectorBreakingChanges const usage
- Add validate_metadata_file() function with Pydantic validation
- Add 'airbyte-cdk metadata validate' CLI command with JSON and text output
- Update generation script to use vendored YAML files instead of cloning from GitHub
- Regenerate metadata_schema.json with fixed YAML files
- Reorganize connector_metadata module into package structure

This makes the CDK self-contained for metadata validation and removes dependency
on the monorepo for schema files.

Co-Authored-By: AJ Steers <[email protected]>

Author: devin-ai-integration[bot]

airbyte_cdk/cli/airbyte_cdk/__init__.py

@@ -44,6 +44,7 @@
 from airbyte_cdk.cli.airbyte_cdk._connector import connector_cli_group
 from airbyte_cdk.cli.airbyte_cdk._image import image_cli_group
 from airbyte_cdk.cli.airbyte_cdk._manifest import manifest_cli_group
+from airbyte_cdk.cli.airbyte_cdk._metadata import metadata_cli_group
 from airbyte_cdk.cli.airbyte_cdk._secrets import secrets_cli_group
 from airbyte_cdk.cli.airbyte_cdk._version import print_version
 
@@ -78,6 +79,7 @@ def cli(
 
 cli.add_command(connector_cli_group)
 cli.add_command(manifest_cli_group)
+cli.add_command(metadata_cli_group)
 cli.add_command(image_cli_group)
 cli.add_command(secrets_cli_group)
 

airbyte_cdk/cli/airbyte_cdk/_metadata.py

@@ -0,0 +1,66 @@
+# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
+"""CLI commands for metadata validation."""
+
+import json
+import sys
+from pathlib import Path
+
+import rich_click as click
+
+from airbyte_cdk.models.connector_metadata import validate_metadata_file
+
+
+@click.group(name="metadata")
+def metadata_cli_group() -> None:
+    """Commands for working with connector metadata."""
+    pass
+
+
+@metadata_cli_group.command(name="validate")
+@click.option(
+    "--file",
+    "-f",
+    "file_path",
+    type=click.Path(exists=True, path_type=Path),
+    required=True,
+    help="Path to the metadata.yaml file to validate",
+)
+@click.option(
+    "--format",
+    "output_format",
+    type=click.Choice(["json", "text"]),
+    default="text",
+    help="Output format (json or text)",
+)
+def validate_command(file_path: Path, output_format: str) -> None:
+    """Validate a connector metadata.yaml file.
+
+    This command validates a metadata.yaml file against the connector metadata schema
+    and reports any validation errors.
+
+    Examples:
+        airbyte-cdk metadata validate --file metadata.yaml
+        airbyte-cdk metadata validate --file metadata.yaml --format json
+    """
+    result = validate_metadata_file(file_path)
+
+    if output_format == "json":
+        click.echo(result.model_dump_json(indent=2))
+    else:
+        if result.valid:
+            click.secho("✓ Metadata file is valid", fg="green")
+        else:
+            click.secho("✗ Metadata file is invalid", fg="red")
+            click.echo()
+            click.echo("Errors:")
+            for error in result.errors:
+                error_type = error.get("type", "unknown")
+                path = error.get("path", "")
+                message = error.get("message", "")
+
+                if path:
+                    click.echo(f"  • {path}: {message} (type: {error_type})")
+                else:
+                    click.echo(f"  • {message} (type: {error_type})")
+
+    sys.exit(0 if result.valid else 1)

airbyte_cdk/models/connector_metadata/__init__.py

@@ -0,0 +1,21 @@
+"""Connector metadata models and validation."""
+
+from airbyte_cdk.models.connector_metadata.metadata_file import (
+    ConnectorBuildOptions,
+    ConnectorLanguage,
+    ConnectorMetadata,
+    MetadataFile,
+    SuggestedStreams,
+    ValidationResult,
+    validate_metadata_file,
+)
+
+__all__ = [
+    "ConnectorBuildOptions",
+    "ConnectorLanguage",
+    "ConnectorMetadata",
+    "MetadataFile",
+    "SuggestedStreams",
+    "ValidationResult",
+    "validate_metadata_file",
+]

airbyte_cdk/models/connector_metadata.py …dels/connector_metadata/metadata_file.pyairbyte_cdk/models/connector_metadata.py renamed to airbyte_cdk/models/connector_metadata/metadata_file.py airbyte_cdk/models/connector_metadata.py renamed to airbyte_cdk/models/connector_metadata/metadata_file.py

@@ -2,11 +2,13 @@
 
 from __future__ import annotations
 
+import json
 from enum import Enum
 from pathlib import Path
+from typing import Any
 
 import yaml
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, ValidationError
 
 
 class ConnectorLanguage(str, Enum):
@@ -109,3 +111,89 @@ def from_file(
 
         metadata_file = MetadataFile.model_validate(metadata_dict)
         return metadata_file
+
+
+class ValidationResult(BaseModel):
+    """Result of metadata validation."""
+
+    valid: bool = Field(..., description="Whether the metadata is valid")
+    errors: list[dict[str, Any]] = Field(
+        default_factory=list, description="List of validation errors"
+    )
+    metadata: dict[str, Any] | None = Field(None, description="Parsed metadata if available")
+
+
+def validate_metadata_file(file_path: Path) -> ValidationResult:
+    """Validate a metadata.yaml file.
+
+    Args:
+        file_path: Path to the metadata.yaml file to validate
+
+    Returns:
+        ValidationResult with validation status, errors, and parsed metadata
+    """
+    errors = []
+    metadata_dict = None
+
+    try:
+        if not file_path.exists():
+            return ValidationResult(
+                valid=False,
+                errors=[
+                    {"type": "file_not_found", "message": f"Metadata file not found: {file_path}"}
+                ],
+                metadata=None,
+            )
+
+        try:
+            metadata_content = file_path.read_text()
+            metadata_dict = yaml.safe_load(metadata_content)
+        except yaml.YAMLError as e:
+            return ValidationResult(
+                valid=False,
+                errors=[{"type": "yaml_parse_error", "message": f"Failed to parse YAML: {e}"}],
+                metadata=None,
+            )
+
+        if not metadata_dict or "data" not in metadata_dict:
+            return ValidationResult(
+                valid=False,
+                errors=[
+                    {
+                        "type": "missing_field",
+                        "path": "data",
+                        "message": "Missing 'data' field in metadata",
+                    }
+                ],
+                metadata=metadata_dict,
+            )
+
+        try:
+            metadata_file = MetadataFile.model_validate(metadata_dict)
+            return ValidationResult(
+                valid=True,
+                errors=[],
+                metadata=metadata_dict,
+            )
+        except ValidationError as e:
+            for error in e.errors():
+                errors.append(
+                    {
+                        "type": error["type"],
+                        "path": ".".join(str(loc) for loc in error["loc"]),
+                        "message": error["msg"],
+                    }
+                )
+
+            return ValidationResult(
+                valid=False,
+                errors=errors,
+                metadata=metadata_dict,
+            )
+
+    except Exception as e:
+        return ValidationResult(
+            valid=False,
+            errors=[{"type": "unexpected_error", "message": f"Unexpected error: {e}"}],
+            metadata=metadata_dict,
+        )

airbyte_cdk/models/connector_metadata/resources/ActorDefinitionResourceRequirements.yaml

@@ -0,0 +1,30 @@
+---
+"$schema": http://json-schema.org/draft-07/schema#
+"$id": https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ActorDefinitionResourceRequirements.yaml
+title: ActorDefinitionResourceRequirements
+description: actor definition specific resource requirements
+type: object
+# set to false because we need the validations on seeds to be strict. otherwise, we will just add whatever is in the seed file into the db.
+additionalProperties: false
+properties:
+  default:
+    description: if set, these are the requirements that should be set for ALL jobs run for this actor definition.
+    "$ref": ResourceRequirements.yaml
+  jobSpecific:
+    type: array
+    items:
+      "$ref": "#/definitions/JobTypeResourceLimit"
+definitions:
+  JobTypeResourceLimit:
+    description: sets resource requirements for a specific job type for an actor definition. these values override the default, if both are set.
+    type: object
+    # set to false because we need the validations on seeds to be strict. otherwise, we will just add whatever is in the seed file into the db.
+    additionalProperties: false
+    required:
+      - jobType
+      - resourceRequirements
+    properties:
+      jobType:
+        "$ref": JobType.yaml
+      resourceRequirements:
+        "$ref": ResourceRequirements.yaml

airbyte_cdk/models/connector_metadata/resources/AirbyteInternal.yaml

@@ -0,0 +1,32 @@
+---
+"$schema": http://json-schema.org/draft-07/schema#
+"$id": https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/AirbyteInternal.yml
+title: AirbyteInternal
+description: Fields for internal use only
+type: object
+additionalProperties: true
+properties:
+  sl:
+    type: integer
+    enum:
+      - 0
+      - 100
+      - 200
+      - 300
+  ql:
+    type: integer
+    enum:
+      - 0
+      - 100
+      - 200
+      - 300
+      - 400
+      - 500
+      - 600
+  isEnterprise:
+    type: boolean
+    default: false
+  requireVersionIncrementsInPullRequests:
+    type: boolean
+    default: true
+    description: When false, version increment checks will be skipped for this connector

airbyte_cdk/models/connector_metadata/resources/AllowedHosts.yaml

@@ -0,0 +1,13 @@
+---
+"$schema": http://json-schema.org/draft-07/schema#
+"$id": https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/AllowedHosts.yaml
+title: AllowedHosts
+description: A connector's allowed hosts.  If present, the platform will limit communication to only hosts which are listed in `AllowedHosts.hosts`.
+type: object
+additionalProperties: true
+properties:
+  hosts:
+    type: array
+    description: An array of hosts that this connector can connect to.  AllowedHosts not being present for the source or destination means that access to all hosts is allowed.  An empty list here means that no network access is granted.
+    items:
+      type: string

airbyte_cdk/models/connector_metadata/resources/ConnectorBreakingChanges.yaml

@@ -0,0 +1,65 @@
+---
+"$schema": http://json-schema.org/draft-07/schema#
+"$id": https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorBreakingChanges.yaml
+title: ConnectorBreakingChanges
+description: Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.
+type: object
+additionalProperties: false
+minProperties: 1
+patternProperties:
+  "^\\d+\\.\\d+\\.\\d+$":
+    $ref: "#/definitions/VersionBreakingChange"
+definitions:
+  VersionBreakingChange:
+    description: Contains information about a breaking change, including the deadline to upgrade and a message detailing the change.
+    type: object
+    additionalProperties: false
+    required:
+      - upgradeDeadline
+      - message
+    properties:
+      upgradeDeadline:
+        description: The deadline by which to upgrade before the breaking change takes effect.
+        type: string
+        format: date
+      message:
+        description: Descriptive message detailing the breaking change.
+        type: string
+      deadlineAction:
+        description: Action to do when the deadline is reached.
+        type: string
+        enum:
+          - auto_upgrade
+          - disable
+      migrationDocumentationUrl:
+        description: URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}
+        type: string
+        format: uri
+      scopedImpact:
+        description: List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.
+        type: array
+        minItems: 1
+        items:
+          $ref: "#/definitions/BreakingChangeScope"
+  BreakingChangeScope:
+    description: A scope that can be used to limit the impact of a breaking change.
+    type: object
+    oneOf:
+      - $ref: "#/definitions/StreamBreakingChangeScope"
+  StreamBreakingChangeScope:
+    description: A scope that can be used to limit the impact of a breaking change to specific streams.
+    type: object
+    additionalProperties: false
+    required:
+      - scopeType
+      - impactedScopes
+    properties:
+      scopeType:
+        type: string
+        const: stream
+      impactedScopes:
+        description: List of streams that are impacted by the breaking change.
+        type: array
+        minItems: 1
+        items:
+          type: string

airbyte_cdk/models/connector_metadata/resources/ConnectorBuildOptions.yaml

@@ -0,0 +1,10 @@
+---
+"$schema": http://json-schema.org/draft-07/schema#
+"$id": https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorBuildOptions.yaml
+title: ConnectorBuildOptions
+description: metadata specific to the build process.
+type: object
+additionalProperties: false
+properties:
+  baseImage:
+    type: string

airbyte_cdk/models/connector_metadata/resources/ConnectorIPCOptions.yaml

@@ -0,0 +1,29 @@
+---
+"$schema": http://json-schema.org/draft-07/schema#
+"$id": https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorIPCOptions.yaml
+title: ConnectorIPCOptions
+type: object
+required:
+  - dataChannel
+additionalProperties: false
+properties:
+  dataChannel:
+    type: object
+    required:
+      - version
+      - supportedSerialization
+      - supportedTransport
+    additionalProperties: false
+    properties:
+      version:
+        type: string
+      supportedSerialization:
+        type: array
+        items:
+          type: string
+          enum: ["JSONL", "PROTOBUF", "FLATBUFFERS"]
+      supportedTransport:
+        type: array
+        items:
+          type: string
+          enum: ["STDIO", "SOCKET"]