[ACIX-1034] Implement dda build bin core-agent

src/dda/build/artifacts/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: 2025-present Datadog, Inc. <[email protected]>
+#
+# SPDX-License-Identifier: MIT

src/dda/build/artifacts/base.py

@@ -0,0 +1,46 @@
+# SPDX-FileCopyrightText: 2025-present Datadog, Inc. <[email protected]>
+#
+# SPDX-License-Identifier: MIT
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING
+
+from dda.build.metadata.metadata import BuildMetadata, analyze_context
+
+if TYPE_CHECKING:
+    from typing import Any
+
+    from dda.build.metadata.digests import ArtifactDigest
+    from dda.build.metadata.formats import ArtifactFormat
+    from dda.cli.application import Application
+
+
+# NOTE: Very much speculative - this API might be subject to signficant changes !
+class BuildArtifact(ABC):
+    """
+    Base class for all build artifacts.
+    """
+
+    @abstractmethod
+    def build(self, app: Application, *args: Any, **kwargs: Any) -> None:
+        """
+        Build the artifact. This function can have arbitrary side effects (creating files, running commands, etc.), and is not expected to return anything.
+        """
+
+    @abstractmethod
+    def get_build_components(self) -> tuple[set[str], ArtifactFormat]:
+        """
+        Gets the build components and artifact format for this artifact.
+
+        Returns:
+            A tuple containing the build components and artifact format for this artifact.
+        """
+
+    def compute_metadata(self, app: Application, artifact_digest: ArtifactDigest) -> BuildMetadata:
+        """
+        Creates a BuildMetadata instance for this artifact.
+        """
+        from dda.build.metadata.metadata import BuildMetadata
+
+        return BuildMetadata.spawn_from_context(analyze_context(app), self.get_build_components(), artifact_digest)

src/dda/build/artifacts/binaries/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: 2025-present Datadog, Inc. <[email protected]>
+#
+# SPDX-License-Identifier: MIT

src/dda/build/artifacts/binaries/base.py

@@ -0,0 +1,31 @@
+# SPDX-FileCopyrightText: 2025-present Datadog, Inc. <[email protected]>
+#
+# SPDX-License-Identifier: MIT
+from __future__ import annotations
+
+from abc import abstractmethod
+from typing import TYPE_CHECKING, override
+
+from dda.build.artifacts.base import BuildArtifact
+
+if TYPE_CHECKING:
+    from dda.build.metadata.formats import ArtifactFormat
+
+
+class BinaryArtifact(BuildArtifact):
+    """
+    Base class for all binary build artifacts.
+    """
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """
+        Get the name of the binary artifact this object represents.
+        """
+
+    @override
+    def get_build_components(self) -> tuple[set[str], ArtifactFormat]:
+        from dda.build.metadata.formats import ArtifactFormat
+
+        return {self.name}, ArtifactFormat.BIN

src/dda/build/artifacts/binaries/core_agent.py

@@ -0,0 +1,113 @@
+# SPDX-FileCopyrightText: 2025-present Datadog, Inc. <[email protected]>
+#
+# SPDX-License-Identifier: MIT
+from __future__ import annotations
+
+from functools import cache
+from typing import TYPE_CHECKING, Any, override
+
+from dda.build.artifacts.binaries.base import BinaryArtifact
+from dda.build.languages.go import GoArtifact
+
+if TYPE_CHECKING:
+    from dda.cli.application import Application
+    from dda.utils.fs import Path
+
+
+@cache
+def get_repo_root(app: Application) -> Path:
+    return app.tools.git.get_repo_root()
+
+
+class CoreAgent(BinaryArtifact, GoArtifact):
+    """
+    Build artifact for the `core-agent` binary.
+    """
+
+    @override
+    def get_build_tags(self, *args: Any, **kwargs: Any) -> set[str]:
+        # TODO: Implement a properly dynamic function, matching the old invoke task
+        return {
+            "ec2",
+            "python",
+            "kubeapiserver",
+            "oracle",
+            "etcd",
+            "jmx",
+            "grpcnotrace",
+            "consul",
+            "systemprobechecks",
+            "ncm",
+            "otlp",
+            "zstd",
+            "orchestrator",
+            "zk",
+            "datadog.no_waf",
+            "trivy_no_javadb",
+            "zlib",
+            "bundle_agent",
+            "fargateprocess",
+            "kubelet",
+            "cel",
+        }
+
+    @override
+    def get_gcflags(self, *args: Any, **kwargs: Any) -> list[str]:
+        return []
+
+    @override
+    def get_ldflags(self, app: Application, *args: Any, **kwargs: Any) -> list[str]:
+        from dda.build.versioning import parse_describe_result
+
+        repo_root = get_repo_root(app)
+        with repo_root.as_cwd():
+            commit = app.tools.git.get_commit().sha1
+            agent_version = parse_describe_result(app.tools.git.capture(["describe", "--tags"]).strip())
+
+        return [
+            "-X",
+            f"github.com/DataDog/datadog-agent/pkg/version.Commit={commit[:10]}",
+            "-X",
+            f"github.com/DataDog/datadog-agent/pkg/version.AgentVersion={agent_version}",
+            "-X",
+            # TODO: Make this dynamic
+            "github.com/DataDog/datadog-agent/pkg/version.AgentPayloadVersion=v5.0.174",
+            "-X",
+            f"github.com/DataDog/datadog-agent/pkg/version.AgentPackageVersion={agent_version}",
+            "-r",
+            f"{repo_root}/dev/lib",
+            "'-extldflags=-Wl,-bind_at_load,-no_warn_duplicate_libraries'",
+        ]
+
+    @override
+    def get_build_env(self, app: Application, *args: Any, **kwargs: Any) -> dict[str, str]:
+        # TODO: Implement a properly dynamic function, matching the old invoke task
+        repo_root = get_repo_root(app)
+        return {
+            "GO111MODULE": "on",
+            "CGO_LDFLAGS_ALLOW": "-Wl,--wrap=.*",
+            "DYLD_LIBRARY_PATH": f"{repo_root}/dev/lib",
+            "LD_LIBRARY_PATH": f"{repo_root}/dev/lib",
+            "CGO_LDFLAGS": f" -L{repo_root}/dev/lib",
+            "CGO_CFLAGS": f" -Werror -Wno-deprecated-declarations -I{repo_root}/dev/include",
+            "CGO_ENABLED": "1",
+            "PATH": f"{repo_root}/go/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin",
+        }
+
+    @property
+    @override
+    def name(self) -> str:
+        return "core-agent"
+
+    @override
+    def build(self, app: Application, output: Path, *args: Any, **kwargs: Any) -> None:
+        # TODO: Build rtloader first if needed
+        # TODO: Make this build in a devenv ? Or at least add a flag
+        app.tools.go.build(
+            "github.com/DataDog/datadog-agent/cmd/agent",
+            output=output,
+            build_tags=self.get_build_tags(),
+            gcflags=self.get_gcflags(),
+            ldflags=self.get_ldflags(app),
+            env_vars=self.get_build_env(app),
+        )

src/dda/build/artifacts/distributions/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: 2025-present Datadog, Inc. <[email protected]>
+#
+# SPDX-License-Identifier: MIT

src/dda/build/artifacts/distributions/base.py

@@ -0,0 +1,12 @@
+# SPDX-FileCopyrightText: 2025-present Datadog, Inc. <[email protected]>
+#
+# SPDX-License-Identifier: MIT
+from __future__ import annotations
+
+from dda.build.artifacts.base import BuildArtifact
+
+
+class DistributionArtifact(BuildArtifact):
+    """
+    Base class for all distribution build artifacts.
+    """

src/dda/build/languages/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: 2025-present Datadog, Inc. <[email protected]>
+#
+# SPDX-License-Identifier: MIT

src/dda/build/languages/go.py

@@ -0,0 +1,43 @@
+# SPDX-FileCopyrightText: 2025-present Datadog, Inc. <[email protected]>
+#
+# SPDX-License-Identifier: MIT
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from typing import Any
+
+
+class GoArtifact(ABC):
+    """
+    Base class for all Go artifacts.
+    Any artifact class for an artifact built with the Go compiler should inherit from this class and implement its methods.
+    """
+
+    @abstractmethod
+    def get_build_tags(self, *args: Any, **kwargs: Any) -> set[str]:
+        """
+        Get the build tags to pass to the Go compiler for the artifact.
+        """
+
+    @abstractmethod
+    def get_gcflags(self, *args: Any, **kwargs: Any) -> list[str]:
+        """
+        Get the gcflags to pass to the Go compiler for the artifact.
+        """
+
+    @abstractmethod
+    def get_ldflags(self, *args: Any, **kwargs: Any) -> list[str]:
+        """
+        Get the ldflags to pass to the Go compiler for the artifact.
+        """
+        return []
+
+    @abstractmethod
+    def get_build_env(self, *args: Any, **kwargs: Any) -> dict[str, str]:
+        """
+        Get the build environment variables to pass to the Go compiler for the artifact.
+        """

src/dda/build/metadata/metadata.py

@@ -18,8 +18,6 @@
 from dda.utils.git.commit import Commit  # noqa: TC001
 
 if TYPE_CHECKING:
-    from click import Context
-
     from dda.cli.application import Application
 
 
@@ -70,13 +68,17 @@ def artifact_type(self) -> ArtifactType:
     def spawn_from_context(
         cls,
         context_data: _MetadataRequiredContext,
+        artifact_components: _MetadataAgentComponents,
         artifact_digest: ArtifactDigest,
     ) -> BuildMetadata:
         """
         Create a BuildMetadata instance for the current build.
-        Takes two arguments:
+        Takes three arguments:
         - context_data: A _MetadataRequiredContext instance containing the required data to generate build metadata.
             This can be created with the `analyze_context` function.
+        - artifact_components: A tuple containing the agent components and artifact format for the artifact.
+            The first element is a set of strings representing the agent components.
+            The second element is an ArtifactFormat instance representing the format of the artifact.
         - artifact_digest: An ArtifactDigest instance containing the digest of the artifact.
             This can be calculated with the `DigestType.calculate_digest` method.
             For example, starting from the _MetadataRequiredContext instance, you can do:
@@ -98,6 +100,8 @@ def spawn_from_context(
         return cls(
             id=artifact_id,
             build_time=build_time,
+            agent_components=artifact_components[0],
+            artifact_format=artifact_components[1],
             digest=artifact_digest,
             **context_data.dump(),
         )
@@ -176,37 +180,6 @@ def get_canonical_filename(self) -> str:
         return f"{components}-{compatibility}-{source_info}-{short_uuid}{artifact_format_identifier}"
 
 
-def get_build_components(command: str) -> tuple[set[str], ArtifactFormat]:
-    """
-    Parse calling command to get the agent components and artifact format.
-
-    Ex:
-        `dda build bin core-agent` -> (`core-agent`), `bin` and `bin`
-        `dda build dist deb -c core-agent -c process-agent` -> (`core-agent`, `process-agent`), `dist` and `deb`
-    """
-    command_parts = command.split(" ")
-    # Remove the first two parts, which are `dda` and `build`, if they exist
-    if command_parts[:2] != ["dda", "build"]:
-        msg = f"Unexpected command, only build commands can be used to extract build components: {command}"
-        raise ValueError(msg)
-
-    artifact_format: ArtifactFormat
-    artifact_type_str = command_parts[2]
-    match artifact_type_str:
-        case "dist":
-            artifact_format = ArtifactFormat[command_parts[3].upper()]
-            # TODO: Implement this in a more robust way, write a proper parser for the command line
-            agent_components = {part for part in command_parts[4:] if part != "-c"}
-        case "bin":
-            artifact_format = ArtifactFormat.BIN
-            agent_components = {command_parts[3]}
-        case _:
-            msg = f"Unsupported artifact type: {artifact_type_str}"
-            raise NotImplementedError(msg)
-
-    return agent_components, artifact_format
-
-
 def generate_build_id() -> UUID:
     """
     Generate a unique build ID.
@@ -216,11 +189,14 @@ def generate_build_id() -> UUID:
     return uuid4()
 
 
-def analyze_context(ctx: Context, app: Application) -> _MetadataRequiredContext:
+def analyze_context(app: Application) -> _MetadataRequiredContext:
     """
     Analyze the context to get the required data to generate build metadata.
     """
-    return _MetadataRequiredContext.from_context(ctx, app)
+    return _MetadataRequiredContext.from_context(app)
+
+
+_MetadataAgentComponents = tuple[set[str], ArtifactFormat]
 
 
 class _MetadataRequiredContext(Struct):
@@ -229,9 +205,6 @@ class _MetadataRequiredContext(Struct):
     Having this as a separate struct allows for easier overriding - this struct is explicitely not frozen.
     """
 
-    agent_components: set[str]
-    artifact_format: ArtifactFormat
-
     # Source tree fields
     commit: Commit
     worktree_diff: ChangeSet
@@ -243,7 +216,7 @@ class _MetadataRequiredContext(Struct):
     build_platform: Platform
 
     @classmethod
-    def from_context(cls, ctx: Context, app: Application) -> _MetadataRequiredContext:
+    def from_context(cls, app: Application) -> _MetadataRequiredContext:
         """
         Create a _MetadataRequiredContext instance from the application and build context.
         Some values might not be correct for some artifacts, in which case they should be overridden afterwards.
@@ -259,16 +232,10 @@ def from_context(cls, ctx: Context, app: Application) -> _MetadataRequiredContex
 
         import platform
 
-        # Build components
-        build_components = get_build_components(ctx.command_path)
-        agent_components, artifact_format = build_components
-
         # Build platform
         build_platform = Platform.from_alias(platform.system().lower(), platform.machine())
 
         return cls(
-            agent_components=agent_components,
-            artifact_format=artifact_format,
             commit=app.tools.git.get_commit(),
             worktree_diff=app.tools.git.get_changes("HEAD", start="HEAD", working_tree=True),
             compatible_platforms={build_platform},