feat: add build config samples

.gitignore

@@ -123,3 +123,6 @@ integrations/**/charts/*
 
 # Kubeconfig
 integrations/kubeconfig.yaml
+
+samples/**/report.md
+samples/evaluation/.deepeval*

integrations/Taskfile.yaml

@@ -121,6 +121,12 @@ tasks:
       - docker pull {{.IMAGE_REPO}}/dir-ctl:{{.DIRECTORY_IMAGE_TAG}}
       - REMOVE_CONTAINERS={{.REMOVE_CONTAINERS}} IMAGE_REPO={{.IMAGE_REPO}} DIRECTORY_IMAGE_TAG={{.DIRECTORY_IMAGE_TAG}} go test ./agntcy-dir/tests -v -failfast -test.v -test.paniconexit0 -ginkgo.timeout 10m -timeout 10m -ginkgo.v -ginkgo.focus "agent compilation"
 
+  test:directory:compile:samples:
+    desc: Agntcy compiler test in samples
+    cmds:
+      - docker pull {{.IMAGE_REPO}}/dir-ctl:{{.DIRECTORY_IMAGE_TAG}}
+      - REMOVE_CONTAINERS={{.REMOVE_CONTAINERS}} IMAGE_REPO={{.IMAGE_REPO}} DIRECTORY_IMAGE_TAG={{.DIRECTORY_IMAGE_TAG}} go test ./agntcy-dir/tests -v -failfast -test.v -test.paniconexit0 -ginkgo.timeout 10m -timeout 10m -ginkgo.v -ginkgo.focus "Samples build test"
+
   test:directory:push:
     desc: Directory agent push test
     cmds:

integrations/agntcy-dir/tests/compiler_test.go → integrations/agntcy-dir/tests/compiler_sanity_test.go

@@ -16,7 +16,7 @@ import (
 	"github.com/agntcy/csit/integrations/testutils"
 )
 
-var _ = ginkgo.Describe("Agntcy compiler tests", func() {
+var _ = ginkgo.Describe("Agntcy compiler sanity tests", func() {
 	var (
 		tempAgentPath          string
 		dockerImage            string
@@ -42,7 +42,6 @@ var _ = ginkgo.Describe("Agntcy compiler tests", func() {
 
 	ginkgo.Context("agent compilation", func() {
 		ginkgo.It("should compile an agent", func() {
-
 			dirctlArgs := []string{
 				"build",
 				"--config",
@@ -80,15 +79,16 @@ var _ = ginkgo.Describe("Agntcy compiler tests", func() {
 				// Ensure the path is deep enough
 				if len(p) >= 3 {
 					if mapStep, ok := p[len(p)-3].(cmp.MapIndex); ok {
-						if key, ok := mapStep.Key().Interface().(string); ok && key == "created_at" {
+						if key, ok := mapStep.Key().Interface().(string); ok && key == "created_at" || key == "extensions" {
 							return true // Ignore these paths
 						}
 					}
 				}
 				return false // Include all other paths
 			}, cmp.Ignore())
 
-			gomega.Expect(expected).To(gomega.BeComparableTo(compiled, filter))
+			gomega.Expect(expected).Should(gomega.BeComparableTo(compiled, filter))
+			gomega.Expect(expected["extensions"]).Should(gomega.ConsistOf(compiled["extensions"]))
 		})
 	})
 })

integrations/agntcy-dir/tests/find_samples.go

@@ -0,0 +1,41 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025 Cisco and/or its affiliates.
+// SPDX-License-Identifier: Apache-2.0
+
+package tests
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+func FindFilePairs(rootDir, buildConfigName, expectedModelName string) ([]string, error) {
+	directories := make(map[string]int)
+
+	err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			dir := filepath.Dir(path)
+			if info.Name() == buildConfigName || info.Name() == expectedModelName {
+				directories[dir]++
+			}
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("error walking the path %q: %v", rootDir, err)
+	}
+
+	dirs := make([]string, 0)
+	for dir, count := range directories {
+		if count >= 2 {
+			dirs = append(dirs, dir)
+		}
+	}
+
+	return dirs, nil
+}

integrations/agntcy-dir/tests/samples_compile_test.go

@@ -0,0 +1,119 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025 Cisco and/or its affiliates.
+// SPDX-License-Identifier: Apache-2.0
+
+package tests
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/agntcy/csit/integrations/testutils"
+	"github.com/google/go-cmp/cmp"
+	ginkgo "github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+)
+
+const (
+	buildConfigName   = "build.config.yaml"
+	expectedModelName = "model.json"
+	samplesPath       = "../../../samples"
+)
+
+var _ = ginkgo.Describe("Samples build test", func() {
+	var (
+		dockerImage string
+		samples     []string
+	)
+
+	samplesDir, err := filepath.Abs(samplesPath)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+	samples, err = FindFilePairs(samplesDir, buildConfigName, expectedModelName)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+	dockerImage = fmt.Sprintf("%s/dir-ctl:%s", os.Getenv("IMAGE_REPO"), os.Getenv("DIRECTORY_IMAGE_TAG"))
+
+	_, err = fmt.Fprintf(ginkgo.GinkgoWriter, "samples: %v\n", samples)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+	for _, entry := range samples {
+		entry := entry
+		ginkgo.Context(entry, func() {
+			var (
+				tempAgentPath          string
+				mountDest              string
+				mountString            string
+				modelConfigFilePath    string
+				expectedAgentModelFile string
+			)
+
+			ginkgo.BeforeEach(func() {
+				mountDest := fmt.Sprintf("/%s", filepath.Base(entry))
+				mountString = fmt.Sprintf("%s:%s", entry, mountDest)
+				modelConfigFilePath = filepath.Join(mountDest, buildConfigName)
+				expectedAgentModelFile = filepath.Join(entry, expectedModelName)
+				tempFileName := fmt.Sprintf("%s.json", strings.ReplaceAll(entry, "/", "-"))
+				tempAgentPath = filepath.Join(os.TempDir(), tempFileName)
+			})
+
+			ginkgo.It("Should compile", func() {
+				_, err := fmt.Fprintf(ginkgo.GinkgoWriter, "Compiling agent model: %v\n", entry)
+				gomega.Expect(err).NotTo(gomega.HaveOccurred())
+				_, err = fmt.Fprintf(ginkgo.GinkgoWriter, "tempagent path: %v\n", tempAgentPath)
+				gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+				dirctlArgs := []string{
+					"build",
+					"--config",
+					modelConfigFilePath,
+					mountDest,
+				}
+				runner := testutils.NewDockerRunner(dockerImage, mountString, nil)
+				outputBuffer, err := runner.Run(dirctlArgs...)
+				gomega.Expect(err).NotTo(gomega.HaveOccurred(), outputBuffer.String())
+
+				err = os.WriteFile(tempAgentPath, outputBuffer.Bytes(), 0644)
+				gomega.Expect(err).NotTo(gomega.HaveOccurred())
+			})
+
+			ginkgo.It("Agent model should be the expected", func() {
+				var expected, compiled map[string]any
+
+				_, err = fmt.Fprintf(ginkgo.GinkgoWriter, "tempagent path: %v\n", tempAgentPath)
+				gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+				expactedModelJSON, err := os.ReadFile(expectedAgentModelFile)
+				gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+				// Unmarshal or Decode the JSON to the interface.
+				err = json.Unmarshal([]byte(expactedModelJSON), &expected)
+				gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+				compiledModelJSON, err := os.ReadFile(tempAgentPath)
+				gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+				// Unmarshal or Decode the JSON to the interface.
+				err = json.Unmarshal([]byte(compiledModelJSON), &compiled)
+				gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+				// Filter "created_at" field and extensions
+				filter := cmp.FilterPath(func(p cmp.Path) bool {
+					// Ensure the path is deep enough
+					if len(p) >= 3 {
+						if mapStep, ok := p[len(p)-3].(cmp.MapIndex); ok {
+							if key, ok := mapStep.Key().Interface().(string); ok && key == "created_at" || key == "extensions" {
+								return true // Ignore these paths
+							}
+						}
+					}
+					return false // Include all other paths
+				}, cmp.Ignore())
+
+				gomega.Expect(expected).Should(gomega.BeComparableTo(compiled, filter))
+				gomega.Expect(expected["extensions"]).Should(gomega.ConsistOf(compiled["extensions"]))
+			})
+		})
+	}
+})

samples/crewai/simple_crew/build.config.yaml

@@ -0,0 +1,70 @@
+model:
+  name: "research_crew"
+
+  version: "v0.1.18"
+
+  authors:
+    - "Your Name <you@example.com>"
+
+  skills:
+    - "Natural Language Understanding"
+    - "Natural Language Generation"
+    - "Information Retrieval and Synthesis"
+    - "Fact Extraction"
+    - "Knowledge Synthesis"
+
+  locators:
+    - type: "source-code"
+      url: "https://github.com/agntcy/csit/tree/main/samples/crewai/simple_crew"
+
+  extensions:
+    - name: "oasf.agntcy.org/features/runtime/io-mapper"
+      version: "v1.0.0"
+      specs:
+        input_name: "topic"
+        input_type: "string"
+        output_name: "report"
+        output_type: "file"
+        output_description: "A fully fledge reports with the mains topics, each with a full section of information. Formatted as markdown without '```'"
+    - name: "oasf.agntcy.org/features/observability/logging"
+      version: "v1.0.0"
+      specs:
+        type: "stdout"
+        format: "<string>"
+    - name: "oasf.agntcy.org/features/observability/metrics"
+      version: "v1.0.0"
+      specs:
+        token_usage:
+          - "total_tokens"
+          - "prompt_tokens"
+          - "cached_prompt_tokens"
+          - "completion_tokens"
+          - "successful_requests"
+        task_duration: "task_duration"
+    - name: "oasf.agntcy.org/features/framework/llm"
+      version: "v1.0.0"
+      specs:
+        model: "ollama/llama3.1"
+        base_url: "http://localhost:11434"
+    - name: "oasf.agntcy.org/features/framework/evaluation"
+      version: "v1.0.0"
+      specs:
+        provider: "local"
+        type: "evaluator agent"
+    - name: "oasf.agntcy.org/features/framework/orchestration"
+      version: "v1.0.0"
+      specs:
+        type: "sequential"
+    - name: "oasf.agntcy.org/features/framework/memory"
+      version: "v1.0.0"
+      specs:
+        enabled: false
+
+builder:
+  source: "/simple_crew" # TODO(pbalogh-sa) please update
+  source-ignore:
+    - ".venv/"
+
+  llmanalyzer: false
+  crewai: true
+  runtime: true