Skip to content

Commit 935030e

Browse files
tarilabstarilabs
committed
chore: make CM content aligned to current EH Service dev
Signed-off-by: tarilabs <matteo.mortari@gmail.com>
1 parent d510911 commit 935030e

File tree

6 files changed

+192
-127
lines changed

6 files changed

+192
-127
lines changed

‎controllers/evalhub/configmap.go‎

Lines changed: 45 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,6 @@ import (
1616
"sigs.k8s.io/yaml"
1717
)
1818

19-
// ProviderConfig represents the provider configuration structure
20-
type ProviderConfig struct {
21-
Name string `yaml:"name"`
22-
Type string `yaml:"type"`
23-
Enabled bool `yaml:"enabled"`
24-
Benchmarks []string `yaml:"benchmarks,omitempty"`
25-
Config map[string]string `yaml:"config,omitempty"`
26-
}
27-
2819
// DatabaseConfig represents the database configuration in config.yaml
2920
type DatabaseConfig struct {
3021
Driver string `yaml:"driver"`
@@ -40,7 +31,7 @@ type SecretsMapping struct {
4031

4132
// EvalHubConfig represents the eval-hub configuration structure
4233
type EvalHubConfig struct {
43-
Providers []ProviderConfig `yaml:"providers"`
34+
Providers []ProviderResource `yaml:"providers"`
4435
Collections []string `yaml:"collections,omitempty"`
4536
Database *DatabaseConfig `yaml:"database,omitempty"`
4637
Secrets *SecretsMapping `yaml:"secrets,omitempty"`
@@ -91,56 +82,52 @@ func (r *EvalHubReconciler) reconcileConfigMap(ctx context.Context, instance *ev
9182
// generateConfigData generates the configuration data for the ConfigMap
9283
func (r *EvalHubReconciler) generateConfigData(instance *evalhubv1alpha1.EvalHub) (map[string]string, error) {
9384
config := EvalHubConfig{
94-
Providers: make([]ProviderConfig, 0),
85+
Providers: make([]ProviderResource, 0),
9586
Collections: []string{},
9687
}
9788

9889
// Default providers configuration set by the controller
99-
config.Providers = []ProviderConfig{
90+
config.Providers = []ProviderResource{
10091
{
101-
Name: "lm-eval-harness",
102-
Type: "lm_evaluation_harness",
103-
Enabled: true,
104-
Benchmarks: []string{
105-
"arc_challenge", "hellaswag", "mmlu", "truthfulqa",
106-
},
107-
Config: map[string]string{
108-
"batch_size": "8",
109-
"max_length": "2048",
92+
ID: "lm-eval-harness",
93+
Name: "lm-eval-harness",
94+
Type: "lm_evaluation_harness",
95+
Benchmarks: []BenchmarkResource{
96+
{ID: "arc_challenge", Name: "arc_challenge"},
97+
{ID: "hellaswag", Name: "hellaswag"},
98+
{ID: "mmlu", Name: "mmlu"},
99+
{ID: "truthfulqa", Name: "truthfulqa"},
110100
},
111101
},
112102
{
113-
Name: "ragas-provider",
114-
Type: "ragas",
115-
Enabled: true,
116-
Benchmarks: []string{
117-
"faithfulness", "answer_relevancy", "context_precision", "context_recall",
118-
},
119-
Config: map[string]string{
120-
"llm_model": "gpt-3.5-turbo",
121-
"embeddings_model": "text-embedding-ada-002",
103+
ID: "ragas-provider",
104+
Name: "ragas-provider",
105+
Type: "ragas",
106+
Benchmarks: []BenchmarkResource{
107+
{ID: "faithfulness", Name: "faithfulness"},
108+
{ID: "answer_relevancy", Name: "answer_relevancy"},
109+
{ID: "context_precision", Name: "context_precision"},
110+
{ID: "context_recall", Name: "context_recall"},
122111
},
123112
},
124113
{
125-
Name: "garak-security",
126-
Type: "garak",
127-
Enabled: false,
128-
Benchmarks: []string{
129-
"encoding", "injection", "malware", "prompt_injection",
130-
},
131-
Config: map[string]string{
132-
"probe_set": "basic",
114+
ID: "garak-security",
115+
Name: "garak-security",
116+
Type: "garak",
117+
Benchmarks: []BenchmarkResource{
118+
{ID: "encoding", Name: "encoding"},
119+
{ID: "injection", Name: "injection"},
120+
{ID: "malware", Name: "malware"},
121+
{ID: "prompt_injection", Name: "prompt_injection"},
133122
},
134123
},
135124
{
136-
Name: "trustyai-custom",
137-
Type: "trustyai_custom",
138-
Enabled: true,
139-
Benchmarks: []string{
140-
"bias_detection", "fairness_metrics",
141-
},
142-
Config: map[string]string{
143-
"bias_threshold": "0.1",
125+
ID: "trustyai-custom",
126+
Name: "trustyai-custom",
127+
Type: "trustyai_custom",
128+
Benchmarks: []BenchmarkResource{
129+
{ID: "bias_detection", Name: "bias_detection"},
130+
{ID: "fairness_metrics", Name: "fairness_metrics"},
144131
},
145132
},
146133
}
@@ -179,31 +166,24 @@ func (r *EvalHubReconciler) generateConfigData(instance *evalhubv1alpha1.EvalHub
179166
return nil, err
180167
}
181168

182-
// Generate providers.yaml content
183-
providersYAML, err := r.generateProvidersYAML(config.Providers)
184-
if err != nil {
185-
return nil, err
169+
data := map[string]string{
170+
"config.yaml": string(configYAML),
186171
}
187172

188-
return map[string]string{
189-
"config.yaml": string(configYAML),
190-
"providers.yaml": providersYAML,
191-
}, nil
192-
}
193-
194-
// generateProvidersYAML generates the providers.yaml configuration
195-
func (r *EvalHubReconciler) generateProvidersYAML(providers []ProviderConfig) (string, error) {
196-
providersData := make(map[string]interface{})
197-
providersData["providers"] = providers
198-
199-
yamlData, err := yaml.Marshal(providersData)
200-
if err != nil {
201-
return "", err
173+
// Generate one file per provider: providers/<name>.yaml
174+
for _, provider := range config.Providers {
175+
providerYAML, err := yaml.Marshal(provider)
176+
if err != nil {
177+
return nil, fmt.Errorf("failed to marshal provider %s: %w", provider.Name, err)
178+
}
179+
key := fmt.Sprintf("providers/%s.yaml", provider.Name)
180+
data[key] = string(providerYAML)
202181
}
203182

204-
return string(yamlData), nil
183+
return data, nil
205184
}
206185

186+
207187
// getImageFromConfigMap gets a required image value from the operator's ConfigMap
208188
// Returns error if ConfigMap is not found, key is missing, or value is empty
209189
// This ensures explicit configuration and prevents deployment with unconfigured images

‎controllers/evalhub/configmap_test.go‎

Lines changed: 69 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,14 @@ import (
1313
"sigs.k8s.io/yaml"
1414
)
1515

16+
func benchmarkResourceNames(benchmarks []BenchmarkResource) []string {
17+
names := make([]string, len(benchmarks))
18+
for i, b := range benchmarks {
19+
names[i] = b.Name
20+
}
21+
return names
22+
}
23+
1624
var _ = Describe("EvalHub ConfigMap", func() {
1725
const (
1826
testNamespacePrefix = "evalhub-configmap-test"
@@ -88,7 +96,10 @@ var _ = Describe("EvalHub ConfigMap", func() {
8896

8997
By("Checking required keys exist")
9098
Expect(configMap.Data).To(HaveKey("config.yaml"))
91-
Expect(configMap.Data).To(HaveKey("providers.yaml"))
99+
Expect(configMap.Data).To(HaveKey("providers/lm-eval-harness.yaml"))
100+
Expect(configMap.Data).To(HaveKey("providers/ragas-provider.yaml"))
101+
Expect(configMap.Data).To(HaveKey("providers/garak-security.yaml"))
102+
Expect(configMap.Data).To(HaveKey("providers/trustyai-custom.yaml"))
92103
})
93104

94105
It("should have valid YAML configuration", func() {
@@ -122,24 +133,28 @@ var _ = Describe("EvalHub ConfigMap", func() {
122133
))
123134
})
124135

125-
It("should have valid providers.yaml", func() {
136+
It("should have valid per-provider YAML files", func() {
126137
By("Reconciling configmap")
127138
err := reconciler.reconcileConfigMap(ctx, evalHub)
128139
Expect(err).NotTo(HaveOccurred())
129140

130141
By("Getting configmap")
131142
configMap := waitForConfigMap(evalHubName+"-config", testNamespace)
132143

133-
By("Parsing providers.yaml")
134-
var providersData map[string]interface{}
135-
err = yaml.Unmarshal([]byte(configMap.Data["providers.yaml"]), &providersData)
136-
Expect(err).NotTo(HaveOccurred())
137-
138-
By("Checking providers structure")
139-
Expect(providersData).To(HaveKey("providers"))
140-
providers, ok := providersData["providers"].([]interface{})
141-
Expect(ok).To(BeTrue())
142-
Expect(providers).To(HaveLen(4))
144+
By("Checking each provider file exists and is valid")
145+
expectedProviders := []string{
146+
"providers/lm-eval-harness.yaml",
147+
"providers/ragas-provider.yaml",
148+
"providers/garak-security.yaml",
149+
"providers/trustyai-custom.yaml",
150+
}
151+
for _, key := range expectedProviders {
152+
Expect(configMap.Data).To(HaveKey(key))
153+
var provider ProviderResource
154+
err = yaml.Unmarshal([]byte(configMap.Data[key]), &provider)
155+
Expect(err).NotTo(HaveOccurred())
156+
Expect(provider.Name).NotTo(BeEmpty())
157+
}
143158
})
144159

145160
It("should configure lm-eval-harness provider correctly", func() {
@@ -156,7 +171,7 @@ var _ = Describe("EvalHub ConfigMap", func() {
156171
Expect(err).NotTo(HaveOccurred())
157172

158173
By("Finding lm-eval-harness provider")
159-
var lmEvalProvider *ProviderConfig
174+
var lmEvalProvider *ProviderResource
160175
for _, provider := range config.Providers {
161176
if provider.Name == "lm-eval-harness" {
162177
lmEvalProvider = &provider
@@ -167,12 +182,10 @@ var _ = Describe("EvalHub ConfigMap", func() {
167182

168183
By("Checking lm-eval-harness configuration")
169184
Expect(lmEvalProvider.Type).To(Equal("lm_evaluation_harness"))
170-
Expect(lmEvalProvider.Enabled).To(BeTrue())
171-
Expect(lmEvalProvider.Benchmarks).To(ContainElements(
185+
benchmarkNames := benchmarkResourceNames(lmEvalProvider.Benchmarks)
186+
Expect(benchmarkNames).To(ContainElements(
172187
"arc_challenge", "hellaswag", "mmlu", "truthfulqa",
173188
))
174-
Expect(lmEvalProvider.Config["batch_size"]).To(Equal("8"))
175-
Expect(lmEvalProvider.Config["max_length"]).To(Equal("2048"))
176189
})
177190

178191
It("should configure ragas provider correctly", func() {
@@ -189,7 +202,7 @@ var _ = Describe("EvalHub ConfigMap", func() {
189202
Expect(err).NotTo(HaveOccurred())
190203

191204
By("Finding ragas provider")
192-
var ragasProvider *ProviderConfig
205+
var ragasProvider *ProviderResource
193206
for _, provider := range config.Providers {
194207
if provider.Name == "ragas-provider" {
195208
ragasProvider = &provider
@@ -200,12 +213,10 @@ var _ = Describe("EvalHub ConfigMap", func() {
200213

201214
By("Checking ragas configuration")
202215
Expect(ragasProvider.Type).To(Equal("ragas"))
203-
Expect(ragasProvider.Enabled).To(BeTrue())
204-
Expect(ragasProvider.Benchmarks).To(ContainElements(
216+
benchmarkNames := benchmarkResourceNames(ragasProvider.Benchmarks)
217+
Expect(benchmarkNames).To(ContainElements(
205218
"faithfulness", "answer_relevancy", "context_precision", "context_recall",
206219
))
207-
Expect(ragasProvider.Config["llm_model"]).To(Equal("gpt-3.5-turbo"))
208-
Expect(ragasProvider.Config["embeddings_model"]).To(Equal("text-embedding-ada-002"))
209220
})
210221

211222
It("should configure garak security provider correctly", func() {
@@ -222,7 +233,7 @@ var _ = Describe("EvalHub ConfigMap", func() {
222233
Expect(err).NotTo(HaveOccurred())
223234

224235
By("Finding garak provider")
225-
var garakProvider *ProviderConfig
236+
var garakProvider *ProviderResource
226237
for _, provider := range config.Providers {
227238
if provider.Name == "garak-security" {
228239
garakProvider = &provider
@@ -233,11 +244,10 @@ var _ = Describe("EvalHub ConfigMap", func() {
233244

234245
By("Checking garak configuration")
235246
Expect(garakProvider.Type).To(Equal("garak"))
236-
Expect(garakProvider.Enabled).To(BeFalse()) // Disabled by default
237-
Expect(garakProvider.Benchmarks).To(ContainElements(
247+
benchmarkNames := benchmarkResourceNames(garakProvider.Benchmarks)
248+
Expect(benchmarkNames).To(ContainElements(
238249
"encoding", "injection", "malware", "prompt_injection",
239250
))
240-
Expect(garakProvider.Config["probe_set"]).To(Equal("basic"))
241251
})
242252

243253
It("should configure trustyai custom provider correctly", func() {
@@ -254,7 +264,7 @@ var _ = Describe("EvalHub ConfigMap", func() {
254264
Expect(err).NotTo(HaveOccurred())
255265

256266
By("Finding trustyai provider")
257-
var trustyaiProvider *ProviderConfig
267+
var trustyaiProvider *ProviderResource
258268
for _, provider := range config.Providers {
259269
if provider.Name == "trustyai-custom" {
260270
trustyaiProvider = &provider
@@ -265,11 +275,10 @@ var _ = Describe("EvalHub ConfigMap", func() {
265275

266276
By("Checking trustyai configuration")
267277
Expect(trustyaiProvider.Type).To(Equal("trustyai_custom"))
268-
Expect(trustyaiProvider.Enabled).To(BeTrue())
269-
Expect(trustyaiProvider.Benchmarks).To(ContainElements(
278+
benchmarkNames := benchmarkResourceNames(trustyaiProvider.Benchmarks)
279+
Expect(benchmarkNames).To(ContainElements(
270280
"bias_detection", "fairness_metrics",
271281
))
272-
Expect(trustyaiProvider.Config["bias_threshold"]).To(Equal("0.1"))
273282
})
274283

275284
It("should update existing configmap", func() {
@@ -413,7 +422,10 @@ var _ = Describe("EvalHub ConfigMap", func() {
413422

414423
By("Checking required keys are present")
415424
Expect(configData).To(HaveKey("config.yaml"))
416-
Expect(configData).To(HaveKey("providers.yaml"))
425+
Expect(configData).To(HaveKey("providers/lm-eval-harness.yaml"))
426+
Expect(configData).To(HaveKey("providers/ragas-provider.yaml"))
427+
Expect(configData).To(HaveKey("providers/garak-security.yaml"))
428+
Expect(configData).To(HaveKey("providers/trustyai-custom.yaml"))
417429

418430
By("Validating config.yaml content")
419431
var config EvalHubConfig
@@ -422,14 +434,23 @@ var _ = Describe("EvalHub ConfigMap", func() {
422434
Expect(config.Providers).To(HaveLen(4))
423435
Expect(config.Collections).To(HaveLen(4))
424436

425-
By("Validating providers.yaml content")
426-
var providersData map[string]interface{}
427-
err = yaml.Unmarshal([]byte(configData["providers.yaml"]), &providersData)
428-
Expect(err).NotTo(HaveOccurred())
429-
Expect(providersData).To(HaveKey("providers"))
437+
By("Validating per-provider YAML files")
438+
expectedProviders := []string{
439+
"providers/lm-eval-harness.yaml",
440+
"providers/ragas-provider.yaml",
441+
"providers/garak-security.yaml",
442+
"providers/trustyai-custom.yaml",
443+
}
444+
for _, key := range expectedProviders {
445+
Expect(configData).To(HaveKey(key))
446+
var provider ProviderResource
447+
err = yaml.Unmarshal([]byte(configData[key]), &provider)
448+
Expect(err).NotTo(HaveOccurred())
449+
Expect(provider.Name).NotTo(BeEmpty())
450+
}
430451
})
431452

432-
It("should generate providers YAML correctly", func() {
453+
It("should generate per-provider YAML files correctly", func() {
433454
By("Generating configuration data")
434455
configData, err := reconciler.generateConfigData(evalHub)
435456
Expect(err).NotTo(HaveOccurred())
@@ -439,18 +460,17 @@ var _ = Describe("EvalHub ConfigMap", func() {
439460
err = yaml.Unmarshal([]byte(configData["config.yaml"]), &config)
440461
Expect(err).NotTo(HaveOccurred())
441462

442-
By("Generating providers YAML")
443-
providersYAML, err := reconciler.generateProvidersYAML(config.Providers)
444-
Expect(err).NotTo(HaveOccurred())
445-
446-
By("Verifying providers YAML matches configmap data")
447-
Expect(providersYAML).To(Equal(configData["providers.yaml"]))
448-
449-
By("Verifying providers YAML is valid")
450-
var providersData map[string]interface{}
451-
err = yaml.Unmarshal([]byte(providersYAML), &providersData)
452-
Expect(err).NotTo(HaveOccurred())
453-
Expect(providersData).To(HaveKey("providers"))
463+
By("Verifying each provider has its own YAML file")
464+
for _, provider := range config.Providers {
465+
key := fmt.Sprintf("providers/%s.yaml", provider.Name)
466+
Expect(configData).To(HaveKey(key))
467+
468+
var parsed ProviderResource
469+
err = yaml.Unmarshal([]byte(configData[key]), &parsed)
470+
Expect(err).NotTo(HaveOccurred())
471+
Expect(parsed.Name).To(Equal(provider.Name))
472+
Expect(parsed.Type).To(Equal(provider.Type))
473+
}
454474
})
455475
})
456476
})

‎controllers/evalhub/deployment.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ func (r *EvalHubReconciler) buildDeploymentSpec(ctx context.Context, instance *e
109109
},
110110
{
111111
Name: "PROVIDERS_CONFIG_PATH",
112-
Value: "/etc/evalhub/providers.yaml",
112+
Value: "/etc/evalhub/providers/",
113113
},
114114
{
115115
Name: "SERVICE_URL",

0 commit comments

Comments
 (0)