Skip to content

chore(evalhub): hardcoded providers not needed #649

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
tarilabs wants to merge 1 commit into from
Closed

chore(evalhub): hardcoded providers not needed #649

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions config/rbac/kustomization.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,14 @@ resources:
- auth_proxy_role_binding.yaml
- auth_proxy_client_clusterrole.yaml
- evalhub/evalhub_auth_reviewer_role.yaml
- evalhub/evalhub_jobs_writer_role.yaml
- evalhub/evalhub_jobs_writer_binding.yaml
- evalhub/evalhub_job_config_role.yaml
- evalhub/evalhub_job_config_binding.yaml
- evalhub/evalhub_mlflow_access_role.yaml
- evalhub/evalhub_job_config_role.yaml
- evalhub/evalhub_jobs_writer_binding.yaml
- evalhub/evalhub_jobs_writer_role.yaml
- evalhub/evalhub_mlflow_access_binding.yaml
- evalhub/evalhub_mlflow_jobs_role.yaml
- evalhub/evalhub_mlflow_access_role.yaml
- evalhub/evalhub_mlflow_jobs_binding.yaml
- evalhub/evalhub_mlflow_jobs_role.yaml
- nemoguardrail_editor_role.yaml
- nemoguardrail_viewer_role.yaml
- trustyaiservice_editor_role.yaml
Expand Down
100 changes: 4 additions & 96 deletions controllers/evalhub/configmap.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,6 @@ import (
"sigs.k8s.io/yaml"
)

// ProviderConfig represents the provider configuration structure
type ProviderConfig struct {
Name string `yaml:"name"`
Type string `yaml:"type"`
Enabled bool `yaml:"enabled"`
Benchmarks []string `yaml:"benchmarks,omitempty"`
Config map[string]string `yaml:"config,omitempty"`
}

// DatabaseConfig represents the database configuration in config.yaml
type DatabaseConfig struct {
Driver string `yaml:"driver"`
Expand All @@ -40,10 +31,8 @@ type SecretsMapping struct {

// EvalHubConfig represents the eval-hub configuration structure
type EvalHubConfig struct {
Providers []ProviderConfig `yaml:"providers"`
Collections []string `yaml:"collections,omitempty"`
Database *DatabaseConfig `yaml:"database,omitempty"`
Secrets *SecretsMapping `yaml:"secrets,omitempty"`
Database *DatabaseConfig `yaml:"database,omitempty"`
Secrets *SecretsMapping `yaml:"secrets,omitempty"`
}

// reconcileConfigMap creates or updates the ConfigMap for EvalHub configuration
Expand Down Expand Up @@ -90,68 +79,7 @@ func (r *EvalHubReconciler) reconcileConfigMap(ctx context.Context, instance *ev

// generateConfigData generates the configuration data for the ConfigMap
func (r *EvalHubReconciler) generateConfigData(instance *evalhubv1alpha1.EvalHub) (map[string]string, error) {
config := EvalHubConfig{
Providers: make([]ProviderConfig, 0),
Collections: []string{},
}

// Default providers configuration set by the controller
config.Providers = []ProviderConfig{
{
Name: "lm-eval-harness",
Type: "lm_evaluation_harness",
Enabled: true,
Benchmarks: []string{
"arc_challenge", "hellaswag", "mmlu", "truthfulqa",
},
Config: map[string]string{
"batch_size": "8",
"max_length": "2048",
},
},
{
Name: "ragas-provider",
Type: "ragas",
Enabled: true,
Benchmarks: []string{
"faithfulness", "answer_relevancy", "context_precision", "context_recall",
},
Config: map[string]string{
"llm_model": "gpt-3.5-turbo",
"embeddings_model": "text-embedding-ada-002",
},
},
{
Name: "garak-security",
Type: "garak",
Enabled: false,
Benchmarks: []string{
"encoding", "injection", "malware", "prompt_injection",
},
Config: map[string]string{
"probe_set": "basic",
},
},
{
Name: "trustyai-custom",
Type: "trustyai_custom",
Enabled: true,
Benchmarks: []string{
"bias_detection", "fairness_metrics",
},
Config: map[string]string{
"bias_threshold": "0.1",
},
},
}

// Default collections
config.Collections = []string{
"healthcare_safety_v1",
"automotive_safety_v1",
"finance_compliance_v1",
"general_llm_eval_v1",
}
config := EvalHubConfig{}

// Conditionally add database configuration
if instance.Spec.IsDatabaseConfigured() {
Expand Down Expand Up @@ -179,31 +107,11 @@ func (r *EvalHubReconciler) generateConfigData(instance *evalhubv1alpha1.EvalHub
return nil, err
}

// Generate providers.yaml content
providersYAML, err := r.generateProvidersYAML(config.Providers)
if err != nil {
return nil, err
}

return map[string]string{
"config.yaml": string(configYAML),
"providers.yaml": providersYAML,
"config.yaml": string(configYAML),
}, nil
}

// generateProvidersYAML generates the providers.yaml configuration
func (r *EvalHubReconciler) generateProvidersYAML(providers []ProviderConfig) (string, error) {
providersData := make(map[string]interface{})
providersData["providers"] = providers

yamlData, err := yaml.Marshal(providersData)
if err != nil {
return "", err
}

return string(yamlData), nil
}

// getImageFromConfigMap gets a required image value from the operator's ConfigMap
// Returns error if ConfigMap is not found, key is missing, or value is empty
// This ensures explicit configuration and prevents deployment with unconfigured images
Expand Down
201 changes: 0 additions & 201 deletions controllers/evalhub/configmap_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,6 @@ var _ = Describe("EvalHub ConfigMap", func() {

By("Checking required keys exist")
Expect(configMap.Data).To(HaveKey("config.yaml"))
Expect(configMap.Data).To(HaveKey("providers.yaml"))
})

It("should have valid YAML configuration", func() {
Expand All @@ -103,173 +102,6 @@ var _ = Describe("EvalHub ConfigMap", func() {
var config EvalHubConfig
err = yaml.Unmarshal([]byte(configMap.Data["config.yaml"]), &config)
Expect(err).NotTo(HaveOccurred())

By("Checking default providers are present")
Expect(config.Providers).To(HaveLen(4))

var providerNames []string
for _, provider := range config.Providers {
providerNames = append(providerNames, provider.Name)
}
Expect(providerNames).To(ContainElements(
"lm-eval-harness", "ragas-provider", "garak-security", "trustyai-custom",
))

By("Checking default collections are present")
Expect(config.Collections).To(ContainElements(
"healthcare_safety_v1", "automotive_safety_v1",
"finance_compliance_v1", "general_llm_eval_v1",
))
})

It("should have valid providers.yaml", func() {
By("Reconciling configmap")
err := reconciler.reconcileConfigMap(ctx, evalHub)
Expect(err).NotTo(HaveOccurred())

By("Getting configmap")
configMap := waitForConfigMap(evalHubName+"-config", testNamespace)

By("Parsing providers.yaml")
var providersData map[string]interface{}
err = yaml.Unmarshal([]byte(configMap.Data["providers.yaml"]), &providersData)
Expect(err).NotTo(HaveOccurred())

By("Checking providers structure")
Expect(providersData).To(HaveKey("providers"))
providers, ok := providersData["providers"].([]interface{})
Expect(ok).To(BeTrue())
Expect(providers).To(HaveLen(4))
})

It("should configure lm-eval-harness provider correctly", func() {
By("Reconciling configmap")
err := reconciler.reconcileConfigMap(ctx, evalHub)
Expect(err).NotTo(HaveOccurred())

By("Getting configmap")
configMap := waitForConfigMap(evalHubName+"-config", testNamespace)

By("Parsing config.yaml")
var config EvalHubConfig
err = yaml.Unmarshal([]byte(configMap.Data["config.yaml"]), &config)
Expect(err).NotTo(HaveOccurred())

By("Finding lm-eval-harness provider")
var lmEvalProvider *ProviderConfig
for _, provider := range config.Providers {
if provider.Name == "lm-eval-harness" {
lmEvalProvider = &provider
break
}
}
Expect(lmEvalProvider).NotTo(BeNil())

By("Checking lm-eval-harness configuration")
Expect(lmEvalProvider.Type).To(Equal("lm_evaluation_harness"))
Expect(lmEvalProvider.Enabled).To(BeTrue())
Expect(lmEvalProvider.Benchmarks).To(ContainElements(
"arc_challenge", "hellaswag", "mmlu", "truthfulqa",
))
Expect(lmEvalProvider.Config["batch_size"]).To(Equal("8"))
Expect(lmEvalProvider.Config["max_length"]).To(Equal("2048"))
})

It("should configure ragas provider correctly", func() {
By("Reconciling configmap")
err := reconciler.reconcileConfigMap(ctx, evalHub)
Expect(err).NotTo(HaveOccurred())

By("Getting configmap")
configMap := waitForConfigMap(evalHubName+"-config", testNamespace)

By("Parsing config.yaml")
var config EvalHubConfig
err = yaml.Unmarshal([]byte(configMap.Data["config.yaml"]), &config)
Expect(err).NotTo(HaveOccurred())

By("Finding ragas provider")
var ragasProvider *ProviderConfig
for _, provider := range config.Providers {
if provider.Name == "ragas-provider" {
ragasProvider = &provider
break
}
}
Expect(ragasProvider).NotTo(BeNil())

By("Checking ragas configuration")
Expect(ragasProvider.Type).To(Equal("ragas"))
Expect(ragasProvider.Enabled).To(BeTrue())
Expect(ragasProvider.Benchmarks).To(ContainElements(
"faithfulness", "answer_relevancy", "context_precision", "context_recall",
))
Expect(ragasProvider.Config["llm_model"]).To(Equal("gpt-3.5-turbo"))
Expect(ragasProvider.Config["embeddings_model"]).To(Equal("text-embedding-ada-002"))
})

It("should configure garak security provider correctly", func() {
By("Reconciling configmap")
err := reconciler.reconcileConfigMap(ctx, evalHub)
Expect(err).NotTo(HaveOccurred())

By("Getting configmap")
configMap := waitForConfigMap(evalHubName+"-config", testNamespace)

By("Parsing config.yaml")
var config EvalHubConfig
err = yaml.Unmarshal([]byte(configMap.Data["config.yaml"]), &config)
Expect(err).NotTo(HaveOccurred())

By("Finding garak provider")
var garakProvider *ProviderConfig
for _, provider := range config.Providers {
if provider.Name == "garak-security" {
garakProvider = &provider
break
}
}
Expect(garakProvider).NotTo(BeNil())

By("Checking garak configuration")
Expect(garakProvider.Type).To(Equal("garak"))
Expect(garakProvider.Enabled).To(BeFalse()) // Disabled by default
Expect(garakProvider.Benchmarks).To(ContainElements(
"encoding", "injection", "malware", "prompt_injection",
))
Expect(garakProvider.Config["probe_set"]).To(Equal("basic"))
})

It("should configure trustyai custom provider correctly", func() {
By("Reconciling configmap")
err := reconciler.reconcileConfigMap(ctx, evalHub)
Expect(err).NotTo(HaveOccurred())

By("Getting configmap")
configMap := waitForConfigMap(evalHubName+"-config", testNamespace)

By("Parsing config.yaml")
var config EvalHubConfig
err = yaml.Unmarshal([]byte(configMap.Data["config.yaml"]), &config)
Expect(err).NotTo(HaveOccurred())

By("Finding trustyai provider")
var trustyaiProvider *ProviderConfig
for _, provider := range config.Providers {
if provider.Name == "trustyai-custom" {
trustyaiProvider = &provider
break
}
}
Expect(trustyaiProvider).NotTo(BeNil())

By("Checking trustyai configuration")
Expect(trustyaiProvider.Type).To(Equal("trustyai_custom"))
Expect(trustyaiProvider.Enabled).To(BeTrue())
Expect(trustyaiProvider.Benchmarks).To(ContainElements(
"bias_detection", "fairness_metrics",
))
Expect(trustyaiProvider.Config["bias_threshold"]).To(Equal("0.1"))
})

It("should update existing configmap", func() {
Expand Down Expand Up @@ -413,44 +245,11 @@ var _ = Describe("EvalHub ConfigMap", func() {

By("Checking required keys are present")
Expect(configData).To(HaveKey("config.yaml"))
Expect(configData).To(HaveKey("providers.yaml"))

By("Validating config.yaml content")
var config EvalHubConfig
err = yaml.Unmarshal([]byte(configData["config.yaml"]), &config)
Expect(err).NotTo(HaveOccurred())
Expect(config.Providers).To(HaveLen(4))
Expect(config.Collections).To(HaveLen(4))

By("Validating providers.yaml content")
var providersData map[string]interface{}
err = yaml.Unmarshal([]byte(configData["providers.yaml"]), &providersData)
Expect(err).NotTo(HaveOccurred())
Expect(providersData).To(HaveKey("providers"))
})

It("should generate providers YAML correctly", func() {
By("Generating configuration data")
configData, err := reconciler.generateConfigData(evalHub)
Expect(err).NotTo(HaveOccurred())

By("Parsing config.yaml to get providers")
var config EvalHubConfig
err = yaml.Unmarshal([]byte(configData["config.yaml"]), &config)
Expect(err).NotTo(HaveOccurred())

By("Generating providers YAML")
providersYAML, err := reconciler.generateProvidersYAML(config.Providers)
Expect(err).NotTo(HaveOccurred())

By("Verifying providers YAML matches configmap data")
Expect(providersYAML).To(Equal(configData["providers.yaml"]))

By("Verifying providers YAML is valid")
var providersData map[string]interface{}
err = yaml.Unmarshal([]byte(providersYAML), &providersData)
Expect(err).NotTo(HaveOccurred())
Expect(providersData).To(HaveKey("providers"))
})
})
})
9 changes: 5 additions & 4 deletions controllers/evalhub/deployment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,10 +107,11 @@ func (r *EvalHubReconciler) buildDeploymentSpec(ctx context.Context, instance *e
Name: "CONFIG_PATH",
Value: "/etc/evalhub/config.yaml",
},
{
Name: "PROVIDERS_CONFIG_PATH",
Value: "/etc/evalhub/providers.yaml",
},
// TODO: reconsider when the Operator reads OOTB providers from ConfigMaps
// {
// Name: "PROVIDERS_CONFIG_PATH",
// Value: "/etc/evalhub/providers.yaml",
// },
{
Name: "SERVICE_URL",
Value: fmt.Sprintf("https://%s.%s.svc.cluster.local:8443", instance.Name, instance.Namespace),
Expand Down
Loading
Loading