From cd3f07729eb9f424f3f83ef1bdda35ef40506832 Mon Sep 17 00:00:00 2001 From: Filinto Duran <1373693+filintod@users.noreply.github.com> Date: Fri, 3 Oct 2025 10:39:37 -0500 Subject: [PATCH 1/2] make it possible to use env vars in components Signed-off-by: Filinto Duran <1373693+filintod@users.noreply.github.com> --- cmd/run.go | 16 +- pkg/runexec/runexec_test.go | 5 +- pkg/standalone/run.go | 67 ++- pkg/templateprocessor/templateprocessor.go | 278 ++++++++++++ .../templateprocessor_test.go | 428 ++++++++++++++++++ pkg/templateprocessor/testdata/config.yaml | 11 + .../testdata/example-with-defaults.yaml | 51 +++ pkg/templateprocessor/testdata/pubsub.yaml | 16 + .../testdata/statestore.yaml | 15 + tests/e2e/standalone/template_test.go | 190 ++++++++ 10 files changed, 1066 insertions(+), 11 deletions(-) create mode 100644 pkg/templateprocessor/templateprocessor.go create mode 100644 pkg/templateprocessor/templateprocessor_test.go create mode 100644 pkg/templateprocessor/testdata/config.yaml create mode 100644 pkg/templateprocessor/testdata/example-with-defaults.yaml create mode 100644 pkg/templateprocessor/testdata/pubsub.yaml create mode 100644 pkg/templateprocessor/testdata/statestore.yaml create mode 100644 tests/e2e/standalone/template_test.go diff --git a/cmd/run.go b/cmd/run.go index 29b06d37a..80e8f52b5 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -210,7 +210,7 @@ dapr run --run-file /path/to/directory -k SchedulerHostAddress: schedulerHostAddress, DaprdInstallPath: daprRuntimePath, } - output, err := runExec.NewOutput(&standalone.RunConfig{ + runConfig := &standalone.RunConfig{ AppID: appID, AppChannelAddress: appChannelAddress, AppPort: appPort, @@ -222,7 +222,8 @@ dapr run --run-file /path/to/directory -k UnixDomainSocket: unixDomainSocket, InternalGRPCPort: internalGRPCPort, SharedRunConfig: *sharedRunConfig, - }) + } + output, err := runExec.NewOutput(runConfig) if err != nil { print.FailureStatusEvent(os.Stderr, err.Error()) os.Exit(1) @@ -464,6 +465,12 @@ dapr run --run-file /path/to/directory -k } } + // Cleanup temporary resources directory if it was created. + err = runConfig.CleanupTempResources() + if err != nil { + print.WarningStatusEvent(os.Stdout, "Failed to cleanup temporary resources: %s", err.Error()) + } + if exitWithError { os.Exit(1) } @@ -659,6 +666,11 @@ func gracefullyShutdownAppsAndCloseResources(runState []*runExec.RunExec, apps [ if err == nil && hasErr != nil { err = hasErr } + // Cleanup temporary resources for each app. + hasErr = app.RunConfig.CleanupTempResources() + if err == nil && hasErr != nil { + err = hasErr + } } return err } diff --git a/pkg/runexec/runexec_test.go b/pkg/runexec/runexec_test.go index 740cdb159..19e24cacc 100644 --- a/pkg/runexec/runexec_test.go +++ b/pkg/runexec/runexec_test.go @@ -108,7 +108,7 @@ func assertCommonArgs(t *testing.T, basicConfig *standalone.RunConfig, output *R assert.Equal(t, 8000, output.DaprHTTPPort) assert.Equal(t, 50001, output.DaprGRPCPort) - daprPath, err := standalone.GetDaprRuntimePath("") + _, err := standalone.GetDaprRuntimePath("") assert.NoError(t, err) assert.Contains(t, output.DaprCMD.Args[0], "daprd") @@ -119,7 +119,8 @@ func assertCommonArgs(t *testing.T, basicConfig *standalone.RunConfig, output *R assertArgumentEqual(t, "app-max-concurrency", "-1", output.DaprCMD.Args) assertArgumentEqual(t, "app-protocol", "http", output.DaprCMD.Args) assertArgumentEqual(t, "app-port", "3000", output.DaprCMD.Args) - assertArgumentEqual(t, "components-path", standalone.GetDaprComponentsPath(daprPath), output.DaprCMD.Args) + // Note: components-path is now a processed temporary directory with template substitution + assertArgumentContains(t, "components-path", "dapr-resources", output.DaprCMD.Args) assertArgumentEqual(t, "app-ssl", "", output.DaprCMD.Args) assertArgumentEqual(t, "metrics-port", "9001", output.DaprCMD.Args) assertArgumentEqual(t, "max-body-size", "-1", output.DaprCMD.Args) diff --git a/pkg/standalone/run.go b/pkg/standalone/run.go index fddfa6bf4..430c33125 100644 --- a/pkg/standalone/run.go +++ b/pkg/standalone/run.go @@ -31,6 +31,7 @@ import ( "gopkg.in/yaml.v2" "github.com/dapr/cli/pkg/print" + "github.com/dapr/cli/pkg/templateprocessor" localloader "github.com/dapr/dapr/pkg/components/loader" ) @@ -95,6 +96,8 @@ type SharedRunConfig struct { DaprdLogDestination LogDestType `yaml:"daprdLogDestination"` AppLogDestination LogDestType `yaml:"appLogDestination"` SchedulerHostAddress string `arg:"scheduler-host-address" yaml:"schedulerHostAddress"` + // Internal field to track temporary directory for processed resources. Not exposed to user. + tempResourcesDir string } func (meta *DaprMeta) newAppID() string { @@ -116,11 +119,33 @@ func (config *RunConfig) validateResourcesPaths() error { return fmt.Errorf("error validating resources path %q : %w", dirPath, err) } } - localLoader := localloader.NewLocalLoader(config.AppID, dirPath) - err := localLoader.Validate(context.Background()) + + // Process resources with environment variable substitution. + // This creates a temporary directory with processed files. + processedResources, err := templateprocessor.ProcessResourcesWithEnvVars(dirPath) + if err != nil { + return fmt.Errorf("error processing resource templates: %w", err) + } + + // Store temp directory for cleanup and use processed paths for validation. + config.tempResourcesDir = processedResources.TempDir + validationPaths := processedResources.ProcessedPaths + + // Validate the processed resources. + localLoader := localloader.NewLocalLoader(config.AppID, validationPaths) + err = localLoader.Validate(context.Background()) if err != nil { return fmt.Errorf("error validating components in resources path %q : %w", dirPath, err) } + + // Update ResourcesPaths to point to processed resources. + // This ensures daprd uses the processed files with substituted values. + if len(config.ResourcesPaths) > 0 { + config.ResourcesPaths = processedResources.ProcessedPaths + } else { + config.ComponentsPath = processedResources.ProcessedPaths[0] + } + return nil } @@ -380,8 +405,12 @@ func (config *RunConfig) getArgs() []string { // This is needed because the config struct has embedded struct. func getArgsFromSchema(schema reflect.Value, args []string) []string { for i := range schema.NumField() { - valueField := schema.Field(i).Interface() typeField := schema.Type().Field(i) + // Skip unexported fields. + if !typeField.IsExported() { + continue + } + valueField := schema.Field(i).Interface() key := typeField.Tag.Get("arg") if typeField.Type.Kind() == reflect.Struct { args = getArgsFromSchema(schema.Field(i), args) @@ -422,8 +451,12 @@ func (config *RunConfig) SetDefaultFromSchema() { func (config *RunConfig) setDefaultFromSchemaRecursive(schema reflect.Value) { for i := range schema.NumField() { - valueField := schema.Field(i) typeField := schema.Type().Field(i) + // Skip unexported fields. + if !typeField.IsExported() { + continue + } + valueField := schema.Field(i) if typeField.Type.Kind() == reflect.Struct { config.setDefaultFromSchemaRecursive(valueField) continue @@ -448,8 +481,12 @@ func (config *RunConfig) getEnv() []string { // Handle values from config that have an "env" tag. schema := reflect.ValueOf(*config) for i := range schema.NumField() { - valueField := schema.Field(i).Interface() typeField := schema.Type().Field(i) + // Skip unexported fields. + if !typeField.IsExported() { + continue + } + valueField := schema.Field(i).Interface() key := typeField.Tag.Get("env") if len(key) == 0 { continue @@ -508,8 +545,12 @@ func (config *RunConfig) GetEnv() map[string]string { env := map[string]string{} schema := reflect.ValueOf(*config) for i := range schema.NumField() { - valueField := schema.Field(i).Interface() typeField := schema.Type().Field(i) + // Skip unexported fields. + if !typeField.IsExported() { + continue + } + valueField := schema.Field(i).Interface() key := typeField.Tag.Get("env") if len(key) == 0 { continue @@ -532,8 +573,12 @@ func (config *RunConfig) GetAnnotations() map[string]string { annotations := map[string]string{} schema := reflect.ValueOf(*config) for i := range schema.NumField() { - valueField := schema.Field(i).Interface() typeField := schema.Type().Field(i) + // Skip unexported fields. + if !typeField.IsExported() { + continue + } + valueField := schema.Field(i).Interface() key := typeField.Tag.Get("annotation") if len(key) == 0 { continue @@ -612,3 +657,11 @@ func (l LogDestType) IsValid() error { } return fmt.Errorf("invalid log destination type: %s", l) } + +// CleanupTempResources removes the temporary directory created for processed resources. +func (config *RunConfig) CleanupTempResources() error { + if config.tempResourcesDir == "" { + return nil + } + return templateprocessor.Cleanup(config.tempResourcesDir) +} diff --git a/pkg/templateprocessor/templateprocessor.go b/pkg/templateprocessor/templateprocessor.go new file mode 100644 index 000000000..cbb6ffc61 --- /dev/null +++ b/pkg/templateprocessor/templateprocessor.go @@ -0,0 +1,278 @@ +/* +Copyright 2025 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package templateprocessor + +import ( + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "strings" +) + +// Template pattern: {{ENV_VAR_NAME}} or {{ENV_VAR_NAME:default value}} +// Matches uppercase letters, digits, and underscores for variable names. +// Optionally matches a colon followed by a default value. +var templatePattern = regexp.MustCompile(`\{\{([A-Z_][A-Z0-9_]*)(?::([^}]*))?\}\}`) + +// ProcessedResources contains information about the processed resources. +type ProcessedResources struct { + TempDir string // Temporary directory containing processed files + ProcessedPaths []string // Paths to processed resource directories + HasTemplates bool // Whether any templates were found and processed +} + +// ProcessResourcesWithEnvVars processes resource files by substituting environment variables +// in template placeholders like {{ENV_VAR_NAME}}. It creates a temporary directory with +// processed files and returns the paths to use for daprd. +func ProcessResourcesWithEnvVars(resourcesPaths []string) (*ProcessedResources, error) { + if len(resourcesPaths) == 0 { + return nil, fmt.Errorf("no resource paths provided") + } + + // Create temporary directory for processed files. + tempDir, err := os.MkdirTemp("", "dapr-resources-*") + if err != nil { + return nil, fmt.Errorf("failed to create temporary directory: %w", err) + } + + result := &ProcessedResources{ + TempDir: tempDir, + ProcessedPaths: make([]string, 0, len(resourcesPaths)), + HasTemplates: false, + } + + // Process each resource path. + for i, resourcePath := range resourcesPaths { + // Create a subdirectory in temp dir for each resource path. + destDir := filepath.Join(tempDir, fmt.Sprintf("resources-%d", i)) + err := os.MkdirAll(destDir, 0755) + if err != nil { + Cleanup(result.TempDir) + return nil, fmt.Errorf("failed to create destination directory: %w", err) + } + + // Check if resource path exists and is a directory. + info, err := os.Stat(resourcePath) + if err != nil { + Cleanup(result.TempDir) + return nil, fmt.Errorf("resource path %q does not exist: %w", resourcePath, err) + } + + if !info.IsDir() { + Cleanup(result.TempDir) + return nil, fmt.Errorf("resource path %q is not a directory", resourcePath) + } + + // Process all files in the resource directory. + hasTemplates, err := processDirectory(resourcePath, destDir) + if err != nil { + Cleanup(result.TempDir) + return nil, fmt.Errorf("failed to process resource path %q: %w", resourcePath, err) + } + + if hasTemplates { + result.HasTemplates = true + } + + result.ProcessedPaths = append(result.ProcessedPaths, destDir) + } + + return result, nil +} + +// processDirectory recursively processes all files in a directory. +func processDirectory(srcDir, destDir string) (bool, error) { + hasTemplates := false + + entries, err := os.ReadDir(srcDir) + if err != nil { + return false, fmt.Errorf("failed to read directory %q: %w", srcDir, err) + } + + for _, entry := range entries { + srcPath := filepath.Join(srcDir, entry.Name()) + destPath := filepath.Join(destDir, entry.Name()) + + if entry.IsDir() { + // Create subdirectory and process recursively. + err := os.MkdirAll(destPath, 0755) + if err != nil { + return false, fmt.Errorf("failed to create directory %q: %w", destPath, err) + } + + subHasTemplates, err := processDirectory(srcPath, destPath) + if err != nil { + return false, err + } + if subHasTemplates { + hasTemplates = true + } + } else { + // Process file. + fileHasTemplates, err := processFile(srcPath, destPath) + if err != nil { + return false, fmt.Errorf("failed to process file %q: %w", srcPath, err) + } + if fileHasTemplates { + hasTemplates = true + } + } + } + + return hasTemplates, nil +} + +// processFile processes a single file, substituting environment variables if it's a +// text file (YAML, JSON), otherwise copying it as-is. +func processFile(srcPath, destPath string) (bool, error) { + // Check if file should be processed for templates based on extension. + shouldProcess := shouldProcessFile(srcPath) + + if !shouldProcess { + // Copy file as-is. + return false, copyFile(srcPath, destPath) + } + + // Read source file. + content, err := os.ReadFile(srcPath) + if err != nil { + return false, fmt.Errorf("failed to read file: %w", err) + } + + // Substitute environment variables. + processedContent, hasTemplates := substituteEnvVars(content) + + // Get original file permissions. + info, err := os.Stat(srcPath) + if err != nil { + return false, fmt.Errorf("failed to get file info: %w", err) + } + + // Write processed content to destination. + err = os.WriteFile(destPath, processedContent, info.Mode()) + if err != nil { + return false, fmt.Errorf("failed to write file: %w", err) + } + + return hasTemplates, nil +} + +// shouldProcessFile determines if a file should be processed for templates +// based on its extension. +func shouldProcessFile(path string) bool { + ext := strings.ToLower(filepath.Ext(path)) + processableExtensions := []string{".yaml", ".yml", ".json"} + + for _, processable := range processableExtensions { + if ext == processable { + return true + } + } + return false +} + +// substituteEnvVars replaces template placeholders {{ENV_VAR_NAME}} or +// {{ENV_VAR_NAME:default value}} with environment variable values. +// If the environment variable doesn't exist: +// - If a default value is provided, use it +// - Otherwise, leave the template as-is +// +// Returns the processed content and whether any substitutions were made. +func substituteEnvVars(content []byte) ([]byte, bool) { + hasTemplates := false + strContent := string(content) + + // Find all template matches. + result := templatePattern.ReplaceAllStringFunc(strContent, func(match string) string { + // Extract variable name and optional default value. + // The regex captures: {{VAR_NAME}} or {{VAR_NAME:default}} + matches := templatePattern.FindStringSubmatch(match) + if len(matches) < 2 { + return match + } + + varName := matches[1] + // Check if the match contains a colon to determine if default value syntax is used + hasDefaultSyntax := strings.Contains(match, ":") + defaultValue := "" + if len(matches) > 2 { + defaultValue = matches[2] + } + + // Get environment variable value. + if value, exists := os.LookupEnv(varName); exists { + hasTemplates = true + return value + } + + // If env var doesn't exist but default value syntax is present (even if empty), + // use the default value. + if hasDefaultSyntax { + hasTemplates = true + return defaultValue + } + + // If no env var and no default syntax, leave the template as-is. + return match + }) + + return []byte(result), hasTemplates +} + +// copyFile copies a file from src to dest, preserving permissions. +func copyFile(src, dest string) error { + // Open source file. + srcFile, err := os.Open(src) + if err != nil { + return fmt.Errorf("failed to open source file: %w", err) + } + defer srcFile.Close() + + // Get source file info for permissions. + info, err := srcFile.Stat() + if err != nil { + return fmt.Errorf("failed to get file info: %w", err) + } + + // Create destination file. + destFile, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode()) + if err != nil { + return fmt.Errorf("failed to create destination file: %w", err) + } + defer destFile.Close() + + // Copy content. + _, err = io.Copy(destFile, srcFile) + if err != nil { + return fmt.Errorf("failed to copy file content: %w", err) + } + + return nil +} + +// Cleanup removes the temporary directory and all its contents. +func Cleanup(tempDir string) error { + if tempDir == "" { + return nil + } + + err := os.RemoveAll(tempDir) + if err != nil { + return fmt.Errorf("failed to cleanup temporary directory %q: %w", tempDir, err) + } + + return nil +} diff --git a/pkg/templateprocessor/templateprocessor_test.go b/pkg/templateprocessor/templateprocessor_test.go new file mode 100644 index 000000000..adc1574fa --- /dev/null +++ b/pkg/templateprocessor/templateprocessor_test.go @@ -0,0 +1,428 @@ +/* +Copyright 2025 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package templateprocessor + +import ( + "fmt" + "os" + "path/filepath" + "testing" +) + +func TestSubstituteEnvVars(t *testing.T) { + tests := []struct { + name string + input string + envVars map[string]string + expected string + expectTemplate bool + }{ + { + name: "single env var substitution", + input: "host: {{REDIS_HOST}}", + envVars: map[string]string{"REDIS_HOST": "localhost"}, + expected: "host: localhost", + expectTemplate: true, + }, + { + name: "multiple env var substitution", + input: "host: {{REDIS_HOST}}\nport: {{REDIS_PORT}}", + envVars: map[string]string{"REDIS_HOST": "localhost", "REDIS_PORT": "6379"}, + expected: "host: localhost\nport: 6379", + expectTemplate: true, + }, + { + name: "env var not set - leave as template", + input: "host: {{MISSING_VAR}}", + envVars: map[string]string{}, + expected: "host: {{MISSING_VAR}}", + expectTemplate: false, + }, + { + name: "mixed set and unset env vars", + input: "host: {{REDIS_HOST}}\nport: {{MISSING_PORT}}", + envVars: map[string]string{"REDIS_HOST": "localhost"}, + expected: "host: localhost\nport: {{MISSING_PORT}}", + expectTemplate: true, + }, + { + name: "no templates", + input: "host: localhost\nport: 6379", + envVars: map[string]string{}, + expected: "host: localhost\nport: 6379", + expectTemplate: false, + }, + { + name: "env var with underscores and numbers", + input: "key: {{MY_VAR_123}}", + envVars: map[string]string{"MY_VAR_123": "value123"}, + expected: "key: value123", + expectTemplate: true, + }, + { + name: "lowercase should not match", + input: "key: {{lowercase_var}}", + envVars: map[string]string{"lowercase_var": "value"}, + expected: "key: {{lowercase_var}}", + expectTemplate: false, + }, + { + name: "empty string substitution", + input: "key: {{EMPTY_VAR}}", + envVars: map[string]string{"EMPTY_VAR": ""}, + expected: "key: ", + expectTemplate: true, + }, + { + name: "default value when env var not set", + input: "host: {{REDIS_HOST:localhost}}", + envVars: map[string]string{}, + expected: "host: localhost", + expectTemplate: true, + }, + { + name: "env var overrides default value", + input: "host: {{REDIS_HOST:localhost}}", + envVars: map[string]string{"REDIS_HOST": "redis.example.com"}, + expected: "host: redis.example.com", + expectTemplate: true, + }, + { + name: "default value with spaces", + input: "name: {{APP_NAME:My Application}}", + envVars: map[string]string{}, + expected: "name: My Application", + expectTemplate: true, + }, + { + name: "default value with special characters", + input: "url: {{DATABASE_URL:postgresql://localhost:5432/db}}", + envVars: map[string]string{}, + expected: "url: postgresql://localhost:5432/db", + expectTemplate: true, + }, + { + name: "empty default value", + input: "key: {{OPTIONAL_VAR:}}", + envVars: map[string]string{}, + expected: "key: ", + expectTemplate: true, + }, + { + name: "multiple templates with defaults", + input: "host: {{HOST:localhost}}\nport: {{PORT:6379}}", + envVars: map[string]string{"HOST": "redis-server"}, + expected: "host: redis-server\nport: 6379", + expectTemplate: true, + }, + { + name: "default value with colon in it", + input: "url: {{URL:http://localhost:8080}}", + envVars: map[string]string{}, + expected: "url: http://localhost:8080", + expectTemplate: true, + }, + { + name: "template without default stays unchanged when var missing", + input: "key: {{MISSING_VAR}}", + envVars: map[string]string{}, + expected: "key: {{MISSING_VAR}}", + expectTemplate: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Set up environment variables. + for k, v := range tt.envVars { + os.Setenv(k, v) + defer os.Unsetenv(k) + } + + result, hasTemplates := substituteEnvVars([]byte(tt.input)) + + if string(result) != tt.expected { + t.Errorf("substituteEnvVars() = %q, want %q", string(result), tt.expected) + } + + if hasTemplates != tt.expectTemplate { + t.Errorf("substituteEnvVars() hasTemplates = %v, want %v", hasTemplates, tt.expectTemplate) + } + }) + } +} + +func TestShouldProcessFile(t *testing.T) { + tests := []struct { + name string + path string + expected bool + }{ + {"yaml file", "component.yaml", true}, + {"yml file", "component.yml", true}, + {"json file", "config.json", true}, + {"YAML uppercase", "component.YAML", true}, + {"text file", "readme.txt", false}, + {"go file", "main.go", false}, + {"no extension", "Dockerfile", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := shouldProcessFile(tt.path) + if result != tt.expected { + t.Errorf("shouldProcessFile(%q) = %v, want %v", tt.path, result, tt.expected) + } + }) + } +} + +func TestProcessResourcesWithEnvVars(t *testing.T) { + // Create a temporary directory with test files. + tempDir := t.TempDir() + + // Create test directory structure. + resourceDir := filepath.Join(tempDir, "resources") + err := os.MkdirAll(resourceDir, 0755) + if err != nil { + t.Fatalf("Failed to create test directory: %v", err) + } + + // Create test component file with template. + componentContent := `apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: statestore +spec: + type: state.redis + metadata: + - name: redisHost + value: {{TEST_REDIS_HOST}} + - name: redisPort + value: {{TEST_REDIS_PORT}} +` + componentPath := filepath.Join(resourceDir, "statestore.yaml") + err = os.WriteFile(componentPath, []byte(componentContent), 0644) + if err != nil { + t.Fatalf("Failed to create test component file: %v", err) + } + + // Create test config file without template. + configContent := `apiVersion: dapr.io/v1alpha1 +kind: Configuration +metadata: + name: appconfig +spec: + tracing: + samplingRate: "1" +` + configPath := filepath.Join(resourceDir, "config.yaml") + err = os.WriteFile(configPath, []byte(configContent), 0644) + if err != nil { + t.Fatalf("Failed to create test config file: %v", err) + } + + // Create a non-processable file. + readmePath := filepath.Join(resourceDir, "README.txt") + err = os.WriteFile(readmePath, []byte("This is a readme"), 0644) + if err != nil { + t.Fatalf("Failed to create test readme file: %v", err) + } + + // Set environment variables. + os.Setenv("TEST_REDIS_HOST", "localhost") + os.Setenv("TEST_REDIS_PORT", "6379") + defer os.Unsetenv("TEST_REDIS_HOST") + defer os.Unsetenv("TEST_REDIS_PORT") + + // Process resources. + result, err := ProcessResourcesWithEnvVars([]string{resourceDir}) + if err != nil { + t.Fatalf("ProcessResourcesWithEnvVars() failed: %v", err) + } + defer Cleanup(result.TempDir) + + // Verify temp directory was created. + if result.TempDir == "" { + t.Error("TempDir is empty") + } + + // Verify processed paths. + if len(result.ProcessedPaths) != 1 { + t.Errorf("Expected 1 processed path, got %d", len(result.ProcessedPaths)) + } + + // Verify templates were found. + if !result.HasTemplates { + t.Error("Expected HasTemplates to be true") + } + + // Read processed component file. + processedComponentPath := filepath.Join(result.ProcessedPaths[0], "statestore.yaml") + processedContent, err := os.ReadFile(processedComponentPath) + if err != nil { + t.Fatalf("Failed to read processed component file: %v", err) + } + + expectedContent := `apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: statestore +spec: + type: state.redis + metadata: + - name: redisHost + value: localhost + - name: redisPort + value: 6379 +` + if string(processedContent) != expectedContent { + t.Errorf("Processed content doesn't match.\nGot:\n%s\nWant:\n%s", string(processedContent), expectedContent) + } + + // Verify config file was copied (no substitution). + processedConfigPath := filepath.Join(result.ProcessedPaths[0], "config.yaml") + processedConfigContent, err := os.ReadFile(processedConfigPath) + if err != nil { + t.Fatalf("Failed to read processed config file: %v", err) + } + + if string(processedConfigContent) != configContent { + t.Error("Config file content was modified when it shouldn't be") + } + + // Verify non-processable file was copied. + processedReadmePath := filepath.Join(result.ProcessedPaths[0], "README.txt") + if _, err := os.Stat(processedReadmePath); os.IsNotExist(err) { + t.Error("README.txt was not copied") + } +} + +func TestProcessResourcesWithMultiplePaths(t *testing.T) { + // Create temporary directories with test files. + tempDir := t.TempDir() + + // Create first resource directory. + resourceDir1 := filepath.Join(tempDir, "resources1") + err := os.MkdirAll(resourceDir1, 0755) + if err != nil { + t.Fatalf("Failed to create test directory 1: %v", err) + } + + componentContent1 := `apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: component1 +spec: + type: state.redis + metadata: + - name: host + value: {{MULTI_TEST_HOST}} +` + err = os.WriteFile(filepath.Join(resourceDir1, "component1.yaml"), []byte(componentContent1), 0644) + if err != nil { + t.Fatalf("Failed to create component1 file: %v", err) + } + + // Create second resource directory. + resourceDir2 := filepath.Join(tempDir, "resources2") + err = os.MkdirAll(resourceDir2, 0755) + if err != nil { + t.Fatalf("Failed to create test directory 2: %v", err) + } + + componentContent2 := `apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: component2 +spec: + type: pubsub.redis + metadata: + - name: host + value: {{MULTI_TEST_HOST}} +` + err = os.WriteFile(filepath.Join(resourceDir2, "component2.yaml"), []byte(componentContent2), 0644) + if err != nil { + t.Fatalf("Failed to create component2 file: %v", err) + } + + // Set environment variable. + os.Setenv("MULTI_TEST_HOST", "redis-server") + defer os.Unsetenv("MULTI_TEST_HOST") + + // Process multiple resource paths. + result, err := ProcessResourcesWithEnvVars([]string{resourceDir1, resourceDir2}) + if err != nil { + t.Fatalf("ProcessResourcesWithEnvVars() failed: %v", err) + } + defer Cleanup(result.TempDir) + + // Verify processed paths. + if len(result.ProcessedPaths) != 2 { + t.Errorf("Expected 2 processed paths, got %d", len(result.ProcessedPaths)) + } + + // Verify both files were processed. + for i, processedPath := range result.ProcessedPaths { + componentFile := filepath.Join(processedPath, fmt.Sprintf("component%d.yaml", i+1)) + content, err := os.ReadFile(componentFile) + if err != nil { + t.Errorf("Failed to read processed component%d file: %v", i+1, err) + continue + } + + if !contains(string(content), "redis-server") { + t.Errorf("Component%d file does not contain substituted value", i+1) + } + } +} + +func TestCleanup(t *testing.T) { + // Create a temporary directory. + tempDir, err := os.MkdirTemp("", "test-cleanup-*") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + + // Create a file in it. + testFile := filepath.Join(tempDir, "test.txt") + err = os.WriteFile(testFile, []byte("test"), 0644) + if err != nil { + t.Fatalf("Failed to create test file: %v", err) + } + + // Cleanup. + err = Cleanup(tempDir) + if err != nil { + t.Errorf("Cleanup() failed: %v", err) + } + + // Verify directory was removed. + if _, err := os.Stat(tempDir); !os.IsNotExist(err) { + t.Error("Temp directory still exists after cleanup") + } +} + +func TestCleanupEmptyString(t *testing.T) { + // Cleanup with empty string should not error. + err := Cleanup("") + if err != nil { + t.Errorf("Cleanup(\"\") should not error, got: %v", err) + } +} + +func contains(s, substr string) bool { + return len(s) >= len(substr) && (s == substr || len(substr) == 0 || + (len(s) > 0 && (s[0:len(substr)] == substr || contains(s[1:], substr)))) +} diff --git a/pkg/templateprocessor/testdata/config.yaml b/pkg/templateprocessor/testdata/config.yaml new file mode 100644 index 000000000..3ed887913 --- /dev/null +++ b/pkg/templateprocessor/testdata/config.yaml @@ -0,0 +1,11 @@ +apiVersion: dapr.io/v1alpha1 +kind: Configuration +metadata: + name: appconfig +spec: + tracing: + samplingRate: "1" + metrics: + enabled: true + + diff --git a/pkg/templateprocessor/testdata/example-with-defaults.yaml b/pkg/templateprocessor/testdata/example-with-defaults.yaml new file mode 100644 index 000000000..b1dbda46a --- /dev/null +++ b/pkg/templateprocessor/testdata/example-with-defaults.yaml @@ -0,0 +1,51 @@ +# Example Component with Default Values +# This demonstrates the template variable substitution feature with defaults +# +# Syntax: {{ENV_VAR_NAME:default_value}} +# +# Behavior: +# - If ENV_VAR_NAME is set, use its value +# - If ENV_VAR_NAME is not set, use default_value +# - If no default is provided ({{ENV_VAR_NAME}}), leave unchanged when var is missing + +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: redis-statestore +spec: + type: state.redis + version: v1 + metadata: + # Basic default - simple value + - name: redisHost + value: {{REDIS_HOST:localhost}} + + # Numeric default + - name: redisPort + value: {{REDIS_PORT:6379}} + + # Boolean default + - name: enableTLS + value: {{REDIS_TLS:false}} + + # URL with protocol and port + - name: connectionString + value: {{REDIS_CONNECTION:redis://localhost:6379}} + + # Empty default - explicitly set to empty if not provided + - name: redisDB + value: {{REDIS_DB:0}} + + # No default - must be set or will remain as template + - name: redisPassword + value: {{REDIS_PASSWORD}} + + # Default with special characters + - name: clientName + value: {{REDIS_CLIENT_NAME:dapr-app-v1.0}} + + # Default with spaces + - name: description + value: {{REDIS_DESCRIPTION:Default Redis State Store}} + + diff --git a/pkg/templateprocessor/testdata/pubsub.yaml b/pkg/templateprocessor/testdata/pubsub.yaml new file mode 100644 index 000000000..11aa39a4b --- /dev/null +++ b/pkg/templateprocessor/testdata/pubsub.yaml @@ -0,0 +1,16 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: pubsub +spec: + type: pubsub.redis + version: v1 + metadata: + - name: redisHost + value: {{PUBSUB_HOST}} + - name: redisPort + value: "6379" + - name: consumerID + value: myapp + + diff --git a/pkg/templateprocessor/testdata/statestore.yaml b/pkg/templateprocessor/testdata/statestore.yaml new file mode 100644 index 000000000..9fde120ab --- /dev/null +++ b/pkg/templateprocessor/testdata/statestore.yaml @@ -0,0 +1,15 @@ +apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: statestore +spec: + type: state.redis + version: v1 + metadata: + - name: redisHost + value: {{REDIS_HOST:localhost}} + - name: redisPort + value: {{REDIS_PORT:6379}} + - name: redisPassword + value: {{REDIS_PASSWORD}} + diff --git a/tests/e2e/standalone/template_test.go b/tests/e2e/standalone/template_test.go new file mode 100644 index 000000000..8de2d72f0 --- /dev/null +++ b/tests/e2e/standalone/template_test.go @@ -0,0 +1,190 @@ +/* +Copyright 2025 The Dapr Authors +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package standalone_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/dapr/cli/pkg/templateprocessor" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTemplateProcessorIntegration(t *testing.T) { + // Create a temporary directory for test resources. + tempDir := t.TempDir() + resourcesDir := filepath.Join(tempDir, "resources") + err := os.MkdirAll(resourcesDir, 0755) + require.NoError(t, err) + + // Create a component file with templates. + componentContent := `apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: statestore +spec: + type: state.redis + version: v1 + metadata: + - name: redisHost + value: {{TEST_REDIS_HOST}} + - name: redisPort + value: {{TEST_REDIS_PORT}} + - name: enableTLS + value: {{TEST_ENABLE_TLS}} +` + componentPath := filepath.Join(resourcesDir, "statestore.yaml") + err = os.WriteFile(componentPath, []byte(componentContent), 0644) + require.NoError(t, err) + + // Set environment variables. + os.Setenv("TEST_REDIS_HOST", "my-redis.example.com") + os.Setenv("TEST_REDIS_PORT", "6380") + os.Setenv("TEST_ENABLE_TLS", "true") + defer os.Unsetenv("TEST_REDIS_HOST") + defer os.Unsetenv("TEST_REDIS_PORT") + defer os.Unsetenv("TEST_ENABLE_TLS") + + // Process resources. + result, err := templateprocessor.ProcessResourcesWithEnvVars([]string{resourcesDir}) + require.NoError(t, err) + require.NotNil(t, result) + defer templateprocessor.Cleanup(result.TempDir) + + // Verify temp directory was created. + assert.NotEmpty(t, result.TempDir) + assert.True(t, result.HasTemplates) + assert.Len(t, result.ProcessedPaths, 1) + + // Read and verify the processed component file. + processedComponentPath := filepath.Join(result.ProcessedPaths[0], "statestore.yaml") + processedContent, err := os.ReadFile(processedComponentPath) + require.NoError(t, err) + + // Verify substitutions were made. + processedStr := string(processedContent) + assert.Contains(t, processedStr, "my-redis.example.com") + assert.Contains(t, processedStr, "6380") + assert.Contains(t, processedStr, "true") + assert.NotContains(t, processedStr, "{{TEST_REDIS_HOST}}") + assert.NotContains(t, processedStr, "{{TEST_REDIS_PORT}}") + assert.NotContains(t, processedStr, "{{TEST_ENABLE_TLS}}") + + // Verify cleanup works. + err = templateprocessor.Cleanup(result.TempDir) + assert.NoError(t, err) + + // Verify temp directory was removed. + _, err = os.Stat(result.TempDir) + assert.True(t, os.IsNotExist(err)) +} + +func TestTemplateProcessorWithDefaultValues(t *testing.T) { + // Create a temporary directory for test resources. + tempDir := t.TempDir() + resourcesDir := filepath.Join(tempDir, "resources") + err := os.MkdirAll(resourcesDir, 0755) + require.NoError(t, err) + + // Create a component file with default values. + componentContent := `apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: statestore +spec: + type: state.redis + version: v1 + metadata: + - name: redisHost + value: {{REDIS_HOST:localhost}} + - name: redisPort + value: {{REDIS_PORT:6379}} + - name: enableTLS + value: {{ENABLE_TLS:false}} +` + componentPath := filepath.Join(resourcesDir, "statestore.yaml") + err = os.WriteFile(componentPath, []byte(componentContent), 0644) + require.NoError(t, err) + + // Set only one environment variable, others should use defaults. + os.Setenv("REDIS_HOST", "my-redis.example.com") + defer os.Unsetenv("REDIS_HOST") + + // Process resources. + result, err := templateprocessor.ProcessResourcesWithEnvVars([]string{resourcesDir}) + require.NoError(t, err) + require.NotNil(t, result) + defer templateprocessor.Cleanup(result.TempDir) + + // Verify templates were processed. + assert.True(t, result.HasTemplates) + + // Read the processed component file. + processedComponentPath := filepath.Join(result.ProcessedPaths[0], "statestore.yaml") + processedContent, err := os.ReadFile(processedComponentPath) + require.NoError(t, err) + + // Verify: REDIS_HOST from env, others from defaults. + processedStr := string(processedContent) + assert.Contains(t, processedStr, "my-redis.example.com") // From env var + assert.Contains(t, processedStr, "6379") // From default + assert.Contains(t, processedStr, "false") // From default + assert.NotContains(t, processedStr, "{{REDIS_HOST:") + assert.NotContains(t, processedStr, "{{REDIS_PORT:") + assert.NotContains(t, processedStr, "{{ENABLE_TLS:") +} + +func TestTemplateProcessorWithMissingEnvVars(t *testing.T) { + // Create a temporary directory for test resources. + tempDir := t.TempDir() + resourcesDir := filepath.Join(tempDir, "resources") + err := os.MkdirAll(resourcesDir, 0755) + require.NoError(t, err) + + // Create a component file with templates. + componentContent := `apiVersion: dapr.io/v1alpha1 +kind: Component +metadata: + name: statestore +spec: + type: state.redis + version: v1 + metadata: + - name: redisHost + value: {{MISSING_REDIS_HOST}} + - name: redisPort + value: 6379 +` + componentPath := filepath.Join(resourcesDir, "statestore.yaml") + err = os.WriteFile(componentPath, []byte(componentContent), 0644) + require.NoError(t, err) + + // Process resources without setting the env var. + result, err := templateprocessor.ProcessResourcesWithEnvVars([]string{resourcesDir}) + require.NoError(t, err) + require.NotNil(t, result) + defer templateprocessor.Cleanup(result.TempDir) + + // Read the processed component file. + processedComponentPath := filepath.Join(result.ProcessedPaths[0], "statestore.yaml") + processedContent, err := os.ReadFile(processedComponentPath) + require.NoError(t, err) + + // Verify template was left as-is since env var doesn't exist. + processedStr := string(processedContent) + assert.Contains(t, processedStr, "{{MISSING_REDIS_HOST}}") + assert.Contains(t, processedStr, "6379") +} From f9ec0d39f855e51ba1496f468e8cbecd9c6cc6b4 Mon Sep 17 00:00:00 2001 From: Filinto Duran <1373693+filintod@users.noreply.github.com> Date: Fri, 3 Oct 2025 11:00:48 -0500 Subject: [PATCH 2/2] correct tests Signed-off-by: Filinto Duran <1373693+filintod@users.noreply.github.com> --- .../templateprocessor_test.go | 73 +++++++++++++++++++ .../testdata/example-with-defaults.yaml | 6 +- 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/pkg/templateprocessor/templateprocessor_test.go b/pkg/templateprocessor/templateprocessor_test.go index adc1574fa..0010f7bd5 100644 --- a/pkg/templateprocessor/templateprocessor_test.go +++ b/pkg/templateprocessor/templateprocessor_test.go @@ -422,6 +422,79 @@ func TestCleanupEmptyString(t *testing.T) { } } +func TestExampleWithDefaultsFile(t *testing.T) { + // Test that the example-with-defaults.yaml file is valid and processes correctly + exampleFile := filepath.Join("testdata", "example-with-defaults.yaml") + + // Read the example file + content, err := os.ReadFile(exampleFile) + if err != nil { + t.Fatalf("Failed to read example-with-defaults.yaml: %v", err) + } + + // Test without setting any env vars (should use defaults) + processed, hasTemplates := substituteEnvVars(content) + if !hasTemplates { + t.Error("Expected templates to be found in example-with-defaults.yaml") + } + + processedStr := string(processed) + + // Verify defaults are applied + if !contains(processedStr, "localhost") { + t.Error("Expected REDIS_HOST default 'localhost' to be applied") + } + if !contains(processedStr, "6379") { + t.Error("Expected REDIS_PORT default '6379' to be applied") + } + if !contains(processedStr, "false") { + t.Error("Expected REDIS_TLS default 'false' to be applied") + } + + // Verify empty default is applied for OPTIONAL_TAG + // The line should have "value: " with nothing after it (or newline) + if !contains(processedStr, "optionalTag\n value: \n") && !contains(processedStr, "optionalTag\n value:\n") { + t.Error("Expected OPTIONAL_TAG empty default to result in 'value: ' (empty)") + } + + // Verify template without default stays as template + if !contains(processedStr, "{{REDIS_PASSWORD}}") { + t.Error("Expected REDIS_PASSWORD without default to remain as template") + } + + // Now test with env vars set + os.Setenv("REDIS_HOST", "custom-redis") + os.Setenv("REDIS_PASSWORD", "secret123") + os.Setenv("OPTIONAL_TAG", "my-tag") + defer func() { + os.Unsetenv("REDIS_HOST") + os.Unsetenv("REDIS_PASSWORD") + os.Unsetenv("OPTIONAL_TAG") + }() + + processed2, hasTemplates2 := substituteEnvVars(content) + if !hasTemplates2 { + t.Error("Expected templates to be found in example-with-defaults.yaml") + } + + processedStr2 := string(processed2) + + // Verify env vars override defaults + if !contains(processedStr2, "custom-redis") { + t.Error("Expected REDIS_HOST env var 'custom-redis' to override default") + } + + // Verify env var substituted for template without default + if !contains(processedStr2, "secret123") { + t.Error("Expected REDIS_PASSWORD to be substituted with 'secret123'") + } + + // Verify env var overrides empty default + if !contains(processedStr2, "my-tag") { + t.Error("Expected OPTIONAL_TAG env var 'my-tag' to override empty default") + } +} + func contains(s, substr string) bool { return len(s) >= len(substr) && (s == substr || len(substr) == 0 || (len(s) > 0 && (s[0:len(substr)] == substr || contains(s[1:], substr)))) diff --git a/pkg/templateprocessor/testdata/example-with-defaults.yaml b/pkg/templateprocessor/testdata/example-with-defaults.yaml index b1dbda46a..5fa26b30b 100644 --- a/pkg/templateprocessor/testdata/example-with-defaults.yaml +++ b/pkg/templateprocessor/testdata/example-with-defaults.yaml @@ -32,10 +32,14 @@ spec: - name: connectionString value: {{REDIS_CONNECTION:redis://localhost:6379}} - # Empty default - explicitly set to empty if not provided + # Numeric default - database index - name: redisDB value: {{REDIS_DB:0}} + # Empty default - explicitly set to empty if not provided + - name: optionalTag + value: {{OPTIONAL_TAG:}} + # No default - must be set or will remain as template - name: redisPassword value: {{REDIS_PASSWORD}}