From dd0b9b1ddbc6d67339e5e1863934b9633bd9d87e Mon Sep 17 00:00:00 2001 From: "Alex Ellis (OpenFaaS Ltd)" Date: Thu, 13 Nov 2025 12:10:07 +0000 Subject: [PATCH 1/2] Version pinning for templates Templates can now be pinned in three ways: - no pinning - get HEAD from default branch - Git tag / branch name - pass @ then the release i.e. 0.9.0 - Git SHA - requires full depth clone of whole respository followed by a git checkout Tested using the following: * Via build and publish with a templates configuration in stack.yaml * Via build and publish with no templates configuration in stack.yaml - the templates were resolved via the store * Via new - with and without a template being present - the template is pulled from the store if not found locally Signed-off-by: Alex Ellis (OpenFaaS Ltd) --- .ssyncignore | 1 + builder/build.go | 266 ++++++++++++++----------- builder/copy.go | 4 +- commands/build.go | 71 ++++--- commands/fetch_templates.go | 285 ++++++++++++++++++++------- commands/fetch_templates_test.go | 20 +- commands/local_run.go | 5 +- commands/new_function.go | 17 +- commands/plugin_get.go | 2 +- commands/publish.go | 64 +++--- commands/secret_apply.go | 166 ++++++++++++++++ commands/template_pull.go | 15 +- commands/template_pull_stack.go | 51 +++-- commands/template_pull_stack_test.go | 104 +++++++--- commands/template_pull_test.go | 49 ++--- commands/template_store_pull.go | 30 ++- commands/watch.go | 2 +- config/config_file.go | 4 +- testing/.gitignore | 3 + testing/go-090/go.mod | 3 + testing/go-090/handler.go | 22 +++ testing/go-latest/go.mod | 3 + testing/go-latest/handler.go | 22 +++ testing/go-sha-2e6e262/go.mod | 3 + testing/go-sha-2e6e262/handler.go | 22 +++ testing/inproc-fn/go.mod | 3 + testing/inproc-fn/handler.go | 25 +++ testing/no-template-config.yaml | 20 ++ testing/pyfn/handler.py | 5 + testing/pyfn/handler_test.py | 10 + testing/pyfn/requirements.txt | 0 testing/pyfn/tox.ini | 41 ++++ testing/stack.yaml | 17 ++ testing/template-config.yaml | 27 +++ vendor/modules.txt | 70 ------- versioncontrol/core.go | 5 + versioncontrol/git.go | 35 +++- versioncontrol/parse.go | 13 +- versioncontrol/parse_test.go | 31 +-- 39 files changed, 1082 insertions(+), 454 deletions(-) create mode 100644 .ssyncignore create mode 100644 commands/secret_apply.go create mode 100644 testing/.gitignore create mode 100644 testing/go-090/go.mod create mode 100644 testing/go-090/handler.go create mode 100644 testing/go-latest/go.mod create mode 100644 testing/go-latest/handler.go create mode 100644 testing/go-sha-2e6e262/go.mod create mode 100644 testing/go-sha-2e6e262/handler.go create mode 100644 testing/inproc-fn/go.mod create mode 100644 testing/inproc-fn/handler.go create mode 100644 testing/no-template-config.yaml create mode 100644 testing/pyfn/handler.py create mode 100644 testing/pyfn/handler_test.py create mode 100644 testing/pyfn/requirements.txt create mode 100644 testing/pyfn/tox.ini create mode 100644 testing/stack.yaml create mode 100644 testing/template-config.yaml diff --git a/.ssyncignore b/.ssyncignore new file mode 100644 index 000000000..c3fba8cab --- /dev/null +++ b/.ssyncignore @@ -0,0 +1 @@ +/.git diff --git a/builder/build.go b/builder/build.go index af694769e..6c6e9b5c3 100644 --- a/builder/build.go +++ b/builder/build.go @@ -30,158 +30,186 @@ import ( // Can also be passed as a build arg hence needs to be accessed from commands const AdditionalPackageBuildArg = "ADDITIONAL_PACKAGE" -// BuildImage construct Docker image from function parameters -// TODO: refactor signature to a struct to simplify the length of the method header -func BuildImage(image string, handler string, functionName string, language string, nocache bool, squash bool, shrinkwrap bool, buildArgMap map[string]string, buildOptions []string, tagFormat schema.BuildFormat, buildLabelMap map[string]string, quietBuild bool, copyExtraPaths []string, remoteBuilder, payloadSecretPath string, forcePull bool) error { +func getTemplate(lang string) (string, *stack.LanguageTemplate, error) { - if stack.IsValidTemplate(language) { - pathToTemplateYAML := fmt.Sprintf("./template/%s/template.yml", language) - if _, err := os.Stat(pathToTemplateYAML); err != nil && os.IsNotExist(err) { - return err - } + cwd, err := os.Getwd() + if err != nil { + return "", nil, fmt.Errorf("can't get current working directory: %w", err) + } - langTemplate, err := stack.ParseYAMLForLanguageTemplate(pathToTemplateYAML) - if err != nil { - return fmt.Errorf("error reading language template: %s", err.Error()) - } + templateDir := filepath.Join(cwd, "template") + if _, err := os.Stat(templateDir); err != nil { + return "", nil, fmt.Errorf("template directory not found") + } - mountSSH := false - if langTemplate.MountSSH { - mountSSH = true - } + files, err := os.ReadDir(templateDir) + if err != nil { + return "", nil, fmt.Errorf("can't read template directory: %w", err) + } - if err := ensureHandlerPath(handler); err != nil { - return fmt.Errorf("building %s, %s is an invalid path", functionName, handler) + found := "" + for _, file := range files { + if file.IsDir() { + if file.Name() == lang { + found = filepath.Join(templateDir, file.Name()) + break + } } + } - opts := []builder.BuildContextOption{} - if len(langTemplate.HandlerFolder) > 0 { - opts = append(opts, builder.WithHandlerOverlay(langTemplate.HandlerFolder)) - } + if len(found) == 0 { + return "", nil, fmt.Errorf("template %s not found", lang) + } + parsed, err := stack.ParseYAMLForLanguageTemplate(filepath.Join(found, "template.yml")) + if err != nil { + return "", nil, fmt.Errorf("can't parse template: %w", err) + } + return found, parsed, nil +} - buildContext, err := builder.CreateBuildContext(functionName, handler, language, copyExtraPaths, opts...) - if err != nil { - return err - } +// BuildImage construct Docker image from function parameters +// TODO: refactor signature to a struct to simplify the length of the method header +func BuildImage(image string, handler string, functionName string, language string, nocache bool, squash bool, shrinkwrap bool, buildArgMap map[string]string, buildOptions []string, tagFormat schema.BuildFormat, buildLabelMap map[string]string, quietBuild bool, copyExtraPaths []string, remoteBuilder, payloadSecretPath string, forcePull bool) error { - if shrinkwrap { - fmt.Printf("%s shrink-wrapped to %s\n", functionName, buildContext) - return nil - } + _, langTemplate, err := getTemplate(language) + if err != nil { + return fmt.Errorf("language template: %s not supported, build a custom Dockerfile, error: %w", language, err) + } - branch, version, err := GetImageTagValues(tagFormat, handler) - if err != nil { - return err - } + mountSSH := false + if langTemplate.MountSSH { + mountSSH = true + } - imageName := schema.BuildImageName(tagFormat, image, version, branch) + if err := ensureHandlerPath(handler); err != nil { + return fmt.Errorf("building %s, %s is an invalid path", functionName, handler) + } - buildOptPackages, err := getBuildOptionPackages(buildOptions, language, langTemplate.BuildOptions) - if err != nil { - return err + opts := []builder.BuildContextOption{} + if len(langTemplate.HandlerFolder) > 0 { + opts = append(opts, builder.WithHandlerOverlay(langTemplate.HandlerFolder)) + } - } - buildArgMap = appendAdditionalPackages(buildArgMap, buildOptPackages) + buildContext, err := builder.CreateBuildContext(functionName, handler, language, copyExtraPaths, opts...) + if err != nil { + return err + } - fmt.Printf("Building: %s with %s template. Please wait..\n", imageName, language) + if shrinkwrap { + fmt.Printf("%s shrink-wrapped to %s\n", functionName, buildContext) + return nil + } - if remoteBuilder != "" { - tempDir, err := os.MkdirTemp(os.TempDir(), "openfaas-build-*") - if err != nil { - return fmt.Errorf("failed to create temporary directory: %w", err) - } - defer os.RemoveAll(tempDir) + branch, version, err := GetImageTagValues(tagFormat, handler) + if err != nil { + return err + } - tarPath := path.Join(tempDir, "req.tar") + imageName := schema.BuildImageName(tagFormat, image, version, branch) - buildConfig := builder.BuildConfig{ - Image: imageName, - BuildArgs: buildArgMap, - } + buildOptPackages, err := getBuildOptionPackages(buildOptions, language, langTemplate.BuildOptions) + if err != nil { + return err - // Prepare a tar archive that contains the build config and build context. - if err := builder.MakeTar(tarPath, path.Join("build", functionName), &buildConfig); err != nil { - return fmt.Errorf("failed to create tar file for %s, error: %w", functionName, err) - } + } + buildArgMap = appendAdditionalPackages(buildArgMap, buildOptPackages) - // Get the HMAC secret used for payload authentication with the builder API. - payloadSecret, err := os.ReadFile(payloadSecretPath) - if err != nil { - return fmt.Errorf("failed to read payload secret: %w", err) - } - payloadSecret = bytes.TrimSpace(payloadSecret) + fmt.Printf("Building: %s with %s template. Please wait..\n", imageName, language) - // Initialize a new builder client. - u, _ := url.Parse(remoteBuilder) - builderURL := &url.URL{ - Scheme: u.Scheme, - Host: u.Host, - } - b := builder.NewFunctionBuilder(builderURL, http.DefaultClient, builder.WithHmacAuth(string(payloadSecret))) + if remoteBuilder != "" { + tempDir, err := os.MkdirTemp(os.TempDir(), "openfaas-build-*") + if err != nil { + return fmt.Errorf("failed to create temporary directory: %w", err) + } + defer os.RemoveAll(tempDir) - stream, err := b.BuildWithStream(tarPath) - if err != nil { - return fmt.Errorf("failed to invoke builder: %w", err) - } - defer stream.Close() + tarPath := path.Join(tempDir, "req.tar") - for result := range stream.Results() { - if !quietBuild { - for _, logMsg := range result.Log { - fmt.Printf("%s\n", logMsg) - } - } + buildConfig := builder.BuildConfig{ + Image: imageName, + BuildArgs: buildArgMap, + } + + // Prepare a tar archive that contains the build config and build context. + if err := builder.MakeTar(tarPath, path.Join("build", functionName), &buildConfig); err != nil { + return fmt.Errorf("failed to create tar file for %s, error: %w", functionName, err) + } + + // Get the HMAC secret used for payload authentication with the builder API. + payloadSecret, err := os.ReadFile(payloadSecretPath) + if err != nil { + return fmt.Errorf("failed to read payload secret: %w", err) + } + payloadSecret = bytes.TrimSpace(payloadSecret) + + // Initialize a new builder client. + u, _ := url.Parse(remoteBuilder) + builderURL := &url.URL{ + Scheme: u.Scheme, + Host: u.Host, + } + b := builder.NewFunctionBuilder(builderURL, http.DefaultClient, builder.WithHmacAuth(string(payloadSecret))) - switch result.Status { - case builder.BuildSuccess: - log.Printf("%s success building and pushing image: %s", functionName, result.Image) - case builder.BuildFailed: - return fmt.Errorf("%s failure while building or pushing image %s: %s", functionName, imageName, result.Error) + stream, err := b.BuildWithStream(tarPath) + if err != nil { + return fmt.Errorf("failed to invoke builder: %w", err) + } + defer stream.Close() + + for result := range stream.Results() { + if !quietBuild { + for _, logMsg := range result.Log { + fmt.Printf("%s\n", logMsg) } } - } else { - dockerBuildVal := dockerBuild{ - Image: imageName, - NoCache: nocache, - Squash: squash, - HTTPProxy: os.Getenv("http_proxy"), - HTTPSProxy: os.Getenv("https_proxy"), - BuildArgMap: buildArgMap, - BuildLabelMap: buildLabelMap, - ForcePull: forcePull, + switch result.Status { + case builder.BuildSuccess: + log.Printf("%s success building and pushing image: %s", functionName, result.Image) + case builder.BuildFailed: + return fmt.Errorf("%s failure while building or pushing image %s: %s", functionName, imageName, result.Error) } + } - command, args := getDockerBuildCommand(dockerBuildVal) + } else { + dockerBuildVal := dockerBuild{ + Image: imageName, + NoCache: nocache, + Squash: squash, + HTTPProxy: os.Getenv("http_proxy"), + HTTPSProxy: os.Getenv("https_proxy"), + BuildArgMap: buildArgMap, + BuildLabelMap: buildLabelMap, + ForcePull: forcePull, + } - envs := os.Environ() - if mountSSH { - envs = append(envs, "DOCKER_BUILDKIT=1") - } - log.Printf("Build flags: %+v\n", args) - - task := v2execute.ExecTask{ - Cwd: buildContext, - Command: command, - Args: args, - StreamStdio: !quietBuild, - Env: envs, - } + command, args := getDockerBuildCommand(dockerBuildVal) - res, err := task.Execute(context.TODO()) + envs := os.Environ() + if mountSSH { + envs = append(envs, "DOCKER_BUILDKIT=1") + } + log.Printf("Build flags: %+v\n", args) + + task := v2execute.ExecTask{ + Cwd: buildContext, + Command: command, + Args: args, + StreamStdio: !quietBuild, + Env: envs, + } - if err != nil { - return err - } + res, err := task.Execute(context.TODO()) - if res.ExitCode != 0 { - return fmt.Errorf("[%s] received non-zero exit code from build, error: %s", functionName, res.Stderr) - } + if err != nil { + return err + } - fmt.Printf("Image: %s built.\n", imageName) + if res.ExitCode != 0 { + return fmt.Errorf("[%s] received non-zero exit code from build, error: %s", functionName, res.Stderr) } - } else { - return fmt.Errorf("language template: %s not supported, build a custom Dockerfile", language) + + fmt.Printf("Image: %s built.\n", imageName) } return nil diff --git a/builder/copy.go b/builder/copy.go index b5c978245..3dc4cfdfc 100644 --- a/builder/copy.go +++ b/builder/copy.go @@ -33,8 +33,8 @@ func copyDir(src, dest string) error { return fmt.Errorf("error reading dest stats: %s", err.Error()) } - if err := os.MkdirAll(dest, info.Mode()); err != nil { - return fmt.Errorf("error creating path: %s - %s", dest, err.Error()) + if err := os.MkdirAll(dest, info.Mode()); err != nil && !os.IsExist(err) { + return fmt.Errorf("error creating directory: %s - %w", dest, err) } infos, err := ioutil.ReadDir(src) diff --git a/commands/build.go b/commands/build.go index 85756da8d..022a6dc9f 100644 --- a/commands/build.go +++ b/commands/build.go @@ -5,7 +5,9 @@ package commands import ( "fmt" + "log" "os" + "path/filepath" "strings" "sync" "time" @@ -61,6 +63,9 @@ func init() { buildCmd.Flags().BoolVar(&disableStackPull, "disable-stack-pull", false, "Disables the template configuration in the stack.yaml") buildCmd.Flags().BoolVar(&forcePull, "pull", false, "Force a re-pull of base images in template during build, useful for publishing images") + buildCmd.Flags().BoolVar(&pullDebug, "debug", false, "Enable debug output when pulling templates") + buildCmd.Flags().BoolVar(&overwrite, "overwrite", true, "Overwrite existing templates from the template repository") + // Set bash-completion. _ = buildCmd.Flags().SetAnnotation("handler", cobra.BashCompSubdirsInDir, []string{}) @@ -81,23 +86,23 @@ var buildCmd = &cobra.Command{ [--build-arg KEY=VALUE] [--build-option VALUE] [--copy-extra PATH] - [--tag ] + [--tag ] [--forcePull]`, Short: "Builds OpenFaaS function containers", Long: `Builds OpenFaaS function containers either via the supplied YAML config using the "--yaml" flag (which may contain multiple function definitions), or directly via flags.`, Example: ` faas-cli build -f https://domain/path/myfunctions.yml - faas-cli build -f stack.yaml --no-cache --build-arg NPM_VERSION=0.2.2 - faas-cli build -f stack.yaml --build-option dev - faas-cli build -f stack.yaml --tag sha - faas-cli build -f stack.yaml --tag branch - faas-cli build -f stack.yaml --tag describe - faas-cli build -f stack.yaml --filter "*gif*" - faas-cli build -f stack.yaml --regex "fn[0-9]_.*" + faas-cli build -f functions.yaml + faas-cli build --no-cache --build-arg NPM_VERSION=0.2.2 + faas-cli build --build-option dev + faas-cli build --tag sha + faas-cli build --parallel 4 + faas-cli build --filter "*gif*" + faas-cli build --regex "fn[0-9]_.*" faas-cli build --image=my_image --lang=python --handler=/path/to/fn/ \ --name=my_fn --squash - faas-cli build -f stack.yaml --build-label org.label-schema.label-name="value"`, + faas-cli build --build-label org.label-schema.label-name="value"`, PreRunE: preRunBuild, RunE: runBuild, } @@ -154,8 +159,6 @@ func parseBuildArgs(args []string) (map[string]string, error) { func runBuild(cmd *cobra.Command, args []string) error { - templateName := "" // templateName may not be known at this point - var services stack.Services if len(yamlFile) > 0 { parsedServices, err := stack.ParseYAMLFile(yamlFile, regex, filter, envsubst) @@ -168,24 +171,42 @@ func runBuild(cmd *cobra.Command, args []string) error { } } + cwd, _ := os.Getwd() + templatesPath := filepath.Join(cwd, TemplateDirectory) + if len(services.StackConfiguration.TemplateConfigs) > 0 && !disableStackPull { - newTemplateInfos, err := filterExistingTemplates(services.StackConfiguration.TemplateConfigs, "./template") + missingTemplates, err := getMissingTemplates(services.Functions, templatesPath) if err != nil { - return fmt.Errorf("already pulled templates directory has issue: %s", err.Error()) + return fmt.Errorf("error accessing existing templates folder: %s", err.Error()) } - if err = pullStackTemplates(newTemplateInfos, cmd); err != nil { - return fmt.Errorf("could not pull templates from function yaml file: %s", err.Error()) + if err := pullStackTemplates(missingTemplates, services.StackConfiguration.TemplateConfigs, cmd); err != nil { + return fmt.Errorf("error pulling templates: %s", err.Error()) } + if len(missingTemplates) > 0 { + log.Printf("Pulled templates: %v", missingTemplates) + } + } else { - templateAddress := getTemplateURL("", os.Getenv(templateURLEnvironment), DefaultTemplateRepository) - if err := pullTemplates(templateAddress, templateName); err != nil { - return fmt.Errorf("could not pull templates: %v", err) + + // When the configuration.templates section is empty, it's only possible to pull from the store + // this store can be overridden by a flag or environment variable + + missingTemplates, err := getMissingTemplates(services.Functions, templatesPath) + if err != nil { + return fmt.Errorf("error accessing existing templates folder: %s", err.Error()) + } + + for _, missingTemplate := range missingTemplates { + + if err := runTemplateStorePull(cmd, []string{missingTemplate}); err != nil { + return fmt.Errorf("error pulling template: %s", err.Error()) + } } + } if len(services.Functions) == 0 { - if len(image) == 0 { return fmt.Errorf("please provide a valid --image name for your Docker image") } @@ -306,18 +327,22 @@ func build(services *stack.Services, queueDepth int, shrinkwrap, quietBuild bool // pullTemplates pulls templates from specified git remote. templateURL may be a pinned repository. func pullTemplates(templateURL, templateName string) error { + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("can't get current working directory: %s", err) + } - if _, err := os.Stat("./template"); err != nil { - if os.IsNotExist(err) { + templatePath := filepath.Join(cwd, TemplateDirectory) + if _, err := os.Stat(templatePath); err != nil { + if os.IsNotExist(err) { fmt.Printf("No templates found in current directory.\n") templateURL, refName := versioncontrol.ParsePinnedRemote(templateURL) - if err := fetchTemplates(templateURL, refName, templateName, false); err != nil { + if err := fetchTemplates(templateURL, refName, templateName, overwrite); err != nil { return err } } else { - // Perhaps there was a permissions issue or something else. return err } diff --git a/commands/fetch_templates.go b/commands/fetch_templates.go index c159f2695..81b01abd6 100644 --- a/commands/fetch_templates.go +++ b/commands/fetch_templates.go @@ -4,11 +4,18 @@ package commands import ( + "context" + "encoding/json" "fmt" "log" "os" "path/filepath" + "regexp" + "slices" + "strings" + "time" + execute "github.com/alexellis/go-execute/v2" "github.com/openfaas/faas-cli/builder" "github.com/openfaas/faas-cli/versioncontrol" ) @@ -16,47 +23,119 @@ import ( // DefaultTemplateRepository contains the Git repo for the official templates const DefaultTemplateRepository = "https://github.com/openfaas/templates.git" -const templateDirectory = "./template/" +const TemplateDirectory = "./template/" + +const ShaPrefix = "sha-" // fetchTemplates fetch code templates using git clone. -func fetchTemplates(templateURL, refName, templateName string, overwrite bool) error { +func fetchTemplates(templateURL, refName, templateName string, overwriteTemplates bool) error { if len(templateURL) == 0 { return fmt.Errorf("pass valid templateURL") } - dir, err := os.MkdirTemp("", "openfaas-templates-*") + refMsg := "" + if len(refName) > 0 { + refMsg = " [" + refName + "]" + } + + log.Printf("Fetching templates from %s%s", templateURL, refMsg) + + extractedPath, err := os.MkdirTemp("", "openfaas-templates-*") if err != nil { - return err + return fmt.Errorf("unable to create temporary directory: %s", err) } if !pullDebug { - defer os.RemoveAll(dir) + defer os.RemoveAll(extractedPath) } - pullDebugPrint(fmt.Sprintf("Temp files in %s", dir)) + pullDebugPrint(fmt.Sprintf("Temp files in %s", extractedPath)) - args := map[string]string{"dir": dir, "repo": templateURL} + args := map[string]string{"dir": extractedPath, "repo": templateURL} cmd := versioncontrol.GitCloneDefault - if refName != "" { - args["refname"] = refName - cmd = versioncontrol.GitClone + if len(refName) > 0 { + if strings.HasPrefix(refName, ShaPrefix) { + cmd = versioncontrol.GitCloneFullDepth + } else { + args["refname"] = refName + + cmd = versioncontrol.GitCloneBranch + args["refname"] = refName + } } if err := cmd.Invoke(".", args); err != nil { + return fmt.Errorf("error invoking git clone: %w", err) + } + + if len(refName) > 0 && strings.HasPrefix(refName, ShaPrefix) { + + targetCommit := strings.TrimPrefix(refName, ShaPrefix) + + if !regexp.MustCompile(`^[a-fA-F0-9]{7,40}$`).MatchString(targetCommit) { + return fmt.Errorf("invalid SHA format: %s - must be 7-40 hex characters", targetCommit) + } + + t := execute.ExecTask{ + Command: "git", + Args: []string{"-C", extractedPath, "checkout", targetCommit}, + } + res, err := t.Execute(context.Background()) + if err != nil { + return fmt.Errorf("error checking out ref %s: %w", targetCommit, err) + } + if res.ExitCode != 0 { + out := res.Stdout + " " + res.Stderr + return fmt.Errorf("error checking out ref %s: %s", targetCommit, out) + } + } + + if os.Getenv("FAAS_DEBUG") == "1" { + task := execute.ExecTask{ + Command: "git", + Args: []string{"-C", extractedPath, "log", "-1", "--oneline"}, + } + + res, err := task.Execute(context.Background()) + if err != nil { + return fmt.Errorf("error executing git log: %w", err) + } + if res.ExitCode != 0 { + e := fmt.Errorf("exit code: %d, stderr: %s, stdout: %s", res.ExitCode, res.Stderr, res.Stdout) + return fmt.Errorf("error from: git log: %w", e) + } + + log.Printf("[git] log: %s", strings.TrimSpace(res.Stdout)) + } + + // Get the long SHA digest from the clone repository. + sha, err := versioncontrol.GetGitSHAFor(extractedPath, false) + if err != nil { return err } - preExistingLanguages, fetchedLanguages, err := moveTemplates(dir, templateName, overwrite) + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("can't get current working directory: %s", err) + } + localTemplatesDir := filepath.Join(cwd, TemplateDirectory) + if _, err := os.Stat(localTemplatesDir); err != nil && os.IsNotExist(err) { + if err := os.MkdirAll(localTemplatesDir, 0755); err != nil && !os.IsExist(err) { + return fmt.Errorf("error creating template directory: %s - %w", localTemplatesDir, err) + } + } + + protectedLanguages, fetchedLanguages, err := moveTemplates(localTemplatesDir, extractedPath, templateName, overwriteTemplates, templateURL, refName, sha) if err != nil { return err } - if len(preExistingLanguages) > 0 { - log.Printf("Cannot overwrite the following %d template(s): %v\n", len(preExistingLanguages), preExistingLanguages) + if len(protectedLanguages) > 0 { + return fmt.Errorf("unable to overwrite the following: %v", protectedLanguages) } - fmt.Printf("Wrote %d template(s) : %v from %s\n", len(fetchedLanguages), fetchedLanguages, templateURL) + fmt.Printf("Wrote %d template(s) : %v\n", len(fetchedLanguages), fetchedLanguages) return err } @@ -64,88 +143,135 @@ func fetchTemplates(templateURL, refName, templateName string, overwrite bool) e // canWriteLanguage tells whether the language can be expanded from the zip or not. // availableLanguages map keeps track of which languages we know to be okay to copy. // overwrite flag will allow to force copy the language template -func canWriteLanguage(availableLanguages map[string]bool, language string, overwrite bool) bool { - canWrite := false - if availableLanguages != nil && len(language) > 0 { - if _, found := availableLanguages[language]; found { - return availableLanguages[language] - } - canWrite = templateFolderExists(language, overwrite) - availableLanguages[language] = canWrite +func canWriteLanguage(existingLanguages []string, language string, overwriteTemplate bool) bool { + if overwriteTemplate { + return true } - return canWrite + return !slices.Contains(existingLanguages, language) } -// Takes a language input (e.g. "node"), tells whether or not it is OK to download -func templateFolderExists(language string, overwrite bool) bool { - dir := templateDirectory + language - if _, err := os.Stat(dir); err == nil && !overwrite { - // The directory template/language/ exists - return false - } - return true -} +// moveTemplates moves the templates from the repository to the template directory +// It returns the existing languages and the fetched languages +// It also returns an error if the templates cannot be read +func moveTemplates(localTemplatesDir, extractedPath, templateName string, overwriteTemplate bool, repository string, refName string, sha string) ([]string, []string, error) { -func moveTemplates(repoPath, templateName string, overwrite bool) ([]string, []string, error) { var ( - existingLanguages []string - fetchedLanguages []string - err error + existingLanguages []string + fetchedLanguages []string + protectedLanguages []string + err error ) - availableLanguages := make(map[string]bool) + templateEntries, err := os.ReadDir(localTemplatesDir) + if err != nil { + return nil, nil, fmt.Errorf("unable to read directory: %s", localTemplatesDir) + } + + // OK if nothing exists yet + for _, entry := range templateEntries { + if !entry.IsDir() { + continue + } - templateDir := filepath.Join(repoPath, templateDirectory) - templates, err := os.ReadDir(templateDir) + templateFile := filepath.Join(localTemplatesDir, entry.Name(), "template.yml") + if _, err := os.Stat(templateFile); err != nil && !os.IsNotExist(err) { + return nil, nil, fmt.Errorf("can't find template.yml in: %s", templateFile) + } + + existingLanguages = append(existingLanguages, entry.Name()) + } + + extractedTemplates, err := os.ReadDir(filepath.Join(extractedPath, TemplateDirectory)) if err != nil { - return nil, nil, fmt.Errorf("can't find templates in: %s", repoPath) + return nil, nil, fmt.Errorf("can't find templates in: %s", filepath.Join(extractedPath, TemplateDirectory)) } - for _, file := range templates { - if !file.IsDir() { + for _, entry := range extractedTemplates { + if !entry.IsDir() { continue } - language := file.Name() + language := entry.Name() + refSuffix := "" + if refName != "" { + refSuffix = "@" + refName + } - if len(templateName) == 0 { + if canWriteLanguage(existingLanguages, language, overwriteTemplate) { + // Do cp here + languageSrc := filepath.Join(extractedPath, TemplateDirectory, language) + languageDest := filepath.Join(localTemplatesDir, language) + langName := language + if refName != "" { + languageDest += "@" + refName + langName = language + "@" + refName + } + fetchedLanguages = append(fetchedLanguages, langName) - canWrite := canWriteLanguage(availableLanguages, language, overwrite) - if canWrite { - fetchedLanguages = append(fetchedLanguages, language) - // Do cp here - languageSrc := filepath.Join(templateDir, language) - languageDest := filepath.Join(templateDirectory, language) - builder.CopyFiles(languageSrc, languageDest) - } else { - existingLanguages = append(existingLanguages, language) - continue + if err := builder.CopyFiles(languageSrc, languageDest); err != nil { + return nil, nil, err } - } else if language == templateName { - - if canWriteLanguage(availableLanguages, language, overwrite) { - fetchedLanguages = append(fetchedLanguages, language) - // Do cp here - languageSrc := filepath.Join(templateDir, language) - languageDest := filepath.Join(templateDirectory, language) - builder.CopyFiles(languageSrc, languageDest) + + if err := writeTemplateMeta(languageDest, repository, refName, sha); err != nil { + return nil, nil, err } + } else { + protectedLanguages = append(protectedLanguages, language+refSuffix) + continue } } - return existingLanguages, fetchedLanguages, nil + return protectedLanguages, fetchedLanguages, nil +} + +func writeTemplateMeta(languageDest, repository, refName, sha string) error { + templateMeta := TemplateMeta{ + Repository: repository, + WrittenAt: time.Now(), + RefName: refName, + Sha: sha, + } + + metaBytes, err := json.Marshal(templateMeta) + if err != nil { + return fmt.Errorf("error marshalling template meta: %s", err) + } + + metaPath := filepath.Join(languageDest, "meta.json") + if err := os.WriteFile(metaPath, metaBytes, 0644); err != nil { + return fmt.Errorf("error writing template meta: %s", err) + } + + return nil } -func pullTemplate(repository, templateName string) error { - if _, err := os.Stat(repository); err != nil { - if !versioncontrol.IsGitRemote(repository) && !versioncontrol.IsPinnedGitRemote(repository) { +func pullTemplate(repository, templateName string, overwriteTemplates bool) error { + + baseRepository := repository + + // Sometimes a templates git repo can be a local path + if _, err := os.Stat(repository); err != nil && os.IsNotExist(err) { + base, _, found := strings.Cut(repository, "#") + if found { + baseRepository = base + } else { + + _, ref, found := strings.Cut(templateName, "#") + if found { + repository = baseRepository + "#" + ref + } + } + } + + if !isValidFilesystemPath(repository) { + if !versioncontrol.IsGitRemote(baseRepository) && !versioncontrol.IsPinnedGitRemote(baseRepository) { return fmt.Errorf("the repository URL must be a valid git repo uri") } } repository, refName := versioncontrol.ParsePinnedRemote(repository) - - if refName != "" { + isShaRefName := strings.HasPrefix(refName, ShaPrefix) + if refName != "" && !isShaRefName { err := versioncontrol.GitCheckRefName.Invoke("", map[string]string{"refname": refName}) if err != nil { fmt.Printf("Invalid tag or branch name `%s`\n", refName) @@ -155,14 +281,23 @@ func pullTemplate(repository, templateName string) error { } } - refStr := "" - if len(refName) > 0 { - refStr = fmt.Sprintf(" @ %s", refName) - } - fmt.Printf("Fetch templates from repository: %s%s\n", repository, refStr) - if err := fetchTemplates(repository, refName, templateName, overwrite); err != nil { - return fmt.Errorf("error while fetching templates: %s", err) + if err := fetchTemplates(repository, refName, templateName, overwriteTemplates); err != nil { + return fmt.Errorf("error while fetching templates: %w", err) } return nil } + +type TemplateMeta struct { + Repository string `json:"repository"` + RefName string `json:"ref_name,omitempty"` + Sha string `json:"sha,omitempty"` + WrittenAt time.Time `json:"written_at"` +} + +func isValidFilesystemPath(path string) bool { + if _, err := os.Stat(path); err == nil { + return true + } + return false +} diff --git a/commands/fetch_templates_test.go b/commands/fetch_templates_test.go index 911e6bcb9..ebb1dc3c2 100644 --- a/commands/fetch_templates_test.go +++ b/commands/fetch_templates_test.go @@ -29,10 +29,11 @@ func Test_PullTemplates(t *testing.T) { } }) + overwrite := true t.Run("fetchTemplates with master ref", func(t *testing.T) { defer tearDownFetchTemplates(t) - if err := fetchTemplates(localTemplateRepository, "master", templateName, false); err != nil { + if err := fetchTemplates(localTemplateRepository, "master", templateName, overwrite); err != nil { t.Fatal(err) } @@ -42,7 +43,7 @@ func Test_PullTemplates(t *testing.T) { defer tearDownFetchTemplates(t) templateName := "" - if err := fetchTemplates(localTemplateRepository, "", templateName, false); err != nil { + if err := fetchTemplates(localTemplateRepository, "", templateName, overwrite); err != nil { t.Error(err) } @@ -105,14 +106,21 @@ func setupLocalTemplateRepo(t *testing.T) string { // tearDownFetchTemplates cleans all files and directories created by the test func tearDownFetchTemplates(t *testing.T) { + cwd, err := os.Getwd() + if err != nil { + t.Logf("Could not get current working directory: %s", err) + return + } + + fullPath := filepath.Join(cwd, TemplateDirectory) // Remove existing templates folder, if it exist - if _, err := os.Stat("./template/"); err == nil { - t.Log("Found a ./template/ directory, removing it.") + if _, err := os.Stat(fullPath); err == nil { + t.Log("Found a template directory, removing it.") - if err := os.RemoveAll("./template/"); err != nil { + if err := os.RemoveAll(fullPath); err != nil { t.Log(err) } } else { - t.Logf("Directory template was not created: %s", err) + t.Logf("Template directory %s was not created: %s", fullPath, err) } } diff --git a/commands/local_run.go b/commands/local_run.go index 1afc97796..8d088d0c5 100644 --- a/commands/local_run.go +++ b/commands/local_run.go @@ -362,9 +362,8 @@ func buildDockerRun(ctx context.Context, name string, fnc stack.Function, opts r return nil, fmt.Errorf("can't determine secrets folder: %w", err) } - err = os.MkdirAll(secretsPath, 0700) - if err != nil { - return nil, fmt.Errorf("can't create local secrets folder %q: %w", secretsPath, err) + if err = os.MkdirAll(secretsPath, 0700); err != nil && !os.IsExist(err) { + return nil, fmt.Errorf("error creating local secrets folder %q: %w", secretsPath, err) } if !opts.print { diff --git a/commands/new_function.go b/commands/new_function.go index 3406f1a52..f8aff5de1 100644 --- a/commands/new_function.go +++ b/commands/new_function.go @@ -108,7 +108,7 @@ func runNewFunction(cmd *cobra.Command, args []string) error { if list { var availableTemplates []string - templateFolders, err := os.ReadDir(templateDirectory) + templateFolders, err := os.ReadDir(TemplateDirectory) if err != nil { return fmt.Errorf(`no language templates were found. @@ -140,9 +140,11 @@ Download templates: return fmt.Errorf("error while getting templates info: %s", err) } + _, ref, _ := strings.Cut(language, "@") + var templateInfo *TemplateInfo for _, info := range templatesInfo { - if info.TemplateName == language { + if info.TemplateName == language || (info.TemplateName+"@"+ref == language) { templateInfo = &info break } @@ -153,8 +155,12 @@ Download templates: } templateName := templateInfo.TemplateName + if ref != "" { + // Reformat as a Git-native reference + templateName += "#" + ref + } - if err := pullTemplate(templateInfo.Repository, templateName); err != nil { + if err := pullTemplate(templateInfo.Repository, templateName, overwrite); err != nil { return fmt.Errorf("error while pulling template: %s", err) } @@ -225,7 +231,10 @@ Download templates: fromTemplateHandler := filepath.Join("template", language, templateHandlerFolder) // Create function directory from template. - builder.CopyFiles(fromTemplateHandler, handlerDir) + if err := builder.CopyFiles(fromTemplateHandler, handlerDir); err != nil { + return fmt.Errorf("error copying template handler: %s", err) + } + printLogo() fmt.Printf("\nFunction created in folder: %s\n", handlerDir) diff --git a/commands/plugin_get.go b/commands/plugin_get.go index d7a0566b5..ac6103f28 100644 --- a/commands/plugin_get.go +++ b/commands/plugin_get.go @@ -100,7 +100,7 @@ func runPluginGetCmd(cmd *cobra.Command, args []string) error { } if _, err := os.Stat(pluginDir); err != nil && os.IsNotExist(err) { - if err := os.MkdirAll(pluginDir, 0755); err != nil && os.ErrExist != err { + if err := os.MkdirAll(pluginDir, 0755); err != nil && !os.IsExist(err) { return fmt.Errorf("failed to create plugin directory %s: %w", pluginDir, err) } } diff --git a/commands/publish.go b/commands/publish.go index cd001ef95..3c8b37888 100644 --- a/commands/publish.go +++ b/commands/publish.go @@ -5,7 +5,9 @@ package commands import ( "fmt" + "log" "os" + "path/filepath" "sync" "time" @@ -54,6 +56,9 @@ func init() { publishCmd.Flags().StringVar(&payloadSecretPath, "payload-secret", "", "Path to payload secret file") publishCmd.Flags().BoolVar(&forcePull, "pull", false, "Force a re-pull of base images in template during build, useful for publishing images") + publishCmd.Flags().BoolVar(&pullDebug, "debug", false, "Enable debug output when pulling templates") + publishCmd.Flags().BoolVar(&overwrite, "overwrite", true, "Overwrite existing templates from the template repository") + // Set bash-completion. _ = publishCmd.Flags().SetAnnotation("handler", cobra.BashCompSubdirsInDir, []string{}) @@ -92,7 +97,7 @@ correctly configured TARGETPLATFORM and BUILDPLATFORM arguments. See also: faas-cli build`, Example: ` faas-cli publish --platforms linux/amd64,linux/arm64 faas-cli publish --platforms linux/arm64 --filter webhook-arm - faas-cli publish -f go.yml --no-cache --build-arg NPM_VERSION=0.2.2 + faas-cli publish -f custom.yml --no-cache --build-arg NPM_VERSION=0.2.2 faas-cli publish --build-option dev faas-cli publish --tag sha faas-cli publish --reset-qemu @@ -126,8 +131,8 @@ func preRunPublish(cmd *cobra.Command, args []string) error { } func runPublish(cmd *cobra.Command, args []string) error { - var services stack.Services + if len(yamlFile) > 0 { parsedServices, err := stack.ParseYAMLFile(yamlFile, regex, filter, envsubst) if err != nil { @@ -139,35 +144,37 @@ func runPublish(cmd *cobra.Command, args []string) error { } } - needTemplates := false - for _, function := range services.Functions { - if len(function.Language) > 0 { - needTemplates = true - break - } - } + cwd, _ := os.Getwd() + templatesPath := filepath.Join(cwd, TemplateDirectory) - templatesFound := false - if stat, err := os.Stat("./template"); err == nil && stat.IsDir() { - templatesFound = true - } + if len(services.StackConfiguration.TemplateConfigs) > 0 && !disableStackPull { + missingTemplates, err := getMissingTemplates(services.Functions, templatesPath) + if err != nil { + return fmt.Errorf("error accessing existing templates folder: %s", err.Error()) + } - // if no templates are configured, but they exist in the configuration section, - // attempt to pull them first - if !templatesFound && needTemplates { - if len(services.StackConfiguration.TemplateConfigs) > 0 { - if err := pullStackTemplates(services.StackConfiguration.TemplateConfigs, cmd); err != nil { - return err - } + if err := pullStackTemplates(missingTemplates, services.StackConfiguration.TemplateConfigs, cmd); err != nil { + return fmt.Errorf("error pulling templates: %s", err.Error()) + } + if len(missingTemplates) > 0 { + log.Printf("Pulled templates: %v", missingTemplates) + } + } else { + // When the configuration.templates section is empty, it's only possible to pull from the store + // this store can be overridden by a flag or environment variable + missingTemplates, err := getMissingTemplates(services.Functions, templatesPath) + if err != nil { + return fmt.Errorf("error accessing existing templates folder: %s", err.Error()) } - } - if needTemplates { - if _, err := os.Stat("./template"); err != nil && os.IsNotExist(err) { + for _, missingTemplate := range missingTemplates { - return fmt.Errorf(`the "template" directory is missing but required by at least one function`) + if err := runTemplateStorePull(cmd, []string{missingTemplate}); err != nil { + return fmt.Errorf("error pulling template: %s", err.Error()) + } } + } if resetQemu { @@ -223,14 +230,13 @@ func runPublish(cmd *cobra.Command, args []string) error { } if len(services.StackConfiguration.TemplateConfigs) != 0 && !disableStackPull { - newTemplateInfos, err := filterExistingTemplates(services.StackConfiguration.TemplateConfigs, "./template") + newTemplateInfos, err := getMissingTemplates(services.Functions, "./template") if err != nil { - return fmt.Errorf("already pulled templates directory has issue: %s", err.Error()) + return fmt.Errorf("already pulled templates directory has issue: %w", err) } - err = pullStackTemplates(newTemplateInfos, cmd) - if err != nil { - return fmt.Errorf("could not pull templates from function yaml file: %s", err.Error()) + if err := pullStackTemplates(newTemplateInfos, services.StackConfiguration.TemplateConfigs, cmd); err != nil { + return fmt.Errorf("could not pull templates from function yaml file: %w", err) } } diff --git a/commands/secret_apply.go b/commands/secret_apply.go new file mode 100644 index 000000000..6f14e6eae --- /dev/null +++ b/commands/secret_apply.go @@ -0,0 +1,166 @@ +// Copyright (c) OpenFaaS Author(s) 2019. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +package commands + +import ( + "context" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/openfaas/faas-cli/proxy" + types "github.com/openfaas/faas-provider/types" + "github.com/spf13/cobra" +) + +var secretApplyCmd = &cobra.Command{ + Use: `apply [--tls-no-verify]`, + Short: "Apply secrets from .secrets folder", + Long: `Apply all secrets from the .secrets folder to the gateway. Each file in .secrets/ will be synced to the gateway, replacing existing secrets with the same name.`, + Example: ` # Apply all secrets from .secrets folder + faas-cli secret apply + + # Apply secrets to a specific namespace + faas-cli secret apply --namespace=my-namespace + + # Apply secrets with a custom gateway + faas-cli secret apply --gateway=http://127.0.0.1:8080`, + RunE: runSecretApply, +} + +func init() { + secretApplyCmd.Flags().StringVarP(&gateway, "gateway", "g", defaultGateway, "Gateway URL starting with http(s)://") + secretApplyCmd.Flags().BoolVar(&tlsInsecure, "tls-no-verify", false, "Disable TLS validation") + secretApplyCmd.Flags().StringVarP(&token, "token", "k", "", "Pass a JWT token to use instead of basic auth") + secretApplyCmd.Flags().StringVarP(&functionNamespace, "namespace", "n", "", "Namespace of the function") + secretApplyCmd.Flags().BoolVar(&trimSecret, "trim", true, "Trim whitespace from the start and end of the secret value") + + secretCmd.AddCommand(secretApplyCmd) +} + +func runSecretApply(cmd *cobra.Command, args []string) error { + gatewayAddress := getGatewayURL(gateway, defaultGateway, "", os.Getenv(openFaaSURLEnvironment)) + + if msg := checkTLSInsecure(gatewayAddress, tlsInsecure); len(msg) > 0 { + fmt.Println(msg) + } + + cliAuth, err := proxy.NewCLIAuth(token, gatewayAddress) + if err != nil { + return err + } + transport := GetDefaultCLITransport(tlsInsecure, &commandTimeout) + client, err := proxy.NewClient(cliAuth, gatewayAddress, transport, &commandTimeout) + if err != nil { + return err + } + + // Get the absolute path to .secrets directory + secretsPath, err := filepath.Abs(localSecretsDir) + if err != nil { + return fmt.Errorf("can't determine secrets folder: %w", err) + } + + // Check if .secrets directory exists + if _, err := os.Stat(secretsPath); os.IsNotExist(err) { + return fmt.Errorf("secrets directory does not exist: %s", secretsPath) + } + + // Read all files from .secrets directory + files, err := os.ReadDir(secretsPath) + if err != nil { + return fmt.Errorf("failed to read secrets directory: %w", err) + } + + if len(files) == 0 { + fmt.Println("No secrets found in .secrets directory") + return nil + } + + // Get list of existing secrets from gateway to check if they exist + existingSecrets, err := client.GetSecretList(context.Background(), functionNamespace) + if err != nil { + return fmt.Errorf("failed to get secret list: %w", err) + } + + // Create a map of existing secrets for quick lookup + secretMap := make(map[string]bool) + for _, secret := range existingSecrets { + // Match by name and namespace + if secret.Namespace == functionNamespace { + secretMap[secret.Name] = true + } + } + + // Process each file in .secrets directory + for _, file := range files { + if file.IsDir() { + continue + } + + secretName := file.Name() + + // Validate secret name + isValid, err := validateSecretName(secretName) + if !isValid { + fmt.Printf("Skipping invalid secret name: %s - %v\n", secretName, err) + continue + } + + // Read secret file content + secretFilePath := filepath.Join(secretsPath, secretName) + fileData, err := os.ReadFile(secretFilePath) + if err != nil { + fmt.Printf("Failed to read secret file %s: %v\n", secretName, err) + continue + } + + secretValue := string(fileData) + if trimSecret { + secretValue = strings.TrimSpace(secretValue) + } + + if len(secretValue) == 0 { + fmt.Printf("Skipping empty secret: %s\n", secretName) + continue + } + + secret := types.Secret{ + Name: secretName, + Namespace: functionNamespace, + Value: secretValue, + RawValue: fileData, + } + + // Check if secret exists and delete it if it does + if secretMap[secretName] { + fmt.Printf("Secret %s exists, deleting before recreating...\n", secretName) + deleteSecret := types.Secret{ + Name: secretName, + Namespace: functionNamespace, + } + err = client.RemoveSecret(context.Background(), deleteSecret) + if err != nil { + fmt.Printf("Failed to remove existing secret %s: %v\n", secretName, err) + // Continue anyway to try creating it + } + } + + // Create the secret + fmt.Printf("Creating secret: %s.%s\n", secret.Name, functionNamespace) + status, output := client.CreateSecret(context.Background(), secret) + + if status == http.StatusConflict { + // If secret still exists (race condition), try update instead + fmt.Printf("Secret %s still exists, updating...\n", secretName) + _, output = client.UpdateSecret(context.Background(), secret) + } + + fmt.Print(output) + } + + return nil +} diff --git a/commands/template_pull.go b/commands/template_pull.go index af496f035..f6effe8d9 100644 --- a/commands/template_pull.go +++ b/commands/template_pull.go @@ -16,7 +16,7 @@ var ( ) func init() { - templatePullCmd.Flags().BoolVar(&overwrite, "overwrite", false, "Overwrite existing templates?") + templatePullCmd.Flags().BoolVar(&overwrite, "overwrite", true, "Overwrite existing templates?") templatePullCmd.Flags().BoolVar(&pullDebug, "debug", false, "Enable debug output") templateCmd.AddCommand(templatePullCmd) @@ -35,7 +35,9 @@ directory from the root of the repo, if it exists. faas-cli template pull https://github.com/openfaas/templates faas-cli template pull https://github.com/openfaas/templates#1.0 `, - RunE: runTemplatePull, + RunE: runTemplatePull, + SilenceErrors: true, + SilenceUsage: true, } func runTemplatePull(cmd *cobra.Command, args []string) error { @@ -44,10 +46,15 @@ func runTemplatePull(cmd *cobra.Command, args []string) error { repository = args[0] } - templateName := "" // templateName may not be known at this point + return templatePull(repository, overwrite) +} + +func templatePull(repository string, overwriteTemplates bool) error { + templateName := "" repository = getTemplateURL(repository, os.Getenv(templateURLEnvironment), DefaultTemplateRepository) - return pullTemplate(repository, templateName) + + return pullTemplate(repository, templateName, overwriteTemplates) } func pullDebugPrint(message string) { diff --git a/commands/template_pull_stack.go b/commands/template_pull_stack.go index 794b2d605..6b5bc1edb 100644 --- a/commands/template_pull_stack.go +++ b/commands/template_pull_stack.go @@ -16,7 +16,7 @@ var ( ) func init() { - templatePullStackCmd.Flags().BoolVar(&overwrite, "overwrite", false, "Overwrite existing templates?") + templatePullStackCmd.Flags().BoolVar(&overwrite, "overwrite", true, "Overwrite existing templates?") templatePullStackCmd.Flags().BoolVar(&pullDebug, "debug", false, "Enable debug output") templatePullCmd.AddCommand(templatePullStackCmd) @@ -40,7 +40,8 @@ func runTemplatePullStack(cmd *cobra.Command, args []string) error { if err != nil { return err } - return pullStackTemplates(templatesConfig, cmd) + + return pullStackTemplates([]string{}, templatesConfig, cmd) } func loadTemplateConfig() ([]stack.TemplateSource, error) { @@ -68,35 +69,47 @@ func readStackConfig() (stack.Configuration, error) { return configField, nil } -func pullStackTemplates(templateInfo []stack.TemplateSource, cmd *cobra.Command) error { - for _, val := range templateInfo { - fmt.Printf("Pulling template: %s from configuration file: %s\n", val.Name, yamlFile) - if len(val.Source) == 0 { - pullErr := runTemplateStorePull(cmd, []string{val.Name}) - if pullErr != nil { - return pullErr +func pullStackTemplates(missingTemplates []string, templateSources []stack.TemplateSource, cmd *cobra.Command) error { + + for _, val := range missingTemplates { + + var templateConfig stack.TemplateSource + for _, config := range templateSources { + if config.Name == val { + templateConfig = config + break + } + } + + if templateConfig.Source == "" { + fmt.Printf("Pulling template: %s from store\n", val) + + if err := runTemplateStorePull(cmd, []string{val}); err != nil { + return err } } else { + fmt.Printf("Pulling template: %s from %s\n", val, templateConfig.Source) - templateName := val.Name - pullErr := pullTemplate(val.Source, templateName) - if pullErr != nil { - return pullErr + templateName := templateConfig.Name + if err := pullTemplate(templateConfig.Source, templateName, overwrite); err != nil { + return err } } } + return nil } // filter templates which are already available on filesystem -func filterExistingTemplates(templateInfo []stack.TemplateSource, templatesDir string) ([]stack.TemplateSource, error) { - var newTemplates []stack.TemplateSource - for _, info := range templateInfo { - templatePath := fmt.Sprintf("%s/%s", templatesDir, info.Name) +func getMissingTemplates(functions map[string]stack.Function, templatesDir string) ([]string, error) { + var missing []string + + for _, function := range functions { + templatePath := fmt.Sprintf("%s/%s", templatesDir, function.Language) if _, err := os.Stat(templatePath); err != nil && os.IsNotExist(err) { - newTemplates = append(newTemplates, info) + missing = append(missing, function.Language) } } - return newTemplates, nil + return missing, nil } diff --git a/commands/template_pull_stack_test.go b/commands/template_pull_stack_test.go index f335f773d..3baeaa2bc 100644 --- a/commands/template_pull_stack_test.go +++ b/commands/template_pull_stack_test.go @@ -9,54 +9,52 @@ import ( "github.com/openfaas/go-sdk/stack" ) -func Test_pullAllTemplates(t *testing.T) { +func Test_pullStackTemplates(t *testing.T) { tests := []struct { - title string - existingTemplates []stack.TemplateSource - expectedError bool + title string + templateSources []stack.TemplateSource + missingTemplates []string + expectedError bool }{ { - title: "Pull specific Template", - existingTemplates: []stack.TemplateSource{ + title: "Pull specific template", + templateSources: []stack.TemplateSource{ {Name: "my_powershell", Source: "https://github.com/openfaas-incubator/powershell-http-template"}, {Name: "my_rust", Source: "https://github.com/openfaas-incubator/openfaas-rust-template"}, }, - expectedError: false, + missingTemplates: []string{"my_powershell"}, + expectedError: false, }, { title: "Pull all templates", - existingTemplates: []stack.TemplateSource{ + templateSources: []stack.TemplateSource{ {Name: "my_powershell", Source: "https://github.com/openfaas-incubator/powershell-http-template"}, {Name: "my_rust", Source: "https://github.com/openfaas-incubator/openfaas-rust-template"}, }, - expectedError: false, + missingTemplates: []string{"my_powershell", "my_rust"}, + expectedError: false, }, { title: "Pull custom template and template from store without source", - existingTemplates: []stack.TemplateSource{ + templateSources: []stack.TemplateSource{ {Name: "perl-alpine"}, {Name: "my_rust", Source: "https://github.com/openfaas-incubator/openfaas-rust-template"}, }, - expectedError: false, + missingTemplates: []string{"perl-alpine"}, + expectedError: false, }, { - title: "Pull non-existant template", - existingTemplates: []stack.TemplateSource{ + title: "Pull template from invalid URL", + templateSources: []stack.TemplateSource{ {Name: "my_powershell", Source: "invalidURL"}, {Name: "my_rust", Source: "https://github.com/openfaas-incubator/openfaas-rust-template"}, }, - expectedError: true, + missingTemplates: []string{"my_powershell"}, + expectedError: true, }, { - title: "Pull template with invalid URL", - existingTemplates: []stack.TemplateSource{ - {Name: "my_powershell", Source: "invalidURL"}, - }, - expectedError: true, - }, - { - title: "Pull template which does not exist in store", - existingTemplates: []stack.TemplateSource{ + title: "Pull template which does not exist in store, which has no URL given", + templateSources: []stack.TemplateSource{ {Name: "my_powershell"}, }, expectedError: true, @@ -64,7 +62,7 @@ func Test_pullAllTemplates(t *testing.T) { } for _, test := range tests { t.Run(test.title, func(t *testing.T) { - actualError := pullStackTemplates(test.existingTemplates, templatePullStackCmd) + actualError := pullStackTemplates(test.missingTemplates, test.templateSources, templatePullStackCmd) if actualError != nil && test.expectedError == false { t.Errorf("Unexpected error: %s", actualError.Error()) } @@ -72,21 +70,61 @@ func Test_pullAllTemplates(t *testing.T) { } } -func Test_filterExistingTemplates(t *testing.T) { +func Test_getMissingTemplates_finds_missing_template(t *testing.T) { templatesDir := "./template" defer os.RemoveAll(templatesDir) - templates := []stack.TemplateSource{ - {Name: "dockerfile", Source: "https://github.com/openfaas/templates"}, - {Name: "ruby", Source: "https://github.com/openfaas/classic-templates"}, - {Name: "perl", Source: "https://github.com/openfaas-incubator/perl-template"}, + stackYaml := `version: 1.0 +provider: + name: openfaas + gateway: http://127.0.0.1:8080 +functions: + docker-fn: + lang: dockerfile + handler: ./dockerfile + ruby-fn: + lang: ruby + handler: ./ruby + image: ttl.sh/alexellis/ruby:latest + perl-fn: + lang: perl + handler: ./perl + image: ttl.sh/alexellis/perl:latest + +configuration: + templates: + - name: dockerfile + source: https://github.com/openfaas/templates + - name: ruby + source: https://github.com/openfaas/classic-templates + - name: perl + source: https://github.com/openfaas-incubator/perl-template +` + + parsedFns, err := stack.ParseYAMLData([]byte(stackYaml), "", "", false) + if err != nil { + t.Errorf("Unexpected error: %s", err.Error()) } + functions := parsedFns.Functions + // Copy the submodule to temp directory to avoid altering it during tests testRepoGit := filepath.Join("testdata", "templates", "template") builder.CopyFiles(testRepoGit, templatesDir) - newTemplateInfos, err := filterExistingTemplates(templates, templatesDir) + // Assert that dockerfile and ruby exist, and that perl is missing + + if _, err := os.Stat(filepath.Join(templatesDir, "dockerfile")); err != nil { + t.Errorf("Unexpected error: %s", err.Error()) + } + if _, err := os.Stat(filepath.Join(templatesDir, "ruby")); err != nil { + t.Errorf("Unexpected error: %s", err.Error()) + } + if _, err := os.Stat(filepath.Join(templatesDir, "perl")); err == nil { + t.Errorf("perl should not exist at this point") + } + + newTemplateInfos, err := getMissingTemplates(functions, templatesDir) if err != nil { t.Errorf("Unexpected error: %s", err.Error()) } @@ -95,7 +133,9 @@ func Test_filterExistingTemplates(t *testing.T) { t.Errorf("Wanted new templates: `%d` got `%d`", 1, len(newTemplateInfos)) } - if newTemplateInfos[0].Name != "perl" { - t.Errorf("Wanted template: `%s` got `%s`", "perl", newTemplateInfos[0].Name) + wantMissingName := "perl" + gotMissingName := newTemplateInfos[0] + if gotMissingName != wantMissingName { + t.Errorf("Wanted template: `%s` got `%s`", wantMissingName, gotMissingName) } } diff --git a/commands/template_pull_test.go b/commands/template_pull_test.go index 72ba12563..87fb25df9 100644 --- a/commands/template_pull_test.go +++ b/commands/template_pull_test.go @@ -3,11 +3,9 @@ package commands import ( - "bytes" "fmt" - "log" "os" - "regexp" + "os/exec" "strings" "testing" ) @@ -20,52 +18,38 @@ func Test_templatePull(t *testing.T) { defer tearDownFetchTemplates(t) faasCmd.SetArgs([]string{"template", "pull", localTemplateRepository}) - err := faasCmd.Execute() - if err != nil { - t.Errorf("unexpected error while puling valid repo: %s", err.Error()) + if err := faasCmd.Execute(); err != nil { + t.Fatalf("unexpected error while puling valid repo (%q): %s", localTemplateRepository, err.Error()) } // Verify created directories if _, err := os.Stat("template"); err != nil { t.Fatalf("The directory %s was not created", "template") } + }) t.Run("WithOverwriting", func(t *testing.T) { defer tearDownFetchTemplates(t) - faasCmd.SetArgs([]string{"template", "pull", localTemplateRepository}) - err := faasCmd.Execute() - if err != nil { - t.Errorf("unexpected error while executing template pull: %s", err.Error()) + if err := templatePull(localTemplateRepository, true); err != nil { + t.Fatalf("unexpected error while executing initial template pull: %s", err.Error()) } - var buf bytes.Buffer - log.SetOutput(&buf) - - r := regexp.MustCompile(`(?m:Cannot overwrite the following \d+ template\(s\):)`) + //execute command to ls in the template directory - faasCmd.SetArgs([]string{"template", "pull", localTemplateRepository}) - err = faasCmd.Execute() + lsCmd := exec.Command("ls", "template") + _, err := lsCmd.CombinedOutput() if err != nil { - t.Errorf("unexpected error while executing template pull: %s", err.Error()) - } - - if !r.MatchString(buf.String()) { - t.Fatal(buf.String()) + t.Fatalf("error while listing template directory: %s", err.Error()) } - buf.Reset() - - faasCmd.SetArgs([]string{"template", "pull", localTemplateRepository, "--overwrite"}) - err = faasCmd.Execute() - if err != nil { - t.Errorf("unexpected error while executing template pull with --overwrite: %s", err.Error()) + if err := templatePull(localTemplateRepository, false); err == nil { + t.Fatalf("error expected overwriting existing templates with --overwrite=false:") } - str := buf.String() - if r.MatchString(str) { - t.Fatal() + if err := templatePull(localTemplateRepository, true); err != nil { + t.Fatalf("unexpected error while executing template pull with --overwrite: %s", err.Error()) } // Verify created directories @@ -80,10 +64,11 @@ func Test_templatePull(t *testing.T) { want := "the repository URL must be a valid git repo uri" got := err.Error() if !strings.Contains(err.Error(), want) { - t.Errorf("The error should contain:\n%q\n, but was:\n%q", want, got) + t.Fatalf("The error should contain:\n%q\n, but was:\n%q", want, got) } }) } + func Test_templatePullPriority(t *testing.T) { templateURLs := []struct { name string @@ -113,7 +98,7 @@ func Test_templatePullPriority(t *testing.T) { }, } for _, scenario := range templateURLs { - t.Run(fmt.Sprintf("%s", scenario.name), func(t *testing.T) { + t.Run(scenario.name, func(t *testing.T) { repository = getTemplateURL(scenario.cliURL, scenario.envURL, DefaultTemplateRepository) if repository != scenario.resultURL { t.Errorf("result URL, want %s got %s", scenario.resultURL, repository) diff --git a/commands/template_store_pull.go b/commands/template_store_pull.go index dfa25e081..afc26ec43 100644 --- a/commands/template_store_pull.go +++ b/commands/template_store_pull.go @@ -6,6 +6,7 @@ package commands import ( "fmt" "os" + "strings" "github.com/spf13/cobra" ) @@ -32,33 +33,46 @@ var templateStorePullCmd = &cobra.Command{ func runTemplateStorePull(cmd *cobra.Command, args []string) error { if len(args) == 0 { - return fmt.Errorf("\nNeed to specify one of the store templates, check available ones by running the command:\n\nfaas-cli template store list\n") + return fmt.Errorf("need to specify one of the store templates, check available ones by running the command:\n\nfaas-cli template store list") } if len(args) > 1 { - return fmt.Errorf("\nNeed to specify single template from the store, check available ones by running the command:\n\nfaas-cli template store list\n") + return fmt.Errorf("need to specify single template from the store, check available ones by running the command:\n\nfaas-cli template store list") } envTemplateRepoStore := os.Getenv(templateStoreURLEnvironment) storeURL := getTemplateStoreURL(templateStoreURL, envTemplateRepoStore, DefaultTemplatesStore) - storeTemplates, templatesErr := getTemplateInfo(storeURL) - if templatesErr != nil { - return fmt.Errorf("error while fetching templates from store: %s", templatesErr) + storeTemplates, err := getTemplateInfo(storeURL) + if err != nil { + return fmt.Errorf("error while fetching templates from store: %s", err) } templateName := args[0] found := false + for _, storeTemplate := range storeTemplates { + + _, ref, _ := strings.Cut(templateName, "@") + sourceName := fmt.Sprintf("%s/%s", storeTemplate.Source, storeTemplate.TemplateName) - if templateName == storeTemplate.TemplateName || templateName == sourceName { - err := runTemplatePull(cmd, []string{storeTemplate.Repository}) - if err != nil { + + // len(ref) > 0 && storeTemplate.TemplateName+"@"+ref == templateName+"@"+ref + if templateName == storeTemplate.TemplateName || (len(ref) > 0 && templateName == storeTemplate.TemplateName+"@"+ref) || templateName == sourceName { + + repository := storeTemplate.Repository + if ref != "" { + repository = repository + "#" + ref + } + + if err := runTemplatePull(cmd, []string{repository}); err != nil { return fmt.Errorf("error while pulling template: %s : %s", storeTemplate.TemplateName, err.Error()) } + found = true break } } + if !found { return fmt.Errorf("template with name: `%s` does not exist in the repo", templateName) } diff --git a/commands/watch.go b/commands/watch.go index e521e6fe4..60aa94355 100644 --- a/commands/watch.go +++ b/commands/watch.go @@ -36,7 +36,7 @@ func watchLoop(cmd *cobra.Command, args []string, onChange func(cmd *cobra.Comma } fnNames := []string{} - for name, _ := range services.Functions { + for name := range services.Functions { fnNames = append(fnNames, name) } diff --git a/config/config_file.go b/config/config_file.go index 97add8bd8..4720f13ef 100644 --- a/config/config_file.go +++ b/config/config_file.go @@ -131,8 +131,8 @@ func EnsureFile() (string, error) { } filePath := path.Clean(filepath.Join(dirPath, DefaultFile)) - if err := os.MkdirAll(filepath.Dir(filePath), permission); err != nil { - return "", err + if err := os.MkdirAll(filepath.Dir(filePath), permission); err != nil && !os.IsExist(err) { + return "", fmt.Errorf("error creating directory: %s - %w", filepath.Dir(filePath), err) } if _, err := os.Stat(filePath); err != nil && os.IsNotExist(err) { diff --git a/testing/.gitignore b/testing/.gitignore new file mode 100644 index 000000000..4f0356525 --- /dev/null +++ b/testing/.gitignore @@ -0,0 +1,3 @@ +template +build +.secrets diff --git a/testing/go-090/go.mod b/testing/go-090/go.mod new file mode 100644 index 000000000..c2d61626c --- /dev/null +++ b/testing/go-090/go.mod @@ -0,0 +1,3 @@ +module handler/function + +go 1.24.0 diff --git a/testing/go-090/handler.go b/testing/go-090/handler.go new file mode 100644 index 000000000..450322896 --- /dev/null +++ b/testing/go-090/handler.go @@ -0,0 +1,22 @@ +package function + +import ( + "fmt" + "io" + "net/http" +) + +func Handle(w http.ResponseWriter, r *http.Request) { + var input []byte + + if r.Body != nil { + defer r.Body.Close() + + body, _ := io.ReadAll(r.Body) + + input = body + } + + w.WriteHeader(http.StatusOK) + w.Write([]byte(fmt.Sprintf("Body: %s", string(input)))) +} diff --git a/testing/go-latest/go.mod b/testing/go-latest/go.mod new file mode 100644 index 000000000..c2d61626c --- /dev/null +++ b/testing/go-latest/go.mod @@ -0,0 +1,3 @@ +module handler/function + +go 1.24.0 diff --git a/testing/go-latest/handler.go b/testing/go-latest/handler.go new file mode 100644 index 000000000..450322896 --- /dev/null +++ b/testing/go-latest/handler.go @@ -0,0 +1,22 @@ +package function + +import ( + "fmt" + "io" + "net/http" +) + +func Handle(w http.ResponseWriter, r *http.Request) { + var input []byte + + if r.Body != nil { + defer r.Body.Close() + + body, _ := io.ReadAll(r.Body) + + input = body + } + + w.WriteHeader(http.StatusOK) + w.Write([]byte(fmt.Sprintf("Body: %s", string(input)))) +} diff --git a/testing/go-sha-2e6e262/go.mod b/testing/go-sha-2e6e262/go.mod new file mode 100644 index 000000000..c2d61626c --- /dev/null +++ b/testing/go-sha-2e6e262/go.mod @@ -0,0 +1,3 @@ +module handler/function + +go 1.24.0 diff --git a/testing/go-sha-2e6e262/handler.go b/testing/go-sha-2e6e262/handler.go new file mode 100644 index 000000000..450322896 --- /dev/null +++ b/testing/go-sha-2e6e262/handler.go @@ -0,0 +1,22 @@ +package function + +import ( + "fmt" + "io" + "net/http" +) + +func Handle(w http.ResponseWriter, r *http.Request) { + var input []byte + + if r.Body != nil { + defer r.Body.Close() + + body, _ := io.ReadAll(r.Body) + + input = body + } + + w.WriteHeader(http.StatusOK) + w.Write([]byte(fmt.Sprintf("Body: %s", string(input)))) +} diff --git a/testing/inproc-fn/go.mod b/testing/inproc-fn/go.mod new file mode 100644 index 000000000..45f94acfd --- /dev/null +++ b/testing/inproc-fn/go.mod @@ -0,0 +1,3 @@ +module handler/function + +go 1.18 diff --git a/testing/inproc-fn/handler.go b/testing/inproc-fn/handler.go new file mode 100644 index 000000000..824e6d3de --- /dev/null +++ b/testing/inproc-fn/handler.go @@ -0,0 +1,25 @@ +package function + +import ( + "fmt" + "io" + + "net/http" +) + +func Handle(w http.ResponseWriter, r *http.Request) { + var input []byte + + if r.Body != nil { + defer r.Body.Close() + + body, _ := io.ReadAll(r.Body) + + input = body + } + + w.WriteHeader(http.StatusOK) + w.Write([]byte(fmt.Sprintf("Body: %s", string(input)))) + + input = nil +} diff --git a/testing/no-template-config.yaml b/testing/no-template-config.yaml new file mode 100644 index 000000000..73818a00f --- /dev/null +++ b/testing/no-template-config.yaml @@ -0,0 +1,20 @@ +version: 1.0 +provider: + name: openfaas + gateway: http://127.0.0.1:8080 + +functions: + go-latest: + lang: golang-middleware + handler: ./go-latest + image: ttl.sh/alexellis/go-latest:latest + + go-090: + lang: golang-middleware@0.9.0 + handler: ./go-090 + image: ttl.sh/alexellis/go-090:latest + + go-sha-2e6e262: + lang: golang-middleware@sha-2e6e262 + handler: ./go-sha-2e6e262 + image: ttl.sh/alexellis/go-sha-2e6e262:latest diff --git a/testing/pyfn/handler.py b/testing/pyfn/handler.py new file mode 100644 index 000000000..a1a099cd6 --- /dev/null +++ b/testing/pyfn/handler.py @@ -0,0 +1,5 @@ +def handle(event, context): + return { + "statusCode": 200, + "body": "Hello from OpenFaaS!" + } diff --git a/testing/pyfn/handler_test.py b/testing/pyfn/handler_test.py new file mode 100644 index 000000000..b07d5bf78 --- /dev/null +++ b/testing/pyfn/handler_test.py @@ -0,0 +1,10 @@ +from .handler import handle + +# Test your handler here + +# To disable testing, you can set the build_arg `TEST_ENABLED=false` on the CLI or in your stack.yml +# https://docs.openfaas.com/reference/yaml/#function-build-args-build-args + +def test_handle(): + # assert handle("input") == "input" + pass diff --git a/testing/pyfn/requirements.txt b/testing/pyfn/requirements.txt new file mode 100644 index 000000000..e69de29bb diff --git a/testing/pyfn/tox.ini b/testing/pyfn/tox.ini new file mode 100644 index 000000000..a64a80087 --- /dev/null +++ b/testing/pyfn/tox.ini @@ -0,0 +1,41 @@ +# If you would like to disable +# automated testing during faas-cli build, + +# Replace the content of this file with +# [tox] +# skipsdist = true + +# You can also edit, remove, or add additional test steps +# by editing, removing, or adding new testenv sections + + +# find out more about tox: https://tox.readthedocs.io/en/latest/ +[tox] +envlist = lint,test +skipsdist = true + +[testenv:test] +deps = + flask + pytest + -rrequirements.txt +commands = + # run unit tests with pytest + # https://docs.pytest.org/en/stable/ + # configure by adding a pytest.ini to your handler + pytest + +[testenv:lint] +deps = + flake8 +commands = + flake8 . + +[flake8] +count = true +max-line-length = 127 +max-complexity = 10 +statistics = true +# stop the build if there are Python syntax errors or undefined names +select = E9,F63,F7,F82 +show-source = true diff --git a/testing/stack.yaml b/testing/stack.yaml new file mode 100644 index 000000000..6d5bf5d72 --- /dev/null +++ b/testing/stack.yaml @@ -0,0 +1,17 @@ +version: 1.0 +provider: + name: openfaas + gateway: http://127.0.0.1:8080 + +functions: + + inproc-fn: + lang: golang-middleware-inproc + handler: ./inproc-fn + image: ttl.sh/go-inproc-fn:latest + + go-fn: + lang: golang-middleware + handler: ./inproc-fn + image: ttl.sh/go-middleware-fn:latest + diff --git a/testing/template-config.yaml b/testing/template-config.yaml new file mode 100644 index 000000000..5f9e70dc3 --- /dev/null +++ b/testing/template-config.yaml @@ -0,0 +1,27 @@ +version: 1.0 +provider: + name: openfaas + gateway: http://127.0.0.1:8080 + +functions: + go-latest: + lang: golang-middleware + handler: ./go-latest + image: ttl.sh/alexellis/go-latest:latest + + go-090: + lang: golang-middleware@0.9.0 + handler: ./go-090 + image: ttl.sh/alexellis/go-090:latest + + go-sha-2e6e262: + lang: golang-middleware@sha-2e6e262 + handler: ./go-sha-2e6e262 + image: ttl.sh/alexellis/go-sha-2e6e262:latest + + +configuration: + templates: + - name: golang-middleware + source: https://github.com/openfaas/golang-http-template + diff --git a/vendor/modules.txt b/vendor/modules.txt index 20438bf70..36714e122 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,5 +1,3 @@ -# dario.cat/mergo v1.0.0 -## explicit; go 1.13 # github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c ## explicit; go 1.16 github.com/Azure/go-ansiterm @@ -7,10 +5,6 @@ github.com/Azure/go-ansiterm/winterm # github.com/Masterminds/semver v1.5.0 ## explicit github.com/Masterminds/semver -# github.com/Microsoft/go-winio v0.6.2 -## explicit; go 1.21 -# github.com/ProtonMail/go-crypto v1.1.6 -## explicit; go 1.17 # github.com/VividCortex/ewma v1.2.0 ## explicit; go 1.12 github.com/VividCortex/ewma @@ -27,27 +21,17 @@ github.com/alexellis/go-execute/v2 # github.com/alexellis/hmac/v2 v2.0.0 ## explicit; go 1.16 github.com/alexellis/hmac/v2 -# github.com/beorn7/perks v1.0.1 -## explicit; go 1.11 # github.com/bep/debounce v1.2.1 ## explicit github.com/bep/debounce -# github.com/cespare/xxhash/v2 v2.3.0 -## explicit; go 1.11 # github.com/cheggaaa/pb/v3 v3.1.7 ## explicit; go 1.18 github.com/cheggaaa/pb/v3 github.com/cheggaaa/pb/v3/termutil -# github.com/cloudflare/circl v1.6.1 -## explicit; go 1.22.0 # github.com/containerd/stargz-snapshotter/estargz v0.17.0 ## explicit; go 1.23.0 github.com/containerd/stargz-snapshotter/estargz github.com/containerd/stargz-snapshotter/estargz/errorutil -# github.com/cyphar/filepath-securejoin v0.4.1 -## explicit; go 1.18 -# github.com/distribution/reference v0.6.0 -## explicit; go 1.20 # github.com/docker/cli v28.3.3+incompatible ## explicit github.com/docker/cli/cli/config @@ -62,15 +46,11 @@ github.com/docker/distribution/registry/client/auth/challenge ## explicit; go 1.21 github.com/docker/docker-credential-helpers/client github.com/docker/docker-credential-helpers/credentials -# github.com/docker/go-units v0.5.0 -## explicit # github.com/drone/envsubst v1.0.3 ## explicit; go 1.13 github.com/drone/envsubst github.com/drone/envsubst/parse github.com/drone/envsubst/path -# github.com/emirpasic/gods v1.18.1 -## explicit; go 1.2 # github.com/fatih/color v1.18.0 ## explicit; go 1.17 github.com/fatih/color @@ -93,10 +73,6 @@ github.com/go-git/go-git/v5/internal/path_util github.com/go-git/go-git/v5/plumbing/format/config github.com/go-git/go-git/v5/plumbing/format/gitignore github.com/go-git/go-git/v5/utils/ioutil -# github.com/gogo/protobuf v1.3.2 -## explicit; go 1.15 -# github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 -## explicit; go 1.20 # github.com/google/go-cmp v0.7.0 ## explicit; go 1.21 github.com/google/go-cmp/cmp @@ -134,16 +110,12 @@ github.com/google/go-containerregistry/pkg/v1/remote/transport github.com/google/go-containerregistry/pkg/v1/stream github.com/google/go-containerregistry/pkg/v1/tarball github.com/google/go-containerregistry/pkg/v1/types -# github.com/gorilla/mux v1.8.1 -## explicit; go 1.20 # github.com/inconshreveable/mousetrap v1.1.0 ## explicit; go 1.18 github.com/inconshreveable/mousetrap # github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 ## explicit github.com/jbenet/go-context/io -# github.com/kevinburke/ssh_config v1.2.0 -## explicit # github.com/klauspost/compress v1.18.0 ## explicit; go 1.22 github.com/klauspost/compress @@ -154,8 +126,6 @@ github.com/klauspost/compress/internal/le github.com/klauspost/compress/internal/snapref github.com/klauspost/compress/zstd github.com/klauspost/compress/zstd/internal/xxhash -# github.com/magefile/mage v1.14.0 -## explicit; go 1.12 # github.com/mattn/go-colorable v0.1.14 ## explicit; go 1.18 github.com/mattn/go-colorable @@ -178,16 +148,6 @@ github.com/moby/term/windows # github.com/morikuni/aec v1.0.0 ## explicit github.com/morikuni/aec -# github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 -## explicit -# github.com/nats-io/nats.go v1.37.0 -## explicit; go 1.20 -# github.com/nats-io/nkeys v0.4.8 -## explicit; go 1.20 -# github.com/nats-io/nuid v1.0.1 -## explicit -# github.com/nats-io/stan.go v0.10.4 -## explicit; go 1.14 # github.com/olekukonko/cat v0.0.0-20250817074551-3280053e4e00 ## explicit; go 1.21 github.com/olekukonko/cat @@ -228,40 +188,18 @@ github.com/openfaas/go-sdk github.com/openfaas/go-sdk/builder github.com/openfaas/go-sdk/internal/httpclient github.com/openfaas/go-sdk/stack -# github.com/openfaas/nats-queue-worker v0.0.0-20231219105451-b94918cb8a24 -## explicit; go 1.20 -# github.com/otiai10/copy v1.14.1 -## explicit; go 1.18 -# github.com/otiai10/mint v1.6.3 -## explicit; go 1.18 -# github.com/pjbgf/sha1cd v0.3.2 -## explicit; go 1.21 # github.com/pkg/errors v0.9.1 ## explicit github.com/pkg/errors -# github.com/prometheus/client_golang v1.20.5 -## explicit; go 1.20 -# github.com/prometheus/client_model v0.6.1 -## explicit; go 1.19 -# github.com/prometheus/common v0.62.0 -## explicit; go 1.21 -# github.com/prometheus/procfs v0.15.1 -## explicit; go 1.20 # github.com/rivo/uniseg v0.4.7 ## explicit; go 1.18 github.com/rivo/uniseg # github.com/ryanuber/go-glob v1.0.0 ## explicit github.com/ryanuber/go-glob -# github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 -## explicit; go 1.13 -# github.com/sethvargo/go-password v0.3.1 -## explicit; go 1.21 # github.com/sirupsen/logrus v1.9.3 ## explicit; go 1.13 github.com/sirupsen/logrus -# github.com/skeema/knownhosts v1.3.1 -## explicit; go 1.22 # github.com/spf13/cobra v1.9.1 ## explicit; go 1.15 github.com/spf13/cobra @@ -271,12 +209,6 @@ github.com/spf13/pflag # github.com/vbatts/tar-split v0.12.1 ## explicit; go 1.17 github.com/vbatts/tar-split/archive/tar -# github.com/xanzy/ssh-agent v0.3.3 -## explicit; go 1.16 -# golang.org/x/crypto v0.41.0 -## explicit; go 1.23.0 -# golang.org/x/mod v0.27.0 -## explicit; go 1.23.0 # golang.org/x/net v0.43.0 ## explicit; go 1.23.0 golang.org/x/net/context @@ -287,8 +219,6 @@ golang.org/x/sync/errgroup ## explicit; go 1.23.0 golang.org/x/sys/unix golang.org/x/sys/windows -# google.golang.org/protobuf v1.36.4 -## explicit; go 1.21 # gopkg.in/warnings.v0 v0.1.2 ## explicit gopkg.in/warnings.v0 diff --git a/versioncontrol/core.go b/versioncontrol/core.go index a9dd2f149..e1aaa7b79 100644 --- a/versioncontrol/core.go +++ b/versioncontrol/core.go @@ -5,6 +5,7 @@ package versioncontrol import ( "bytes" "fmt" + "log" "os" "os/exec" "strings" @@ -48,6 +49,10 @@ func (v *vcsCmd) run(dir string, cmdline string, keyval map[string]string, verbo return nil, err } + if os.Getenv("FAAS_DEBUG") == "1" { + log.Printf("[git] %s %s", v.cmd, strings.Join(args, " ")) + } + cmd := exec.Command(v.cmd, args...) cmd.Dir = dir cmd.Env = envWithPWD(cmd.Dir) diff --git a/versioncontrol/git.go b/versioncontrol/git.go index 45385d62d..2dcf212c8 100644 --- a/versioncontrol/git.go +++ b/versioncontrol/git.go @@ -1,20 +1,30 @@ package versioncontrol import ( + "fmt" "strings" "github.com/openfaas/faas-cli/exec" ) -// GitClone defines the command to clone a repo into a directory -var GitClone = &vcsCmd{ +// GitCloneBranch clones a specific branch of a repo into a directory +var GitCloneBranch = &vcsCmd{ name: "Git", cmd: "git", cmds: []string{"clone {repo} {dir} --depth=1 --config core.autocrlf=false -b {refname}"}, scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, } -// GitClone defines the command to clone the default branch of a repo into a directory +// GitCloneFullDepth clones a repo into a directory with full depth so that a +// SHA can be checked out after +var GitCloneFullDepth = &vcsCmd{ + name: "Git", + cmd: "git", + cmds: []string{"clone {repo} {dir} --config core.autocrlf=false"}, + scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, +} + +// GitCloneDefault clones the default branch of a repo into a directory var GitCloneDefault = &vcsCmd{ name: "Git", cmd: "git", @@ -22,7 +32,7 @@ var GitCloneDefault = &vcsCmd{ scheme: []string{"git", "https", "http", "git+ssh", "ssh"}, } -// GitCheckout defines the command to clone a specific REF of repo into a directory +// GitCheckout checks out a specific REF of a repo into a directory var GitCheckout = &vcsCmd{ name: "Git", cmd: "git", @@ -87,6 +97,23 @@ func GetGitDescribe() string { return sha } +// GetGitSHA returns the short Git commit SHA from local repo +func GetGitSHAFor(path string, short bool) (string, error) { + args := []string{"-C", path, "rev-parse"} + if short { + args = append(args, "--short") + } + args = append(args, "HEAD") + getShaCommand := []string{"git"} + getShaCommand = append(getShaCommand, args...) + sha := exec.CommandWithOutput(getShaCommand, true) + if strings.Contains(sha, "Not a git repository") { + return "", fmt.Errorf("not a git repository") + } + + return strings.TrimSpace(sha), nil +} + // GetGitSHA returns the short Git commit SHA from local repo func GetGitSHA() string { getShaCommand := []string{"git", "rev-parse", "--short", "HEAD"} diff --git a/versioncontrol/parse.go b/versioncontrol/parse.go index e5f3a73c4..119da1b29 100644 --- a/versioncontrol/parse.go +++ b/versioncontrol/parse.go @@ -6,9 +6,9 @@ import ( ) const ( - pinCharater = `#` + pinCharacter = `#` gitRemoteRegexpStr = `(git|ssh|https?|git@[-\w.]+):(\/\/)?([^#]*?(?:\.git)?\/?)` - gitPinnedRemoteRegexpStr = gitRemoteRegexpStr + pinCharater + `[-\/\d\w._]+$` + gitPinnedRemoteRegexpStr = gitRemoteRegexpStr + pinCharacter + `[-\/\d\w._]+$` gitRemoteRepoRegexpStr = gitRemoteRegexpStr + `$` ) @@ -50,10 +50,13 @@ func ParsePinnedRemote(repoURL string) (remoteURL, refName string) { // handle ssh special case - atIndex := strings.LastIndex(repoURL, pinCharater) + atIndex := strings.LastIndex(repoURL, pinCharacter) if atIndex > 0 { - refName = repoURL[atIndex+len(pinCharater):] - remoteURL = repoURL[:atIndex] + + remoteURL, refName, _ = strings.Cut(repoURL, pinCharacter) + + // refName = repoURL[atIndex+len(pinCharacter):] + // remoteURL = repoURL[:atIndex] } if !IsGitRemote(remoteURL) { diff --git a/versioncontrol/parse_test.go b/versioncontrol/parse_test.go index 4fa3c1787..b405e8408 100644 --- a/versioncontrol/parse_test.go +++ b/versioncontrol/parse_test.go @@ -61,18 +61,18 @@ func Test_IsPinnedGitRemote(t *testing.T) { name string url string }{ - {name: "git protocol without .git suffix", url: "git://host.xz/path/to/repo" + pinCharater + "feature-branch"}, - {name: "git protocol", url: "git://host.xz/path/to/repo.git/" + pinCharater + "tagname"}, - {name: "scp style with ip address", url: "git@192.168.101.127:user/project.git" + pinCharater + "v1.2.3"}, - {name: "scp style with hostname", url: "git@github.com:user/project.git" + pinCharater + "feature-branch"}, - {name: "http protocol with ip address", url: "http://192.168.101.127/user/project.git" + pinCharater + "tagname"}, - {name: "http protocol", url: "http://github.com/user/project.git" + pinCharater + "v1.2.3"}, - {name: "http protocol without .git suffix", url: "http://github.com/user/project" + pinCharater + "feature-branch"}, - {name: "https protocol with ip address", url: "https://192.168.101.127/user/project.git" + pinCharater + "tagname"}, - {name: "https protocol with hostname", url: "https://github.com/user/project.git" + pinCharater + "v1.2.3"}, - {name: "https protocol with basic auth", url: "https://username:password@github.com/username/repository.git" + pinCharater + "feature/branch"}, - {name: "ssh protocol with hostname no port", url: "ssh://user@host.xz/path/to/repo.git/" + pinCharater + "v1.2.3"}, - {name: "ssh protocol with hostname and port", url: "ssh://user@host.xz:port/path/to/repo.git/" + pinCharater + "tagname"}, + {name: "git protocol without .git suffix", url: "git://host.xz/path/to/repo" + pinCharacter + "feature-branch"}, + {name: "git protocol", url: "git://host.xz/path/to/repo.git/" + pinCharacter + "tagname"}, + {name: "scp style with ip address", url: "git@192.168.101.127:user/project.git" + pinCharacter + "v1.2.3"}, + {name: "scp style with hostname", url: "git@github.com:user/project.git" + pinCharacter + "feature-branch"}, + {name: "http protocol with ip address", url: "http://192.168.101.127/user/project.git" + pinCharacter + "tagname"}, + {name: "http protocol", url: "http://github.com/user/project.git" + pinCharacter + "v1.2.3"}, + {name: "http protocol without .git suffix", url: "http://github.com/user/project" + pinCharacter + "feature-branch"}, + {name: "https protocol with ip address", url: "https://192.168.101.127/user/project.git" + pinCharacter + "tagname"}, + {name: "https protocol with hostname", url: "https://github.com/user/project.git" + pinCharacter + "v1.2.3"}, + {name: "https protocol with basic auth", url: "https://username:password@github.com/username/repository.git" + pinCharacter + "feature/branch"}, + {name: "ssh protocol with hostname no port", url: "ssh://user@host.xz/path/to/repo.git/" + pinCharacter + "v1.2.3"}, + {name: "ssh protocol with hostname and port", url: "ssh://user@host.xz:port/path/to/repo.git/" + pinCharacter + "tagname"}, } for _, scenario := range validURLs { @@ -95,8 +95,8 @@ func Test_IsPinnedGitRemote(t *testing.T) { {name: "git protocol without .git suffix and no tag", url: "git://host.xz/path/to/repo"}, {name: "git protocol with hash", url: "git://github.com/openfaas/faas.git#ff78lf9h@feature/branch"}, {name: "local repo file protocol", url: "file:///path/to/repo.git/@feature/branch"}, - {name: "ssh missing username and port", url: "host.xz:/path/to/repo.git" + pinCharater + "feature-branch"}, - {name: "ssh username and missing port", url: "user@host.xz:path/to/repo.git" + pinCharater + "v1.2.3"}, + {name: "ssh missing username and port", url: "host.xz:/path/to/repo.git" + pinCharacter + "feature-branch"}, + {name: "ssh username and missing port", url: "user@host.xz:path/to/repo.git" + pinCharacter + "v1.2.3"}, {name: "relative local path", url: "path/to/repo.git/@feature/branch"}, {name: "magic relative local", url: "~/path/to/repo.git@feature/branch"}, } @@ -129,11 +129,12 @@ func Test_ParsePinnedRemote(t *testing.T) { {name: "https protocol with basic auth", url: "https://username:password@github.com/username/repository.git", refName: "feature/branch"}, {name: "ssh protocol with hostname no port", url: "ssh://user@host.xz/path/to/repo.git/", refName: "v1.2.3"}, {name: "ssh protocol with hostname and port", url: "ssh://user@host.xz:port/path/to/repo.git/", refName: "tagname"}, + {name: "https with custom pinned SHA", url: "https://github.com/user/project.git", refName: "sha-1234567890"}, } for _, scenario := range cases { t.Run(fmt.Sprintf("can parse refname from url with %s", scenario.name), func(t *testing.T) { - remote, refName := ParsePinnedRemote(scenario.url + pinCharater + scenario.refName) + remote, refName := ParsePinnedRemote(scenario.url + pinCharacter + scenario.refName) if remote != scenario.url { t.Errorf("expected remote url: %s, got: %s", scenario.url, remote) } From 28524d89fbe2b332dcf754e1f03ed87198cd23fa Mon Sep 17 00:00:00 2001 From: "Han Verstraete (OpenFaaS Ltd)" Date: Wed, 19 Nov 2025 14:05:11 +0100 Subject: [PATCH 2/2] Fix template pull stack command Signed-off-by: Han Verstraete (OpenFaaS Ltd) --- commands/fetch_templates.go | 22 ++++++++++++++++++++-- commands/template_pull_stack.go | 13 ++++++++++++- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/commands/fetch_templates.go b/commands/fetch_templates.go index 81b01abd6..ec0eff356 100644 --- a/commands/fetch_templates.go +++ b/commands/fetch_templates.go @@ -155,6 +155,8 @@ func canWriteLanguage(existingLanguages []string, language string, overwriteTemp // It returns the existing languages and the fetched languages // It also returns an error if the templates cannot be read func moveTemplates(localTemplatesDir, extractedPath, templateName string, overwriteTemplate bool, repository string, refName string, sha string) ([]string, []string, error) { + // Get the template name without prefix + template := strings.SplitN(templateName, "@", 2)[0] var ( existingLanguages []string @@ -197,13 +199,29 @@ func moveTemplates(localTemplatesDir, extractedPath, templateName string, overwr refSuffix = "@" + refName } + // Only copy the requested template when a template name is provided + // copy all templates otherwise. + if len(template) > 0 && language != template { + continue + } + if canWriteLanguage(existingLanguages, language, overwriteTemplate) { // Do cp here languageSrc := filepath.Join(extractedPath, TemplateDirectory, language) - languageDest := filepath.Join(localTemplatesDir, language) + + var languageDest string + + if len(templateName) > 0 { + languageDest = filepath.Join(localTemplatesDir, templateName) + } else { + languageDest = filepath.Join(localTemplatesDir, language) + if refName != "" { + languageDest += "@" + refName + } + } + langName := language if refName != "" { - languageDest += "@" + refName langName = language + "@" + refName } fetchedLanguages = append(fetchedLanguages, langName) diff --git a/commands/template_pull_stack.go b/commands/template_pull_stack.go index 6b5bc1edb..5858326f5 100644 --- a/commands/template_pull_stack.go +++ b/commands/template_pull_stack.go @@ -41,7 +41,7 @@ func runTemplatePullStack(cmd *cobra.Command, args []string) error { return err } - return pullStackTemplates([]string{}, templatesConfig, cmd) + return pullConfigTemplates(templatesConfig) } func loadTemplateConfig() ([]stack.TemplateSource, error) { @@ -69,6 +69,17 @@ func readStackConfig() (stack.Configuration, error) { return configField, nil } +func pullConfigTemplates(templateSources []stack.TemplateSource) error { + for _, config := range templateSources { + fmt.Printf("Pulling template: %s from %s\n", config.Name, config.Source) + + if err := pullTemplate(config.Source, config.Name, overwrite); err != nil { + return err + } + } + return nil +} + func pullStackTemplates(missingTemplates []string, templateSources []stack.TemplateSource, cmd *cobra.Command) error { for _, val := range missingTemplates {