[release-v1.16] Pick Python scaffolding (#1236)

* Pick knative#2685

Signed-off-by: Matej Vašek <[email protected]>

* Fix Python S2I On-Cluster build

Signed-off-by: Matej Vašek <[email protected]>

---------

Signed-off-by: Matej Vašek <[email protected]>
Signed-off-by: Matej Vašek <[email protected]>

Author: matejvasek

.github/workflows/test-e2e-runtime.yaml

@@ -12,7 +12,7 @@ jobs:
     continue-on-error: true
     strategy:
       matrix:
-        runtime: ["node", "go", "python", "quarkus", "springboot", "typescript", "rust"]
+        runtime: ["node", "go", "quarkus", "springboot", "typescript", "rust"]
     runs-on: ubuntu-latest
     steps:
       - name: Set Environment Variables

.gitignore

@@ -2,8 +2,6 @@
 /func
 /func_*
 /cmd/func.yaml
-/templates/typescript/cloudevents/build
-/templates/typescript/http/build
 /coverage.out
 /coverage.txt
 /.coverage
@@ -16,11 +14,15 @@
 
 /pkg/functions/testdata/migrations/*/.gitignore
 
-# Nodejs
+# JS
 node_modules
+/templates/typescript/cloudevents/build
+/templates/typescript/http/build
 
 # Python
 __pycache__
+/templates/python/cloudevents/.venv
+/templates/python/http/.venv
 
 # VSCode
 .vscode

Makefile

@@ -85,21 +85,27 @@ generate/zz_filesystem_generated.go: clean_templates templates/certs/ca-certific
 .PHONY: clean_templates
 clean_templates:
 	# Removing temporary template files
+	@rm -rf templates/**/.DS_Store
 	@rm -rf templates/node/cloudevents/node_modules
 	@rm -rf templates/node/http/node_modules
-	@rm -rf templates/python/cloudevents/__pycache__
-	@rm -rf templates/python/http/__pycache__
-	@rm -rf templates/typescript/cloudevents/node_modules
-	@rm -rf templates/typescript/http/node_modules
-	@rm -rf templates/typescript/cloudevents/build
-	@rm -rf templates/typescript/http/build
-	@rm -rf templates/rust/cloudevents/target
-	@rm -rf templates/rust/http/target
+	@rm -rf templates/python/cloudevents/.venv
+	@rm -rf templates/python/cloudevents/.pytest_cache
+	@rm -rf templates/python/cloudevents/function/__pycache__
+	@rm -rf templates/python/cloudevents/tests/__pycache__
+	@rm -rf templates/python/http/.venv
+	@rm -rf templates/python/http/.pytest_cache
+	@rm -rf templates/python/http/function/__pycache__
+	@rm -rf templates/python/http/tests/__pycache__
 	@rm -rf templates/quarkus/cloudevents/target
 	@rm -rf templates/quarkus/http/target
+	@rm -rf templates/rust/cloudevents/target
+	@rm -rf templates/rust/http/target
 	@rm -rf templates/springboot/cloudevents/target
 	@rm -rf templates/springboot/http/target
-	@rm -f templates/**/.DS_Store
+	@rm -rf templates/typescript/cloudevents/build
+	@rm -rf templates/typescript/cloudevents/node_modules
+	@rm -rf templates/typescript/http/build
+	@rm -rf templates/typescript/http/node_modules
 
 .PHONY: clean
 clean: clean_templates ## Remove generated artifacts such as binaries and schemas
@@ -151,9 +157,8 @@ test-node: ## Test Node templates
 	cd templates/node/cloudevents && npm ci && npm test && rm -rf node_modules
 	cd templates/node/http && npm ci && npm test && rm -rf node_modules
 
-test-python: ## Test Python templates
-	cd templates/python/cloudevents && pip3 install -r requirements.txt && python3 test_func.py && rm -rf __pycache__
-	cd templates/python/http && python3 test_func.py && rm -rf __pycache__
+test-python: ## Test Python templates and Scaffolding
+	test/test_python.sh
 
 test-quarkus: ## Test Quarkus templates
 	cd templates/quarkus/cloudevents && ./mvnw -q test && ./mvnw clean && rm .mvn/wrapper/maven-wrapper.jar
@@ -176,14 +181,15 @@ test-typescript: ## Test Typescript templates
 ###############
 
 # Pulls runtimes then rebuilds the embedded filesystem
-update-runtimes:  pull-runtimes generate/zz_filesystem_generated.go ## Update Scaffolding Runtimes
+update-runtimes:  update-runtime-go generate/zz_filesystem_generated.go ## Update Scaffolding Runtimes
 
-pull-runtimes:
+update-runtime-go:
 	cd templates/go/scaffolding/instanced-http && go get -u knative.dev/func-go/http
 	cd templates/go/scaffolding/static-http && go get -u knative.dev/func-go/http
 	cd templates/go/scaffolding/instanced-cloudevents && go get -u knative.dev/func-go/cloudevents
 	cd templates/go/scaffolding/static-cloudevents && go get -u knative.dev/func-go/cloudevents
 
+
 .PHONY: cert
 certs: templates/certs/ca-certificates.crt ## Update root certificates
 

cmd/build.go

@@ -130,10 +130,7 @@ EXAMPLES
 
 	// Temporarily Hidden Basic Auth Flags
 	// Username, Password and Token flags, which plumb through basic auth, are
-	// currently only available on the experimental "host" builder, which is
-	// itself behind a feature flag FUNC_ENABLE_HOST_BUILDER.  So set these
-	// flags to hidden until it's out of preview and they are plumbed through
-	// the docker pusher as well.
+	// currently only available on the "host" builder.
 	_ = cmd.Flags().MarkHidden("username")
 	_ = cmd.Flags().MarkHidden("password")
 	_ = cmd.Flags().MarkHidden("token")
@@ -165,13 +162,13 @@ func runBuild(cmd *cobra.Command, _ []string, newClient ClientFactory) (err erro
 	if err = cfg.Validate(); err != nil { // Perform any pre-validation
 		return
 	}
-	if f, err = fn.NewFunction(cfg.Path); err != nil {
+	if f, err = fn.NewFunction(cfg.Path); err != nil { // Read in the Function
 		return
 	}
 	if !f.Initialized() {
 		return fn.NewErrNotInitialized(f.Root)
 	}
-	f = cfg.Configure(f) // Updates f at path to include build request values
+	f = cfg.Configure(f) // Returns an f updated with values from the config (flags, envs, etc)
 
 	cmd.SetContext(cfg.WithValues(cmd.Context())) // Some optional settings are passed via context
 
@@ -184,7 +181,7 @@ func runBuild(cmd *cobra.Command, _ []string, newClient ClientFactory) (err erro
 	defer done()
 
 	// Build
-	buildOptions, err := cfg.buildOptions()
+	buildOptions, err := cfg.buildOptions() // build-specific options from the finalized cfg
 	if err != nil {
 		return
 	}

cmd/deploy.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"os"
 	"strconv"
 	"strings"
 
@@ -205,10 +204,7 @@ EXAMPLES
 
 	// Temporarily Hidden Basic Auth Flags
 	// Username, Password and Token flags, which plumb through basic auth, are
-	// currently only available on the experimental "host" builder, which is
-	// itself behind a feature flag FUNC_ENABLE_HOST_BUILDER.  So set these
-	// flags to hidden until it's out of preview and they are plumbed through
-	// the docker pusher as well.
+	// currently only available on "host" builder.
 	_ = cmd.Flags().MarkHidden("username")
 	_ = cmd.Flags().MarkHidden("password")
 	_ = cmd.Flags().MarkHidden("token")
@@ -431,18 +427,7 @@ func KnownBuilders() builders.Known {
 	// However, future third-party integrations may support less than, or more
 	// builders, and certain environmental considerations may alter this list.
 
-	// Also a good place to stick feature-flags; to wit:
-	enable_host, _ := strconv.ParseBool(os.Getenv("FUNC_ENABLE_HOST_BUILDER"))
-	if !enable_host {
-		bb := []string{}
-		for _, b := range builders.All() {
-			if b != builders.Host {
-				bb = append(bb, b)
-			}
-		}
-		return bb
-	}
-
+	// Also a good place to stick feature-flags.
 	return builders.All()
 }
 

cmd/deploy_test.go

@@ -1531,7 +1531,6 @@ func testAuthentication(cmdFn commandConstructor, t *testing.T) {
 	}
 
 	cmd := cmdFn(NewTestClient(fn.WithPusher(pusher)))
-	t.Setenv("FUNC_ENABLE_HOST_BUILDER", "true") // host builder is currently behind this feature flag
 	cmd.SetArgs([]string{"--builder", "host", "--username", testUser, "--password", testPass})
 	if err := cmd.Execute(); err != nil {
 		t.Fatal(err)

cmd/func-util/main.go

@@ -78,7 +78,7 @@ func scaffold(ctx context.Context) error {
 		return fmt.Errorf("cannot load func project: %w", err)
 	}
 
-	if f.Runtime != "go" {
+	if f.Runtime != "go" && f.Runtime != "python" {
 		// Scaffolding is for now supported/needed only for Go.
 		return nil
 	}
@@ -100,7 +100,17 @@ func scaffold(ctx context.Context) error {
 		return fmt.Errorf("unable to create .s2i bin dir. %w", err)
 	}
 
-	if err := os.WriteFile(filepath.Join(f.Root, ".s2i", "bin", "assemble"), []byte(s2i.GoAssembler), 0755); err != nil {
+	var asm string
+	switch f.Runtime {
+	case "go":
+		asm = s2i.GoAssembler
+	case "python":
+		asm = s2i.PythonAssembler
+	default:
+		panic("unreachable")
+	}
+
+	if err := os.WriteFile(filepath.Join(f.Root, ".s2i", "bin", "assemble"), []byte(asm), 0755); err != nil {
 		return fmt.Errorf("unable to write go assembler. %w", err)
 	}
 

cmd/run.go

@@ -14,6 +14,7 @@ import (
 	"knative.dev/func/pkg/config"
 	"knative.dev/func/pkg/docker"
 	fn "knative.dev/func/pkg/functions"
+	"knative.dev/func/pkg/oci"
 )
 
 func NewRunCmd(newClient ClientFactory) *cobra.Command {
@@ -38,9 +39,14 @@ DESCRIPTION
 	  The --container flag indicates that the function's container should be
 	  run rather than running the source code directly.  This may require that
 	  the function's container first be rebuilt.  Building the container on or
-	  off can be altered using the --build flag.  The default value --build=auto
-	  indicates the system should automatically build the container only if
-	  necessary.
+	  off can be altered using the --build flag.  The value --build=auto
+	  can be used to indicate the function should be run in a container, with
+	  the container automatically built if necessary.
+
+	  The --container flag defaults to true if the builder defined for the
+	  function is a containerized builder such as Pack or S2I, and in the case
+	  where the function's runtime requires containerized builds (is not yet
+	  supported by the Host builder.
 
 	Process Scaffolding
 	  This is an Experimental Feature currently available only to Go projects.
@@ -103,6 +109,8 @@ EXAMPLES
 			"You may provide this flag multiple times for setting multiple environment variables. "+
 			"To unset, specify the environment variable name followed by a \"-\" (e.g., NAME-).")
 	cmd.Flags().Duration("start-timeout", f.Run.StartTimeout, fmt.Sprintf("time this function needs in order to start. If not provided, the client default %v will be in effect. ($FUNC_START_TIMEOUT)", fn.DefaultStartTimeout))
+	cmd.Flags().BoolP("container", "t", runContainerizedByDefault(f),
+		"Run the function in a container. ($FUNC_CONTAINER)")
 
 	// TODO: Without the "Host" builder enabled, this code-path is unreachable,
 	// so remove hidden flag when either the Host builder path is available,
@@ -116,8 +124,6 @@ EXAMPLES
 	cmd.Flags().String("build", "auto",
 		"Build the function. [auto|true|false]. ($FUNC_BUILD)")
 	cmd.Flags().Lookup("build").NoOptDefVal = "true" // register `--build` as equivalient to `--build=true`
-	cmd.Flags().BoolP("container", "t", true,
-		"Run the function in a container. ($FUNC_CONTAINER)")
 
 	// Oft-shared flags:
 	addConfirmFlag(cmd, cfg.Confirm)
@@ -136,14 +142,17 @@ EXAMPLES
 	return cmd
 }
 
+func runContainerizedByDefault(f fn.Function) bool {
+	return f.Build.Builder == "pack" || f.Build.Builder == "s2i" || !oci.IsSupported(f.Runtime)
+}
+
 func runRun(cmd *cobra.Command, newClient ClientFactory) (err error) {
 	var (
 		cfg runConfig
 		f   fn.Function
 	)
-	if cfg, err = newRunConfig(cmd).Prompt(); err != nil {
-		return
-	}
+	cfg = newRunConfig(cmd) // Will add Prompt on upcoming UX refactor
+
 	if f, err = fn.NewFunction(cfg.Path); err != nil {
 		return
 	}
@@ -340,15 +349,8 @@ func (c runConfig) Validate(cmd *cobra.Command, f fn.Function) (err error) {
 		}
 	}
 
-	// There is currently no local host runner implemented, so specifying
-	// --container=false should return an informative error for runtimes other
-	// than Go  that is more helpful than the cryptic, though correct, error
-	// from the Client that it was instantated without a runner.
-	// TODO: modify this check when the local host runner is available to
-	// only generate this error when --container==false && the --language is
-	// not yet implemented.
-	if !c.Container && f.Runtime != "go" {
-		return errors.New("the ability to run functions outside of a container via 'func run' is coming soon.")
+	if !c.Container && !oci.IsSupported(f.Runtime) {
+		return fmt.Errorf("The %q runtime currently requires being run in a container", f.Runtime)
 	}
 
 	// When the docker runner respects the StartTimeout, this validation check

cmd/templates_test.go

@@ -27,9 +27,7 @@ go           http
 node         cloudevents
 node         http
 python       cloudevents
-python       flask
 python       http
-python       wsgi
 quarkus      cloudevents
 quarkus      http
 rust         cloudevents
@@ -67,9 +65,7 @@ func TestTemplates_JSON(t *testing.T) {
   ],
   "python": [
     "cloudevents",
-    "flask",
-    "http",
-    "wsgi"
+    "http"
   ],
   "quarkus": [
     "cloudevents",

docs/reference/func_build.md

@@ -58,7 +58,7 @@ func build
 
 ```
       --build-timestamp        Use the actual time as the created time for the docker image. This is only useful for buildpacks builder.
-  -b, --builder string         Builder to use when creating the function's container. Currently supported builders are "pack" and "s2i". ($FUNC_BUILDER) (default "s2i")
+  -b, --builder string         Builder to use when creating the function's container. Currently supported builders are "host", "pack" and "s2i". ($FUNC_BUILDER) (default "s2i")
       --builder-image string   Specify a custom builder image for use by the builder other than its default. ($FUNC_BUILDER_IMAGE)
   -c, --confirm                Prompt to confirm options interactively ($FUNC_CONFIRM)
   -h, --help                   help for build