[pip] Add pip support (via venv) DEV-1146 (#147)

mikeland73 · gcurtis · web-flow · commit 9cea339cd3d4 · 2022-09-23T15:44:40.000-07:00
## Summary This is stacked on #105. Please review that one first. Adds basic pip support. * Uses requirements.txt for detection. * Uses venv for shell * Uses venv + pex for build Does not yet libraries with native extensions like `pandas` but there's a plan for that. ## How was it tested? * `devbox shell`. * `devbox build` * Unit tests Co-authored-by: Greg Curtis <greg.curtis@jetpack.io>
diff --git a/README.md b/README.md
@@ -142,7 +142,7 @@ Devbox makes it easy to package your application into an OCI-compliant container
 Devbox currently detects the following languages/stacks:
 
 - Go
-- Python (Poetry)
+- Python ([See notes](planner/languages/python/))
 - NodeJS
 - PHP
 - static nginx (Experimental - [see notes](planner/languages/nginx))
diff --git a/devbox.go b/devbox.go
@@ -140,7 +140,7 @@ func (d *Devbox) Shell() error {
 		return errors.WithStack(err)
 	}
 	nixDir := filepath.Join(d.srcDir, ".devbox/gen/shell.nix")
-	sh, err := nix.DetectShell(nix.WithWelcomeMessage(plan.ShellWelcomeMessage))
+	sh, err := nix.DetectShell(nix.WithPlanInitHook(plan.ShellInitHook))
 	if err != nil {
 		// Fall back to using a plain Nix shell.
 		sh = &nix.Shell{}
diff --git a/devbox_test.go b/devbox_test.go
@@ -12,6 +12,7 @@ import (
 )
 
 func TestDevbox(t *testing.T) {
+	t.Setenv("TMPDIR", "/tmp")
 	testPaths, err := doublestar.FilepathGlob("./testdata/**/devbox.json")
 	assert.NoError(t, err, "Reading testdata/ should not fail")
 
@@ -89,7 +90,7 @@ func assertPlansMatch(t *testing.T, expected *plansdk.Plan, actual *plansdk.Plan
 	)
 
 	assert.ElementsMatch(expected.Definitions, actual.Definitions, "Definitions should match")
-	assert.Equal(expected.ShellWelcomeMessage, actual.ShellWelcomeMessage, "ShellWelcomeMessage should match")
+	assert.Equal(expected.ShellInitHook, actual.ShellInitHook, "ShellInitHook should match")
 	if expected.GeneratedFiles != nil {
 		assert.Equal(expected.GeneratedFiles, actual.GeneratedFiles, "GeneratedFiles should match")
 	}
diff --git a/nix/shell.go b/nix/shell.go
@@ -37,10 +37,10 @@ type Shell struct {
 	name            name
 	binPath         string
 	userShellrcPath string
+	planInitHook    string
 
 	// UserInitHook contains commands that will run at shell startup.
-	UserInitHook   string
-	welcomeMessage string
+	UserInitHook string
 }
 
 type ShellOption func(*Shell)
@@ -91,9 +91,9 @@ func DetectShell(opts ...ShellOption) (*Shell, error) {
 	return sh, nil
 }
 
-func WithWelcomeMessage(message string) ShellOption {
+func WithPlanInitHook(hook string) ShellOption {
 	return func(s *Shell) {
-		s.welcomeMessage = message
+		s.planInitHook = hook
 	}
 }
 
@@ -249,12 +249,12 @@ func (s *Shell) writeDevboxShellrc() (path string, err error) {
 		OriginalInit     string
 		OriginalInitPath string
 		UserHook         string
-		WelcomeMessage   string
+		PlanInitHook     string
 	}{
 		OriginalInit:     string(bytes.TrimSpace(userShellrc)),
 		OriginalInitPath: filepath.Clean(s.userShellrcPath),
 		UserHook:         strings.TrimSpace(s.UserInitHook),
-		WelcomeMessage:   strings.TrimSpace(s.welcomeMessage),
+		PlanInitHook:     strings.TrimSpace(s.planInitHook),
 	})
 	if err != nil {
 		return "", fmt.Errorf("execute shellrc template: %v", err)
diff --git a/nix/shell_test.go b/nix/shell_test.go
@@ -51,7 +51,7 @@ func TestWriteDevboxShellrc(t *testing.T) {
 			s := &Shell{
 				userShellrcPath: test.shellrcPath,
 				UserInitHook:    test.hook,
-				welcomeMessage:  "Welcome to the devbox!",
+				planInitHook:    `echo "Welcome to the devbox!"`,
 			}
 			gotPath, err := s.writeDevboxShellrc()
 			if err != nil {
diff --git a/nix/shellrc.tmpl b/nix/shellrc.tmpl
@@ -75,12 +75,12 @@ export PS1="(devbox) $PS1"
 
 {{- end }}
 
-# Begin Shell Welcome Message
+# Begin Plan Init Hook
 
-{{- if .WelcomeMessage }}
+{{- if .PlanInitHook }}
 
-echo "{{ .WelcomeMessage }}"
+{{ .PlanInitHook }}
 
 {{- end }}
 
-# End Shell Welcome Message
+# End Plan Init Hook
diff --git a/nix/testdata/shellrc/basic/shellrc.golden b/nix/testdata/shellrc/basic/shellrc.golden
@@ -85,8 +85,8 @@ echo "Hello from a devbox shell hook!"
 
 # End Devbox User Hook
 
-# Begin Shell Welcome Message
+# Begin Plan Init Hook
 
 echo "Welcome to the devbox!"
 
-# End Shell Welcome Message
+# End Plan Init Hook
diff --git a/nix/testdata/shellrc/nohook/shellrc.golden b/nix/testdata/shellrc/nohook/shellrc.golden
@@ -79,8 +79,8 @@ export PS1="(devbox) $PS1"
 
 # End Devbox Post-init Hook
 
-# Begin Shell Welcome Message
+# Begin Plan Init Hook
 
 echo "Welcome to the devbox!"
 
-# End Shell Welcome Message
+# End Plan Init Hook
diff --git a/nix/testdata/shellrc/noshellrc/shellrc.golden b/nix/testdata/shellrc/noshellrc/shellrc.golden
@@ -43,8 +43,8 @@ echo "Hello from a devbox shell hook!"
 
 # End Devbox User Hook
 
-# Begin Shell Welcome Message
+# Begin Plan Init Hook
 
 echo "Welcome to the devbox!"
 
-# End Shell Welcome Message
+# End Plan Init Hook
diff --git a/planner/languages/nginx/nginx_planner.go b/planner/languages/nginx/nginx_planner.go
@@ -29,7 +29,9 @@ func (p *Planner) IsRelevant(srcDir string) bool {
 
 func (p *Planner) GetPlan(srcDir string) *plansdk.Plan {
 	return &plansdk.Plan{
-		ShellWelcomeMessage: fmt.Sprintf(welcomeMessage, p.shellConfig(srcDir)),
+		ShellInitHook: plansdk.WelcomeMessage(
+			fmt.Sprintf(welcomeMessage, p.shellConfig(srcDir)),
+		),
 		DevPackages: []string{
 			"nginx",
 			"shell-nginx",
diff --git a/planner/languages/python/README.md b/planner/languages/python/README.md
@@ -0,0 +1,41 @@
+# Python Planner
+
+* We currently support shell and build using poetry and pip. We generally recommend using poetry, but will do our best to support both. 
+
+# Python Poetry Planner
+
+## Detection
+
+* This planner looks for `poetry.lock` or `pyproject.toml` in your devbox.json directory.
+
+## Shell
+
+* poetry by default uses virtual environment so there should be no changes to dev workflow.
+
+## Build
+
+This planner uses pex to build an executable that has all dependencies. It looks for an entrypoint in the following order:
+
+* If there's a module with same name as the project, it uses that as the entrypoint.
+* If there's a script with same name as project, it uses that as the entrypoint.
+* Use first script in alphabetical order as the entrypoint.
+
+# Python Pip Planner
+
+## Detection
+
+* Looks for `requirements.txt` in your devbox.json directory. We default to python3 as provided by nix packages.
+
+## Shell
+
+* Uses venv (automatically created in .venv) to create a virtual environment.
+
+## Build
+
+* Uses pex to build an executable that has all dependencies. 
+* Requires barebones `setup.py`
+* Uses `__main__` module in package that matches the lowercase `setup.py` name. Any dashes in the name are replaced with underscores.
+
+# Limitations
+
+Currently `build` does not support projects that depend on libraries with native extensions (e.g. `pandas`).
diff --git a/planner/languages/python/python_pip_planner.go b/planner/languages/python/python_pip_planner.go
@@ -0,0 +1,85 @@
+// Copyright 2022 Jetpack Technologies Inc and contributors. All rights reserved.
+// Use of this source code is governed by the license in the LICENSE file.
+
+package python
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"go.jetpack.io/devbox/boxcli/usererr"
+	"go.jetpack.io/devbox/planner/plansdk"
+)
+
+// TODO: Doesn't work with libraries like Pandas that have C extensions
+// We get error
+// ImportError: libstdc++.so.6: cannot open shared object file: No such file or directory
+// possible solution is to set $LD_LIBRARY_PATH
+// https://nixos.wiki/wiki/Packaging/Quirks_and_Caveats
+type PIPPlanner struct{}
+
+// PythonPoetryPlanner implements interface Planner (compile-time check)
+var _ plansdk.Planner = (*PIPPlanner)(nil)
+
+func (p *PIPPlanner) Name() string {
+	return "python.Planner"
+}
+
+func (p *PIPPlanner) IsRelevant(srcDir string) bool {
+	return plansdk.FileExists(filepath.Join(srcDir, "requirements.txt"))
+}
+func (p *PIPPlanner) GetPlan(srcDir string) *plansdk.Plan {
+	plan := &plansdk.Plan{
+		DevPackages: []string{
+			"python3",
+		},
+		RuntimePackages: []string{
+			`python3`,
+		},
+		ShellInitHook: p.shellInitHook(srcDir),
+	}
+	if err := p.isBuildable(srcDir); err != nil {
+		return plan.WithError(err)
+	}
+	plan.InstallStage = &plansdk.Stage{
+		Command:    "python -m venv .venv && source .venv/bin/activate && pip install -r requirements.txt",
+		InputFiles: plansdk.AllFiles(),
+	}
+	plan.BuildStage = &plansdk.Stage{Command: pipBuildCommand}
+	plan.StartStage = &plansdk.Stage{
+		Command:    "python ./app.pex",
+		InputFiles: []string{"app.pex"},
+	}
+	return plan
+}
+
+func (p *PIPPlanner) isBuildable(srcDir string) error {
+	if plansdk.FileExists(filepath.Join(srcDir, "setup.py")) {
+		return nil
+	}
+
+	return usererr.New(
+		"setup.py not found. Please create a setup.py file to build your project." +
+			" The distribution name must be a case-insensitive match of the package" +
+			" (dir) name. Dashes are converted to underscores.",
+	)
+}
+
+func (p *PIPPlanner) shellInitHook(srcDir string) string {
+	venvPath := filepath.Join(srcDir, ".venv")
+	venvActivatePath := filepath.Join(srcDir, ".venv", "bin", "activate")
+	script := strings.TrimSpace(`
+echo "Creating/Using virtual environment in %[1]s";
+python -m venv "%[1]s";
+source "%[2]s";`)
+	return fmt.Sprintf(script, venvPath, venvActivatePath)
+}
+
+var pipBuildCommand = strings.TrimSpace(`
+source .venv/bin/activate && \
+pip install pex && \
+PACKAGE_NAME=$(python setup.py --name |  tr '[:upper:]-' '[:lower:]_') && \
+pex . -o app.pex -m $PACKAGE_NAME -r requirements.txt
+`,
+)
diff --git a/planner/languages/python/python_poetry_planner.go b/planner/languages/python/python_poetry_planner.go
@@ -16,21 +16,21 @@ import (
 	"golang.org/x/exp/slices"
 )
 
-type Planner struct{}
+type PoetryPlanner struct{}
 
 // PythonPoetryPlanner implements interface Planner (compile-time check)
-var _ plansdk.Planner = (*Planner)(nil)
+var _ plansdk.Planner = (*PoetryPlanner)(nil)
 
-func (p *Planner) Name() string {
+func (p *PoetryPlanner) Name() string {
 	return "python.Planner"
 }
 
-func (p *Planner) IsRelevant(srcDir string) bool {
+func (p *PoetryPlanner) IsRelevant(srcDir string) bool {
 	return plansdk.FileExists(filepath.Join(srcDir, "poetry.lock")) ||
 		plansdk.FileExists(filepath.Join(srcDir, "pyproject.toml"))
 }
 
-func (p *Planner) GetPlan(srcDir string) *plansdk.Plan {
+func (p *PoetryPlanner) GetPlan(srcDir string) *plansdk.Plan {
 	version := p.PythonVersion(srcDir)
 	pythonPkg := fmt.Sprintf("python%s", version.MajorMinorConcatenated())
 	plan := &plansdk.Plan{
@@ -62,7 +62,7 @@ func (p *Planner) GetPlan(srcDir string) *plansdk.Plan {
 }
 
 // TODO: This can be generalized to all python planners
-func (p *Planner) PythonVersion(srcDir string) *plansdk.Version {
+func (p *PoetryPlanner) PythonVersion(srcDir string) *plansdk.Version {
 	defaultVersion, _ := plansdk.NewVersion("3.10.6")
 	project := p.PyProject(srcDir)
 
@@ -76,7 +76,7 @@ func (p *Planner) PythonVersion(srcDir string) *plansdk.Version {
 	return defaultVersion
 }
 
-func (p *Planner) buildCommand(srcDir string) string {
+func (p *PoetryPlanner) buildCommand(srcDir string) string {
 	project := p.PyProject(srcDir)
 	// Assume name follows https://peps.python.org/pep-0508/#names
 	// Do simple replacement "-" -> "_" and check if any script matches name.
@@ -113,7 +113,7 @@ type pyProject struct {
 	} `toml:"tool"`
 }
 
-func (p *Planner) PyProject(srcDir string) *pyProject {
+func (p *PoetryPlanner) PyProject(srcDir string) *pyProject {
 	pyProjectPath := filepath.Join(srcDir, "pyproject.toml")
 	content, err := os.ReadFile(pyProjectPath)
 	if err != nil {
@@ -124,7 +124,7 @@ func (p *Planner) PyProject(srcDir string) *pyProject {
 	return &proj
 }
 
-func (p *Planner) isBuildable(srcDir string) (bool, error) {
+func (p *PoetryPlanner) isBuildable(srcDir string) (bool, error) {
 	project := p.PyProject(srcDir)
 	if project == nil {
 		return false, usererr.New("Could not build container for python " +
@@ -164,7 +164,7 @@ func (p *Planner) isBuildable(srcDir string) (bool, error) {
 	return true, nil
 }
 
-func (p *Planner) formatBuildCommand(module, script string) string {
+func (p *PoetryPlanner) formatBuildCommand(module, script string) string {
 
 	// If no scripts, just run the module directly always.
 	if script == "" {
diff --git a/planner/planner.go b/planner/planner.go
@@ -59,7 +59,8 @@ var PLANNERS = []plansdk.Planner{
 	&ocaml.Planner{},
 	&perl.Planner{},
 	&php.Planner{},
-	&python.Planner{},
+	&python.PoetryPlanner{},
+	&python.PIPPlanner{},
 	&ruby.Planner{},
 	&rust.Planner{},
 	&scala.Planner{},
diff --git a/planner/plansdk/plansdk.go b/planner/plansdk/plansdk.go
@@ -5,6 +5,7 @@ package plansdk
 
 import (
 	"encoding/json"
+	"fmt"
 	"os"
 
 	"github.com/imdario/mergo"
@@ -26,7 +27,7 @@ type PlanError struct {
 
 // Plan tells devbox how to start shells and build projects.
 type Plan struct {
-	ShellWelcomeMessage string `json:"shell_welcome_message,omitempty"`
+	ShellInitHook string `json:"shell_init_hook,omitempty"`
 
 	NixOverlays []string `cur:"[...string]" json:"nix_overlays,omitempty"`
 
@@ -183,3 +184,7 @@ func FileExists(path string) bool {
 	_, err := os.Stat(path)
 	return err == nil
 }
+
+func WelcomeMessage(s string) string {
+	return fmt.Sprintf(`echo "%s";`, s)
+}
diff --git a/testdata/nginx/plan.json b/testdata/nginx/plan.json
@@ -20,5 +20,8 @@
   "definitions": [
     "\nshell-nginx = pkgs.writeShellScriptBin \"shell-nginx\" ''\n\necho \"Starting nginx with command:\"\necho \"nginx -p testdata/nginx -c shell-nginx.conf -e /tmp/error.log -g \\\"pid /tmp/mynginx.pid;daemon off;\\\"\"\nnginx -p testdata/nginx -c shell-nginx.conf -e /tmp/error.log -g \"pid /tmp/shell-nginx.pid;daemon off;\"\n'';"
   ],
-  "shell_welcome_message": "\n##### WARNING: nginx planner is experimental #####\n\nYou may need to add\n\n\\\"include ./.devbox/gen/shell-helper-nginx.conf;\\\"\n\nto your shell-nginx.conf file to ensure the server can start in the nix shell.\n\nUse \\\"shell-nginx\\\" to start the server\n"
+  "shell_init_hook": "echo \"\n##### WARNING: nginx planner is experimental #####\n\nYou may need to add\n\n\\\"include ./.devbox/gen/shell-helper-nginx.conf;\\\"\n\nto your shell-nginx.conf file to ensure the server can start in the nix shell.\n\nUse \\\"shell-nginx\\\" to start the server\n\";",
+  "generated_files": {
+    "shell-helper-nginx.conf": "access_log /tmp/access.log;\nclient_body_temp_path /tmp/client_body;\nproxy_temp_path /tmp/proxy;\nfastcgi_temp_path /tmp/fastcgi;\nuwsgi_temp_path /tmp/uwsgi;\nscgi_temp_path /tmp/scgi;\n"
+  }
 }
diff --git a/testdata/python/pip-example/.gitignore b/testdata/python/pip-example/.gitignore
@@ -0,0 +1 @@
+.venv
diff --git a/testdata/python/pip-example/devbox.json b/testdata/python/pip-example/devbox.json
@@ -0,0 +1,6 @@
+{
+  "packages": [],
+  "shell": {
+    "init_hook": null
+  }
+}
diff --git a/testdata/python/pip-example/mymod/__main__.py b/testdata/python/pip-example/mymod/__main__.py
diff --git a/testdata/python/pip-example/plan.json b/testdata/python/pip-example/plan.json
diff --git a/testdata/python/pip-example/requirements.txt b/testdata/python/pip-example/requirements.txt
diff --git a/testdata/python/pip-example/setup.py b/testdata/python/pip-example/setup.py
diff --git a/tmpl/Dockerfile.dockerignore.tmpl b/tmpl/Dockerfile.dockerignore.tmpl

Original file line number	Diff line number	Diff line change
`@@ -140,7 +140,7 @@ func (d *Devbox) Shell() error {`
`140`	`140`	`return errors.WithStack(err)`
`141`	`141`	`}`
`142`	`142`	`nixDir := filepath.Join(d.srcDir, ".devbox/gen/shell.nix")`
`143`		`- sh, err := nix.DetectShell(nix.WithWelcomeMessage(plan.ShellWelcomeMessage))`
	`143`	`+ sh, err := nix.DetectShell(nix.WithPlanInitHook(plan.ShellInitHook))`
`144`	`144`	`if err != nil {`
`145`	`145`	`// Fall back to using a plain Nix shell.`
`146`	`146`	`sh = &nix.Shell{}`
Original file line number	Diff line number	Diff line change
`@@ -12,6 +12,7 @@ import (`
`12`	`12`	`)`
`13`	`13`
`14`	`14`	`func TestDevbox(t *testing.T) {`
	`15`	`+ t.Setenv("TMPDIR", "/tmp")`
`15`	`16`	`testPaths, err := doublestar.FilepathGlob("./testdata/**/devbox.json")`
`16`	`17`	`assert.NoError(t, err, "Reading testdata/ should not fail")`
`17`	`18`
`@@ -89,7 +90,7 @@ func assertPlansMatch(t testing.T, expected plansdk.Plan, actual *plansdk.Plan`
`89`	`90`	`)`
`90`	`91`
`91`	`92`	`assert.ElementsMatch(expected.Definitions, actual.Definitions, "Definitions should match")`
`92`		`- assert.Equal(expected.ShellWelcomeMessage, actual.ShellWelcomeMessage, "ShellWelcomeMessage should match")`
	`93`	`+ assert.Equal(expected.ShellInitHook, actual.ShellInitHook, "ShellInitHook should match")`
`93`	`94`	`if expected.GeneratedFiles != nil {`
`94`	`95`	`assert.Equal(expected.GeneratedFiles, actual.GeneratedFiles, "GeneratedFiles should match")`
`95`	`96`	`}`
Original file line number	Diff line number	Diff line change
`@@ -51,7 +51,7 @@ func TestWriteDevboxShellrc(t *testing.T) {`
`51`	`51`	`s := &Shell{`
`52`	`52`	`userShellrcPath: test.shellrcPath,`
`53`	`53`	`UserInitHook: test.hook,`
`54`		`- welcomeMessage: "Welcome to the devbox!",`
	`54`	+ planInitHook: `echo "Welcome to the devbox!"`,
`55`	`55`	`}`
`56`	`56`	`gotPath, err := s.writeDevboxShellrc()`
`57`	`57`	`if err != nil {`