[python] Improve python poetry planner to auto-detect main module (#84)

## Summary

This PR allows the poetry planner to detect and use `__main__` module.
It does this by

1) Using the pypoetry.toml and auto-detect directories to try and find
the `__main__.py`
2) It uses an inline python script in the build phase to determine the
correct flag to pass to pex. (This part is a bit hacky).

I'm not 100% sure this will work in all cases, but it works in more
cases than the previous implementation.

Unrelated fix: fixed how we combine errors when plan fails.

cc: @Lagoja 

## How was it tested?

* `devbox build` with console script only
* `devbox build` with `__main__` module only
* `devbox build` without scripts or `__main__` module (failed, showed
correct error).

Author: mikeland73

devbox_test.go

@@ -53,6 +53,9 @@ func testIndividualPlan(t *testing.T, testPath string) {
 		// For now we only compare the DevPackages and RuntimePackages fields:
 		assert.ElementsMatch(expected.DevPackages, plan.DevPackages, "DevPackages should match")
 		assert.ElementsMatch(expected.RuntimePackages, plan.RuntimePackages, "RuntimePackages should match")
+		assert.Equal(expected.InstallStage.GetCommand(), plan.InstallStage.GetCommand(), "Install stage should match")
+		assert.Equal(expected.BuildStage.GetCommand(), plan.BuildStage.GetCommand(), "Build stage should match")
+		assert.Equal(expected.StartStage.GetCommand(), plan.StartStage.GetCommand(), "Start stage should match")
 	})
 }
 

examples/python-poetry/python_poetry/__main__.py

@@ -0,0 +1,4 @@
+from python_poetry.server import run_server
+
+print("Running server using __main__ module")
+run_server()

planner/plan.go

@@ -52,6 +52,13 @@ type Stage struct {
 	InputFiles []string `cue:"[...string]" json:"input_files,omitempty"`
 }
 
+func (s *Stage) GetCommand() string {
+	if s == nil {
+		return ""
+	}
+	return s.Command
+}
+
 func (p *Plan) String() string {
 	b, err := json.MarshalIndent(p, "", "  ")
 	if err != nil {
@@ -85,11 +92,11 @@ func (p *Plan) Error() error {
 	if len(p.Errors) == 0 {
 		return nil
 	}
-	var err error = p.Errors[0]
-	for _, err = range p.Errors[1:] {
-		err = errors.Wrap(err, err.Error())
+	combined := p.Errors[0].error
+	for _, err := range p.Errors[1:] {
+		combined = errors.Wrap(combined, err.Error())
 	}
-	return err
+	return combined
 }
 
 func (p *Plan) WithError(err error) *Plan {

planner/python_poetry_planner.go

@@ -42,10 +42,7 @@ func (g *PythonPoetryPlanner) GetPlan(srcDir string) *Plan {
 	if buildable, err := g.isBuildable(srcDir); !buildable {
 		return plan.WithError(err)
 	}
-	entrypoint, err := g.GetEntrypoint(srcDir)
-	if err != nil {
-		return plan.WithError(err)
-	}
+
 	plan.InstallStage = &Stage{
 		// pex is is incompatible with certain less common python versions,
 		// but because versions are sometimes expressed open-ended (e.g. ^3.10)
@@ -54,12 +51,8 @@ func (g *PythonPoetryPlanner) GetPlan(srcDir string) *Plan {
 		Command: "poetry add pex -n --no-ansi && " +
 			"poetry install --no-dev -n --no-ansi",
 	}
-	plan.BuildStage = &Stage{
-		Command: "PEX_ROOT=/tmp/.pex poetry run pex . -o app.pex --script " + entrypoint,
-	}
-	plan.StartStage = &Stage{
-		Command: "PEX_ROOT=/tmp/.pex python ./app.pex",
-	}
+	plan.BuildStage = &Stage{Command: g.buildCommand(srcDir)}
+	plan.StartStage = &Stage{Command: "python ./app.pex"}
 	return plan
 }
 
@@ -78,20 +71,25 @@ func (g *PythonPoetryPlanner) PythonVersion(srcDir string) *version {
 	return defaultVersion
 }
 
-func (g *PythonPoetryPlanner) GetEntrypoint(srcDir string) (string, error) {
+func (g *PythonPoetryPlanner) buildCommand(srcDir string) string {
 	project := g.PyProject(srcDir)
 	// Assume name follows https://peps.python.org/pep-0508/#names
 	// Do simple replacement "-" -> "_" and check if any script matches name.
 	// This could be improved.
 	moduleName := strings.ReplaceAll(project.Tool.Poetry.Name, "-", "_")
 	if _, ok := project.Tool.Poetry.Scripts[moduleName]; ok {
-		return moduleName, nil
+		// return moduleName, nil
+		return g.formatBuildCommand(moduleName, moduleName)
 	}
 	// otherwise use the first script alphabetically
 	// (go-toml doesn't preserve order, we could parse ourselves)
 	scripts := maps.Keys(project.Tool.Poetry.Scripts)
 	slices.Sort(scripts)
-	return scripts[0], nil
+	script := ""
+	if len(scripts) > 0 {
+		script = scripts[0]
+	}
+	return g.formatBuildCommand(moduleName, script)
 }
 
 type pyProject struct {
@@ -101,6 +99,10 @@ type pyProject struct {
 			Dependencies struct {
 				Python string `toml:"python"`
 			} `toml:"dependencies"`
+			Packages []struct {
+				Include string `toml:"include"`
+				From    string `toml:"from"`
+			} `toml:"packages"`
 			Scripts map[string]string `toml:"scripts"`
 		} `toml:"poetry"`
 	} `toml:"tool"`
@@ -124,11 +126,62 @@ func (g *PythonPoetryPlanner) isBuildable(srcDir string) (bool, error) {
 			"application. pyproject.toml is missing and needed to install python " +
 			"dependencies.")
 	}
+
+	// is this the right way to determine package name?
+	packageName := strings.ReplaceAll(project.Tool.Poetry.Name, "-", "_")
+
+	// First try to find a __main__ module as entry point
+	if len(project.Tool.Poetry.Packages) > 0 {
+		// If package has custom directory, check that.
+		// Using packages disables auto-detection of __main__ module.
+		for _, pkg := range project.Tool.Poetry.Packages {
+			if pkg.Include == packageName &&
+				fileExists(filepath.Join(srcDir, pkg.From, pkg.Include, "__main__.py")) {
+				return true, nil
+			}
+		}
+
+		// Use setup tools auto-detect directory structure
+	} else if fileExists(filepath.Join(srcDir, packageName, "__main__.py")) ||
+		fileExists(filepath.Join(srcDir, "src", packageName, "__main__.py")) {
+
+		return true, nil
+	}
+
+	// Fallback to using poetry scripts
 	if len(project.Tool.Poetry.Scripts) == 0 {
 		return false,
-			usererr.New("Project is not buildable: no scripts found in " +
-				"pyproject.toml. Please define a script to use as an entrypoint for " +
-				"your app:\n\n[tool.poetry.scripts]\nmy_app = \"my_app:my_function\"\n")
+			usererr.New(
+				"Project is not buildable: no __main__.py file found and " +
+					"no scripts defined in pyproject.toml",
+			)
 	}
 	return true, nil
 }
+
+func (g *PythonPoetryPlanner) formatBuildCommand(module, script string) string {
+
+	// If no scripts, just run the module directly always.
+	if script == "" {
+		return fmt.Sprintf(
+			"poetry run pex . -o app.pex -m %s --validate-entry-point",
+			module,
+		)
+	}
+
+	entrypointScript := fmt.Sprintf(
+		`$(poetry run python -c "import pkgutil;
+import %[1]s;
+modules = [name for _, name, _ in pkgutil.iter_modules(%[1]s.__path__)];
+print('-m %[1]s' if '__main__' in modules else '--script %[2]s');")
+`,
+		module,
+		script,
+	)
+
+	return fmt.Sprintf(
+		"poetry run pex . -o app.pex %s --validate-entry-point &>/dev/null || "+
+			"(echo 'Build failed. Could not find entrypoint' && exit 1)",
+		strings.TrimSpace(strings.ReplaceAll(entrypointScript, "\n", "")),
+	)
+}

testdata/go/go-1.17/plan.json

@@ -2,5 +2,14 @@
   "dev_packages": [
     "go_1_17"
   ],
-  "runtime_packages": []
-}
+  "runtime_packages": [],
+  "install_stage": {
+    "command": "go get"
+  },
+  "build_stage": {
+    "command": "CGO_ENABLED=0 go build -o app"
+  },
+  "start_stage": {
+    "command": "./app"
+  }
+}

testdata/go/go-1.18/plan.json

@@ -2,5 +2,14 @@
   "dev_packages": [
     "go"
   ],
-  "runtime_packages": []
-}
+  "runtime_packages": [],
+  "install_stage": {
+    "command": "go get"
+  },
+  "build_stage": {
+    "command": "CGO_ENABLED=0 go build -o app"
+  },
+  "start_stage": {
+    "command": "./app"
+  }
+}

testdata/go/go-1.19/plan.json

@@ -2,5 +2,14 @@
   "dev_packages": [
     "go_1_19"
   ],
-  "runtime_packages": []
-}
+  "runtime_packages": [],
+  "install_stage": {
+    "command": "go get"
+  },
+  "build_stage": {
+    "command": "CGO_ENABLED=0 go build -o app"
+  },
+  "start_stage": {
+    "command": "./app"
+  }
+}

testdata/python/poetry-no-lock-file/plan.json

@@ -6,4 +6,4 @@
   "runtime_packages": [
     "python310"
   ]
-}
+}

testdata/python/poetry-with-custom-dir-structure/devbox.json

@@ -0,0 +1,3 @@
+{
+  "packages": []
+}