diff --git a/rules/builtins.build_defs b/rules/builtins.build_defs index 7380ef8a6..490ebf013 100644 --- a/rules/builtins.build_defs +++ b/rules/builtins.build_defs @@ -2,18 +2,65 @@ # Do not change the order of arguments to this function without updating the iota in targets.go to match it. -def build_rule(name:str, cmd:str|dict='', test_cmd:str|dict='', debug_cmd:str='', srcs:list|dict=None, data:list|dict=None, - debug_data:list|dict=None, outs:list|dict=None, deps:list=None, exported_deps:list=None, secrets:list|dict=None, - tools:str|list|dict=None, test_tools:str|list|dict=None, debug_tools:str|list|dict=None, labels:list=None, - visibility:list=CONFIG.DEFAULT_VISIBILITY, hashes:list=None, binary:bool=False, test:bool=False, - test_only:bool=CONFIG.DEFAULT_TESTONLY, building_description:str=None, needs_transitive_deps:bool=False, - output_is_complete:bool=False, sandbox:bool=CONFIG.BUILD_SANDBOX, test_sandbox:bool=CONFIG.TEST_SANDBOX, - no_test_output:bool=False, flaky:bool|int=0, build_timeout:int|str=0, test_timeout:int|str=0, pre_build:function=None, - post_build:function=None, requires:list=None, provides:dict=None, licences:list=CONFIG.DEFAULT_LICENCES, - test_outputs:list=None, system_srcs:list=None, stamp:bool=False, tag:str='', optional_outs:list=None, progress:bool=False, - size:str=None, _urls:list=None, internal_deps:list=None, pass_env:list=None, local:bool=False, output_dirs:list=[], - exit_on_error:bool=CONFIG.EXIT_ON_ERROR, entry_points:dict={}, env:dict={}, _file_content:str=None, - _subrepo:bool=False, no_test_coverage:bool=False): +def build_rule( + name:str, + cmd:str|dict="", + test_cmd:str|dict="", + debug_cmd:str="", + srcs:list|dict=None, + data:list|dict=None, + debug_data:list|dict=None, + outs:list|dict=None, + deps:list=None, + exported_deps:list=None, + secrets:list|dict=None, + tools:str|list|dict=None, + test_tools:str|list|dict=None, + debug_tools:str|list|dict=None, + labels:list=None, + visibility:list=CONFIG.DEFAULT_VISIBILITY, + hashes:list=None, + binary:bool=False, + test:bool=False, + test_only:bool=CONFIG.DEFAULT_TESTONLY, + building_description:str=None, + needs_transitive_deps:bool=False, + output_is_complete:bool=False, + sandbox:bool=CONFIG.BUILD_SANDBOX, + test_sandbox:bool=CONFIG.TEST_SANDBOX, + no_test_output:bool=False, + flaky:bool|int=0, + build_timeout:int|str=0, + test_timeout:int|str=0, + pre_build:function=None, + post_build:function=None, + requires:list=None, + provides:dict=None, + licences:list=CONFIG.DEFAULT_LICENCES, + test_outputs:list=None, + system_srcs:list=None, + stamp:bool=False, + tag:str="", + optional_outs:list=None, + progress:bool=False, + size:str=None, + _urls:list=None, + internal_deps:list=None, + pass_env:list=None, + local:bool=False, + output_dirs:list=[], + exit_on_error:bool=CONFIG.EXIT_ON_ERROR, + entry_points:dict={}, + env:dict={}, + _file_content:str=None, + _subrepo:bool=False, + no_test_coverage:bool=False, + # This matches the default `BuildEntrypoint` is defined in + #`src/core/build_entrypoint.go`. + build_entry_point:list=None, + build_entry_point_exit_on_error_args:list=None, + build_entry_point_interactive_args:list=None, + build_entry_point_exec_command_args:list=None): pass def chr(i:int) -> str: diff --git a/src/build/build_step.go b/src/build/build_step.go index b2c8e4c05..7d05ce29b 100644 --- a/src/build/build_step.go +++ b/src/build/build_step.go @@ -3,6 +3,7 @@ package build import ( "bytes" + "context" "encoding/hex" "errors" "fmt" @@ -515,12 +516,28 @@ func runBuildCommand(state *core.BuildState, target *core.BuildTarget, command s if target.IsTextFile { return nil, buildTextFile(state, target) } + env := core.StampedBuildEnvironment(state, target, inputHash, filepath.Join(core.RepoRoot, target.TmpDir()), target.Stamp).ToSlice() log.Debug("Building target %s\nENVIRONMENT:\n%s\n%s", target.Label, env, command) - out, combined, err := state.ProcessExecutor.ExecWithTimeoutShell(target, target.TmpDir(), env, target.BuildTimeout, state.ShowAllOutput, false, process.NewSandboxConfig(target.Sandbox, target.Sandbox), command) + + buildArgvOpts := []core.BuildArgvOpt{target.BuildEntryPoint.WithBuildArgvCommand(command)} + if target.ShouldExitOnError() { + buildArgvOpts = append(buildArgvOpts, target.BuildEntryPoint.WithBuildArgvExitOnError()) + } + argv, err := target.BuildEntryPoint.BuildArgv(state, target, buildArgvOpts...) + if err != nil { + return nil, err + } + out, combined, err := state.ProcessExecutor.ExecWithTimeout( + context.Background(), + target, target.TmpDir(), env, target.BuildTimeout, state.ShowAllOutput, + false, false, false, process.NewSandboxConfig(target.Sandbox, target.Sandbox), + argv, + ) if err != nil { return nil, fmt.Errorf("Error building target %s: %s\n%s", target.Label, err, combined) } + return out, nil } diff --git a/src/build/incrementality_test.go b/src/build/incrementality_test.go index dfb35d8ac..10c2b3713 100644 --- a/src/build/incrementality_test.go +++ b/src/build/incrementality_test.go @@ -26,6 +26,7 @@ var KnownFields = map[string]bool{ "IsTextFile": true, "FileContent": true, "IsRemoteFile": true, + "BuildEntryPoint": true, "Command": true, "Commands": true, "NeedsTransitiveDependencies": true, diff --git a/src/core/build_entrypoint.go b/src/core/build_entrypoint.go new file mode 100644 index 000000000..52159facc --- /dev/null +++ b/src/core/build_entrypoint.go @@ -0,0 +1,96 @@ +package core + +type BuildEntrypoint struct { + Entrypoint []string + ExecCommandArgs []string + ExitOnErrorArgs []string + InteractiveArgs []string +} + +type BuildEntrypointOpt func(*BuildEntrypoint) + +func WithBuildEntrypointEntrypoint(entrypoint []string) BuildEntrypointOpt { + return func(be *BuildEntrypoint) { + be.Entrypoint = entrypoint + } +} + +func WithBuildEntrypointExitOnErrorArgs(args []string) BuildEntrypointOpt { + return func(be *BuildEntrypoint) { + be.ExitOnErrorArgs = args + } +} + +func WithBuildEntrypointExecCommandArgs(args []string) BuildEntrypointOpt { + return func(be *BuildEntrypoint) { + be.ExecCommandArgs = args + } +} + +func WithBuildEntrypointInteractiveArgs(args []string) BuildEntrypointOpt { + return func(be *BuildEntrypoint) { + be.InteractiveArgs = args + } +} + +func NewBuildEntrypoint(opts ...BuildEntrypointOpt) *BuildEntrypoint { + be := &BuildEntrypoint{ + Entrypoint: []string{}, + ExecCommandArgs: []string{}, + ExitOnErrorArgs: []string{}, + InteractiveArgs: []string{}, + } + + for _, opt := range opts { + opt(be) + } + + // Default to Bash if Entrypoint not set. + if len(be.Entrypoint) < 1 { + be.Entrypoint = []string{"bash", "--noprofile", "--norc", "-u", "-o", "pipefail"} + be.ExecCommandArgs = []string{"-c"} + be.ExitOnErrorArgs = []string{"-e"} + be.InteractiveArgs = []string{} + } + + return be +} + +type BuildArgv struct{ Argv []string } +type BuildArgvOpt func(*BuildArgv) + +func (be *BuildEntrypoint) WithBuildArgvExitOnError() BuildArgvOpt { + return func(ba *BuildArgv) { + ba.Argv = append(ba.Argv, be.ExitOnErrorArgs...) + } +} + +func (be *BuildEntrypoint) WithBuildArgvInteractive() BuildArgvOpt { + return func(ba *BuildArgv) { + log.Debugf("pre interactive argv: %#v", ba.Argv) + ba.Argv = append(ba.Argv, be.InteractiveArgs...) + + log.Debugf("post interactive argv: %#v", ba.Argv) + } +} + +func (be *BuildEntrypoint) WithBuildArgvCommand(command string) BuildArgvOpt { + return func(ba *BuildArgv) { + ba.Argv = append(ba.Argv, append(be.ExecCommandArgs, command)...) + } +} + +func (be *BuildEntrypoint) BuildArgv(buildState *BuildState, target *BuildTarget, opts ...BuildArgvOpt) ([]string, error) { + argv := &BuildArgv{Argv: be.Entrypoint} + for _, opt := range opts { + opt(argv) + } + + newArg0, err := ReplaceSequences(buildState, target, argv.Argv[0]) + if err != nil { + return nil, err + } + argv.Argv[0] = newArg0 + + return argv.Argv, nil +} diff --git a/src/core/build_entrypoint_test.go b/src/core/build_entrypoint_test.go new file mode 100644 index 000000000..be3d8d1ba --- /dev/null +++ b/src/core/build_entrypoint_test.go @@ -0,0 +1,75 @@ +package core + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewBuildEntrypoint(t *testing.T) { + var tests = []struct { + description string + opts []BuildEntrypointOpt + expected *BuildEntrypoint + }{ + { + "DefaultConfigIsBash", + nil, + &BuildEntrypoint{ + Entrypoint: []string{"bash", "--noprofile", "--norc", "-u", "-o", "pipefail"}, + ExecCommandArgs: []string{"-c"}, + ExitOnErrorArgs: []string{"-e"}, + InteractiveArgs: []string{}, + }, + }, + { + "NuShell", + []BuildEntrypointOpt{ + WithBuildEntrypointEntrypoint([]string{"nu", "--no-config-file", "--no-history"}), + WithBuildEntrypointInteractiveArgs([]string{"--execute", "$env.config.show_banner = false"}), + WithBuildEntrypointExecCommandArgs([]string{"--commands"}), + }, + &BuildEntrypoint{ + Entrypoint: []string{"nu", "--no-config-file", "--no-history"}, + ExecCommandArgs: []string{"--commands"}, + ExitOnErrorArgs: []string{}, + InteractiveArgs: []string{"--execute", "$env.config.show_banner = false"}, + }, + }, + { + "Powershell", + []BuildEntrypointOpt{ + WithBuildEntrypointEntrypoint([]string{"pwsh", "-NoProfile"}), + WithBuildEntrypointInteractiveArgs([]string{"-Interactive"}), + WithBuildEntrypointExecCommandArgs([]string{"-Command"}), + }, + &BuildEntrypoint{ + Entrypoint: []string{"pwsh", "-NoProfile"}, + ExecCommandArgs: []string{"-Command"}, + ExitOnErrorArgs: []string{}, + InteractiveArgs: []string{"-Interactive"}, + }, + }, + { + "Elvish", + []BuildEntrypointOpt{ + WithBuildEntrypointEntrypoint([]string{"elvish", "-norc"}), + WithBuildEntrypointInteractiveArgs([]string{"-i"}), + WithBuildEntrypointExecCommandArgs([]string{"-c"}), + }, + &BuildEntrypoint{ + Entrypoint: []string{"elvish", "-norc"}, + ExecCommandArgs: []string{"-c"}, + ExitOnErrorArgs: []string{}, + InteractiveArgs: []string{"-i"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.description, func(t *testing.T) { + actualBec := NewBuildEntrypoint(tt.opts...) + assert.Equal(t, tt.expected, actualBec) + }) + } +} diff --git a/src/core/build_target.go b/src/core/build_target.go index 5e523be46..d2fef1581 100644 --- a/src/core/build_target.go +++ b/src/core/build_target.go @@ -134,7 +134,9 @@ type BuildTarget struct { OptionalOutputs []string `name:"optional_outs"` // Optional labels applied to this rule. Used for including/excluding rules. Labels []string - // Shell command to run. + // Build Entrypoint configuration. + BuildEntryPoint *BuildEntrypoint `name:"build_entry_point" hide:"filegroup"` + // Shell command to run, this is passed as the last argument to the Binary. Command string `name:"cmd" hide:"filegroup"` // Per-configuration shell commands to run. Commands map[string]string `name:"cmd" hide:"filegroup"` @@ -384,6 +386,7 @@ func NewBuildTarget(label BuildLabel) *BuildTarget { state: int32(Inactive), BuildingDescription: DefaultBuildingDescription, finishedBuilding: make(chan struct{}), + BuildEntryPoint: NewBuildEntrypoint(), } } diff --git a/src/output/shell_output.go b/src/output/shell_output.go index 613361db4..5fba650a1 100644 --- a/src/output/shell_output.go +++ b/src/output/shell_output.go @@ -443,11 +443,16 @@ func printTempDirs(state *core.BuildState, duration time.Duration, shell, shellR fmt.Printf(" Expanded: %s\n", os.Expand(cmd, env.ReplaceEnvironment)) } else { fmt.Printf("\n") - argv := []string{"bash", "--noprofile", "--norc", "-o", "pipefail"} + buildArgvOpts := []core.BuildArgvOpt{target.BuildEntryPoint.WithBuildArgvInteractive()} if shellRun { - argv = append(argv, "-c", cmd) + buildArgvOpts = append(buildArgvOpts, target.BuildEntryPoint.WithBuildArgvCommand(cmd)) } - log.Debug("Full command: %s", strings.Join(argv, " ")) + argv, err := target.BuildEntryPoint.BuildArgv(state, target, buildArgvOpts...) + if err != nil { + log.Errorf("Could not build shell args: %s", err) + } + + log.Debug("Full command(shellRun: %v): %#v", shellRun, argv) cmd := state.ProcessExecutor.ExecCommand(process.NewSandboxConfig(shouldSandbox, shouldSandbox), false, argv[0], argv[1:]...) cmd.Dir = dir cmd.Env = append(cmd.Env, env.ToSlice()...) diff --git a/src/parse/asp/targets.go b/src/parse/asp/targets.go index d916a1a13..7145392a5 100644 --- a/src/parse/asp/targets.go +++ b/src/parse/asp/targets.go @@ -72,6 +72,10 @@ const ( fileContentArgIdx subrepoArgIdx noTestCoverageArgIdx + buildEntryPointArgIdx + buildEntryPointExitOnErrorArgsArgIdx + buildEntryPointInteractiveArgsArgIdx + buildEntryPointExecCommandArgsArgIdx ) // createTarget creates a new build target as part of build_rule(). @@ -140,6 +144,8 @@ func createTarget(s *scope, args []pyObject) *core.BuildTarget { target.AddLabel("remote") } target.Command, target.Commands = decodeCommands(s, args[cmdBuildRuleArgIdx]) + target.BuildEntryPoint = decodeBuildEntrypointFromArgs(s, args) + if test { target.Test = new(core.TestFields) @@ -252,6 +258,37 @@ func decodeCommands(s *scope, obj pyObject) (string, map[string]string) { return "", m } +// decodeBuildEntrypointFromArgs takes a Python object and returns it as a BuildEntrypoint. +func decodeBuildEntrypointFromArgs(s *scope, args []pyObject) *core.BuildEntrypoint { + buildEntrypointOpts := []core.BuildEntrypointOpt{} + + if obj := args[buildEntryPointArgIdx]; obj != nil && obj != None { + buildEntrypointOpts = append(buildEntrypointOpts, + core.WithBuildEntrypointEntrypoint(asStringList(s, mustList(obj), "build_entry_point")), + ) + } + + if obj := args[buildEntryPointExecCommandArgsArgIdx]; obj != nil && obj != None { + buildEntrypointOpts = append(buildEntrypointOpts, + core.WithBuildEntrypointExecCommandArgs(asStringList(s, mustList(obj), "build_entry_point_exec_command_args")), + ) + } + + if obj := args[buildEntryPointExitOnErrorArgsArgIdx]; obj != nil && obj != None { + buildEntrypointOpts = append(buildEntrypointOpts, + core.WithBuildEntrypointExitOnErrorArgs(asStringList(s, mustList(obj), "build_entry_point_exit_on_error_args")), + ) + } + + if obj := args[buildEntryPointInteractiveArgsArgIdx]; obj != nil && obj != None { + buildEntrypointOpts = append(buildEntrypointOpts, + core.WithBuildEntrypointInteractiveArgs(asStringList(s, mustList(obj), "build_entry_point_interactive_args")), + ) + } + + return core.NewBuildEntrypoint(buildEntrypointOpts...) +} + // populateTarget sets the assorted attributes on a build target. func populateTarget(s *scope, t *core.BuildTarget, args []pyObject) { if t.IsRemoteFile { diff --git a/test/build_entrypoints/BUILD b/test/build_entrypoints/BUILD new file mode 100644 index 000000000..09fba7f2a --- /dev/null +++ b/test/build_entrypoints/BUILD @@ -0,0 +1,107 @@ +build_rule( + name = "default_entrypoint", + outs = ["default_entrypoint.txt"], + # Default is Bash + cmd = "printf '%s' \"$NAME\" > $OUTS", +) + +sh_test( + name = "default_entrypoint_test", + src = "test_build_entrypoint.sh", + data = [":default_entrypoint"], +) + +build_rule( + name = "bash_entrypoint", + outs = ["bash_entrypoint.txt"], + build_entry_point = ["bash", "--noprofile", "--norc", "-u", "-o", "pipefail"], + build_entry_point_exit_on_error_args = ["-e"], + build_entry_point_exec_command_args = ["-c"], + cmd = "printf '%s' \"$NAME\" > $OUTS", +) + +sh_test( + name = "bash_entrypoint_test", + src = "test_build_entrypoint.sh", + data = [":bash_entrypoint"], +) + +subinclude("///go//build_defs:go") + +go_binary( + name = "main", + srcs = ["main.go"], +) + +build_rule( + name = "binary_entrypoint", + outs = ["binary_entrypoint.txt"], + build_entry_point = ["$(exe :main)"], + tools = [":main"], +) + +sh_test( + name = "binary_entrypoint_test", + src = "test_build_entrypoint.sh", + data = [":binary_entrypoint"], +) + +linux_libc="gnu" +if CONFIG.OS == "linux": + genrule( + name = "libc_impl", + outs = ["libc_impl"], + cmd = """ + lddVersionOut="$(ldd --version 2>&1 || true)" + if echo "$lddVersionOut" | grep -i "GNU\|GLIBC"; then + echo "libc='gnu'" > $OUTS + exit + fi + + if echo "$lddVersionOut" | grep -i "musl"; then + echo "libc='musl'" > $OUTS + exit + fi + + echo "$lddVersionOut" + exit 1 + """, + ) + + subinclude(":libc_impl") + linux_libc=libc + + +nushell_version = "0.104.1" +nushell_release_asset_suffix_by_os_arch = { + "linux_amd64": f"-x86_64-unknown-linux-{linux_libc}.tar.gz", + "linux_arm64": f"-aarch64-unknown-linux-{linux_libc}.tar.gz", + "darwin_amd64": "-x86_64-apple-darwin.tar.gz", + "darwin_arm64": "-aarch64-apple-darwin.tar.gz", +} +nushell_release_asset_suffix = nushell_release_asset_suffix_by_os_arch[CONFIG.OS+"_"+CONFIG.ARCH] +nushell_release_asset_extracted_suffix = nushell_release_asset_suffix.split(".")[0] + +remote_file( + name = "nu", + url = f"https://github.com/nushell/nushell/releases/download/{nushell_version}/nu-{nushell_version}{nushell_release_asset_suffix}", + extract = True, + binary = True, + exported_files = [f"nu-{nushell_version}{nushell_release_asset_extracted_suffix}/nu"], +) + +build_rule( + name = "nushell_entrypoint", + outs = ["nushell_entrypoint.txt"], + build_entry_point = ["$(exe :nu)", "--no-config-file", "--no-history"], + build_entry_point_interactive_args = ["--execute", "$env.config.show_banner = false"], + build_entry_point_exec_command_args = ["--commands"], + cmd = "$env.NAME | save $env.OUTS", + tools = [":nu"], +) + +sh_test( + name = "nushell_entrypoint_test", + src = "test_build_entrypoint.sh", + data = [":nushell_entrypoint"], +) diff --git a/test/build_entrypoints/main.go b/test/build_entrypoints/main.go new file mode 100644 index 000000000..955d2727d --- /dev/null +++ b/test/build_entrypoints/main.go @@ -0,0 +1,12 @@ +package main + +import ( + "log" + "os" +) + +func main() { + if err := os.WriteFile(os.Getenv("OUT"), []byte(os.Getenv("NAME")), 0600); err != nil { + log.Fatalf("write '%s': %s", os.Getenv("OUT"), err) + } +} diff --git a/test/build_entrypoints/test_build_entrypoint.sh b/test/build_entrypoints/test_build_entrypoint.sh new file mode 100644 index 000000000..f620e3d8b --- /dev/null +++ b/test/build_entrypoints/test_build_entrypoint.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# This script tests that the Please test name exists in the expected output +# file for an entrypoint. +# This expects the pattern of build rules with Build Entrypoints to output a +# file with the name of their build target into a file with the name of the +# build target. +set -Eeuo pipefail + +exec grep "${NAME//_test/}" "${PKG_DIR}/${NAME//_test/}.txt"