[tests] Fix docs about example tests (#1302)

## Summary
We were abusing the "example test" term to really mean "run `devbox run
run_test` on any devbox.json". Updated code and docs to reflect reality,
since it was confusing while I was debugging a failed test.

## How was it tested?
`DEVBOX_RUN_DEVBOX_JSON_TESTS=1 go test testscripts [-run
TestExamples/flake_remote_run_test`

Author: ipince

.github/workflows/cli-tests.yaml

@@ -19,7 +19,7 @@ on:
         type: boolean
   workflow_dispatch:
     inputs:
-      run-example-tests:
+      run-devbox-json-tests:
         type: boolean
       run-mac-tests:
         type: boolean
@@ -73,15 +73,15 @@ jobs:
           - ${{ github.ref == 'refs/heads/main' }}
         os: [ubuntu-latest, macos-latest]
         # This is an optimization that runs tests twice, with and without
-        # the examples. We can require non-example tests to complete before
+        # the devbox.json tests. We can require the other tests to complete before
         # merging, while keeping the others as an additional non-required signal
-        run-example-tests: [true, false]
+        run-devbox-json-tests: [true, false]
         exclude:
           - is-main: false
             os: "${{ inputs.run-mac-tests && 'dummy' || 'macos-latest' }}"
           - is-main: true
-            run-example-tests: false
-          - run-example-tests: true
+            run-devbox-json-tests: false
+          - run-devbox-json-tests: true
             os: macos-latest
     runs-on: ${{ matrix.os }}
     timeout-minutes: ${{ (github.ref == 'refs/heads/main' || inputs.run-mac-tests) && 37 || 20 }}
@@ -108,10 +108,10 @@ jobs:
           nix-build-user-count: 4
       - name: Run tests
         env:
-          # For example tests, we default to non-debug mode since the debug output is less useful than for unit testscripts.
+          # For devbox.json tests, we default to non-debug mode since the debug output is less useful than for unit testscripts.
           # But we allow overriding via inputs.example-debug
-          DEVBOX_DEBUG: ${{ (!matrix.run-example-tests || inputs.example-debug) && '1' || '0' }}
-          DEVBOX_EXAMPLE_TESTS: ${{ matrix.run-example-tests }}
+          DEVBOX_DEBUG: ${{ (!matrix.run-devbox-json-tests || inputs.example-debug) && '1' || '0' }}
+          DEVBOX_RUN_DEVBOX_JSON_TESTS: ${{ matrix.run-devbox-json-tests }}
           # Used in `go test -timeout` flag. Needs a value that time.ParseDuration can parse.
           DEVBOX_GOLANG_TEST_TIMEOUT: "${{ (github.ref == 'refs/heads/main' || inputs.run-mac-tests) && '35m' || '20m' }}"
         run: |

testscripts/testrunner/examplesrunner.go

@@ -22,17 +22,18 @@ import (
 // virtenvs of devbox plugins in this directory. We need to use a custom
 // path that is intentionally short, since some plugins store unix sockets in
 // their virtenv and unix sockets require their paths to be short.
-const xdgStateHomeDir = "/tmp/devbox-example-testscripts"
+const xdgStateHomeDir = "/tmp/devbox-testscripts"
 
-// RunExamplesTestscripts generates testscripts for each example devbox-project.
-func RunExamplesTestscripts(t *testing.T, examplesDir string) {
+// RunDevboxTestscripts generates and runs a testscript test for each Devbox project in dir.
+// For each project, runs `devbox run run_test` (if script exists) and asserts it succeeds.
+func RunDevboxTestscripts(t *testing.T, dir string) {
 	// ensure the state home dir for devbox exists
 	err := os.MkdirAll(xdgStateHomeDir, 0700)
 	if err != nil && !errors.Is(err, fs.ErrNotExist) {
 		t.Error(err)
 	}
 
-	err = filepath.WalkDir(examplesDir, func(path string, entry os.DirEntry, err error) error {
+	err = filepath.WalkDir(dir, func(path string, entry os.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
@@ -75,16 +76,16 @@ func RunExamplesTestscripts(t *testing.T, examplesDir string) {
 		}
 
 		t.Logf("running testscript for example: %s\n", path)
-		runSingleExampleTestscript(t, examplesDir, path)
+		runSingleDevboxTestscript(t, dir, path)
 		return nil
 	})
 	if err != nil {
 		t.Error(err)
 	}
 }
 
-func runSingleExampleTestscript(t *testing.T, examplesDir, projectDir string) {
-	testscriptDir, err := generateTestscript(t, examplesDir, projectDir)
+func runSingleDevboxTestscript(t *testing.T, dir, projectDir string) {
+	testscriptDir, err := generateTestscript(t, dir, projectDir)
 	if err != nil {
 		t.Error(err)
 	}
@@ -127,10 +128,10 @@ func runSingleExampleTestscript(t *testing.T, examplesDir, projectDir string) {
 }
 
 // generateTestscript will create a temp-directory and place the generic
-// testscript file (.test.txt) for all examples devbox-projects in it.
+// testscript file (.test.txt) for all devbox-projects in the dir.
 // It returns the directory containing the testscript file.
-func generateTestscript(t *testing.T, examplesDir, projectDir string) (string, error) {
-	testPath, err := filepath.Rel(examplesDir, projectDir)
+func generateTestscript(t *testing.T, dir, projectDir string) (string, error) {
+	testPath, err := filepath.Rel(dir, projectDir)
 	if err != nil {
 		return "", errors.WithStack(err)
 	}

testscripts/testscripts_test.go

@@ -8,9 +8,13 @@ import (
 	"go.jetpack.io/devbox/testscripts/testrunner"
 )
 
-const exampleTestsEnvName = "DEVBOX_EXAMPLE_TESTS"
+// When true, tests that `devbox run run_test` succeeds on every devbox.json
+// found in examples/.. and testscripts/..
+const runDevboxJSONTests = "DEVBOX_RUN_DEVBOX_JSON_TESTS"
 
 func TestScripts(t *testing.T) {
+	// To run a specific test, say, testscripts/foo/bar.test.text, then run
+	// go test ./testscripts -run TestScripts/bar
 	testrunner.RunTestscripts(t, ".")
 }
 
@@ -20,19 +24,22 @@ func TestMain(m *testing.M) {
 
 // TestExamples runs testscripts on the devbox-projects in the examples folder.
 func TestExamples(t *testing.T) {
-	isOn, err := strconv.ParseBool(os.Getenv(exampleTestsEnvName))
+	isOn, err := strconv.ParseBool(os.Getenv(runDevboxJSONTests))
 	if err != nil || !isOn {
-		t.Skipf("Skipping TestExamples. To enable, set %s=1.", exampleTestsEnvName)
+		t.Skipf("Skipping TestExamples. To enable, set %s=1.", runDevboxJSONTests)
 	}
 
-	testrunner.RunExamplesTestscripts(t, "../examples")
+	// To run a specific test, say, examples/foo/bar, then run
+	// go test ./testscripts -run TestExamples/foo_bar_run_test
+	testrunner.RunDevboxTestscripts(t, "../examples")
 }
 
+// TestScriptsWithDevboxJSON runs testscripts on the devbox-projects in the testscripts folder.
 func TestScriptsWithDevboxJSON(t *testing.T) {
-	isOn, err := strconv.ParseBool(os.Getenv(exampleTestsEnvName))
+	isOn, err := strconv.ParseBool(os.Getenv(runDevboxJSONTests))
 	if err != nil || !isOn {
-		t.Skipf("Skipping TestExamples. To enable, set %s=1.", exampleTestsEnvName)
+		t.Skipf("Skipping TestExamples. To enable, set %s=1.", runDevboxJSONTests)
 	}
 
-	testrunner.RunExamplesTestscripts(t, ".")
+	testrunner.RunDevboxTestscripts(t, ".")
 }