test: Faster tests by being consistent with what counts as TEST_ACCEPTANCE

Makefile

@@ -9,9 +9,13 @@ lint:
 format:
 	scripts/run_formatters.sh
 
+test-short:
+	scripts/run_tests.sh -short
+
 test:
 	scripts/run_tests.sh
 
+
 clean:
 	rm -f osv-scanner
 	rm -r cmd/osv-scanner/scan/image/testdata/test-*.tar

cmd/osv-scanner/fix/__snapshots__/command_test.snap

@@ -11206,7 +11206,83 @@ Rewriting <tempdir>/package-lock.json...
 
 ---
 
-[TestCommand/fix_non_interactive_in_place_package_lock_json_with_offline_vulns - 1]
+[TestCommand/fix_non_interactive_override_pom_xml_with_native_data_source - 1]
+Resolving <tempdir>/pom.xml...
+Found 12 vulnerabilities matching the filter
+Can fix 12/12 matching vulnerabilities by overriding 4 dependencies
+OVERRIDE-PACKAGE: org.apache.httpcomponents:httpclient,4.5.13
+OVERRIDE-PACKAGE: org.codehaus.plexus:plexus-utils,3.0.24
+OVERRIDE-PACKAGE: commons-io:commons-io,2.14.0
+OVERRIDE-PACKAGE: org.jsoup:jsoup,1.15.3
+FIXED-VULN-IDS: GHSA-2x83-r56g-cv47,GHSA-78wr-2p64-hpwj,GHSA-7r82-7xv7-xcpj,GHSA-8vhq-qq4p-grq3,GHSA-cfh5-3ghh-wfjx,GHSA-fmj5-wv96-r2ch,GHSA-g6ph-x5wf-g337,GHSA-gp7f-rwcx-9369,GHSA-gw85-4gmf-m7rh,GHSA-gwrp-pvrq-jmwv,GHSA-jcwr-x25h-x5fh,GHSA-m72m-mhq2-9p6c
+REMAINING-VULNS: 0
+UNFIXABLE-VULNS: 0
+Rewriting <tempdir>/pom.xml...
+
+---
+
+[TestCommand/fix_non_interactive_override_pom_xml_with_native_data_source - 2]
+
+---
+
+[TestCommand/fix_non_interactive_override_pom_xml_with_native_data_source - 3]
+<project>
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>dev.osv</groupId>
+  <artifactId>osv-fix</artifactId>
+  <version>1</version>
+
+  <properties>
+    <httpclient.version>4.5.13</httpclient.version>
+  </properties>
+
+    <dependencyManagement>
+      <dependencies>
+        <dependency>
+          <groupId>commons-io</groupId>
+          <artifactId>commons-io</artifactId>
+          <version>2.14.0</version>
+        </dependency>
+        <dependency>
+          <groupId>org.jsoup</groupId>
+          <artifactId>jsoup</artifactId>
+          <version>1.15.3</version>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.httpcomponents</groupId>
+          <artifactId>httpclient</artifactId>
+          <version>${httpclient.version}</version>
+        </dependency>
+      </dependencies>
+    </dependencyManagement>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.maven.wagon</groupId>
+      <artifactId>wagon-http</artifactId>
+      <version>3.0.0</version>
+    </dependency>
+    <dependency>
+      <groupId>org.codehaus.plexus</groupId>
+      <artifactId>plexus-utils</artifactId>
+      <version>3.0.24</version>
+    </dependency>
+  </dependencies>
+</project>
+
+---
+
+[TestCommand/no_args_provided - 1]
+
+---
+
+[TestCommand/no_args_provided - 2]
+manifest or lockfile is required
+
+---
+
+[TestCommandOffline/fix_non_interactive_in_place_package_lock_json_with_offline_vulns - 1]
 Loaded npm local db from <tempdir>/osv-scanner/npm/all.zip
 Scanning <tempdir>/package-lock.json...
 Found 9 vulnerabilities matching the filter
@@ -11221,11 +11297,11 @@ Rewriting <tempdir>/package-lock.json...
 
 ---
 
-[TestCommand/fix_non_interactive_in_place_package_lock_json_with_offline_vulns - 2]
+[TestCommandOffline/fix_non_interactive_in_place_package_lock_json_with_offline_vulns - 2]
 
 ---
 
-[TestCommand/fix_non_interactive_in_place_package_lock_json_with_offline_vulns - 3]
+[TestCommandOffline/fix_non_interactive_in_place_package_lock_json_with_offline_vulns - 3]
 {
   "name": "osv-fix",
   "version": "1.0.0",
@@ -12951,74 +13027,7 @@ Rewriting <tempdir>/package-lock.json...
 
 ---
 
-[TestCommand/fix_non_interactive_override_pom_xml_with_native_data_source - 1]
-Resolving <tempdir>/pom.xml...
-Found 12 vulnerabilities matching the filter
-Can fix 12/12 matching vulnerabilities by overriding 4 dependencies
-OVERRIDE-PACKAGE: org.apache.httpcomponents:httpclient,4.5.13
-OVERRIDE-PACKAGE: org.codehaus.plexus:plexus-utils,3.0.24
-OVERRIDE-PACKAGE: commons-io:commons-io,2.14.0
-OVERRIDE-PACKAGE: org.jsoup:jsoup,1.15.3
-FIXED-VULN-IDS: GHSA-2x83-r56g-cv47,GHSA-78wr-2p64-hpwj,GHSA-7r82-7xv7-xcpj,GHSA-8vhq-qq4p-grq3,GHSA-cfh5-3ghh-wfjx,GHSA-fmj5-wv96-r2ch,GHSA-g6ph-x5wf-g337,GHSA-gp7f-rwcx-9369,GHSA-gw85-4gmf-m7rh,GHSA-gwrp-pvrq-jmwv,GHSA-jcwr-x25h-x5fh,GHSA-m72m-mhq2-9p6c
-REMAINING-VULNS: 0
-UNFIXABLE-VULNS: 0
-Rewriting <tempdir>/pom.xml...
-
----
-
-[TestCommand/fix_non_interactive_override_pom_xml_with_native_data_source - 2]
-
----
-
-[TestCommand/fix_non_interactive_override_pom_xml_with_native_data_source - 3]
-<project>
-  <modelVersion>4.0.0</modelVersion>
-
-  <groupId>dev.osv</groupId>
-  <artifactId>osv-fix</artifactId>
-  <version>1</version>
-
-  <properties>
-    <httpclient.version>4.5.13</httpclient.version>
-  </properties>
-
-    <dependencyManagement>
-      <dependencies>
-        <dependency>
-          <groupId>commons-io</groupId>
-          <artifactId>commons-io</artifactId>
-          <version>2.14.0</version>
-        </dependency>
-        <dependency>
-          <groupId>org.jsoup</groupId>
-          <artifactId>jsoup</artifactId>
-          <version>1.15.3</version>
-        </dependency>
-        <dependency>
-          <groupId>org.apache.httpcomponents</groupId>
-          <artifactId>httpclient</artifactId>
-          <version>${httpclient.version}</version>
-        </dependency>
-      </dependencies>
-    </dependencyManagement>
-
-  <dependencies>
-    <dependency>
-      <groupId>org.apache.maven.wagon</groupId>
-      <artifactId>wagon-http</artifactId>
-      <version>3.0.0</version>
-    </dependency>
-    <dependency>
-      <groupId>org.codehaus.plexus</groupId>
-      <artifactId>plexus-utils</artifactId>
-      <version>3.0.24</version>
-    </dependency>
-  </dependencies>
-</project>
-
----
-
-[TestCommand/fix_non_interactive_relax_package_json_with_offline_vulns - 1]
+[TestCommandOffline/fix_non_interactive_relax_package_json_with_offline_vulns - 1]
 Loaded npm local db from <tempdir>/osv-scanner/npm/all.zip
 Resolving <tempdir>/package.json...
 Found 6 vulnerabilities matching the filter
@@ -13031,11 +13040,11 @@ Rewriting <tempdir>/package.json...
 
 ---
 
-[TestCommand/fix_non_interactive_relax_package_json_with_offline_vulns - 2]
+[TestCommandOffline/fix_non_interactive_relax_package_json_with_offline_vulns - 2]
 
 ---
 
-[TestCommand/fix_non_interactive_relax_package_json_with_offline_vulns - 3]
+[TestCommandOffline/fix_non_interactive_relax_package_json_with_offline_vulns - 3]
 {
   "name": "osv-fix",
   "version": "1.0.0",
@@ -13052,12 +13061,3 @@ Rewriting <tempdir>/package.json...
 }
 
 ---
-
-[TestCommand/no_args_provided - 1]
-
----
-
-[TestCommand/no_args_provided - 2]
-manifest or lockfile is required
-
----

cmd/osv-scanner/fix/command_test.go

@@ -25,6 +25,8 @@ func matchFile(t *testing.T, file string) {
 func TestCommand(t *testing.T) {
 	t.Parallel()
 
+	testutility.SkipIfShort(t)
+
 	tests := []testcmd.Case{
 		{
 			Name: "no_args_provided",
@@ -36,11 +38,6 @@ func TestCommand(t *testing.T) {
 			Args: []string{"", "fix", "--strategy=in-place", "-L", "./testdata/in-place-npm/package-lock.json"},
 			Exit: 0,
 		},
-		{
-			Name: "fix_non_interactive_in_place_package_lock_json_with_offline_vulns",
-			Args: []string{"", "fix", "--strategy=in-place", "--offline-vulnerabilities", "--download-offline-databases", "-L", "./testdata/in-place-npm/package-lock.json"},
-			Exit: 0,
-		},
 		{
 			Name: "fix_non_interactive_in_place_package_lock_json_with_native_data_source",
 			Args: []string{"", "fix", "--strategy=in-place", "--data-source", "native", "-L", "./testdata/in-place-npm/package-lock.json"},
@@ -51,11 +48,6 @@ func TestCommand(t *testing.T) {
 			Args: []string{"", "fix", "--strategy=relax", "-M", "./testdata/relax-npm/package.json"},
 			Exit: 0,
 		},
-		{
-			Name: "fix_non_interactive_relax_package_json_with_offline_vulns",
-			Args: []string{"", "fix", "--strategy=relax", "--offline-vulnerabilities", "--download-offline-databases", "-M", "./testdata/relax-npm/package.json"},
-			Exit: 0,
-		},
 		{
 			Name: "fix non-interactive override pom.xml",
 			Args: []string{"", "fix", "--strategy=override", "-M", "./testdata/override-maven/pom.xml"},
@@ -139,6 +131,45 @@ func TestCommand(t *testing.T) {
 	}
 }
 
+func TestCommandOffline(t *testing.T) {
+	t.Parallel()
+
+	testutility.SkipIfShort(t)
+
+	tests := []testcmd.Case{
+		{
+			Name: "fix_non_interactive_in_place_package_lock_json_with_offline_vulns",
+			Args: []string{"", "fix", "--strategy=in-place", "--offline-vulnerabilities", "--download-offline-databases", "-L", "./testdata/in-place-npm/package-lock.json"},
+			Exit: 0,
+		},
+		{
+			Name: "fix_non_interactive_relax_package_json_with_offline_vulns",
+			Args: []string{"", "fix", "--strategy=relax", "--offline-vulnerabilities", "--download-offline-databases", "-M", "./testdata/relax-npm/package.json"},
+			Exit: 0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.Name, func(t *testing.T) {
+			t.Parallel()
+
+			// fix action overwrites files, copy them to a temporary directory
+			testDir := testutility.CreateTestDir(t)
+
+			lockfile := testcmd.CopyFileFlagTo(t, tt, "-L", testDir)
+			manifest := testcmd.CopyFileFlagTo(t, tt, "-M", testDir)
+
+			testcmd.RunAndMatchSnapshots(t, tt)
+
+			if lockfile != "" {
+				matchFile(t, lockfile)
+			}
+			if manifest != "" {
+				matchFile(t, manifest)
+			}
+		})
+	}
+}
+
 func parseFlags(t *testing.T, flags []string, arguments []string) (*cli.Command, error) {
 	// This is a bit hacky: make a mock App with only the flags we care about.
 	// Then use app.RunAndMatchSnapshots() to parse the flags into the cli.Context, which is returned.

cmd/osv-scanner/internal/testcmd/run.go

@@ -53,6 +53,8 @@ func run(t *testing.T, tc Case) (string, string) {
 
 	if ec != tc.Exit {
 		t.Errorf("cli exited with code %d, not %d", ec, tc.Exit)
+		t.Errorf("stdout: %s", stdout.String())
+		t.Errorf("stderr: %s", stderr.String())
 	}
 
 	return stdout.String(), stderr.String()
@@ -67,6 +69,10 @@ func RunAndNormalize(t *testing.T, tc Case) (string, string) {
 	stderr = normalizeDirScanOrder(t, stderr)
 
 	if len(tc.ReplaceRules) > 0 {
+		if len(stdout) == 0 || !json.Valid([]byte(stdout)) {
+			t.Fatalf("invalid JSON when expecting json\n stdout: %s\n stderr: %s", stdout, stderr)
+		}
+
 		stdout = normalizeJSON(t, stdout, tc.ReplaceRules...)
 	}
 

cmd/osv-scanner/internal/testcmd/vcr.go

@@ -102,12 +102,16 @@ func InsertCassette(t *testing.T) *http.Client {
 
 			delete(i.Request.Headers, "User-Agent")
 
-			i.Request.Body = string(pretty.Pretty([]byte(i.Request.Body)))
+			// Force copy of default options
+			prettyOptions := *pretty.DefaultOptions
+			prettyOptions.SortKeys = true
+
+			i.Request.Body = string(pretty.PrettyOptions([]byte(i.Request.Body), &prettyOptions))
 			i.Request.ContentLength = int64(len(i.Request.Body))
 
 			// use a static duration since we don't care about replicating latency
 			i.Response.Duration = 0
-			i.Response.Body = string(pretty.Pretty([]byte(i.Response.Body)))
+			i.Response.Body = string(pretty.PrettyOptions([]byte(i.Response.Body), &prettyOptions))
 
 			return nil
 		}, recorder.AfterCaptureHook),