[dartfuzz] Extend VM flag fuzzing to dartfuzz and iso-stress.

Extend fuzzing to run dart2wasm.

Increase flag fuzzing timeout to equal generated program fuzzing timeout.

Adjust dartfuzz_test and iso-stress to use relative paths for ease of copy-pasting commands for reproduction.

Cq-Include-Trybots: luci.dart.try:iso-stress-linux-arm64-try,iso-stress-linux-x64-try
Bug: #60804
Bug: #60805
Change-Id: I6898f30b710f26f2c624726f7affee3187804334
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/430280
Commit-Queue: Ryan Macnak <[email protected]>
Reviewed-by: Alexander Aprelev <[email protected]>

Author: rmacnak-google

runtime/tests/concurrency/README.md

@@ -18,10 +18,8 @@ as part of the stress test.
 `generate_stress_test.dart`: Can be used to consume `stress_test_list.json` and
 build the stress test files.
 
-`stress_test_list.json`: Contains two lists of tests (one for NNBD and one for
-non-NNBD) that can be used to generate a stress test.
+`stress_test_list.json`: Contains a list of tests that can be used to generate a
+stress test.
 
-To ensure the list doesn't get out-of-date we have two tests on regular bots
-that will try to compile the stress test to kernel, thereby ensuring that the
-files at least exist and compile, see
-`runtime/tests/vm/dart/isolates/concurrency_stress_sanity_test.dart`.
+The stress test is run on the `iso-stress-linux-arm64` and 
+`iso-stress-linux-x64` builders.

runtime/tests/concurrency/generate_stress_test.dart

@@ -24,11 +24,7 @@ main(List<String> args) async {
 }
 
 Future<String> generateStressTest(List<String> testFiles) async {
-  testFiles =
-      testFiles
-          .map((String file) => path.absolute(path.join(thisDirectory, file)))
-          .toList()
-        ..shuffle();
+  testFiles = List.from(testFiles)..shuffle();
 
   final sb = StringBuffer();
   sb.writeln(r'''
@@ -42,7 +38,7 @@ import 'dart:io';
     sb.writeln('import "$testFile" as test$i;');
   }
   for (int i = 0; i < testFiles.length; ++i) {
-    final testFile = testFiles[i];
+    final testFile = path.normalize(path.join(thisDirectory, testFiles[i]));
     sb.writeln('''
         wrapper$i(dynamic _) {
           print('[$testFile] starting ...');
@@ -67,7 +63,7 @@ import 'dart:io';
   ''');
   sb.writeln('final List<Test> tests = [');
   for (int i = 0; i < testFiles.length; ++i) {
-    final testFile = testFiles[i];
+    final testFile = path.normalize(path.join(thisDirectory, testFiles[i]));
     sb.writeln('  Test("$testFile", wrapper$i),');
   }
   sb.writeln('];');

runtime/tests/concurrency/run_stress_test_shards.dart

@@ -12,6 +12,7 @@ import 'package:path/path.dart' as path;
 import 'package:test_runner/src/options.dart';
 
 import '../vm/dart/snapshot_test_helper.dart';
+import '../../tools/dartfuzz/flag_fuzzer.dart';
 
 int crashCounter = 0;
 
@@ -36,7 +37,8 @@ Future<bool> run(
   List<String> args,
   List<PotentialCrash> crashes,
 ) async {
-  print('Running "$executable ${args.join(' ')}"');
+  print('\n\nRunning "$executable ${args.join(' ')}"');
+  final sw = Stopwatch()..start();
   final Process process = await Process.start(
     executable,
     args,
@@ -45,6 +47,7 @@ Future<bool> run(
   forwardStream(process.stdout, stdout);
   forwardStream(process.stderr, stderr);
   final int exitCode = await process.exitCode;
+  print('Completed in ${sw.elapsed}');
   if (exitCode != 0) {
     // Ignore normal exceptions and compile-time errors for the purpose of
     // crashdump reporting.
@@ -141,7 +144,7 @@ void writeUnexpectedCrashesFile(List<PotentialCrash> crashes) {
   File(unexpectedCrashesFile).writeAsStringSync(buffer.toString());
 }
 
-const int tsanShards = 200;
+const int tsanShards = 64;
 
 late final List<TestRunner> configurations;
 
@@ -173,29 +176,42 @@ main(List<String> arguments) async {
   configurations = <TestRunner>[
     JitTestRunner('out/Debug$arch', [
       '--disable-dart-dev',
+      ...someJitRuntimeFlags(),
       'runtime/tests/concurrency/generated_stress_test.dart.jit.dill',
     ]),
     JitTestRunner('out/Release$arch', [
       '--disable-dart-dev',
-      '--no-inline-alloc',
-      '--use-slow-path',
-      '--deoptimize-on-runtime-call-every=300',
+      ...someJitRuntimeFlags(),
       'runtime/tests/concurrency/generated_stress_test.dart.jit.dill',
     ]),
+    AotTestRunner(
+      'out/Debug$arch',
+      [
+        ...someGenSnapshotFlags(),
+        'runtime/tests/concurrency/generated_stress_test.dart.aot.dill',
+      ],
+      [...someAotRuntimeFlags()],
+    ),
+    AotTestRunner(
+      'out/Release$arch',
+      [
+        ...someGenSnapshotFlags(),
+        'runtime/tests/concurrency/generated_stress_test.dart.aot.dill',
+      ],
+      [...someAotRuntimeFlags()],
+    ),
+    // TSAN last so the other steps are evenly distributed.
     for (int i = 0; i < tsanShards; ++i)
       JitTestRunner('out/ReleaseTSAN$arch', [
         '--disable-dart-dev',
+        ...someJitRuntimeFlags(),
+        '--no-profiler', // TODO(https://github.com/dart-lang/sdk/issues/60804, https://github.com/dart-lang/sdk/issues/60805)
+        '--no-gc_at_throw', // Too slow under TSAN
         '-Drepeat=4',
         '-Dshard=$i',
         '-Dshards=$tsanShards',
         'runtime/tests/concurrency/generated_stress_test.dart.jit.dill',
       ]),
-    AotTestRunner('out/Release$arch', [
-      'runtime/tests/concurrency/generated_stress_test.dart.aot.dill',
-    ], []),
-    AotTestRunner('out/Debug$arch', [
-      'runtime/tests/concurrency/generated_stress_test.dart.aot.dill',
-    ], []),
   ];
 
   // Tasks will eventually be killed if they do not have any output for some

runtime/tests/concurrency/stress_test_list.json

@@ -60,7 +60,6 @@
   "../vm/dart/inline_TypedList_getUint32_il_test.dart",
   "../vm/dart/inline_local_functions_il_test.dart",
   "../vm/dart/inline_stack_frame_test.dart",
-  "../vm/dart/isolates/concurrency_stress_sanity_test.dart",
   "../vm/dart/keep_class_names_implementing_test.dart",
   "../vm/dart/licm_and_assert_strengthening_test.dart",
   "../vm/dart/load_indexed_trivial_type_il_test.dart",

runtime/tests/vm/dart/isolates/concurrency_stress_sanity_test.dart

@@ -12,6 +12,7 @@ import 'package:args/args.dart';
 import 'package:matcher/matcher.dart';
 
 import 'dartfuzz.dart';
+import 'flag_fuzzer.dart';
 
 const debug = false;
 const sigkill = 9;
@@ -64,50 +65,10 @@ abstract class TestRunner {
     var prefix = mode.substring(0, 3).toUpperCase();
     var tag = getTag(mode);
     var extraFlags = <String>[];
-    // Every once in a while, stress test JIT.
-    if (mode.startsWith('jit') && rand.nextInt(4) == 0) {
-      final r = rand.nextInt(7);
-      if (r == 0) {
-        prefix += '-NOFIELDGUARDS';
-        extraFlags += ['--use_field_guards=false'];
-      } else if (r == 1) {
-        prefix += '-NOINTRINSIFY';
-        extraFlags += ['--intrinsify=false'];
-      } else if (r == 2) {
-        final freq = rand.nextInt(1000) + 500;
-        prefix += '-COMPACTEVERY-$freq';
-        extraFlags += ['--gc_every=$freq', '--use_compactor=true'];
-      } else if (r == 3) {
-        final freq = rand.nextInt(1000) + 500;
-        prefix += '-MARKSWEEPEVERY-$freq';
-        extraFlags += ['--gc_every=$freq', '--use_compactor=false'];
-      } else if (r == 4) {
-        final freq = rand.nextInt(100) + 50;
-        prefix += '-DEPOPTEVERY-$freq';
-        extraFlags += ['--deoptimize_every=$freq'];
-      } else if (r == 5) {
-        final freq = rand.nextInt(100) + 50;
-        prefix += '-STACKTRACEEVERY-$freq';
-        extraFlags += ['--stacktrace_every=$freq'];
-      } else if (r == 6) {
-        prefix += '-OPTCOUNTER';
-        extraFlags += ['--optimization_counter_threshold=1'];
-      }
-    }
-    // Every once in a while, use -O3 compiler.
-    if (!mode.startsWith('djs') && rand.nextInt(4) == 0) {
-      prefix += '-O3';
-      extraFlags += ['--optimization_level=3'];
-    }
-    // Every once in a while, use the slowpath flag.
-    if (!mode.startsWith('djs') && rand.nextInt(4) == 0) {
-      prefix += '-SLOWPATH';
-      extraFlags += ['--use-slow-path'];
-    }
-    // Every once in a while, use the deterministic flag.
-    if (!mode.startsWith('djs') && rand.nextInt(4) == 0) {
-      prefix += '-DET';
-      extraFlags += ['--deterministic'];
+    if (mode.startsWith('jit')) {
+      extraFlags += someJitRuntimeFlags();
+    } else if (mode.startsWith('aot')) {
+      extraFlags += someGenSnapshotFlags();
     }
     // Construct runner.
     if (mode.startsWith('jit')) {
@@ -698,7 +659,7 @@ class DartFuzzTestSession {
       top = Platform.environment['DART_TOP'];
     }
     if (top == null || top == '') {
-      top = Directory.current.path;
+      top = '.';
     }
     return top;
   }