Add --suite-load-timeout option (#2505)

Remove the hardcoded timeout and allow specifying a timeout for loading
individual test suites.

Author: nilsreichardt

pkgs/test/CHANGELOG.md

@@ -1,5 +1,9 @@
-## 1.29.1-wip
+## 1.30.0-wip
 
+* Add a `--suite-load-timeout` argument to allow configuring a timeout for
+  compiling and loading individual test suites.
+* Remove the default 12 minute timeout to compile and load test suites.
+* Bump `test_core` to 0.6.16
 * Add comments to the top-level libraries in the package.
 
 ## 1.29.0

pkgs/test/doc/configuration.md

@@ -26,6 +26,7 @@ tags:
 
 * [Test Configuration](#test-configuration)
   * [`timeout`](#timeout)
+  * [`suite_load_timeout`](#suite_load_timeout)
   * [`ignore-timeouts`](#ignore-timeouts)
   * [`verbose_trace`](#verbose_trace)
   * [`chain_stack_traces`](#chain_stack_traces)
@@ -97,9 +98,27 @@ formats:
 timeout: 1m
 ```
 
+### `suite_load_timeout`
+
+This field indicates how much time the test runner should allow for compiling
+and loading a test suite before it considers that suite to have failed. It has
+two possible formats:
+
+* The string "none" indicates that loading suites should never time out.
+
+* A number followed by a unit abbreviation indicates an exact time. For example,
+  "1m" means a timeout of one minute, and "30s" means a timeout of thirty
+  seconds. Multiple numbers can be combined, as in "1m 30s".
+
+```yaml
+suite_load_timeout: 1m
+```
+
 ###  `ignore-timeouts`
 
-This field disables all timeouts for all tests. This can be useful when debugging, so tests don't time out during debug sessions. It defaults to `false`.
+This field disables all timeouts for all tests. This can be useful when
+debugging, so tests don't time out during debug sessions. It defaults to
+`false`.
 
 ```yaml
 ignore-timeouts: true

pkgs/test/pubspec.yaml

@@ -1,5 +1,5 @@
 name: test
-version: 1.29.1-wip
+version: 1.30.0-wip
 description: >-
   A full featured library for writing and running Dart tests across platforms.
 repository: https://github.com/dart-lang/test/tree/master/pkgs/test
@@ -36,7 +36,7 @@ dependencies:
 
   # Use an exact version until the test_api and test_core package are stable.
   test_api: 0.7.9
-  test_core: 0.6.15
+  test_core: 0.6.16-wip
 
   typed_data: ^1.3.0
   web_socket_channel: '>=2.0.0 <4.0.0'

pkgs/test/test/runner/configuration/top_level_test.dart

@@ -448,6 +448,34 @@ void main() {
     await test.shouldExit(1);
   });
 
+  test('uses the specified suite load timeout', () async {
+    await d
+        .file('dart_test.yaml', jsonEncode({'suite_load_timeout': '1s'}))
+        .create();
+
+    await d.file('test.dart', '''
+      import 'dart:async';
+
+      import 'package:test/test.dart';
+
+      Future<void> main() async {
+        await Future.delayed(Duration(seconds: 2));
+        test('success', () {});
+      }
+    ''').create();
+
+    var test = await runTest(['test.dart']);
+    expect(
+      test.stdout,
+      containsInOrder([
+        'loading test.dart [E]',
+        'Test timed out after 1 seconds.',
+        '-1: Some tests failed.',
+      ]),
+    );
+    await test.shouldExit(1);
+  });
+
   test('runs on the specified platforms', () async {
     await d
         .file(

pkgs/test/test/runner/runner_test.dart

@@ -85,6 +85,8 @@ $_runtimeCompilers
     --shard-index                     The index of this test runner invocation (of --total-shards).
     --timeout                         The default test timeout. For example: 15s, 2x, none
                                       (defaults to "30s")
+    --suite-load-timeout              The timeout for loading a test suite. Loading the test suite includes compiling the test suite. For example: 15s, 2m, none
+                                      (defaults to "none")
     --ignore-timeouts                 Ignore all timeouts (useful if debugging)
     --pause-after-load                Pause for debugging before any tests execute.
                                       Implies --concurrency=1, --debug, and --ignore-timeouts.

pkgs/test/test/runner/suite_load_timeout_test.dart

@@ -0,0 +1,39 @@
+// Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+@TestOn('vm')
+library;
+
+import 'package:test/test.dart';
+import 'package:test_descriptor/test_descriptor.dart' as d;
+
+import '../io.dart';
+
+void main() {
+  setUpAll(precompileTestExecutable);
+
+  test('uses the passed suite load timeout', () async {
+    await d.file('test.dart', '''
+      import 'dart:async';
+
+      import 'package:test/test.dart';
+
+      Future<void> main() async {
+        await Future.delayed(Duration(seconds: 2));
+        test('success', () {});
+      }
+    ''').create();
+
+    var test = await runTest(['--suite-load-timeout=1s', 'test.dart']);
+    expect(
+      test.stdout,
+      containsInOrder([
+        'loading test.dart [E]',
+        'Test timed out after 1 seconds.',
+        '-1: Some tests failed.',
+      ]),
+    );
+    await test.shouldExit(1);
+  });
+}

pkgs/test/test/utils.dart

@@ -181,6 +181,7 @@ SuiteConfiguration suiteConfiguration({
   Map<BooleanSelector, SuiteConfiguration>? tags,
   Map<PlatformSelector, SuiteConfiguration>? onPlatform,
   bool? ignoreTimeouts,
+  Timeout? suiteLoadTimeout,
 
   // Test-level configuration
   Timeout? timeout,
@@ -203,6 +204,7 @@ SuiteConfiguration suiteConfiguration({
   tags: tags,
   onPlatform: onPlatform,
   ignoreTimeouts: ignoreTimeouts,
+  suiteLoadTimeout: suiteLoadTimeout,
   timeout: timeout,
   verboseTrace: verboseTrace,
   chainStackTraces: chainStackTraces,
@@ -256,6 +258,7 @@ Configuration configuration({
   Map<BooleanSelector, SuiteConfiguration>? tags,
   Map<PlatformSelector, SuiteConfiguration>? onPlatform,
   int? testRandomizeOrderingSeed,
+  Timeout? suiteLoadTimeout,
 
   // Test-level configuration
   Timeout? timeout,
@@ -307,6 +310,7 @@ Configuration configuration({
   tags: tags,
   onPlatform: onPlatform,
   testRandomizeOrderingSeed: testRandomizeOrderingSeed,
+  suiteLoadTimeout: suiteLoadTimeout,
   stopOnFirstFailure: false,
   timeout: timeout,
   verboseTrace: verboseTrace,

pkgs/test_core/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.6.16-wip
+
+* Add `SuiteConfiguration.suiteLoadTimeout` to configure the timeout for loading a test suite.
+* Removed hard-coded timeout of 12m for loading a test suite and set default to `none`.
+
 ## 0.6.15
 
 * Add `--coverage-package` flag, which filters the coverage report to specific

pkgs/test_core/lib/src/runner/configuration.dart

@@ -311,6 +311,7 @@ class Configuration {
     required Map<BooleanSelector, SuiteConfiguration>? tags,
     required Map<PlatformSelector, SuiteConfiguration>? onPlatform,
     required bool? ignoreTimeouts,
+    required Timeout? suiteLoadTimeout,
 
     // Test-level configuration
     required Timeout? timeout,
@@ -366,6 +367,7 @@ class Configuration {
         tags: tags,
         onPlatform: onPlatform,
         ignoreTimeouts: ignoreTimeouts,
+        suiteLoadTimeout: suiteLoadTimeout,
 
         // Test-level configuration
         timeout: timeout,
@@ -429,6 +431,7 @@ class Configuration {
     Map<BooleanSelector, SuiteConfiguration>? tags,
     Map<PlatformSelector, SuiteConfiguration>? onPlatform,
     bool? ignoreTimeouts,
+    Timeout? suiteLoadTimeout,
 
     // Test-level configuration
     Timeout? timeout,
@@ -481,6 +484,7 @@ class Configuration {
     tags: tags,
     onPlatform: onPlatform,
     ignoreTimeouts: ignoreTimeouts,
+    suiteLoadTimeout: suiteLoadTimeout,
     timeout: timeout,
     verboseTrace: verboseTrace,
     chainStackTraces: chainStackTraces,
@@ -500,6 +504,7 @@ class Configuration {
     required bool? verboseTrace,
     required bool? jsTrace,
     required Timeout? timeout,
+    required Timeout? suiteLoadTimeout,
     required Map<String, Configuration>? presets,
     required bool? chainStackTraces,
     required Iterable<String>? foldTraceExcept,
@@ -537,6 +542,7 @@ class Configuration {
     testRandomizeOrderingSeed: null,
     stopOnFirstFailure: null,
     ignoreTimeouts: null,
+    suiteLoadTimeout: suiteLoadTimeout,
     allowDuplicateTestNames: null,
     allowTestRandomization: null,
     runSkipped: null,
@@ -617,6 +623,7 @@ class Configuration {
     tags: null,
     onPlatform: null,
     ignoreTimeouts: null,
+    suiteLoadTimeout: null,
     timeout: null,
     verboseTrace: null,
     chainStackTraces: null,
@@ -683,6 +690,7 @@ class Configuration {
     tags: null,
     onPlatform: null,
     ignoreTimeouts: null,
+    suiteLoadTimeout: null,
     timeout: null,
     verboseTrace: null,
     chainStackTraces: null,
@@ -747,6 +755,7 @@ class Configuration {
     tags: null,
     onPlatform: null,
     ignoreTimeouts: null,
+    suiteLoadTimeout: null,
     timeout: null,
     verboseTrace: null,
     chainStackTraces: null,

pkgs/test_core/lib/src/runner/configuration/args.dart

@@ -141,6 +141,13 @@ final ArgParser _parser =
         help: 'The default test timeout. For example: 15s, 2x, none',
         defaultsTo: '30s',
       );
+      parser.addOption(
+        'suite-load-timeout',
+        help:
+            'The timeout for loading a test suite. Loading the test suite '
+            'includes compiling the test suite. For example: 15s, 2m, none',
+        defaultsTo: 'none',
+      );
       parser.addFlag(
         'ignore-timeouts',
         help: 'Ignore all timeouts (useful if debugging)',
@@ -476,6 +483,7 @@ class _Parser {
       shardIndex: shardIndex,
       totalShards: totalShards,
       timeout: _parseOption('timeout', Timeout.parse),
+      suiteLoadTimeout: _parseOption('suite-load-timeout', Timeout.parse),
       globalPatterns: patterns,
       compilerSelections: compilerSelections,
       runtimes: runtimes,