Add --suite-load-timeout option

pkgs/test/CHANGELOG.md

@@ -1,9 +1,12 @@
 ## 1.27.0-wip
 
-* Restrict to latest version of analyzer package.
-* Require Dart 3.7
 * Add `--coverage-path` and `--branch-coverage` options to `dart test`.
+* Remove the default 12 minute timeout to compile and load test suites.
+* Add a `--suite-load-timeout` argument to allow configuring a timeout for
+  compiling and loading individual test suites.
 * Serve dart2wasm source map files.
+* Restrict to latest version of analyzer package.
+* Require Dart 3.7
 
 ## 1.26.3
 

pkgs/test/test/runner/configuration/top_level_test.dart

@@ -448,6 +448,34 @@ void main() {
     await test.shouldExit(1);
   });
 
+  test('uses the specified suite load timeout', () async {
+    await d
+        .file('dart_test.yaml', jsonEncode({'suite_load_timeout': '1s'}))
+        .create();
+
+    await d.file('test.dart', '''
+      import 'dart:async';
+
+      import 'package:test/test.dart';
+
+      Future<void> main() async {
+        await Future.delayed(Duration(seconds: 2));
+        test('success', () {});
+      }
+    ''').create();
+
+    var test = await runTest(['test.dart']);
+    expect(
+      test.stdout,
+      containsInOrder([
+        'loading test.dart [E]',
+        'Test timed out after 1 seconds.',
+        '-1: Some tests failed.',
+      ]),
+    );
+    await test.shouldExit(1);
+  });
+
   test('runs on the specified platforms', () async {
     await d
         .file(

pkgs/test/test/utils.dart

@@ -181,6 +181,7 @@ SuiteConfiguration suiteConfiguration({
   Map<BooleanSelector, SuiteConfiguration>? tags,
   Map<PlatformSelector, SuiteConfiguration>? onPlatform,
   bool? ignoreTimeouts,
+  Timeout? suiteLoadTimeout,
 
   // Test-level configuration
   Timeout? timeout,
@@ -203,6 +204,7 @@ SuiteConfiguration suiteConfiguration({
   tags: tags,
   onPlatform: onPlatform,
   ignoreTimeouts: ignoreTimeouts,
+  suiteLoadTimeout: suiteLoadTimeout,
   timeout: timeout,
   verboseTrace: verboseTrace,
   chainStackTraces: chainStackTraces,
@@ -239,6 +241,7 @@ Configuration configuration({
   Map<String, CustomRuntime>? defineRuntimes,
   bool? noRetry,
   bool? ignoreTimeouts,
+  Timeout? suiteLoadTimeout,
 
   // Suite-level configuration
   bool? allowDuplicateTestNames,
@@ -291,6 +294,7 @@ Configuration configuration({
   defineRuntimes: defineRuntimes,
   noRetry: noRetry,
   ignoreTimeouts: ignoreTimeouts,
+  suiteLoadTimeout: suiteLoadTimeout,
   allowDuplicateTestNames: allowDuplicateTestNames,
   allowTestRandomization: allowTestRandomization,
   jsTrace: jsTrace,

pkgs/test_core/lib/src/runner/configuration.dart

@@ -303,6 +303,7 @@ class Configuration {
     required Map<BooleanSelector, SuiteConfiguration>? tags,
     required Map<PlatformSelector, SuiteConfiguration>? onPlatform,
     required bool? ignoreTimeouts,
+    required Timeout? suiteLoadTimeout,
 
     // Test-level configuration
     required Timeout? timeout,
@@ -357,6 +358,7 @@ class Configuration {
         tags: tags,
         onPlatform: onPlatform,
         ignoreTimeouts: ignoreTimeouts,
+        suiteLoadTimeout: suiteLoadTimeout,
 
         // Test-level configuration
         timeout: timeout,
@@ -419,6 +421,7 @@ class Configuration {
     Map<BooleanSelector, SuiteConfiguration>? tags,
     Map<PlatformSelector, SuiteConfiguration>? onPlatform,
     bool? ignoreTimeouts,
+    Timeout? suiteLoadTimeout,
 
     // Test-level configuration
     Timeout? timeout,
@@ -470,6 +473,7 @@ class Configuration {
     tags: tags,
     onPlatform: onPlatform,
     ignoreTimeouts: ignoreTimeouts,
+    suiteLoadTimeout: suiteLoadTimeout,
     timeout: timeout,
     verboseTrace: verboseTrace,
     chainStackTraces: chainStackTraces,
@@ -489,6 +493,7 @@ class Configuration {
     required bool? verboseTrace,
     required bool? jsTrace,
     required Timeout? timeout,
+    required Timeout? suiteLoadTimeout,
     required Map<String, Configuration>? presets,
     required bool? chainStackTraces,
     required Iterable<String>? foldTraceExcept,
@@ -498,6 +503,7 @@ class Configuration {
     foldTraceOnly: foldTraceOnly,
     jsTrace: jsTrace,
     timeout: timeout,
+    suiteLoadTimeout: suiteLoadTimeout,
     verboseTrace: verboseTrace,
     chainStackTraces: chainStackTraces,
     help: null,
@@ -604,6 +610,7 @@ class Configuration {
     tags: null,
     onPlatform: null,
     ignoreTimeouts: null,
+    suiteLoadTimeout: null,
     timeout: null,
     verboseTrace: null,
     chainStackTraces: null,
@@ -669,6 +676,7 @@ class Configuration {
     tags: null,
     onPlatform: null,
     ignoreTimeouts: null,
+    suiteLoadTimeout: null,
     timeout: null,
     verboseTrace: null,
     chainStackTraces: null,
@@ -732,6 +740,7 @@ class Configuration {
     tags: null,
     onPlatform: null,
     ignoreTimeouts: null,
+    suiteLoadTimeout: null,
     timeout: null,
     verboseTrace: null,
     chainStackTraces: null,
@@ -1046,6 +1055,7 @@ class Configuration {
     bool? noRetry,
     int? testRandomizeOrderingSeed,
     bool? ignoreTimeouts,
+    Timeout? suiteLoadTimeout,
 
     // Suite-level configuration
     bool? allowDuplicateTestNames,
@@ -1117,6 +1127,7 @@ class Configuration {
         testOn: testOn,
         addTags: addTags,
         ignoreTimeouts: ignoreTimeouts,
+        suiteLoadTimeout: suiteLoadTimeout,
       ),
     );
     return config._resolvePresets();

pkgs/test_core/lib/src/runner/configuration/args.dart

@@ -141,6 +141,13 @@ final ArgParser _parser =
         help: 'The default test timeout. For example: 15s, 2x, none',
         defaultsTo: '30s',
       );
+      parser.addOption(
+        'suite-load-timeout',
+        help:
+            'The timeout for loading a test suite. Loading the test suite '
+            'includes compiling the test suite. For example: 15s, none',
+        defaultsTo: 'none',
+      );
       parser.addFlag(
         'ignore-timeouts',
         help: 'Ignore all timeouts (useful if debugging)',
@@ -466,6 +473,7 @@ class _Parser {
       shardIndex: shardIndex,
       totalShards: totalShards,
       timeout: _parseOption('timeout', Timeout.parse),
+      suiteLoadTimeout: _parseOption('suite-load-timeout', Timeout.parse),
       globalPatterns: patterns,
       compilerSelections: compilerSelections,
       runtimes: runtimes,

pkgs/test_core/lib/src/runner/configuration/load.dart

@@ -134,6 +134,7 @@ class _ConfigurationLoader {
     var jsTrace = _getBool('js_trace');
 
     var timeout = _parseValue('timeout', Timeout.parse);
+    var suiteLoadTimeout = _parseValue('suite_load_timeout', Timeout.parse);
 
     var onPlatform = _getMap(
       'on_platform',
@@ -182,6 +183,7 @@ class _ConfigurationLoader {
       verboseTrace: verboseTrace,
       jsTrace: jsTrace,
       timeout: timeout,
+      suiteLoadTimeout: suiteLoadTimeout,
       presets: presets,
       chainStackTraces: chainStackTraces,
       foldTraceExcept: foldStackFrames['except'],

pkgs/test_core/lib/src/runner/load_suite.dart

@@ -5,7 +5,6 @@
 import 'dart:async';
 
 import 'package:stack_trace/stack_trace.dart';
-import 'package:test_api/scaffolding.dart' show Timeout;
 import 'package:test_api/src/backend/group.dart'; // ignore: implementation_imports
 import 'package:test_api/src/backend/invoker.dart'; // ignore: implementation_imports
 import 'package:test_api/src/backend/metadata.dart'; // ignore: implementation_imports
@@ -20,14 +19,6 @@ import 'plugin/environment.dart';
 import 'runner_suite.dart';
 import 'suite.dart';
 
-/// The timeout for loading a test suite.
-///
-/// We want this to be long enough that even a very large application being
-/// compiled with dart2js doesn't trigger it, but short enough that it fires
-/// before the host kills it. For example, Google's Forge service has a
-/// 15-minute timeout.
-final _timeout = const Duration(minutes: 12);
-
 /// A [Suite] emitted by a [Loader] that provides a test-like interface for
 /// loading a test file.
 ///
@@ -177,7 +168,7 @@ class LoadSuite extends Suite implements RunnerSuite {
     String? path,
   }) : super(
          Group.root([
-           LocalTest(name, Metadata(timeout: Timeout(_timeout)), body),
+           LocalTest(name, Metadata(timeout: config.suiteLoadTimeout), body),
          ]),
          platform,
          path: path,

pkgs/test_core/lib/src/runner/suite.dart

@@ -58,6 +58,7 @@ final class SuiteConfiguration {
     onPlatform: null,
     metadata: null,
     ignoreTimeouts: null,
+    suiteLoadTimeout: null,
   );
 
   /// Whether or not duplicate test (or group) names are allowed within the same
@@ -153,6 +154,10 @@ final class SuiteConfiguration {
   final bool? _ignoreTimeouts;
   bool get ignoreTimeouts => _ignoreTimeouts ?? false;
 
+  /// The timeout for loading a test suite.
+  final Timeout? _suiteLoadTimeout;
+  Timeout get suiteLoadTimeout => _suiteLoadTimeout ?? Timeout.none;
+
   factory SuiteConfiguration({
     required bool? allowDuplicateTestNames,
     required bool? allowTestRandomization,
@@ -165,7 +170,7 @@ final class SuiteConfiguration {
     required Map<BooleanSelector, SuiteConfiguration>? tags,
     required Map<PlatformSelector, SuiteConfiguration>? onPlatform,
     required bool? ignoreTimeouts,
-
+    required Timeout? suiteLoadTimeout,
     // Test-level configuration
     required Timeout? timeout,
     required bool? verboseTrace,
@@ -189,6 +194,7 @@ final class SuiteConfiguration {
       tags: tags,
       onPlatform: onPlatform,
       ignoreTimeouts: ignoreTimeouts,
+      suiteLoadTimeout: suiteLoadTimeout,
       metadata: Metadata(
         timeout: timeout,
         verboseTrace: verboseTrace,
@@ -219,6 +225,7 @@ final class SuiteConfiguration {
     Map<BooleanSelector, SuiteConfiguration>? tags,
     Map<PlatformSelector, SuiteConfiguration>? onPlatform,
     bool? ignoreTimeouts,
+    Timeout? suiteLoadTimeout,
 
     // Test-level configuration
     Timeout? timeout,
@@ -241,6 +248,7 @@ final class SuiteConfiguration {
     tags: tags,
     onPlatform: onPlatform,
     ignoreTimeouts: ignoreTimeouts,
+    suiteLoadTimeout: suiteLoadTimeout,
     timeout: timeout,
     verboseTrace: verboseTrace,
     chainStackTraces: chainStackTraces,
@@ -281,6 +289,7 @@ final class SuiteConfiguration {
     required Map<PlatformSelector, SuiteConfiguration>? onPlatform,
     required Metadata? metadata,
     required bool? ignoreTimeouts,
+    required Timeout? suiteLoadTimeout,
   }) : _allowDuplicateTestNames = allowDuplicateTestNames,
        _allowTestRandomization = allowTestRandomization,
        _jsTrace = jsTrace,
@@ -291,6 +300,7 @@ final class SuiteConfiguration {
        tags = _map(tags),
        onPlatform = _map(onPlatform),
        _ignoreTimeouts = ignoreTimeouts,
+       _suiteLoadTimeout = suiteLoadTimeout,
        _metadata = metadata ?? Metadata.empty;
 
   /// Creates a new [SuiteConfiguration] that takes its configuration from
@@ -314,6 +324,7 @@ final class SuiteConfiguration {
         runtimes: null,
         compilerSelections: null,
         ignoreTimeouts: null,
+        suiteLoadTimeout: null,
       );
 
   /// Returns an unmodifiable copy of [input].
@@ -359,6 +370,7 @@ final class SuiteConfiguration {
       onPlatform: _mergeConfigMaps(onPlatform, other.onPlatform),
       ignoreTimeouts: other._ignoreTimeouts ?? _ignoreTimeouts,
       metadata: metadata.merge(other.metadata),
+      suiteLoadTimeout: other._suiteLoadTimeout ?? _suiteLoadTimeout,
     );
     return config._resolveTags();
   }
@@ -379,6 +391,7 @@ final class SuiteConfiguration {
     Map<BooleanSelector, SuiteConfiguration>? tags,
     Map<PlatformSelector, SuiteConfiguration>? onPlatform,
     bool? ignoreTimeouts,
+    Timeout? suiteLoadTimeout,
 
     // Test-level configuration
     Timeout? timeout,
@@ -404,6 +417,7 @@ final class SuiteConfiguration {
       tags: tags ?? this.tags,
       onPlatform: onPlatform ?? this.onPlatform,
       ignoreTimeouts: ignoreTimeouts ?? _ignoreTimeouts,
+      suiteLoadTimeout: suiteLoadTimeout ?? _suiteLoadTimeout,
       metadata: _metadata.change(
         timeout: timeout,
         verboseTrace: verboseTrace,
@@ -441,6 +455,7 @@ final class SuiteConfiguration {
       tags: tags,
       onPlatform: onPlatform,
       ignoreTimeouts: _ignoreTimeouts,
+      suiteLoadTimeout: suiteLoadTimeout,
       metadata: _metadata,
     );
   }