diff --git a/pkgs/test/CHANGELOG.md b/pkgs/test/CHANGELOG.md index 2a9474861..8c9fce2f0 100644 --- a/pkgs/test/CHANGELOG.md +++ b/pkgs/test/CHANGELOG.md @@ -1,9 +1,12 @@ ## 1.27.0-wip -* Restrict to latest version of analyzer package. -* Require Dart 3.7 * Add `--coverage-path` and `--branch-coverage` options to `dart test`. +* Remove the default 12 minute timeout to compile and load test suites. +* Add a `--suite-load-timeout` argument to allow configuring a timeout for + compiling and loading individual test suites. * Serve dart2wasm source map files. +* Restrict to latest version of analyzer package. +* Require Dart 3.7 ## 1.26.3 diff --git a/pkgs/test/test/runner/configuration/top_level_test.dart b/pkgs/test/test/runner/configuration/top_level_test.dart index 815d01f6d..833dd50e8 100644 --- a/pkgs/test/test/runner/configuration/top_level_test.dart +++ b/pkgs/test/test/runner/configuration/top_level_test.dart @@ -448,6 +448,34 @@ void main() { await test.shouldExit(1); }); + test('uses the specified suite load timeout', () async { + await d + .file('dart_test.yaml', jsonEncode({'suite_load_timeout': '1s'})) + .create(); + + await d.file('test.dart', ''' + import 'dart:async'; + + import 'package:test/test.dart'; + + Future main() async { + await Future.delayed(Duration(seconds: 2)); + test('success', () {}); + } + ''').create(); + + var test = await runTest(['test.dart']); + expect( + test.stdout, + containsInOrder([ + 'loading test.dart [E]', + 'Test timed out after 1 seconds.', + '-1: Some tests failed.', + ]), + ); + await test.shouldExit(1); + }); + test('runs on the specified platforms', () async { await d .file( diff --git a/pkgs/test/test/utils.dart b/pkgs/test/test/utils.dart index cfa26cff2..8787a042c 100644 --- a/pkgs/test/test/utils.dart +++ b/pkgs/test/test/utils.dart @@ -181,6 +181,7 @@ SuiteConfiguration suiteConfiguration({ Map? tags, Map? onPlatform, bool? ignoreTimeouts, + Timeout? suiteLoadTimeout, // Test-level configuration Timeout? timeout, @@ -203,6 +204,7 @@ SuiteConfiguration suiteConfiguration({ tags: tags, onPlatform: onPlatform, ignoreTimeouts: ignoreTimeouts, + suiteLoadTimeout: suiteLoadTimeout, timeout: timeout, verboseTrace: verboseTrace, chainStackTraces: chainStackTraces, @@ -239,6 +241,7 @@ Configuration configuration({ Map? defineRuntimes, bool? noRetry, bool? ignoreTimeouts, + Timeout? suiteLoadTimeout, // Suite-level configuration bool? allowDuplicateTestNames, @@ -291,6 +294,7 @@ Configuration configuration({ defineRuntimes: defineRuntimes, noRetry: noRetry, ignoreTimeouts: ignoreTimeouts, + suiteLoadTimeout: suiteLoadTimeout, allowDuplicateTestNames: allowDuplicateTestNames, allowTestRandomization: allowTestRandomization, jsTrace: jsTrace, diff --git a/pkgs/test_core/lib/src/runner/configuration.dart b/pkgs/test_core/lib/src/runner/configuration.dart index e9c3bebe9..a1eb76690 100644 --- a/pkgs/test_core/lib/src/runner/configuration.dart +++ b/pkgs/test_core/lib/src/runner/configuration.dart @@ -303,6 +303,7 @@ class Configuration { required Map? tags, required Map? onPlatform, required bool? ignoreTimeouts, + required Timeout? suiteLoadTimeout, // Test-level configuration required Timeout? timeout, @@ -357,6 +358,7 @@ class Configuration { tags: tags, onPlatform: onPlatform, ignoreTimeouts: ignoreTimeouts, + suiteLoadTimeout: suiteLoadTimeout, // Test-level configuration timeout: timeout, @@ -419,6 +421,7 @@ class Configuration { Map? tags, Map? onPlatform, bool? ignoreTimeouts, + Timeout? suiteLoadTimeout, // Test-level configuration Timeout? timeout, @@ -470,6 +473,7 @@ class Configuration { tags: tags, onPlatform: onPlatform, ignoreTimeouts: ignoreTimeouts, + suiteLoadTimeout: suiteLoadTimeout, timeout: timeout, verboseTrace: verboseTrace, chainStackTraces: chainStackTraces, @@ -489,6 +493,7 @@ class Configuration { required bool? verboseTrace, required bool? jsTrace, required Timeout? timeout, + required Timeout? suiteLoadTimeout, required Map? presets, required bool? chainStackTraces, required Iterable? foldTraceExcept, @@ -498,6 +503,7 @@ class Configuration { foldTraceOnly: foldTraceOnly, jsTrace: jsTrace, timeout: timeout, + suiteLoadTimeout: suiteLoadTimeout, verboseTrace: verboseTrace, chainStackTraces: chainStackTraces, help: null, @@ -604,6 +610,7 @@ class Configuration { tags: null, onPlatform: null, ignoreTimeouts: null, + suiteLoadTimeout: null, timeout: null, verboseTrace: null, chainStackTraces: null, @@ -669,6 +676,7 @@ class Configuration { tags: null, onPlatform: null, ignoreTimeouts: null, + suiteLoadTimeout: null, timeout: null, verboseTrace: null, chainStackTraces: null, @@ -732,6 +740,7 @@ class Configuration { tags: null, onPlatform: null, ignoreTimeouts: null, + suiteLoadTimeout: null, timeout: null, verboseTrace: null, chainStackTraces: null, @@ -1046,6 +1055,7 @@ class Configuration { bool? noRetry, int? testRandomizeOrderingSeed, bool? ignoreTimeouts, + Timeout? suiteLoadTimeout, // Suite-level configuration bool? allowDuplicateTestNames, @@ -1117,6 +1127,7 @@ class Configuration { testOn: testOn, addTags: addTags, ignoreTimeouts: ignoreTimeouts, + suiteLoadTimeout: suiteLoadTimeout, ), ); return config._resolvePresets(); diff --git a/pkgs/test_core/lib/src/runner/configuration/args.dart b/pkgs/test_core/lib/src/runner/configuration/args.dart index e248bf244..84582aa32 100644 --- a/pkgs/test_core/lib/src/runner/configuration/args.dart +++ b/pkgs/test_core/lib/src/runner/configuration/args.dart @@ -141,6 +141,13 @@ final ArgParser _parser = help: 'The default test timeout. For example: 15s, 2x, none', defaultsTo: '30s', ); + parser.addOption( + 'suite-load-timeout', + help: + 'The timeout for loading a test suite. Loading the test suite ' + 'includes compiling the test suite. For example: 15s, none', + defaultsTo: 'none', + ); parser.addFlag( 'ignore-timeouts', help: 'Ignore all timeouts (useful if debugging)', @@ -466,6 +473,7 @@ class _Parser { shardIndex: shardIndex, totalShards: totalShards, timeout: _parseOption('timeout', Timeout.parse), + suiteLoadTimeout: _parseOption('suite-load-timeout', Timeout.parse), globalPatterns: patterns, compilerSelections: compilerSelections, runtimes: runtimes, diff --git a/pkgs/test_core/lib/src/runner/configuration/load.dart b/pkgs/test_core/lib/src/runner/configuration/load.dart index fc74fa008..31a4deec0 100644 --- a/pkgs/test_core/lib/src/runner/configuration/load.dart +++ b/pkgs/test_core/lib/src/runner/configuration/load.dart @@ -134,6 +134,7 @@ class _ConfigurationLoader { var jsTrace = _getBool('js_trace'); var timeout = _parseValue('timeout', Timeout.parse); + var suiteLoadTimeout = _parseValue('suite_load_timeout', Timeout.parse); var onPlatform = _getMap( 'on_platform', @@ -182,6 +183,7 @@ class _ConfigurationLoader { verboseTrace: verboseTrace, jsTrace: jsTrace, timeout: timeout, + suiteLoadTimeout: suiteLoadTimeout, presets: presets, chainStackTraces: chainStackTraces, foldTraceExcept: foldStackFrames['except'], diff --git a/pkgs/test_core/lib/src/runner/load_suite.dart b/pkgs/test_core/lib/src/runner/load_suite.dart index 2fe5ae2b1..509cc39d3 100644 --- a/pkgs/test_core/lib/src/runner/load_suite.dart +++ b/pkgs/test_core/lib/src/runner/load_suite.dart @@ -5,7 +5,6 @@ import 'dart:async'; import 'package:stack_trace/stack_trace.dart'; -import 'package:test_api/scaffolding.dart' show Timeout; import 'package:test_api/src/backend/group.dart'; // ignore: implementation_imports import 'package:test_api/src/backend/invoker.dart'; // ignore: implementation_imports import 'package:test_api/src/backend/metadata.dart'; // ignore: implementation_imports @@ -20,14 +19,6 @@ import 'plugin/environment.dart'; import 'runner_suite.dart'; import 'suite.dart'; -/// The timeout for loading a test suite. -/// -/// We want this to be long enough that even a very large application being -/// compiled with dart2js doesn't trigger it, but short enough that it fires -/// before the host kills it. For example, Google's Forge service has a -/// 15-minute timeout. -final _timeout = const Duration(minutes: 12); - /// A [Suite] emitted by a [Loader] that provides a test-like interface for /// loading a test file. /// @@ -177,7 +168,7 @@ class LoadSuite extends Suite implements RunnerSuite { String? path, }) : super( Group.root([ - LocalTest(name, Metadata(timeout: Timeout(_timeout)), body), + LocalTest(name, Metadata(timeout: config.suiteLoadTimeout), body), ]), platform, path: path, diff --git a/pkgs/test_core/lib/src/runner/suite.dart b/pkgs/test_core/lib/src/runner/suite.dart index e4dbfed2c..b753bed3f 100644 --- a/pkgs/test_core/lib/src/runner/suite.dart +++ b/pkgs/test_core/lib/src/runner/suite.dart @@ -58,6 +58,7 @@ final class SuiteConfiguration { onPlatform: null, metadata: null, ignoreTimeouts: null, + suiteLoadTimeout: null, ); /// Whether or not duplicate test (or group) names are allowed within the same @@ -153,6 +154,10 @@ final class SuiteConfiguration { final bool? _ignoreTimeouts; bool get ignoreTimeouts => _ignoreTimeouts ?? false; + /// The timeout for loading a test suite. + final Timeout? _suiteLoadTimeout; + Timeout get suiteLoadTimeout => _suiteLoadTimeout ?? Timeout.none; + factory SuiteConfiguration({ required bool? allowDuplicateTestNames, required bool? allowTestRandomization, @@ -165,7 +170,7 @@ final class SuiteConfiguration { required Map? tags, required Map? onPlatform, required bool? ignoreTimeouts, - + required Timeout? suiteLoadTimeout, // Test-level configuration required Timeout? timeout, required bool? verboseTrace, @@ -189,6 +194,7 @@ final class SuiteConfiguration { tags: tags, onPlatform: onPlatform, ignoreTimeouts: ignoreTimeouts, + suiteLoadTimeout: suiteLoadTimeout, metadata: Metadata( timeout: timeout, verboseTrace: verboseTrace, @@ -219,6 +225,7 @@ final class SuiteConfiguration { Map? tags, Map? onPlatform, bool? ignoreTimeouts, + Timeout? suiteLoadTimeout, // Test-level configuration Timeout? timeout, @@ -241,6 +248,7 @@ final class SuiteConfiguration { tags: tags, onPlatform: onPlatform, ignoreTimeouts: ignoreTimeouts, + suiteLoadTimeout: suiteLoadTimeout, timeout: timeout, verboseTrace: verboseTrace, chainStackTraces: chainStackTraces, @@ -281,6 +289,7 @@ final class SuiteConfiguration { required Map? onPlatform, required Metadata? metadata, required bool? ignoreTimeouts, + required Timeout? suiteLoadTimeout, }) : _allowDuplicateTestNames = allowDuplicateTestNames, _allowTestRandomization = allowTestRandomization, _jsTrace = jsTrace, @@ -291,6 +300,7 @@ final class SuiteConfiguration { tags = _map(tags), onPlatform = _map(onPlatform), _ignoreTimeouts = ignoreTimeouts, + _suiteLoadTimeout = suiteLoadTimeout, _metadata = metadata ?? Metadata.empty; /// Creates a new [SuiteConfiguration] that takes its configuration from @@ -314,6 +324,7 @@ final class SuiteConfiguration { runtimes: null, compilerSelections: null, ignoreTimeouts: null, + suiteLoadTimeout: null, ); /// Returns an unmodifiable copy of [input]. @@ -359,6 +370,7 @@ final class SuiteConfiguration { onPlatform: _mergeConfigMaps(onPlatform, other.onPlatform), ignoreTimeouts: other._ignoreTimeouts ?? _ignoreTimeouts, metadata: metadata.merge(other.metadata), + suiteLoadTimeout: other._suiteLoadTimeout ?? _suiteLoadTimeout, ); return config._resolveTags(); } @@ -379,6 +391,7 @@ final class SuiteConfiguration { Map? tags, Map? onPlatform, bool? ignoreTimeouts, + Timeout? suiteLoadTimeout, // Test-level configuration Timeout? timeout, @@ -404,6 +417,7 @@ final class SuiteConfiguration { tags: tags ?? this.tags, onPlatform: onPlatform ?? this.onPlatform, ignoreTimeouts: ignoreTimeouts ?? _ignoreTimeouts, + suiteLoadTimeout: suiteLoadTimeout ?? _suiteLoadTimeout, metadata: _metadata.change( timeout: timeout, verboseTrace: verboseTrace, @@ -441,6 +455,7 @@ final class SuiteConfiguration { tags: tags, onPlatform: onPlatform, ignoreTimeouts: _ignoreTimeouts, + suiteLoadTimeout: suiteLoadTimeout, metadata: _metadata, ); }