[ddc] Update hot reload test suite to use recompile-restart

A new optional suffix to the file names is added: '.restart'.
This allows us to differentiate between hot reload generations
and hot restart generations. All files with the same generation
should either have the suffix or don't. If present, uses the
recompile-request instruction in the frontend server. If not,
uses the recompile instruction. Tests are renamed and a WIP
README is added to document this change.

Change-Id: I2691c5ec0b7336115a76dc0e0369213e74b6ea21
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/406183
Commit-Queue: Srujan Gaddam <[email protected]>
Reviewed-by: Nicholas Shahan <[email protected]>

Author: srujzs

pkg/dev_compiler/test/hot_reload_suite.dart

@@ -169,8 +169,12 @@ class HotReloadTest {
   // TODO(nshahan): Support multiple expected errors for a single generation.
   final Map<int, String> expectedErrors;
 
+  /// Map of generation number to whether a hot restart was triggered before the
+  /// generation.
+  final Map<int, bool> isHotRestart;
+
   HotReloadTest(this.directory, this.name, this.generationCount, this.files,
-      ReloadTestConfiguration config)
+      ReloadTestConfiguration config, this.isHotRestart)
       : excludedPlatforms = config.excludedPlatforms,
         expectedErrors = config.expectedErrors;
 
@@ -376,8 +380,9 @@ abstract class HotReloadSuiteRunner {
         r'^(?<name>[\w,-]+)'
         // Followed by a dot and 1 or more digits.
         r'\.(?<generation>\d+)'
-        // Optionally a dot and the word 'reject'.
-        r'(?<reject>\.reject)?'
+        // Optionally a dot and either the word 'restart' indicating a hot
+        // restart, or the word 'reject', indicating a hot reload rejection.
+        r'((?<restart>\.restart)|(?<reject>\.reject))?'
         // Ending with a dot and the word 'dart'
         r'\.dart$');
     final testSuite = <HotReloadTest>[];
@@ -409,6 +414,7 @@ abstract class HotReloadSuiteRunner {
             label: testName);
         continue;
       }
+      final isHotRestart = <int, bool>{};
       final expectedErrors = testConfig.expectedErrors;
       final dartFiles = testDir
           .listSync()
@@ -420,15 +426,26 @@ abstract class HotReloadSuiteRunner {
         final matches = validTestSourceName.allMatches(fileName);
         if (matches.length != 1) {
           throw Exception('Invalid test source file name: $fileName\n'
-              'Valid names look like "file_name.10.dart" '
-              'or "file_name.10.reject.dart".');
+              'Valid names look like "file_name.10.dart", '
+              '"file_name.10.restart.dart" or "file_name.10.reject.dart".');
         }
         final match = matches.single;
         final name = match.namedGroup('name');
         final restoredName = '$name.dart';
         final generation = int.parse(match.namedGroup('generation')!);
         maxGenerations = max(maxGenerations, generation);
+        final restart = match.namedGroup('restart') != null;
+        if (!isHotRestart.containsKey(generation)) {
+          isHotRestart[generation] = restart;
+        } else {
+          if (restart != isHotRestart[generation]) {
+            throw Exception('Expected all files for generation $generation to '
+                "be consistent about having a '.restart' suffix, but $fileName "
+                'does not match other files in the same generation.');
+          }
+        }
         final rejectExpected = match.namedGroup('reject') != null;
+        assert(!(rejectExpected && restart));
         if (rejectExpected && !expectedErrors.containsKey(generation)) {
           throw Exception(
               'Expected error for generation file missing from config.json: '
@@ -465,8 +482,8 @@ abstract class HotReloadSuiteRunner {
             {for (final edit in fileEdits) edit.generation: edit});
         testFiles.add(TestFile(fileName, editsByGeneration));
       }
-      testSuite.add(HotReloadTest(
-          testDir, testName, maxGenerations + 1, testFiles, testConfig));
+      testSuite.add(HotReloadTest(testDir, testName, maxGenerations + 1,
+          testFiles, testConfig, isHotRestart));
     }
     return testSuite;
   }
@@ -986,7 +1003,8 @@ abstract class DdcSuiteRunner extends HotReloadSuiteRunner {
       outputDillPath = outputIncrementalDillUri.toFilePath();
       compilerOutput = await controller.sendRecompile(
           snapshotEntrypointWithScheme,
-          invalidatedFiles: updatedFiles);
+          invalidatedFiles: updatedFiles,
+          recompileRestart: test.isHotRestart[generation]!);
     }
     final expectedError = test.expectedErrors[generation];
     if (compilerOutput.errorCount > 0) {
@@ -1289,7 +1307,10 @@ class VMSuiteRunner extends HotReloadSuiteRunner {
       // TODO(markzipan): Add logic to reject bad compiles.
       compilerOutput = await controller.sendRecompile(
           snapshotEntrypointWithScheme,
-          invalidatedFiles: updatedFiles);
+          invalidatedFiles: updatedFiles,
+          // The VM never uses the `recompile-restart` instruction as it does
+          // not recompile during a hot restart.
+          recompileRestart: false);
     }
     var hasExpectedCompileError = false;
     final expectedError = test.expectedErrors[generation];

pkg/reload_test/README.md

@@ -0,0 +1,18 @@
+# Testing Framework for Hot Reload/Restart Tests for DDC
+
+<!--- TODO(srujzs): Document this package more. For now, document some
+conventions that may not be obvious. --->
+
+This package contains utilities to test multiple generations of files in order
+to validate hot reload and hot restart.
+
+## Test File Conventions
+
+- Different generations of files have a generation number after the file name
+e.g. `file_name.1.dart` is the generation 1-version of the file.
+- If a generation file is intended to be rejected, it should contain `.reject`
+in the file name. The `config.json` file should contain a key `"expectedErrors"`
+with its value being a map of generation number string to the error string.
+- If a generation does a hot restart instead of a reload, it should contain
+`.restart` in every file name in the same generation.
+- It is an error to specify both `.reject` and `.restart`.

pkg/reload_test/lib/frontend_server_controller.dart

@@ -227,7 +227,8 @@ class HotReloadFrontendServerController {
 
   Future<CompilerOutput> sendRecompile(String entrypointPath,
       {List<String> invalidatedFiles = const [],
-      String boundaryKey = fakeBoundaryKey}) async {
+      String boundaryKey = fakeBoundaryKey,
+      required bool recompileRestart}) async {
     // Currently the `FrontendCompiler` used in this test suite clears errors
     // before performing a recompile but not an initial compile. Since we reuse
     // the same instance and issue an initial compile request for each test we
@@ -237,7 +238,8 @@ class HotReloadFrontendServerController {
     totalErrors = 0;
     if (!started) throw Exception('Frontend Server has not been started yet.');
     _state = FrontendServerState.awaitingResult;
-    final command = 'recompile $entrypointPath $boundaryKey\n'
+    final instruction = recompileRestart ? 'recompile-restart' : 'recompile';
+    final command = '$instruction $entrypointPath $boundaryKey\n'
         '${invalidatedFiles.join('\n')}\n$boundaryKey\n';
     if (debug) print('Sending instruction to Frontend Server:\n$command');
     input.add(command.codeUnits);
@@ -247,9 +249,12 @@ class HotReloadFrontendServerController {
 
   Future<void> sendRecompileAndAccept(String entrypointPath,
       {List<String> invalidatedFiles = const [],
-      String boundaryKey = fakeBoundaryKey}) async {
+      String boundaryKey = fakeBoundaryKey,
+      required bool recompileRestart}) async {
     await sendRecompile(entrypointPath,
-        invalidatedFiles: invalidatedFiles, boundaryKey: boundaryKey);
+        invalidatedFiles: invalidatedFiles,
+        boundaryKey: boundaryKey,
+        recompileRestart: recompileRestart);
     sendAccept();
   }
 

tests/hot_reload/framework_timing_test/main.0.dart

@@ -1,4 +1,4 @@
-// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 

tests/hot_reload/framework_timing_test/main.1.dart renamed to tests/hot_reload/framework_timing_test/main.1.restart.dart

@@ -1,4 +1,4 @@
-// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 

tests/hot_reload/framework_timing_test/main.2.dart renamed to tests/hot_reload/framework_timing_test/main.2.restart.dart

@@ -1,4 +1,4 @@
-// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 

tests/hot_reload/hot_restart_constant_equality/library_b.0.dart

@@ -1,4 +1,4 @@
-// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 

tests/hot_reload/hot_restart_constant_equality/library_b.1.dart renamed to tests/hot_reload/hot_restart_constant_equality/library_b.1.restart.dart

@@ -1,4 +1,4 @@
-// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 

tests/hot_reload/hot_restart_constant_equality/main.0.dart

@@ -1,4 +1,4 @@
-// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 

tests/hot_reload/hot_restart_constant_equality/main.1.dart renamed to tests/hot_reload/hot_restart_constant_equality/main.1.restart.dart

@@ -1,4 +1,4 @@
-// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.