feat(jest-haste-map): handle injected scm clocks (#10966)

Author: scotthovestadt

CHANGELOG.md

@@ -9,6 +9,7 @@
 - `[jest-config, jest-runtime]` Support ESM for files other than `.js` and `.mjs` ([#10823](https://github.com/facebook/jest/pull/10823))
 - `[jest-config, jest-runtime]` [**BREAKING**] Use "modern" implementation as default for fake timers ([#10874](https://github.com/facebook/jest/pull/10874))
 - `[jest-core]` make `TestWatcher` extend `emittery` ([#10324](https://github.com/facebook/jest/pull/10324))
+- `[jest-haste-map]` Handle injected scm clocks ([#10966](https://github.com/facebook/jest/pull/10966))
 - `[jest-repl, jest-runner]` [**BREAKING**] Run transforms over environment ([#8751](https://github.com/facebook/jest/pull/8751))
 - `[jest-runner]` [**BREAKING**] set exit code to 1 if test logs after teardown ([#10728](https://github.com/facebook/jest/pull/10728))
 - `[jest-snapshot]` [**BREAKING**] Make prettier optional for inline snapshots - fall back to string replacement ([#7792](https://github.com/facebook/jest/pull/7792))

packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js

@@ -589,4 +589,89 @@ describe('watchman watch', () => {
     expect(calls[0][0]).toEqual(['list-capabilities']);
     expect(calls[2][0][2].fields).not.toContain('content.sha1hex');
   });
+
+  test('source control query', async () => {
+    mockResponse = {
+      'list-capabilities': {
+        [undefined]: {
+          capabilities: ['field-content.sha1hex'],
+        },
+      },
+      query: {
+        [ROOT_MOCK]: {
+          clock: {
+            clock: 'c:1608612057:79675:1:139410',
+            scm: {
+              mergebase: 'master',
+              'mergebase-with': 'master',
+            },
+          },
+          files: [
+            {
+              exists: true,
+              mtime_ms: {toNumber: () => 42},
+              name: 'fruits/kiwi.js',
+              size: 40,
+            },
+            {
+              exists: false,
+              mtime_ms: null,
+              name: 'fruits/tomato.js',
+              size: 0,
+            },
+          ],
+          // Watchman is going to tell us that we have a fresh instance.
+          is_fresh_instance: true,
+          version: '4.5.0',
+        },
+      },
+      'watch-project': WATCH_PROJECT_MOCK,
+    };
+
+    // Start with a source-control clock.
+    const clocks = createMap({
+      '': {scm: {'mergebase-with': 'master'}},
+    });
+
+    const {changedFiles, hasteMap, removedFiles} = await watchmanCrawl({
+      data: {
+        clocks,
+        files: mockFiles,
+      },
+      extensions: ['js', 'json'],
+      ignore: pearMatcher,
+      rootDir: ROOT_MOCK,
+      roots: ROOTS,
+    });
+
+    // The object was reused.
+    expect(hasteMap.files).toBe(mockFiles);
+
+    // Transformed into a normal clock.
+    expect(hasteMap.clocks).toEqual(
+      createMap({
+        '': 'c:1608612057:79675:1:139410',
+      }),
+    );
+
+    expect(changedFiles).toEqual(
+      createMap({
+        [KIWI_RELATIVE]: ['', 42, 40, 0, '', null],
+      }),
+    );
+
+    expect(hasteMap.files).toEqual(
+      createMap({
+        [KIWI_RELATIVE]: ['', 42, 40, 0, '', null],
+        [MELON_RELATIVE]: ['', 33, 43, 0, '', null],
+        [STRAWBERRY_RELATIVE]: ['', 30, 40, 0, '', null],
+      }),
+    );
+
+    expect(removedFiles).toEqual(
+      createMap({
+        [TOMATO_RELATIVE]: ['', 31, 41, 0, '', null],
+      }),
+    );
+  });
 });

packages/jest-haste-map/src/crawlers/watchman.ts

@@ -20,6 +20,34 @@ import type {
 
 type WatchmanRoots = Map<string, Array<string>>;
 
+type WatchmanListCapabilitiesResponse = {
+  capabilities: Array<string>;
+};
+
+type WatchmanWatchProjectResponse = {
+  watch: string;
+  relative_path: string;
+};
+
+type WatchmanQueryResponse = {
+  warning?: string;
+  is_fresh_instance: boolean;
+  version: string;
+  clock:
+    | string
+    | {
+        scm: {'mergebase-with': string; mergebase: string};
+        clock: string;
+      };
+  files: Array<{
+    name: string;
+    exists: boolean;
+    mtime_ms: number | {toNumber: () => number};
+    size: number;
+    'content.sha1hex'?: string;
+  }>;
+};
+
 const watchmanURL = 'https://facebook.github.io/watchman/docs/troubleshooting';
 
 function WatchmanError(error: Error): Error {
@@ -49,16 +77,17 @@ export = async function watchmanCrawl(
   let clientError;
   client.on('error', error => (clientError = WatchmanError(error)));
 
-  // TODO: type better than `any`
-  const cmd = (...args: Array<any>): Promise<any> =>
+  const cmd = <T>(...args: Array<any>): Promise<T> =>
     new Promise((resolve, reject) =>
       client.command(args, (error, result) =>
         error ? reject(WatchmanError(error)) : resolve(result),
       ),
     );
 
   if (options.computeSha1) {
-    const {capabilities} = await cmd('list-capabilities');
+    const {capabilities} = await cmd<WatchmanListCapabilitiesResponse>(
+      'list-capabilities',
+    );
 
     if (capabilities.indexOf('field-content.sha1hex') !== -1) {
       fields.push('content.sha1hex');
@@ -71,7 +100,10 @@ export = async function watchmanCrawl(
     const watchmanRoots = new Map();
     await Promise.all(
       roots.map(async root => {
-        const response = await cmd('watch-project', root);
+        const response = await cmd<WatchmanWatchProjectResponse>(
+          'watch-project',
+          root,
+        );
         const existing = watchmanRoots.get(response.watch);
         // A root can only be filtered if it was never seen with a
         // relative_path before.
@@ -96,7 +128,7 @@ export = async function watchmanCrawl(
   }
 
   async function queryWatchmanForDirs(rootProjectDirMappings: WatchmanRoots) {
-    const files = new Map();
+    const results = new Map<string, WatchmanQueryResponse>();
     let isFresh = false;
     await Promise.all(
       Array.from(rootProjectDirMappings).map(
@@ -121,35 +153,58 @@ export = async function watchmanCrawl(
             }
           }
 
-          const relativeRoot = fastPath.relative(rootDir, root);
-          const query = clocks.has(relativeRoot)
-            ? // Use the `since` generator if we have a clock available
-              {expression, fields, since: clocks.get(relativeRoot)}
-            : // Otherwise use the `glob` filter
-              {expression, fields, glob, glob_includedotfiles: true};
-
-          const response = await cmd('query', root, query);
+          // Jest is only going to store one type of clock; a string that
+          // represents a local clock. However, the Watchman crawler supports
+          // a second type of clock that can be written by automation outside of
+          // Jest, called an "scm query", which fetches changed files based on
+          // source control mergebases. The reason this is necessary is because
+          // local clocks are not portable across systems, but scm queries are.
+          // By using scm queries, we can create the haste map on a different
+          // system and import it, transforming the clock into a local clock.
+          const since = clocks.get(fastPath.relative(rootDir, root));
+
+          const query =
+            since !== undefined
+              ? // Use the `since` generator if we have a clock available
+                {expression, fields, since}
+              : // Otherwise use the `glob` filter
+                {expression, fields, glob, glob_includedotfiles: true};
+
+          const response = await cmd<WatchmanQueryResponse>(
+            'query',
+            root,
+            query,
+          );
 
           if ('warning' in response) {
             console.warn('watchman warning: ', response.warning);
           }
 
-          isFresh = isFresh || response.is_fresh_instance;
-          files.set(root, response);
+          // When a source-control query is used, we ignore the "is fresh"
+          // response from Watchman because it will be true despite the query
+          // being incremental.
+          const isSourceControlQuery =
+            typeof since !== 'string' &&
+            since?.scm?.['mergebase-with'] !== undefined;
+          if (!isSourceControlQuery) {
+            isFresh = isFresh || response.is_fresh_instance;
+          }
+
+          results.set(root, response);
         },
       ),
     );
 
     return {
-      files,
       isFresh,
+      results,
     };
   }
 
   let files = data.files;
   let removedFiles = new Map();
   const changedFiles = new Map();
-  let watchmanFiles: Map<string, any>;
+  let results: Map<string, WatchmanQueryResponse>;
   let isFresh = false;
   try {
     const watchmanRoots = await getWatchmanRoots(roots);
@@ -163,7 +218,7 @@ export = async function watchmanCrawl(
       isFresh = true;
     }
 
-    watchmanFiles = watchmanFileResults.files;
+    results = watchmanFileResults.results;
   } finally {
     client.end();
   }
@@ -172,11 +227,16 @@ export = async function watchmanCrawl(
     throw clientError;
   }
 
-  // TODO: remove non-null
-  for (const [watchRoot, response] of watchmanFiles!) {
+  for (const [watchRoot, response] of results) {
     const fsRoot = normalizePathSep(watchRoot);
     const relativeFsRoot = fastPath.relative(rootDir, fsRoot);
-    clocks.set(relativeFsRoot, response.clock);
+    clocks.set(
+      relativeFsRoot,
+      // Ensure we persist only the local clock.
+      typeof response.clock === 'string'
+        ? response.clock
+        : response.clock.clock,
+    );
 
     for (const fileData of response.files) {
       const filePath = fsRoot + path.sep + normalizePathSep(fileData.name);
@@ -209,7 +269,7 @@ export = async function watchmanCrawl(
 
         let sha1hex = fileData['content.sha1hex'];
         if (typeof sha1hex !== 'string' || sha1hex.length !== 40) {
-          sha1hex = null;
+          sha1hex = undefined;
         }
 
         let nextData: FileMetaData;
@@ -231,7 +291,7 @@ export = async function watchmanCrawl(
           ];
         } else {
           // See ../constants.ts
-          nextData = ['', mtime, size, 0, '', sha1hex];
+          nextData = ['', mtime, size, 0, '', sha1hex ?? null];
         }
 
         files.set(relativeFilePath, nextData);

packages/jest-haste-map/src/types.ts

@@ -55,7 +55,8 @@ export type FileMetaData = [
 
 export type MockData = Map<string, Config.Path>;
 export type ModuleMapData = Map<string, ModuleMapItem>;
-export type WatchmanClocks = Map<Config.Path, string>;
+export type WatchmanClockSpec = string | {scm: {'mergebase-with': string}};
+export type WatchmanClocks = Map<Config.Path, WatchmanClockSpec>;
 export type HasteRegExp = RegExp | ((str: string) => boolean);
 
 export type DuplicatesSet = Map<string, /* type */ number>;