Guard against recursion in inferTypeForHomomorphicMappedType (microsoft#38224)

* Guard against recursion in inferTypeForHomomorphicMappedType

* Add regression test

Author: ahejlsberg

src/compiler/checker.ts

@@ -827,6 +827,7 @@ namespace ts {
         /** Key is "/path/to/a.ts|/path/to/b.ts". */
         let amalgamatedDuplicates: Map<DuplicateInfoForFiles> | undefined;
         const reverseMappedCache = createMap<Type | undefined>();
+        let inInferTypeForHomomorphicMappedType = false;
         let ambientModulesCache: Symbol[] | undefined;
         /**
          * List of every ambient module with a "*" wildcard.
@@ -18366,12 +18367,16 @@ namespace ts {
          * variable T[P] (i.e. we treat the type T[P] as the type variable we're inferring for).
          */
         function inferTypeForHomomorphicMappedType(source: Type, target: MappedType, constraint: IndexType): Type | undefined {
+            if (inInferTypeForHomomorphicMappedType) {
+                return undefined;
+            }
             const key = source.id + "," + target.id + "," + constraint.id;
             if (reverseMappedCache.has(key)) {
                 return reverseMappedCache.get(key);
             }
-            reverseMappedCache.set(key, undefined);
+            inInferTypeForHomomorphicMappedType = true;
             const type = createReverseMappedType(source, target, constraint);
+            inInferTypeForHomomorphicMappedType = false;
             reverseMappedCache.set(key, type);
             return type;
         }

tests/baselines/reference/recursiveReverseMappedType.js

@@ -0,0 +1,32 @@
+//// [recursiveReverseMappedType.ts]
+// Repro from #38198
+
+type Recur<T> = (
+    T extends (unknown[]) ? {} : { [K in keyof T]?: Recur<T[K]> }
+) | ['marker', ...Recur<T>[]];
+
+function join<T>(l: Recur<T>[]): Recur<T> {
+    return ['marker', ...l];
+}
+
+function a<T>(l: Recur<T>[]): void {
+    const x: Recur<T> | undefined = join(l);
+}
+
+
+//// [recursiveReverseMappedType.js]
+"use strict";
+// Repro from #38198
+var __spreadArrays = (this && this.__spreadArrays) || function () {
+    for (var s = 0, i = 0, il = arguments.length; i < il; i++) s += arguments[i].length;
+    for (var r = Array(s), k = 0, i = 0; i < il; i++)
+        for (var a = arguments[i], j = 0, jl = a.length; j < jl; j++, k++)
+            r[k] = a[j];
+    return r;
+};
+function join(l) {
+    return __spreadArrays(['marker'], l);
+}
+function a(l) {
+    var x = join(l);
+}

tests/baselines/reference/recursiveReverseMappedType.symbols

@@ -0,0 +1,47 @@
+=== tests/cases/compiler/recursiveReverseMappedType.ts ===
+// Repro from #38198
+
+type Recur<T> = (
+>Recur : Symbol(Recur, Decl(recursiveReverseMappedType.ts, 0, 0))
+>T : Symbol(T, Decl(recursiveReverseMappedType.ts, 2, 11))
+
+    T extends (unknown[]) ? {} : { [K in keyof T]?: Recur<T[K]> }
+>T : Symbol(T, Decl(recursiveReverseMappedType.ts, 2, 11))
+>K : Symbol(K, Decl(recursiveReverseMappedType.ts, 3, 36))
+>T : Symbol(T, Decl(recursiveReverseMappedType.ts, 2, 11))
+>Recur : Symbol(Recur, Decl(recursiveReverseMappedType.ts, 0, 0))
+>T : Symbol(T, Decl(recursiveReverseMappedType.ts, 2, 11))
+>K : Symbol(K, Decl(recursiveReverseMappedType.ts, 3, 36))
+
+) | ['marker', ...Recur<T>[]];
+>Recur : Symbol(Recur, Decl(recursiveReverseMappedType.ts, 0, 0))
+>T : Symbol(T, Decl(recursiveReverseMappedType.ts, 2, 11))
+
+function join<T>(l: Recur<T>[]): Recur<T> {
+>join : Symbol(join, Decl(recursiveReverseMappedType.ts, 4, 30))
+>T : Symbol(T, Decl(recursiveReverseMappedType.ts, 6, 14))
+>l : Symbol(l, Decl(recursiveReverseMappedType.ts, 6, 17))
+>Recur : Symbol(Recur, Decl(recursiveReverseMappedType.ts, 0, 0))
+>T : Symbol(T, Decl(recursiveReverseMappedType.ts, 6, 14))
+>Recur : Symbol(Recur, Decl(recursiveReverseMappedType.ts, 0, 0))
+>T : Symbol(T, Decl(recursiveReverseMappedType.ts, 6, 14))
+
+    return ['marker', ...l];
+>l : Symbol(l, Decl(recursiveReverseMappedType.ts, 6, 17))
+}
+
+function a<T>(l: Recur<T>[]): void {
+>a : Symbol(a, Decl(recursiveReverseMappedType.ts, 8, 1))
+>T : Symbol(T, Decl(recursiveReverseMappedType.ts, 10, 11))
+>l : Symbol(l, Decl(recursiveReverseMappedType.ts, 10, 14))
+>Recur : Symbol(Recur, Decl(recursiveReverseMappedType.ts, 0, 0))
+>T : Symbol(T, Decl(recursiveReverseMappedType.ts, 10, 11))
+
+    const x: Recur<T> | undefined = join(l);
+>x : Symbol(x, Decl(recursiveReverseMappedType.ts, 11, 9))
+>Recur : Symbol(Recur, Decl(recursiveReverseMappedType.ts, 0, 0))
+>T : Symbol(T, Decl(recursiveReverseMappedType.ts, 10, 11))
+>join : Symbol(join, Decl(recursiveReverseMappedType.ts, 4, 30))
+>l : Symbol(l, Decl(recursiveReverseMappedType.ts, 10, 14))
+}
+

tests/baselines/reference/recursiveReverseMappedType.types

@@ -0,0 +1,31 @@
+=== tests/cases/compiler/recursiveReverseMappedType.ts ===
+// Repro from #38198
+
+type Recur<T> = (
+>Recur : Recur<T>
+
+    T extends (unknown[]) ? {} : { [K in keyof T]?: Recur<T[K]> }
+) | ['marker', ...Recur<T>[]];
+
+function join<T>(l: Recur<T>[]): Recur<T> {
+>join : <T>(l: Recur<T>[]) => Recur<T>
+>l : Recur<T>[]
+
+    return ['marker', ...l];
+>['marker', ...l] : ["marker", ...Recur<T>[]]
+>'marker' : "marker"
+>...l : Recur<T>
+>l : Recur<T>[]
+}
+
+function a<T>(l: Recur<T>[]): void {
+>a : <T>(l: Recur<T>[]) => void
+>l : Recur<T>[]
+
+    const x: Recur<T> | undefined = join(l);
+>x : (T extends unknown[] ? {} : { [K in keyof T]?: (T[K] extends unknown[] ? {} : { [K in keyof T[K]]?: (T[K][K] extends unknown[] ? {} : { [K in keyof T[K][K]]?: (T[K][K][K] extends unknown[] ? {} : { [K in keyof T[K][K][K]]?: (T[K][K][K][K] extends unknown[] ? {} : { [K in keyof T[K][K][K][K]]?: (T[K][K][K][K][K] extends unknown[] ? {} : { [K in keyof T[K][K][K][K][K]]?: (T[K][K][K][K][K][K] extends unknown[] ? {} : { [K in keyof T[K][K][K][K][K][K]]?: (T[K][K][K][K][K][K][K] extends unknown[] ? {} : { [K in keyof T[K][K][K][K][K][K][K]]?: (T[K][K][K][K][K][K][K][K] extends unknown[] ? {} : { [K in keyof T[K][K][K][K][K][K][K][K]]?: (T[K][K][K][K][K][K][K][K][K] extends unknown[] ? {} : { [K in keyof T[K][K][K][K][K][K][K][K][K]]?: (T[K][K][K][K][K][K][K][K][K][K] extends unknown[] ? {} : { [K in keyof T[K][K][K][K][K][K][K][K][K][K]]?: (T[K][K][K][K][K][K][K][K][K][K][K] extends unknown[] ? {} : any) | ["marker", ...Recur<T[K][K][K][K][K][K][K][K][K][K][K]>[]] | undefined; }) | ["marker", ...Recur<T[K][K][K][K][K][K][K][K][K][K]>[]] | undefined; }) | ["marker", ...Recur<T[K][K][K][K][K][K][K][K][K]>[]] | undefined; }) | ["marker", ...Recur<T[K][K][K][K][K][K][K][K]>[]] | undefined; }) | ["marker", ...Recur<T[K][K][K][K][K][K][K]>[]] | undefined; }) | ["marker", ...Recur<T[K][K][K][K][K][K]>[]] | undefined; }) | ["marker", ...Recur<T[K][K][K][K][K]>[]] | undefined; }) | ["marker", ...Recur<T[K][K][K][K]>[]] | undefined; }) | ["marker", ...Recur<T[K][K][K]>[]] | undefined; }) | ["marker", ...Recur<T[K][K]>[]] | undefined; }) | ["marker", ...Recur<T[K]>[]] | undefined; }) | ["marker", ...Recur<T>[]] | undefined
+>join(l) : Recur<T>
+>join : <T>(l: Recur<T>[]) => Recur<T>
+>l : Recur<T>[]
+}
+

tests/cases/compiler/recursiveReverseMappedType.ts

@@ -0,0 +1,15 @@
+// @strict: true
+
+// Repro from #38198
+
+type Recur<T> = (
+    T extends (unknown[]) ? {} : { [K in keyof T]?: Recur<T[K]> }
+) | ['marker', ...Recur<T>[]];
+
+function join<T>(l: Recur<T>[]): Recur<T> {
+    return ['marker', ...l];
+}
+
+function a<T>(l: Recur<T>[]): void {
+    const x: Recur<T> | undefined = join(l);
+}