Merge pull request #17634 from Microsoft/fixSignatureStackOverflow

Fix stack overflow related to contextual signature instantiations

Author: ahejlsberg

src/compiler/checker.ts

@@ -8074,7 +8074,7 @@ namespace ts {
 
         function cloneTypeMapper(mapper: TypeMapper): TypeMapper {
             return mapper && isInferenceContext(mapper) ?
-                createInferenceContext(mapper.signature, mapper.flags | InferenceFlags.NoDefault, mapper.inferences) :
+                createInferenceContext(mapper.signature, mapper.flags | InferenceFlags.NoDefault, mapper.compareTypes, mapper.inferences) :
                 mapper;
         }
 
@@ -8515,7 +8515,7 @@ namespace ts {
             ignoreReturnTypes: boolean,
             reportErrors: boolean,
             errorReporter: ErrorReporter,
-            compareTypes: (s: Type, t: Type, reportErrors?: boolean) => Ternary): Ternary {
+            compareTypes: TypeComparer): Ternary {
             // TODO (drosen): De-duplicate code between related functions.
             if (source === target) {
                 return Ternary.True;
@@ -8525,7 +8525,7 @@ namespace ts {
             }
 
             if (source.typeParameters) {
-                source = instantiateSignatureInContextOf(source, target);
+                source = instantiateSignatureInContextOf(source, target, /*contextualMapper*/ undefined, compareTypes);
             }
 
             let result = Ternary.True;
@@ -10273,13 +10273,14 @@ namespace ts {
             }
         }
 
-        function createInferenceContext(signature: Signature, flags: InferenceFlags, baseInferences?: InferenceInfo[]): InferenceContext {
+        function createInferenceContext(signature: Signature, flags: InferenceFlags, compareTypes?: TypeComparer, baseInferences?: InferenceInfo[]): InferenceContext {
             const inferences = baseInferences ? map(baseInferences, cloneInferenceInfo) : map(signature.typeParameters, createInferenceInfo);
             const context = mapper as InferenceContext;
             context.mappedTypes = signature.typeParameters;
             context.signature = signature;
             context.inferences = inferences;
             context.flags = flags;
+            context.compareTypes = compareTypes || compareTypesAssignable;
             return context;
 
             function mapper(t: Type): Type {
@@ -10727,7 +10728,7 @@ namespace ts {
                 const constraint = getConstraintOfTypeParameter(context.signature.typeParameters[index]);
                 if (constraint) {
                     const instantiatedConstraint = instantiateType(constraint, context);
-                    if (!isTypeAssignableTo(inferredType, getTypeWithThisArgument(instantiatedConstraint, inferredType))) {
+                    if (!context.compareTypes(inferredType, getTypeWithThisArgument(instantiatedConstraint, inferredType))) {
                         inference.inferredType = inferredType = instantiatedConstraint;
                     }
                 }
@@ -15128,8 +15129,8 @@ namespace ts {
         }
 
         // Instantiate a generic signature in the context of a non-generic signature (section 3.8.5 in TypeScript spec)
-        function instantiateSignatureInContextOf(signature: Signature, contextualSignature: Signature, contextualMapper?: TypeMapper): Signature {
-            const context = createInferenceContext(signature, InferenceFlags.InferUnionTypes);
+        function instantiateSignatureInContextOf(signature: Signature, contextualSignature: Signature, contextualMapper?: TypeMapper, compareTypes?: TypeComparer): Signature {
+            const context = createInferenceContext(signature, InferenceFlags.InferUnionTypes, compareTypes);
             forEachMatchingParameterType(contextualSignature, signature, (source, target) => {
                 // Type parameters from outer context referenced by source type are fixed by instantiation of the source type
                 inferTypes(context.inferences, instantiateType(source, contextualMapper || identityMapper), target);

src/compiler/core.ts

@@ -11,20 +11,6 @@ namespace ts {
 
 /* @internal */
 namespace ts {
-    /**
-     * Ternary values are defined such that
-     * x & y is False if either x or y is False.
-     * x & y is Maybe if either x or y is Maybe, but neither x or y is False.
-     * x & y is True if both x and y are True.
-     * x | y is False if both x and y are False.
-     * x | y is Maybe if either x or y is Maybe, but neither x or y is True.
-     * x | y is True if either x or y is True.
-     */
-    export const enum Ternary {
-        False = 0,
-        Maybe = 1,
-        True = -1
-    }
 
     // More efficient to create a collator once and use its `compare` than to call `a.localeCompare(b)` many times.
     export const collator: { compare(a: string, b: string): number } = typeof Intl === "object" && typeof Intl.Collator === "function" ? new Intl.Collator(/*locales*/ undefined, { usage: "sort", sensitivity: "accent" }) : undefined;

src/compiler/types.ts

@@ -3425,11 +3425,29 @@ namespace ts {
         AnyDefault      = 1 << 2,  // Infer anyType for no inferences (otherwise emptyObjectType)
     }
 
+    /**
+     * Ternary values are defined such that
+     * x & y is False if either x or y is False.
+     * x & y is Maybe if either x or y is Maybe, but neither x or y is False.
+     * x & y is True if both x and y are True.
+     * x | y is False if both x and y are False.
+     * x | y is Maybe if either x or y is Maybe, but neither x or y is True.
+     * x | y is True if either x or y is True.
+     */
+    export const enum Ternary {
+        False = 0,
+        Maybe = 1,
+        True = -1
+    }
+
+    export type TypeComparer = (s: Type, t: Type, reportErrors?: boolean) => Ternary;
+
     /* @internal */
     export interface InferenceContext extends TypeMapper {
         signature: Signature;               // Generic signature for which inferences are made
         inferences: InferenceInfo[];        // Inferences made for each type parameter
         flags: InferenceFlags;              // Inference flags
+        compareTypes: TypeComparer;         // Type comparer function
     }
 
     /* @internal */

tests/baselines/reference/signatureInstantiationWithRecursiveConstraints.js

@@ -0,0 +1,30 @@
+//// [signatureInstantiationWithRecursiveConstraints.ts]
+// Repro from #17148
+
+class Foo {
+  myFunc<T extends Foo>(arg: T) {}
+}
+
+class Bar {
+  myFunc<T extends Bar>(arg: T) {}
+}
+
+const myVar: Foo = new Bar();
+
+
+//// [signatureInstantiationWithRecursiveConstraints.js]
+"use strict";
+// Repro from #17148
+var Foo = (function () {
+    function Foo() {
+    }
+    Foo.prototype.myFunc = function (arg) { };
+    return Foo;
+}());
+var Bar = (function () {
+    function Bar() {
+    }
+    Bar.prototype.myFunc = function (arg) { };
+    return Bar;
+}());
+var myVar = new Bar();

tests/baselines/reference/signatureInstantiationWithRecursiveConstraints.symbols

@@ -0,0 +1,30 @@
+=== tests/cases/compiler/signatureInstantiationWithRecursiveConstraints.ts ===
+// Repro from #17148
+
+class Foo {
+>Foo : Symbol(Foo, Decl(signatureInstantiationWithRecursiveConstraints.ts, 0, 0))
+
+  myFunc<T extends Foo>(arg: T) {}
+>myFunc : Symbol(Foo.myFunc, Decl(signatureInstantiationWithRecursiveConstraints.ts, 2, 11))
+>T : Symbol(T, Decl(signatureInstantiationWithRecursiveConstraints.ts, 3, 9))
+>Foo : Symbol(Foo, Decl(signatureInstantiationWithRecursiveConstraints.ts, 0, 0))
+>arg : Symbol(arg, Decl(signatureInstantiationWithRecursiveConstraints.ts, 3, 24))
+>T : Symbol(T, Decl(signatureInstantiationWithRecursiveConstraints.ts, 3, 9))
+}
+
+class Bar {
+>Bar : Symbol(Bar, Decl(signatureInstantiationWithRecursiveConstraints.ts, 4, 1))
+
+  myFunc<T extends Bar>(arg: T) {}
+>myFunc : Symbol(Bar.myFunc, Decl(signatureInstantiationWithRecursiveConstraints.ts, 6, 11))
+>T : Symbol(T, Decl(signatureInstantiationWithRecursiveConstraints.ts, 7, 9))
+>Bar : Symbol(Bar, Decl(signatureInstantiationWithRecursiveConstraints.ts, 4, 1))
+>arg : Symbol(arg, Decl(signatureInstantiationWithRecursiveConstraints.ts, 7, 24))
+>T : Symbol(T, Decl(signatureInstantiationWithRecursiveConstraints.ts, 7, 9))
+}
+
+const myVar: Foo = new Bar();
+>myVar : Symbol(myVar, Decl(signatureInstantiationWithRecursiveConstraints.ts, 10, 5))
+>Foo : Symbol(Foo, Decl(signatureInstantiationWithRecursiveConstraints.ts, 0, 0))
+>Bar : Symbol(Bar, Decl(signatureInstantiationWithRecursiveConstraints.ts, 4, 1))
+

tests/baselines/reference/signatureInstantiationWithRecursiveConstraints.types

@@ -0,0 +1,31 @@
+=== tests/cases/compiler/signatureInstantiationWithRecursiveConstraints.ts ===
+// Repro from #17148
+
+class Foo {
+>Foo : Foo
+
+  myFunc<T extends Foo>(arg: T) {}
+>myFunc : <T extends Foo>(arg: T) => void
+>T : T
+>Foo : Foo
+>arg : T
+>T : T
+}
+
+class Bar {
+>Bar : Bar
+
+  myFunc<T extends Bar>(arg: T) {}
+>myFunc : <T extends Bar>(arg: T) => void
+>T : T
+>Bar : Bar
+>arg : T
+>T : T
+}
+
+const myVar: Foo = new Bar();
+>myVar : Foo
+>Foo : Foo
+>new Bar() : Bar
+>Bar : typeof Bar
+

tests/cases/compiler/signatureInstantiationWithRecursiveConstraints.ts

@@ -0,0 +1,13 @@
+// @strict: true
+
+// Repro from #17148
+
+class Foo {
+  myFunc<T extends Foo>(arg: T) {}
+}
+
+class Bar {
+  myFunc<T extends Bar>(arg: T) {}
+}
+
+const myVar: Foo = new Bar();