From 73317dc5c407568fe56a8b746690b25f1f29c39e Mon Sep 17 00:00:00 2001
From: Wesley Wigham <wwigham@gmail.com>
Date: Thu, 16 Apr 2020 14:14:14 -0700
Subject: [PATCH 1/2] Handle recursive type references up to a certain level of
 expansion in inference

---
 src/compiler/checker.ts                       | 31 +++++++++++-
 .../selfReferencingTypeReferenceInference.js  | 23 +++++++++
 ...fReferencingTypeReferenceInference.symbols | 49 +++++++++++++++++++
 ...elfReferencingTypeReferenceInference.types | 27 ++++++++++
 .../selfReferencingTypeReferenceInference.ts  | 18 +++++++
 5 files changed, 147 insertions(+), 1 deletion(-)
 create mode 100644 tests/baselines/reference/selfReferencingTypeReferenceInference.js
 create mode 100644 tests/baselines/reference/selfReferencingTypeReferenceInference.symbols
 create mode 100644 tests/baselines/reference/selfReferencingTypeReferenceInference.types
 create mode 100644 tests/cases/compiler/selfReferencingTypeReferenceInference.ts

diff --git a/src/compiler/checker.ts b/src/compiler/checker.ts
index 087353fb9fdc3..0efef2d9a585c 100644
--- a/src/compiler/checker.ts
+++ b/src/compiler/checker.ts
@@ -17332,6 +17332,10 @@ namespace ts {
         // In addition, this will also detect when an indexed access has been chained off of 5 or more times (which is essentially
         // the dual of the structural comparison), and likewise mark the type as deeply nested, potentially adding false positives
         // for finite but deeply expanding indexed accesses (eg, for `Q[P1][P2][P3][P4][P5]`).
+        // It also detects when a recursive type reference has expanded 5 or more times, eg, if the true branch of
+        // `type A<T> = null extends T ? [A<NonNullable<T>>] : [T]`
+        // has expanded into `[A<NonNullable<NonNullable<NonNullable<NonNullable<NonNullable<T>>>>>>]`
+        // in such cases we need to terminate the expansion, and we do so here.
         function isDeeplyNestedType(type: Type, stack: Type[], depth: number): boolean {
             // We track all object types that have an associated symbol (representing the origin of the type)
             if (depth >= 5 && type.flags & TypeFlags.Object && !isObjectOrArrayLiteralType(type)) {
@@ -17358,6 +17362,17 @@ namespace ts {
                     }
                 }
             }
+            if (depth >= 5 && getObjectFlags(type) && ObjectFlags.Reference && !!(type as TypeReference).node) {
+                const root = (type as TypeReference).target;
+                let count = 0;
+                for (let i = 0; i < depth; i++) {
+                    const t = stack[i];
+                    if (getObjectFlags(t) && ObjectFlags.Reference && !!(t as TypeReference).node && (t as TypeReference).target === root) {
+                        count++;
+                        if (count >= 5) return true;
+                    }
+                }
+            }
             return false;
         }
 
@@ -18389,6 +18404,8 @@ namespace ts {
             let propagationType: Type;
             let inferencePriority = InferencePriority.MaxValue;
             let allowComplexConstraintInference = true;
+            let objectTypeComparisonDepth = 0;
+            const targetStack: Type[] = [];
             inferFromTypes(originalSource, originalTarget);
 
             function inferFromTypes(source: Type, target: Type): void {
@@ -18822,15 +18839,27 @@ namespace ts {
                 // its symbol with the instance side which would lead to false positives.
                 const isNonConstructorObject = target.flags & TypeFlags.Object &&
                     !(getObjectFlags(target) & ObjectFlags.Anonymous && target.symbol && target.symbol.flags & SymbolFlags.Class);
-                const symbolOrType = isNonConstructorObject ? isTupleType(target) ? target.target : target.symbol : undefined;
+                const symbolOrType = getObjectFlags(target) & ObjectFlags.Reference && (target as TypeReference).node ? getNormalizedType(target, /*writing*/ false) : isNonConstructorObject ? isTupleType(target) ? target.target : target.symbol : undefined;
                 if (symbolOrType) {
                     if (contains(symbolOrTypeStack, symbolOrType)) {
+                        if (getObjectFlags(target) & ObjectFlags.Reference && (target as TypeReference).node) {
+                            // Don't set the circularity flag for re-encountered recursive type references just because we're already exploring them
+                            return;
+                        }
+                        inferencePriority = InferencePriority.Circularity;
+                        return;
+                    }
+                    targetStack[objectTypeComparisonDepth] = target;
+                    objectTypeComparisonDepth++;
+                    if (isDeeplyNestedType(target, targetStack, objectTypeComparisonDepth)) {
                         inferencePriority = InferencePriority.Circularity;
+                        objectTypeComparisonDepth--;
                         return;
                     }
                     (symbolOrTypeStack || (symbolOrTypeStack = [])).push(symbolOrType);
                     inferFromObjectTypesWorker(source, target);
                     symbolOrTypeStack.pop();
+                    objectTypeComparisonDepth--;
                 }
                 else {
                     inferFromObjectTypesWorker(source, target);
diff --git a/tests/baselines/reference/selfReferencingTypeReferenceInference.js b/tests/baselines/reference/selfReferencingTypeReferenceInference.js
new file mode 100644
index 0000000000000..cf629fb363381
--- /dev/null
+++ b/tests/baselines/reference/selfReferencingTypeReferenceInference.js
@@ -0,0 +1,23 @@
+//// [selfReferencingTypeReferenceInference.ts]
+interface Box<T> {
+    __: T
+}
+
+type Recursive<T> =
+    | T
+    | Box<Recursive<T>>
+
+type InferRecursive<T> = T extends Recursive<infer R> ? R : "never!"
+
+// the type we are testing with
+type t1 = Box<string | Box<number | boolean>>
+
+type t2 = InferRecursive<t1>
+type t3 = InferRecursive<Box<string | Box<number | boolean>>> // write t1 explicitly
+
+  // Why is t2 and t3 different??
+  // They have same input type!
+
+//// [selfReferencingTypeReferenceInference.js]
+// Why is t2 and t3 different??
+// They have same input type!
diff --git a/tests/baselines/reference/selfReferencingTypeReferenceInference.symbols b/tests/baselines/reference/selfReferencingTypeReferenceInference.symbols
new file mode 100644
index 0000000000000..4058c21cbb817
--- /dev/null
+++ b/tests/baselines/reference/selfReferencingTypeReferenceInference.symbols
@@ -0,0 +1,49 @@
+=== tests/cases/compiler/selfReferencingTypeReferenceInference.ts ===
+interface Box<T> {
+>Box : Symbol(Box, Decl(selfReferencingTypeReferenceInference.ts, 0, 0))
+>T : Symbol(T, Decl(selfReferencingTypeReferenceInference.ts, 0, 14))
+
+    __: T
+>__ : Symbol(Box.__, Decl(selfReferencingTypeReferenceInference.ts, 0, 18))
+>T : Symbol(T, Decl(selfReferencingTypeReferenceInference.ts, 0, 14))
+}
+
+type Recursive<T> =
+>Recursive : Symbol(Recursive, Decl(selfReferencingTypeReferenceInference.ts, 2, 1))
+>T : Symbol(T, Decl(selfReferencingTypeReferenceInference.ts, 4, 15))
+
+    | T
+>T : Symbol(T, Decl(selfReferencingTypeReferenceInference.ts, 4, 15))
+
+    | Box<Recursive<T>>
+>Box : Symbol(Box, Decl(selfReferencingTypeReferenceInference.ts, 0, 0))
+>Recursive : Symbol(Recursive, Decl(selfReferencingTypeReferenceInference.ts, 2, 1))
+>T : Symbol(T, Decl(selfReferencingTypeReferenceInference.ts, 4, 15))
+
+type InferRecursive<T> = T extends Recursive<infer R> ? R : "never!"
+>InferRecursive : Symbol(InferRecursive, Decl(selfReferencingTypeReferenceInference.ts, 6, 23))
+>T : Symbol(T, Decl(selfReferencingTypeReferenceInference.ts, 8, 20))
+>T : Symbol(T, Decl(selfReferencingTypeReferenceInference.ts, 8, 20))
+>Recursive : Symbol(Recursive, Decl(selfReferencingTypeReferenceInference.ts, 2, 1))
+>R : Symbol(R, Decl(selfReferencingTypeReferenceInference.ts, 8, 50))
+>R : Symbol(R, Decl(selfReferencingTypeReferenceInference.ts, 8, 50))
+
+// the type we are testing with
+type t1 = Box<string | Box<number | boolean>>
+>t1 : Symbol(t1, Decl(selfReferencingTypeReferenceInference.ts, 8, 68))
+>Box : Symbol(Box, Decl(selfReferencingTypeReferenceInference.ts, 0, 0))
+>Box : Symbol(Box, Decl(selfReferencingTypeReferenceInference.ts, 0, 0))
+
+type t2 = InferRecursive<t1>
+>t2 : Symbol(t2, Decl(selfReferencingTypeReferenceInference.ts, 11, 45))
+>InferRecursive : Symbol(InferRecursive, Decl(selfReferencingTypeReferenceInference.ts, 6, 23))
+>t1 : Symbol(t1, Decl(selfReferencingTypeReferenceInference.ts, 8, 68))
+
+type t3 = InferRecursive<Box<string | Box<number | boolean>>> // write t1 explicitly
+>t3 : Symbol(t3, Decl(selfReferencingTypeReferenceInference.ts, 13, 28))
+>InferRecursive : Symbol(InferRecursive, Decl(selfReferencingTypeReferenceInference.ts, 6, 23))
+>Box : Symbol(Box, Decl(selfReferencingTypeReferenceInference.ts, 0, 0))
+>Box : Symbol(Box, Decl(selfReferencingTypeReferenceInference.ts, 0, 0))
+
+  // Why is t2 and t3 different??
+  // They have same input type!
diff --git a/tests/baselines/reference/selfReferencingTypeReferenceInference.types b/tests/baselines/reference/selfReferencingTypeReferenceInference.types
new file mode 100644
index 0000000000000..a33b3d9ceb75f
--- /dev/null
+++ b/tests/baselines/reference/selfReferencingTypeReferenceInference.types
@@ -0,0 +1,27 @@
+=== tests/cases/compiler/selfReferencingTypeReferenceInference.ts ===
+interface Box<T> {
+    __: T
+>__ : T
+}
+
+type Recursive<T> =
+>Recursive : Recursive<T>
+
+    | T
+    | Box<Recursive<T>>
+
+type InferRecursive<T> = T extends Recursive<infer R> ? R : "never!"
+>InferRecursive : InferRecursive<T>
+
+// the type we are testing with
+type t1 = Box<string | Box<number | boolean>>
+>t1 : t1
+
+type t2 = InferRecursive<t1>
+>t2 : string | number | boolean
+
+type t3 = InferRecursive<Box<string | Box<number | boolean>>> // write t1 explicitly
+>t3 : string | number | boolean
+
+  // Why is t2 and t3 different??
+  // They have same input type!
diff --git a/tests/cases/compiler/selfReferencingTypeReferenceInference.ts b/tests/cases/compiler/selfReferencingTypeReferenceInference.ts
new file mode 100644
index 0000000000000..80aa7805f08a9
--- /dev/null
+++ b/tests/cases/compiler/selfReferencingTypeReferenceInference.ts
@@ -0,0 +1,18 @@
+interface Box<T> {
+    __: T
+}
+
+type Recursive<T> =
+    | T
+    | Box<Recursive<T>>
+
+type InferRecursive<T> = T extends Recursive<infer R> ? R : "never!"
+
+// the type we are testing with
+type t1 = Box<string | Box<number | boolean>>
+
+type t2 = InferRecursive<t1>
+type t3 = InferRecursive<Box<string | Box<number | boolean>>> // write t1 explicitly
+
+  // Why is t2 and t3 different??
+  // They have same input type!
\ No newline at end of file

From 0917694098a4821401ba30d228ef93fb1fc8770d Mon Sep 17 00:00:00 2001
From: Wesley Wigham <wwigham@gmail.com>
Date: Wed, 24 Jun 2020 13:55:40 -0700
Subject: [PATCH 2/2] Move object cases into the same conditional branch

---
 src/compiler/checker.ts | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/src/compiler/checker.ts b/src/compiler/checker.ts
index 5593bbece961e..a4253006a925f 100644
--- a/src/compiler/checker.ts
+++ b/src/compiler/checker.ts
@@ -18096,13 +18096,26 @@ namespace ts {
         // in such cases we need to terminate the expansion, and we do so here.
         function isDeeplyNestedType(type: Type, stack: Type[], depth: number): boolean {
             // We track all object types that have an associated symbol (representing the origin of the type)
-            if (depth >= 5 && type.flags & TypeFlags.Object && !isObjectOrArrayLiteralType(type)) {
-                const symbol = type.symbol;
-                if (symbol) {
+            if (depth >= 5 && type.flags & TypeFlags.Object) {
+                if (!isObjectOrArrayLiteralType(type)) {
+                    const symbol = type.symbol;
+                    if (symbol) {
+                        let count = 0;
+                        for (let i = 0; i < depth; i++) {
+                            const t = stack[i];
+                            if (t.flags & TypeFlags.Object && t.symbol === symbol) {
+                                count++;
+                                if (count >= 5) return true;
+                            }
+                        }
+                    }
+                }
+                if (getObjectFlags(type) && ObjectFlags.Reference && !!(type as TypeReference).node) {
+                    const root = (type as TypeReference).target;
                     let count = 0;
                     for (let i = 0; i < depth; i++) {
                         const t = stack[i];
-                        if (t.flags & TypeFlags.Object && t.symbol === symbol) {
+                        if (getObjectFlags(t) && ObjectFlags.Reference && !!(t as TypeReference).node && (t as TypeReference).target === root) {
                             count++;
                             if (count >= 5) return true;
                         }
@@ -18120,17 +18133,6 @@ namespace ts {
                     }
                 }
             }
-            if (depth >= 5 && getObjectFlags(type) && ObjectFlags.Reference && !!(type as TypeReference).node) {
-                const root = (type as TypeReference).target;
-                let count = 0;
-                for (let i = 0; i < depth; i++) {
-                    const t = stack[i];
-                    if (getObjectFlags(t) && ObjectFlags.Reference && !!(t as TypeReference).node && (t as TypeReference).target === root) {
-                        count++;
-                        if (count >= 5) return true;
-                    }
-                }
-            }
             return false;
         }