Use Ternary.Unknown to signal variance recursion

Author: ahejlsberg

src/compiler/checker.ts

@@ -16407,7 +16407,6 @@ namespace ts {
             let lastSkippedInfo: [Type, Type] | undefined;
             let incompatibleStack: [DiagnosticMessage, (string | number)?, (string | number)?, (string | number)?, (string | number)?][] = [];
             let inPropertyCheck = false;
-            let recursiveVariances = false;
 
             Debug.assert(relation !== identityRelation || !errorNode, "no error reporting in identity checking");
 
@@ -17256,7 +17255,7 @@ namespace ts {
                 depth--;
                 if (result) {
                     if (result === Ternary.True || depth === 0) {
-                        if (result === Ternary.True || !recursiveVariances) {
+                        if (result === Ternary.True || result === Ternary.Maybe) {
                             // If result is definitely true, record all maybe keys as having succeeded
                             for (let i = maybeStart; i < maybeCount; i++) {
                                 relation.set(maybeKeys[i], RelationComparisonResult.Succeeded | propagatingVarianceFlags);
@@ -17330,8 +17329,7 @@ namespace ts {
                     !(source.aliasTypeArgumentsContainsMarker || target.aliasTypeArgumentsContainsMarker)) {
                     const variances = getAliasVariances(source.aliasSymbol);
                     if (variances === emptyArray) {
-                        recursiveVariances = true;
-                        return Ternary.Maybe;
+                        return Ternary.Unknown;
                     }
                     const varianceResult = relateVariances(source.aliasTypeArguments, target.aliasTypeArguments, variances, intersectionState);
                     if (varianceResult !== undefined) {
@@ -17604,8 +17602,7 @@ namespace ts {
                         // effectively means we measure variance only from type parameter occurrences that aren't nested in
                         // recursive instantiations of the generic type.
                         if (variances === emptyArray) {
-                            recursiveVariances = true;
-                            return Ternary.Maybe;
+                            return Ternary.Unknown;
                         }
                         const varianceResult = relateVariances(getTypeArguments(<TypeReference>source), getTypeArguments(<TypeReference>target), variances, intersectionState);
                         if (varianceResult !== undefined) {

src/compiler/types.ts

@@ -5541,7 +5541,8 @@ namespace ts {
     /* @internal */
     export const enum Ternary {
         False = 0,
-        Maybe = 1,
+        Unknown = 1,
+        Maybe = 3,
         True = -1
     }