Try alternative to track hard typevars in constraint

Try alternative to track hard typevars in constraint instead of typer state. This avoids the problem
that a failing subtype comparison can also mark type variables as hard.

Author: odersky

compiler/src/dotty/tools/dotc/core/Constraint.scala

@@ -126,6 +126,12 @@ abstract class Constraint extends Showable {
    */
   def subst(from: TypeLambda, to: TypeLambda)(using Context): This
 
+  /** Is `tv` marked as hard in the constraint? */
+  def isHard(tv: TypeVar): Boolean
+  
+  /** The same as this constraint, but with `tv` marked as hard. */
+  def withHard(tv: TypeVar)(using Context): This
+
   /** Gives for each instantiated type var that does not yet have its `inst` field
    *  set, the instance value stored in the constraint. Storing instances in constraints
    *  is done only in a temporary way for contexts that may be retracted

compiler/src/dotty/tools/dotc/core/GadtConstraint.scala

@@ -6,7 +6,7 @@ import Decorators._
 import Contexts._
 import Types._
 import Symbols._
-import util.SimpleIdentityMap
+import util.{SimpleIdentitySet, SimpleIdentityMap}
 import collection.mutable
 import printing._
 
@@ -68,7 +68,7 @@ final class ProperGadtConstraint private(
   import dotty.tools.dotc.config.Printers.{gadts, gadtsConstr}
 
   def this() = this(
-    myConstraint = new OrderingConstraint(SimpleIdentityMap.empty, SimpleIdentityMap.empty, SimpleIdentityMap.empty),
+    myConstraint = new OrderingConstraint(SimpleIdentityMap.empty, SimpleIdentityMap.empty, SimpleIdentityMap.empty, SimpleIdentitySet.empty),
     mapping = SimpleIdentityMap.empty,
     reverseMapping = SimpleIdentityMap.empty,
     wasConstrained = false

compiler/src/dotty/tools/dotc/core/OrderingConstraint.scala

@@ -3,7 +3,7 @@ package dotc
 package core
 
 import Types._, Contexts._, Symbols._, Decorators._, TypeApplications._
-import util.SimpleIdentityMap
+import util.{SimpleIdentitySet, SimpleIdentityMap}
 import collection.mutable
 import printing.Printer
 import printing.Texts._
@@ -24,12 +24,14 @@ object OrderingConstraint {
   /** The type of `OrderingConstraint#lowerMap`, `OrderingConstraint#upperMap` */
   type ParamOrdering = ArrayValuedMap[List[TypeParamRef]]
 
-  /** A new constraint with given maps */
-  private def newConstraint(boundsMap: ParamBounds, lowerMap: ParamOrdering, upperMap: ParamOrdering)(using Context) : OrderingConstraint =
+  /** A new constraint with given maps and given set of hard typevars */
+  private def newConstraint(
+      boundsMap: ParamBounds, lowerMap: ParamOrdering, upperMap: ParamOrdering,
+      hardVars: TypeVars)(using Context) : OrderingConstraint =
     if boundsMap.isEmpty && lowerMap.isEmpty && upperMap.isEmpty then
       empty
     else
-      val result = new OrderingConstraint(boundsMap, lowerMap, upperMap)
+      val result = new OrderingConstraint(boundsMap, lowerMap, upperMap, hardVars)
       if ctx.run != null then ctx.run.nn.recordConstraintSize(result, result.boundsMap.size)
       result
 
@@ -91,28 +93,28 @@ object OrderingConstraint {
     def entries(c: OrderingConstraint, poly: TypeLambda): Array[Type] | Null =
       c.boundsMap(poly)
     def updateEntries(c: OrderingConstraint, poly: TypeLambda, entries: Array[Type])(using Context): OrderingConstraint =
-      newConstraint(c.boundsMap.updated(poly, entries), c.lowerMap, c.upperMap)
+      newConstraint(c.boundsMap.updated(poly, entries), c.lowerMap, c.upperMap, c.hardVars)
     def initial = NoType
   }
 
   val lowerLens: ConstraintLens[List[TypeParamRef]] = new ConstraintLens[List[TypeParamRef]] {
     def entries(c: OrderingConstraint, poly: TypeLambda): Array[List[TypeParamRef]] | Null =
       c.lowerMap(poly)
     def updateEntries(c: OrderingConstraint, poly: TypeLambda, entries: Array[List[TypeParamRef]])(using Context): OrderingConstraint =
-      newConstraint(c.boundsMap, c.lowerMap.updated(poly, entries), c.upperMap)
+      newConstraint(c.boundsMap, c.lowerMap.updated(poly, entries), c.upperMap, c.hardVars)
     def initial = Nil
   }
 
   val upperLens: ConstraintLens[List[TypeParamRef]] = new ConstraintLens[List[TypeParamRef]] {
     def entries(c: OrderingConstraint, poly: TypeLambda): Array[List[TypeParamRef]] | Null =
       c.upperMap(poly)
     def updateEntries(c: OrderingConstraint, poly: TypeLambda, entries: Array[List[TypeParamRef]])(using Context): OrderingConstraint =
-      newConstraint(c.boundsMap, c.lowerMap, c.upperMap.updated(poly, entries))
+      newConstraint(c.boundsMap, c.lowerMap, c.upperMap.updated(poly, entries), c.hardVars)
     def initial = Nil
   }
 
   @sharable
-  val empty = new OrderingConstraint(SimpleIdentityMap.empty, SimpleIdentityMap.empty, SimpleIdentityMap.empty)
+  val empty = new OrderingConstraint(SimpleIdentityMap.empty, SimpleIdentityMap.empty, SimpleIdentityMap.empty, SimpleIdentitySet.empty)
 }
 
 import OrderingConstraint._
@@ -134,10 +136,13 @@ import OrderingConstraint._
  *  @param upperMap a map from TypeLambdas to arrays. Each array entry corresponds
  *               to a parameter P of the type lambda; it contains all constrained parameters
  *               Q that are known to be greater than P, i.e. P <: Q.
+ *  @param hardVars a set of type variables that are marked as hard and therefore will not
+ *               undergo a `widenUnion` when instantiated to their lower bound.
  */
 class OrderingConstraint(private val boundsMap: ParamBounds,
                          private val lowerMap : ParamOrdering,
-                         private val upperMap : ParamOrdering) extends Constraint {
+                         private val upperMap : ParamOrdering,
+                         private val hardVars : TypeVars) extends Constraint {
 
   import UnificationDirection.*
 
@@ -277,7 +282,7 @@ class OrderingConstraint(private val boundsMap: ParamBounds,
     val entries1 = new Array[Type](nparams * 2)
     poly.paramInfos.copyToArray(entries1, 0)
     tvars.copyToArray(entries1, nparams)
-    newConstraint(boundsMap.updated(poly, entries1), lowerMap, upperMap).init(poly)
+    newConstraint(boundsMap.updated(poly, entries1), lowerMap, upperMap, hardVars).init(poly)
   }
 
   /** Split dependent parameters off the bounds for parameters in `poly`.
@@ -478,7 +483,8 @@ class OrderingConstraint(private val boundsMap: ParamBounds,
       }
       po.remove(pt).mapValuesNow(removeFromBoundss)
     }
-    newConstraint(boundsMap.remove(pt), removeFromOrdering(lowerMap), removeFromOrdering(upperMap))
+    val hardVars1 = pt.paramRefs.foldLeft(hardVars)((hvs, param) => hvs - typeVarOfParam(param))
+    newConstraint(boundsMap.remove(pt), removeFromOrdering(lowerMap), removeFromOrdering(upperMap), hardVars1)
       .checkNonCyclic()
   }
 
@@ -505,7 +511,7 @@ class OrderingConstraint(private val boundsMap: ParamBounds,
     def swapKey[T](m: ArrayValuedMap[T]) =
       val info = m(from)
       if info == null then m else m.remove(from).updated(to, info)
-    var current = newConstraint(swapKey(boundsMap), swapKey(lowerMap), swapKey(upperMap))
+    var current = newConstraint(swapKey(boundsMap), swapKey(lowerMap), swapKey(upperMap), hardVars)
     def subst[T <: Type](x: T): T = x.subst(from, to).asInstanceOf[T]
     current.foreachParam {(p, i) =>
       current = boundsLens.map(this, current, p, i, subst)
@@ -515,6 +521,11 @@ class OrderingConstraint(private val boundsMap: ParamBounds,
     constr.println(i"renamed $this to $current")
     current.checkNonCyclic()
 
+  def isHard(tv: TypeVar) = hardVars.contains(tv)
+
+  def withHard(tv: TypeVar)(using Context) =
+    newConstraint(boundsMap, lowerMap, upperMap, hardVars + tv)
+
   def instType(tvar: TypeVar): Type = entry(tvar.origin) match
     case _: TypeBounds => NoType
     case tp: TypeParamRef => typeVarOfParam(tp).orElse(tp)

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -495,10 +495,11 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
           case AndType(tp21, tp22) => constrainRHSVars(tp21) && constrainRHSVars(tp22)
           case _ => true
 
-        /** Mark toplevel type vars in `tp2` as hard in the current typerState */
+        /** Mark toplevel type vars in `tp2` as hard in the current constraint */
         def hardenTypeVars(tp2: Type): Unit = tp2.dealiasKeepRefiningAnnots match
           case tvar: TypeVar if constraint.contains(tvar.origin) =>
             state.hardenTypeVar(tvar)
+            constraint = constraint.withHard(tvar)
           case tp2: TypeParamRef if constraint.contains(tp2) =>
             hardenTypeVars(constraint.typeVarOfParam(tp2))
           case tp2: AndOrType =>
@@ -524,14 +525,11 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
               // recursively, so we do it only once. See i14870.scala as a test case, which would
               // loop for a very long time without the recursion brake.
 
-        if res && !tp1.isSoft then
+        if res && !tp1.isSoft  && state.isCommittable then
           // We use a heuristic here where every toplevel type variable on the right hand side
           // is marked so that it converts all soft unions in its lower bound to hard unions
-          // before it is instantiated. The reason is that the union might have come from
-          // (decomposed and reconstituted) `tp1`. But of course there might be false positives
-          // where we also treat unions that come from elsewhere as hard unions. Or the constraint
-          // that created the union is ultimately thrown away, but the type variable will
-          // stay marked. So it is a coarse measure to take. But it works in the obvious cases.
+          // before it is instantiated. The reason is that the variable's instance type will
+          // be a supertype of (decomposed and reconstituted) `tp1`.
           hardenTypeVars(tp2)
 
         res

compiler/src/dotty/tools/dotc/core/TyperState.scala

@@ -246,7 +246,8 @@ class TyperState() {
         constraint.contains(tl) || other.isRemovable(tl) || {
           val tvars = tl.paramRefs.map(other.typeVarOfParam(_)).collect { case tv: TypeVar => tv }
           if this.isCommittable then
-            tvars.foreach(tvar => if !tvar.inst.exists && !isOwnedAnywhere(this, tvar) then includeVar(tvar))
+            tvars.foreach(tvar =>
+              if !tvar.inst.exists && !isOwnedAnywhere(this, tvar) then includeVar(tvar))
           typeComparer.addToConstraint(tl, tvars)
         }) &&
         // Integrate the additional constraints on type variables from `other`

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -4721,7 +4721,9 @@ object Types {
         instantiateWith(tp)
 
     /** Widen unions when instantiating this variable in the current context? */
-    def widenUnions(using Context): Boolean = !ctx.typerState.isHard(this)
+    def widenUnions(using Context): Boolean =
+      if true then !ctx.typerState.constraint.isHard(this)
+      else !ctx.typerState.isHard(this)
 
     /** For uninstantiated type variables: the entry in the constraint (either bounds or
      *  provisional instance value)