Fix #8905: break cycles in instantiateToSubType

compiler/src/dotty/tools/dotc/core/TypeOps.scala

@@ -641,6 +641,9 @@ object TypeOps:
           if (maximize) lo else hi
 
       def apply(tp: Type): Type = tp match {
+        case _: MatchType =>
+          tp // break cycles
+
         case tp: TypeRef if isBounds(tp.underlying) =>
           val lo = this(tp.info.loBound)
           val hi = this(tp.info.hiBound)

compiler/src/dotty/tools/dotc/transform/TypeTestsCasts.scala

@@ -81,6 +81,8 @@ object TypeTestsCasts {
     /** Approximate type parameters depending on variance */
     def stripTypeParam(tp: Type)(implicit ctx: Context) = new ApproximatingTypeMap {
       def apply(tp: Type): Type = tp match {
+        case _: MatchType =>
+          tp // break cycles
         case tp: TypeRef if isBounds(tp.underlying) =>
           val lo = apply(tp.info.loBound)
           val hi = apply(tp.info.hiBound)

tests/pos/8905-2.scala

@@ -0,0 +1,3 @@
+class Test[T1 <: Tuple, T2 <: Tuple] {
+  def test6[Y <: Int, X <: Function1[Tuple.Concat[T1, T2], Unit]](x: X) = x.isInstanceOf[Function1[Int, Any]]
+}

tests/pos/8905.scala

@@ -0,0 +1,4 @@
+object Test {
+  def f[T1 <: Tuple](o: Option[Tuple.Concat[T1, T1]]): Unit =
+    o match { case Some(x) => }
+}