when widening tuple types in tmerge, only widen the complex parts (#50929)

This is the part of #50927
required to fix #49249.
Specifically, before this change `tmerge(Tuple{Any, Int}, Nothing)`
would be `Union{Nothing, Tuple{Any, Int}}` but `tmerge(Tuple{BIG_UNION,
Int}, Nothing)` would be `Union{Nothing, Tuple{Any, Any}}`. This feels
bad intuitively because giving the compiler more type information led it
to forget type information that it already knew about, and is especially
damaging because it led to unnecessary type instability when iterating
tuples with complex element types (because the iterator state should be
inferrable as an `Int` even if you have no idea what the tuple type is).

This is tagged for backport to 1.10 since it is a relatively unobtrusive
change and it fixes the string regression in a more proper way.

Author: oscardssmith

base/compiler/typelimits.jl

@@ -764,23 +764,24 @@ end
         return u
     end
     # don't let the slow widening of Tuple cause the whole type to grow too fast
+    # Specifically widen Tuple{..., Union{lots of stuff}...} to Tuple{..., Any, ...}
     for i in 1:length(types)
         if typenames[i] === Tuple.name
-            widen = unwrap_unionall(types[i])
-            if isa(widen, DataType) && !isvatuple(widen)
-                widen = NTuple{length(widen.parameters), Any}
-            else
-                widen = Tuple
-            end
-            types[i] = widen
-            u = Union{types...}
-            if issimpleenoughtype(u)
-                return u
+            ti = types[i]
+            tip = (unwrap_unionall(types[i])::DataType).parameters
+            lt = length(tip)
+            p = Vector{Any}(undef, lt)
+            for j = 1:lt
+                ui = tip[j]
+                p[j] = (unioncomplexity(ui)==0) ? ui : isvarargtype(ui) ? Vararg : Any
             end
-            break
+            types[i] = rewrap_unionall(Tuple{p...}, ti)
         end
     end
-    # finally, just return the widest possible type
+    u = Union{types...}
+    if issimpleenoughtype(u)
+        return u
+    end
     return Any
 end
 

test/compiler/inference.jl

@@ -213,6 +213,21 @@ tmerge_test(Tuple{}, Tuple{Complex, Vararg{Union{ComplexF32, ComplexF64}}},
 @test Core.Compiler.tmerge(Base.BitIntegerType, Union{}) === Base.BitIntegerType
 @test Core.Compiler.tmerge(Union{}, Base.BitIntegerType) === Base.BitIntegerType
 @test Core.Compiler.tmerge(Core.Compiler.fallback_ipo_lattice, Core.Compiler.InterConditional(1, Int, Union{}), Core.Compiler.InterConditional(2, String, Union{})) === Core.Compiler.Const(true)
+# test issue behind https://github.com/JuliaLang/julia/issues/50458
+@test Core.Compiler.tmerge(Nothing, Tuple{Base.BitInteger, Int}) == Union{Nothing, Tuple{Any, Int}}
+@test Core.Compiler.tmerge(Nothing, Tuple{Union{Char, String, SubString{String}, Symbol}, Int}) == Union{Nothing, Tuple{Any, Int}}
+@test Core.Compiler.tmerge(Nothing, Tuple{Integer, Int}) == Union{Nothing, Tuple{Integer, Int}}
+
+# test that recursively more complicated types don't widen all the way to Any when there is a useful valid type upper bound
+# Specificially test with base types of a trivial type, a simple union, a complicated union, and a tuple.
+for T in (Nothing, Base.BitInteger, Union{Int, Int32, Int16, Int8}, Tuple{Int, Int})
+    Ta, Tb = T, T
+    for i in 1:10
+        Ta = Union{Tuple{Ta}, Nothing}
+        Tb = Core.Compiler.tmerge(Tuple{Tb}, Nothing)
+        @test Ta <: Tb <: Union{Nothing, Tuple}
+    end
+end
 
 struct SomethingBits
     x::Base.BitIntegerType