Skip to content

Commit 26a7dbb

Browse files
authored
intersect: fix a minor soundness issue with supertypes (#47813)
When doing intersection, we might end up with a value in `env` (as the only possible *value* for that parameter) without properly considering that the parameter might be a TypeVar.
1 parent 847cdde commit 26a7dbb

File tree

3 files changed

+17
-41
lines changed

3 files changed

+17
-41
lines changed

src/subtype.c

Lines changed: 6 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -2882,48 +2882,14 @@ static void flip_vars(jl_stenv_t *e)
28822882
// intersection where xd nominally inherits from yd
28832883
static jl_value_t *intersect_sub_datatype(jl_datatype_t *xd, jl_datatype_t *yd, jl_stenv_t *e, int R, int param)
28842884
{
2885+
// attempt to populate additional constraints into `e`
2886+
// if that attempt fails, then return bottom
2887+
// otherwise return xd (finish_unionall will later handle propagating those constraints)
28852888
jl_value_t *isuper = R ? intersect((jl_value_t*)yd, (jl_value_t*)xd->super, e, param) :
28862889
intersect((jl_value_t*)xd->super, (jl_value_t*)yd, e, param);
2887-
if (isuper == jl_bottom_type) return jl_bottom_type;
2888-
if (jl_nparams(xd) == 0 || jl_nparams(xd->super) == 0 || !jl_has_free_typevars((jl_value_t*)xd))
2889-
return (jl_value_t*)xd;
2890-
jl_value_t *super_pattern=NULL;
2891-
JL_GC_PUSH2(&isuper, &super_pattern);
2892-
jl_value_t *wrapper = xd->name->wrapper;
2893-
super_pattern = jl_rewrap_unionall_((jl_value_t*)((jl_datatype_t*)jl_unwrap_unionall(wrapper))->super,
2894-
wrapper);
2895-
int envsz = jl_subtype_env_size(super_pattern);
2896-
jl_value_t *ii = jl_bottom_type;
2897-
{
2898-
jl_value_t **env;
2899-
JL_GC_PUSHARGS(env, envsz);
2900-
jl_stenv_t tempe;
2901-
init_stenv(&tempe, env, envsz);
2902-
tempe.intersection = tempe.ignore_free = 1;
2903-
if (subtype_in_env(isuper, super_pattern, &tempe)) {
2904-
jl_value_t *wr = wrapper;
2905-
int i;
2906-
for(i=0; i<envsz; i++) {
2907-
// if a parameter is not constrained by the supertype, use the original
2908-
// parameter value from `x`. this is detected by the value in `env` being
2909-
// the exact typevar from the type's `wrapper`, or a free typevar.
2910-
jl_value_t *ei = env[i];
2911-
if (ei == (jl_value_t*)((jl_unionall_t*)wr)->var ||
2912-
(jl_is_typevar(ei) && lookup(e, (jl_tvar_t*)ei) == NULL))
2913-
env[i] = jl_tparam(xd,i);
2914-
wr = ((jl_unionall_t*)wr)->body;
2915-
}
2916-
JL_TRY {
2917-
ii = jl_apply_type(wrapper, env, envsz);
2918-
}
2919-
JL_CATCH {
2920-
ii = jl_bottom_type;
2921-
}
2922-
}
2923-
JL_GC_POP();
2924-
}
2925-
JL_GC_POP();
2926-
return ii;
2890+
if (isuper == jl_bottom_type)
2891+
return jl_bottom_type;
2892+
return (jl_value_t*)xd;
29272893
}
29282894

29292895
static jl_value_t *intersect_invariant(jl_value_t *x, jl_value_t *y, jl_stenv_t *e)

test/docs.jl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -970,6 +970,7 @@ abstract type $(curmod_prefix)Undocumented.at1{T>:Integer, N}
970970
971971
```
972972
$(curmod_prefix)Undocumented.mt6{Integer, N}
973+
$(curmod_prefix)Undocumented.st5{T>:Integer, N}
973974
```
974975
975976
# Supertype Hierarchy

test/subtype.jl

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2188,7 +2188,16 @@ for T in (B46871{Int, N} where {N}, B46871{Int}) # intentional duplication
21882188
end
21892189
abstract type C38497{e,g<:Tuple,i} end
21902190
struct Q38497{o,e<:NTuple{o},g} <: C38497{e,g,Array{o}} end
2191-
@testintersect(Q38497{<:Any, Tuple{Int}}, C38497, Q38497{1, Tuple{Int}, <:Tuple})
2191+
@testintersect(Q38497{<:Any, Tuple{Int}}, C38497, Q38497{<:Any, Tuple{Int}, <:Tuple})
2192+
# n.b. the only concrete instance of this type is Q38497{1, Tuple{Int}, <:Tuple} (since NTuple{o} also adds an ::Int constraint)
2193+
# but this abstract type is also part of the intersection abstractly
2194+
2195+
abstract type X38497{T<:Number} end
2196+
abstract type Y38497{T>:Integer} <: X38497{T} end
2197+
struct Z38497{T>:Int} <: Y38497{T} end
2198+
@testintersect(Z38497, X38497, Z38497{T} where Int<:T<:Number)
2199+
@testintersect(Z38497, Y38497, Z38497{T} where T>:Integer)
2200+
@testintersect(X38497, Y38497, Y38497{T} where Integer<:T<:Number)
21922201

21932202
#issue #33138
21942203
@test Vector{Vector{Tuple{T,T}} where Int<:T<:Int} <: Vector{Vector{Tuple{S1,S1} where S<:S1<:S}} where S

0 commit comments

Comments
 (0)