@@ -322,8 +322,7 @@ inline_node_is_duplicate(topline::LineInfoNode, line::LineInfoNode) =
322
322
topline. line === line. line
323
323
324
324
function ir_inline_linetable! (linetable:: Vector{LineInfoNode} , inlinee_ir:: IRCode ,
325
- inlinee:: MethodInstance ,
326
- inlined_at:: Int32 )
325
+ inlinee:: MethodInstance , inlined_at:: Int32 )
327
326
inlinee_def = inlinee. def:: Method
328
327
coverage = coverage_enabled (inlinee_def. module)
329
328
linetable_offset:: Int32 = length (linetable)
@@ -358,18 +357,18 @@ function ir_inline_linetable!(linetable::Vector{LineInfoNode}, inlinee_ir::IRCod
358
357
end
359
358
360
359
function ir_prepare_inlining! (insert_node!:: Inserter , inline_target:: Union{IRCode, IncrementalCompact} ,
361
- linetable:: Vector{LineInfoNode} , ir′:: IRCode , sparam_vals:: SimpleVector ,
362
- mi:: MethodInstance , inlined_at:: Int32 , argexprs:: Vector{Any} )
360
+ ir:: IRCode , mi:: MethodInstance , inlined_at:: Int32 , argexprs:: Vector{Any} )
363
361
def = mi. def:: Method
362
+ linetable = inline_target isa IRCode ? inline_target. linetable : inline_target. ir. linetable
364
363
topline:: Int32 = length (linetable) + Int32 (1 )
365
- linetable_offset, extra_coverage_line = ir_inline_linetable! (linetable, ir′ , mi, inlined_at)
364
+ linetable_offset, extra_coverage_line = ir_inline_linetable! (linetable, ir, mi, inlined_at)
366
365
if extra_coverage_line != 0
367
366
insert_node! (NewInstruction (Expr (:code_coverage_effect ), Nothing, extra_coverage_line))
368
367
end
369
- sp_ssa = nothing
370
- if ! validate_sparams (sparam_vals)
368
+ spvals_ssa = nothing
369
+ if ! validate_sparams (mi . sparam_vals)
371
370
# N.B. This works on the caller-side argexprs, (i.e. before the va fixup below)
372
- sp_ssa = insert_node! (
371
+ spvals_ssa = insert_node! (
373
372
effect_free_and_nothrow (NewInstruction (Expr (:call , Core. _compute_sparams, def, argexprs... ), SimpleVector, topline)))
374
373
end
375
374
if def. isva
@@ -382,20 +381,17 @@ function ir_prepare_inlining!(insert_node!::Inserter, inline_target::Union{IRCod
382
381
# Replace the first argument by a load of the capture environment
383
382
argexprs[1 ] = insert_node! (
384
383
NewInstruction (Expr (:call , GlobalRef (Core, :getfield ), argexprs[1 ], QuoteNode (:captures )),
385
- ir′ . argtypes[1 ], topline))
384
+ ir. argtypes[1 ], topline))
386
385
end
387
- return ( Pair {Union{Nothing, SSAValue}, Vector{Any}} (sp_ssa, argexprs) , linetable_offset)
386
+ return SSASubstitute (mi, argexprs, spvals_ssa , linetable_offset)
388
387
end
389
388
390
389
function ir_inline_item! (compact:: IncrementalCompact , idx:: Int , argexprs:: Vector{Any} ,
391
- linetable:: Vector{LineInfoNode} , item:: InliningTodo ,
392
- boundscheck:: Symbol , todo_bbs:: Vector{Tuple{Int, Int}} )
390
+ item:: InliningTodo , boundscheck:: Symbol , todo_bbs:: Vector{Tuple{Int, Int}} )
393
391
# Ok, do the inlining here
394
- sparam_vals = item. mi. sparam_vals
395
392
inlined_at = compact. result[idx][:line ]
396
393
397
- ((sp_ssa, argexprs), linetable_offset) = ir_prepare_inlining! (InsertHere (compact),
398
- compact, linetable, item. ir, sparam_vals, item. mi, inlined_at, argexprs)
394
+ ssa_substitute = ir_prepare_inlining! (InsertHere (compact), compact, item. ir, item. mi, inlined_at, argexprs)
399
395
400
396
if boundscheck === :default || boundscheck === :propagate
401
397
if (compact. result[idx][:flag ] & IR_FLAG_INBOUNDS) != 0
@@ -405,8 +401,6 @@ function ir_inline_item!(compact::IncrementalCompact, idx::Int, argexprs::Vector
405
401
# If the iterator already moved on to the next basic block,
406
402
# temporarily re-open in again.
407
403
local return_value
408
- def = item. mi. def:: Method
409
- sig = def. sig
410
404
# Special case inlining that maintains the current basic block if there's only one BB in the target
411
405
new_new_offset = length (compact. new_new_nodes)
412
406
late_fixup_offset = length (compact. late_fixup)
@@ -418,7 +412,9 @@ function ir_inline_item!(compact::IncrementalCompact, idx::Int, argexprs::Vector
418
412
# face of rename_arguments! mutating in place - should figure out
419
413
# something better eventually.
420
414
inline_compact[idx′] = nothing
421
- stmt′ = ssa_substitute! (InsertBefore (inline_compact, SSAValue (idx′)), inline_compact[SSAValue (idx′)], stmt′, argexprs, sig, sparam_vals, sp_ssa, linetable_offset, boundscheck)
415
+ insert_node! = InsertBefore (inline_compact, SSAValue (idx′))
416
+ stmt′ = ssa_substitute! (insert_node!, inline_compact[SSAValue (idx′)], stmt′,
417
+ ssa_substitute, boundscheck)
422
418
if isa (stmt′, ReturnNode)
423
419
val = stmt′. val
424
420
return_value = SSAValue (idx′)
@@ -445,7 +441,9 @@ function ir_inline_item!(compact::IncrementalCompact, idx::Int, argexprs::Vector
445
441
inline_compact = IncrementalCompact (compact, item. ir, compact. result_idx)
446
442
for ((_, idx′), stmt′) in inline_compact
447
443
inline_compact[idx′] = nothing
448
- stmt′ = ssa_substitute! (InsertBefore (inline_compact, SSAValue (idx′)), inline_compact[SSAValue (idx′)], stmt′, argexprs, sig, sparam_vals, sp_ssa, linetable_offset, boundscheck)
444
+ insert_node! = InsertBefore (inline_compact, SSAValue (idx′))
445
+ stmt′ = ssa_substitute! (insert_node!, inline_compact[SSAValue (idx′)], stmt′,
446
+ ssa_substitute, boundscheck)
449
447
if isa (stmt′, ReturnNode)
450
448
if isdefined (stmt′, :val )
451
449
val = stmt′. val
@@ -554,11 +552,10 @@ excluding cases where `case.sig::UnionAll`.
554
552
In short, here we can process the dispatch candidates in order, assuming we haven't changed
555
553
their order somehow somewhere up to this point.
556
554
"""
557
- function ir_inline_unionsplit! (compact:: IncrementalCompact , idx:: Int ,
558
- argexprs:: Vector{Any} , linetable:: Vector{LineInfoNode} ,
559
- (; fully_covered, atype, cases, bbs):: UnionSplit ,
560
- boundscheck:: Symbol , todo_bbs:: Vector{Tuple{Int, Int}} ,
561
- params:: OptimizationParams )
555
+ function ir_inline_unionsplit! (compact:: IncrementalCompact , idx:: Int , argexprs:: Vector{Any} ,
556
+ union_split:: UnionSplit , boundscheck:: Symbol ,
557
+ todo_bbs:: Vector{Tuple{Int,Int}} , params:: OptimizationParams )
558
+ (; fully_covered, atype, cases, bbs) = union_split
562
559
stmt, typ, line = compact. result[idx][:inst ], compact. result[idx][:type ], compact. result[idx][:line ]
563
560
join_bb = bbs[end ]
564
561
pn = PhiNode ()
@@ -606,7 +603,7 @@ function ir_inline_unionsplit!(compact::IncrementalCompact, idx::Int,
606
603
end
607
604
end
608
605
if isa (case, InliningTodo)
609
- val = ir_inline_item! (compact, idx, argexprs′, linetable, case, boundscheck, todo_bbs)
606
+ val = ir_inline_item! (compact, idx, argexprs′, case, boundscheck, todo_bbs)
610
607
elseif isa (case, InvokeCase)
611
608
inst = Expr (:invoke , case. invoke, argexprs′... )
612
609
flag = flags_for_effects (case. effects)
@@ -698,9 +695,9 @@ function batch_inline!(ir::IRCode, todo::Vector{Pair{Int,Any}}, propagate_inboun
698
695
end
699
696
end
700
697
if isa (item, InliningTodo)
701
- compact. ssa_rename[old_idx] = ir_inline_item! (compact, idx, argexprs, ir . linetable, item, boundscheck, state. todo_bbs)
698
+ compact. ssa_rename[old_idx] = ir_inline_item! (compact, idx, argexprs, item, boundscheck, state. todo_bbs)
702
699
elseif isa (item, UnionSplit)
703
- compact. ssa_rename[old_idx] = ir_inline_unionsplit! (compact, idx, argexprs, ir . linetable, item, boundscheck, state. todo_bbs, params)
700
+ compact. ssa_rename[old_idx] = ir_inline_unionsplit! (compact, idx, argexprs, item, boundscheck, state. todo_bbs, params)
704
701
end
705
702
compact[idx] = nothing
706
703
refinish && finish_current_bb! (compact, 0 )
@@ -1795,15 +1792,18 @@ function late_inline_special_case!(
1795
1792
return nothing
1796
1793
end
1797
1794
1798
- function ssa_substitute! (insert_node!:: Inserter ,
1799
- subst_inst:: Instruction , @nospecialize (val), arg_replacements:: Vector{Any} ,
1800
- @nospecialize (spsig), spvals:: SimpleVector ,
1801
- spvals_ssa:: Union{Nothing, SSAValue} ,
1802
- linetable_offset:: Int32 , boundscheck:: Symbol )
1795
+ struct SSASubstitute
1796
+ mi:: MethodInstance
1797
+ arg_replacements:: Vector{Any}
1798
+ spvals_ssa:: Union{Nothing,SSAValue}
1799
+ linetable_offset:: Int32
1800
+ end
1801
+
1802
+ function ssa_substitute! (insert_node!:: Inserter , subst_inst:: Instruction , @nospecialize (val),
1803
+ ssa_substitute:: SSASubstitute , boundscheck:: Symbol )
1803
1804
subst_inst[:flag ] &= ~ IR_FLAG_INBOUNDS
1804
- subst_inst[:line ] += linetable_offset
1805
- return ssa_substitute_op! (insert_node!, subst_inst,
1806
- val, arg_replacements, spsig, spvals, spvals_ssa, boundscheck)
1805
+ subst_inst[:line ] += ssa_substitute. linetable_offset
1806
+ return ssa_substitute_op! (insert_node!, subst_inst, val, ssa_substitute, boundscheck)
1807
1807
end
1808
1808
1809
1809
function insert_spval! (insert_node!:: Inserter , spvals_ssa:: SSAValue , spidx:: Int , do_isdefined:: Bool )
@@ -1819,26 +1819,24 @@ function insert_spval!(insert_node!::Inserter, spvals_ssa::SSAValue, spidx::Int,
1819
1819
return (ret, tcheck_not)
1820
1820
end
1821
1821
1822
- function ssa_substitute_op! (insert_node!:: Inserter , subst_inst:: Instruction ,
1823
- @nospecialize (val), arg_replacements:: Vector{Any} ,
1824
- @nospecialize (spsig), spvals:: SimpleVector ,
1825
- spvals_ssa:: Union{Nothing, SSAValue} ,
1826
- boundscheck:: Symbol )
1822
+ function ssa_substitute_op! (insert_node!:: Inserter , subst_inst:: Instruction , @nospecialize (val),
1823
+ ssa_substitute:: SSASubstitute , boundscheck:: Symbol )
1827
1824
if isa (val, Argument)
1828
- return arg_replacements[val. n]
1825
+ return ssa_substitute . arg_replacements[val. n]
1829
1826
end
1830
1827
if isa (val, Expr)
1831
1828
e = val:: Expr
1832
1829
head = e. head
1830
+ sparam_vals = ssa_substitute. mi. sparam_vals
1833
1831
if head === :static_parameter
1834
1832
spidx = e. args[1 ]:: Int
1835
- val = spvals [spidx]
1833
+ val = sparam_vals [spidx]
1836
1834
if ! isa (val, TypeVar) && val != = Vararg
1837
1835
return quoted (val)
1838
1836
else
1839
1837
flag = subst_inst[:flag ]
1840
1838
maybe_undef = (flag & IR_FLAG_NOTHROW) == 0 && isa (val, TypeVar)
1841
- (ret, tcheck_not) = insert_spval! (insert_node!, spvals_ssa:: SSAValue , spidx, maybe_undef)
1839
+ (ret, tcheck_not) = insert_spval! (insert_node!, ssa_substitute . spvals_ssa:: SSAValue , spidx, maybe_undef)
1842
1840
if maybe_undef
1843
1841
insert_node! (
1844
1842
NewInstruction (Expr (:throw_undef_if_not , val. name, tcheck_not), Nothing))
@@ -1847,27 +1845,29 @@ function ssa_substitute_op!(insert_node!::Inserter, subst_inst::Instruction,
1847
1845
end
1848
1846
elseif head === :isdefined && isa (e. args[1 ], Expr) && e. args[1 ]. head === :static_parameter
1849
1847
spidx = (e. args[1 ]:: Expr ). args[1 ]:: Int
1850
- val = spvals [spidx]
1848
+ val = sparam_vals [spidx]
1851
1849
if ! isa (val, TypeVar)
1852
1850
return true
1853
1851
else
1854
- (_, tcheck_not) = insert_spval! (insert_node!, spvals_ssa:: SSAValue , spidx, true )
1852
+ (_, tcheck_not) = insert_spval! (insert_node!, ssa_substitute . spvals_ssa:: SSAValue , spidx, true )
1855
1853
return tcheck_not
1856
1854
end
1857
- elseif head === :cfunction && spvals_ssa === nothing
1858
- @assert ! isa (spsig, UnionAll) || ! isempty (spvals)
1859
- e. args[3 ] = ccall (:jl_instantiate_type_in_env , Any, (Any, Any, Ptr{Any}), e. args[3 ], spsig, spvals)
1855
+ elseif head === :cfunction && ssa_substitute. spvals_ssa === nothing
1856
+ msig = (ssa_substitute. mi. def:: Method ). sig
1857
+ @assert ! isa (msig, UnionAll) || ! isempty (sparam_vals)
1858
+ e. args[3 ] = ccall (:jl_instantiate_type_in_env , Any, (Any, Any, Ptr{Any}), e. args[3 ], msig, sparam_vals)
1860
1859
e. args[4 ] = svec (Any[
1861
- ccall (:jl_instantiate_type_in_env , Any, (Any, Any, Ptr{Any}), argt, spsig, spvals )
1860
+ ccall (:jl_instantiate_type_in_env , Any, (Any, Any, Ptr{Any}), argt, msig, sparam_vals )
1862
1861
for argt in e. args[4 ]:: SimpleVector ]. .. )
1863
- elseif head === :foreigncall && spvals_ssa === nothing
1864
- @assert ! isa (spsig, UnionAll) || ! isempty (spvals)
1862
+ elseif head === :foreigncall && ssa_substitute. spvals_ssa === nothing
1863
+ msig = (ssa_substitute. mi. def:: Method ). sig
1864
+ @assert ! isa (msig, UnionAll) || ! isempty (sparam_vals)
1865
1865
for i = 1 : length (e. args)
1866
1866
if i == 2
1867
- e. args[2 ] = ccall (:jl_instantiate_type_in_env , Any, (Any, Any, Ptr{Any}), e. args[2 ], spsig, spvals )
1867
+ e. args[2 ] = ccall (:jl_instantiate_type_in_env , Any, (Any, Any, Ptr{Any}), e. args[2 ], msig, sparam_vals )
1868
1868
elseif i == 3
1869
1869
e. args[3 ] = svec (Any[
1870
- ccall (:jl_instantiate_type_in_env , Any, (Any, Any, Ptr{Any}), argt, spsig, spvals )
1870
+ ccall (:jl_instantiate_type_in_env , Any, (Any, Any, Ptr{Any}), argt, msig, sparam_vals )
1871
1871
for argt in e. args[3 ]:: SimpleVector ]. .. )
1872
1872
end
1873
1873
end
@@ -1884,7 +1884,7 @@ function ssa_substitute_op!(insert_node!::Inserter, subst_inst::Instruction,
1884
1884
isa (val, Union{SSAValue, NewSSAValue}) && return val # avoid infinite loop
1885
1885
urs = userefs (val)
1886
1886
for op in urs
1887
- op[] = ssa_substitute_op! (insert_node!, subst_inst, op[], arg_replacements, spsig, spvals, spvals_ssa , boundscheck)
1887
+ op[] = ssa_substitute_op! (insert_node!, subst_inst, op[], ssa_substitute , boundscheck)
1888
1888
end
1889
1889
return urs[]
1890
1890
end
0 commit comments