Skip to content

Commit bfe84f7

Browse files
VyacheslavLevytskyyVyacheslavLevytskyy
authored
[SPIR-V] Implement support of the SPV_INTEL_split_barrier SPIRV extension (#112359)
This PR implements support of the SPV_EXT_arithmetic_fence SPIRV extension (https://github.com/KhronosGroup/SPIRV-Registry/blob/main/extensions/INTEL/SPV_INTEL_split_barrier.asciidoc) and adds builtins from https://registry.khronos.org/OpenCL/extensions/intel/cl_intel_split_work_group_barrier.html
1 parent 8d8996d commit bfe84f7

10 files changed

+412
-6
lines changed

‎llvm/docs/SPIRVUsage.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,8 @@ list of supported SPIR-V extensions, sorted alphabetically by their extension na
165165
- Adds decorations that can be applied to global (module scope) variables to help code generation for FPGA devices.
166166
* - ``SPV_INTEL_optnone``
167167
- Adds OptNoneINTEL value for Function Control mask that indicates a request to not optimize the function.
168+
* - ``SPV_INTEL_split_barrier``
169+
- Adds SPIR-V instructions to split a control barrier into two separate operations: the first indicates that an invocation has "arrived" at the barrier but should continue executing, and the second indicates that an invocation should "wait" for other invocations to arrive at the barrier before executing further.
168170
* - ``SPV_INTEL_subgroups``
169171
- Allows work items in a subgroup to share data without the use of local memory and work group barriers, and to utilize specialized hardware to load and store blocks of data from images or buffers.
170172
* - ``SPV_INTEL_usm_storage_classes``

‎llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -880,6 +880,18 @@ static bool buildAtomicFlagInst(const SPIRV::IncomingCall *Call,
880880
static bool buildBarrierInst(const SPIRV::IncomingCall *Call, unsigned Opcode,
881881
MachineIRBuilder &MIRBuilder,
882882
SPIRVGlobalRegistry *GR) {
883+
const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
884+
const auto *ST =
885+
static_cast<const SPIRVSubtarget *>(&MIRBuilder.getMF().getSubtarget());
886+
if ((Opcode == SPIRV::OpControlBarrierArriveINTEL ||
887+
Opcode == SPIRV::OpControlBarrierWaitINTEL) &&
888+
!ST->canUseExtension(SPIRV::Extension::SPV_INTEL_split_barrier)) {
889+
std::string DiagMsg = std::string(Builtin->Name) +
890+
": the builtin requires the following SPIR-V "
891+
"extension: SPV_INTEL_split_barrier";
892+
report_fatal_error(DiagMsg.c_str(), false);
893+
}
894+
883895
if (Call->isSpirvOp())
884896
return buildOpFromWrapper(MIRBuilder, Opcode, Call, Register(0));
885897

@@ -896,13 +908,16 @@ static bool buildBarrierInst(const SPIRV::IncomingCall *Call, unsigned Opcode,
896908
if (MemFlags & SPIRV::CLK_IMAGE_MEM_FENCE)
897909
MemSemantics |= SPIRV::MemorySemantics::ImageMemory;
898910

899-
if (Opcode == SPIRV::OpMemoryBarrier) {
900-
std::memory_order MemOrder =
901-
static_cast<std::memory_order>(getIConstVal(Call->Arguments[1], MRI));
902-
MemSemantics = getSPIRVMemSemantics(MemOrder) | MemSemantics;
903-
} else {
911+
if (Opcode == SPIRV::OpMemoryBarrier)
912+
MemSemantics = getSPIRVMemSemantics(static_cast<std::memory_order>(
913+
getIConstVal(Call->Arguments[1], MRI))) |
914+
MemSemantics;
915+
else if (Opcode == SPIRV::OpControlBarrierArriveINTEL)
916+
MemSemantics |= SPIRV::MemorySemantics::Release;
917+
else if (Opcode == SPIRV::OpControlBarrierWaitINTEL)
918+
MemSemantics |= SPIRV::MemorySemantics::Acquire;
919+
else
904920
MemSemantics |= SPIRV::MemorySemantics::SequentiallyConsistent;
905-
}
906921

907922
Register MemSemanticsReg =
908923
MemFlags == MemSemantics

‎llvm/lib/Target/SPIRV/SPIRVBuiltins.td

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -628,6 +628,12 @@ defm : DemangledNativeBuiltin<"barrier", OpenCL_std, Barrier, 1, 3, OpControlBar
628628
defm : DemangledNativeBuiltin<"work_group_barrier", OpenCL_std, Barrier, 1, 3, OpControlBarrier>;
629629
defm : DemangledNativeBuiltin<"__spirv_ControlBarrier", OpenCL_std, Barrier, 3, 3, OpControlBarrier>;
630630

631+
// cl_intel_split_work_group_barrier
632+
defm : DemangledNativeBuiltin<"intel_work_group_barrier_arrive", OpenCL_std, Barrier, 1, 2, OpControlBarrierArriveINTEL>;
633+
defm : DemangledNativeBuiltin<"__spirv_ControlBarrierArriveINTEL", OpenCL_std, Barrier, 3, 3, OpControlBarrierArriveINTEL>;
634+
defm : DemangledNativeBuiltin<"intel_work_group_barrier_wait", OpenCL_std, Barrier, 1, 2, OpControlBarrierWaitINTEL>;
635+
defm : DemangledNativeBuiltin<"__spirv_ControlBarrierWaitINTEL", OpenCL_std, Barrier, 3, 3, OpControlBarrierWaitINTEL>;
636+
631637
// Kernel enqueue builtin records:
632638
defm : DemangledNativeBuiltin<"__enqueue_kernel_basic", OpenCL_std, Enqueue, 5, 5, OpEnqueueKernel>;
633639
defm : DemangledNativeBuiltin<"__enqueue_kernel_basic_events", OpenCL_std, Enqueue, 8, 8, OpEnqueueKernel>;

‎llvm/lib/Target/SPIRV/SPIRVCommandLine.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ static const std::map<std::string, SPIRV::Extension::Extension>
4242
{"SPV_INTEL_optnone", SPIRV::Extension::Extension::SPV_INTEL_optnone},
4343
{"SPV_INTEL_usm_storage_classes",
4444
SPIRV::Extension::Extension::SPV_INTEL_usm_storage_classes},
45+
{"SPV_INTEL_split_barrier",
46+
SPIRV::Extension::Extension::SPV_INTEL_split_barrier},
4547
{"SPV_INTEL_subgroups",
4648
SPIRV::Extension::Extension::SPV_INTEL_subgroups},
4749
{"SPV_KHR_uniform_group_instructions",

‎llvm/lib/Target/SPIRV/SPIRVInstrInfo.td

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -703,6 +703,12 @@ def OpNamedBarrierInitialize: UnOp<"OpNamedBarrierInitialize", 328>;
703703
def OpMemoryNamedBarrier: Op<329, (outs), (ins ID:$barr, ID:$mem, ID:$sem),
704704
"OpMemoryNamedBarrier $barr $mem $sem">;
705705

706+
// SPV_INTEL_split_barrier
707+
def OpControlBarrierArriveINTEL: Op<6142, (outs), (ins ID:$exec, ID:$mem, ID:$sem),
708+
"OpControlBarrierArriveINTEL $exec $mem $sem">;
709+
def OpControlBarrierWaitINTEL: Op<6143, (outs), (ins ID:$exec, ID:$mem, ID:$sem),
710+
"OpControlBarrierWaitINTEL $exec $mem $sem">;
711+
706712
// 3.42.21. Group and Subgroup Instructions
707713

708714
def OpGroupAsyncCopy: Op<259, (outs ID:$res), (ins TYPE:$ty, ID:$scope,

‎llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1211,6 +1211,13 @@ void addInstrRequirements(const MachineInstr &MI,
12111211
Reqs.addExtension(SPIRV::Extension::SPV_EXT_arithmetic_fence);
12121212
Reqs.addCapability(SPIRV::Capability::ArithmeticFenceEXT);
12131213
break;
1214+
case SPIRV::OpControlBarrierArriveINTEL:
1215+
case SPIRV::OpControlBarrierWaitINTEL:
1216+
if (ST.canUseExtension(SPIRV::Extension::SPV_INTEL_split_barrier)) {
1217+
Reqs.addExtension(SPIRV::Extension::SPV_INTEL_split_barrier);
1218+
Reqs.addCapability(SPIRV::Capability::SplitBarrierINTEL);
1219+
}
1220+
break;
12141221
default:
12151222
break;
12161223
}

‎llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -482,6 +482,7 @@ defm GlobalVariableFPGADecorationsINTEL : CapabilityOperand<6189, 0, 0, [SPV_INT
482482
defm CacheControlsINTEL : CapabilityOperand<6441, 0, 0, [SPV_INTEL_cache_controls], []>;
483483
defm CooperativeMatrixKHR : CapabilityOperand<6022, 0, 0, [SPV_KHR_cooperative_matrix], []>;
484484
defm ArithmeticFenceEXT : CapabilityOperand<6144, 0, 0, [SPV_EXT_arithmetic_fence], []>;
485+
defm SplitBarrierINTEL : CapabilityOperand<6141, 0, 0, [SPV_INTEL_split_barrier], []>;
485486

486487
//===----------------------------------------------------------------------===//
487488
// Multiclass used to define SourceLanguage enum values and at the same time

‎llvm/test/CodeGen/SPIRV/extensions/SPV_INTEL_split_barrier/split_work_group_barrier_12.ll

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
; Adapted from Khronos Translator test suite: test/CodeGen/SPIRV/extensions/SPV_INTEL_split_barrier/
2+
3+
;; kernel void test(global uint* dst)
4+
;; {
5+
;; intel_work_group_barrier_arrive(CLK_LOCAL_MEM_FENCE);
6+
;; intel_work_group_barrier_wait(CLK_LOCAL_MEM_FENCE);
7+
;; intel_work_group_barrier_arrive(CLK_GLOBAL_MEM_FENCE);
8+
;; intel_work_group_barrier_wait(CLK_GLOBAL_MEM_FENCE);
9+
;;
10+
;; intel_work_group_barrier_arrive(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);
11+
;; intel_work_group_barrier_wait(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);
12+
;;}
13+
14+
; RUN: not llc -O0 -mtriple=spirv64-unknown-unknown %s -o %t.spvt 2>&1 | FileCheck %s --check-prefix=CHECK-ERROR
15+
16+
; CHECK-ERROR: LLVM ERROR: intel_work_group_barrier_arrive: the builtin requires the following SPIR-V extension: SPV_INTEL_split_barrier
17+
18+
; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_split_barrier %s -o - | FileCheck %s
19+
; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown --spirv-ext=+SPV_INTEL_split_barrier %s -o - -filetype=obj | spirv-val %}
20+
21+
; CHECK: Capability SplitBarrierINTEL
22+
; CHECK: Extension "SPV_INTEL_split_barrier"
23+
; CHECK: %[[#UINT:]] = OpTypeInt 32 0
24+
;
25+
; Scopes:
26+
; CHECK-DAG: %[[#SCOPE_WORK_GROUP:]] = OpConstant %[[#UINT]] 2{{$}}
27+
;
28+
; Memory Semantics:
29+
; 0x2 Acquire + 0x100 WorkgroupMemory
30+
; CHECK-DAG: %[[#ACQUIRE_LOCAL:]] = OpConstant %[[#UINT]] 258
31+
; 0x4 Release + 0x100 WorkgroupMemory
32+
; CHECK-DAG: %[[#RELEASE_LOCAL:]] = OpConstant %[[#UINT]] 260
33+
; 0x2 Acquire + 0x200 CrossWorkgroupMemory
34+
; CHECK-DAG: %[[#ACQUIRE_GLOBAL:]] = OpConstant %[[#UINT]] 514
35+
; 0x4 Release + 0x200 CrossWorkgroupMemory
36+
; CHECK-DAG: %[[#RELEASE_GLOBAL:]] = OpConstant %[[#UINT]] 516
37+
; 0x2 Acquire + 0x100 WorkgroupMemory + 0x200 CrossWorkgroupMemory
38+
; CHECK-DAG: %[[#ACQUIRE_LOCAL_GLOBAL:]] = OpConstant %[[#UINT]] 770
39+
; 0x4 Release + 0x100 WorkgroupMemory + 0x200 CrossWorkgroupMemory
40+
; CHECK-DAG: %[[#RELEASE_LOCAL_GLOBAL:]] = OpConstant %[[#UINT]] 772
41+
;
42+
; CHECK: OpControlBarrierArriveINTEL %[[#SCOPE_WORK_GROUP]] %[[#SCOPE_WORK_GROUP]] %[[#RELEASE_LOCAL]]
43+
; CHECK: OpControlBarrierWaitINTEL %[[#SCOPE_WORK_GROUP]] %[[#SCOPE_WORK_GROUP]] %[[#ACQUIRE_LOCAL]]
44+
; CHECK: OpControlBarrierArriveINTEL %[[#SCOPE_WORK_GROUP]] %[[#SCOPE_WORK_GROUP]] %[[#RELEASE_GLOBAL]]
45+
; CHECK: OpControlBarrierWaitINTEL %[[#SCOPE_WORK_GROUP]] %[[#SCOPE_WORK_GROUP]] %[[#ACQUIRE_GLOBAL]]
46+
;
47+
; CHECK: OpControlBarrierArriveINTEL %[[#SCOPE_WORK_GROUP]] %[[#SCOPE_WORK_GROUP]] %[[#RELEASE_LOCAL_GLOBAL]]
48+
; CHECK: OpControlBarrierWaitINTEL %[[#SCOPE_WORK_GROUP]] %[[#SCOPE_WORK_GROUP]] %[[#ACQUIRE_LOCAL_GLOBAL]]
49+
50+
; Function Attrs: convergent norecurse nounwind
51+
define dso_local spir_kernel void @test(ptr addrspace(1) nocapture noundef readnone align 4 %0) local_unnamed_addr #0 !kernel_arg_addr_space !4 !kernel_arg_access_qual !5 !kernel_arg_type !6 !kernel_arg_base_type !6 !kernel_arg_type_qual !7 {
52+
tail call spir_func void @_Z31intel_work_group_barrier_arrivej(i32 noundef 1) #2
53+
tail call spir_func void @_Z29intel_work_group_barrier_waitj(i32 noundef 1) #2
54+
tail call spir_func void @_Z31intel_work_group_barrier_arrivej(i32 noundef 2) #2
55+
tail call spir_func void @_Z29intel_work_group_barrier_waitj(i32 noundef 2) #2
56+
tail call spir_func void @_Z31intel_work_group_barrier_arrivej(i32 noundef 3) #2
57+
tail call spir_func void @_Z29intel_work_group_barrier_waitj(i32 noundef 3) #2
58+
ret void
59+
}
60+
61+
; Function Attrs: convergent
62+
declare dso_local spir_func void @_Z31intel_work_group_barrier_arrivej(i32 noundef) local_unnamed_addr #1
63+
64+
; Function Attrs: convergent
65+
declare dso_local spir_func void @_Z29intel_work_group_barrier_waitj(i32 noundef) local_unnamed_addr #1
66+
67+
attributes #0 = { convergent norecurse nounwind "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "uniform-work-group-size"="true" }
68+
attributes #1 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
69+
attributes #2 = { convergent nounwind }
70+
71+
!llvm.module.flags = !{!0, !1}
72+
!opencl.ocl.version = !{!2}
73+
!opencl.spir.version = !{!2}
74+
!llvm.ident = !{!3}
75+
76+
!0 = !{i32 1, !"wchar_size", i32 4}
77+
!1 = !{i32 7, !"frame-pointer", i32 2}
78+
!2 = !{i32 1, i32 2}
79+
!3 = !{!"clang version 15.0.0 (https://github.com/llvm/llvm-project 861386dbd6ff0d91636b7c674c2abb2eccd9d3f2)"}
80+
!4 = !{i32 1}
81+
!5 = !{!"none"}
82+
!6 = !{!"uint*"}
83+
!7 = !{!""}

0 commit comments

Comments
 (0)