[DirectX] Legalize memcpy #139173
Open
[DirectX] Legalize memcpy #139173
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
|
@llvm/pr-subscribers-backend-directx Author: Deric C. (Icohedron) ChangesFixes #137188 This PR legalizes memcpy for DXIL in cases where:
These assumption simplify the legalization and, with the addition of #138991, covers the currently-known cases of memcpy that appear when compiling DML shaders. Full diff: https://github.com/llvm/llvm-project/pull/139173.diff 2 Files Affected:
diff --git a/llvm/lib/Target/DirectX/DXILLegalizePass.cpp b/llvm/lib/Target/DirectX/DXILLegalizePass.cpp
index be77a70fa46ba..d9e560a9f8859 100644
--- a/llvm/lib/Target/DirectX/DXILLegalizePass.cpp
+++ b/llvm/lib/Target/DirectX/DXILLegalizePass.cpp
@@ -15,6 +15,7 @@
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Module.h"
#include "llvm/Pass.h"
+#include "llvm/Support/ErrorHandling.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include <functional>
@@ -246,6 +247,58 @@ downcastI64toI32InsertExtractElements(Instruction &I,
}
}
+static void emitMemcpyExpansion(IRBuilder<> &Builder, Value *Dst, Value *Src,
+ ConstantInt *Length) {
+
+ uint64_t ByteLength = Length->getZExtValue();
+ if (ByteLength == 0)
+ return;
+
+ LLVMContext &Ctx = Builder.getContext();
+ const DataLayout &DL = Builder.GetInsertBlock()->getModule()->getDataLayout();
+
+ auto GetArrTyFromVal = [](Value *Val) {
+ if (auto *Alloca = dyn_cast<AllocaInst>(Val))
+ return dyn_cast<ArrayType>(Alloca->getAllocatedType());
+ if (auto *GlobalVar = dyn_cast<GlobalVariable>(Val))
+ return dyn_cast<ArrayType>(GlobalVar->getValueType());
+ llvm_unreachable(
+ "Expected an Alloca or GlobalVariable in memcpy Src and Dst");
+ };
+
+ ArrayType *ArrTy = GetArrTyFromVal(Dst);
+ assert(ArrTy && "Expected Dst of memcpy to be a Pointer to an Array Type");
+ if (auto *DstGlobalVar = dyn_cast<GlobalVariable>(Dst))
+ assert(!DstGlobalVar->isConstant() &&
+ "The Dst of memcpy must not be a constant Global Variable");
+
+ [[maybe_unused]] ArrayType *SrcArrTy = GetArrTyFromVal(Src);
+ assert(SrcArrTy && "Expected Src of memcpy to be a Pointer to an Array Type");
+
+ // This assumption simplifies implementation and covers currently-known
+ // use-cases for DXIL. It may be relaxed in the future if required.
+ assert(ArrTy == SrcArrTy && "Array Types of Src and Dst in memcpy must match");
+
+ Type *ElemTy = ArrTy->getElementType();
+ uint64_t ElemSize = DL.getTypeStoreSize(ElemTy);
+ assert(ElemSize > 0 && "Size must be set");
+
+ [[maybe_unused]] uint64_t Size = ArrTy->getArrayNumElements();
+ assert(ElemSize * Size >= ByteLength &&
+ "Array size must be at least as large as the memcpy length");
+
+ uint64_t NumElemsToCopy = ByteLength / ElemSize;
+ assert(ByteLength % ElemSize == 0 &&
+ "memcpy length must be divisible by array element type");
+ for (uint64_t I = 0; I < NumElemsToCopy; ++I) {
+ Value *Offset = ConstantInt::get(Type::getInt32Ty(Ctx), I);
+ Value *SrcPtr = Builder.CreateGEP(ElemTy, Src, Offset, "gep");
+ Value *SrcVal = Builder.CreateLoad(ElemTy, SrcPtr);
+ Value *DstPtr = Builder.CreateGEP(ElemTy, Dst, Offset, "gep");
+ Builder.CreateStore(SrcVal, DstPtr);
+ }
+}
+
static void emitMemsetExpansion(IRBuilder<> &Builder, Value *Dst, Value *Val,
ConstantInt *SizeCI,
DenseMap<Value *, Value *> &ReplacedValues) {
@@ -296,6 +349,30 @@ static void emitMemsetExpansion(IRBuilder<> &Builder, Value *Dst, Value *Val,
}
}
+static void removeMemCpy(Instruction &I,
+ SmallVectorImpl<Instruction *> &ToRemove,
+ DenseMap<Value *, Value *> &ReplacedValues) {
+
+ CallInst *CI = dyn_cast<CallInst>(&I);
+ if (!CI)
+ return;
+
+ Intrinsic::ID ID = CI->getIntrinsicID();
+ if (ID != Intrinsic::memcpy)
+ return;
+
+ IRBuilder<> Builder(&I);
+ Value *Dst = CI->getArgOperand(0);
+ Value *Src = CI->getArgOperand(1);
+ ConstantInt *Length = dyn_cast<ConstantInt>(CI->getArgOperand(2));
+ assert(Length && "Expected Length to be a ConstantInt");
+ ConstantInt *IsVolatile = dyn_cast<ConstantInt>(CI->getArgOperand(3));
+ assert(IsVolatile && "Expected IsVolatile to be a ConstantInt");
+ assert(IsVolatile->getZExtValue() == 0 && "Expected IsVolatile to be false");
+ emitMemcpyExpansion(Builder, Dst, Src, Length);
+ ToRemove.push_back(CI);
+}
+
static void removeMemSet(Instruction &I,
SmallVectorImpl<Instruction *> &ToRemove,
DenseMap<Value *, Value *> &ReplacedValues) {
@@ -348,6 +425,7 @@ class DXILLegalizationPipeline {
LegalizationPipeline.push_back(fixI8UseChain);
LegalizationPipeline.push_back(downcastI64toI32InsertExtractElements);
LegalizationPipeline.push_back(legalizeFreeze);
+ LegalizationPipeline.push_back(removeMemCpy);
LegalizationPipeline.push_back(removeMemSet);
}
};
diff --git a/llvm/test/CodeGen/DirectX/legalize-memcpy.ll b/llvm/test/CodeGen/DirectX/legalize-memcpy.ll
new file mode 100644
index 0000000000000..09a4d051a2822
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/legalize-memcpy.ll
@@ -0,0 +1,174 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -dxil-legalize -dxil-finalize-linkage -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
+
+@outputStrides = external local_unnamed_addr addrspace(2) global [2 x <4 x i32>], align 4
+
+define void @replace_2x4xint_global_memcpy_test() #0 {
+; CHECK-LABEL: define void @replace_2x4xint_global_memcpy_test(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[TMP1:%.*]] = alloca [2 x <4 x i32>], align 16
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 32, ptr nonnull [[TMP1]])
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i32>, ptr addrspace(2) @outputStrides, align 16
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr <4 x i32>, ptr [[TMP1]], i32 0
+; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[GEP]], align 16
+; CHECK-NEXT: [[TMP3:%.*]] = load <4 x i32>, ptr addrspace(2) getelementptr (<4 x i32>, ptr addrspace(2) @outputStrides, i32 1), align 16
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr <4 x i32>, ptr [[TMP1]], i32 1
+; CHECK-NEXT: store <4 x i32> [[TMP3]], ptr [[GEP1]], align 16
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 32, ptr nonnull [[TMP1]])
+; CHECK-NEXT: ret void
+;
+ %1 = alloca [2 x <4 x i32>], align 16
+ call void @llvm.lifetime.start.p0(i64 32, ptr nonnull %1)
+ call void @llvm.memcpy.p0.p2.i32(ptr nonnull align 16 dereferenceable(32) %1, ptr addrspace(2) align 16 dereferenceable(32) @outputStrides, i32 32, i1 false)
+ call void @llvm.lifetime.end.p0(i64 32, ptr nonnull %1)
+ ret void
+}
+
+define void @replace_int_memcpy_test() #0 {
+; CHECK-LABEL: define void @replace_int_memcpy_test(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = alloca [1 x i32], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = alloca [1 x i32], align 4
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[TMP1]])
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[TMP2]])
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i32, ptr [[TMP2]], i32 0
+; CHECK-NEXT: store i32 [[TMP3]], ptr [[GEP1]], align 4
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[TMP2]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[TMP1]])
+; CHECK-NEXT: ret void
+;
+ %1 = alloca [1 x i32], align 4
+ %2 = alloca [1 x i32], align 4
+ call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %1)
+ call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %2)
+ call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 4 dereferenceable(4) %2, ptr align 4 dereferenceable(4) %1, i32 4, i1 false)
+ call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %2)
+ call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %1)
+ ret void
+}
+
+define void @replace_int16_memcpy_test() #0 {
+; CHECK-LABEL: define void @replace_int16_memcpy_test(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = alloca [2 x i16], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = alloca [2 x i16], align 2
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[TMP1]])
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[TMP2]])
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i16, ptr [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = load i16, ptr [[GEP]], align 2
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i16, ptr [[TMP2]], i32 0
+; CHECK-NEXT: store i16 [[TMP3]], ptr [[GEP1]], align 2
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr i16, ptr [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = load i16, ptr [[GEP2]], align 2
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr i16, ptr [[TMP2]], i32 1
+; CHECK-NEXT: store i16 [[TMP4]], ptr [[GEP3]], align 2
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[TMP2]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[TMP1]])
+; CHECK-NEXT: ret void
+;
+ %1 = alloca [2 x i16], align 2
+ %2 = alloca [2 x i16], align 2
+ call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %1)
+ call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %2)
+ call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 2 dereferenceable(4) %2, ptr align 2 dereferenceable(4) %1, i32 4, i1 false)
+ call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %2)
+ call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %1)
+ ret void
+}
+
+define void @replace_float_memcpy_test() #0 {
+; CHECK-LABEL: define void @replace_float_memcpy_test(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = alloca [2 x float], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = alloca [2 x float], align 4
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[TMP1]])
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 8, ptr nonnull [[TMP2]])
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr float, ptr [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[GEP]], align 4
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr float, ptr [[TMP2]], i32 0
+; CHECK-NEXT: store float [[TMP3]], ptr [[GEP1]], align 4
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr float, ptr [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[GEP2]], align 4
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr float, ptr [[TMP2]], i32 1
+; CHECK-NEXT: store float [[TMP4]], ptr [[GEP3]], align 4
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[TMP2]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 8, ptr nonnull [[TMP1]])
+; CHECK-NEXT: ret void
+;
+ %1 = alloca [2 x float], align 4
+ %2 = alloca [2 x float], align 4
+ call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %1)
+ call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %2)
+ call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 4 dereferenceable(8) %2, ptr align 4 dereferenceable(8) %1, i32 8, i1 false)
+ call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %2)
+ call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %1)
+ ret void
+}
+
+define void @replace_double_memcpy_test() #0 {
+; CHECK-LABEL: define void @replace_double_memcpy_test(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = alloca [2 x double], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = alloca [2 x double], align 4
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP1]])
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 16, ptr nonnull [[TMP2]])
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr double, ptr [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = load double, ptr [[GEP]], align 8
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr double, ptr [[TMP2]], i32 0
+; CHECK-NEXT: store double [[TMP3]], ptr [[GEP1]], align 8
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr double, ptr [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = load double, ptr [[GEP2]], align 8
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr double, ptr [[TMP2]], i32 1
+; CHECK-NEXT: store double [[TMP4]], ptr [[GEP3]], align 8
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP2]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 16, ptr nonnull [[TMP1]])
+; CHECK-NEXT: ret void
+;
+ %1 = alloca [2 x double], align 4
+ %2 = alloca [2 x double], align 4
+ call void @llvm.lifetime.start.p0(i64 16, ptr nonnull %1)
+ call void @llvm.lifetime.start.p0(i64 16, ptr nonnull %2)
+ call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 4 dereferenceable(8) %2, ptr align 4 dereferenceable(8) %1, i32 16, i1 false)
+ call void @llvm.lifetime.end.p0(i64 16, ptr nonnull %2)
+ call void @llvm.lifetime.end.p0(i64 16, ptr nonnull %1)
+ ret void
+}
+
+define void @replace_half_memcpy_test() #0 {
+; CHECK-LABEL: define void @replace_half_memcpy_test(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = alloca [2 x half], align 2
+; CHECK-NEXT: [[TMP2:%.*]] = alloca [2 x half], align 2
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[TMP1]])
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4, ptr nonnull [[TMP2]])
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr half, ptr [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = load half, ptr [[GEP]], align 2
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr half, ptr [[TMP2]], i32 0
+; CHECK-NEXT: store half [[TMP3]], ptr [[GEP1]], align 2
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr half, ptr [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = load half, ptr [[GEP2]], align 2
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr half, ptr [[TMP2]], i32 1
+; CHECK-NEXT: store half [[TMP4]], ptr [[GEP3]], align 2
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[TMP2]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4, ptr nonnull [[TMP1]])
+; CHECK-NEXT: ret void
+;
+ %1 = alloca [2 x half], align 2
+ %2 = alloca [2 x half], align 2
+ call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %1)
+ call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %2)
+ call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 2 dereferenceable(4) %2, ptr align 2 dereferenceable(4) %1, i32 4, i1 false)
+ call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %2)
+ call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %1)
+ ret void
+}
+
+attributes #0 = {"hlsl.export"}
+
+
+declare void @llvm.lifetime.end.p0(i64 immarg, ptr captures(none))
+declare void @llvm.lifetime.start.p0(i64 immarg, ptr captures(none))
+declare void @llvm.memcpy.p0.p2.i32(ptr noalias, ptr addrspace(2) noalias readonly, i32, i1)
+declare void @llvm.memcpy.p0.p0.i32(ptr noalias, ptr noalias readonly, i32, i1)
|
inbelic
approved these changes
May 9, 2025
Co-authored-by: Finn Plummer <[email protected]>
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Fixes #137188
This PR legalizes memcpy for DXIL in cases where:
These assumptions simplify the legalization and, with the addition of #138991, covers the currently-known cases of memcpy that appear when compiling DML shaders.