diff --git a/llvm/lib/Target/DirectX/DXILLegalizePass.cpp b/llvm/lib/Target/DirectX/DXILLegalizePass.cpp index be77a70fa46ba..65441fe7ba5bf 100644 --- a/llvm/lib/Target/DirectX/DXILLegalizePass.cpp +++ b/llvm/lib/Target/DirectX/DXILLegalizePass.cpp @@ -246,6 +246,62 @@ downcastI64toI32InsertExtractElements(Instruction &I, } } +static void emitMemcpyExpansion(IRBuilder<> &Builder, Value *Dst, Value *Src, + ConstantInt *Length) { + + uint64_t ByteLength = Length->getZExtValue(); + // If length to copy is zero, no memcpy is needed. + if (ByteLength == 0) + return; + + LLVMContext &Ctx = Builder.getContext(); + const DataLayout &DL = Builder.GetInsertBlock()->getModule()->getDataLayout(); + + auto GetArrTyFromVal = [](Value *Val) -> ArrayType * { + assert(isa(Val) || + isa(Val) && + "Expected Val to be an Alloca or Global Variable"); + if (auto *Alloca = dyn_cast(Val)) + return dyn_cast(Alloca->getAllocatedType()); + if (auto *GlobalVar = dyn_cast(Val)) + return dyn_cast(GlobalVar->getValueType()); + return nullptr; + }; + + ArrayType *ArrTy = GetArrTyFromVal(Dst); + assert(ArrTy && "Expected Dst of memcpy to be a Pointer to an Array Type"); + if (auto *DstGlobalVar = dyn_cast(Dst)) + assert(!DstGlobalVar->isConstant() && + "The Dst of memcpy must not be a constant Global Variable"); + + [[maybe_unused]] ArrayType *SrcArrTy = GetArrTyFromVal(Src); + assert(SrcArrTy && "Expected Src of memcpy to be a Pointer to an Array Type"); + + // This assumption simplifies implementation and covers currently-known + // use-cases for DXIL. It may be relaxed in the future if required. + assert(ArrTy == SrcArrTy && + "Array Types of Src and Dst in memcpy must match"); + + Type *ElemTy = ArrTy->getElementType(); + uint64_t ElemSize = DL.getTypeStoreSize(ElemTy); + assert(ElemSize > 0 && "Size must be set"); + + [[maybe_unused]] uint64_t Size = ArrTy->getArrayNumElements(); + assert(ElemSize * Size >= ByteLength && + "Array size must be at least as large as the memcpy length"); + + uint64_t NumElemsToCopy = ByteLength / ElemSize; + assert(ByteLength % ElemSize == 0 && + "memcpy length must be divisible by array element type"); + for (uint64_t I = 0; I < NumElemsToCopy; ++I) { + Value *Offset = ConstantInt::get(Type::getInt32Ty(Ctx), I); + Value *SrcPtr = Builder.CreateInBoundsGEP(ElemTy, Src, Offset, "gep"); + Value *SrcVal = Builder.CreateLoad(ElemTy, SrcPtr); + Value *DstPtr = Builder.CreateInBoundsGEP(ElemTy, Dst, Offset, "gep"); + Builder.CreateStore(SrcVal, DstPtr); + } +} + static void emitMemsetExpansion(IRBuilder<> &Builder, Value *Dst, Value *Val, ConstantInt *SizeCI, DenseMap &ReplacedValues) { @@ -296,6 +352,33 @@ static void emitMemsetExpansion(IRBuilder<> &Builder, Value *Dst, Value *Val, } } +// Expands the instruction `I` into corresponding loads and stores if it is a +// memcpy call. In that case, the call instruction is added to the `ToRemove` +// vector. `ReplacedValues` is unused. +static void legalizeMemCpy(Instruction &I, + SmallVectorImpl &ToRemove, + DenseMap &ReplacedValues) { + + CallInst *CI = dyn_cast(&I); + if (!CI) + return; + + Intrinsic::ID ID = CI->getIntrinsicID(); + if (ID != Intrinsic::memcpy) + return; + + IRBuilder<> Builder(&I); + Value *Dst = CI->getArgOperand(0); + Value *Src = CI->getArgOperand(1); + ConstantInt *Length = dyn_cast(CI->getArgOperand(2)); + assert(Length && "Expected Length to be a ConstantInt"); + ConstantInt *IsVolatile = dyn_cast(CI->getArgOperand(3)); + assert(IsVolatile && "Expected IsVolatile to be a ConstantInt"); + assert(IsVolatile->getZExtValue() == 0 && "Expected IsVolatile to be false"); + emitMemcpyExpansion(Builder, Dst, Src, Length); + ToRemove.push_back(CI); +} + static void removeMemSet(Instruction &I, SmallVectorImpl &ToRemove, DenseMap &ReplacedValues) { @@ -348,6 +431,7 @@ class DXILLegalizationPipeline { LegalizationPipeline.push_back(fixI8UseChain); LegalizationPipeline.push_back(downcastI64toI32InsertExtractElements); LegalizationPipeline.push_back(legalizeFreeze); + LegalizationPipeline.push_back(legalizeMemCpy); LegalizationPipeline.push_back(removeMemSet); } }; diff --git a/llvm/test/CodeGen/DirectX/legalize-memcpy.ll b/llvm/test/CodeGen/DirectX/legalize-memcpy.ll new file mode 100644 index 0000000000000..bb460bce1143a --- /dev/null +++ b/llvm/test/CodeGen/DirectX/legalize-memcpy.ll @@ -0,0 +1,133 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 +; RUN: opt -S -dxil-legalize -dxil-finalize-linkage -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s + +define void @replace_int_memcpy_test() #0 { +; CHECK-LABEL: define void @replace_int_memcpy_test( +; CHECK-SAME: ) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[TMP1:%.*]] = alloca [1 x i32], align 4 +; CHECK-NEXT: [[TMP2:%.*]] = alloca [1 x i32], align 4 +; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0 +; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[GEP]], align 4 +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 0 +; CHECK-NEXT: store i32 [[TMP3]], ptr [[GEP1]], align 4 +; CHECK-NEXT: ret void +; + %1 = alloca [1 x i32], align 4 + %2 = alloca [1 x i32], align 4 + call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 4 dereferenceable(4) %2, ptr align 4 dereferenceable(4) %1, i32 4, i1 false) + ret void +} + +define void @replace_3int_memcpy_test() #0 { +; CHECK-LABEL: define void @replace_3int_memcpy_test( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = alloca [3 x i32], align 4 +; CHECK-NEXT: [[TMP2:%.*]] = alloca [3 x i32], align 4 +; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0 +; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[GEP]], align 4 +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 0 +; CHECK-NEXT: store i32 [[TMP3]], ptr [[GEP1]], align 4 +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 1 +; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[GEP2]], align 4 +; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 1 +; CHECK-NEXT: store i32 [[TMP4]], ptr [[GEP3]], align 4 +; CHECK-NEXT: [[GEP4:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 2 +; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[GEP4]], align 4 +; CHECK-NEXT: [[GEP5:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i32 2 +; CHECK-NEXT: store i32 [[TMP5]], ptr [[GEP5]], align 4 +; CHECK-NEXT: ret void +; + %1 = alloca [3 x i32], align 4 + %2 = alloca [3 x i32], align 4 + call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 4 dereferenceable(12) %2, ptr align 4 dereferenceable(12) %1, i32 12, i1 false) + ret void +} + +define void @replace_int16_memcpy_test() #0 { +; CHECK-LABEL: define void @replace_int16_memcpy_test( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = alloca [2 x i16], align 2 +; CHECK-NEXT: [[TMP2:%.*]] = alloca [2 x i16], align 2 +; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 0 +; CHECK-NEXT: [[TMP3:%.*]] = load i16, ptr [[GEP]], align 2 +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i16, ptr [[TMP2]], i32 0 +; CHECK-NEXT: store i16 [[TMP3]], ptr [[GEP1]], align 2 +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i32 1 +; CHECK-NEXT: [[TMP4:%.*]] = load i16, ptr [[GEP2]], align 2 +; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds i16, ptr [[TMP2]], i32 1 +; CHECK-NEXT: store i16 [[TMP4]], ptr [[GEP3]], align 2 +; CHECK-NEXT: ret void +; + %1 = alloca [2 x i16], align 2 + %2 = alloca [2 x i16], align 2 + call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 2 dereferenceable(4) %2, ptr align 2 dereferenceable(4) %1, i32 4, i1 false) + ret void +} + +define void @replace_float_memcpy_test() #0 { +; CHECK-LABEL: define void @replace_float_memcpy_test( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = alloca [2 x float], align 4 +; CHECK-NEXT: [[TMP2:%.*]] = alloca [2 x float], align 4 +; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i32 0 +; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[GEP]], align 4 +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds float, ptr [[TMP2]], i32 0 +; CHECK-NEXT: store float [[TMP3]], ptr [[GEP1]], align 4 +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i32 1 +; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[GEP2]], align 4 +; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds float, ptr [[TMP2]], i32 1 +; CHECK-NEXT: store float [[TMP4]], ptr [[GEP3]], align 4 +; CHECK-NEXT: ret void +; + %1 = alloca [2 x float], align 4 + %2 = alloca [2 x float], align 4 + call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 4 dereferenceable(8) %2, ptr align 4 dereferenceable(8) %1, i32 8, i1 false) + ret void +} + +define void @replace_double_memcpy_test() #0 { +; CHECK-LABEL: define void @replace_double_memcpy_test( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = alloca [2 x double], align 4 +; CHECK-NEXT: [[TMP2:%.*]] = alloca [2 x double], align 4 +; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds double, ptr [[TMP1]], i32 0 +; CHECK-NEXT: [[TMP3:%.*]] = load double, ptr [[GEP]], align 8 +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds double, ptr [[TMP2]], i32 0 +; CHECK-NEXT: store double [[TMP3]], ptr [[GEP1]], align 8 +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds double, ptr [[TMP1]], i32 1 +; CHECK-NEXT: [[TMP4:%.*]] = load double, ptr [[GEP2]], align 8 +; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds double, ptr [[TMP2]], i32 1 +; CHECK-NEXT: store double [[TMP4]], ptr [[GEP3]], align 8 +; CHECK-NEXT: ret void +; + %1 = alloca [2 x double], align 4 + %2 = alloca [2 x double], align 4 + call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 4 dereferenceable(8) %2, ptr align 4 dereferenceable(8) %1, i32 16, i1 false) + ret void +} + +define void @replace_half_memcpy_test() #0 { +; CHECK-LABEL: define void @replace_half_memcpy_test( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: [[TMP1:%.*]] = alloca [2 x half], align 2 +; CHECK-NEXT: [[TMP2:%.*]] = alloca [2 x half], align 2 +; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds half, ptr [[TMP1]], i32 0 +; CHECK-NEXT: [[TMP3:%.*]] = load half, ptr [[GEP]], align 2 +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds half, ptr [[TMP2]], i32 0 +; CHECK-NEXT: store half [[TMP3]], ptr [[GEP1]], align 2 +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds half, ptr [[TMP1]], i32 1 +; CHECK-NEXT: [[TMP4:%.*]] = load half, ptr [[GEP2]], align 2 +; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds half, ptr [[TMP2]], i32 1 +; CHECK-NEXT: store half [[TMP4]], ptr [[GEP3]], align 2 +; CHECK-NEXT: ret void +; + %1 = alloca [2 x half], align 2 + %2 = alloca [2 x half], align 2 + call void @llvm.memcpy.p0.p0.i32(ptr nonnull align 2 dereferenceable(4) %2, ptr align 2 dereferenceable(4) %1, i32 4, i1 false) + ret void +} + +attributes #0 = {"hlsl.export"} + +declare void @llvm.memcpy.p0.p2.i32(ptr noalias, ptr addrspace(2) noalias readonly, i32, i1) +declare void @llvm.memcpy.p0.p0.i32(ptr noalias, ptr noalias readonly, i32, i1)