diff --git a/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h b/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h index f127d16b8de12..8010c1c091e40 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h +++ b/llvm/include/llvm/Transforms/Instrumentation/PGOCtxProfLowering.h @@ -24,5 +24,20 @@ class PGOCtxProfLoweringPass : public PassInfoMixin { PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); }; + +// Utility pass blocking inlining for any function that may be overridden during +// linking by a prevailing copy. +// This avoids confusingly collecting profiles for the same GUID corresponding +// to different variants of the function. We could do like PGO and identify +// functions by a (GUID, Hash) tuple, but since the ctxprof "use" waits for +// thinlto to happen before performing any further optimizations, it's +// unnecessary to collect profiles for non-prevailing copies. +class NoinlineNonPrevailing : public PassInfoMixin { +public: + explicit NoinlineNonPrevailing() = default; + + PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); +}; + } // namespace llvm #endif diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index 53fdaa8d9a2a8..d4c3fe9562f9e 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -1252,6 +1252,13 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, MPM.addPass(AssignGUIDPass()); if (IsCtxProfUse) return MPM; + // Block further inlining in the instrumented ctxprof case. This avoids + // confusingly collecting profiles for the same GUID corresponding to + // different variants of the function. We could do like PGO and identify + // functions by a (GUID, Hash) tuple, but since the ctxprof "use" waits for + // thinlto to happen before performing any further optimizations, it's + // unnecessary to collect profiles for non-prevailing copies. + MPM.addPass(NoinlineNonPrevailing()); addPostPGOLoopRotation(MPM, Level); MPM.addPass(PGOCtxProfLoweringPass()); } else if (IsColdFuncOnlyInstrGen) { diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index a664d6fd7085f..bfd952df25e98 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -62,6 +62,7 @@ MODULE_PASS("cross-dso-cfi", CrossDSOCFIPass()) MODULE_PASS("ctx-instr-gen", PGOInstrumentationGen(PGOInstrumentationType::CTXPROF)) MODULE_PASS("ctx-prof-flatten", PGOCtxProfFlatteningPass()) +MODULE_PASS("noinline-nonprevailing", NoinlineNonPrevailing()) MODULE_PASS("deadargelim", DeadArgumentEliminationPass()) MODULE_PASS("debugify", NewPMDebugifyPass()) MODULE_PASS("dfsan", DataFlowSanitizerPass()) diff --git a/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp b/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp index e7b7c26c493e5..aa6bee23ad5ff 100644 --- a/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp +++ b/llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp @@ -351,3 +351,25 @@ bool CtxInstrumentationLowerer::lowerFunction(Function &F) { F.getName()); return true; } + +PreservedAnalyses NoinlineNonPrevailing::run(Module &M, + ModuleAnalysisManager &MAM) { + bool Changed = false; + for (auto &F : M) { + if (F.isDeclaration()) + continue; + if (F.hasFnAttribute(Attribute::NoInline)) + continue; + if (!F.isWeakForLinker()) + continue; + + if (F.hasFnAttribute(Attribute::AlwaysInline)) + F.removeFnAttr(Attribute::AlwaysInline); + + F.addFnAttr(Attribute::NoInline); + Changed = true; + } + if (Changed) + return PreservedAnalyses::none(); + return PreservedAnalyses::all(); +} diff --git a/llvm/test/Transforms/PGOProfile/ctx-instrumentation-block-inline.ll b/llvm/test/Transforms/PGOProfile/ctx-instrumentation-block-inline.ll new file mode 100644 index 0000000000000..b455f327bf5a6 --- /dev/null +++ b/llvm/test/Transforms/PGOProfile/ctx-instrumentation-block-inline.ll @@ -0,0 +1,25 @@ +; RUN: opt -passes=noinline-nonprevailing -S < %s 2>&1 | FileCheck %s + +define void @a() { + ret void +} + +define void @b() #0 { + ret void +} + +define weak_odr void @c() { + ret void +} + +define weak_odr void @d() #0{ + ret void +} + +attributes #0 = { alwaysinline } + +; CHECK: void @a() { +; CHECK: void @b() #0 +; CHECK: void @c() #1 +; CHECK: void @d() #1 +; CHECK: attributes #1 = { noinline }