diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp index 6cf1cfb50a9c1..0fe417c469c4e 100644 --- a/bolt/lib/Core/BinaryFunction.cpp +++ b/bolt/lib/Core/BinaryFunction.cpp @@ -1761,7 +1761,8 @@ bool BinaryFunction::postProcessIndirectBranches( uint64_t LastJT = 0; uint16_t LastJTIndexReg = BC.MIB->getNoRegister(); for (BinaryBasicBlock &BB : blocks()) { - for (MCInst &Instr : BB) { + for (BinaryBasicBlock::iterator II = BB.begin(); II != BB.end(); ++II) { + MCInst &Instr = *II; if (!BC.MIB->isIndirectBranch(Instr)) continue; @@ -1789,7 +1790,7 @@ bool BinaryFunction::postProcessIndirectBranches( const MCExpr *DispExpr; MCInst *PCRelBaseInstr; IndirectBranchType Type = BC.MIB->analyzeIndirectBranch( - Instr, BB.begin(), BB.end(), PtrSize, MemLocInstr, BaseRegNum, + Instr, BB.begin(), II, PtrSize, MemLocInstr, BaseRegNum, IndexRegNum, DispValue, DispExpr, PCRelBaseInstr); if (Type != IndirectBranchType::UNKNOWN || MemLocInstr != nullptr) continue; diff --git a/bolt/lib/Passes/FixRISCVCallsPass.cpp b/bolt/lib/Passes/FixRISCVCallsPass.cpp index 963a8b04cf9db..e2984deda16dc 100644 --- a/bolt/lib/Passes/FixRISCVCallsPass.cpp +++ b/bolt/lib/Passes/FixRISCVCallsPass.cpp @@ -43,7 +43,12 @@ void FixRISCVCallsPass::runOnFunction(BinaryFunction &BF) { MCInst OldCall = *NextII; auto L = BC.scopeLock(); - MIB->createCall(*II, Target, Ctx); + + if (MIB->isTailCall(*NextII)) + MIB->createTailCall(*II, Target, Ctx); + else + MIB->createCall(*II, Target, Ctx); + MIB->moveAnnotations(std::move(OldCall), *II); // The original offset was set on the jalr of the auipc+jalr pair. Since diff --git a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp index 0acfedabd08bd..96c5cddfe00e2 100644 --- a/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp +++ b/bolt/lib/Target/RISCV/RISCVMCPlusBuilder.cpp @@ -83,6 +83,7 @@ class RISCVMCPlusBuilder : public MCPlusBuilder { return false; case RISCV::JALR: case RISCV::C_JALR: + case RISCV::C_JR: return true; } } @@ -154,6 +155,17 @@ class RISCVMCPlusBuilder : public MCPlusBuilder { DispValue = 0; DispExpr = nullptr; PCRelBaseOut = nullptr; + + // Check for the following long tail call sequence: + // 1: auipc xi, %pcrel_hi(sym) + // jalr zero, %pcrel_lo(1b)(xi) + if (Instruction.getOpcode() == RISCV::JALR && Begin != End) { + MCInst &PrevInst = *std::prev(End); + if (isRISCVCall(PrevInst, Instruction) && + Instruction.getOperand(0).getReg() == RISCV::X0) + return IndirectBranchType::POSSIBLE_TAIL_CALL; + } + return IndirectBranchType::UNKNOWN; } diff --git a/bolt/test/RISCV/call-annotations.s b/bolt/test/RISCV/call-annotations.s index e2c5a1040faee..bc539bb0ec779 100644 --- a/bolt/test/RISCV/call-annotations.s +++ b/bolt/test/RISCV/call-annotations.s @@ -4,7 +4,7 @@ // RUN: llvm-mc -triple riscv64 -filetype obj -o %t.o %s // RUN: ld.lld --emit-relocs -o %t %t.o // RUN: llvm-bolt --enable-bat --print-cfg --print-fix-riscv-calls \ -// RUN: --print-only=_start -o /dev/null %t | FileCheck %s +// RUN: -o /dev/null %t | FileCheck %s .text .global f @@ -19,11 +19,24 @@ f: // CHECK-NEXT: jal ra, f # Offset: 8 // CHECK-NEXT: jal zero, f # TAILCALL # Offset: 12 +// CHECK-LABEL: Binary Function "long_tail" after building cfg { +// CHECK: auipc t1, f +// CHECK-NEXT: jalr zero, -24(t1) # TAILCALL # Offset: 8 + +// CHECK-LABEL: Binary Function "compressed_tail" after building cfg { +// CHECK: jr a0 # TAILCALL # Offset: 0 + // CHECK-LABEL: Binary Function "_start" after fix-riscv-calls { // CHECK: call f # Offset: 0 // CHECK-NEXT: call f # Offset: 8 // CHECK-NEXT: tail f # TAILCALL # Offset: 12 +// CHECK-LABEL: Binary Function "long_tail" after fix-riscv-calls { +// CHECK: tail f # TAILCALL # Offset: 4 + +// CHECK-LABEL: Binary Function "compressed_tail" after fix-riscv-calls { +// CHECK: jr a0 # TAILCALL # Offset: 0 + .globl _start .p2align 1 _start: @@ -31,3 +44,21 @@ _start: jal f jal zero, f .size _start, .-_start + + .globl long_tail + .p2align 1 +long_tail: + // NOTE: BOLT assumes indirect calls in single-BB functions are tail calls + // so artificially introduce a second BB to force RISC-V-specific analysis + // to get triggered. + beq a0, a1, 1f +1: + tail f + .size long_tail, .-long_tail + + .globl compressed_tail + .p2align 1 + .option rvc +compressed_tail: + c.jr a0 + .size compressed_tail, .-compressed_tail