diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp index 69d07f27fa8e1..9bec782ca8ce9 100644 --- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -2730,6 +2730,54 @@ void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) { EmitToStreamer(*OutStreamer, TmpInstSB); return; } + case AArch64::TLSDESC_AUTH_CALLSEQ: { + /// lower this to: + /// adrp x0, :tlsdesc_auth:var + /// ldr x16, [x0, #:tlsdesc_auth_lo12:var] + /// add x0, x0, #:tlsdesc_auth_lo12:var + /// blraa x16, x0 + /// (TPIDR_EL0 offset now in x0) + const MachineOperand &MO_Sym = MI->getOperand(0); + MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym); + MCOperand SymTLSDescLo12, SymTLSDesc; + MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF); + MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE); + MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12); + MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc); + + MCInst Adrp; + Adrp.setOpcode(AArch64::ADRP); + Adrp.addOperand(MCOperand::createReg(AArch64::X0)); + Adrp.addOperand(SymTLSDesc); + EmitToStreamer(*OutStreamer, Adrp); + + MCInst Ldr; + Ldr.setOpcode(AArch64::LDRXui); + Ldr.addOperand(MCOperand::createReg(AArch64::X16)); + Ldr.addOperand(MCOperand::createReg(AArch64::X0)); + Ldr.addOperand(SymTLSDescLo12); + Ldr.addOperand(MCOperand::createImm(0)); + EmitToStreamer(*OutStreamer, Ldr); + + MCInst Add; + Add.setOpcode(AArch64::ADDXri); + Add.addOperand(MCOperand::createReg(AArch64::X0)); + Add.addOperand(MCOperand::createReg(AArch64::X0)); + Add.addOperand(SymTLSDescLo12); + Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0))); + EmitToStreamer(*OutStreamer, Add); + + // Authenticated TLSDESC accesses are not relaxed. + // Thus, do not emit .tlsdesccall for AUTH TLSDESC. + + MCInst Blraa; + Blraa.setOpcode(AArch64::BLRAA); + Blraa.addOperand(MCOperand::createReg(AArch64::X16)); + Blraa.addOperand(MCOperand::createReg(AArch64::X0)); + EmitToStreamer(*OutStreamer, Blraa); + + return; + } case AArch64::TLSDESC_CALLSEQ: { /// lower this to: /// adrp x0, :tlsdesc:var diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index a6f8f47f31fa5..24e1ebd8421fb 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -2669,6 +2669,7 @@ const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const { MAKE_CASE(AArch64ISD::CSINC) MAKE_CASE(AArch64ISD::THREAD_POINTER) MAKE_CASE(AArch64ISD::TLSDESC_CALLSEQ) + MAKE_CASE(AArch64ISD::TLSDESC_AUTH_CALLSEQ) MAKE_CASE(AArch64ISD::PROBED_ALLOCA) MAKE_CASE(AArch64ISD::ABDS_PRED) MAKE_CASE(AArch64ISD::ABDU_PRED) @@ -10123,8 +10124,11 @@ SDValue AArch64TargetLowering::LowerELFTLSDescCallSeq(SDValue SymAddr, SDValue Chain = DAG.getEntryNode(); SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); - Chain = - DAG.getNode(AArch64ISD::TLSDESC_CALLSEQ, DL, NodeTys, {Chain, SymAddr}); + unsigned Opcode = + DAG.getMachineFunction().getInfo()->hasELFSignedGOT() + ? AArch64ISD::TLSDESC_AUTH_CALLSEQ + : AArch64ISD::TLSDESC_CALLSEQ; + Chain = DAG.getNode(Opcode, DL, NodeTys, {Chain, SymAddr}); SDValue Glue = Chain.getValue(1); return DAG.getCopyFromReg(Chain, DL, AArch64::X0, PtrVT, Glue); @@ -10136,8 +10140,12 @@ AArch64TargetLowering::LowerELFGlobalTLSAddress(SDValue Op, assert(Subtarget->isTargetELF() && "This function expects an ELF target"); const GlobalAddressSDNode *GA = cast(Op); + AArch64FunctionInfo *MFI = + DAG.getMachineFunction().getInfo(); - TLSModel::Model Model = getTargetMachine().getTLSModel(GA->getGlobal()); + TLSModel::Model Model = MFI->hasELFSignedGOT() + ? TLSModel::GeneralDynamic + : getTargetMachine().getTLSModel(GA->getGlobal()); if (!EnableAArch64ELFLocalDynamicTLSGeneration) { if (Model == TLSModel::LocalDynamic) @@ -10174,8 +10182,6 @@ AArch64TargetLowering::LowerELFGlobalTLSAddress(SDValue Op, // calculation. // These accesses will need deduplicating if there's more than one. - AArch64FunctionInfo *MFI = - DAG.getMachineFunction().getInfo(); MFI->incNumLocalDynamicTLSAccesses(); // The call needs a relocation too for linker relaxation. It doesn't make diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h index 1b7f328fa729a..85b62be5dd30d 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h @@ -83,6 +83,7 @@ enum NodeType : unsigned { // Produces the full sequence of instructions for getting the thread pointer // offset of a variable into X0, using the TLSDesc model. TLSDESC_CALLSEQ, + TLSDESC_AUTH_CALLSEQ, ADRP, // Page address of a TargetGlobalAddress operand. ADR, // ADR ADDlow, // Add the low 12 bits of a TargetGlobalAddress operand. diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index 629098cda0c4e..ec891ea4bac85 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -883,6 +883,9 @@ def AArch64tlsdesc_callseq : SDNode<"AArch64ISD::TLSDESC_CALLSEQ", SDT_AArch64TLSDescCallSeq, [SDNPOutGlue, SDNPHasChain, SDNPVariadic]>; +def AArch64tlsdesc_auth_callseq : SDNode<"AArch64ISD::TLSDESC_AUTH_CALLSEQ", + SDT_AArch64TLSDescCallSeq, + [SDNPOutGlue, SDNPHasChain, SDNPVariadic]>; def AArch64WrapperLarge : SDNode<"AArch64ISD::WrapperLarge", SDT_AArch64WrapperLarge>; @@ -3312,8 +3315,16 @@ def TLSDESC_CALLSEQ : Pseudo<(outs), (ins i64imm:$sym), [(AArch64tlsdesc_callseq tglobaltlsaddr:$sym)]>, Sched<[WriteI, WriteLD, WriteI, WriteBrReg]>; +let isCall = 1, Defs = [NZCV, LR, X0, X16], hasSideEffects = 1, Size = 16, + isCodeGenOnly = 1 in +def TLSDESC_AUTH_CALLSEQ + : Pseudo<(outs), (ins i64imm:$sym), + [(AArch64tlsdesc_auth_callseq tglobaltlsaddr:$sym)]>, + Sched<[WriteI, WriteLD, WriteI, WriteBrReg]>; def : Pat<(AArch64tlsdesc_callseq texternalsym:$sym), (TLSDESC_CALLSEQ texternalsym:$sym)>; +def : Pat<(AArch64tlsdesc_auth_callseq texternalsym:$sym), + (TLSDESC_AUTH_CALLSEQ texternalsym:$sym)>; //===----------------------------------------------------------------------===// // Conditional branch (immediate) instruction. diff --git a/llvm/test/CodeGen/AArch64/ptrauth-arm64-tls-dynamics.ll b/llvm/test/CodeGen/AArch64/ptrauth-arm64-tls-dynamics.ll new file mode 100644 index 0000000000000..89731e62dcc1e --- /dev/null +++ b/llvm/test/CodeGen/AArch64/ptrauth-arm64-tls-dynamics.ll @@ -0,0 +1,114 @@ +; RUN: llc -mtriple=aarch64-unknown-linux-gnu -mattr=+pauth -relocation-model=pic \ +; RUN: -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -mtriple=aarch64-unknown-linux-gnu -mattr=+pauth -relocation-model=pic \ +; RUN: -filetype=obj < %s | llvm-readelf -r -s - | FileCheck --check-prefix=CHECK-OBJ %s +; RUN: not --crash llc -mtriple=aarch64-unknown-linux-gnu -mattr=+pauth -relocation-model=pic \ +; RUN: -global-isel=1 < %s 2>&1 | FileCheck --check-prefix=CHECK-ERR %s + +@general_dynamic_var = external thread_local global i32 + +define i32 @test_generaldynamic() { +; CHECK-LABEL: test_generaldynamic: + + %val = load i32, ptr @general_dynamic_var + ret i32 %val + +; CHECK: adrp x[[TLSDESC_HI:[0-9]+]], :tlsdesc_auth:general_dynamic_var +; CHECK-NEXT: ldr x16, [x[[TLSDESC_HI]], :tlsdesc_auth_lo12:general_dynamic_var] +; CHECK-NEXT: add x0, x[[TLSDESC_HI]], :tlsdesc_auth_lo12:general_dynamic_var +; CHECK-NEXT: blraa x16, x0 +; CHECK-NEXT: mrs x[[TPIDR:[0-9]+]], TPIDR_EL0 +; CHECK-NEXT: ldr w0, [x[[TPIDR]], x0] + +; CHECK-OBJ: R_AARCH64_AUTH_TLSDESC_ADR_PAGE21 +; CHECK-OBJ: R_AARCH64_AUTH_TLSDESC_LD64_LO12 +; CHECK-OBJ: R_AARCH64_AUTH_TLSDESC_ADD_LO12 +; CHECK-OBJ-NOT: R_AARCH64_TLSDESC_CALL + +; CHECK-ERR: LLVM ERROR: cannot select: %1:gpr64sp(p0) = G_GLOBAL_VALUE @general_dynamic_var (in function: test_generaldynamic) +} + +define ptr @test_generaldynamic_addr() { +; CHECK-LABEL: test_generaldynamic_addr: + + ret ptr @general_dynamic_var + +; CHECK: adrp x[[TLSDESC_HI:[0-9]+]], :tlsdesc_auth:general_dynamic_var +; CHECK-NEXT: ldr x16, [x[[TLSDESC_HI]], :tlsdesc_auth_lo12:general_dynamic_var] +; CHECK-NEXT: add x0, x[[TLSDESC_HI]], :tlsdesc_auth_lo12:general_dynamic_var +; CHECK-NEXT: blraa x16, x0 +; CHECK-NEXT: mrs [[TP:x[0-9]+]], TPIDR_EL0 +; CHECK-NEXT: add x0, [[TP]], x0 + +; CHECK-OBJ: R_AARCH64_AUTH_TLSDESC_ADR_PAGE21 +; CHECK-OBJ: R_AARCH64_AUTH_TLSDESC_LD64_LO12 +; CHECK-OBJ: R_AARCH64_AUTH_TLSDESC_ADD_LO12 +; CHECK-OBJ-NOT: R_AARCH64_TLSDESC_CALL +} + +;; Note: with signed TLSDESC, general dynamic model is always used, +;; even when local dynamic is requested. + +@local_dynamic_var = external thread_local(localdynamic) global i32 + +define i32 @test_localdynamic() { +; CHECK-LABEL: test_localdynamic: + + %val = load i32, ptr @local_dynamic_var + ret i32 %val + +; CHECK: adrp x[[TLSDESC_HI:[0-9]+]], :tlsdesc_auth:local_dynamic_var +; CHECK-NEXT: ldr x16, [x[[TLSDESC_HI]], :tlsdesc_auth_lo12:local_dynamic_var] +; CHECK-NEXT: add x0, x[[TLSDESC_HI]], :tlsdesc_auth_lo12:local_dynamic_var +; CHECK-NEXT: blraa x16, x0 +; CHECK-NEXT: mrs x[[TPIDR:[0-9]+]], TPIDR_EL0 +; CHECK-NEXT: ldr w0, [x[[TPIDR]], x0] + +; CHECK-OBJ: R_AARCH64_AUTH_TLSDESC_ADR_PAGE21 +; CHECK-OBJ: R_AARCH64_AUTH_TLSDESC_LD64_LO12 +; CHECK-OBJ: R_AARCH64_AUTH_TLSDESC_ADD_LO12 +; CHECK-OBJ-NOT: R_AARCH64_TLSDESC_CALL +} + +define ptr @test_localdynamic_addr() { +; CHECK-LABEL: test_localdynamic_addr: + + ret ptr @local_dynamic_var + +; CHECK: adrp x[[TLSDESC_HI:[0-9]+]], :tlsdesc_auth:local_dynamic_var +; CHECK-NEXT: ldr x16, [x[[TLSDESC_HI]], :tlsdesc_auth_lo12:local_dynamic_var] +; CHECK-NEXT: add x0, x[[TLSDESC_HI]], :tlsdesc_auth_lo12:local_dynamic_var +; CHECK-NEXT: blraa x16, x0 +; CHECK-NEXT: mrs x[[TPIDR:[0-9]+]], TPIDR_EL0 +; CHECK-NEXT: add x0, x[[TPIDR]], x0 + +; CHECK-OBJ: R_AARCH64_AUTH_TLSDESC_ADR_PAGE21 +; CHECK-OBJ: R_AARCH64_AUTH_TLSDESC_LD64_LO12 +; CHECK-OBJ: R_AARCH64_AUTH_TLSDESC_ADD_LO12 +; CHECK-OBJ-NOT: R_AARCH64_TLSDESC_CALL +} + +@extern_weak_var = extern_weak thread_local global i32 + +define i32 @test_extern_weak() { +; CHECK-LABEL: test_extern_weak: + + %val = load i32, ptr @extern_weak_var + ret i32 %val + +; CHECK: adrp x[[TLSDESC_HI:[0-9]+]], :tlsdesc_auth:extern_weak_var +; CHECK-NEXT: ldr x16, [x[[TLSDESC_HI]], :tlsdesc_auth_lo12:extern_weak_var] +; CHECK-NEXT: add x0, x[[TLSDESC_HI]], :tlsdesc_auth_lo12:extern_weak_var +; CHECK-NEXT: blraa x16, x0 +; CHECK-NEXT: mrs x[[TPIDR:[0-9]+]], TPIDR_EL0 +; CHECK-NEXT: ldr w0, [x[[TPIDR]], x0] + +; CHECK-OBJ: R_AARCH64_AUTH_TLSDESC_ADR_PAGE21 +; CHECK-OBJ: R_AARCH64_AUTH_TLSDESC_LD64_LO12 +; CHECK-OBJ: R_AARCH64_AUTH_TLSDESC_ADD_LO12 +; CHECK-OBJ-NOT: R_AARCH64_TLSDESC_CALL +; CHECK-OBJ: 0000000000000000 0 TLS WEAK DEFAULT UND extern_weak_var +} + +!llvm.module.flags = !{!0} +!0 = !{i32 8, !"ptrauth-elf-got", i32 1}