Skip to content

Commit d094353

Browse files
arsenmarsenm
committed
GlobalISel: Apply target MMO flags to atomics
Unify MMO flag handling with SelectionDAG like with loads and stores.
1 parent 0d0fce4 commit d094353

File tree

5 files changed

+117
-17
lines changed

5 files changed

+117
-17
lines changed

‎llvm/include/llvm/CodeGen/TargetLowering.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,8 @@ class TargetLoweringBase {
295295
const DataLayout &DL) const;
296296
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI,
297297
const DataLayout &DL) const;
298+
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI,
299+
const DataLayout &DL) const;
298300

299301
virtual bool isSelectSupported(SelectSupportKind /*kind*/) const {
300302
return true;

‎llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1955,9 +1955,8 @@ bool IRTranslator::translateAtomicCmpXchg(const User &U,
19551955
if (I.isWeak())
19561956
return false;
19571957

1958-
auto Flags = I.isVolatile() ? MachineMemOperand::MOVolatile
1959-
: MachineMemOperand::MONone;
1960-
Flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1958+
auto &TLI = *MF->getSubtarget().getTargetLowering();
1959+
auto Flags = TLI.getAtomicMemOperandFlags(I, *DL);
19611960

19621961
Type *ResType = I.getType();
19631962
Type *ValType = ResType->Type::getStructElementType(0);
@@ -1985,10 +1984,8 @@ bool IRTranslator::translateAtomicCmpXchg(const User &U,
19851984
bool IRTranslator::translateAtomicRMW(const User &U,
19861985
MachineIRBuilder &MIRBuilder) {
19871986
const AtomicRMWInst &I = cast<AtomicRMWInst>(U);
1988-
1989-
auto Flags = I.isVolatile() ? MachineMemOperand::MOVolatile
1990-
: MachineMemOperand::MONone;
1991-
Flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
1987+
auto &TLI = *MF->getSubtarget().getTargetLowering();
1988+
auto Flags = TLI.getAtomicMemOperandFlags(I, *DL);
19921989

19931990
Type *ResType = I.getType();
19941991

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4619,11 +4619,8 @@ void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
46194619
SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
46204620

46214621
auto Alignment = DAG.getEVTAlignment(MemVT);
4622-
4623-
auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
4624-
if (I.isVolatile())
4625-
Flags |= MachineMemOperand::MOVolatile;
4626-
Flags |= DAG.getTargetLoweringInfo().getTargetMMOFlags(I);
4622+
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4623+
auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
46274624

46284625
MachineFunction &MF = DAG.getMachineFunction();
46294626
MachineMemOperand *MMO =
@@ -4670,11 +4667,8 @@ void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
46704667

46714668
auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
46724669
auto Alignment = DAG.getEVTAlignment(MemVT);
4673-
4674-
auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
4675-
if (I.isVolatile())
4676-
Flags |= MachineMemOperand::MOVolatile;
4677-
Flags |= DAG.getTargetLoweringInfo().getTargetMMOFlags(I);
4670+
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4671+
auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout());
46784672

46794673
MachineFunction &MF = DAG.getMachineFunction();
46804674
MachineMemOperand *MMO =

‎llvm/lib/CodeGen/TargetLoweringBase.cpp

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2042,3 +2042,22 @@ TargetLoweringBase::getStoreMemOperandFlags(const StoreInst &SI,
20422042
Flags |= getTargetMMOFlags(SI);
20432043
return Flags;
20442044
}
2045+
2046+
MachineMemOperand::Flags
2047+
TargetLoweringBase::getAtomicMemOperandFlags(const Instruction &AI,
2048+
const DataLayout &DL) const {
2049+
auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
2050+
2051+
if (const AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(&AI)) {
2052+
if (RMW->isVolatile())
2053+
Flags |= MachineMemOperand::MOVolatile;
2054+
} else if (const AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(&AI)) {
2055+
if (CmpX->isVolatile())
2056+
Flags |= MachineMemOperand::MOVolatile;
2057+
} else
2058+
llvm_unreachable("not an atomic instruction");
2059+
2060+
// FIXME: Not preserving dereferenceable
2061+
Flags |= getTargetMMOFlags(AI);
2062+
return Flags;
2063+
}

‎llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-atomic-metadata.ll

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
2+
; RUN: llc -mtriple=aarch64-- -mcpu=falkor -mattr=+lse -O0 -aarch64-enable-atomic-cfg-tidy=0 -stop-after=irtranslator -global-isel -verify-machineinstrs %s -o - | FileCheck %s
3+
4+
define i32 @atomicrmw_volatile(i32* %ptr) {
5+
; CHECK-LABEL: name: atomicrmw_volatile
6+
; CHECK: bb.1 (%ir-block.0):
7+
; CHECK: liveins: $x0
8+
; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
9+
; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
10+
; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:_(s32) = G_ATOMICRMW_ADD [[COPY]](p0), [[C]] :: (volatile load store monotonic 4 on %ir.ptr)
11+
; CHECK: $w0 = COPY [[ATOMICRMW_ADD]](s32)
12+
; CHECK: RET_ReallyLR implicit $w0
13+
%oldval = atomicrmw volatile add i32* %ptr, i32 1 monotonic
14+
ret i32 %oldval
15+
}
16+
17+
define i32 @atomicrmw_falkor(i32* %ptr) {
18+
; CHECK-LABEL: name: atomicrmw_falkor
19+
; CHECK: bb.1 (%ir-block.0):
20+
; CHECK: liveins: $x0
21+
; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
22+
; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
23+
; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:_(s32) = G_ATOMICRMW_ADD [[COPY]](p0), [[C]] :: ("aarch64-strided-access" load store monotonic 4 on %ir.ptr)
24+
; CHECK: $w0 = COPY [[ATOMICRMW_ADD]](s32)
25+
; CHECK: RET_ReallyLR implicit $w0
26+
%oldval = atomicrmw add i32* %ptr, i32 1 monotonic, !falkor.strided.access !0
27+
ret i32 %oldval
28+
}
29+
30+
define i32 @atomicrmw_volatile_falkor(i32* %ptr) {
31+
; CHECK-LABEL: name: atomicrmw_volatile_falkor
32+
; CHECK: bb.1 (%ir-block.0):
33+
; CHECK: liveins: $x0
34+
; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
35+
; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
36+
; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:_(s32) = G_ATOMICRMW_ADD [[COPY]](p0), [[C]] :: (volatile "aarch64-strided-access" load store monotonic 4 on %ir.ptr)
37+
; CHECK: $w0 = COPY [[ATOMICRMW_ADD]](s32)
38+
; CHECK: RET_ReallyLR implicit $w0
39+
%oldval = atomicrmw volatile add i32* %ptr, i32 1 monotonic, !falkor.strided.access !0
40+
ret i32 %oldval
41+
}
42+
43+
define i32 @cmpxchg_volatile(i32* %addr) {
44+
; CHECK-LABEL: name: cmpxchg_volatile
45+
; CHECK: bb.1 (%ir-block.0):
46+
; CHECK: liveins: $x0
47+
; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
48+
; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
49+
; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
50+
; CHECK: [[ATOMIC_CMPXCHG_WITH_SUCCESS:%[0-9]+]]:_(s32), [[ATOMIC_CMPXCHG_WITH_SUCCESS1:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[COPY]](p0), [[C]], [[C1]] :: (volatile load store monotonic monotonic 4 on %ir.addr)
51+
; CHECK: $w0 = COPY [[ATOMIC_CMPXCHG_WITH_SUCCESS]](s32)
52+
; CHECK: RET_ReallyLR implicit $w0
53+
%val_success = cmpxchg volatile i32* %addr, i32 0, i32 1 monotonic monotonic
54+
%value_loaded = extractvalue { i32, i1 } %val_success, 0
55+
ret i32 %value_loaded
56+
}
57+
58+
define i32 @cmpxchg_falkor(i32* %addr) {
59+
; CHECK-LABEL: name: cmpxchg_falkor
60+
; CHECK: bb.1 (%ir-block.0):
61+
; CHECK: liveins: $x0
62+
; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
63+
; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
64+
; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
65+
; CHECK: [[ATOMIC_CMPXCHG_WITH_SUCCESS:%[0-9]+]]:_(s32), [[ATOMIC_CMPXCHG_WITH_SUCCESS1:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[COPY]](p0), [[C]], [[C1]] :: ("aarch64-strided-access" load store monotonic monotonic 4 on %ir.addr)
66+
; CHECK: $w0 = COPY [[ATOMIC_CMPXCHG_WITH_SUCCESS]](s32)
67+
; CHECK: RET_ReallyLR implicit $w0
68+
%val_success = cmpxchg i32* %addr, i32 0, i32 1 monotonic monotonic, !falkor.strided.access !0
69+
%value_loaded = extractvalue { i32, i1 } %val_success, 0
70+
ret i32 %value_loaded
71+
}
72+
73+
define i32 @cmpxchg_volatile_falkor(i32* %addr) {
74+
; CHECK-LABEL: name: cmpxchg_volatile_falkor
75+
; CHECK: bb.1 (%ir-block.0):
76+
; CHECK: liveins: $x0
77+
; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
78+
; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
79+
; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
80+
; CHECK: [[ATOMIC_CMPXCHG_WITH_SUCCESS:%[0-9]+]]:_(s32), [[ATOMIC_CMPXCHG_WITH_SUCCESS1:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[COPY]](p0), [[C]], [[C1]] :: (volatile "aarch64-strided-access" load store monotonic monotonic 4 on %ir.addr)
81+
; CHECK: $w0 = COPY [[ATOMIC_CMPXCHG_WITH_SUCCESS]](s32)
82+
; CHECK: RET_ReallyLR implicit $w0
83+
%val_success = cmpxchg volatile i32* %addr, i32 0, i32 1 monotonic monotonic, !falkor.strided.access !0
84+
%value_loaded = extractvalue { i32, i1 } %val_success, 0
85+
ret i32 %value_loaded
86+
}
87+
88+
!0 = !{}

0 commit comments

Comments
 (0)