Skip to content

Commit a722cbf

Browse files
arsenmarsenm
committed
AMDGPU/GlobalISel: Handle atomic_inc/atomic_dec
The intermediate instruction drops the extra volatile argument. We are missing an atomic ordering on these.
1 parent 9c92864 commit a722cbf

10 files changed

+3829
-164
lines changed

‎llvm/lib/Target/AMDGPU/AMDGPUGISel.td

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,12 @@ def : GINodeEquiv<G_ATOMICRMW_FADD, atomic_load_fadd_glue>;
119119
def : GINodeEquiv<G_AMDGPU_FFBH_U32, AMDGPUffbh_u32>;
120120
def : GINodeEquiv<G_AMDGPU_ATOMIC_CMPXCHG, AMDGPUatomic_cmp_swap>;
121121

122+
// FIXME: Check MMO is atomic
123+
def : GINodeEquiv<G_AMDGPU_ATOMIC_INC, SIatomic_inc>;
124+
def : GINodeEquiv<G_AMDGPU_ATOMIC_DEC, SIatomic_dec>;
125+
def : GINodeEquiv<G_AMDGPU_ATOMIC_INC, atomic_inc_glue>;
126+
def : GINodeEquiv<G_AMDGPU_ATOMIC_DEC, atomic_dec_glue>;
127+
122128

123129
class GISelSop2Pat <
124130
SDPatternOperator node,

‎llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1956,6 +1956,10 @@ bool AMDGPUInstructionSelector::select(MachineInstr &I) {
19561956
return selectG_PTR_MASK(I);
19571957
case TargetOpcode::G_EXTRACT_VECTOR_ELT:
19581958
return selectG_EXTRACT_VECTOR_ELT(I);
1959+
case AMDGPU::G_AMDGPU_ATOMIC_INC:
1960+
case AMDGPU::G_AMDGPU_ATOMIC_DEC:
1961+
initM0(I);
1962+
return selectImpl(I, *CoverageInfo);
19591963
default:
19601964
return selectImpl(I, *CoverageInfo);
19611965
}

‎llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2340,6 +2340,22 @@ bool AMDGPULegalizerInfo::legalizeRawBufferStore(MachineInstr &MI,
23402340
return Ty == S32;
23412341
}
23422342

2343+
bool AMDGPULegalizerInfo::legalizeAtomicIncDec(MachineInstr &MI,
2344+
MachineIRBuilder &B,
2345+
bool IsInc) const {
2346+
B.setInstr(MI);
2347+
unsigned Opc = IsInc ? AMDGPU::G_AMDGPU_ATOMIC_INC :
2348+
AMDGPU::G_AMDGPU_ATOMIC_DEC;
2349+
B.buildInstr(Opc)
2350+
.addDef(MI.getOperand(0).getReg())
2351+
.addUse(MI.getOperand(2).getReg())
2352+
.addUse(MI.getOperand(3).getReg())
2353+
.cloneMemRefs(MI);
2354+
MI.eraseFromParent();
2355+
return true;
2356+
}
2357+
2358+
// FIMXE: Needs observer like custom
23432359
bool AMDGPULegalizerInfo::legalizeIntrinsic(MachineInstr &MI,
23442360
MachineRegisterInfo &MRI,
23452361
MachineIRBuilder &B) const {
@@ -2458,6 +2474,10 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(MachineInstr &MI,
24582474
return legalizeRawBufferStore(MI, MRI, B, false);
24592475
case Intrinsic::amdgcn_raw_buffer_store_format:
24602476
return legalizeRawBufferStore(MI, MRI, B, true);
2477+
case Intrinsic::amdgcn_atomic_inc:
2478+
return legalizeAtomicIncDec(MI, B, true);
2479+
case Intrinsic::amdgcn_atomic_dec:
2480+
return legalizeAtomicIncDec(MI, B, false);
24612481
default:
24622482
return true;
24632483
}

‎llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,10 @@ class AMDGPULegalizerInfo : public LegalizerInfo {
106106
Register Reg) const;
107107
bool legalizeRawBufferStore(MachineInstr &MI, MachineRegisterInfo &MRI,
108108
MachineIRBuilder &B, bool IsFormat) const;
109+
110+
bool legalizeAtomicIncDec(MachineInstr &MI, MachineIRBuilder &B,
111+
bool IsInc) const;
112+
109113
bool legalizeIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI,
110114
MachineIRBuilder &B) const override;
111115

‎llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3183,8 +3183,6 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
31833183
case Intrinsic::amdgcn_ds_fadd:
31843184
case Intrinsic::amdgcn_ds_fmin:
31853185
case Intrinsic::amdgcn_ds_fmax:
3186-
case Intrinsic::amdgcn_atomic_inc:
3187-
case Intrinsic::amdgcn_atomic_dec:
31883186
return getDefaultMappingAllVGPR(MI);
31893187
case Intrinsic::amdgcn_ds_ordered_add:
31903188
case Intrinsic::amdgcn_ds_ordered_swap: {
@@ -3380,7 +3378,9 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
33803378
case AMDGPU::G_ATOMICRMW_UMIN:
33813379
case AMDGPU::G_ATOMICRMW_FADD:
33823380
case AMDGPU::G_ATOMIC_CMPXCHG:
3383-
case AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG: {
3381+
case AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG:
3382+
case AMDGPU::G_AMDGPU_ATOMIC_INC:
3383+
case AMDGPU::G_AMDGPU_ATOMIC_DEC: {
33843384
return getDefaultMappingAllVGPR(MI);
33853385
}
33863386
case AMDGPU::G_BRCOND: {

‎llvm/lib/Target/AMDGPU/SIInstructions.td

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2150,8 +2150,13 @@ def G_AMDGPU_FFBH_U32 : AMDGPUGenericInstruction {
21502150
// operands.
21512151
def G_AMDGPU_ATOMIC_CMPXCHG : AMDGPUGenericInstruction {
21522152
let OutOperandList = (outs type0:$oldval);
2153-
let InOperandList = (ins ptype1:$addr, type0:$cmpval_nnenwval);
2153+
let InOperandList = (ins ptype1:$addr, type0:$cmpval_newval);
21542154
let hasSideEffects = 0;
21552155
let mayLoad = 1;
21562156
let mayStore = 1;
21572157
}
2158+
2159+
let Namespace = "AMDGPU" in {
2160+
def G_AMDGPU_ATOMIC_INC : G_ATOMICRMW_OP;
2161+
def G_AMDGPU_ATOMIC_DEC : G_ATOMICRMW_OP;
2162+
}

0 commit comments

Comments
 (0)