[RISCV][GlobalISel] Legalize and select G_ATOMICRMW_ADD instruction #153791

ReVe1uv · 2025-08-15T11:30:48Z

This patch adds legalization and instruction selection support for the G_ATOMICRMW_ADD opcode in the RISCV GlobalISel backend. Support for other opcodes will be added in subsequent PRs.

llvmbot · 2025-08-15T11:31:18Z

@llvm/pr-subscribers-llvm-globalisel

Author: Kane Wang (ReVe1uv)

Changes

This patch adds legalization and instruction selection support for the G_ATOMICRMW_ADD opcode in the RISCV GlobalISel backend.

Patch is 26.17 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/153791.diff

7 Files Affected:

(modified) llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp (+69)
(modified) llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp (+7)
(added) llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/atomicrmw-add-rv32.mir (+98)
(added) llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/atomicrmw-add-rv64.mir (+195)
(modified) llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir (+2-2)
(added) llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-add-rv32.mir (+79)
(added) llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-add-rv64.mir (+168)

diff --git a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
index f83c2b6da8923..de34850536e6d 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
@@ -92,6 +92,7 @@ class RISCVInstructionSelector : public InstructionSelector {
   void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
                  MachineIRBuilder &MIB) const;
   bool selectUnmergeValues(MachineInstr &MI, MachineIRBuilder &MIB) const;
+  bool selectAtomicRMWAdd(MachineInstr &MI, MachineIRBuilder &MIB) const;
 
   ComplexRendererFns selectShiftMask(MachineOperand &Root,
                                      unsigned ShiftWidth) const;
@@ -815,6 +816,8 @@ bool RISCVInstructionSelector::select(MachineInstr &MI) {
     return selectImplicitDef(MI, MIB);
   case TargetOpcode::G_UNMERGE_VALUES:
     return selectUnmergeValues(MI, MIB);
+  case TargetOpcode::G_ATOMICRMW_ADD:
+    return selectAtomicRMWAdd(MI, MIB);
   default:
     return false;
   }
@@ -1415,6 +1418,72 @@ void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
   MIB.buildInstr(RISCV::FENCE, {}, {}).addImm(Pred).addImm(Succ);
 }
 
+bool RISCVInstructionSelector::selectAtomicRMWAdd(MachineInstr &MI,
+                                                  MachineIRBuilder &MIB) const {
+  MachineBasicBlock &MBB = *MI.getParent();
+  MachineFunction &MF = *MBB.getParent();
+  MachineRegisterInfo &MRI = MF.getRegInfo();
+
+  Register DstReg = MI.getOperand(0).getReg();
+  Register PtrReg = MI.getOperand(1).getReg();
+  Register ValReg = MI.getOperand(2).getReg();
+
+  if (MI.memoperands_empty())
+    return false;
+  MachineMemOperand *MMO = *MI.memoperands_begin();
+  AtomicOrdering Ordering = MMO->getSuccessOrdering();
+
+  unsigned BaseOpcode;
+  LLT Ty = MRI.getType(DstReg);
+  if (Ty == LLT::scalar(32)) {
+    switch (Ordering) {
+    case AtomicOrdering::Monotonic:
+      BaseOpcode = RISCV::AMOADD_W;
+      break;
+    case AtomicOrdering::Acquire:
+      BaseOpcode = RISCV::AMOADD_W_AQ;
+      break;
+    case AtomicOrdering::Release:
+      BaseOpcode = RISCV::AMOADD_W_RL;
+      break;
+    case AtomicOrdering::AcquireRelease:
+    case AtomicOrdering::SequentiallyConsistent:
+      BaseOpcode = RISCV::AMOADD_W_AQ_RL;
+      break;
+    default:
+      return false;
+    }
+  } else if (Ty == LLT::scalar(64)) {
+    switch (Ordering) {
+    case AtomicOrdering::Monotonic:
+      BaseOpcode = RISCV::AMOADD_D;
+      break;
+    case AtomicOrdering::Acquire:
+      BaseOpcode = RISCV::AMOADD_D_AQ;
+      break;
+    case AtomicOrdering::Release:
+      BaseOpcode = RISCV::AMOADD_D_RL;
+      break;
+    case AtomicOrdering::AcquireRelease:
+    case AtomicOrdering::SequentiallyConsistent:
+      BaseOpcode = RISCV::AMOADD_D_AQ_RL;
+      break;
+    default:
+      return false;
+    }
+  } else {
+    return false;
+  }
+
+  auto NewMI =
+      MIB.buildInstr(BaseOpcode).addDef(DstReg).addUse(ValReg).addUse(PtrReg);
+  NewMI.addMemOperand(MMO);
+
+  MI.eraseFromParent();
+
+  return constrainSelectedInstRegOperands(*NewMI, TII, TRI, RBI);
+}
+
 namespace llvm {
 InstructionSelector *
 createRISCVInstructionSelector(const RISCVTargetMachine &TM,
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index e88f33d6859ec..9986cb39b7965 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -692,6 +692,13 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
       .customIf(all(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST),
                     typeIsLegalIntOrFPVec(1, IntOrFPVecTys, ST)));
 
+  bool ForceAtomics = ST.hasForcedAtomics() && !ST.hasStdExtA();
+
+  getActionDefinitionsBuilder(G_ATOMICRMW_ADD)
+      .legalFor(!ForceAtomics, {{s32, p0}, {sXLen, p0}})
+      .libcallFor(ForceAtomics, {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})
+      .clampScalar(0, s32, sXLen);
+
   getLegacyLegalizerInfo().computeTables();
   verify(*ST.getInstrInfo());
 }
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/atomicrmw-add-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/atomicrmw-add-rv32.mir
new file mode 100644
index 0000000000000..27a531a1a5c2b
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/atomicrmw-add-rv32.mir
@@ -0,0 +1,98 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -mattr=+A -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name:            atomicrmw_add_i32_monotonic
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i32_monotonic
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_W:%[0-9]+]]:gpr = AMOADD_W [[COPY]], [[ADDI]] :: (load store monotonic (s32))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_W]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s32) = G_CONSTANT i32 1
+    %2:gprb(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store monotonic (s32))
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+...
+---
+name:            atomicrmw_add_i32_acquire
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i32_acquire
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_W_AQ:%[0-9]+]]:gpr = AMOADD_W_AQ [[COPY]], [[ADDI]] :: (load store acquire (s32))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_W_AQ]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s32) = G_CONSTANT i32 1
+    %2:gprb(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store acquire (s32))
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+...
+---
+---
+name:            atomicrmw_add_i32_release
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i32_release
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_W_RL:%[0-9]+]]:gpr = AMOADD_W_RL [[COPY]], [[ADDI]] :: (load store release (s32))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_W_RL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s32) = G_CONSTANT i32 1
+    %2:gprb(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store release (s32))
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+...
+---
+---
+name:            atomicrmw_add_i32_acq_rel
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i32_acq_rel
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_W_AQ_RL:%[0-9]+]]:gpr = AMOADD_W_AQ_RL [[COPY]], [[ADDI]] :: (load store acq_rel (s32))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_W_AQ_RL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s32) = G_CONSTANT i32 1
+    %2:gprb(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store acq_rel (s32))
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+...
+---
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/atomicrmw-add-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/atomicrmw-add-rv64.mir
new file mode 100644
index 0000000000000..fb7d87f02a1d5
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/atomicrmw-add-rv64.mir
@@ -0,0 +1,195 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -mattr=+A -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name:            atomicrmw_add_i32_monotonic
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i32_monotonic
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_W:%[0-9]+]]:gpr = AMOADD_W [[ADDI]], [[COPY]] :: (load store monotonic (s32))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_W]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s64) = G_CONSTANT i64 1
+    %2:gprb(s32) = G_TRUNC %1(s64)
+    %3:gprb(s32) = G_ATOMICRMW_ADD %0, %2 :: (load store monotonic (s32))
+    %4:gprb(s64) = G_ANYEXT %3(s32)
+    $x10 = COPY %4(s64)
+    PseudoRET implicit $x10
+...
+---
+name:            atomicrmw_add_i32_acquire
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i32_acquire
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_W_AQ:%[0-9]+]]:gpr = AMOADD_W_AQ [[ADDI]], [[COPY]] :: (load store acquire (s32))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_W_AQ]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s64) = G_CONSTANT i64 1
+    %2:gprb(s32) = G_TRUNC %1(s64)
+    %3:gprb(s32) = G_ATOMICRMW_ADD %0, %2 :: (load store acquire (s32))
+    %4:gprb(s64) = G_ANYEXT %3(s32)
+    $x10 = COPY %4(s64)
+    PseudoRET implicit $x10
+...
+---
+name:            atomicrmw_add_i32_release
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i32_release
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_W_RL:%[0-9]+]]:gpr = AMOADD_W_RL [[ADDI]], [[COPY]] :: (load store release (s32))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_W_RL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s64) = G_CONSTANT i64 1
+    %2:gprb(s32) = G_TRUNC %1(s64)
+    %3:gprb(s32) = G_ATOMICRMW_ADD %0, %2 :: (load store release (s32))
+    %4:gprb(s64) = G_ANYEXT %3(s32)
+    $x10 = COPY %4(s64)
+    PseudoRET implicit $x10
+...
+---
+name:            atomicrmw_add_i32_acq_rel
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i32_acq_rel
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_W_AQ_RL:%[0-9]+]]:gpr = AMOADD_W_AQ_RL [[ADDI]], [[COPY]] :: (load store acq_rel (s32))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_W_AQ_RL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s64) = G_CONSTANT i64 1
+    %2:gprb(s32) = G_TRUNC %1(s64)
+    %3:gprb(s32) = G_ATOMICRMW_ADD %0, %2 :: (load store acq_rel (s32))
+    %4:gprb(s64) = G_ANYEXT %3(s32)
+    $x10 = COPY %4(s64)
+    PseudoRET implicit $x10
+...
+---
+name:            atomicrmw_add_i64_monotonic
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i64_monotonic
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_D:%[0-9]+]]:gpr = AMOADD_D [[COPY]], [[ADDI]] :: (load store monotonic (s64))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_D]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s64) = G_CONSTANT i64 1
+    %2:gprb(s64) = G_ATOMICRMW_ADD %0, %1 :: (load store monotonic (s64))
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+...
+---
+name:            atomicrmw_add_i64_acquire
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i64_acquire
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_D_AQ:%[0-9]+]]:gpr = AMOADD_D_AQ [[COPY]], [[ADDI]] :: (load store acquire (s64))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_D_AQ]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s64) = G_CONSTANT i64 1
+    %2:gprb(s64) = G_ATOMICRMW_ADD %0, %1 :: (load store acquire (s64))
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+...
+---
+name:            atomicrmw_add_i64_release
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i64_release
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_D_RL:%[0-9]+]]:gpr = AMOADD_D_RL [[COPY]], [[ADDI]] :: (load store release (s64))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_D_RL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s64) = G_CONSTANT i64 1
+    %2:gprb(s64) = G_ATOMICRMW_ADD %0, %1 :: (load store release (s64))
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+...
+---
+name:            atomicrmw_add_i64_acq_rel
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i64_acq_rel
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_D_AQ_RL:%[0-9]+]]:gpr = AMOADD_D_AQ_RL [[COPY]], [[ADDI]] :: (load store acq_rel (s64))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_D_AQ_RL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s64) = G_CONSTANT i64 1
+    %2:gprb(s64) = G_ATOMICRMW_ADD %0, %1 :: (load store acq_rel (s64))
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
index 82cc6829838a0..c47ce4d5f1153 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
@@ -222,8 +222,8 @@
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_ATOMICRMW_ADD (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. the first uncovered type index: 2, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_ATOMICRMW_SUB (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-add-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-add-rv32.mir
new file mode 100644
index 0000000000000..2b4201ce08b99
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-add-rv32.mir
@@ -0,0 +1,79 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -mattr=+A -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            atomicrmw_add_i32_monotonic
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    ; CHECK-LABEL: name: atomicrmw_add_i32_monotonic
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(p0) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[ATOMICRMW_ADD:%[0-9]+]]:gprb(s32) = G_ATOMICRMW_ADD [[COPY]](p0), [[C]] :: (load store monotonic (s32))
+    ; CHECK-NEXT: $x10 = COPY [[ATOMICRMW_ADD]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s32) = G_CONSTANT i32 1
+    %2:gprb(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store monotonic (s32))
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+...
+---
+name:            atomicrmw_add_i32_acquire
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    ; CHECK-LABEL: name: atomicrmw_add_i32_acquire
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(p0) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[ATOMICRMW_ADD:%[0-9]+]]:gprb(s32) = G_ATOMICRMW_ADD [[COPY]](p0), [[C]] :: (load store acquire (s32))
+    ; CHECK-NEXT: $x10 = COPY [[ATOMICRMW_ADD]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s32) = G_CONSTANT i32 1
+    %2:gprb(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store acquire (s32))
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+...
+---
+name:            atomicrmw_add_i32_release
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    ; CHECK-LABEL: name: atomicrmw_add_i32_release
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(p0) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[ATOMICRMW_ADD:%[0-9]+]]:gprb(s32) = G_ATOMICRMW_ADD [[COPY]](p0), [[C]] :: (load store release (s32))
+    ; CHECK-NEXT: $x10 = COPY [[ATOMICRMW_ADD]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s32) = G_CONSTANT i32 1
+    %2:gprb(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store release (s32))
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+...
+---
+name:            atomicrmw_add_i32_acq_rel
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    ; CHECK-LABEL: name: atomicrmw_add_i32_acq_rel
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(p0) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[ATOMICRMW_ADD:%[0-9]+]]:gprb(s32) = G_ATOMICRMW_ADD [[COPY]](p0), [[C]] :: (load store acq_rel (s32))
+    ; CHECK-NEXT: $x10 = COPY [[ATOMICRMW_ADD]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s32) = G_CONSTANT i32 1
+    %2:gprb(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store acq_rel (s32))
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-add-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-add-rv64.mir
new file mode 100644
index 0000000000000..47a174e134999
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-add-rv64.mir
@@ -0,0 +1,168 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -mattr=+A -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            atomicrmw_add_i32_monotonic
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i32_monotonic
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(p0) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:gprb(s32) = G_TRUNC [[C]](s64)
+ ...
[truncated]

llvmbot · 2025-08-15T11:31:20Z

@llvm/pr-subscribers-backend-risc-v

Author: Kane Wang (ReVe1uv)

Changes

This patch adds legalization and instruction selection support for the G_ATOMICRMW_ADD opcode in the RISCV GlobalISel backend.

Patch is 26.17 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/153791.diff

7 Files Affected:

(modified) llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp (+69)
(modified) llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp (+7)
(added) llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/atomicrmw-add-rv32.mir (+98)
(added) llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/atomicrmw-add-rv64.mir (+195)
(modified) llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir (+2-2)
(added) llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-add-rv32.mir (+79)
(added) llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-add-rv64.mir (+168)

diff --git a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
index f83c2b6da8923..de34850536e6d 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
@@ -92,6 +92,7 @@ class RISCVInstructionSelector : public InstructionSelector {
   void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
                  MachineIRBuilder &MIB) const;
   bool selectUnmergeValues(MachineInstr &MI, MachineIRBuilder &MIB) const;
+  bool selectAtomicRMWAdd(MachineInstr &MI, MachineIRBuilder &MIB) const;
 
   ComplexRendererFns selectShiftMask(MachineOperand &Root,
                                      unsigned ShiftWidth) const;
@@ -815,6 +816,8 @@ bool RISCVInstructionSelector::select(MachineInstr &MI) {
     return selectImplicitDef(MI, MIB);
   case TargetOpcode::G_UNMERGE_VALUES:
     return selectUnmergeValues(MI, MIB);
+  case TargetOpcode::G_ATOMICRMW_ADD:
+    return selectAtomicRMWAdd(MI, MIB);
   default:
     return false;
   }
@@ -1415,6 +1418,72 @@ void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
   MIB.buildInstr(RISCV::FENCE, {}, {}).addImm(Pred).addImm(Succ);
 }
 
+bool RISCVInstructionSelector::selectAtomicRMWAdd(MachineInstr &MI,
+                                                  MachineIRBuilder &MIB) const {
+  MachineBasicBlock &MBB = *MI.getParent();
+  MachineFunction &MF = *MBB.getParent();
+  MachineRegisterInfo &MRI = MF.getRegInfo();
+
+  Register DstReg = MI.getOperand(0).getReg();
+  Register PtrReg = MI.getOperand(1).getReg();
+  Register ValReg = MI.getOperand(2).getReg();
+
+  if (MI.memoperands_empty())
+    return false;
+  MachineMemOperand *MMO = *MI.memoperands_begin();
+  AtomicOrdering Ordering = MMO->getSuccessOrdering();
+
+  unsigned BaseOpcode;
+  LLT Ty = MRI.getType(DstReg);
+  if (Ty == LLT::scalar(32)) {
+    switch (Ordering) {
+    case AtomicOrdering::Monotonic:
+      BaseOpcode = RISCV::AMOADD_W;
+      break;
+    case AtomicOrdering::Acquire:
+      BaseOpcode = RISCV::AMOADD_W_AQ;
+      break;
+    case AtomicOrdering::Release:
+      BaseOpcode = RISCV::AMOADD_W_RL;
+      break;
+    case AtomicOrdering::AcquireRelease:
+    case AtomicOrdering::SequentiallyConsistent:
+      BaseOpcode = RISCV::AMOADD_W_AQ_RL;
+      break;
+    default:
+      return false;
+    }
+  } else if (Ty == LLT::scalar(64)) {
+    switch (Ordering) {
+    case AtomicOrdering::Monotonic:
+      BaseOpcode = RISCV::AMOADD_D;
+      break;
+    case AtomicOrdering::Acquire:
+      BaseOpcode = RISCV::AMOADD_D_AQ;
+      break;
+    case AtomicOrdering::Release:
+      BaseOpcode = RISCV::AMOADD_D_RL;
+      break;
+    case AtomicOrdering::AcquireRelease:
+    case AtomicOrdering::SequentiallyConsistent:
+      BaseOpcode = RISCV::AMOADD_D_AQ_RL;
+      break;
+    default:
+      return false;
+    }
+  } else {
+    return false;
+  }
+
+  auto NewMI =
+      MIB.buildInstr(BaseOpcode).addDef(DstReg).addUse(ValReg).addUse(PtrReg);
+  NewMI.addMemOperand(MMO);
+
+  MI.eraseFromParent();
+
+  return constrainSelectedInstRegOperands(*NewMI, TII, TRI, RBI);
+}
+
 namespace llvm {
 InstructionSelector *
 createRISCVInstructionSelector(const RISCVTargetMachine &TM,
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index e88f33d6859ec..9986cb39b7965 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -692,6 +692,13 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
       .customIf(all(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST),
                     typeIsLegalIntOrFPVec(1, IntOrFPVecTys, ST)));
 
+  bool ForceAtomics = ST.hasForcedAtomics() && !ST.hasStdExtA();
+
+  getActionDefinitionsBuilder(G_ATOMICRMW_ADD)
+      .legalFor(!ForceAtomics, {{s32, p0}, {sXLen, p0}})
+      .libcallFor(ForceAtomics, {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})
+      .clampScalar(0, s32, sXLen);
+
   getLegacyLegalizerInfo().computeTables();
   verify(*ST.getInstrInfo());
 }
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/atomicrmw-add-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/atomicrmw-add-rv32.mir
new file mode 100644
index 0000000000000..27a531a1a5c2b
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/atomicrmw-add-rv32.mir
@@ -0,0 +1,98 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -mattr=+A -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name:            atomicrmw_add_i32_monotonic
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i32_monotonic
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_W:%[0-9]+]]:gpr = AMOADD_W [[COPY]], [[ADDI]] :: (load store monotonic (s32))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_W]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s32) = G_CONSTANT i32 1
+    %2:gprb(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store monotonic (s32))
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+...
+---
+name:            atomicrmw_add_i32_acquire
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i32_acquire
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_W_AQ:%[0-9]+]]:gpr = AMOADD_W_AQ [[COPY]], [[ADDI]] :: (load store acquire (s32))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_W_AQ]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s32) = G_CONSTANT i32 1
+    %2:gprb(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store acquire (s32))
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+...
+---
+---
+name:            atomicrmw_add_i32_release
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i32_release
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_W_RL:%[0-9]+]]:gpr = AMOADD_W_RL [[COPY]], [[ADDI]] :: (load store release (s32))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_W_RL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s32) = G_CONSTANT i32 1
+    %2:gprb(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store release (s32))
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+...
+---
+---
+name:            atomicrmw_add_i32_acq_rel
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i32_acq_rel
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_W_AQ_RL:%[0-9]+]]:gpr = AMOADD_W_AQ_RL [[COPY]], [[ADDI]] :: (load store acq_rel (s32))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_W_AQ_RL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s32) = G_CONSTANT i32 1
+    %2:gprb(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store acq_rel (s32))
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+...
+---
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/atomicrmw-add-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/atomicrmw-add-rv64.mir
new file mode 100644
index 0000000000000..fb7d87f02a1d5
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/atomicrmw-add-rv64.mir
@@ -0,0 +1,195 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -mattr=+A -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name:            atomicrmw_add_i32_monotonic
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i32_monotonic
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_W:%[0-9]+]]:gpr = AMOADD_W [[ADDI]], [[COPY]] :: (load store monotonic (s32))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_W]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s64) = G_CONSTANT i64 1
+    %2:gprb(s32) = G_TRUNC %1(s64)
+    %3:gprb(s32) = G_ATOMICRMW_ADD %0, %2 :: (load store monotonic (s32))
+    %4:gprb(s64) = G_ANYEXT %3(s32)
+    $x10 = COPY %4(s64)
+    PseudoRET implicit $x10
+...
+---
+name:            atomicrmw_add_i32_acquire
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i32_acquire
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_W_AQ:%[0-9]+]]:gpr = AMOADD_W_AQ [[ADDI]], [[COPY]] :: (load store acquire (s32))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_W_AQ]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s64) = G_CONSTANT i64 1
+    %2:gprb(s32) = G_TRUNC %1(s64)
+    %3:gprb(s32) = G_ATOMICRMW_ADD %0, %2 :: (load store acquire (s32))
+    %4:gprb(s64) = G_ANYEXT %3(s32)
+    $x10 = COPY %4(s64)
+    PseudoRET implicit $x10
+...
+---
+name:            atomicrmw_add_i32_release
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i32_release
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_W_RL:%[0-9]+]]:gpr = AMOADD_W_RL [[ADDI]], [[COPY]] :: (load store release (s32))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_W_RL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s64) = G_CONSTANT i64 1
+    %2:gprb(s32) = G_TRUNC %1(s64)
+    %3:gprb(s32) = G_ATOMICRMW_ADD %0, %2 :: (load store release (s32))
+    %4:gprb(s64) = G_ANYEXT %3(s32)
+    $x10 = COPY %4(s64)
+    PseudoRET implicit $x10
+...
+---
+name:            atomicrmw_add_i32_acq_rel
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i32_acq_rel
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_W_AQ_RL:%[0-9]+]]:gpr = AMOADD_W_AQ_RL [[ADDI]], [[COPY]] :: (load store acq_rel (s32))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_W_AQ_RL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s64) = G_CONSTANT i64 1
+    %2:gprb(s32) = G_TRUNC %1(s64)
+    %3:gprb(s32) = G_ATOMICRMW_ADD %0, %2 :: (load store acq_rel (s32))
+    %4:gprb(s64) = G_ANYEXT %3(s32)
+    $x10 = COPY %4(s64)
+    PseudoRET implicit $x10
+...
+---
+name:            atomicrmw_add_i64_monotonic
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i64_monotonic
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_D:%[0-9]+]]:gpr = AMOADD_D [[COPY]], [[ADDI]] :: (load store monotonic (s64))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_D]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s64) = G_CONSTANT i64 1
+    %2:gprb(s64) = G_ATOMICRMW_ADD %0, %1 :: (load store monotonic (s64))
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+...
+---
+name:            atomicrmw_add_i64_acquire
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i64_acquire
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_D_AQ:%[0-9]+]]:gpr = AMOADD_D_AQ [[COPY]], [[ADDI]] :: (load store acquire (s64))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_D_AQ]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s64) = G_CONSTANT i64 1
+    %2:gprb(s64) = G_ATOMICRMW_ADD %0, %1 :: (load store acquire (s64))
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+...
+---
+name:            atomicrmw_add_i64_release
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i64_release
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_D_RL:%[0-9]+]]:gpr = AMOADD_D_RL [[COPY]], [[ADDI]] :: (load store release (s64))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_D_RL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s64) = G_CONSTANT i64 1
+    %2:gprb(s64) = G_ATOMICRMW_ADD %0, %1 :: (load store release (s64))
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+...
+---
+name:            atomicrmw_add_i64_acq_rel
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i64_acq_rel
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; CHECK-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI $x0, 1
+    ; CHECK-NEXT: [[AMOADD_D_AQ_RL:%[0-9]+]]:gpr = AMOADD_D_AQ_RL [[COPY]], [[ADDI]] :: (load store acq_rel (s64))
+    ; CHECK-NEXT: $x10 = COPY [[AMOADD_D_AQ_RL]]
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s64) = G_CONSTANT i64 1
+    %2:gprb(s64) = G_ATOMICRMW_ADD %0, %1 :: (load store acq_rel (s64))
+    $x10 = COPY %2(s64)
+    PseudoRET implicit $x10
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
index 82cc6829838a0..c47ce4d5f1153 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
@@ -222,8 +222,8 @@
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: G_ATOMICRMW_ADD (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. the first uncovered type index: 2, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_ATOMICRMW_SUB (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-add-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-add-rv32.mir
new file mode 100644
index 0000000000000..2b4201ce08b99
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-add-rv32.mir
@@ -0,0 +1,79 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv32 -mattr=+A -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            atomicrmw_add_i32_monotonic
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    ; CHECK-LABEL: name: atomicrmw_add_i32_monotonic
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(p0) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[ATOMICRMW_ADD:%[0-9]+]]:gprb(s32) = G_ATOMICRMW_ADD [[COPY]](p0), [[C]] :: (load store monotonic (s32))
+    ; CHECK-NEXT: $x10 = COPY [[ATOMICRMW_ADD]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s32) = G_CONSTANT i32 1
+    %2:gprb(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store monotonic (s32))
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+...
+---
+name:            atomicrmw_add_i32_acquire
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    ; CHECK-LABEL: name: atomicrmw_add_i32_acquire
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(p0) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[ATOMICRMW_ADD:%[0-9]+]]:gprb(s32) = G_ATOMICRMW_ADD [[COPY]](p0), [[C]] :: (load store acquire (s32))
+    ; CHECK-NEXT: $x10 = COPY [[ATOMICRMW_ADD]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s32) = G_CONSTANT i32 1
+    %2:gprb(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store acquire (s32))
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+...
+---
+name:            atomicrmw_add_i32_release
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    ; CHECK-LABEL: name: atomicrmw_add_i32_release
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(p0) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[ATOMICRMW_ADD:%[0-9]+]]:gprb(s32) = G_ATOMICRMW_ADD [[COPY]](p0), [[C]] :: (load store release (s32))
+    ; CHECK-NEXT: $x10 = COPY [[ATOMICRMW_ADD]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s32) = G_CONSTANT i32 1
+    %2:gprb(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store release (s32))
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+...
+---
+name:            atomicrmw_add_i32_acq_rel
+body:             |
+  bb.0.entry:
+    liveins: $x10
+    ; CHECK-LABEL: name: atomicrmw_add_i32_acq_rel
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(p0) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[ATOMICRMW_ADD:%[0-9]+]]:gprb(s32) = G_ATOMICRMW_ADD [[COPY]](p0), [[C]] :: (load store acq_rel (s32))
+    ; CHECK-NEXT: $x10 = COPY [[ATOMICRMW_ADD]](s32)
+    ; CHECK-NEXT: PseudoRET implicit $x10
+    %0:gprb(p0) = COPY $x10
+    %1:gprb(s32) = G_CONSTANT i32 1
+    %2:gprb(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store acq_rel (s32))
+    $x10 = COPY %2(s32)
+    PseudoRET implicit $x10
+...
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-add-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-add-rv64.mir
new file mode 100644
index 0000000000000..47a174e134999
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-add-rv64.mir
@@ -0,0 +1,168 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=riscv64 -mattr=+A -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name:            atomicrmw_add_i32_monotonic
+body:             |
+  bb.0.entry:
+    liveins: $x10
+
+    ; CHECK-LABEL: name: atomicrmw_add_i32_monotonic
+    ; CHECK: liveins: $x10
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprb(p0) = COPY $x10
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:gprb(s32) = G_TRUNC [[C]](s64)
+ ...
[truncated]

ReVe1uv · 2025-08-15T11:34:44Z

Hi @topperc — I can’t assign reviewers on this repo, so I’m tagging you here for a review when you get a chance. Thanks!
This PR adds instruction selection and legalization support for G_ATOMICRMW_ADD on RISC-V.

topperc · 2025-08-15T15:23:22Z

llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp

@@ -1415,6 +1418,72 @@ void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
  MIB.buildInstr(RISCV::FENCE, {}, {}).addImm(Pred).addImm(Succ);
 }

+bool RISCVInstructionSelector::selectAtomicRMWAdd(MachineInstr &MI,
+                                                  MachineIRBuilder &MIB) const {
+  MachineBasicBlock &MBB = *MI.getParent();


Why aren't we able to import the patterns from tablegen?

I tried adding
def : GINodeEquiv<G_ATOMICRMW_ADD, atomic_load_add>;
in llvm/lib/Target/RISCV/RISCVInstrGISel.td. With this change, on riscv64a, G_ATOMICRMW_ADD instructions with 64-bit data width correctly select AMOADD_D. However, G_ATOMICRMW_ADD instructions with 32-bit data width do not select AMOADD_W on riscv64a (selection works correctly on riscv32a).

Isn't that GINodeEquiv already present in llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td?

Yes, it’s already defined in SelectionDAGCompat.td.

llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/atomicrmw-add-rv32.mir

llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp

topperc · 2025-08-20T02:48:09Z

Please add end to end IR tests.

ReVe1uv · 2025-08-21T01:56:30Z

Please add end to end IR tests.

While working on the end-to-end IR tests, I found that with -mattr=+a only, sub-i32 atomicrmw add operations are expanded during the atomic-expand phase into intrinsics (via shouldExpandAtomicRMWInIR), e.g., call i64 @llvm.riscv.masked.atomicrmw.add.i64.p0. To handle this, I added cases for these intrinsics in RISCVLegalizerInfo::legalizeIntrinsic. The end-to-end IR test is in llvm/test/CodeGen/RISCV/GlobalISel/atomicrmw-add.ll.

llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp

topperc

LGTM

topperc · 2025-08-25T19:20:55Z

@ReVe1uv do you have commit access?

ReVe1uv · 2025-08-26T01:23:40Z

do you have commit access?

Hi @topperc , I don’t have the access. Could you please help merge this change?

topperc · 2025-08-26T02:00:39Z

llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-atomicrmw-add-rv32.mir

+  bb.0.entry:
+    liveins: $x10, $x11
+
+    ; CHECK-LABEL: name: atomicrmw_add_i8_monotonic


There is no RUN line with CHECK in its prefix list. Can you remove them?

There is no RUN line with CHECK in its prefix list. Can you remove them?

Thanks! I’ve removed them from all test cases and will avoid this in future.

topperc

I missed the unused CHECK lines before

This patch adds legalization and instruction selection support for the G_ATOMICRMW_ADD opcode in the RISCV GlobalISel backend.

topperc

LGTM

llvmbot added backend:RISC-V llvm:globalisel labels Aug 15, 2025

ReVe1uv force-pushed the riscv-g-atomic-add branch from a15e2a3 to 324f6cf Compare August 15, 2025 11:51

topperc reviewed Aug 15, 2025

View reviewed changes

llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/atomicrmw-add-rv32.mir Outdated Show resolved Hide resolved

topperc reviewed Aug 15, 2025

View reviewed changes

llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp Outdated Show resolved Hide resolved

ReVe1uv force-pushed the riscv-g-atomic-add branch 2 times, most recently from fcdd168 to e12782c Compare August 19, 2025 08:01

ReVe1uv requested a review from topperc August 19, 2025 08:04

topperc reviewed Aug 19, 2025

View reviewed changes

llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp Outdated Show resolved Hide resolved

ReVe1uv force-pushed the riscv-g-atomic-add branch from e12782c to 89463ee Compare August 20, 2025 01:52

ReVe1uv requested a review from topperc August 20, 2025 01:58

ReVe1uv force-pushed the riscv-g-atomic-add branch from 89463ee to 775c60c Compare August 21, 2025 01:45

ReVe1uv force-pushed the riscv-g-atomic-add branch 2 times, most recently from dc2684d to 8f07ef3 Compare August 21, 2025 09:22

topperc reviewed Aug 21, 2025

View reviewed changes

llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp Outdated Show resolved Hide resolved

ReVe1uv force-pushed the riscv-g-atomic-add branch from bcee23f to 9bf77a3 Compare August 23, 2025 16:50

ReVe1uv requested a review from topperc August 23, 2025 16:55

topperc reviewed Aug 23, 2025

View reviewed changes

llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp Outdated Show resolved Hide resolved

ReVe1uv force-pushed the riscv-g-atomic-add branch from 9bf77a3 to 6fffd8d Compare August 23, 2025 17:10

ReVe1uv requested a review from topperc August 23, 2025 17:13

topperc approved these changes Aug 25, 2025

View reviewed changes

topperc reviewed Aug 26, 2025

View reviewed changes

topperc requested changes Aug 26, 2025

View reviewed changes

[RISCV][GlobalISel] Legalize and select G_ATOMICRMW_ADD instruction

fc3865c

This patch adds legalization and instruction selection support for the G_ATOMICRMW_ADD opcode in the RISCV GlobalISel backend.

ReVe1uv force-pushed the riscv-g-atomic-add branch from 23ba362 to fc3865c Compare August 26, 2025 03:08

ReVe1uv requested a review from topperc August 26, 2025 03:08

topperc approved these changes Aug 26, 2025

View reviewed changes

topperc merged commit 81740e0 into llvm:main Aug 26, 2025
8 of 9 checks passed

[RISCV][GlobalISel] Legalize and select G_ATOMICRMW_ADD instruction #153791

[RISCV][GlobalISel] Legalize and select G_ATOMICRMW_ADD instruction #153791

Uh oh!

Conversation

ReVe1uv commented Aug 15, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

llvmbot commented Aug 15, 2025

Uh oh!

llvmbot commented Aug 15, 2025

Uh oh!

ReVe1uv commented Aug 15, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

topperc Aug 15, 2025

Choose a reason for hiding this comment

Uh oh!

ReVe1uv Aug 19, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Choose a reason for hiding this comment

Uh oh!

topperc Aug 19, 2025

Choose a reason for hiding this comment

Uh oh!

ReVe1uv Aug 20, 2025

Choose a reason for hiding this comment

Uh oh!

Uh oh!

Uh oh!

Uh oh!

topperc commented Aug 20, 2025

Uh oh!

ReVe1uv commented Aug 21, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Uh oh!

Uh oh!

Uh oh!

topperc left a comment

Choose a reason for hiding this comment

Uh oh!

topperc commented Aug 25, 2025

Uh oh!

ReVe1uv commented Aug 26, 2025

Uh oh!

topperc Aug 26, 2025

Choose a reason for hiding this comment

Uh oh!

ReVe1uv Aug 26, 2025

Choose a reason for hiding this comment

Uh oh!

topperc left a comment

Choose a reason for hiding this comment

Uh oh!

topperc left a comment

Choose a reason for hiding this comment

Uh oh!

Uh oh!

Uh oh!

ReVe1uv commented Aug 15, 2025 •

edited

Loading

ReVe1uv commented Aug 15, 2025 •

edited

Loading

ReVe1uv Aug 19, 2025 •

edited

Loading

ReVe1uv commented Aug 21, 2025 •

edited

Loading