Skip to content

Commit 5af9cf0

Browse files
arsenmarsenm
committed
GlobalISel: Change representation of shuffle masks
Currently shufflemasks get emitted as any other constant, and you end up with a bunch of virtual registers of G_CONSTANT with a G_BUILD_VECTOR. The AArch64 selector then asserts on anything that doesn't fit this pattern. This isn't an ideal representation, and should avoid legalization and have fewer opportunities for a representational error. Rather than invent a new shuffle mask operand type, similar to what ShuffleVectorSDNode does, just track the original IR Constant mask operand. I don't completely like the idea of adding another link to the IR, but MIR is already quite dependent on IR constants already, and this will allow sharing the shuffle mask utility functions with the IR. llvm-svn: 368704
1 parent 8a033a9 commit 5af9cf0

25 files changed

+446
-150
lines changed

‎llvm/include/llvm/CodeGen/MachineInstrBuilder.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,11 @@ class MachineInstrBuilder {
250250
return *this;
251251
}
252252

253+
const MachineInstrBuilder &addShuffleMask(const Constant *Val) const {
254+
MI->addOperand(*MF, MachineOperand::CreateShuffleMask(Val));
255+
return *this;
256+
}
257+
253258
const MachineInstrBuilder &addSym(MCSymbol *Sym,
254259
unsigned char TargetFlags = 0) const {
255260
MI->addOperand(*MF, MachineOperand::CreateMCSymbol(Sym, TargetFlags));

‎llvm/include/llvm/CodeGen/MachineOperand.h

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
namespace llvm {
2424

2525
class BlockAddress;
26+
class Constant;
2627
class ConstantFP;
2728
class ConstantInt;
2829
class GlobalValue;
@@ -68,7 +69,8 @@ class MachineOperand {
6869
MO_CFIIndex, ///< MCCFIInstruction index.
6970
MO_IntrinsicID, ///< Intrinsic ID for ISel
7071
MO_Predicate, ///< Generic predicate for ISel
71-
MO_Last = MO_Predicate,
72+
MO_ShuffleMask, ///< Other IR Constant for ISel (shuffle masks)
73+
MO_Last = MO_ShuffleMask
7274
};
7375

7476
private:
@@ -172,6 +174,7 @@ class MachineOperand {
172174
unsigned CFIIndex; // For MO_CFI.
173175
Intrinsic::ID IntrinsicID; // For MO_IntrinsicID.
174176
unsigned Pred; // For MO_Predicate
177+
const Constant *ShuffleMask; // For MO_ShuffleMask
175178

176179
struct { // For MO_Register.
177180
// Register number is in SmallContents.RegNo.
@@ -341,6 +344,7 @@ class MachineOperand {
341344
bool isCFIIndex() const { return OpKind == MO_CFIIndex; }
342345
bool isIntrinsicID() const { return OpKind == MO_IntrinsicID; }
343346
bool isPredicate() const { return OpKind == MO_Predicate; }
347+
bool isShuffleMask() const { return OpKind == MO_ShuffleMask; }
344348
//===--------------------------------------------------------------------===//
345349
// Accessors for Register Operands
346350
//===--------------------------------------------------------------------===//
@@ -579,6 +583,11 @@ class MachineOperand {
579583
return Contents.Pred;
580584
}
581585

586+
const Constant *getShuffleMask() const {
587+
assert(isShuffleMask() && "Wrong MachineOperand accessor");
588+
return Contents.ShuffleMask;
589+
}
590+
582591
/// Return the offset from the symbol in this operand. This always returns 0
583592
/// for ExternalSymbol operands.
584593
int64_t getOffset() const {
@@ -902,6 +911,12 @@ class MachineOperand {
902911
return Op;
903912
}
904913

914+
static MachineOperand CreateShuffleMask(const Constant *C) {
915+
MachineOperand Op(MachineOperand::MO_ShuffleMask);
916+
Op.Contents.ShuffleMask = C;
917+
return Op;
918+
}
919+
905920
friend class MachineInstr;
906921
friend class MachineRegisterInfo;
907922

‎llvm/include/llvm/Target/GenericOpcodes.td

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -967,9 +967,12 @@ def G_EXTRACT_VECTOR_ELT : GenericInstruction {
967967
}
968968

969969
// Generic shufflevector.
970+
//
971+
// The mask operand should be an IR Constant which exactly matches the
972+
// corresponding mask for the IR shufflevector instruction.
970973
def G_SHUFFLE_VECTOR: GenericInstruction {
971974
let OutOperandList = (outs type0:$dst);
972-
let InOperandList = (ins type1:$v1, type1:$v2, type2:$mask);
975+
let InOperandList = (ins type1:$v1, type1:$v2, unknown:$mask);
973976
let hasSideEffects = 0;
974977
}
975978

‎llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1902,7 +1902,7 @@ bool IRTranslator::translateShuffleVector(const User &U,
19021902
.addDef(getOrCreateVReg(U))
19031903
.addUse(getOrCreateVReg(*U.getOperand(0)))
19041904
.addUse(getOrCreateVReg(*U.getOperand(1)))
1905-
.addUse(getOrCreateVReg(*U.getOperand(2)));
1905+
.addShuffleMask(cast<Constant>(U.getOperand(2)));
19061906
return true;
19071907
}
19081908

‎llvm/lib/CodeGen/MIRParser/MILexer.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,7 @@ static MIToken::TokenKind getIdentifierKind(StringRef Identifier) {
249249
.Case("successors", MIToken::kw_successors)
250250
.Case("floatpred", MIToken::kw_floatpred)
251251
.Case("intpred", MIToken::kw_intpred)
252+
.Case("shufflemask", MIToken::kw_shufflemask)
252253
.Case("pre-instr-symbol", MIToken::kw_pre_instr_symbol)
253254
.Case("post-instr-symbol", MIToken::kw_post_instr_symbol)
254255
.Case("unknown-size", MIToken::kw_unknown_size)

‎llvm/lib/CodeGen/MIRParser/MILexer.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ struct MIToken {
117117
kw_successors,
118118
kw_floatpred,
119119
kw_intpred,
120+
kw_shufflemask,
120121
kw_pre_instr_symbol,
121122
kw_post_instr_symbol,
122123
kw_unknown_size,
@@ -146,6 +147,7 @@ struct MIToken {
146147
IntegerLiteral,
147148
FloatingPointLiteral,
148149
HexLiteral,
150+
VectorLiteral,
149151
VirtualRegister,
150152
ConstantPoolItem,
151153
JumpTableIndex,

‎llvm/lib/CodeGen/MIRParser/MIParser.cpp

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -451,6 +451,7 @@ class MIParser {
451451
bool parseBlockAddressOperand(MachineOperand &Dest);
452452
bool parseIntrinsicOperand(MachineOperand &Dest);
453453
bool parsePredicateOperand(MachineOperand &Dest);
454+
bool parseShuffleMaskOperand(MachineOperand &Dest);
454455
bool parseTargetIndexOperand(MachineOperand &Dest);
455456
bool parseCustomRegisterMaskOperand(MachineOperand &Dest);
456457
bool parseLiveoutRegisterMaskOperand(MachineOperand &Dest);
@@ -2285,6 +2286,49 @@ bool MIParser::parsePredicateOperand(MachineOperand &Dest) {
22852286
return false;
22862287
}
22872288

2289+
bool MIParser::parseShuffleMaskOperand(MachineOperand &Dest) {
2290+
assert(Token.is(MIToken::kw_shufflemask));
2291+
2292+
lex();
2293+
if (expectAndConsume(MIToken::lparen))
2294+
return error("expected syntax shufflemask(<integer or undef>, ...)");
2295+
2296+
SmallVector<Constant *, 32> ShufMask;
2297+
LLVMContext &Ctx = MF.getFunction().getContext();
2298+
Type *I32Ty = Type::getInt32Ty(Ctx);
2299+
2300+
bool AllZero = true;
2301+
bool AllUndef = true;
2302+
2303+
do {
2304+
if (Token.is(MIToken::kw_undef)) {
2305+
ShufMask.push_back(UndefValue::get(I32Ty));
2306+
AllZero = false;
2307+
} else if (Token.is(MIToken::IntegerLiteral)) {
2308+
AllUndef = false;
2309+
const APSInt &Int = Token.integerValue();
2310+
if (!Int.isNullValue())
2311+
AllZero = false;
2312+
ShufMask.push_back(ConstantInt::get(I32Ty, Int.getExtValue()));
2313+
} else
2314+
return error("expected integer constant");
2315+
2316+
lex();
2317+
} while (consumeIfPresent(MIToken::comma));
2318+
2319+
if (expectAndConsume(MIToken::rparen))
2320+
return error("shufflemask should be terminated by ')'.");
2321+
2322+
if (AllZero || AllUndef) {
2323+
VectorType *VT = VectorType::get(I32Ty, ShufMask.size());
2324+
Constant *C = AllZero ? Constant::getNullValue(VT) : UndefValue::get(VT);
2325+
Dest = MachineOperand::CreateShuffleMask(C);
2326+
} else
2327+
Dest = MachineOperand::CreateShuffleMask(ConstantVector::get(ShufMask));
2328+
2329+
return false;
2330+
}
2331+
22882332
bool MIParser::parseTargetIndexOperand(MachineOperand &Dest) {
22892333
assert(Token.is(MIToken::kw_target_index));
22902334
lex();
@@ -2432,6 +2476,8 @@ bool MIParser::parseMachineOperand(MachineOperand &Dest,
24322476
case MIToken::kw_floatpred:
24332477
case MIToken::kw_intpred:
24342478
return parsePredicateOperand(Dest);
2479+
case MIToken::kw_shufflemask:
2480+
return parseShuffleMaskOperand(Dest);
24352481
case MIToken::Error:
24362482
return true;
24372483
case MIToken::Identifier:

‎llvm/lib/CodeGen/MIRPrinter.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -842,7 +842,8 @@ void MIPrinter::print(const MachineInstr &MI, unsigned OpIdx,
842842
case MachineOperand::MO_CFIIndex:
843843
case MachineOperand::MO_IntrinsicID:
844844
case MachineOperand::MO_Predicate:
845-
case MachineOperand::MO_BlockAddress: {
845+
case MachineOperand::MO_BlockAddress:
846+
case MachineOperand::MO_ShuffleMask: {
846847
unsigned TiedOperandIdx = 0;
847848
if (ShouldPrintRegisterTies && Op.isReg() && Op.isTied() && !Op.isDef())
848849
TiedOperandIdx = Op.getParent()->findTiedOperandIdx(OpIdx);

‎llvm/lib/CodeGen/MachineOperand.cpp

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -333,6 +333,8 @@ bool MachineOperand::isIdenticalTo(const MachineOperand &Other) const {
333333
return getIntrinsicID() == Other.getIntrinsicID();
334334
case MachineOperand::MO_Predicate:
335335
return getPredicate() == Other.getPredicate();
336+
case MachineOperand::MO_ShuffleMask:
337+
return getShuffleMask() == Other.getShuffleMask();
336338
}
337339
llvm_unreachable("Invalid machine operand type");
338340
}
@@ -381,6 +383,8 @@ hash_code llvm::hash_value(const MachineOperand &MO) {
381383
return hash_combine(MO.getType(), MO.getTargetFlags(), MO.getIntrinsicID());
382384
case MachineOperand::MO_Predicate:
383385
return hash_combine(MO.getType(), MO.getTargetFlags(), MO.getPredicate());
386+
case MachineOperand::MO_ShuffleMask:
387+
return hash_combine(MO.getType(), MO.getTargetFlags(), MO.getShuffleMask());
384388
}
385389
llvm_unreachable("Invalid machine operand type");
386390
}
@@ -936,6 +940,20 @@ void MachineOperand::print(raw_ostream &OS, ModuleSlotTracker &MST,
936940
<< CmpInst::getPredicateName(Pred) << ')';
937941
break;
938942
}
943+
case MachineOperand::MO_ShuffleMask:
944+
OS << "shufflemask(";
945+
const Constant* C = getShuffleMask();
946+
const int NumElts = C->getType()->getVectorNumElements();
947+
948+
StringRef Separator;
949+
for (int I = 0; I != NumElts; ++I) {
950+
OS << Separator;
951+
C->getAggregateElement(I)->printAsOperand(OS, false, MST);
952+
Separator = ", ";
953+
}
954+
955+
OS << ')';
956+
break;
939957
}
940958
}
941959

‎llvm/lib/CodeGen/MachineVerifier.cpp

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1387,6 +1387,22 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
13871387
report("G_SEXT_INREG size must be less than source bit width", MI);
13881388
break;
13891389
}
1390+
case TargetOpcode::G_SHUFFLE_VECTOR: {
1391+
const MachineOperand &MaskOp = MI->getOperand(3);
1392+
if (!MaskOp.isShuffleMask()) {
1393+
report("Incorrect mask operand type for G_SHUFFLE_VECTOR", MI);
1394+
break;
1395+
}
1396+
1397+
const Constant *Mask = MaskOp.getShuffleMask();
1398+
if (!Mask->getAggregateElement(0u)) {
1399+
report("Invalid shufflemask constant type", MI);
1400+
break;
1401+
}
1402+
1403+
// TODO: Verify element numbers consistent
1404+
break;
1405+
}
13901406
default:
13911407
break;
13921408
}

‎llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp

Lines changed: 7 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -115,8 +115,6 @@ class AArch64InstructionSelector : public InstructionSelector {
115115
bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI) const;
116116
bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI) const;
117117

118-
void collectShuffleMaskIndices(MachineInstr &I, MachineRegisterInfo &MRI,
119-
SmallVectorImpl<Optional<int>> &Idxs) const;
120118
bool selectShuffleVector(MachineInstr &I, MachineRegisterInfo &MRI) const;
121119
bool selectExtractElt(MachineInstr &I, MachineRegisterInfo &MRI) const;
122120
bool selectConcatVectors(MachineInstr &I, MachineRegisterInfo &MRI) const;
@@ -3065,29 +3063,6 @@ bool AArch64InstructionSelector::selectConcatVectors(
30653063
return true;
30663064
}
30673065

3068-
void AArch64InstructionSelector::collectShuffleMaskIndices(
3069-
MachineInstr &I, MachineRegisterInfo &MRI,
3070-
SmallVectorImpl<Optional<int>> &Idxs) const {
3071-
MachineInstr *MaskDef = MRI.getVRegDef(I.getOperand(3).getReg());
3072-
assert(
3073-
MaskDef->getOpcode() == TargetOpcode::G_BUILD_VECTOR &&
3074-
"G_SHUFFLE_VECTOR should have a constant mask operand as G_BUILD_VECTOR");
3075-
// Find the constant indices.
3076-
for (unsigned i = 1, e = MaskDef->getNumOperands(); i < e; ++i) {
3077-
// Look through copies.
3078-
MachineInstr *ScalarDef =
3079-
getDefIgnoringCopies(MaskDef->getOperand(i).getReg(), MRI);
3080-
assert(ScalarDef && "Could not find vreg def of shufflevec index op");
3081-
if (ScalarDef->getOpcode() != TargetOpcode::G_CONSTANT) {
3082-
// This be an undef if not a constant.
3083-
assert(ScalarDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
3084-
Idxs.push_back(None);
3085-
} else {
3086-
Idxs.push_back(ScalarDef->getOperand(1).getCImm()->getSExtValue());
3087-
}
3088-
}
3089-
}
3090-
30913066
unsigned
30923067
AArch64InstructionSelector::emitConstantPoolEntry(Constant *CPVal,
30933068
MachineFunction &MF) const {
@@ -3630,16 +3605,9 @@ bool AArch64InstructionSelector::tryOptVectorDup(MachineInstr &I) const {
36303605
return false;
36313606

36323607
// The shuffle's second operand doesn't matter if the mask is all zero.
3633-
auto *ZeroVec = getOpcodeDef(G_BUILD_VECTOR, I.getOperand(3).getReg(), MRI);
3634-
if (!ZeroVec)
3608+
const Constant *Mask = I.getOperand(3).getShuffleMask();
3609+
if (!isa<ConstantAggregateZero>(Mask))
36353610
return false;
3636-
int64_t Zero = 0;
3637-
if (!mi_match(ZeroVec->getOperand(1).getReg(), MRI, m_ICst(Zero)) || Zero)
3638-
return false;
3639-
for (unsigned i = 1, e = ZeroVec->getNumOperands(); i < e; ++i) {
3640-
if (ZeroVec->getOperand(i).getReg() != ZeroVec->getOperand(1).getReg())
3641-
return false; // This wasn't an all zeros vector.
3642-
}
36433611

36443612
// We're done, now find out what kind of splat we need.
36453613
LLT VecTy = MRI.getType(I.getOperand(0).getReg());
@@ -3687,19 +3655,14 @@ bool AArch64InstructionSelector::selectShuffleVector(
36873655
const LLT Src1Ty = MRI.getType(Src1Reg);
36883656
Register Src2Reg = I.getOperand(2).getReg();
36893657
const LLT Src2Ty = MRI.getType(Src2Reg);
3658+
const Constant *ShuffleMask = I.getOperand(3).getShuffleMask();
36903659

36913660
MachineBasicBlock &MBB = *I.getParent();
36923661
MachineFunction &MF = *MBB.getParent();
36933662
LLVMContext &Ctx = MF.getFunction().getContext();
36943663

3695-
// G_SHUFFLE_VECTOR doesn't really have a strictly enforced constant mask
3696-
// operand, it comes in as a normal vector value which we have to analyze to
3697-
// find the mask indices. If the mask element is undef, then
3698-
// collectShuffleMaskIndices() will add a None entry for that index into
3699-
// the list.
3700-
SmallVector<Optional<int>, 8> Mask;
3701-
collectShuffleMaskIndices(I, MRI, Mask);
3702-
assert(!Mask.empty() && "Expected to find mask indices");
3664+
SmallVector<int, 8> Mask;
3665+
ShuffleVectorInst::getShuffleMask(ShuffleMask, Mask);
37033666

37043667
// G_SHUFFLE_VECTOR is weird in that the source operands can be scalars, if
37053668
// it's originated from a <1 x T> type. Those should have been lowered into
@@ -3712,10 +3675,10 @@ bool AArch64InstructionSelector::selectShuffleVector(
37123675
unsigned BytesPerElt = DstTy.getElementType().getSizeInBits() / 8;
37133676

37143677
SmallVector<Constant *, 64> CstIdxs;
3715-
for (auto &MaybeVal : Mask) {
3678+
for (int Val : Mask) {
37163679
// For now, any undef indexes we'll just assume to be 0. This should be
37173680
// optimized in future, e.g. to select DUP etc.
3718-
int Val = MaybeVal.hasValue() ? *MaybeVal : 0;
3681+
Val = Val < 0 ? 0 : Val;
37193682
for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte) {
37203683
unsigned Offset = Byte + Val * BytesPerElt;
37213684
CstIdxs.emplace_back(ConstantInt::get(Type::getInt8Ty(Ctx), Offset));

‎llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -789,6 +789,7 @@ static bool IsAnAddressOperand(const MachineOperand &MO) {
789789
case MachineOperand::MO_Immediate:
790790
case MachineOperand::MO_CImmediate:
791791
case MachineOperand::MO_FPImmediate:
792+
case MachineOperand::MO_ShuffleMask:
792793
return false;
793794
case MachineOperand::MO_MachineBasicBlock:
794795
return true;

0 commit comments

Comments
 (0)