diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index 936625606e315..179dd86e17d96 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -5143,6 +5143,113 @@ SDValue ARMTargetLowering::LowerUnsignedALUO(SDValue Op, return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow); } +// Lower various (select (icmp CmpVal, 0), LHS, RHS) custom patterns. +static SDValue LowerSELECTWithCmpZero(SDValue CmpVal, SDValue LHS, SDValue RHS, + ISD::CondCode CC, const SDLoc &DL, + SelectionDAG &DAG, + const ARMSubtarget &Subtarget) { + EVT CmpVT = CmpVal.getValueType(); + EVT VT = LHS.getValueType(); + if (!CmpVT.isScalarInteger() || !VT.isScalarInteger()) + return SDValue(); + + if (CC == ISD::SETEQ && CmpVal.getOpcode() == ISD::AND && + isOneConstant(CmpVal.getOperand(1))) { + auto SplatLSB = [&](EVT SplatVT) { + // we need mask of all zeros or ones with same size of the other + // operands. + SDValue Neg = CmpVal; + if (CmpVT.bitsGT(SplatVT)) + Neg = DAG.getNode(ISD::TRUNCATE, DL, SplatVT, CmpVal); + else if (CmpVT.bitsLT(SplatVT)) + Neg = DAG.getNode( + ISD::AND, DL, SplatVT, + DAG.getNode(ISD::ANY_EXTEND, DL, SplatVT, CmpVal.getOperand(0)), + DAG.getConstant(1, DL, SplatVT)); + return DAG.getNegative(Neg, DL, SplatVT); // -(and (x, 0x1)) + }; + + // SELECT (AND(X,1) == 0), 0, -1 -> NEG(AND(X,1)) + if (isNullConstant(LHS) && isAllOnesConstant(RHS)) + return SplatLSB(VT); + + // SELECT (AND(X,1) == 0), C1, C2 -> XOR(C1,AND(NEG(AND(X,1)),XOR(C1,C2)) + if (Subtarget.isThumb1Only() && isa(LHS) && + isa(RHS)) { + SDValue Mask = SplatLSB(VT); + SDValue Diff = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS); + SDValue Flip = DAG.getNode(ISD::AND, DL, VT, Mask, Diff); + return DAG.getNode(ISD::XOR, DL, VT, LHS, Flip); + } + + SDValue Src1, Src2; + auto isIdentityPatternZero = [&]() { + switch (RHS.getOpcode()) { + default: + break; + case ISD::OR: + case ISD::XOR: + case ISD::ADD: + if (RHS.getOperand(0) == LHS || RHS.getOperand(1) == LHS) { + Src1 = RHS.getOperand(RHS.getOperand(0) == LHS ? 1 : 0); + Src2 = LHS; + return true; + } + break; + case ISD::SHL: + case ISD::SRA: + case ISD::SRL: + case ISD::SUB: + if (RHS.getOperand(0) == LHS) { + Src1 = RHS.getOperand(1); + Src2 = LHS; + return true; + } + break; + } + return false; + }; + + auto isIdentityPatternOnes = [&]() { + switch (LHS.getOpcode()) { + default: + break; + case ISD::AND: + if (LHS.getOperand(0) == RHS || LHS.getOperand(1) == RHS) { + Src1 = LHS.getOperand(LHS.getOperand(0) == RHS ? 1 : 0); + Src2 = RHS; + return true; + } + break; + } + return false; + }; + + // Convert 'identity' patterns (iff X is 0 or 1): + // SELECT (AND(X,1) == 0), Y, (OR Y, Z) -> (OR Y, (AND NEG(AND(X,1)), Z)) + // SELECT (AND(X,1) == 0), Y, (XOR Y, Z) -> (XOR Y, (AND NEG(AND(X,1)), Z)) + // SELECT (AND(X,1) == 0), Y, (ADD Y, Z) -> (ADD Y, (AND NEG(AND(X,1)), Z)) + // SELECT (AND(X,1) == 0), Y, (SUB Y, Z) -> (SUB Y, (AND NEG(AND(X,1)), Z)) + // SELECT (AND(X,1) == 0), Y, (SHL Y, Z) -> (SHL Y, (AND NEG(AND(X,1)), Z)) + // SELECT (AND(X,1) == 0), Y, (SRA Y, Z) -> (SRA Y, (AND NEG(AND(X,1)), Z)) + // SELECT (AND(X,1) == 0), Y, (SRL Y, Z) -> (SRL Y, (AND NEG(AND(X,1)), Z)) + if (Subtarget.isThumb1Only() && isIdentityPatternZero()) { + SDValue Mask = SplatLSB(Src1.getValueType()); + SDValue And = DAG.getNode(ISD::AND, DL, Src1.getValueType(), Mask, + Src1); // Mask & z + return DAG.getNode(RHS.getOpcode(), DL, VT, Src2, And); // y Op And + } + // SELECT (AND(X,1) == 0), (AND Y, Z), Y -> (AND Y, (OR NEG(AND(X, 1)), Z)) + if (Subtarget.isThumb1Only() && isIdentityPatternOnes()) { + SDValue Mask = SplatLSB(VT); + SDValue Or = DAG.getNode(ISD::OR, DL, VT, Mask, Src1); // Mask | z + return DAG.getNode(LHS.getOpcode(), DL, VT, Src2, Or); // y Op Or + } + } + + return SDValue(); +} + static SDValue LowerADDSUBSAT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget) { EVT VT = Op.getValueType(); @@ -5521,7 +5628,6 @@ SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { ConstantSDNode *CTVal = dyn_cast(TrueVal); ConstantSDNode *RHSC = dyn_cast(RHS); if (Op.getValueType().isInteger()) { - // Check for SMAX(lhs, 0) and SMIN(lhs, 0) patterns. // (SELECT_CC setgt, lhs, 0, lhs, 0) -> (BIC lhs, (SRA lhs, typesize-1)) // (SELECT_CC setlt, lhs, 0, lhs, 0) -> (AND lhs, (SRA lhs, typesize-1)) @@ -5539,6 +5645,11 @@ SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { return DAG.getNode(ISD::AND, dl, VT, LHS, Shift); } + if (RHSC && RHSC->isZero()) { + if (SDValue R = LowerSELECTWithCmpZero(LHS, TrueVal, FalseVal, CC, dl, + DAG, *Subtarget)) + return R; + } } if (Subtarget->hasV8_1MMainlineOps() && CFVal && CTVal && diff --git a/llvm/test/CodeGen/ARM/arm-and-tst-peephole.ll b/llvm/test/CodeGen/ARM/arm-and-tst-peephole.ll index 0795525fba1b3..74b9328829ff9 100644 --- a/llvm/test/CodeGen/ARM/arm-and-tst-peephole.ll +++ b/llvm/test/CodeGen/ARM/arm-and-tst-peephole.ll @@ -334,14 +334,10 @@ define i32 @test_tst_assessment(i32 %a, i32 %b) { ; ; THUMB-LABEL: test_tst_assessment: ; THUMB: @ %bb.0: -; THUMB-NEXT: movs r2, r0 -; THUMB-NEXT: movs r0, #1 +; THUMB-NEXT: movs r2, #1 +; THUMB-NEXT: ands r1, r2 ; THUMB-NEXT: ands r0, r2 -; THUMB-NEXT: lsls r1, r1, #31 -; THUMB-NEXT: beq .LBB2_2 -; THUMB-NEXT: @ %bb.1: -; THUMB-NEXT: subs r0, r0, #1 -; THUMB-NEXT: .LBB2_2: +; THUMB-NEXT: subs r0, r0, r1 ; THUMB-NEXT: bx lr ; ; T2-LABEL: test_tst_assessment: diff --git a/llvm/test/CodeGen/ARM/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll b/llvm/test/CodeGen/ARM/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll index 7cc623fb0a616..0717c19294a14 100644 --- a/llvm/test/CodeGen/ARM/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll +++ b/llvm/test/CodeGen/ARM/hoist-and-by-const-from-lshr-in-eqcmp-zero.ll @@ -64,9 +64,8 @@ define i1 @scalar_i8_lowestbit_eq(i8 %x, i8 %y) nounwind { ; THUMB6-NEXT: uxtb r1, r1 ; THUMB6-NEXT: lsls r0, r1 ; THUMB6-NEXT: movs r1, #1 -; THUMB6-NEXT: ands r0, r1 -; THUMB6-NEXT: rsbs r1, r0, #0 -; THUMB6-NEXT: adcs r0, r1 +; THUMB6-NEXT: bics r1, r0 +; THUMB6-NEXT: mov r0, r1 ; THUMB6-NEXT: bx lr ; ; THUMB78-LABEL: scalar_i8_lowestbit_eq: @@ -166,9 +165,8 @@ define i1 @scalar_i16_lowestbit_eq(i16 %x, i16 %y) nounwind { ; THUMB6-NEXT: uxth r1, r1 ; THUMB6-NEXT: lsls r0, r1 ; THUMB6-NEXT: movs r1, #1 -; THUMB6-NEXT: ands r0, r1 -; THUMB6-NEXT: rsbs r1, r0, #0 -; THUMB6-NEXT: adcs r0, r1 +; THUMB6-NEXT: bics r1, r0 +; THUMB6-NEXT: mov r0, r1 ; THUMB6-NEXT: bx lr ; ; THUMB78-LABEL: scalar_i16_lowestbit_eq: @@ -261,9 +259,8 @@ define i1 @scalar_i32_lowestbit_eq(i32 %x, i32 %y) nounwind { ; THUMB6: @ %bb.0: ; THUMB6-NEXT: lsls r0, r1 ; THUMB6-NEXT: movs r1, #1 -; THUMB6-NEXT: ands r0, r1 -; THUMB6-NEXT: rsbs r1, r0, #0 -; THUMB6-NEXT: adcs r0, r1 +; THUMB6-NEXT: bics r1, r0 +; THUMB6-NEXT: mov r0, r1 ; THUMB6-NEXT: bx lr ; ; THUMB78-LABEL: scalar_i32_lowestbit_eq: @@ -392,9 +389,8 @@ define i1 @scalar_i64_lowestbit_eq(i64 %x, i64 %y) nounwind { ; THUMB6-NEXT: push {r7, lr} ; THUMB6-NEXT: bl __ashldi3 ; THUMB6-NEXT: movs r1, #1 -; THUMB6-NEXT: ands r0, r1 -; THUMB6-NEXT: rsbs r1, r0, #0 -; THUMB6-NEXT: adcs r0, r1 +; THUMB6-NEXT: bics r1, r0 +; THUMB6-NEXT: mov r0, r1 ; THUMB6-NEXT: pop {r7, pc} ; ; THUMB78-LABEL: scalar_i64_lowestbit_eq: @@ -514,29 +510,28 @@ define <4 x i1> @vec_4xi32_splat_eq(<4 x i32> %x, <4 x i32> %y) nounwind { ; ; THUMB6-LABEL: vec_4xi32_splat_eq: ; THUMB6: @ %bb.0: -; THUMB6-NEXT: push {r4, r5, r7, lr} -; THUMB6-NEXT: ldr r4, [sp, #16] -; THUMB6-NEXT: lsls r0, r4 +; THUMB6-NEXT: push {r4, r5, r6, lr} +; THUMB6-NEXT: mov r5, r0 +; THUMB6-NEXT: ldr r0, [sp, #16] +; THUMB6-NEXT: lsls r5, r0 ; THUMB6-NEXT: movs r4, #1 -; THUMB6-NEXT: ands r0, r4 -; THUMB6-NEXT: rsbs r5, r0, #0 -; THUMB6-NEXT: adcs r0, r5 +; THUMB6-NEXT: mov r0, r4 +; THUMB6-NEXT: bics r0, r5 ; THUMB6-NEXT: ldr r5, [sp, #20] ; THUMB6-NEXT: lsls r1, r5 -; THUMB6-NEXT: ands r1, r4 -; THUMB6-NEXT: rsbs r5, r1, #0 -; THUMB6-NEXT: adcs r1, r5 -; THUMB6-NEXT: ldr r5, [sp, #24] -; THUMB6-NEXT: lsls r2, r5 -; THUMB6-NEXT: ands r2, r4 -; THUMB6-NEXT: rsbs r5, r2, #0 -; THUMB6-NEXT: adcs r2, r5 -; THUMB6-NEXT: ldr r5, [sp, #28] -; THUMB6-NEXT: lsls r3, r5 -; THUMB6-NEXT: ands r3, r4 -; THUMB6-NEXT: rsbs r4, r3, #0 -; THUMB6-NEXT: adcs r3, r4 -; THUMB6-NEXT: pop {r4, r5, r7, pc} +; THUMB6-NEXT: mov r5, r4 +; THUMB6-NEXT: bics r5, r1 +; THUMB6-NEXT: ldr r1, [sp, #24] +; THUMB6-NEXT: lsls r2, r1 +; THUMB6-NEXT: mov r6, r4 +; THUMB6-NEXT: bics r6, r2 +; THUMB6-NEXT: ldr r1, [sp, #28] +; THUMB6-NEXT: lsls r3, r1 +; THUMB6-NEXT: bics r4, r3 +; THUMB6-NEXT: mov r1, r5 +; THUMB6-NEXT: mov r2, r6 +; THUMB6-NEXT: mov r3, r4 +; THUMB6-NEXT: pop {r4, r5, r6, pc} ; ; THUMB78-LABEL: vec_4xi32_splat_eq: ; THUMB78: @ %bb.0: @@ -601,24 +596,24 @@ define <4 x i1> @vec_4xi32_nonsplat_eq(<4 x i32> %x, <4 x i32> %y) nounwind { ; THUMB6-LABEL: vec_4xi32_nonsplat_eq: ; THUMB6: @ %bb.0: ; THUMB6-NEXT: push {r4, lr} -; THUMB6-NEXT: ldr r0, [sp, #12] -; THUMB6-NEXT: lsls r1, r0 +; THUMB6-NEXT: ldr r0, [sp, #16] +; THUMB6-NEXT: lsls r2, r0 +; THUMB6-NEXT: ldr r0, .LCPI13_0 +; THUMB6-NEXT: ands r2, r0 +; THUMB6-NEXT: rsbs r0, r2, #0 +; THUMB6-NEXT: adcs r2, r0 +; THUMB6-NEXT: ldr r0, [sp, #20] +; THUMB6-NEXT: lsls r3, r0 ; THUMB6-NEXT: movs r0, #1 -; THUMB6-NEXT: ands r1, r0 -; THUMB6-NEXT: rsbs r4, r1, #0 -; THUMB6-NEXT: adcs r1, r4 -; THUMB6-NEXT: ldr r4, [sp, #16] -; THUMB6-NEXT: lsls r2, r4 -; THUMB6-NEXT: ldr r4, .LCPI13_0 -; THUMB6-NEXT: ands r2, r4 -; THUMB6-NEXT: rsbs r4, r2, #0 -; THUMB6-NEXT: adcs r2, r4 -; THUMB6-NEXT: ldr r4, [sp, #20] -; THUMB6-NEXT: lsls r3, r4 ; THUMB6-NEXT: lsls r4, r0, #31 ; THUMB6-NEXT: ands r3, r4 ; THUMB6-NEXT: rsbs r4, r3, #0 ; THUMB6-NEXT: adcs r3, r4 +; THUMB6-NEXT: ldr r4, [sp, #12] +; THUMB6-NEXT: lsls r1, r4 +; THUMB6-NEXT: mov r4, r0 +; THUMB6-NEXT: bics r4, r1 +; THUMB6-NEXT: mov r1, r4 ; THUMB6-NEXT: pop {r4, pc} ; THUMB6-NEXT: .p2align 2 ; THUMB6-NEXT: @ %bb.1: @@ -683,24 +678,24 @@ define <4 x i1> @vec_4xi32_nonsplat_undef0_eq(<4 x i32> %x, <4 x i32> %y) nounwi ; ; THUMB6-LABEL: vec_4xi32_nonsplat_undef0_eq: ; THUMB6: @ %bb.0: -; THUMB6-NEXT: push {r4, lr} -; THUMB6-NEXT: ldr r2, [sp, #8] +; THUMB6-NEXT: push {r4, r5, r6, lr} +; THUMB6-NEXT: ldr r2, [sp, #16] ; THUMB6-NEXT: lsls r0, r2 ; THUMB6-NEXT: movs r2, #1 -; THUMB6-NEXT: ands r0, r2 -; THUMB6-NEXT: rsbs r4, r0, #0 -; THUMB6-NEXT: adcs r0, r4 -; THUMB6-NEXT: ldr r4, [sp, #12] -; THUMB6-NEXT: lsls r1, r4 -; THUMB6-NEXT: ands r1, r2 -; THUMB6-NEXT: rsbs r4, r1, #0 -; THUMB6-NEXT: adcs r1, r4 -; THUMB6-NEXT: ldr r4, [sp, #20] -; THUMB6-NEXT: lsls r3, r4 -; THUMB6-NEXT: ands r3, r2 -; THUMB6-NEXT: rsbs r4, r3, #0 -; THUMB6-NEXT: adcs r3, r4 -; THUMB6-NEXT: pop {r4, pc} +; THUMB6-NEXT: mov r4, r2 +; THUMB6-NEXT: bics r4, r0 +; THUMB6-NEXT: ldr r0, [sp, #20] +; THUMB6-NEXT: lsls r1, r0 +; THUMB6-NEXT: mov r5, r2 +; THUMB6-NEXT: bics r5, r1 +; THUMB6-NEXT: ldr r0, [sp, #28] +; THUMB6-NEXT: lsls r3, r0 +; THUMB6-NEXT: mov r6, r2 +; THUMB6-NEXT: bics r6, r3 +; THUMB6-NEXT: mov r0, r4 +; THUMB6-NEXT: mov r1, r5 +; THUMB6-NEXT: mov r3, r6 +; THUMB6-NEXT: pop {r4, r5, r6, pc} ; ; THUMB78-LABEL: vec_4xi32_nonsplat_undef0_eq: ; THUMB78: @ %bb.0: @@ -750,24 +745,23 @@ define <4 x i1> @vec_4xi32_nonsplat_undef1_eq(<4 x i32> %x, <4 x i32> %y) nounwi ; ; THUMB6-LABEL: vec_4xi32_nonsplat_undef1_eq: ; THUMB6: @ %bb.0: -; THUMB6-NEXT: push {r4, lr} -; THUMB6-NEXT: ldr r2, [sp, #8] +; THUMB6-NEXT: push {r4, r5, r7, lr} +; THUMB6-NEXT: ldr r2, [sp, #16] ; THUMB6-NEXT: lsls r0, r2 ; THUMB6-NEXT: movs r2, #1 -; THUMB6-NEXT: ands r0, r2 -; THUMB6-NEXT: rsbs r4, r0, #0 -; THUMB6-NEXT: adcs r0, r4 -; THUMB6-NEXT: ldr r4, [sp, #12] -; THUMB6-NEXT: lsls r1, r4 -; THUMB6-NEXT: ands r1, r2 -; THUMB6-NEXT: rsbs r4, r1, #0 -; THUMB6-NEXT: adcs r1, r4 -; THUMB6-NEXT: ldr r4, [sp, #20] -; THUMB6-NEXT: lsls r3, r4 -; THUMB6-NEXT: ands r3, r2 -; THUMB6-NEXT: rsbs r2, r3, #0 -; THUMB6-NEXT: adcs r3, r2 -; THUMB6-NEXT: pop {r4, pc} +; THUMB6-NEXT: mov r4, r2 +; THUMB6-NEXT: bics r4, r0 +; THUMB6-NEXT: ldr r0, [sp, #20] +; THUMB6-NEXT: lsls r1, r0 +; THUMB6-NEXT: mov r5, r2 +; THUMB6-NEXT: bics r5, r1 +; THUMB6-NEXT: ldr r0, [sp, #28] +; THUMB6-NEXT: lsls r3, r0 +; THUMB6-NEXT: bics r2, r3 +; THUMB6-NEXT: mov r0, r4 +; THUMB6-NEXT: mov r1, r5 +; THUMB6-NEXT: mov r3, r2 +; THUMB6-NEXT: pop {r4, r5, r7, pc} ; ; THUMB78-LABEL: vec_4xi32_nonsplat_undef1_eq: ; THUMB78: @ %bb.0: @@ -818,24 +812,23 @@ define <4 x i1> @vec_4xi32_nonsplat_undef2_eq(<4 x i32> %x, <4 x i32> %y) nounwi ; ; THUMB6-LABEL: vec_4xi32_nonsplat_undef2_eq: ; THUMB6: @ %bb.0: -; THUMB6-NEXT: push {r4, lr} -; THUMB6-NEXT: ldr r2, [sp, #8] +; THUMB6-NEXT: push {r4, r5, r7, lr} +; THUMB6-NEXT: ldr r2, [sp, #16] ; THUMB6-NEXT: lsls r0, r2 ; THUMB6-NEXT: movs r2, #1 -; THUMB6-NEXT: ands r0, r2 -; THUMB6-NEXT: rsbs r4, r0, #0 -; THUMB6-NEXT: adcs r0, r4 -; THUMB6-NEXT: ldr r4, [sp, #12] -; THUMB6-NEXT: lsls r1, r4 -; THUMB6-NEXT: ands r1, r2 -; THUMB6-NEXT: rsbs r4, r1, #0 -; THUMB6-NEXT: adcs r1, r4 -; THUMB6-NEXT: ldr r4, [sp, #20] -; THUMB6-NEXT: lsls r3, r4 -; THUMB6-NEXT: ands r3, r2 -; THUMB6-NEXT: rsbs r2, r3, #0 -; THUMB6-NEXT: adcs r3, r2 -; THUMB6-NEXT: pop {r4, pc} +; THUMB6-NEXT: mov r4, r2 +; THUMB6-NEXT: bics r4, r0 +; THUMB6-NEXT: ldr r0, [sp, #20] +; THUMB6-NEXT: lsls r1, r0 +; THUMB6-NEXT: mov r5, r2 +; THUMB6-NEXT: bics r5, r1 +; THUMB6-NEXT: ldr r0, [sp, #28] +; THUMB6-NEXT: lsls r3, r0 +; THUMB6-NEXT: bics r2, r3 +; THUMB6-NEXT: mov r0, r4 +; THUMB6-NEXT: mov r1, r5 +; THUMB6-NEXT: mov r3, r2 +; THUMB6-NEXT: pop {r4, r5, r7, pc} ; ; THUMB78-LABEL: vec_4xi32_nonsplat_undef2_eq: ; THUMB78: @ %bb.0: @@ -911,10 +904,8 @@ define i1 @scalar_i32_x_is_const_eq(i32 %y) nounwind { ; THUMB6: @ %bb.0: ; THUMB6-NEXT: ldr r1, .LCPI18_0 ; THUMB6-NEXT: lsrs r1, r0 -; THUMB6-NEXT: movs r2, #1 -; THUMB6-NEXT: ands r2, r1 -; THUMB6-NEXT: rsbs r0, r2, #0 -; THUMB6-NEXT: adcs r0, r2 +; THUMB6-NEXT: movs r0, #1 +; THUMB6-NEXT: bics r0, r1 ; THUMB6-NEXT: bx lr ; THUMB6-NEXT: .p2align 2 ; THUMB6-NEXT: @ %bb.1: diff --git a/llvm/test/CodeGen/ARM/hoist-and-by-const-from-shl-in-eqcmp-zero.ll b/llvm/test/CodeGen/ARM/hoist-and-by-const-from-shl-in-eqcmp-zero.ll index a8421ae9a6a89..27d46717aa996 100644 --- a/llvm/test/CodeGen/ARM/hoist-and-by-const-from-shl-in-eqcmp-zero.ll +++ b/llvm/test/CodeGen/ARM/hoist-and-by-const-from-shl-in-eqcmp-zero.ll @@ -64,12 +64,10 @@ define i1 @scalar_i8_lowestbit_eq(i8 %x, i8 %y) nounwind { ; THUMB6-LABEL: scalar_i8_lowestbit_eq: ; THUMB6: @ %bb.0: ; THUMB6-NEXT: uxtb r1, r1 -; THUMB6-NEXT: uxtb r0, r0 -; THUMB6-NEXT: lsrs r0, r1 -; THUMB6-NEXT: movs r1, #1 -; THUMB6-NEXT: ands r1, r0 -; THUMB6-NEXT: rsbs r0, r1, #0 -; THUMB6-NEXT: adcs r0, r1 +; THUMB6-NEXT: uxtb r2, r0 +; THUMB6-NEXT: lsrs r2, r1 +; THUMB6-NEXT: movs r0, #1 +; THUMB6-NEXT: bics r0, r2 ; THUMB6-NEXT: bx lr ; ; THUMB78-LABEL: scalar_i8_lowestbit_eq: @@ -173,12 +171,10 @@ define i1 @scalar_i16_lowestbit_eq(i16 %x, i16 %y) nounwind { ; THUMB6-LABEL: scalar_i16_lowestbit_eq: ; THUMB6: @ %bb.0: ; THUMB6-NEXT: uxth r1, r1 -; THUMB6-NEXT: uxth r0, r0 -; THUMB6-NEXT: lsrs r0, r1 -; THUMB6-NEXT: movs r1, #1 -; THUMB6-NEXT: ands r1, r0 -; THUMB6-NEXT: rsbs r0, r1, #0 -; THUMB6-NEXT: adcs r0, r1 +; THUMB6-NEXT: uxth r2, r0 +; THUMB6-NEXT: lsrs r2, r1 +; THUMB6-NEXT: movs r0, #1 +; THUMB6-NEXT: bics r0, r2 ; THUMB6-NEXT: bx lr ; ; THUMB78-LABEL: scalar_i16_lowestbit_eq: @@ -275,9 +271,8 @@ define i1 @scalar_i32_lowestbit_eq(i32 %x, i32 %y) nounwind { ; THUMB6: @ %bb.0: ; THUMB6-NEXT: lsrs r0, r1 ; THUMB6-NEXT: movs r1, #1 -; THUMB6-NEXT: ands r0, r1 -; THUMB6-NEXT: rsbs r1, r0, #0 -; THUMB6-NEXT: adcs r0, r1 +; THUMB6-NEXT: bics r1, r0 +; THUMB6-NEXT: mov r0, r1 ; THUMB6-NEXT: bx lr ; ; THUMB78-LABEL: scalar_i32_lowestbit_eq: @@ -403,9 +398,8 @@ define i1 @scalar_i64_lowestbit_eq(i64 %x, i64 %y) nounwind { ; THUMB6-NEXT: push {r7, lr} ; THUMB6-NEXT: bl __lshrdi3 ; THUMB6-NEXT: movs r1, #1 -; THUMB6-NEXT: ands r0, r1 -; THUMB6-NEXT: rsbs r1, r0, #0 -; THUMB6-NEXT: adcs r0, r1 +; THUMB6-NEXT: bics r1, r0 +; THUMB6-NEXT: mov r0, r1 ; THUMB6-NEXT: pop {r7, pc} ; ; THUMB78-LABEL: scalar_i64_lowestbit_eq: @@ -529,29 +523,28 @@ define <4 x i1> @vec_4xi32_splat_eq(<4 x i32> %x, <4 x i32> %y) nounwind { ; ; THUMB6-LABEL: vec_4xi32_splat_eq: ; THUMB6: @ %bb.0: -; THUMB6-NEXT: push {r4, r5, r7, lr} -; THUMB6-NEXT: ldr r4, [sp, #16] -; THUMB6-NEXT: lsrs r0, r4 +; THUMB6-NEXT: push {r4, r5, r6, lr} +; THUMB6-NEXT: mov r5, r0 +; THUMB6-NEXT: ldr r0, [sp, #16] +; THUMB6-NEXT: lsrs r5, r0 ; THUMB6-NEXT: movs r4, #1 -; THUMB6-NEXT: ands r0, r4 -; THUMB6-NEXT: rsbs r5, r0, #0 -; THUMB6-NEXT: adcs r0, r5 +; THUMB6-NEXT: mov r0, r4 +; THUMB6-NEXT: bics r0, r5 ; THUMB6-NEXT: ldr r5, [sp, #20] ; THUMB6-NEXT: lsrs r1, r5 -; THUMB6-NEXT: ands r1, r4 -; THUMB6-NEXT: rsbs r5, r1, #0 -; THUMB6-NEXT: adcs r1, r5 -; THUMB6-NEXT: ldr r5, [sp, #24] -; THUMB6-NEXT: lsrs r2, r5 -; THUMB6-NEXT: ands r2, r4 -; THUMB6-NEXT: rsbs r5, r2, #0 -; THUMB6-NEXT: adcs r2, r5 -; THUMB6-NEXT: ldr r5, [sp, #28] -; THUMB6-NEXT: lsrs r3, r5 -; THUMB6-NEXT: ands r3, r4 -; THUMB6-NEXT: rsbs r4, r3, #0 -; THUMB6-NEXT: adcs r3, r4 -; THUMB6-NEXT: pop {r4, r5, r7, pc} +; THUMB6-NEXT: mov r5, r4 +; THUMB6-NEXT: bics r5, r1 +; THUMB6-NEXT: ldr r1, [sp, #24] +; THUMB6-NEXT: lsrs r2, r1 +; THUMB6-NEXT: mov r6, r4 +; THUMB6-NEXT: bics r6, r2 +; THUMB6-NEXT: ldr r1, [sp, #28] +; THUMB6-NEXT: lsrs r3, r1 +; THUMB6-NEXT: bics r4, r3 +; THUMB6-NEXT: mov r1, r5 +; THUMB6-NEXT: mov r2, r6 +; THUMB6-NEXT: mov r3, r4 +; THUMB6-NEXT: pop {r4, r5, r6, pc} ; ; THUMB78-LABEL: vec_4xi32_splat_eq: ; THUMB78: @ %bb.0: @@ -616,24 +609,24 @@ define <4 x i1> @vec_4xi32_nonsplat_eq(<4 x i32> %x, <4 x i32> %y) nounwind { ; THUMB6-LABEL: vec_4xi32_nonsplat_eq: ; THUMB6: @ %bb.0: ; THUMB6-NEXT: push {r4, lr} -; THUMB6-NEXT: ldr r0, [sp, #12] -; THUMB6-NEXT: lsrs r1, r0 +; THUMB6-NEXT: ldr r0, [sp, #16] +; THUMB6-NEXT: lsrs r2, r0 +; THUMB6-NEXT: ldr r0, .LCPI13_0 +; THUMB6-NEXT: ands r2, r0 +; THUMB6-NEXT: rsbs r0, r2, #0 +; THUMB6-NEXT: adcs r2, r0 +; THUMB6-NEXT: ldr r0, [sp, #20] +; THUMB6-NEXT: lsrs r3, r0 ; THUMB6-NEXT: movs r0, #1 -; THUMB6-NEXT: ands r1, r0 -; THUMB6-NEXT: rsbs r4, r1, #0 -; THUMB6-NEXT: adcs r1, r4 -; THUMB6-NEXT: ldr r4, [sp, #16] -; THUMB6-NEXT: lsrs r2, r4 -; THUMB6-NEXT: ldr r4, .LCPI13_0 -; THUMB6-NEXT: ands r2, r4 -; THUMB6-NEXT: rsbs r4, r2, #0 -; THUMB6-NEXT: adcs r2, r4 -; THUMB6-NEXT: ldr r4, [sp, #20] -; THUMB6-NEXT: lsrs r3, r4 ; THUMB6-NEXT: lsls r4, r0, #31 ; THUMB6-NEXT: ands r3, r4 ; THUMB6-NEXT: rsbs r4, r3, #0 ; THUMB6-NEXT: adcs r3, r4 +; THUMB6-NEXT: ldr r4, [sp, #12] +; THUMB6-NEXT: lsrs r1, r4 +; THUMB6-NEXT: mov r4, r0 +; THUMB6-NEXT: bics r4, r1 +; THUMB6-NEXT: mov r1, r4 ; THUMB6-NEXT: pop {r4, pc} ; THUMB6-NEXT: .p2align 2 ; THUMB6-NEXT: @ %bb.1: @@ -698,24 +691,24 @@ define <4 x i1> @vec_4xi32_nonsplat_undef0_eq(<4 x i32> %x, <4 x i32> %y) nounwi ; ; THUMB6-LABEL: vec_4xi32_nonsplat_undef0_eq: ; THUMB6: @ %bb.0: -; THUMB6-NEXT: push {r4, lr} -; THUMB6-NEXT: ldr r2, [sp, #8] +; THUMB6-NEXT: push {r4, r5, r6, lr} +; THUMB6-NEXT: ldr r2, [sp, #16] ; THUMB6-NEXT: lsrs r0, r2 ; THUMB6-NEXT: movs r2, #1 -; THUMB6-NEXT: ands r0, r2 -; THUMB6-NEXT: rsbs r4, r0, #0 -; THUMB6-NEXT: adcs r0, r4 -; THUMB6-NEXT: ldr r4, [sp, #12] -; THUMB6-NEXT: lsrs r1, r4 -; THUMB6-NEXT: ands r1, r2 -; THUMB6-NEXT: rsbs r4, r1, #0 -; THUMB6-NEXT: adcs r1, r4 -; THUMB6-NEXT: ldr r4, [sp, #20] -; THUMB6-NEXT: lsrs r3, r4 -; THUMB6-NEXT: ands r3, r2 -; THUMB6-NEXT: rsbs r4, r3, #0 -; THUMB6-NEXT: adcs r3, r4 -; THUMB6-NEXT: pop {r4, pc} +; THUMB6-NEXT: mov r4, r2 +; THUMB6-NEXT: bics r4, r0 +; THUMB6-NEXT: ldr r0, [sp, #20] +; THUMB6-NEXT: lsrs r1, r0 +; THUMB6-NEXT: mov r5, r2 +; THUMB6-NEXT: bics r5, r1 +; THUMB6-NEXT: ldr r0, [sp, #28] +; THUMB6-NEXT: lsrs r3, r0 +; THUMB6-NEXT: mov r6, r2 +; THUMB6-NEXT: bics r6, r3 +; THUMB6-NEXT: mov r0, r4 +; THUMB6-NEXT: mov r1, r5 +; THUMB6-NEXT: mov r3, r6 +; THUMB6-NEXT: pop {r4, r5, r6, pc} ; ; THUMB78-LABEL: vec_4xi32_nonsplat_undef0_eq: ; THUMB78: @ %bb.0: @@ -765,24 +758,23 @@ define <4 x i1> @vec_4xi32_nonsplat_undef1_eq(<4 x i32> %x, <4 x i32> %y) nounwi ; ; THUMB6-LABEL: vec_4xi32_nonsplat_undef1_eq: ; THUMB6: @ %bb.0: -; THUMB6-NEXT: push {r4, lr} -; THUMB6-NEXT: ldr r2, [sp, #8] +; THUMB6-NEXT: push {r4, r5, r7, lr} +; THUMB6-NEXT: ldr r2, [sp, #16] ; THUMB6-NEXT: lsrs r0, r2 ; THUMB6-NEXT: movs r2, #1 -; THUMB6-NEXT: ands r0, r2 -; THUMB6-NEXT: rsbs r4, r0, #0 -; THUMB6-NEXT: adcs r0, r4 -; THUMB6-NEXT: ldr r4, [sp, #12] -; THUMB6-NEXT: lsrs r1, r4 -; THUMB6-NEXT: ands r1, r2 -; THUMB6-NEXT: rsbs r4, r1, #0 -; THUMB6-NEXT: adcs r1, r4 -; THUMB6-NEXT: ldr r4, [sp, #20] -; THUMB6-NEXT: lsrs r3, r4 -; THUMB6-NEXT: ands r3, r2 -; THUMB6-NEXT: rsbs r2, r3, #0 -; THUMB6-NEXT: adcs r3, r2 -; THUMB6-NEXT: pop {r4, pc} +; THUMB6-NEXT: mov r4, r2 +; THUMB6-NEXT: bics r4, r0 +; THUMB6-NEXT: ldr r0, [sp, #20] +; THUMB6-NEXT: lsrs r1, r0 +; THUMB6-NEXT: mov r5, r2 +; THUMB6-NEXT: bics r5, r1 +; THUMB6-NEXT: ldr r0, [sp, #28] +; THUMB6-NEXT: lsrs r3, r0 +; THUMB6-NEXT: bics r2, r3 +; THUMB6-NEXT: mov r0, r4 +; THUMB6-NEXT: mov r1, r5 +; THUMB6-NEXT: mov r3, r2 +; THUMB6-NEXT: pop {r4, r5, r7, pc} ; ; THUMB78-LABEL: vec_4xi32_nonsplat_undef1_eq: ; THUMB78: @ %bb.0: @@ -831,24 +823,23 @@ define <4 x i1> @vec_4xi32_nonsplat_undef2_eq(<4 x i32> %x, <4 x i32> %y) nounwi ; ; THUMB6-LABEL: vec_4xi32_nonsplat_undef2_eq: ; THUMB6: @ %bb.0: -; THUMB6-NEXT: push {r4, lr} -; THUMB6-NEXT: ldr r2, [sp, #8] +; THUMB6-NEXT: push {r4, r5, r7, lr} +; THUMB6-NEXT: ldr r2, [sp, #16] ; THUMB6-NEXT: lsrs r0, r2 ; THUMB6-NEXT: movs r2, #1 -; THUMB6-NEXT: ands r0, r2 -; THUMB6-NEXT: rsbs r4, r0, #0 -; THUMB6-NEXT: adcs r0, r4 -; THUMB6-NEXT: ldr r4, [sp, #12] -; THUMB6-NEXT: lsrs r1, r4 -; THUMB6-NEXT: ands r1, r2 -; THUMB6-NEXT: rsbs r4, r1, #0 -; THUMB6-NEXT: adcs r1, r4 -; THUMB6-NEXT: ldr r4, [sp, #20] -; THUMB6-NEXT: lsrs r3, r4 -; THUMB6-NEXT: ands r3, r2 -; THUMB6-NEXT: rsbs r2, r3, #0 -; THUMB6-NEXT: adcs r3, r2 -; THUMB6-NEXT: pop {r4, pc} +; THUMB6-NEXT: mov r4, r2 +; THUMB6-NEXT: bics r4, r0 +; THUMB6-NEXT: ldr r0, [sp, #20] +; THUMB6-NEXT: lsrs r1, r0 +; THUMB6-NEXT: mov r5, r2 +; THUMB6-NEXT: bics r5, r1 +; THUMB6-NEXT: ldr r0, [sp, #28] +; THUMB6-NEXT: lsrs r3, r0 +; THUMB6-NEXT: bics r2, r3 +; THUMB6-NEXT: mov r0, r4 +; THUMB6-NEXT: mov r1, r5 +; THUMB6-NEXT: mov r3, r2 +; THUMB6-NEXT: pop {r4, r5, r7, pc} ; ; THUMB78-LABEL: vec_4xi32_nonsplat_undef2_eq: ; THUMB78: @ %bb.0: @@ -923,10 +914,8 @@ define i1 @scalar_i32_x_is_const_eq(i32 %y) nounwind { ; THUMB6: @ %bb.0: ; THUMB6-NEXT: ldr r1, .LCPI18_0 ; THUMB6-NEXT: lsls r1, r0 -; THUMB6-NEXT: movs r2, #1 -; THUMB6-NEXT: ands r2, r1 -; THUMB6-NEXT: rsbs r0, r2, #0 -; THUMB6-NEXT: adcs r0, r2 +; THUMB6-NEXT: movs r0, #1 +; THUMB6-NEXT: bics r0, r1 ; THUMB6-NEXT: bx lr ; THUMB6-NEXT: .p2align 2 ; THUMB6-NEXT: @ %bb.1: