Skip to content

Commit 6baefa1

Browse files
author
Alexei Starovoitov
committed
Merge branch 'support-alu32_arsh'
Jiong Wang says: ==================== BPF_ALU | BPF_ARSH | BPF_* were rejected by commit: 7891a87 ("bpf: arsh is not supported in 32 bit alu thus reject it"). As explained in the commit message, this is due to there is no complete support for them on interpreter and various JIT compilation back-ends. This patch set is a follow-up which completes the missing bits. This also pave the way for running bpf program compiled with ALU32 instruction enabled by specifing -mattr=+alu32 to LLVM for which case there is likely to have more BPF_ALU | BPF_ARSH insns that will trigger the rejection code. test_verifier.c is updated accordingly. I have tested this patch set on x86-64 and NFP, I need help of review and test on the arch changes (mips/ppc/s390). Note, there might be merge confict on mips change which is better to be applied on top of: commit: 20b880a ("mips: bpf: fix encoding bug for mm_srlv32_op"), which is on mips-fixes branch at the moment. Thanks. v1->v2: - Fix ppc implementation bug. Should zero high bits explicitly. ==================== Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2 parents 17f6c83 + c099f3f commit 6baefa1

File tree

14 files changed

+137
-35
lines changed

14 files changed

+137
-35
lines changed

‎arch/mips/include/asm/uasm.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@ Ip_u2u1s3(_slti);
157157
Ip_u2u1s3(_sltiu);
158158
Ip_u3u1u2(_sltu);
159159
Ip_u2u1u3(_sra);
160+
Ip_u3u2u1(_srav);
160161
Ip_u2u1u3(_srl);
161162
Ip_u3u2u1(_srlv);
162163
Ip_u3u1u2(_subu);

‎arch/mips/include/uapi/asm/inst.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -371,6 +371,7 @@ enum mm_32a_minor_op {
371371
mm_srl32_op = 0x040,
372372
mm_srlv32_op = 0x050,
373373
mm_sra_op = 0x080,
374+
mm_srav_op = 0x090,
374375
mm_rotr_op = 0x0c0,
375376
mm_lwxs_op = 0x118,
376377
mm_addu32_op = 0x150,

‎arch/mips/mm/uasm-micromips.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ static const struct insn insn_table_MM[insn_invalid] = {
104104
[insn_sltiu] = {M(mm_sltiu32_op, 0, 0, 0, 0, 0), RT | RS | SIMM},
105105
[insn_sltu] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_sltu_op), RT | RS | RD},
106106
[insn_sra] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_sra_op), RT | RS | RD},
107+
[insn_srav] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_srav_op), RT | RS | RD},
107108
[insn_srl] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_srl32_op), RT | RS | RD},
108109
[insn_srlv] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_srlv32_op), RT | RS | RD},
109110
[insn_rotr] = {M(mm_pool32a_op, 0, 0, 0, 0, mm_rotr_op), RT | RS | RD},

‎arch/mips/mm/uasm-mips.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,7 @@ static const struct insn insn_table[insn_invalid] = {
171171
[insn_sltiu] = {M(sltiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM},
172172
[insn_sltu] = {M(spec_op, 0, 0, 0, 0, sltu_op), RS | RT | RD},
173173
[insn_sra] = {M(spec_op, 0, 0, 0, 0, sra_op), RT | RD | RE},
174+
[insn_srav] = {M(spec_op, 0, 0, 0, 0, srav_op), RS | RT | RD},
174175
[insn_srl] = {M(spec_op, 0, 0, 0, 0, srl_op), RT | RD | RE},
175176
[insn_srlv] = {M(spec_op, 0, 0, 0, 0, srlv_op), RS | RT | RD},
176177
[insn_subu] = {M(spec_op, 0, 0, 0, 0, subu_op), RS | RT | RD},

‎arch/mips/mm/uasm.c

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,10 +61,10 @@ enum opcode {
6161
insn_mthc0, insn_mthi, insn_mtlo, insn_mul, insn_multu, insn_nor,
6262
insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr, insn_sb,
6363
insn_sc, insn_scd, insn_sd, insn_sh, insn_sll, insn_sllv,
64-
insn_slt, insn_slti, insn_sltiu, insn_sltu, insn_sra, insn_srl,
65-
insn_srlv, insn_subu, insn_sw, insn_sync, insn_syscall, insn_tlbp,
66-
insn_tlbr, insn_tlbwi, insn_tlbwr, insn_wait, insn_wsbh, insn_xor,
67-
insn_xori, insn_yield,
64+
insn_slt, insn_slti, insn_sltiu, insn_sltu, insn_sra, insn_srav,
65+
insn_srl, insn_srlv, insn_subu, insn_sw, insn_sync, insn_syscall,
66+
insn_tlbp, insn_tlbr, insn_tlbwi, insn_tlbwr, insn_wait, insn_wsbh,
67+
insn_xor, insn_xori, insn_yield,
6868
insn_invalid /* insn_invalid must be last */
6969
};
7070

@@ -353,6 +353,7 @@ I_u2u1s3(_slti)
353353
I_u2u1s3(_sltiu)
354354
I_u3u1u2(_sltu)
355355
I_u2u1u3(_sra)
356+
I_u3u2u1(_srav)
356357
I_u2u1u3(_srl)
357358
I_u3u2u1(_srlv)
358359
I_u2u1u3(_rotr)

‎arch/mips/net/ebpf_jit.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -854,6 +854,7 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
854854
case BPF_ALU | BPF_MOD | BPF_X: /* ALU_REG */
855855
case BPF_ALU | BPF_LSH | BPF_X: /* ALU_REG */
856856
case BPF_ALU | BPF_RSH | BPF_X: /* ALU_REG */
857+
case BPF_ALU | BPF_ARSH | BPF_X: /* ALU_REG */
857858
src = ebpf_to_mips_reg(ctx, insn, src_reg_no_fp);
858859
dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
859860
if (src < 0 || dst < 0)
@@ -913,6 +914,9 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
913914
case BPF_RSH:
914915
emit_instr(ctx, srlv, dst, dst, src);
915916
break;
917+
case BPF_ARSH:
918+
emit_instr(ctx, srav, dst, dst, src);
919+
break;
916920
default:
917921
pr_err("ALU_REG NOT HANDLED\n");
918922
return -EINVAL;

‎arch/powerpc/include/asm/ppc-opcode.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,8 @@
342342
#define PPC_INST_SLW 0x7c000030
343343
#define PPC_INST_SLD 0x7c000036
344344
#define PPC_INST_SRW 0x7c000430
345+
#define PPC_INST_SRAW 0x7c000630
346+
#define PPC_INST_SRAWI 0x7c000670
345347
#define PPC_INST_SRD 0x7c000436
346348
#define PPC_INST_SRAD 0x7c000634
347349
#define PPC_INST_SRADI 0x7c000674

‎arch/powerpc/net/bpf_jit.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,10 @@
152152
___PPC_RS(a) | ___PPC_RB(s))
153153
#define PPC_SRW(d, a, s) EMIT(PPC_INST_SRW | ___PPC_RA(d) | \
154154
___PPC_RS(a) | ___PPC_RB(s))
155+
#define PPC_SRAW(d, a, s) EMIT(PPC_INST_SRAW | ___PPC_RA(d) | \
156+
___PPC_RS(a) | ___PPC_RB(s))
157+
#define PPC_SRAWI(d, a, i) EMIT(PPC_INST_SRAWI | ___PPC_RA(d) | \
158+
___PPC_RS(a) | __PPC_SH(i))
155159
#define PPC_SRD(d, a, s) EMIT(PPC_INST_SRD | ___PPC_RA(d) | \
156160
___PPC_RS(a) | ___PPC_RB(s))
157161
#define PPC_SRAD(d, a, s) EMIT(PPC_INST_SRAD | ___PPC_RA(d) | \

‎arch/powerpc/net/bpf_jit_comp64.c

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -529,9 +529,15 @@ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
529529
if (imm != 0)
530530
PPC_SRDI(dst_reg, dst_reg, imm);
531531
break;
532+
case BPF_ALU | BPF_ARSH | BPF_X: /* (s32) dst >>= src */
533+
PPC_SRAW(dst_reg, dst_reg, src_reg);
534+
goto bpf_alu32_trunc;
532535
case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */
533536
PPC_SRAD(dst_reg, dst_reg, src_reg);
534537
break;
538+
case BPF_ALU | BPF_ARSH | BPF_K: /* (s32) dst >>= imm */
539+
PPC_SRAWI(dst_reg, dst_reg, imm);
540+
goto bpf_alu32_trunc;
535541
case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */
536542
if (imm != 0)
537543
PPC_SRADI(dst_reg, dst_reg, imm);

‎arch/s390/net/bpf_jit_comp.c

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -821,10 +821,22 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i
821821
/*
822822
* BPF_ARSH
823823
*/
824+
case BPF_ALU | BPF_ARSH | BPF_X: /* ((s32) dst) >>= src */
825+
/* sra %dst,%dst,0(%src) */
826+
EMIT4_DISP(0x8a000000, dst_reg, src_reg, 0);
827+
EMIT_ZERO(dst_reg);
828+
break;
824829
case BPF_ALU64 | BPF_ARSH | BPF_X: /* ((s64) dst) >>= src */
825830
/* srag %dst,%dst,0(%src) */
826831
EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, src_reg, 0);
827832
break;
833+
case BPF_ALU | BPF_ARSH | BPF_K: /* ((s32) dst >> imm */
834+
if (imm == 0)
835+
break;
836+
/* sra %dst,imm(%r0) */
837+
EMIT4_DISP(0x8a000000, dst_reg, REG_0, imm);
838+
EMIT_ZERO(dst_reg);
839+
break;
828840
case BPF_ALU64 | BPF_ARSH | BPF_K: /* ((s64) dst) >>= imm */
829841
if (imm == 0)
830842
break;

‎drivers/net/ethernet/netronome/nfp/bpf/jit.c

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2382,6 +2382,49 @@ static int neg_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
23822382
return 0;
23832383
}
23842384

2385+
static int __ashr_imm(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
2386+
{
2387+
/* Set signedness bit (MSB of result). */
2388+
emit_alu(nfp_prog, reg_none(), reg_a(dst), ALU_OP_OR, reg_imm(0));
2389+
emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR, reg_b(dst),
2390+
SHF_SC_R_SHF, shift_amt);
2391+
wrp_immed(nfp_prog, reg_both(dst + 1), 0);
2392+
2393+
return 0;
2394+
}
2395+
2396+
static int ashr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2397+
{
2398+
const struct bpf_insn *insn = &meta->insn;
2399+
u64 umin, umax;
2400+
u8 dst, src;
2401+
2402+
dst = insn->dst_reg * 2;
2403+
umin = meta->umin_src;
2404+
umax = meta->umax_src;
2405+
if (umin == umax)
2406+
return __ashr_imm(nfp_prog, dst, umin);
2407+
2408+
src = insn->src_reg * 2;
2409+
/* NOTE: the first insn will set both indirect shift amount (source A)
2410+
* and signedness bit (MSB of result).
2411+
*/
2412+
emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst));
2413+
emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR,
2414+
reg_b(dst), SHF_SC_R_SHF);
2415+
wrp_immed(nfp_prog, reg_both(dst + 1), 0);
2416+
2417+
return 0;
2418+
}
2419+
2420+
static int ashr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2421+
{
2422+
const struct bpf_insn *insn = &meta->insn;
2423+
u8 dst = insn->dst_reg * 2;
2424+
2425+
return __ashr_imm(nfp_prog, dst, insn->imm);
2426+
}
2427+
23852428
static int shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
23862429
{
23872430
const struct bpf_insn *insn = &meta->insn;
@@ -3286,6 +3329,8 @@ static const instr_cb_t instr_cb[256] = {
32863329
[BPF_ALU | BPF_DIV | BPF_K] = div_imm,
32873330
[BPF_ALU | BPF_NEG] = neg_reg,
32883331
[BPF_ALU | BPF_LSH | BPF_K] = shl_imm,
3332+
[BPF_ALU | BPF_ARSH | BPF_X] = ashr_reg,
3333+
[BPF_ALU | BPF_ARSH | BPF_K] = ashr_imm,
32893334
[BPF_ALU | BPF_END | BPF_X] = end_reg32,
32903335
[BPF_LD | BPF_IMM | BPF_DW] = imm_ld8,
32913336
[BPF_LD | BPF_ABS | BPF_B] = data_ld1,

‎kernel/bpf/core.c

Lines changed: 30 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -933,32 +933,34 @@ EXPORT_SYMBOL_GPL(__bpf_call_base);
933933
#define BPF_INSN_MAP(INSN_2, INSN_3) \
934934
/* 32 bit ALU operations. */ \
935935
/* Register based. */ \
936-
INSN_3(ALU, ADD, X), \
937-
INSN_3(ALU, SUB, X), \
938-
INSN_3(ALU, AND, X), \
939-
INSN_3(ALU, OR, X), \
940-
INSN_3(ALU, LSH, X), \
941-
INSN_3(ALU, RSH, X), \
942-
INSN_3(ALU, XOR, X), \
943-
INSN_3(ALU, MUL, X), \
944-
INSN_3(ALU, MOV, X), \
945-
INSN_3(ALU, DIV, X), \
946-
INSN_3(ALU, MOD, X), \
936+
INSN_3(ALU, ADD, X), \
937+
INSN_3(ALU, SUB, X), \
938+
INSN_3(ALU, AND, X), \
939+
INSN_3(ALU, OR, X), \
940+
INSN_3(ALU, LSH, X), \
941+
INSN_3(ALU, RSH, X), \
942+
INSN_3(ALU, XOR, X), \
943+
INSN_3(ALU, MUL, X), \
944+
INSN_3(ALU, MOV, X), \
945+
INSN_3(ALU, ARSH, X), \
946+
INSN_3(ALU, DIV, X), \
947+
INSN_3(ALU, MOD, X), \
947948
INSN_2(ALU, NEG), \
948949
INSN_3(ALU, END, TO_BE), \
949950
INSN_3(ALU, END, TO_LE), \
950951
/* Immediate based. */ \
951-
INSN_3(ALU, ADD, K), \
952-
INSN_3(ALU, SUB, K), \
953-
INSN_3(ALU, AND, K), \
954-
INSN_3(ALU, OR, K), \
955-
INSN_3(ALU, LSH, K), \
956-
INSN_3(ALU, RSH, K), \
957-
INSN_3(ALU, XOR, K), \
958-
INSN_3(ALU, MUL, K), \
959-
INSN_3(ALU, MOV, K), \
960-
INSN_3(ALU, DIV, K), \
961-
INSN_3(ALU, MOD, K), \
952+
INSN_3(ALU, ADD, K), \
953+
INSN_3(ALU, SUB, K), \
954+
INSN_3(ALU, AND, K), \
955+
INSN_3(ALU, OR, K), \
956+
INSN_3(ALU, LSH, K), \
957+
INSN_3(ALU, RSH, K), \
958+
INSN_3(ALU, XOR, K), \
959+
INSN_3(ALU, MUL, K), \
960+
INSN_3(ALU, MOV, K), \
961+
INSN_3(ALU, ARSH, K), \
962+
INSN_3(ALU, DIV, K), \
963+
INSN_3(ALU, MOD, K), \
962964
/* 64 bit ALU operations. */ \
963965
/* Register based. */ \
964966
INSN_3(ALU64, ADD, X), \
@@ -1137,6 +1139,12 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn, u64 *stack)
11371139
DST = (u64) (u32) insn[0].imm | ((u64) (u32) insn[1].imm) << 32;
11381140
insn++;
11391141
CONT;
1142+
ALU_ARSH_X:
1143+
DST = (u64) (u32) ((*(s32 *) &DST) >> SRC);
1144+
CONT;
1145+
ALU_ARSH_K:
1146+
DST = (u64) (u32) ((*(s32 *) &DST) >> IMM);
1147+
CONT;
11401148
ALU64_ARSH_X:
11411149
(*(s64 *) &DST) >>= SRC;
11421150
CONT;

‎kernel/bpf/verifier.c

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3649,11 +3649,6 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
36493649
return -EINVAL;
36503650
}
36513651

3652-
if (opcode == BPF_ARSH && BPF_CLASS(insn->code) != BPF_ALU64) {
3653-
verbose(env, "BPF_ARSH not supported for 32 bit ALU\n");
3654-
return -EINVAL;
3655-
}
3656-
36573652
if ((opcode == BPF_LSH || opcode == BPF_RSH ||
36583653
opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) {
36593654
int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32;

‎tools/testing/selftests/bpf/test_verifier.c

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -721,8 +721,18 @@ static struct bpf_test tests[] = {
721721
BPF_ALU32_IMM(BPF_ARSH, BPF_REG_0, 5),
722722
BPF_EXIT_INSN(),
723723
},
724-
.result = REJECT,
725-
.errstr = "unknown opcode c4",
724+
.result = ACCEPT,
725+
.retval = 0,
726+
},
727+
{
728+
"arsh32 on imm 2",
729+
.insns = {
730+
BPF_LD_IMM64(BPF_REG_0, 0x1122334485667788),
731+
BPF_ALU32_IMM(BPF_ARSH, BPF_REG_0, 7),
732+
BPF_EXIT_INSN(),
733+
},
734+
.result = ACCEPT,
735+
.retval = -16069393,
726736
},
727737
{
728738
"arsh32 on reg",
@@ -732,8 +742,19 @@ static struct bpf_test tests[] = {
732742
BPF_ALU32_REG(BPF_ARSH, BPF_REG_0, BPF_REG_1),
733743
BPF_EXIT_INSN(),
734744
},
735-
.result = REJECT,
736-
.errstr = "unknown opcode cc",
745+
.result = ACCEPT,
746+
.retval = 0,
747+
},
748+
{
749+
"arsh32 on reg 2",
750+
.insns = {
751+
BPF_LD_IMM64(BPF_REG_0, 0xffff55667788),
752+
BPF_MOV64_IMM(BPF_REG_1, 15),
753+
BPF_ALU32_REG(BPF_ARSH, BPF_REG_0, BPF_REG_1),
754+
BPF_EXIT_INSN(),
755+
},
756+
.result = ACCEPT,
757+
.retval = 43724,
737758
},
738759
{
739760
"arsh64 on imm",

0 commit comments

Comments
 (0)