Skip to content

Commit a46a5c1

Browse files
committed
Merge branch 'bpf-nfp-programmable-rss'
Jakub Kicinski says: ==================== This small series adds a feature which extends BPF offload beyond a pure host processing offload and firmly into the realm of heterogeneous processing. Allowing offloaded XDP programs to set the RX queue index opens the door for defining fully programmable RSS/n-tuple filter replacement. In fact the device datapath will skip the RSS processing completely if BPF decided on the queue already, making the XDP program replace part of the standard NIC datapath. We hope some day the entire NIC datapath will be defined by BPF :) ==================== Signed-off-by: Daniel Borkmann <daniel@iogearbox.net> Acked-by: Alexei Starovoitov <ast@kernel.org>
2 parents a1d1f07 + d985888 commit a46a5c1

File tree

9 files changed

+115
-15
lines changed

9 files changed

+115
-15
lines changed

drivers/net/ethernet/netronome/nfp/bpf/fw.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ enum bpf_cap_tlv_type {
5050
NFP_BPF_CAP_TYPE_ADJUST_HEAD = 2,
5151
NFP_BPF_CAP_TYPE_MAPS = 3,
5252
NFP_BPF_CAP_TYPE_RANDOM = 4,
53+
NFP_BPF_CAP_TYPE_QUEUE_SELECT = 5,
5354
};
5455

5556
struct nfp_bpf_cap_tlv_func {

drivers/net/ethernet/netronome/nfp/bpf/jit.c

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242

4343
#include "main.h"
4444
#include "../nfp_asm.h"
45+
#include "../nfp_net_ctrl.h"
4546

4647
/* --- NFP prog --- */
4748
/* Foreach "multiple" entries macros provide pos and next<n> pointers.
@@ -1470,6 +1471,38 @@ nfp_perf_event_output(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
14701471
return 0;
14711472
}
14721473

1474+
static int
1475+
nfp_queue_select(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1476+
{
1477+
u32 jmp_tgt;
1478+
1479+
jmp_tgt = nfp_prog_current_offset(nfp_prog) + 5;
1480+
1481+
/* Make sure the queue id fits into FW field */
1482+
emit_alu(nfp_prog, reg_none(), reg_a(meta->insn.src_reg * 2),
1483+
ALU_OP_AND_NOT_B, reg_imm(0xff));
1484+
emit_br(nfp_prog, BR_BEQ, jmp_tgt, 2);
1485+
1486+
/* Set the 'queue selected' bit and the queue value */
1487+
emit_shf(nfp_prog, pv_qsel_set(nfp_prog),
1488+
pv_qsel_set(nfp_prog), SHF_OP_OR, reg_imm(1),
1489+
SHF_SC_L_SHF, PKT_VEL_QSEL_SET_BIT);
1490+
emit_ld_field(nfp_prog,
1491+
pv_qsel_val(nfp_prog), 0x1, reg_b(meta->insn.src_reg * 2),
1492+
SHF_SC_NONE, 0);
1493+
/* Delay slots end here, we will jump over next instruction if queue
1494+
* value fits into the field.
1495+
*/
1496+
emit_ld_field(nfp_prog,
1497+
pv_qsel_val(nfp_prog), 0x1, reg_imm(NFP_NET_RXR_MAX),
1498+
SHF_SC_NONE, 0);
1499+
1500+
if (!nfp_prog_confirm_current_offset(nfp_prog, jmp_tgt))
1501+
return -EINVAL;
1502+
1503+
return 0;
1504+
}
1505+
14731506
/* --- Callbacks --- */
14741507
static int mov_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
14751508
{
@@ -2160,6 +2193,17 @@ mem_stx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
21602193
false, wrp_lmem_store);
21612194
}
21622195

2196+
static int mem_stx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2197+
{
2198+
switch (meta->insn.off) {
2199+
case offsetof(struct xdp_md, rx_queue_index):
2200+
return nfp_queue_select(nfp_prog, meta);
2201+
}
2202+
2203+
WARN_ON_ONCE(1); /* verifier should have rejected bad accesses */
2204+
return -EOPNOTSUPP;
2205+
}
2206+
21632207
static int
21642208
mem_stx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
21652209
unsigned int size)
@@ -2186,6 +2230,9 @@ static int mem_stx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
21862230

21872231
static int mem_stx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
21882232
{
2233+
if (meta->ptr.type == PTR_TO_CTX)
2234+
if (nfp_prog->type == BPF_PROG_TYPE_XDP)
2235+
return mem_stx_xdp(nfp_prog, meta);
21892236
return mem_stx(nfp_prog, meta, 4);
21902237
}
21912238

drivers/net/ethernet/netronome/nfp/bpf/main.c

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,13 @@ nfp_bpf_parse_cap_random(struct nfp_app_bpf *bpf, void __iomem *value,
334334
return 0;
335335
}
336336

337+
static int
338+
nfp_bpf_parse_cap_qsel(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
339+
{
340+
bpf->queue_select = true;
341+
return 0;
342+
}
343+
337344
static int nfp_bpf_parse_capabilities(struct nfp_app *app)
338345
{
339346
struct nfp_cpp *cpp = app->pf->cpp;
@@ -376,6 +383,10 @@ static int nfp_bpf_parse_capabilities(struct nfp_app *app)
376383
if (nfp_bpf_parse_cap_random(app->priv, value, length))
377384
goto err_release_free;
378385
break;
386+
case NFP_BPF_CAP_TYPE_QUEUE_SELECT:
387+
if (nfp_bpf_parse_cap_qsel(app->priv, value, length))
388+
goto err_release_free;
389+
break;
379390
default:
380391
nfp_dbg(cpp, "unknown BPF capability: %d\n", type);
381392
break;

drivers/net/ethernet/netronome/nfp/bpf/main.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,10 +82,16 @@ enum static_regs {
8282
enum pkt_vec {
8383
PKT_VEC_PKT_LEN = 0,
8484
PKT_VEC_PKT_PTR = 2,
85+
PKT_VEC_QSEL_SET = 4,
86+
PKT_VEC_QSEL_VAL = 6,
8587
};
8688

89+
#define PKT_VEL_QSEL_SET_BIT 4
90+
8791
#define pv_len(np) reg_lm(1, PKT_VEC_PKT_LEN)
8892
#define pv_ctm_ptr(np) reg_lm(1, PKT_VEC_PKT_PTR)
93+
#define pv_qsel_set(np) reg_lm(1, PKT_VEC_QSEL_SET)
94+
#define pv_qsel_val(np) reg_lm(1, PKT_VEC_QSEL_VAL)
8995

9096
#define stack_reg(np) reg_a(STATIC_REG_STACK)
9197
#define stack_imm(np) imm_b(np)
@@ -139,6 +145,7 @@ enum pkt_vec {
139145
* @helpers.perf_event_output: output perf event to a ring buffer
140146
*
141147
* @pseudo_random: FW initialized the pseudo-random machinery (CSRs)
148+
* @queue_select: BPF can set the RX queue ID in packet vector
142149
*/
143150
struct nfp_app_bpf {
144151
struct nfp_app *app;
@@ -181,6 +188,7 @@ struct nfp_app_bpf {
181188
} helpers;
182189

183190
bool pseudo_random;
191+
bool queue_select;
184192
};
185193

186194
enum nfp_bpf_map_use {

drivers/net/ethernet/netronome/nfp/bpf/verifier.c

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -467,6 +467,30 @@ nfp_bpf_check_ptr(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
467467
return 0;
468468
}
469469

470+
static int
471+
nfp_bpf_check_store(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
472+
struct bpf_verifier_env *env)
473+
{
474+
const struct bpf_reg_state *reg = cur_regs(env) + meta->insn.dst_reg;
475+
476+
if (reg->type == PTR_TO_CTX) {
477+
if (nfp_prog->type == BPF_PROG_TYPE_XDP) {
478+
/* XDP ctx accesses must be 4B in size */
479+
switch (meta->insn.off) {
480+
case offsetof(struct xdp_md, rx_queue_index):
481+
if (nfp_prog->bpf->queue_select)
482+
goto exit_check_ptr;
483+
pr_vlog(env, "queue selection not supported by FW\n");
484+
return -EOPNOTSUPP;
485+
}
486+
}
487+
pr_vlog(env, "unsupported store to context field\n");
488+
return -EOPNOTSUPP;
489+
}
490+
exit_check_ptr:
491+
return nfp_bpf_check_ptr(nfp_prog, meta, env, meta->insn.dst_reg);
492+
}
493+
470494
static int
471495
nfp_bpf_check_xadd(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
472496
struct bpf_verifier_env *env)
@@ -522,8 +546,8 @@ nfp_verify_insn(struct bpf_verifier_env *env, int insn_idx, int prev_insn_idx)
522546
return nfp_bpf_check_ptr(nfp_prog, meta, env,
523547
meta->insn.src_reg);
524548
if (is_mbpf_store(meta))
525-
return nfp_bpf_check_ptr(nfp_prog, meta, env,
526-
meta->insn.dst_reg);
549+
return nfp_bpf_check_store(nfp_prog, meta, env);
550+
527551
if (is_mbpf_xadd(meta))
528552
return nfp_bpf_check_xadd(nfp_prog, meta, env);
529553

drivers/net/ethernet/netronome/nfp/nfp_asm.h

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -183,16 +183,18 @@ enum shf_sc {
183183
#define OP_ALU_DST_LMEXTN 0x80000000000ULL
184184

185185
enum alu_op {
186-
ALU_OP_NONE = 0x00,
187-
ALU_OP_ADD = 0x01,
188-
ALU_OP_NOT = 0x04,
189-
ALU_OP_ADD_2B = 0x05,
190-
ALU_OP_AND = 0x08,
191-
ALU_OP_SUB_C = 0x0d,
192-
ALU_OP_ADD_C = 0x11,
193-
ALU_OP_OR = 0x14,
194-
ALU_OP_SUB = 0x15,
195-
ALU_OP_XOR = 0x18,
186+
ALU_OP_NONE = 0x00,
187+
ALU_OP_ADD = 0x01,
188+
ALU_OP_NOT = 0x04,
189+
ALU_OP_ADD_2B = 0x05,
190+
ALU_OP_AND = 0x08,
191+
ALU_OP_AND_NOT_A = 0x0c,
192+
ALU_OP_SUB_C = 0x0d,
193+
ALU_OP_AND_NOT_B = 0x10,
194+
ALU_OP_ADD_C = 0x11,
195+
ALU_OP_OR = 0x14,
196+
ALU_OP_SUB = 0x15,
197+
ALU_OP_XOR = 0x18,
196198
};
197199

198200
enum alu_dst_ab {

include/linux/bpf.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -627,7 +627,7 @@ bool bpf_offload_dev_match(struct bpf_prog *prog, struct bpf_map *map);
627627
#if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL)
628628
int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr);
629629

630-
static inline bool bpf_prog_is_dev_bound(struct bpf_prog_aux *aux)
630+
static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux)
631631
{
632632
return aux->offload_requested;
633633
}

kernel/bpf/verifier.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5215,7 +5215,7 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
52155215
}
52165216
}
52175217

5218-
if (!ops->convert_ctx_access)
5218+
if (!ops->convert_ctx_access || bpf_prog_is_dev_bound(env->prog->aux))
52195219
return 0;
52205220

52215221
insn = env->prog->insnsi + delta;

net/core/filter.c

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4645,8 +4645,15 @@ static bool xdp_is_valid_access(int off, int size,
46454645
const struct bpf_prog *prog,
46464646
struct bpf_insn_access_aux *info)
46474647
{
4648-
if (type == BPF_WRITE)
4648+
if (type == BPF_WRITE) {
4649+
if (bpf_prog_is_dev_bound(prog->aux)) {
4650+
switch (off) {
4651+
case offsetof(struct xdp_md, rx_queue_index):
4652+
return __is_valid_xdp_access(off, size);
4653+
}
4654+
}
46494655
return false;
4656+
}
46504657

46514658
switch (off) {
46524659
case offsetof(struct xdp_md, data):

0 commit comments

Comments
 (0)