Skip to content

Commit 7a12b50

Browse files
davem330davem330
committed
sparc64: Add eBPF JIT.
This is an eBPF JIT for sparc64. All major features are supported. All tests under tools/testing/selftests/bpf/ pass. Signed-off-by: David S. Miller <davem@davemloft.net>
1 parent 6b3d4ee commit 7a12b50

File tree

7 files changed

+1424
-60
lines changed

7 files changed

+1424
-60
lines changed

‎arch/sparc/Kconfig

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@ config SPARC
3131
select ARCH_WANT_IPC_PARSE_VERSION
3232
select GENERIC_PCI_IOMAP
3333
select HAVE_NMI_WATCHDOG if SPARC64
34-
select HAVE_CBPF_JIT
34+
select HAVE_CBPF_JIT if SPARC32
35+
select HAVE_EBPF_JIT if SPARC64
3536
select HAVE_DEBUG_BUGVERBOSE
3637
select GENERIC_SMP_IDLE_THREAD
3738
select GENERIC_CLOCKEVENTS

‎arch/sparc/net/bpf_jit_32.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@
3939
#define r_TMP2 G2
4040
#define r_OFF G3
4141

42-
/* assembly code in arch/sparc/net/bpf_jit_asm.S */
42+
/* assembly code in arch/sparc/net/bpf_jit_asm_32.S */
4343
extern u32 bpf_jit_load_word[];
4444
extern u32 bpf_jit_load_half[];
4545
extern u32 bpf_jit_load_byte[];

‎arch/sparc/net/bpf_jit_64.h

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#ifndef _BPF_JIT_H
2+
#define _BPF_JIT_H
3+
4+
#ifndef __ASSEMBLER__
5+
#define G0 0x00
6+
#define G1 0x01
7+
#define G2 0x02
8+
#define G3 0x03
9+
#define G6 0x06
10+
#define G7 0x07
11+
#define O0 0x08
12+
#define O1 0x09
13+
#define O2 0x0a
14+
#define O3 0x0b
15+
#define O4 0x0c
16+
#define O5 0x0d
17+
#define SP 0x0e
18+
#define O7 0x0f
19+
#define L0 0x10
20+
#define L1 0x11
21+
#define L2 0x12
22+
#define L3 0x13
23+
#define L4 0x14
24+
#define L5 0x15
25+
#define L6 0x16
26+
#define L7 0x17
27+
#define I0 0x18
28+
#define I1 0x19
29+
#define I2 0x1a
30+
#define I3 0x1b
31+
#define I4 0x1c
32+
#define I5 0x1d
33+
#define FP 0x1e
34+
#define I7 0x1f
35+
36+
#define r_SKB L0
37+
#define r_HEADLEN L4
38+
#define r_SKB_DATA L5
39+
#define r_TMP G1
40+
#define r_TMP2 G3
41+
42+
/* assembly code in arch/sparc/net/bpf_jit_asm_64.S */
43+
extern u32 bpf_jit_load_word[];
44+
extern u32 bpf_jit_load_half[];
45+
extern u32 bpf_jit_load_byte[];
46+
extern u32 bpf_jit_load_byte_msh[];
47+
extern u32 bpf_jit_load_word_positive_offset[];
48+
extern u32 bpf_jit_load_half_positive_offset[];
49+
extern u32 bpf_jit_load_byte_positive_offset[];
50+
extern u32 bpf_jit_load_byte_msh_positive_offset[];
51+
extern u32 bpf_jit_load_word_negative_offset[];
52+
extern u32 bpf_jit_load_half_negative_offset[];
53+
extern u32 bpf_jit_load_byte_negative_offset[];
54+
extern u32 bpf_jit_load_byte_msh_negative_offset[];
55+
56+
#else
57+
#define r_RESULT %o0
58+
#define r_SKB %o0
59+
#define r_OFF %o1
60+
#define r_HEADLEN %l4
61+
#define r_SKB_DATA %l5
62+
#define r_TMP %g1
63+
#define r_TMP2 %g3
64+
#endif
65+
66+
#endif /* _BPF_JIT_H */

‎arch/sparc/net/bpf_jit_asm_32.S

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,10 @@
22

33
#include "bpf_jit_32.h"
44

5-
#ifdef CONFIG_SPARC64
6-
#define SAVE_SZ 176
7-
#define SCRATCH_OFF STACK_BIAS + 128
8-
#define BE_PTR(label) be,pn %xcc, label
9-
#define SIGN_EXTEND(reg) sra reg, 0, reg
10-
#else
115
#define SAVE_SZ 96
126
#define SCRATCH_OFF 72
137
#define BE_PTR(label) be label
148
#define SIGN_EXTEND(reg)
15-
#endif
169

1710
#define SKF_MAX_NEG_OFF (-0x200000) /* SKF_LL_OFF from filter.h */
1811

‎arch/sparc/net/bpf_jit_asm_64.S

Lines changed: 161 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,161 @@
1-
#include "bpf_jit_asm_32.S"
1+
#include <asm/ptrace.h>
2+
3+
#include "bpf_jit_64.h"
4+
5+
#define SAVE_SZ 176
6+
#define SCRATCH_OFF STACK_BIAS + 128
7+
#define BE_PTR(label) be,pn %xcc, label
8+
#define SIGN_EXTEND(reg) sra reg, 0, reg
9+
10+
#define SKF_MAX_NEG_OFF (-0x200000) /* SKF_LL_OFF from filter.h */
11+
12+
.text
13+
.globl bpf_jit_load_word
14+
bpf_jit_load_word:
15+
cmp r_OFF, 0
16+
bl bpf_slow_path_word_neg
17+
nop
18+
.globl bpf_jit_load_word_positive_offset
19+
bpf_jit_load_word_positive_offset:
20+
sub r_HEADLEN, r_OFF, r_TMP
21+
cmp r_TMP, 3
22+
ble bpf_slow_path_word
23+
add r_SKB_DATA, r_OFF, r_TMP
24+
andcc r_TMP, 3, %g0
25+
bne load_word_unaligned
26+
nop
27+
retl
28+
ld [r_TMP], r_RESULT
29+
load_word_unaligned:
30+
ldub [r_TMP + 0x0], r_OFF
31+
ldub [r_TMP + 0x1], r_TMP2
32+
sll r_OFF, 8, r_OFF
33+
or r_OFF, r_TMP2, r_OFF
34+
ldub [r_TMP + 0x2], r_TMP2
35+
sll r_OFF, 8, r_OFF
36+
or r_OFF, r_TMP2, r_OFF
37+
ldub [r_TMP + 0x3], r_TMP2
38+
sll r_OFF, 8, r_OFF
39+
retl
40+
or r_OFF, r_TMP2, r_RESULT
41+
42+
.globl bpf_jit_load_half
43+
bpf_jit_load_half:
44+
cmp r_OFF, 0
45+
bl bpf_slow_path_half_neg
46+
nop
47+
.globl bpf_jit_load_half_positive_offset
48+
bpf_jit_load_half_positive_offset:
49+
sub r_HEADLEN, r_OFF, r_TMP
50+
cmp r_TMP, 1
51+
ble bpf_slow_path_half
52+
add r_SKB_DATA, r_OFF, r_TMP
53+
andcc r_TMP, 1, %g0
54+
bne load_half_unaligned
55+
nop
56+
retl
57+
lduh [r_TMP], r_RESULT
58+
load_half_unaligned:
59+
ldub [r_TMP + 0x0], r_OFF
60+
ldub [r_TMP + 0x1], r_TMP2
61+
sll r_OFF, 8, r_OFF
62+
retl
63+
or r_OFF, r_TMP2, r_RESULT
64+
65+
.globl bpf_jit_load_byte
66+
bpf_jit_load_byte:
67+
cmp r_OFF, 0
68+
bl bpf_slow_path_byte_neg
69+
nop
70+
.globl bpf_jit_load_byte_positive_offset
71+
bpf_jit_load_byte_positive_offset:
72+
cmp r_OFF, r_HEADLEN
73+
bge bpf_slow_path_byte
74+
nop
75+
retl
76+
ldub [r_SKB_DATA + r_OFF], r_RESULT
77+
78+
#define bpf_slow_path_common(LEN) \
79+
save %sp, -SAVE_SZ, %sp; \
80+
mov %i0, %o0; \
81+
mov %i1, %o1; \
82+
add %fp, SCRATCH_OFF, %o2; \
83+
call skb_copy_bits; \
84+
mov (LEN), %o3; \
85+
cmp %o0, 0; \
86+
restore;
87+
88+
bpf_slow_path_word:
89+
bpf_slow_path_common(4)
90+
bl bpf_error
91+
ld [%sp + SCRATCH_OFF], r_RESULT
92+
retl
93+
nop
94+
bpf_slow_path_half:
95+
bpf_slow_path_common(2)
96+
bl bpf_error
97+
lduh [%sp + SCRATCH_OFF], r_RESULT
98+
retl
99+
nop
100+
bpf_slow_path_byte:
101+
bpf_slow_path_common(1)
102+
bl bpf_error
103+
ldub [%sp + SCRATCH_OFF], r_RESULT
104+
retl
105+
nop
106+
107+
#define bpf_negative_common(LEN) \
108+
save %sp, -SAVE_SZ, %sp; \
109+
mov %i0, %o0; \
110+
mov %i1, %o1; \
111+
SIGN_EXTEND(%o1); \
112+
call bpf_internal_load_pointer_neg_helper; \
113+
mov (LEN), %o2; \
114+
mov %o0, r_TMP; \
115+
cmp %o0, 0; \
116+
BE_PTR(bpf_error); \
117+
restore;
118+
119+
bpf_slow_path_word_neg:
120+
sethi %hi(SKF_MAX_NEG_OFF), r_TMP
121+
cmp r_OFF, r_TMP
122+
bl bpf_error
123+
nop
124+
.globl bpf_jit_load_word_negative_offset
125+
bpf_jit_load_word_negative_offset:
126+
bpf_negative_common(4)
127+
andcc r_TMP, 3, %g0
128+
bne load_word_unaligned
129+
nop
130+
retl
131+
ld [r_TMP], r_RESULT
132+
133+
bpf_slow_path_half_neg:
134+
sethi %hi(SKF_MAX_NEG_OFF), r_TMP
135+
cmp r_OFF, r_TMP
136+
bl bpf_error
137+
nop
138+
.globl bpf_jit_load_half_negative_offset
139+
bpf_jit_load_half_negative_offset:
140+
bpf_negative_common(2)
141+
andcc r_TMP, 1, %g0
142+
bne load_half_unaligned
143+
nop
144+
retl
145+
lduh [r_TMP], r_RESULT
146+
147+
bpf_slow_path_byte_neg:
148+
sethi %hi(SKF_MAX_NEG_OFF), r_TMP
149+
cmp r_OFF, r_TMP
150+
bl bpf_error
151+
nop
152+
.globl bpf_jit_load_byte_negative_offset
153+
bpf_jit_load_byte_negative_offset:
154+
bpf_negative_common(1)
155+
retl
156+
ldub [r_TMP], r_RESULT
157+
158+
bpf_error:
159+
/* Make the JIT program itself return zero. */
160+
ret
161+
restore %g0, %g0, %o0

‎arch/sparc/net/bpf_jit_comp_32.c

Lines changed: 0 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -17,24 +17,6 @@ static inline bool is_simm13(unsigned int value)
1717
return value + 0x1000 < 0x2000;
1818
}
1919

20-
static void bpf_flush_icache(void *start_, void *end_)
21-
{
22-
#ifdef CONFIG_SPARC64
23-
/* Cheetah's I-cache is fully coherent. */
24-
if (tlb_type == spitfire) {
25-
unsigned long start = (unsigned long) start_;
26-
unsigned long end = (unsigned long) end_;
27-
28-
start &= ~7UL;
29-
end = (end + 7UL) & ~7UL;
30-
while (start < end) {
31-
flushi(start);
32-
start += 32;
33-
}
34-
}
35-
#endif
36-
}
37-
3820
#define SEEN_DATAREF 1 /* might call external helpers */
3921
#define SEEN_XREG 2 /* ebx is used */
4022
#define SEEN_MEM 4 /* use mem[] for temporary storage */
@@ -82,11 +64,7 @@ static void bpf_flush_icache(void *start_, void *end_)
8264
#define BE (F2(0, 2) | CONDE)
8365
#define BNE (F2(0, 2) | CONDNE)
8466

85-
#ifdef CONFIG_SPARC64
86-
#define BE_PTR (F2(0, 1) | CONDE | (2 << 20))
87-
#else
8867
#define BE_PTR BE
89-
#endif
9068

9169
#define SETHI(K, REG) \
9270
(F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff))
@@ -116,13 +94,8 @@ static void bpf_flush_icache(void *start_, void *end_)
11694
#define LD64 F3(3, 0x0b)
11795
#define ST32 F3(3, 0x04)
11896

119-
#ifdef CONFIG_SPARC64
120-
#define LDPTR LD64
121-
#define BASE_STACKFRAME 176
122-
#else
12397
#define LDPTR LD32
12498
#define BASE_STACKFRAME 96
125-
#endif
12699

127100
#define LD32I (LD32 | IMMED)
128101
#define LD8I (LD8 | IMMED)
@@ -234,11 +207,7 @@ do { BUILD_BUG_ON(FIELD_SIZEOF(STRUCT, FIELD) != sizeof(u8)); \
234207
__emit_load8(BASE, STRUCT, FIELD, DEST); \
235208
} while (0)
236209

237-
#ifdef CONFIG_SPARC64
238-
#define BIAS (STACK_BIAS - 4)
239-
#else
240210
#define BIAS (-4)
241-
#endif
242211

243212
#define emit_ldmem(OFF, DEST) \
244213
do { *prog++ = LD32I | RS1(SP) | S13(BIAS - (OFF)) | RD(DEST); \
@@ -249,13 +218,8 @@ do { *prog++ = ST32I | RS1(SP) | S13(BIAS - (OFF)) | RD(SRC); \
249218
} while (0)
250219

251220
#ifdef CONFIG_SMP
252-
#ifdef CONFIG_SPARC64
253-
#define emit_load_cpu(REG) \
254-
emit_load16(G6, struct thread_info, cpu, REG)
255-
#else
256221
#define emit_load_cpu(REG) \
257222
emit_load32(G6, struct thread_info, cpu, REG)
258-
#endif
259223
#else
260224
#define emit_load_cpu(REG) emit_clear(REG)
261225
#endif
@@ -486,47 +450,35 @@ void bpf_jit_compile(struct bpf_prog *fp)
486450
if (K == 1)
487451
break;
488452
emit_write_y(G0);
489-
#ifdef CONFIG_SPARC32
490453
/* The Sparc v8 architecture requires
491454
* three instructions between a %y
492455
* register write and the first use.
493456
*/
494457
emit_nop();
495458
emit_nop();
496459
emit_nop();
497-
#endif
498460
emit_alu_K(DIV, K);
499461
break;
500462
case BPF_ALU | BPF_DIV | BPF_X: /* A /= X; */
501463
emit_cmpi(r_X, 0);
502464
if (pc_ret0 > 0) {
503465
t_offset = addrs[pc_ret0 - 1];
504-
#ifdef CONFIG_SPARC32
505466
emit_branch(BE, t_offset + 20);
506-
#else
507-
emit_branch(BE, t_offset + 8);
508-
#endif
509467
emit_nop(); /* delay slot */
510468
} else {
511469
emit_branch_off(BNE, 16);
512470
emit_nop();
513-
#ifdef CONFIG_SPARC32
514471
emit_jump(cleanup_addr + 20);
515-
#else
516-
emit_jump(cleanup_addr + 8);
517-
#endif
518472
emit_clear(r_A);
519473
}
520474
emit_write_y(G0);
521-
#ifdef CONFIG_SPARC32
522475
/* The Sparc v8 architecture requires
523476
* three instructions between a %y
524477
* register write and the first use.
525478
*/
526479
emit_nop();
527480
emit_nop();
528481
emit_nop();
529-
#endif
530482
emit_alu_X(DIV);
531483
break;
532484
case BPF_ALU | BPF_NEG:
@@ -797,7 +749,6 @@ cond_branch: f_offset = addrs[i + filter[i].jf];
797749
bpf_jit_dump(flen, proglen, pass + 1, image);
798750

799751
if (image) {
800-
bpf_flush_icache(image, image + proglen);
801752
fp->bpf_func = (void *)image;
802753
fp->jited = 1;
803754
}

0 commit comments

Comments
 (0)