Skip to content

Commit a263b30

Browse files
davem330davem330
committed
ipv4: Make neigh lookups directly in output packet path.
Do not use the dst cached neigh, we'll be getting rid of that. Signed-off-by: David S. Miller <davem@davemloft.net>
1 parent 1160472 commit a263b30

File tree

5 files changed

+44
-25
lines changed

5 files changed

+44
-25
lines changed

‎include/net/arp.h

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,24 +15,34 @@ static inline u32 arp_hashfn(u32 key, const struct net_device *dev, u32 hash_rnd
1515
return val * hash_rnd;
1616
}
1717

18-
static inline struct neighbour *__ipv4_neigh_lookup(struct net_device *dev, u32 key)
18+
static inline struct neighbour *__ipv4_neigh_lookup_noref(struct net_device *dev, u32 key)
1919
{
20-
struct neigh_hash_table *nht;
20+
struct neigh_hash_table *nht = rcu_dereference_bh(arp_tbl.nht);
2121
struct neighbour *n;
2222
u32 hash_val;
2323

24-
rcu_read_lock_bh();
25-
nht = rcu_dereference_bh(arp_tbl.nht);
24+
if (dev->flags & (IFF_LOOPBACK | IFF_POINTOPOINT))
25+
key = 0;
26+
2627
hash_val = arp_hashfn(key, dev, nht->hash_rnd[0]) >> (32 - nht->hash_shift);
2728
for (n = rcu_dereference_bh(nht->hash_buckets[hash_val]);
2829
n != NULL;
2930
n = rcu_dereference_bh(n->next)) {
30-
if (n->dev == dev && *(u32 *)n->primary_key == key) {
31-
if (!atomic_inc_not_zero(&n->refcnt))
32-
n = NULL;
33-
break;
34-
}
31+
if (n->dev == dev && *(u32 *)n->primary_key == key)
32+
return n;
3533
}
34+
35+
return NULL;
36+
}
37+
38+
static inline struct neighbour *__ipv4_neigh_lookup(struct net_device *dev, u32 key)
39+
{
40+
struct neighbour *n;
41+
42+
rcu_read_lock_bh();
43+
n = __ipv4_neigh_lookup_noref(dev, key);
44+
if (n && !atomic_inc_not_zero(&n->refcnt))
45+
n = NULL;
3646
rcu_read_unlock_bh();
3747

3848
return n;

‎include/net/neighbour.h

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -202,9 +202,16 @@ extern struct neighbour * neigh_lookup(struct neigh_table *tbl,
202202
extern struct neighbour * neigh_lookup_nodev(struct neigh_table *tbl,
203203
struct net *net,
204204
const void *pkey);
205-
extern struct neighbour * neigh_create(struct neigh_table *tbl,
205+
extern struct neighbour * __neigh_create(struct neigh_table *tbl,
206+
const void *pkey,
207+
struct net_device *dev,
208+
bool want_ref);
209+
static inline struct neighbour *neigh_create(struct neigh_table *tbl,
206210
const void *pkey,
207-
struct net_device *dev);
211+
struct net_device *dev)
212+
{
213+
return __neigh_create(tbl, pkey, dev, true);
214+
}
208215
extern void neigh_destroy(struct neighbour *neigh);
209216
extern int __neigh_event_send(struct neighbour *neigh, struct sk_buff *skb);
210217
extern int neigh_update(struct neighbour *neigh, const u8 *lladdr, u8 new,

‎net/core/neighbour.c

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -474,8 +474,8 @@ struct neighbour *neigh_lookup_nodev(struct neigh_table *tbl, struct net *net,
474474
}
475475
EXPORT_SYMBOL(neigh_lookup_nodev);
476476

477-
struct neighbour *neigh_create(struct neigh_table *tbl, const void *pkey,
478-
struct net_device *dev)
477+
struct neighbour *__neigh_create(struct neigh_table *tbl, const void *pkey,
478+
struct net_device *dev, bool want_ref)
479479
{
480480
u32 hash_val;
481481
int key_len = tbl->key_len;
@@ -535,14 +535,16 @@ struct neighbour *neigh_create(struct neigh_table *tbl, const void *pkey,
535535
n1 = rcu_dereference_protected(n1->next,
536536
lockdep_is_held(&tbl->lock))) {
537537
if (dev == n1->dev && !memcmp(n1->primary_key, pkey, key_len)) {
538-
neigh_hold(n1);
538+
if (want_ref)
539+
neigh_hold(n1);
539540
rc = n1;
540541
goto out_tbl_unlock;
541542
}
542543
}
543544

544545
n->dead = 0;
545-
neigh_hold(n);
546+
if (want_ref)
547+
neigh_hold(n);
546548
rcu_assign_pointer(n->next,
547549
rcu_dereference_protected(nht->hash_buckets[hash_val],
548550
lockdep_is_held(&tbl->lock)));
@@ -558,7 +560,7 @@ struct neighbour *neigh_create(struct neigh_table *tbl, const void *pkey,
558560
neigh_release(n);
559561
goto out;
560562
}
561-
EXPORT_SYMBOL(neigh_create);
563+
EXPORT_SYMBOL(__neigh_create);
562564

563565
static u32 pneigh_hash(const void *pkey, int key_len)
564566
{

‎net/ipv4/ip_output.c

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,7 @@ static inline int ip_finish_output2(struct sk_buff *skb)
170170
struct net_device *dev = dst->dev;
171171
unsigned int hh_len = LL_RESERVED_SPACE(dev);
172172
struct neighbour *neigh;
173+
u32 nexthop;
173174

174175
if (rt->rt_type == RTN_MULTICAST) {
175176
IP_UPD_PO_STATS(dev_net(dev), IPSTATS_MIB_OUTMCAST, skb->len);
@@ -191,15 +192,18 @@ static inline int ip_finish_output2(struct sk_buff *skb)
191192
skb = skb2;
192193
}
193194

194-
rcu_read_lock();
195-
neigh = dst_get_neighbour_noref(dst);
195+
rcu_read_lock_bh();
196+
nexthop = rt->rt_gateway ? rt->rt_gateway : ip_hdr(skb)->daddr;
197+
neigh = __ipv4_neigh_lookup_noref(dev, nexthop);
198+
if (unlikely(!neigh))
199+
neigh = __neigh_create(&arp_tbl, &nexthop, dev, false);
196200
if (neigh) {
197201
int res = neigh_output(neigh, skb);
198202

199-
rcu_read_unlock();
203+
rcu_read_unlock_bh();
200204
return res;
201205
}
202-
rcu_read_unlock();
206+
rcu_read_unlock_bh();
203207

204208
net_dbg_ratelimited("%s: No header cache and no neighbour!\n",
205209
__func__);

‎net/ipv4/route.c

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1098,17 +1098,13 @@ static int slow_chain_length(const struct rtable *head)
10981098

10991099
static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst, const void *daddr)
11001100
{
1101-
static const __be32 inaddr_any = 0;
11021101
struct net_device *dev = dst->dev;
11031102
const __be32 *pkey = daddr;
11041103
const struct rtable *rt;
11051104
struct neighbour *n;
11061105

11071106
rt = (const struct rtable *) dst;
1108-
1109-
if (dev->flags & (IFF_LOOPBACK | IFF_POINTOPOINT))
1110-
pkey = &inaddr_any;
1111-
else if (rt->rt_gateway)
1107+
if (rt->rt_gateway)
11121108
pkey = (const __be32 *) &rt->rt_gateway;
11131109

11141110
n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);

0 commit comments

Comments
 (0)