Skip to content

Commit 7dbea3e

Browse files
davem330davem330
committed
Merge branch 'napi_page_frags'
Alexander Duyck says: ==================== net: Alloc NAPI page frags from their own pool This patch series implements a means of allocating page fragments without the need for the local_irq_save/restore in __netdev_alloc_frag. By doing this I am able to decrease packet processing time by 11ns per packet in my test environment. ==================== Signed-off-by: David S. Miller <davem@davemloft.net>
2 parents 6e5f59a + 45abfb1 commit 7dbea3e

File tree

16 files changed

+178
-70
lines changed

16 files changed

+178
-70
lines changed

‎drivers/net/ethernet/broadcom/b44.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -836,7 +836,7 @@ static int b44_rx(struct b44 *bp, int budget)
836836
struct sk_buff *copy_skb;
837837

838838
b44_recycle_rx(bp, cons, bp->rx_prod);
839-
copy_skb = netdev_alloc_skb_ip_align(bp->dev, len);
839+
copy_skb = napi_alloc_skb(&bp->napi, len);
840840
if (copy_skb == NULL)
841841
goto drop_it_no_recycle;
842842

‎drivers/net/ethernet/broadcom/bcm63xx_enet.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -385,7 +385,7 @@ static int bcm_enet_receive_queue(struct net_device *dev, int budget)
385385
if (len < copybreak) {
386386
struct sk_buff *nskb;
387387

388-
nskb = netdev_alloc_skb_ip_align(dev, len);
388+
nskb = napi_alloc_skb(&priv->napi, len);
389389
if (!nskb) {
390390
/* forget packet, just rearm desc */
391391
dev->stats.rx_dropped++;

‎drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1015,7 +1015,7 @@ static int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
10151015
*/
10161016
if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
10171017
(len <= RX_COPY_THRESH)) {
1018-
skb = netdev_alloc_skb_ip_align(bp->dev, len);
1018+
skb = napi_alloc_skb(&fp->napi, len);
10191019
if (skb == NULL) {
10201020
DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
10211021
"ERROR packet dropped because of alloc failure\n");

‎drivers/net/ethernet/chelsio/cxgb/sge.c

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1025,7 +1025,7 @@ MODULE_PARM_DESC(copybreak, "Receive copy threshold");
10251025

10261026
/**
10271027
* get_packet - return the next ingress packet buffer
1028-
* @pdev: the PCI device that received the packet
1028+
* @adapter: the adapter that received the packet
10291029
* @fl: the SGE free list holding the packet
10301030
* @len: the actual packet length, excluding any SGE padding
10311031
*
@@ -1037,14 +1037,15 @@ MODULE_PARM_DESC(copybreak, "Receive copy threshold");
10371037
* threshold and the packet is too big to copy, or (b) the packet should
10381038
* be copied but there is no memory for the copy.
10391039
*/
1040-
static inline struct sk_buff *get_packet(struct pci_dev *pdev,
1040+
static inline struct sk_buff *get_packet(struct adapter *adapter,
10411041
struct freelQ *fl, unsigned int len)
10421042
{
1043-
struct sk_buff *skb;
10441043
const struct freelQ_ce *ce = &fl->centries[fl->cidx];
1044+
struct pci_dev *pdev = adapter->pdev;
1045+
struct sk_buff *skb;
10451046

10461047
if (len < copybreak) {
1047-
skb = netdev_alloc_skb_ip_align(NULL, len);
1048+
skb = napi_alloc_skb(&adapter->napi, len);
10481049
if (!skb)
10491050
goto use_orig_buf;
10501051

@@ -1357,7 +1358,7 @@ static void sge_rx(struct sge *sge, struct freelQ *fl, unsigned int len)
13571358
struct sge_port_stats *st;
13581359
struct net_device *dev;
13591360

1360-
skb = get_packet(adapter->pdev, fl, len - sge->rx_pkt_pad);
1361+
skb = get_packet(adapter, fl, len - sge->rx_pkt_pad);
13611362
if (unlikely(!skb)) {
13621363
sge->stats.rx_drops++;
13631364
return;

‎drivers/net/ethernet/intel/e1000/e1000_main.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4100,7 +4100,7 @@ static bool e1000_tbi_should_accept(struct e1000_adapter *adapter,
41004100
static struct sk_buff *e1000_alloc_rx_skb(struct e1000_adapter *adapter,
41014101
unsigned int bufsz)
41024102
{
4103-
struct sk_buff *skb = netdev_alloc_skb_ip_align(adapter->netdev, bufsz);
4103+
struct sk_buff *skb = napi_alloc_skb(&adapter->napi, bufsz);
41044104

41054105
if (unlikely(!skb))
41064106
adapter->alloc_rx_buff_failed++;

‎drivers/net/ethernet/intel/e1000e/netdev.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1016,7 +1016,7 @@ static bool e1000_clean_rx_irq(struct e1000_ring *rx_ring, int *work_done,
10161016
*/
10171017
if (length < copybreak) {
10181018
struct sk_buff *new_skb =
1019-
netdev_alloc_skb_ip_align(netdev, length);
1019+
napi_alloc_skb(&adapter->napi, length);
10201020
if (new_skb) {
10211021
skb_copy_to_linear_data_offset(new_skb,
10221022
-NET_IP_ALIGN,

‎drivers/net/ethernet/intel/fm10k/fm10k_main.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -308,8 +308,8 @@ static struct sk_buff *fm10k_fetch_rx_buffer(struct fm10k_ring *rx_ring,
308308
#endif
309309

310310
/* allocate a skb to store the frags */
311-
skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
312-
FM10K_RX_HDR_LEN);
311+
skb = napi_alloc_skb(&rx_ring->q_vector->napi,
312+
FM10K_RX_HDR_LEN);
313313
if (unlikely(!skb)) {
314314
rx_ring->rx_stats.alloc_failed++;
315315
return NULL;

‎drivers/net/ethernet/intel/igb/igb_main.c

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6644,8 +6644,7 @@ static struct sk_buff *igb_fetch_rx_buffer(struct igb_ring *rx_ring,
66446644
#endif
66456645

66466646
/* allocate a skb to store the frags */
6647-
skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
6648-
IGB_RX_HDR_LEN);
6647+
skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGB_RX_HDR_LEN);
66496648
if (unlikely(!skb)) {
66506649
rx_ring->rx_stats.alloc_failed++;
66516650
return NULL;

‎drivers/net/ethernet/intel/ixgb/ixgb_main.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1963,7 +1963,7 @@ ixgb_rx_checksum(struct ixgb_adapter *adapter,
19631963
* this should improve performance for small packets with large amounts
19641964
* of reassembly being done in the stack
19651965
*/
1966-
static void ixgb_check_copybreak(struct net_device *netdev,
1966+
static void ixgb_check_copybreak(struct napi_struct *napi,
19671967
struct ixgb_buffer *buffer_info,
19681968
u32 length, struct sk_buff **skb)
19691969
{
@@ -1972,7 +1972,7 @@ static void ixgb_check_copybreak(struct net_device *netdev,
19721972
if (length > copybreak)
19731973
return;
19741974

1975-
new_skb = netdev_alloc_skb_ip_align(netdev, length);
1975+
new_skb = napi_alloc_skb(napi, length);
19761976
if (!new_skb)
19771977
return;
19781978

@@ -2064,7 +2064,7 @@ ixgb_clean_rx_irq(struct ixgb_adapter *adapter, int *work_done, int work_to_do)
20642064
goto rxdesc_done;
20652065
}
20662066

2067-
ixgb_check_copybreak(netdev, buffer_info, length, &skb);
2067+
ixgb_check_copybreak(&adapter->napi, buffer_info, length, &skb);
20682068

20692069
/* Good Receive */
20702070
skb_put(skb, length);

‎drivers/net/ethernet/intel/ixgbe/ixgbe_main.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1913,8 +1913,8 @@ static struct sk_buff *ixgbe_fetch_rx_buffer(struct ixgbe_ring *rx_ring,
19131913
#endif
19141914

19151915
/* allocate a skb to store the frags */
1916-
skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
1917-
IXGBE_RX_HDR_SIZE);
1916+
skb = napi_alloc_skb(&rx_ring->q_vector->napi,
1917+
IXGBE_RX_HDR_SIZE);
19181918
if (unlikely(!skb)) {
19191919
rx_ring->rx_stats.alloc_rx_buff_failed++;
19201920
return NULL;

‎drivers/net/ethernet/realtek/8139cp.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -507,7 +507,7 @@ static int cp_rx_poll(struct napi_struct *napi, int budget)
507507
netif_dbg(cp, rx_status, dev, "rx slot %d status 0x%x len %d\n",
508508
rx_tail, status, len);
509509

510-
new_skb = netdev_alloc_skb_ip_align(dev, buflen);
510+
new_skb = napi_alloc_skb(napi, buflen);
511511
if (!new_skb) {
512512
dev->stats.rx_dropped++;
513513
goto rx_next;

‎drivers/net/ethernet/realtek/8139too.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2037,7 +2037,7 @@ static int rtl8139_rx(struct net_device *dev, struct rtl8139_private *tp,
20372037
/* Malloc up new buffer, compatible with net-2e. */
20382038
/* Omit the four octet CRC from the length. */
20392039

2040-
skb = netdev_alloc_skb_ip_align(dev, pkt_size);
2040+
skb = napi_alloc_skb(&tp->napi, pkt_size);
20412041
if (likely(skb)) {
20422042
#if RX_BUF_IDX == 3
20432043
wrap_copy(skb, rx_ring, ring_offset+4, pkt_size);

‎drivers/net/ethernet/realtek/r8169.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7260,7 +7260,7 @@ static struct sk_buff *rtl8169_try_rx_copy(void *data,
72607260
data = rtl8169_align(data);
72617261
dma_sync_single_for_cpu(d, addr, pkt_size, DMA_FROM_DEVICE);
72627262
prefetch(data);
7263-
skb = netdev_alloc_skb_ip_align(tp->dev, pkt_size);
7263+
skb = napi_alloc_skb(&tp->napi, pkt_size);
72647264
if (skb)
72657265
memcpy(skb->data, data, pkt_size);
72667266
dma_sync_single_for_device(d, addr, pkt_size, DMA_FROM_DEVICE);

‎include/linux/skbuff.h

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@ struct net_device;
151151
struct scatterlist;
152152
struct pipe_inode_info;
153153
struct iov_iter;
154+
struct napi_struct;
154155

155156
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
156157
struct nf_conntrack {
@@ -673,6 +674,7 @@ struct sk_buff {
673674

674675
#define SKB_ALLOC_FCLONE 0x01
675676
#define SKB_ALLOC_RX 0x02
677+
#define SKB_ALLOC_NAPI 0x04
676678

677679
/* Returns true if the skb was allocated from PFMEMALLOC reserves */
678680
static inline bool skb_pfmemalloc(const struct sk_buff *skb)
@@ -2164,6 +2166,15 @@ static inline struct sk_buff *netdev_alloc_skb_ip_align(struct net_device *dev,
21642166
return __netdev_alloc_skb_ip_align(dev, length, GFP_ATOMIC);
21652167
}
21662168

2169+
void *napi_alloc_frag(unsigned int fragsz);
2170+
struct sk_buff *__napi_alloc_skb(struct napi_struct *napi,
2171+
unsigned int length, gfp_t gfp_mask);
2172+
static inline struct sk_buff *napi_alloc_skb(struct napi_struct *napi,
2173+
unsigned int length)
2174+
{
2175+
return __napi_alloc_skb(napi, length, GFP_ATOMIC);
2176+
}
2177+
21672178
/**
21682179
* __dev_alloc_pages - allocate page for network Rx
21692180
* @gfp_mask: allocation priority. Set __GFP_NOMEMALLOC if not for network Rx

‎net/core/dev.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4172,7 +4172,7 @@ struct sk_buff *napi_get_frags(struct napi_struct *napi)
41724172
struct sk_buff *skb = napi->skb;
41734173

41744174
if (!skb) {
4175-
skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
4175+
skb = napi_alloc_skb(napi, GRO_MAX_HEAD);
41764176
napi->skb = skb;
41774177
}
41784178
return skb;

0 commit comments

Comments
 (0)