Skip to content

Commit dfa50be

Browse files
author
Ben Hutchings
committed
sfc: Implement firmware-assisted TSO for EF10
Segmentation remains in the driver, but we generate option descriptors describing the required packet editing rather than making our own copies. Reduce tso_state::ipv4_id to 16 bits, so it doesn't overflow into the TCP_FLAGS field of the option descriptor. Signed-off-by: Ben Hutchings <bhutchings@solarflare.com>
1 parent c78c39e commit dfa50be

File tree

1 file changed

+121
-52
lines changed
  • drivers/net/ethernet/sfc

1 file changed

+121
-52
lines changed

‎drivers/net/ethernet/sfc/tx.c

Lines changed: 121 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include "efx.h"
2222
#include "nic.h"
2323
#include "workarounds.h"
24+
#include "ef10_regs.h"
2425

2526
static void efx_dequeue_buffer(struct efx_tx_queue *tx_queue,
2627
struct efx_tx_buffer *buffer,
@@ -83,8 +84,10 @@ unsigned int efx_tx_max_skb_descs(struct efx_nic *efx)
8384
*/
8485
unsigned int max_descs = EFX_TSO_MAX_SEGS * 2 + MAX_SKB_FRAGS;
8586

86-
/* Possibly one more per segment for the alignment workaround */
87-
if (EFX_WORKAROUND_5391(efx))
87+
/* Possibly one more per segment for the alignment workaround,
88+
* or for option descriptors
89+
*/
90+
if (EFX_WORKAROUND_5391(efx) || efx_nic_rev(efx) >= EFX_REV_HUNT_A0)
8891
max_descs += EFX_TSO_MAX_SEGS;
8992

9093
/* Possibly more for PCIe page boundaries within input fragments */
@@ -628,6 +631,9 @@ void efx_remove_tx_queue(struct efx_tx_queue *tx_queue)
628631
* @tcp_off: Offset of TCP header
629632
* @header_len: Number of bytes of header
630633
* @ip_base_len: IPv4 tot_len or IPv6 payload_len, before TCP payload
634+
* @header_dma_addr: Header DMA address, when using option descriptors
635+
* @header_unmap_len: Header DMA mapped length, or 0 if not using option
636+
* descriptors
631637
*
632638
* The state used during segmentation. It is put into this data structure
633639
* just to make it easy to pass into inline functions.
@@ -636,7 +642,7 @@ struct tso_state {
636642
/* Output position */
637643
unsigned out_len;
638644
unsigned seqnum;
639-
unsigned ipv4_id;
645+
u16 ipv4_id;
640646
unsigned packet_space;
641647

642648
/* Input position */
@@ -651,6 +657,8 @@ struct tso_state {
651657
unsigned int tcp_off;
652658
unsigned header_len;
653659
unsigned int ip_base_len;
660+
dma_addr_t header_dma_addr;
661+
unsigned int header_unmap_len;
654662
};
655663

656664

@@ -825,7 +833,10 @@ static void efx_enqueue_unwind(struct efx_tx_queue *tx_queue)
825833
static int tso_start(struct tso_state *st, struct efx_nic *efx,
826834
const struct sk_buff *skb)
827835
{
836+
bool use_options = efx_nic_rev(efx) >= EFX_REV_HUNT_A0;
837+
struct device *dma_dev = &efx->pci_dev->dev;
828838
unsigned int header_len, in_len;
839+
dma_addr_t dma_addr;
829840

830841
st->ip_off = skb_network_header(skb) - skb->data;
831842
st->tcp_off = skb_transport_header(skb) - skb->data;
@@ -848,22 +859,32 @@ static int tso_start(struct tso_state *st, struct efx_nic *efx,
848859

849860
st->out_len = skb->len - header_len;
850861

851-
if (likely(in_len == 0)) {
852-
st->unmap_len = 0;
862+
if (!use_options) {
863+
st->header_unmap_len = 0;
864+
865+
if (likely(in_len == 0)) {
866+
st->dma_flags = 0;
867+
st->unmap_len = 0;
868+
return 0;
869+
}
870+
871+
dma_addr = dma_map_single(dma_dev, skb->data + header_len,
872+
in_len, DMA_TO_DEVICE);
873+
st->dma_flags = EFX_TX_BUF_MAP_SINGLE;
874+
st->dma_addr = dma_addr;
875+
st->unmap_addr = dma_addr;
876+
st->unmap_len = in_len;
877+
} else {
878+
dma_addr = dma_map_single(dma_dev, skb->data,
879+
skb_headlen(skb), DMA_TO_DEVICE);
880+
st->header_dma_addr = dma_addr;
881+
st->header_unmap_len = skb_headlen(skb);
853882
st->dma_flags = 0;
854-
return 0;
883+
st->dma_addr = dma_addr + header_len;
884+
st->unmap_len = 0;
855885
}
856886

857-
st->unmap_addr = dma_map_single(&efx->pci_dev->dev,
858-
skb->data + header_len, in_len,
859-
DMA_TO_DEVICE);
860-
if (unlikely(dma_mapping_error(&efx->pci_dev->dev, st->unmap_addr)))
861-
return -ENOMEM;
862-
863-
st->dma_flags = EFX_TX_BUF_MAP_SINGLE;
864-
st->unmap_len = in_len;
865-
st->dma_addr = st->unmap_addr;
866-
return 0;
887+
return unlikely(dma_mapping_error(dma_dev, dma_addr)) ? -ENOMEM : 0;
867888
}
868889

869890
static int tso_get_fragment(struct tso_state *st, struct efx_nic *efx,
@@ -948,54 +969,97 @@ static int tso_start_new_packet(struct efx_tx_queue *tx_queue,
948969
{
949970
struct efx_tx_buffer *buffer =
950971
&tx_queue->buffer[tx_queue->insert_count & tx_queue->ptr_mask];
951-
struct tcphdr *tsoh_th;
952-
unsigned ip_length;
953-
u8 *header;
954-
int rc;
955-
956-
/* Allocate and insert a DMA-mapped header buffer. */
957-
header = efx_tsoh_get_buffer(tx_queue, buffer, st->header_len);
958-
if (!header)
959-
return -ENOMEM;
972+
bool is_last = st->out_len <= skb_shinfo(skb)->gso_size;
973+
u8 tcp_flags_clear;
960974

961-
tsoh_th = (struct tcphdr *)(header + st->tcp_off);
962-
963-
/* Copy and update the headers. */
964-
memcpy(header, skb->data, st->header_len);
965-
966-
tsoh_th->seq = htonl(st->seqnum);
967-
st->seqnum += skb_shinfo(skb)->gso_size;
968-
if (st->out_len > skb_shinfo(skb)->gso_size) {
969-
/* This packet will not finish the TSO burst. */
975+
if (!is_last) {
970976
st->packet_space = skb_shinfo(skb)->gso_size;
971-
tsoh_th->fin = 0;
972-
tsoh_th->psh = 0;
977+
tcp_flags_clear = 0x09; /* mask out FIN and PSH */
973978
} else {
974-
/* This packet will be the last in the TSO burst. */
975979
st->packet_space = st->out_len;
976-
tsoh_th->fin = tcp_hdr(skb)->fin;
977-
tsoh_th->psh = tcp_hdr(skb)->psh;
980+
tcp_flags_clear = 0x00;
978981
}
979-
ip_length = st->ip_base_len + st->packet_space;
980982

981-
if (st->protocol == htons(ETH_P_IP)) {
982-
struct iphdr *tsoh_iph = (struct iphdr *)(header + st->ip_off);
983+
if (!st->header_unmap_len) {
984+
/* Allocate and insert a DMA-mapped header buffer. */
985+
struct tcphdr *tsoh_th;
986+
unsigned ip_length;
987+
u8 *header;
988+
int rc;
983989

984-
tsoh_iph->tot_len = htons(ip_length);
990+
header = efx_tsoh_get_buffer(tx_queue, buffer, st->header_len);
991+
if (!header)
992+
return -ENOMEM;
985993

986-
/* Linux leaves suitable gaps in the IP ID space for us to fill. */
987-
tsoh_iph->id = htons(st->ipv4_id);
988-
st->ipv4_id++;
994+
tsoh_th = (struct tcphdr *)(header + st->tcp_off);
995+
996+
/* Copy and update the headers. */
997+
memcpy(header, skb->data, st->header_len);
998+
999+
tsoh_th->seq = htonl(st->seqnum);
1000+
((u8 *)tsoh_th)[13] &= ~tcp_flags_clear;
1001+
1002+
ip_length = st->ip_base_len + st->packet_space;
1003+
1004+
if (st->protocol == htons(ETH_P_IP)) {
1005+
struct iphdr *tsoh_iph =
1006+
(struct iphdr *)(header + st->ip_off);
1007+
1008+
tsoh_iph->tot_len = htons(ip_length);
1009+
tsoh_iph->id = htons(st->ipv4_id);
1010+
} else {
1011+
struct ipv6hdr *tsoh_iph =
1012+
(struct ipv6hdr *)(header + st->ip_off);
1013+
1014+
tsoh_iph->payload_len = htons(ip_length);
1015+
}
1016+
1017+
rc = efx_tso_put_header(tx_queue, buffer, header);
1018+
if (unlikely(rc))
1019+
return rc;
9891020
} else {
990-
struct ipv6hdr *tsoh_iph =
991-
(struct ipv6hdr *)(header + st->ip_off);
1021+
/* Send the original headers with a TSO option descriptor
1022+
* in front
1023+
*/
1024+
u8 tcp_flags = ((u8 *)tcp_hdr(skb))[13] & ~tcp_flags_clear;
1025+
1026+
buffer->flags = EFX_TX_BUF_OPTION;
1027+
buffer->len = 0;
1028+
buffer->unmap_len = 0;
1029+
EFX_POPULATE_QWORD_5(buffer->option,
1030+
ESF_DZ_TX_DESC_IS_OPT, 1,
1031+
ESF_DZ_TX_OPTION_TYPE,
1032+
ESE_DZ_TX_OPTION_DESC_TSO,
1033+
ESF_DZ_TX_TSO_TCP_FLAGS, tcp_flags,
1034+
ESF_DZ_TX_TSO_IP_ID, st->ipv4_id,
1035+
ESF_DZ_TX_TSO_TCP_SEQNO, st->seqnum);
1036+
++tx_queue->insert_count;
9921037

993-
tsoh_iph->payload_len = htons(ip_length);
1038+
/* We mapped the headers in tso_start(). Unmap them
1039+
* when the last segment is completed.
1040+
*/
1041+
buffer = &tx_queue->buffer[tx_queue->insert_count &
1042+
tx_queue->ptr_mask];
1043+
buffer->dma_addr = st->header_dma_addr;
1044+
buffer->len = st->header_len;
1045+
if (is_last) {
1046+
buffer->flags = EFX_TX_BUF_CONT | EFX_TX_BUF_MAP_SINGLE;
1047+
buffer->unmap_len = st->header_unmap_len;
1048+
/* Ensure we only unmap them once in case of a
1049+
* later DMA mapping error and rollback
1050+
*/
1051+
st->header_unmap_len = 0;
1052+
} else {
1053+
buffer->flags = EFX_TX_BUF_CONT;
1054+
buffer->unmap_len = 0;
1055+
}
1056+
++tx_queue->insert_count;
9941057
}
9951058

996-
rc = efx_tso_put_header(tx_queue, buffer, header);
997-
if (unlikely(rc))
998-
return rc;
1059+
st->seqnum += skb_shinfo(skb)->gso_size;
1060+
1061+
/* Linux leaves suitable gaps in the IP ID space for us to fill. */
1062+
++st->ipv4_id;
9991063

10001064
++tx_queue->tso_packets;
10011065

@@ -1091,6 +1155,11 @@ static int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue,
10911155
state.unmap_len, DMA_TO_DEVICE);
10921156
}
10931157

1158+
/* Free the header DMA mapping, if using option descriptors */
1159+
if (state.header_unmap_len)
1160+
dma_unmap_single(&efx->pci_dev->dev, state.header_dma_addr,
1161+
state.header_unmap_len, DMA_TO_DEVICE);
1162+
10941163
efx_enqueue_unwind(tx_queue);
10951164
return NETDEV_TX_OK;
10961165
}

0 commit comments

Comments
 (0)