Line data Source code
1 : /*
2 : * Copyright (C) 2016 Juniper Networks.
3 : *
4 : * This program is free software; you can redistribute it and/or
5 : * modify it under the terms of the GNU General Public License as
6 : * published by the Free Software Foundation version 2.
7 : *
8 : * This program is distributed "as is" WITHOUT ANY WARRANTY of any
9 : * kind, whether express or implied; without even the implied warranty
10 : * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 : * GNU General Public License for more details.
12 : *
13 : * vr_dpdk_gso.c -- TCP Offloads on the sender
14 : *
15 : * This is an adaptation of FreeBSD's patch for gso
16 : * Copyright (C) 2014, Stefano Garzarella - Universita` di Pisa
17 : * All rights reserved.
18 : * BSD LICENSE
19 : *
20 : *
21 : */
22 :
23 : #include "vr_dpdk.h"
24 : #include "vr_dpdk_netlink.h"
25 : #include "vr_dpdk_usocket.h"
26 : #include "vr_dpdk_virtio.h"
27 : #include "vr_dpdk_gro.h"
28 : #include "vr_packet.h"
29 : #include "vr_datapath.h"
30 :
31 : #include <netinet/ip.h>
32 : #include <netinet/tcp.h>
33 :
34 : #include <rte_errno.h>
35 : #include <rte_ethdev.h>
36 : #include <rte_ip_frag.h>
37 : #include <rte_ip.h>
38 : #include <rte_port_ethdev.h>
39 : #include <rte_hash.h>
40 : #include <rte_cycles.h>
41 : #include <rte_malloc.h>
42 : #include <rte_tcp.h>
43 : #include <rte_eth_bond.h>
44 :
45 : /*
46 : * Structure that contains the state during the TCP segmentation
47 : */
48 : struct dpdk_gso_state {
49 : void (*update)
50 : (struct rte_mbuf*, struct dpdk_gso_state*);
51 : void (*internal)
52 : (struct rte_mbuf*, struct dpdk_gso_state*);
53 : union {
54 : struct rte_ipv4_hdr *ip;
55 : struct rte_ipv6_hdr *ip6;
56 : };
57 : struct rte_tcp_hdr *tcp;
58 : int mac_hlen;
59 : int ip_hlen;
60 : int tcp_hlen;
61 : int hlen;
62 : int pay_len;
63 : uint32_t tcp_seq;
64 : uint16_t ip_id;
65 : uint16_t ip_off;
66 : int tx_cksum_offload;
67 : };
68 :
69 : /**
70 : * Process the IPv6 UDP or TCP checksum in a **chained** mbuf.
71 : *
72 : * Layer 4 checksum must be set to 0 in the packet by the caller.
73 : *
74 : * @param ipv6_hdr
75 : * The pointer to the contiguous IPv4 header.
76 : * @param l4_hdr
77 : * The pointer to the beginning of the L4 header.
78 : * @return
79 : * The complemented checksum to set in the IP packet.
80 : */
81 : inline uint16_t
82 8 : dpdk_ipv6_udptcp_cksum(struct rte_mbuf *m, const struct rte_ipv6_hdr *ipv6_hdr,
83 : uint8_t *l4_hdr)
84 : {
85 8 : uint32_t cksum = 0;
86 : uint32_t l4_len;
87 8 : uint32_t data_len = 0, rem_len = 0;
88 8 : uint8_t *data_ptr = NULL;
89 :
90 8 : l4_len = rte_be_to_cpu_16(ipv6_hdr->payload_len);
91 :
92 : do {
93 8 : data_ptr = likely(!!data_ptr)? rte_pktmbuf_mtod(m, uint8_t*):l4_hdr;
94 8 : data_len = likely(!!data_len)? rte_pktmbuf_data_len(m):
95 8 : rte_pktmbuf_mtod(m, uint8_t*) + rte_pktmbuf_data_len(m) - l4_hdr ;
96 8 : if (rem_len + data_len > l4_len)
97 0 : data_len = l4_len - rem_len;
98 8 : cksum += rte_raw_cksum(data_ptr, data_len);
99 8 : rem_len += data_len;
100 8 : m = m->next;
101 8 : } while (m && rem_len < l4_len);
102 :
103 8 : cksum += rte_ipv6_phdr_cksum(ipv6_hdr, 0);
104 8 : cksum = ((cksum & 0xffff0000) >> 16) + (cksum & 0xffff);
105 8 : cksum = (~cksum) & 0xffff;
106 8 : if (cksum == 0)
107 0 : cksum = 0xffff;
108 :
109 8 : return cksum;
110 : }
111 :
112 : /**
113 : * Process the IPv4 UDP or TCP checksum in a **chained** mbuf.
114 : *
115 : * The IPv4 header should not contains options. The IP and layer 4
116 : * checksum must be set to 0 in the packet by the caller.
117 : *
118 : * @param ipv4_hdr
119 : * The pointer to the contiguous IPv4 header.
120 : * @param l4_hdr
121 : * The pointer to the beginning of the L4 header.
122 : * @return
123 : * The complemented checksum to set in the IP packet.
124 : */
125 : inline uint16_t
126 36 : dpdk_ipv4_udptcp_cksum(struct rte_mbuf *m,
127 : const struct rte_ipv4_hdr *ipv4_hdr,
128 : uint8_t *l4_hdr)
129 : {
130 36 : uint32_t cksum = 0;
131 : uint32_t l4_len;
132 36 : uint32_t data_len = 0, rem_len = 0;
133 36 : uint8_t *data_ptr = NULL;
134 :
135 36 : l4_len = rte_be_to_cpu_16(ipv4_hdr->total_length) -
136 36 : ((ipv4_hdr->version_ihl & 0xf) << 2);
137 :
138 : do {
139 40 : data_ptr = likely(!!data_ptr)? rte_pktmbuf_mtod(m, uint8_t*):l4_hdr;
140 40 : data_len = likely(!!data_len)? rte_pktmbuf_data_len(m):
141 36 : rte_pktmbuf_mtod(m, uint8_t*) + rte_pktmbuf_data_len(m) - l4_hdr ;
142 40 : if (rem_len + data_len > l4_len)
143 0 : data_len = l4_len - rem_len;
144 40 : cksum += rte_raw_cksum(data_ptr, data_len);
145 40 : rem_len += data_len;
146 40 : m = m->next;
147 40 : } while (m && rem_len < l4_len);
148 :
149 36 : cksum += rte_ipv4_phdr_cksum(ipv4_hdr, 0);
150 36 : cksum = ((cksum & 0xffff0000) >> 16) + (cksum & 0xffff);
151 36 : cksum = (~cksum) & 0xffff;
152 36 : if (cksum == 0)
153 0 : cksum = 0xffff;
154 :
155 36 : return cksum;
156 : }
157 :
158 : /* Split chained mbuf to individual mbufs and save the pointers in pkts_out */
159 : static int32_t
160 0 : dpdk_split_chained_mbuf(struct rte_mbuf *pkt_in, struct rte_mbuf **pkts_out,
161 : uint16_t nb_pkts_out, uint16_t hdr_len, uint16_t frag_size)
162 : {
163 0 : struct rte_mbuf *curr_pkt = pkt_in, *next_pkt;
164 0 : uint32_t nb_segs = 0, total_segs = pkt_in->nb_segs, i = 0, j;
165 0 : char *in_hdr = rte_pktmbuf_mtod(pkt_in, char*);
166 : char *pkt_addr;
167 :
168 0 : while (curr_pkt) {
169 0 : next_pkt = curr_pkt->next;
170 0 : curr_pkt->next = NULL;
171 0 : curr_pkt->nb_segs = 1;
172 0 : curr_pkt->pkt_len = curr_pkt->data_len;
173 : /* 1st mbuf in the chain already has all the headers */
174 0 : if (likely(nb_segs > 0)) {
175 0 : pkt_addr = rte_pktmbuf_prepend(curr_pkt, hdr_len);
176 0 : if (unlikely(pkt_addr == NULL)) {
177 : /* pkts_out[0] will be freed in the caller
178 : * when -1 is returned
179 : */
180 0 : for(j=1;j<i;j++)
181 : /* We cannot use dpdk_pfree() since the
182 : * fragments may not have vr_packet info
183 : */
184 0 : rte_pktmbuf_free(pkts_out[j]);
185 0 : rte_pktmbuf_free(curr_pkt);
186 0 : rte_pktmbuf_free(next_pkt);
187 0 : return -1;
188 : }
189 0 : rte_memcpy(pkt_addr, in_hdr, hdr_len);
190 : }
191 0 : pkts_out[nb_segs++] = curr_pkt;
192 0 : if ((nb_segs >= nb_pkts_out) || (i >= total_segs)) {
193 : /* pkts_out[0] will be freed in the caller
194 : * when -1 is returned
195 : */
196 0 : for(j=1;j<(i+1);j++)
197 : /* We cannot use dpdk_pfree() since the
198 : * fragments may not have vr_packet info
199 : */
200 0 : rte_pktmbuf_free(pkts_out[j]);
201 0 : rte_pktmbuf_free(next_pkt);
202 0 : return -1;
203 : }
204 0 : curr_pkt = next_pkt;
205 0 : i++;
206 : }
207 0 : return nb_segs;
208 : }
209 :
210 : /*
211 : * Updates the pointers to TCP and IPv6 headers
212 : */
213 : static inline void
214 0 : dpdk_gso_ipv6_tcp_update(struct rte_mbuf *m, struct dpdk_gso_state *state)
215 : {
216 0 : state->ip6 = (struct rte_ipv6_hdr *)(rte_pktmbuf_mtod(m, uint8_t *) + state->mac_hlen);
217 0 : state->tcp = (struct rte_tcp_hdr *)((uint8_t*)(state->ip6) + state->ip_hlen);
218 0 : state->pay_len = m->pkt_len - state->hlen;
219 0 : }
220 :
221 : /*
222 : * Update the pointers to TCP and IPv4 headers
223 : */
224 : static inline void
225 0 : dpdk_gso_ipv4_tcp_update(struct rte_mbuf *m, struct dpdk_gso_state *state)
226 : {
227 0 : state->ip = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(m, uint8_t *) + state->mac_hlen);
228 0 : state->tcp = (struct rte_tcp_hdr *)((uint8_t*)(state->ip) + state->ip_hlen);
229 0 : state->pay_len = m->pkt_len - state->hlen;
230 0 : }
231 :
232 : /*
233 : * Sets properly the TCP and IPv6 headers
234 : */
235 : static inline void
236 0 : dpdk_gso_ipv6_tcp_internal(struct rte_mbuf *m, struct dpdk_gso_state *state)
237 : {
238 0 : state->ip6->payload_len = rte_cpu_to_be_16(m->pkt_len -
239 : state->mac_hlen - state->ip_hlen);
240 :
241 : /* TCP Sequence number */
242 0 : state->tcp->sent_seq = htonl(state->tcp_seq);
243 :
244 0 : if (state->tx_cksum_offload) {
245 0 : m->l2_len = state->mac_hlen;
246 0 : m->l3_len = state->ip_hlen;
247 0 : state->tcp->cksum = 0;
248 0 : state->tcp->cksum = rte_ipv6_phdr_cksum((struct rte_ipv6_hdr *)state->ip6,
249 : m->ol_flags);
250 0 : m->ol_flags |= PKT_TX_IPV6;
251 0 : m->ol_flags |= PKT_TX_TCP_CKSUM;
252 : } else {
253 : /* TCP Checksum */
254 0 : state->tcp->cksum = 0;
255 0 : state->tcp->cksum = rte_ipv6_udptcp_cksum((struct rte_ipv6_hdr *)state->ip6,
256 0 : (uint8_t*)state->tcp);
257 : }
258 0 : state->tcp_seq += state->pay_len;
259 0 : }
260 :
261 : /*
262 : * Set properly the TCP and IPv4 headers
263 : */
264 : static inline void
265 0 : dpdk_gso_ipv4_tcp_internal(struct rte_mbuf *m, struct dpdk_gso_state *state)
266 : {
267 : /* Update IP header */
268 0 : state->ip->packet_id = rte_cpu_to_be_16((state->ip_id)++);
269 0 : state->ip->total_length = rte_cpu_to_be_16(m->pkt_len - state->mac_hlen);
270 :
271 : /* TCP Sequence number */
272 0 : state->tcp->sent_seq = htonl(state->tcp_seq);
273 :
274 0 : if (state->tx_cksum_offload) {
275 0 : m->l2_len = state->mac_hlen;
276 0 : m->l3_len = state->ip_hlen;
277 0 : state->ip->hdr_checksum = 0;
278 0 : state->tcp->cksum = 0;
279 0 : state->tcp->cksum = rte_ipv4_phdr_cksum((struct rte_ipv4_hdr *)state->ip,
280 : m->ol_flags);
281 0 : m->ol_flags |= PKT_TX_IP_CKSUM | PKT_TX_IPV4;
282 0 : m->ol_flags |= PKT_TX_TCP_CKSUM;
283 : } else {
284 : /* TCP Checksum */
285 0 : state->tcp->cksum = 0;
286 0 : state->tcp->cksum = rte_ipv4_udptcp_cksum((struct rte_ipv4_hdr *)state->ip,
287 0 : (uint8_t*)state->tcp);
288 : /* IP Checksum */
289 0 : state->ip->hdr_checksum = 0;
290 0 : state->ip->hdr_checksum = rte_ipv4_cksum(state->ip);
291 : }
292 0 : state->tcp_seq += state->pay_len;
293 0 : }
294 :
295 : /*
296 : * Init the state during the TCP segmentation
297 : */
298 : static int
299 0 : dpdk_gso_init_state(struct dpdk_gso_state *state,
300 : struct rte_mbuf *m, int mac_hlen, int isipv6, int hw_cksum)
301 : {
302 : uint32_t ip_hlen;
303 :
304 0 : if (isipv6) {
305 0 : ip_hlen = sizeof(struct rte_ipv6_hdr);
306 0 : state->ip6 = (struct rte_ipv6_hdr *)(rte_pktmbuf_mtod(m, uint8_t *) + mac_hlen);
307 0 : if (state->ip6->proto != VR_IP_PROTO_TCP)
308 0 : return -1;
309 0 : state->tcp = (struct rte_tcp_hdr *)((uint8_t*)(state->ip6) + ip_hlen);
310 0 : state->update = dpdk_gso_ipv6_tcp_update;
311 0 : state->internal = dpdk_gso_ipv6_tcp_internal;
312 : } else {
313 0 : state->ip = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(m, uint8_t *) + mac_hlen);
314 0 : state->ip_id = rte_be_to_cpu_16(state->ip->packet_id);
315 0 : ip_hlen = ((state->ip->version_ihl) & 0xf) << 2;
316 0 : if (state->ip->next_proto_id != VR_IP_PROTO_TCP)
317 0 : return -1;
318 0 : state->tcp = (struct rte_tcp_hdr *)((uint8_t*)state->ip + ip_hlen);
319 0 : state->update = dpdk_gso_ipv4_tcp_update;
320 0 : state->internal = dpdk_gso_ipv4_tcp_internal;
321 : }
322 :
323 0 : state->tx_cksum_offload = hw_cksum;
324 0 : state->mac_hlen = mac_hlen;
325 0 : state->ip_hlen = ip_hlen;
326 0 : state->tcp_hlen = ((state->tcp->data_off & 0xf0) >> 4) << 2;
327 0 : state->hlen = mac_hlen + ip_hlen + state->tcp_hlen;
328 0 : state->tcp_seq = ntohl(state->tcp->sent_seq);
329 0 : return 0;
330 : }
331 :
332 : /*
333 : * GSO on TCP/IP (v4 or v6)
334 : */
335 : static int
336 0 : dpdk_gso_segment_ip_tcp(struct rte_mbuf *pkt_in,
337 : struct dpdk_gso_state *state,
338 : struct rte_mbuf **mbufs_out,
339 : const unsigned short nb_pkts_out,
340 : uint16_t mss_size)
341 : {
342 0 : int i = 0, nsegs = 0;
343 :
344 : /* Check that pkts_out is big enough to hold all segments */
345 0 : if (unlikely(mss_size * nb_pkts_out <
346 : (uint16_t)(pkt_in->pkt_len - state->hlen)))
347 0 : return -EINVAL;
348 :
349 0 : nsegs = dpdk_split_chained_mbuf(pkt_in, mbufs_out, nb_pkts_out,
350 0 : state->hlen, mss_size);
351 :
352 0 : if (nsegs < 0)
353 0 : goto err;
354 :
355 0 : while (i < nsegs)
356 : {
357 0 : state->update(mbufs_out[i], state);
358 :
359 : /* Update TCP flags -
360 : * => Retain CWR only in the first segment and mask in the rest of
361 : * the segments
362 : * => FIN and PSH flags are only applicable to the last segment
363 : */
364 0 : if (state->tcp) {
365 0 : if (i > 0)
366 0 : state->tcp->tcp_flags &= ~RTE_TCP_CWR_FLAG;
367 0 : if (i < nsegs-1)
368 0 : state->tcp->tcp_flags &= ~(RTE_TCP_FIN_FLAG | RTE_TCP_PSH_FLAG);
369 : }
370 :
371 0 : state->internal(mbufs_out[i], state);
372 :
373 0 : i++;
374 : }
375 :
376 0 : return nsegs;
377 :
378 0 : err:
379 0 : return -1;
380 : }
381 :
382 : static void
383 0 : dpdk_adjust_outer_header(struct rte_mbuf *m, uint16_t outer_header_len)
384 : {
385 0 : struct vr_packet *pkt = vr_dpdk_mbuf_to_pkt(m);
386 0 : struct vr_ip *inner_ip = NULL;
387 :
388 0 : if (pkt->vp_type == VP_TYPE_IPOIP)
389 0 : inner_ip = rte_pktmbuf_mtod(m, struct vr_ip *);
390 :
391 : /* Outer header operations */
392 0 : char *outer_header_ptr = rte_pktmbuf_prepend(m, outer_header_len);
393 :
394 0 : uint16_t eth_hlen = dpdk_get_ether_header_len(outer_header_ptr);
395 0 : struct vr_ip *outer_ip = (struct vr_ip *)(outer_header_ptr + eth_hlen);
396 0 : outer_ip->ip_len = rte_cpu_to_be_16(rte_pktmbuf_pkt_len(m) - eth_hlen);
397 :
398 : /* Copy inner IP id to outer. Currently, the Agent diagnostics depends
399 : * on that. */
400 0 : if (inner_ip)
401 0 : outer_ip->ip_id = inner_ip->ip_id;
402 : else
403 0 : outer_ip->ip_id = rte_cpu_to_be_16(vr_generate_unique_ip_id());
404 :
405 :
406 : /* Adjust UDP length to match IP segment size */
407 0 : if (outer_ip->ip_proto == VR_IP_PROTO_UDP) {
408 0 : unsigned header_len = outer_ip->ip_hl * 4;
409 0 : struct vr_udp *udp = (struct vr_udp *)((char *)outer_ip +
410 : header_len);
411 0 : udp->udp_length = rte_cpu_to_be_16(
412 : rte_be_to_cpu_16(outer_ip->ip_len) - header_len);
413 : }
414 :
415 : /* Calculate the outer header IP checksum */
416 0 : outer_ip->ip_csum = vr_ip_csum(outer_ip);
417 0 : }
418 :
419 : int
420 0 : dpdk_segment_packet(struct vr_packet *pkt, struct rte_mbuf *mbuf_in,
421 : struct rte_mbuf **mbuf_out, const unsigned short out_num,
422 : const unsigned short mss_size, bool do_outer_ip_csum)
423 : {
424 : struct rte_mbuf *m;
425 : struct dpdk_gso_state state;
426 0 : int number_of_packets = 0, i;
427 : uint16_t outer_header_len;
428 :
429 0 : outer_header_len = pkt_get_inner_network_header_off(pkt) -
430 0 : pkt_head_space(pkt);
431 :
432 0 : if (dpdk_gso_init_state(&state, mbuf_in, outer_header_len,
433 0 : (pkt->vp_type == VP_TYPE_IP6OIP), do_outer_ip_csum) < 0)
434 0 : return -1;
435 :
436 0 : number_of_packets = dpdk_gso_segment_ip_tcp(mbuf_in, &state, mbuf_out,
437 : out_num, mss_size);
438 0 : if (number_of_packets < 0)
439 0 : return number_of_packets;
440 :
441 : /* Adjust outer and inner IP headers for each segmented packets */
442 0 : for (i = 0; i < number_of_packets; i++)
443 : {
444 0 : m = mbuf_out[i];
445 : /* Get into the inner IP header */
446 0 : rte_pktmbuf_adj(m, outer_header_len);
447 0 : dpdk_adjust_outer_header(m, outer_header_len);
448 0 : m->l2_len = mbuf_in->l2_len;
449 0 : m->l3_len = mbuf_in->l3_len;
450 0 : m->vlan_tci = mbuf_in->vlan_tci;
451 0 : m->ol_flags |= mbuf_in->ol_flags;
452 : }
453 :
454 0 : return number_of_packets;
455 : }
|
