Line data Source code
1 : /*
2 : * Copyright (C) 2016 Juniper Networks.
3 : * Copyright Contributors to the OpenSDN project
4 : *
5 : * This program is free software; you can redistribute it and/or
6 : * modify it under the terms of the GNU General Public License as
7 : * published by the Free Software Foundation version 2.
8 : *
9 : * This program is distributed "as is" WITHOUT ANY WARRANTY of any
10 : * kind, whether express or implied; without even the implied warranty
11 : * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * vr_dpdk_gso.c -- TCP Offloads on the sender
15 : *
16 : * This is an adaptation of FreeBSD's patch for gso
17 : * Copyright (C) 2014, Stefano Garzarella - Universita` di Pisa
18 : * All rights reserved.
19 : * BSD LICENSE
20 : *
21 : *
22 : */
23 :
24 : #include "vr_dpdk.h"
25 : #include "vr_dpdk_netlink.h"
26 : #include "vr_dpdk_usocket.h"
27 : #include "vr_dpdk_virtio.h"
28 : #include "vr_dpdk_gro.h"
29 : #include "vr_packet.h"
30 : #include "vr_datapath.h"
31 :
32 : #include <netinet/ip.h>
33 : #include <netinet/tcp.h>
34 :
35 : #include <rte_errno.h>
36 : #include <rte_ethdev.h>
37 : #include <rte_ip_frag.h>
38 : #include <rte_ip.h>
39 : #include <rte_port_ethdev.h>
40 : #include <rte_hash.h>
41 : #include <rte_cycles.h>
42 : #include <rte_malloc.h>
43 : #include <rte_tcp.h>
44 : #include <rte_eth_bond.h>
45 :
46 : /*
47 : * Structure that contains the state during the TCP segmentation
48 : */
49 : struct dpdk_gso_state {
50 : void (*update)
51 : (struct rte_mbuf*, struct dpdk_gso_state*);
52 : void (*internal)
53 : (struct rte_mbuf*, struct dpdk_gso_state*);
54 : union {
55 : struct rte_ipv4_hdr *ip;
56 : struct rte_ipv6_hdr *ip6;
57 : };
58 : struct rte_tcp_hdr *tcp;
59 : int mac_hlen;
60 : int ip_hlen;
61 : int tcp_hlen;
62 : int hlen;
63 : int pay_len;
64 : uint32_t tcp_seq;
65 : uint16_t ip_id;
66 : uint16_t ip_off;
67 : int tx_cksum_offload;
68 : };
69 :
70 : /**
71 : * Process the IPv6 UDP or TCP checksum in a **chained** mbuf.
72 : *
73 : * Layer 4 checksum must be set to 0 in the packet by the caller.
74 : *
75 : * @param ipv6_hdr
76 : * The pointer to the contiguous IPv4 header.
77 : * @param l4_hdr
78 : * The pointer to the beginning of the L4 header.
79 : * @return
80 : * The complemented checksum to set in the IP packet.
81 : */
82 : inline uint16_t
83 8 : dpdk_ipv6_udptcp_cksum(struct rte_mbuf *m, const struct rte_ipv6_hdr *ipv6_hdr,
84 : uint8_t *l4_hdr)
85 : {
86 8 : uint32_t cksum = 0;
87 : uint32_t l4_len;
88 8 : uint32_t data_len = 0, rem_len = 0;
89 8 : uint8_t *data_ptr = NULL;
90 :
91 8 : l4_len = rte_be_to_cpu_16(ipv6_hdr->payload_len);
92 :
93 : do {
94 8 : data_ptr = likely(!!data_ptr)? rte_pktmbuf_mtod(m, uint8_t*):l4_hdr;
95 8 : data_len = likely(!!data_len)? rte_pktmbuf_data_len(m):
96 8 : rte_pktmbuf_mtod(m, uint8_t*) + rte_pktmbuf_data_len(m) - l4_hdr ;
97 8 : if (rem_len + data_len > l4_len)
98 0 : data_len = l4_len - rem_len;
99 8 : cksum += rte_raw_cksum(data_ptr, data_len);
100 8 : rem_len += data_len;
101 8 : m = m->next;
102 8 : } while (m && rem_len < l4_len);
103 :
104 8 : cksum += rte_ipv6_phdr_cksum(ipv6_hdr, 0);
105 8 : cksum = ((cksum & 0xffff0000) >> 16) + (cksum & 0xffff);
106 8 : cksum = (~cksum) & 0xffff;
107 8 : if (cksum == 0)
108 0 : cksum = 0xffff;
109 :
110 8 : return cksum;
111 : }
112 :
113 : /**
114 : * Process the IPv4 UDP or TCP checksum in a **chained** mbuf.
115 : *
116 : * The IPv4 header should not contains options. The IP and layer 4
117 : * checksum must be set to 0 in the packet by the caller.
118 : *
119 : * @param ipv4_hdr
120 : * The pointer to the contiguous IPv4 header.
121 : * @param l4_hdr
122 : * The pointer to the beginning of the L4 header.
123 : * @return
124 : * The complemented checksum to set in the IP packet.
125 : */
126 : inline uint16_t
127 36 : dpdk_ipv4_udptcp_cksum(struct rte_mbuf *m,
128 : const struct rte_ipv4_hdr *ipv4_hdr,
129 : uint8_t *l4_hdr)
130 : {
131 36 : uint32_t cksum = 0;
132 : uint32_t l4_len;
133 36 : uint32_t data_len = 0, rem_len = 0;
134 36 : uint8_t *data_ptr = NULL;
135 :
136 36 : l4_len = rte_be_to_cpu_16(ipv4_hdr->total_length) -
137 36 : ((ipv4_hdr->version_ihl & 0xf) << 2);
138 :
139 : do {
140 40 : data_ptr = likely(!!data_ptr)? rte_pktmbuf_mtod(m, uint8_t*):l4_hdr;
141 40 : data_len = likely(!!data_len)? rte_pktmbuf_data_len(m):
142 36 : rte_pktmbuf_mtod(m, uint8_t*) + rte_pktmbuf_data_len(m) - l4_hdr ;
143 40 : if (rem_len + data_len > l4_len)
144 0 : data_len = l4_len - rem_len;
145 40 : cksum += rte_raw_cksum(data_ptr, data_len);
146 40 : rem_len += data_len;
147 40 : m = m->next;
148 40 : } while (m && rem_len < l4_len);
149 :
150 36 : cksum += rte_ipv4_phdr_cksum(ipv4_hdr, 0);
151 36 : cksum = ((cksum & 0xffff0000) >> 16) + (cksum & 0xffff);
152 36 : cksum = (~cksum) & 0xffff;
153 36 : if (cksum == 0)
154 0 : cksum = 0xffff;
155 :
156 36 : return cksum;
157 : }
158 :
159 : /* Split chained mbuf to individual mbufs and save the pointers in pkts_out */
160 : static int32_t
161 0 : dpdk_split_chained_mbuf(struct rte_mbuf *pkt_in, struct rte_mbuf **pkts_out,
162 : uint16_t nb_pkts_out, uint16_t hdr_len, uint16_t frag_size)
163 : {
164 0 : struct rte_mbuf *curr_pkt = pkt_in, *next_pkt;
165 0 : uint32_t nb_segs = 0, total_segs = pkt_in->nb_segs, i = 0, j;
166 0 : char *in_hdr = rte_pktmbuf_mtod(pkt_in, char*);
167 : char *pkt_addr;
168 :
169 0 : while (curr_pkt) {
170 0 : next_pkt = curr_pkt->next;
171 0 : curr_pkt->next = NULL;
172 0 : curr_pkt->nb_segs = 1;
173 0 : curr_pkt->pkt_len = curr_pkt->data_len;
174 : /* 1st mbuf in the chain already has all the headers */
175 0 : if (likely(nb_segs > 0)) {
176 0 : pkt_addr = rte_pktmbuf_prepend(curr_pkt, hdr_len);
177 0 : if (unlikely(pkt_addr == NULL)) {
178 : /* pkts_out[0] will be freed in the caller
179 : * when -1 is returned
180 : */
181 0 : for(j=1;j<i;j++)
182 : /* We cannot use dpdk_pfree() since the
183 : * fragments may not have vr_packet info
184 : */
185 0 : rte_pktmbuf_free(pkts_out[j]);
186 0 : rte_pktmbuf_free(curr_pkt);
187 0 : rte_pktmbuf_free(next_pkt);
188 0 : return -1;
189 : }
190 0 : rte_memcpy(pkt_addr, in_hdr, hdr_len);
191 : }
192 0 : pkts_out[nb_segs++] = curr_pkt;
193 0 : if ((nb_segs >= nb_pkts_out) || (i >= total_segs)) {
194 : /* pkts_out[0] will be freed in the caller
195 : * when -1 is returned
196 : */
197 0 : for(j=1;j<(i+1);j++)
198 : /* We cannot use dpdk_pfree() since the
199 : * fragments may not have vr_packet info
200 : */
201 0 : rte_pktmbuf_free(pkts_out[j]);
202 0 : rte_pktmbuf_free(next_pkt);
203 0 : return -1;
204 : }
205 0 : curr_pkt = next_pkt;
206 0 : i++;
207 : }
208 0 : return nb_segs;
209 : }
210 :
211 : /*
212 : * Updates the pointers to TCP and IPv6 headers
213 : */
214 : static inline void
215 0 : dpdk_gso_ipv6_tcp_update(struct rte_mbuf *m, struct dpdk_gso_state *state)
216 : {
217 0 : state->ip6 = (struct rte_ipv6_hdr *)(rte_pktmbuf_mtod(m, uint8_t *) + state->mac_hlen);
218 0 : state->tcp = (struct rte_tcp_hdr *)((uint8_t*)(state->ip6) + state->ip_hlen);
219 0 : state->pay_len = m->pkt_len - state->hlen;
220 0 : }
221 :
222 : /*
223 : * Update the pointers to TCP and IPv4 headers
224 : */
225 : static inline void
226 0 : dpdk_gso_ipv4_tcp_update(struct rte_mbuf *m, struct dpdk_gso_state *state)
227 : {
228 0 : state->ip = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(m, uint8_t *) + state->mac_hlen);
229 0 : state->tcp = (struct rte_tcp_hdr *)((uint8_t*)(state->ip) + state->ip_hlen);
230 0 : state->pay_len = m->pkt_len - state->hlen;
231 0 : }
232 :
233 : /*
234 : * Sets properly the TCP and IPv6 headers
235 : */
236 : static inline void
237 0 : dpdk_gso_ipv6_tcp_internal(struct rte_mbuf *m, struct dpdk_gso_state *state)
238 : {
239 0 : state->ip6->payload_len = rte_cpu_to_be_16(m->pkt_len -
240 : state->mac_hlen - state->ip_hlen);
241 :
242 : /* TCP Sequence number */
243 0 : state->tcp->sent_seq = htonl(state->tcp_seq);
244 :
245 0 : if (state->tx_cksum_offload) {
246 0 : m->l2_len = state->mac_hlen;
247 0 : m->l3_len = state->ip_hlen;
248 0 : state->tcp->cksum = 0;
249 0 : state->tcp->cksum = rte_ipv6_phdr_cksum((struct rte_ipv6_hdr *)state->ip6,
250 : m->ol_flags);
251 0 : m->ol_flags |= PKT_TX_IPV6;
252 0 : m->ol_flags |= PKT_TX_TCP_CKSUM;
253 : } else {
254 : /* TCP Checksum */
255 0 : state->tcp->cksum = 0;
256 0 : state->tcp->cksum = rte_ipv6_udptcp_cksum((struct rte_ipv6_hdr *)state->ip6,
257 0 : (uint8_t*)state->tcp);
258 : }
259 0 : state->tcp_seq += state->pay_len;
260 0 : }
261 :
262 : /*
263 : * Set properly the TCP and IPv4 headers
264 : */
265 : static inline void
266 0 : dpdk_gso_ipv4_tcp_internal(struct rte_mbuf *m, struct dpdk_gso_state *state)
267 : {
268 : /* Update IP header */
269 0 : state->ip->packet_id = rte_cpu_to_be_16((state->ip_id)++);
270 0 : state->ip->total_length = rte_cpu_to_be_16(m->pkt_len - state->mac_hlen);
271 :
272 : /* TCP Sequence number */
273 0 : state->tcp->sent_seq = htonl(state->tcp_seq);
274 :
275 0 : if (state->tx_cksum_offload) {
276 0 : m->l2_len = state->mac_hlen;
277 0 : m->l3_len = state->ip_hlen;
278 0 : state->ip->hdr_checksum = 0;
279 0 : state->tcp->cksum = 0;
280 0 : state->tcp->cksum = rte_ipv4_phdr_cksum((struct rte_ipv4_hdr *)state->ip,
281 : m->ol_flags);
282 0 : m->ol_flags |= PKT_TX_IP_CKSUM | PKT_TX_IPV4;
283 0 : m->ol_flags |= PKT_TX_TCP_CKSUM;
284 : } else {
285 : /* TCP Checksum */
286 0 : state->tcp->cksum = 0;
287 0 : state->tcp->cksum = rte_ipv4_udptcp_cksum((struct rte_ipv4_hdr *)state->ip,
288 0 : (uint8_t*)state->tcp);
289 : /* IP Checksum */
290 0 : state->ip->hdr_checksum = 0;
291 0 : state->ip->hdr_checksum = rte_ipv4_cksum(state->ip);
292 : }
293 0 : state->tcp_seq += state->pay_len;
294 0 : }
295 :
296 : /*
297 : * Init the state during the TCP segmentation
298 : */
299 : static int
300 0 : dpdk_gso_init_state(struct dpdk_gso_state *state,
301 : struct rte_mbuf *m, int mac_hlen, int isipv6, int hw_cksum)
302 : {
303 : uint32_t ip_hlen;
304 :
305 0 : if (isipv6) {
306 0 : ip_hlen = sizeof(struct rte_ipv6_hdr);
307 0 : state->ip6 = (struct rte_ipv6_hdr *)(rte_pktmbuf_mtod(m, uint8_t *) + mac_hlen);
308 0 : if (state->ip6->proto != VR_IP_PROTO_TCP)
309 0 : return -1;
310 0 : state->tcp = (struct rte_tcp_hdr *)((uint8_t*)(state->ip6) + ip_hlen);
311 0 : state->update = dpdk_gso_ipv6_tcp_update;
312 0 : state->internal = dpdk_gso_ipv6_tcp_internal;
313 : } else {
314 0 : state->ip = (struct rte_ipv4_hdr *)(rte_pktmbuf_mtod(m, uint8_t *) + mac_hlen);
315 0 : state->ip_id = rte_be_to_cpu_16(state->ip->packet_id);
316 0 : ip_hlen = ((state->ip->version_ihl) & 0xf) << 2;
317 0 : if (state->ip->next_proto_id != VR_IP_PROTO_TCP)
318 0 : return -1;
319 0 : state->tcp = (struct rte_tcp_hdr *)((uint8_t*)state->ip + ip_hlen);
320 0 : state->update = dpdk_gso_ipv4_tcp_update;
321 0 : state->internal = dpdk_gso_ipv4_tcp_internal;
322 : }
323 :
324 0 : state->tx_cksum_offload = hw_cksum;
325 0 : state->mac_hlen = mac_hlen;
326 0 : state->ip_hlen = ip_hlen;
327 0 : state->tcp_hlen = ((state->tcp->data_off & 0xf0) >> 4) << 2;
328 0 : state->hlen = mac_hlen + ip_hlen + state->tcp_hlen;
329 0 : state->tcp_seq = ntohl(state->tcp->sent_seq);
330 0 : return 0;
331 : }
332 :
333 : /*
334 : * GSO on TCP/IP (v4 or v6)
335 : */
336 : static int
337 0 : dpdk_gso_segment_ip_tcp(struct rte_mbuf *pkt_in,
338 : struct dpdk_gso_state *state,
339 : struct rte_mbuf **mbufs_out,
340 : const unsigned short nb_pkts_out,
341 : uint16_t mss_size)
342 : {
343 0 : int i = 0, nsegs = 0;
344 :
345 : /* Check that pkts_out is big enough to hold all segments */
346 0 : if (unlikely(mss_size * nb_pkts_out <
347 : (uint16_t)(pkt_in->pkt_len - state->hlen)))
348 0 : return -EINVAL;
349 :
350 0 : nsegs = dpdk_split_chained_mbuf(pkt_in, mbufs_out, nb_pkts_out,
351 0 : state->hlen, mss_size);
352 :
353 0 : if (nsegs < 0)
354 0 : goto err;
355 :
356 0 : while (i < nsegs)
357 : {
358 0 : state->update(mbufs_out[i], state);
359 :
360 : /* Update TCP flags -
361 : * => Retain CWR only in the first segment and mask in the rest of
362 : * the segments
363 : * => FIN and PSH flags are only applicable to the last segment
364 : */
365 0 : if (state->tcp) {
366 0 : if (i > 0)
367 0 : state->tcp->tcp_flags &= ~RTE_TCP_CWR_FLAG;
368 0 : if (i < nsegs-1)
369 0 : state->tcp->tcp_flags &= ~(RTE_TCP_FIN_FLAG | RTE_TCP_PSH_FLAG);
370 : }
371 :
372 0 : state->internal(mbufs_out[i], state);
373 :
374 0 : i++;
375 : }
376 :
377 0 : return nsegs;
378 :
379 0 : err:
380 0 : return -1;
381 : }
382 :
383 : static void
384 0 : dpdk_adjust_outer_header(struct rte_mbuf *m, uint16_t outer_header_len)
385 : {
386 0 : struct vr_packet *pkt = vr_dpdk_mbuf_to_pkt(m);
387 0 : struct vr_ip *inner_ip = NULL;
388 :
389 0 : if (pkt->vp_type == VP_TYPE_IPOIP)
390 0 : inner_ip = rte_pktmbuf_mtod(m, struct vr_ip *);
391 :
392 : /* Outer header operations */
393 0 : char *outer_header_ptr = rte_pktmbuf_prepend(m, outer_header_len);
394 :
395 0 : uint16_t eth_hlen = dpdk_get_ether_header_len(outer_header_ptr);
396 0 : struct vr_ip *outer_ip = (struct vr_ip *)(outer_header_ptr + eth_hlen);
397 0 : outer_ip->ip_len = rte_cpu_to_be_16(rte_pktmbuf_pkt_len(m) - eth_hlen);
398 :
399 : /* Copy inner IP id to outer. Currently, the Agent diagnostics depends
400 : * on that. */
401 0 : if (inner_ip)
402 0 : outer_ip->ip_id = inner_ip->ip_id;
403 : else
404 0 : outer_ip->ip_id = rte_cpu_to_be_16(vr_generate_unique_ip_id());
405 :
406 :
407 : /* Adjust UDP length to match IP segment size */
408 0 : if (outer_ip->ip_proto == VR_IP_PROTO_UDP) {
409 0 : unsigned header_len = outer_ip->ip_hl * 4;
410 0 : struct vr_udp *udp = (struct vr_udp *)((char *)outer_ip +
411 : header_len);
412 0 : udp->udp_length = rte_cpu_to_be_16(
413 : rte_be_to_cpu_16(outer_ip->ip_len) - header_len);
414 : }
415 :
416 : /* Calculate the outer header IP checksum */
417 0 : outer_ip->ip_csum = vr_ip_csum(outer_ip);
418 0 : }
419 :
420 : int
421 0 : dpdk_segment_packet(struct vr_packet *pkt, struct rte_mbuf *mbuf_in,
422 : struct rte_mbuf **mbuf_out, const unsigned short out_num,
423 : const unsigned short mss_size, bool do_outer_ip_csum)
424 : {
425 : struct rte_mbuf *m;
426 : struct dpdk_gso_state state;
427 0 : int number_of_packets = 0, i;
428 : uint16_t outer_header_len;
429 :
430 0 : outer_header_len = pkt_get_inner_network_header_off(pkt) -
431 0 : pkt_head_space(pkt);
432 :
433 0 : if (dpdk_gso_init_state(&state, mbuf_in, outer_header_len,
434 0 : (pkt->vp_type == VP_TYPE_IP6OIP), do_outer_ip_csum) < 0)
435 0 : return -1;
436 :
437 0 : number_of_packets = dpdk_gso_segment_ip_tcp(mbuf_in, &state, mbuf_out,
438 : out_num, mss_size);
439 0 : if (number_of_packets < 0)
440 0 : return number_of_packets;
441 :
442 : /* Adjust outer and inner IP headers for each segmented packets */
443 0 : for (i = 0; i < number_of_packets; i++)
444 : {
445 0 : m = mbuf_out[i];
446 : /* Get into the inner IP header */
447 0 : rte_pktmbuf_adj(m, outer_header_len);
448 0 : dpdk_adjust_outer_header(m, outer_header_len);
449 0 : m->l2_len = mbuf_in->l2_len;
450 0 : m->l3_len = mbuf_in->l3_len;
451 0 : m->vlan_tci = mbuf_in->vlan_tci;
452 0 : m->ol_flags |= mbuf_in->ol_flags;
453 : }
454 :
455 0 : return number_of_packets;
456 : }
|
