Line data Source code
1 : /*
2 : * Copyright (C) 2017 Juniper Networks.
3 : * Copyright Contributors to the OpenSDN project
4 : *
5 : * This program is free software; you can redistribute it and/or
6 : * modify it under the terms of the GNU General Public License as
7 : * published by the Free Software Foundation version 2.
8 : *
9 : * This program is distributed "as is" WITHOUT ANY WARRANTY of any
10 : * kind, whether express or implied; without even the implied warranty
11 : * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 : * GNU General Public License for more details.
13 : *
14 : * vr_dpdk_gro.c -- TCP Offloads on the receiver
15 : *
16 : * This is an adaptation of FreeBSD's tcp_lro.c.
17 : * Copyright (c) 2007, Myricom Inc.
18 : * Copyright (c) 2008, Intel Corporation.
19 : * Copyright (c) 2012 The FreeBSD Foundation
20 : * All rights reserved.
21 : * BSD LICENSE
22 : *
23 : */
24 :
25 : #include "vr_dpdk.h"
26 : #include "vr_dpdk_netlink.h"
27 : #include "vr_dpdk_usocket.h"
28 : #include "vr_dpdk_virtio.h"
29 : #include "vr_dpdk_gro.h"
30 : #include "vr_packet.h"
31 : #include "vr_datapath.h"
32 :
33 : #include <netinet/ip.h>
34 : #include <netinet/tcp.h>
35 :
36 : #include <rte_errno.h>
37 : #include <rte_ethdev.h>
38 : #include <rte_ip_frag.h>
39 : #include <rte_ip.h>
40 : #include <rte_port_ethdev.h>
41 :
42 : #include <rte_hash.h>
43 : #include <rte_cycles.h>
44 : #include <rte_malloc.h>
45 : #include <rte_tcp.h>
46 : #include <rte_eth_bond.h>
47 :
48 : #define GRO_TABLE_NAME_LEN 45
49 :
50 : int
51 106 : vr_dpdk_gro_init(unsigned lcore_id, struct vr_dpdk_lcore *lcore)
52 : {
53 : /* Parameters used for hash tables. Name is set later. */
54 106 : struct rte_hash_parameters gro_tbl_params_v4 = {
55 : .entries = 1<<3,
56 : .key_len = sizeof(struct vr_dpdk_gro_flow_key_v4),
57 : .hash_func = rte_jhash,
58 : .hash_func_init_val = 0,
59 : .socket_id = 0, /* TODO */
60 : };
61 106 : struct rte_hash_parameters gro_tbl_params_v6 = {
62 : .entries = 1<<3,
63 : .key_len = sizeof(struct vr_dpdk_gro_flow_key_v6),
64 : .hash_func = rte_jhash,
65 : .hash_func_init_val = 0,
66 : .socket_id = 0, /* TODO */
67 : };
68 :
69 : char gro_v4_tbl_name[GRO_TABLE_NAME_LEN], gro_v6_tbl_name[GRO_TABLE_NAME_LEN];
70 :
71 106 : snprintf(gro_v4_tbl_name, sizeof(gro_v4_tbl_name), "GRO_Table_v4_lcore_%d", lcore_id);
72 106 : gro_tbl_params_v4.name = gro_v4_tbl_name;
73 106 : lcore->gro.gro_tbl_v4_handle = rte_hash_create(&gro_tbl_params_v4);
74 106 : if (lcore->gro.gro_tbl_v4_handle == NULL) {
75 0 : RTE_LOG(ERR, VROUTER, "Warning! %s: lcore:%d, Unable to allocate memory "
76 : "for IPv4 GRO table\n", __func__, lcore_id);
77 0 : return -ENOMEM;
78 : }
79 106 : snprintf(gro_v6_tbl_name, sizeof(gro_v6_tbl_name), "GRO_Table_v6_lcore_%d", lcore_id);
80 106 : gro_tbl_params_v6.name = gro_v6_tbl_name;
81 106 : lcore->gro.gro_tbl_v6_handle = rte_hash_create(&gro_tbl_params_v6);
82 106 : if (lcore->gro.gro_tbl_v6_handle == NULL) {
83 0 : RTE_LOG(ERR, VROUTER, "Warning! %s: lcore:%d, Unable to allocate memory "
84 : "for IPv6 GRO table\n", __func__, lcore_id);
85 0 : return -ENOMEM;
86 : }
87 106 : return 0;
88 : }
89 :
90 :
91 : static uint16_t
92 0 : dpdk_gro_csum_tcph(struct vr_tcp *tcph)
93 : {
94 : uint32_t ch;
95 : uint16_t *p, l;
96 :
97 0 : ch = tcph->tcp_csum = 0x0000;
98 0 : l = VR_TCP_OFFSET(tcph->tcp_offset_r_flags);
99 0 : p = (uint16_t *)tcph;
100 0 : while (l > 0) {
101 0 : ch += (*p);
102 0 : p++;
103 0 : ch += (*p);
104 0 : p++;
105 0 : l--;
106 : }
107 0 : while (ch > 0xffff)
108 0 : ch = (ch >> 16) + (ch & 0xffff);
109 :
110 0 : return (ch & 0xffff);
111 : }
112 :
113 : static uint16_t
114 0 : dpdk_gro_rx_csum_fixup(struct gro_entry *entry, uint8_t is_ipv6, void *nw_hdr,
115 : struct vr_tcp *tcph, uint16_t tcp_data_len, uint16_t csum)
116 : {
117 : uint32_t c;
118 : uint16_t cs;
119 :
120 0 : c = csum;
121 :
122 : /* For first packet, remove just the length from checksum
123 : * For subsequent packets, remove the checksum of full TCP header
124 : */
125 0 : if (!is_ipv6) {
126 0 : struct vr_ip* ip4 = nw_hdr;
127 0 : if (entry->mbuf_cnt == 1)
128 0 : cs = rte_be_to_cpu_16(ip4->ip_len) - sizeof(*ip4);
129 : else
130 0 : cs = rte_be_to_cpu_16(rte_ipv4_phdr_cksum((const struct rte_ipv4_hdr*)ip4,0));
131 : } else {
132 0 : struct vr_ip6* ip6 = nw_hdr;
133 0 : if (entry->mbuf_cnt == 1)
134 0 : cs = rte_be_to_cpu_16(ip6->ip6_plen);
135 : else
136 0 : cs = rte_be_to_cpu_16(rte_ipv6_phdr_cksum((const struct rte_ipv6_hdr*)ip6,0));
137 : }
138 :
139 0 : cs = ~cs;
140 0 : c += cs;
141 :
142 : /* Remove TCP header csum. */
143 0 : cs = rte_be_to_cpu_16(~dpdk_gro_csum_tcph(tcph));
144 0 : c += cs;
145 0 : while (c > 0xffff)
146 0 : c = (c >> 16) + (c & 0xffff);
147 :
148 0 : return (c & 0xffff);
149 : }
150 :
151 : static int
152 0 : dpdk_gro_flush(struct gro_ctrl *gro, void *key, struct gro_entry *entry)
153 : {
154 : int ret;
155 0 : struct vr_tcp *tcph = NULL;
156 0 : struct vr_interface *vif = NULL;
157 0 : struct vrouter *router = vrouter_get(0);
158 0 : struct rte_mbuf *m = entry->mbuf_head;
159 0 : struct vr_packet *pkt = vr_dpdk_mbuf_to_pkt(m);
160 : struct vr_forwarding_md fmd;
161 : uint32_t cl;
162 : uint16_t c;
163 : unsigned short drop_reason;
164 : struct vr_nexthop *nh;
165 :
166 0 : if (entry->mbuf_cnt > 1) {
167 :
168 0 : if (!entry->is_ipv6) {
169 : /* Fix IP header checksum for new length. */
170 0 : struct vr_ip *ip4 = entry->le_ip4;
171 0 : c = rte_be_to_cpu_16(~ip4->ip_csum);
172 0 : cl = c;
173 0 : c = rte_be_to_cpu_16(~ip4->ip_len);
174 0 : cl += c + entry->p_len;
175 0 : while (cl > 0xffff)
176 0 : cl = (cl >> 16) + (cl & 0xffff);
177 0 : c = cl;
178 0 : ip4->ip_csum = rte_cpu_to_be_16(~c);
179 :
180 0 : entry->le_ip4->ip_len = rte_cpu_to_be_16(entry->p_len);
181 0 : tcph = (struct vr_tcp *)(entry->le_ip4 + 1);
182 0 : m->ol_flags |= PKT_RX_GSO_TCP4;
183 : } else {
184 0 : struct vr_ip6 *ip6 = entry->le_ip6;
185 0 : ip6->ip6_plen = rte_cpu_to_be_16(entry->p_len - sizeof(struct vr_ip6));
186 0 : tcph = (struct vr_tcp *)(entry->le_ip6 + 1);
187 0 : m->ol_flags |= PKT_RX_GSO_TCP6;
188 : }
189 0 : m->pkt_len =
190 0 : entry->p_len + pkt_get_network_header_off(pkt) - pkt_head_space(pkt);
191 0 : m->nb_segs = entry->mbuf_cnt;
192 0 : m->tso_segsz = entry->seg_sz;
193 :
194 : /* Incorporate the latest ACK into the TCP header. */
195 0 : tcph->tcp_ack = rte_cpu_to_be_32(entry->ack_seq);
196 0 : tcph->tcp_win = rte_cpu_to_be_16(entry->window);
197 : /* Incorporate latest timestamp into the TCP header. */
198 0 : if (entry->timestamp != 0) {
199 : uint32_t *ts_ptr;
200 :
201 0 : ts_ptr = (uint32_t *)(tcph + 1);
202 0 : ts_ptr[1] = rte_cpu_to_be_32(entry->tsval);
203 0 : ts_ptr[2] = rte_cpu_to_be_32(entry->tsecr);
204 : }
205 :
206 : /* Update the TCP header checksum. */
207 0 : if (!entry->is_ipv6)
208 0 : entry->ulp_csum += entry->p_len - sizeof(struct vr_ip);
209 : else
210 0 : entry->ulp_csum += entry->p_len - sizeof(struct vr_ip6);
211 :
212 0 : entry->ulp_csum += rte_cpu_to_be_16(dpdk_gro_csum_tcph(tcph));
213 0 : while (entry->ulp_csum > 0xffff)
214 0 : entry->ulp_csum = (entry->ulp_csum >> 16) +
215 0 : (entry->ulp_csum & 0xffff);
216 0 : tcph->tcp_csum = entry->ulp_csum & 0xffff;
217 0 : tcph->tcp_csum = rte_cpu_to_be_16(~tcph->tcp_csum);
218 : }
219 :
220 0 : gro->gro_queued += entry->mbuf_cnt;
221 0 : gro->gro_flushed++;
222 :
223 : /* Delete flow entry from hash table */
224 0 : if (!entry->is_ipv6)
225 0 : ret = rte_hash_del_key(gro->gro_tbl_v4_handle, key);
226 : else
227 0 : ret = rte_hash_del_key(gro->gro_tbl_v6_handle, key);
228 0 : ASSERT(ret >= 0);
229 :
230 0 : nh = __vrouter_get_nexthop(router, entry->nh_id);
231 0 : if (!nh) {
232 0 : drop_reason = VP_DROP_INVALID_NH;
233 0 : goto drop;
234 : }
235 0 : vif = nh->nh_dev;
236 0 : if ((vif == NULL) || (!vif_is_virtual(vif))) {
237 0 : drop_reason = VP_DROP_INVALID_IF;
238 0 : goto drop;
239 : }
240 :
241 : /* Validate the gen_id of the vif */
242 0 : if (unlikely((vif->vif_idx != entry->dst_vif_idx) ||
243 : (vif->vif_gen != entry->dst_vif_gen))) {
244 0 : drop_reason = VP_DROP_INVALID_IF;
245 0 : goto drop;
246 : }
247 :
248 0 : if (nh->nh_family == AF_BRIDGE) {
249 0 : if (!rte_pktmbuf_prepend(m, VR_ETHER_HLEN)) {
250 0 : drop_reason = VP_DROP_INVALID_PACKET;
251 0 : goto drop;
252 : }
253 : }
254 :
255 0 : pkt = vr_dpdk_packet_get(m, NULL);
256 0 : if (!pkt) {
257 0 : drop_reason = VP_DROP_INVALID_IF;
258 0 : goto drop;
259 : }
260 :
261 : /*
262 : * since vif was not available when we did vr_dpdk_packet_get, set vif
263 : * manually here
264 : */
265 0 : vif = __vrouter_get_interface(router, entry->src_vif_idx);
266 0 : if (!vif) {
267 0 : drop_reason = VP_DROP_INVALID_IF;
268 0 : goto drop;
269 : }
270 :
271 : /* Validate the gen_id of the vif */
272 0 : if (unlikely(vif->vif_gen != entry->src_vif_gen)) {
273 0 : drop_reason = VP_DROP_INVALID_IF;
274 0 : goto drop;
275 : }
276 :
277 0 : pkt->vp_if = vif;
278 :
279 0 : vr_init_forwarding_md(&fmd);
280 0 : fmd.fmd_dvrf = nh->nh_dev->vif_vrf;
281 :
282 0 : if (nh->nh_family == AF_BRIDGE) {
283 0 : if (vr_pkt_type(pkt, 0, &fmd)) {
284 0 : drop_reason = VP_DROP_INVALID_PACKET;
285 0 : goto drop;
286 : }
287 : } else {
288 0 : if (vr_ip_is_ip4((struct vr_ip *)pkt_data(pkt))) {
289 0 : pkt->vp_type = VP_TYPE_IP;
290 0 : } else if (vr_ip_is_ip6((struct vr_ip *)pkt_data(pkt))) {
291 0 : pkt->vp_type = VP_TYPE_IP6;
292 : } else {
293 0 : drop_reason = VP_DROP_INVALID_PROTOCOL;
294 0 : goto drop;
295 : }
296 :
297 0 : pkt_set_network_header(pkt, pkt->vp_data);
298 0 : pkt_set_inner_network_header(pkt, pkt->vp_data);
299 : }
300 :
301 : /* For GRO packet, count will be incremented once */
302 0 : if (vif) {
303 : struct vr_interface_stats *gro_vif_stats;
304 0 : gro_vif_stats = vif_get_stats(vif, vr_get_cpu());
305 0 : if (gro_vif_stats) {
306 0 : gro_vif_stats->vis_opackets++;
307 0 : gro_vif_stats->vis_obytes += pkt_len(pkt);
308 : }
309 : }
310 :
311 0 : pkt->vp_flags |= VP_FLAG_FLOW_SET | VP_FLAG_GROED;
312 0 : nh_output(pkt, nh, &fmd);
313 0 : rte_free(entry);
314 0 : return GRO_MERGED;
315 :
316 0 : drop:
317 0 : vr_dpdk_pfree(m, vif, drop_reason);
318 0 : rte_free(entry);
319 0 : return 0;
320 : }
321 :
322 : void
323 53 : dpdk_gro_free_all_flows(struct vr_dpdk_lcore *lcore)
324 : {
325 : struct gro_entry *entry;
326 53 : uint32_t iter = 0;
327 : void *next_key;
328 53 : struct gro_ctrl *gro = &lcore->gro;
329 53 : if (gro->gro_tbl_v4_handle != NULL) {
330 0 : while (rte_hash_iterate(gro->gro_tbl_v4_handle, (const void**)&next_key,
331 0 : (void**)&entry, &iter) >= 0) {
332 0 : vr_dpdk_pfree(entry->mbuf_head, NULL, VP_DROP_DISCARD);
333 0 : rte_free(entry);
334 : }
335 0 : rte_hash_reset(gro->gro_tbl_v4_handle);
336 : }
337 53 : iter = 0;
338 53 : if (gro->gro_tbl_v6_handle != NULL) {
339 0 : while (rte_hash_iterate(gro->gro_tbl_v6_handle, (const void**)&next_key,
340 0 : (void**)&entry, &iter) >= 0) {
341 0 : vr_dpdk_pfree(entry->mbuf_head, NULL, VP_DROP_DISCARD);
342 0 : rte_free(entry);
343 : }
344 0 : rte_hash_reset(gro->gro_tbl_v6_handle);
345 : }
346 53 : }
347 :
348 : void
349 1358892 : dpdk_gro_flush_all_inactive(struct vr_dpdk_lcore *lcore)
350 : {
351 1358892 : uint32_t iter = 0;
352 : void *next_key;
353 : struct gro_entry *entry;
354 1358892 : struct gro_ctrl *gro = &lcore->gro;
355 1358892 : uint64_t cur_cycles = 0, diff_cycles;
356 1358892 : const uint64_t gro_flush_cycles = 100 * VR_DPDK_TX_FLUSH_LOOPS;
357 :
358 1358892 : if (gro->gro_tbl_v4_handle != NULL) {
359 1358893 : cur_cycles = lcore->lcore_fwd_loops;
360 : /* Iterate through the hash table */
361 1358893 : while (rte_hash_iterate(gro->gro_tbl_v4_handle, (const void**)&next_key,
362 1358901 : (void**)&entry, &iter) >= 0) {
363 0 : diff_cycles = cur_cycles - entry->mtime;
364 0 : if (unlikely(gro_flush_cycles < diff_cycles)) {
365 : /* Flush flow */
366 0 : dpdk_gro_flush(gro, next_key, entry);
367 0 : lcore->gro.gro_flush_inactive_flows++;
368 : }
369 : }
370 : }
371 :
372 1358900 : iter = 0;
373 1358900 : if (gro->gro_tbl_v6_handle != NULL) {
374 1358905 : cur_cycles = lcore->lcore_fwd_loops;
375 : /* Iterate through the hash table */
376 1358905 : while (rte_hash_iterate(gro->gro_tbl_v6_handle, (const void**)&next_key,
377 1358893 : (void**)&entry, &iter) >= 0) {
378 0 : diff_cycles = cur_cycles - entry->mtime;
379 0 : if (unlikely(gro_flush_cycles < diff_cycles)) {
380 : /* Flush flow */
381 0 : dpdk_gro_flush(gro, next_key, entry);
382 0 : lcore->gro.gro_flush_inactive_flows++;
383 : }
384 : }
385 : }
386 1358888 : }
387 :
388 : int
389 4 : dpdk_gro_process(struct vr_packet *pkt, struct vr_interface *vif, bool l2_pkt)
390 : {
391 : struct vr_dpdk_gro_flow_key_v4 flow4;
392 : struct vr_dpdk_gro_flow_key_v6 flow6;
393 : struct gro_entry *entry;
394 : struct rte_mbuf *m;
395 : void *nw_hdr;
396 4 : struct vr_tcp *tcph = NULL;
397 : uint16_t tcp_data_len, ip_pkt_len, l;
398 : uint32_t *ts_ptr;
399 : int32_t ret;
400 : unsigned short csum;
401 4 : unsigned short src_vif_idx = 0;
402 4 : uint32_t nh_id = 0;
403 : unsigned lcore_id;
404 : struct vr_dpdk_lcore *lcore;
405 4 : uint8_t is_ipv6 = 0, flush = 0;
406 4 : void *key = NULL;
407 : struct vrouter *router;
408 : struct vr_interface *src_vif;
409 : struct vr_nexthop *nh;
410 : struct vr_gro *gro;
411 :
412 :
413 : /* Normal processing for VMs if -
414 : * => They dont require GRO (like DPDK VM's)
415 : * => They dont support mergeable buffers
416 : */
417 4 : if ((vif->vif_flags &
418 : (VIF_FLAG_GRO_NEEDED | VIF_FLAG_MRG_RXBUF)) !=
419 : (VIF_FLAG_GRO_NEEDED | VIF_FLAG_MRG_RXBUF)) {
420 4 : return 0;
421 : }
422 :
423 0 : lcore_id = rte_lcore_id();
424 0 : lcore = vr_dpdk.lcores[lcore_id];
425 :
426 : /* Packets arriving through non-Fwd cores - Normal processing */
427 0 : if (unlikely((lcore->gro.gro_tbl_v4_handle == NULL) ||
428 : (lcore->gro.gro_tbl_v6_handle == NULL))){
429 0 : return 0;
430 : }
431 :
432 0 : m = vr_dpdk_pkt_to_mbuf(pkt);
433 0 : nw_hdr = m->buf_addr + pkt_get_network_header_off(pkt);
434 0 : router = vrouter_get(0);
435 : /* Adjust mbuf */
436 0 : m->data_off = pkt_head_space(pkt);
437 0 : m->pkt_len = pkt_len(pkt);
438 0 : m->data_len = pkt_head_len(pkt);
439 :
440 : /* get the vif_idx and nh_id */
441 0 : gro = rte_pktmbuf_mtod(m, struct vr_gro *);
442 0 : src_vif_idx = gro->vg_vif_id;
443 0 : nh_id = gro->vg_nh_id;
444 0 : rte_pktmbuf_adj(m, sizeof(struct vr_gro));
445 0 : pkt_pull(pkt, sizeof(struct vr_gro));
446 :
447 0 : src_vif = __vrouter_get_interface(router, src_vif_idx);
448 :
449 : /* Make sure nh points to a virtual vif */
450 0 : nh = __vrouter_get_nexthop(router, nh_id);
451 0 : if ((nh == NULL) || (nh->nh_dev == NULL)
452 0 : || (!vif_is_virtual(nh->nh_dev))) {
453 0 : ret = GRO_NORMAL;
454 0 : goto func_exit;
455 : }
456 :
457 : /* Can't handle chained mbufs */
458 0 : if (m->nb_segs > 1) {
459 0 : flush = 1;
460 : }
461 :
462 : /* Parse IP header */
463 0 : if (vr_ip_is_ip6(nw_hdr)) {
464 0 : struct vr_ip6 *ip6 = nw_hdr;
465 0 : is_ipv6 = 1;
466 0 : flow6.proto = ip6->ip6_nxt;
467 : /* For non TCP packets, no GRO */
468 0 : if (flow6.proto != VR_IP_PROTO_TCP) {
469 0 : ret = GRO_NOT_APPLICABLE;
470 0 : goto func_exit;
471 : }
472 0 : rte_memcpy(&flow6.ip6_src, ip6->ip6_src, sizeof(flow6.ip6_src));
473 0 : rte_memcpy(&flow6.ip6_dst, ip6->ip6_dst, sizeof(flow6.ip6_dst));
474 0 : tcph = (struct vr_tcp *)(nw_hdr + sizeof(*ip6));
475 0 : flow6.port_src = rte_be_to_cpu_16(tcph->tcp_sport);
476 0 : flow6.port_dst = rte_be_to_cpu_16(tcph->tcp_dport);
477 0 : flow6.vif_idx = vif->vif_idx;
478 0 : ip_pkt_len = rte_be_to_cpu_16(ip6->ip6_plen) + sizeof(struct vr_ip6);
479 0 : tcp_data_len = rte_be_to_cpu_16(ip6->ip6_plen);
480 0 : key = &flow6;
481 0 : } else if (vr_ip_is_ip4(nw_hdr)) {
482 0 : struct vr_ip *ip4 = nw_hdr;
483 0 : unsigned int hlen = 0;
484 0 : hlen = ip4->ip_hl * 4;
485 0 : flow4.proto = ip4->ip_proto;
486 : /* For non TCP packets, no GRO */
487 0 : if (flow4.proto != VR_IP_PROTO_TCP) {
488 0 : ret = GRO_NOT_APPLICABLE;
489 0 : goto func_exit;
490 : }
491 : /* Ensure the packet is not fragmented. */
492 0 : else if (ip4->ip_frag_off & rte_cpu_to_be_16(IP_MF|IP_OFFMASK)) {
493 0 : flush = 1;
494 : }
495 : /* .. and there are no options. */
496 0 : else if (hlen != sizeof (*ip4)) {
497 0 : flush = 1;
498 : }
499 0 : flow4.ip_src = ip4->ip_saddr;
500 0 : flow4.ip_dst = ip4->ip_daddr;
501 0 : tcph = (struct vr_tcp *)(nw_hdr + hlen);
502 0 : flow4.port_src = rte_be_to_cpu_16(tcph->tcp_sport);
503 0 : flow4.port_dst = rte_be_to_cpu_16(tcph->tcp_dport);
504 0 : flow4.vif_idx = vif->vif_idx;
505 0 : ip_pkt_len = rte_be_to_cpu_16(ip4->ip_len);
506 0 : tcp_data_len = ip_pkt_len - (ip4->ip_hl << 2);
507 0 : key = &flow4;
508 : } else {
509 0 : ret = GRO_NOT_APPLICABLE;
510 0 : goto func_exit;
511 : }
512 0 : csum = rte_be_to_cpu_16(tcph->tcp_csum);
513 :
514 : /* Check TCP header constraints */
515 :
516 : /* Ensure no bits set besides ACK. */
517 0 : if (VR_TCP_FLAGS(tcph->tcp_offset_r_flags) & (~VR_TCP_FLAG_ACK)) {
518 0 : flush = 1;
519 : }
520 :
521 : /*
522 : * Check for timestamps.
523 : * Since the only option we handle are timestamps, we only have to
524 : * handle the simple case of aligned timestamps.
525 : */
526 0 : l = VR_TCP_OFFSET(tcph->tcp_offset_r_flags) << 2;
527 0 : tcp_data_len -= l;
528 0 : l -= sizeof(*tcph);
529 0 : ts_ptr = (uint32_t *)(tcph + 1);
530 0 : if (l != 0 && (unlikely(l != TCPOLEN_TSTAMP_APPA) ||
531 0 : (*ts_ptr != rte_cpu_to_be_32(TCPOPT_NOP<<24|TCPOPT_NOP<<16|
532 : TCPOPT_TIMESTAMP<<8|TCPOLEN_TIMESTAMP)))) {
533 0 : flush = 1;
534 : }
535 :
536 : /* Cannot handle odd size segments */
537 0 : if (tcp_data_len & 1) {
538 0 : flush = 1;
539 : }
540 :
541 0 : if (likely(flush == 0)) {
542 : /* Check TCP checksum */
543 0 : tcph->tcp_csum = 0;
544 0 : if (!is_ipv6)
545 0 : tcph->tcp_csum = rte_ipv4_udptcp_cksum(nw_hdr, tcph);
546 : else
547 0 : tcph->tcp_csum = rte_ipv6_udptcp_cksum(nw_hdr, tcph);
548 :
549 0 : if (tcph->tcp_csum == 0xffff)
550 0 : tcph->tcp_csum = 0;
551 :
552 0 : if (csum != rte_be_to_cpu_16(tcph->tcp_csum)) {
553 0 : vr_dpdk_pfree(m, vif, VP_DROP_CKSUM_ERR);
554 0 : ret = GRO_MERGED;
555 0 : goto func_exit;
556 : }
557 : }
558 :
559 0 : if (!is_ipv6)
560 0 : ret = rte_hash_lookup_data(lcore->gro.gro_tbl_v4_handle, &flow4, (void*)&entry);
561 : else
562 0 : ret = rte_hash_lookup_data(lcore->gro.gro_tbl_v6_handle, &flow6, (void*)&entry);
563 :
564 0 : if (ret >= 0) {
565 : /* Update flow */
566 :
567 : /* Check if packet needs to be flushed */
568 0 : if (flush) {
569 0 : dpdk_gro_flush(&lcore->gro, key, entry);
570 0 : ret = GRO_CANNOT;
571 0 : goto func_exit;
572 : }
573 :
574 : /* Flush now if appending will result in overflow. */
575 0 : if (entry->p_len > (65535 - tcp_data_len)) {
576 0 : dpdk_gro_flush(&lcore->gro, key, entry);
577 0 : goto create;
578 : }
579 :
580 : /* Try to append the new segment. */
581 0 : if (unlikely(rte_be_to_cpu_32(tcph->tcp_seq) != entry->next_seq ||
582 : (tcp_data_len == 0))) {
583 : /* Out of order packet or duplicate ACK. */
584 0 : dpdk_gro_flush(&lcore->gro, key, entry);
585 0 : ret = GRO_CANNOT;
586 0 : goto func_exit;
587 : }
588 :
589 0 : if (l != 0) {
590 0 : uint32_t tsval = rte_be_to_cpu_32(*(ts_ptr + 1));
591 : /* Make sure timestamp values are increasing.
592 : * Also make sure the timestamp echo reply value is valid.
593 : * Note: timestamp echo reply is valid if its value is not 0.
594 : */
595 0 : if (unlikely(entry->tsval > tsval ||
596 : rte_be_to_cpu_32(*(ts_ptr + 2)) == 0)) {
597 0 : dpdk_gro_flush(&lcore->gro, key, entry);
598 0 : ret = GRO_CANNOT;
599 0 : goto func_exit;
600 : }
601 0 : entry->tsval = tsval;
602 0 : entry->tsecr = rte_be_to_cpu_32(*(ts_ptr + 2));
603 : }
604 :
605 0 : entry->next_seq += tcp_data_len;
606 0 : entry->ack_seq = rte_be_to_cpu_32(tcph->tcp_ack);
607 0 : entry->window = rte_be_to_cpu_16(tcph->tcp_win);
608 0 : entry->mbuf_cnt++;
609 0 : entry->ulp_csum += dpdk_gro_rx_csum_fixup(entry, is_ipv6, nw_hdr, tcph, tcp_data_len, ~csum);
610 :
611 0 : entry->p_len += tcp_data_len;
612 :
613 : /*
614 : * Adjust the mbuf so that rte_pktmbuf_mtod(m) points to the first byte of
615 : * the ULP payload. Adjust the mbuf to avoid complications and
616 : * append new segment to existing mbuf chain.
617 : */
618 0 : rte_pktmbuf_adj(m, rte_pktmbuf_data_len(m) - tcp_data_len);
619 :
620 0 : entry->mbuf_tail->next = m;
621 0 : entry->mbuf_tail = rte_pktmbuf_lastseg(m);
622 :
623 : /* If segment size is different, flush */
624 0 : if (entry->seg_sz != tcp_data_len) {
625 0 : dpdk_gro_flush(&lcore->gro, key, entry);
626 0 : ret = GRO_MERGED;
627 0 : goto func_exit;
628 : }
629 :
630 : /* Save cycles */
631 0 : entry->mtime = lcore->lcore_fwd_loops;
632 :
633 0 : ret = GRO_MERGED;
634 : } else {
635 : /* Create new flow */
636 : struct gro_entry *entry;
637 0 : create:
638 0 : if (tcp_data_len == 0) {
639 0 : ret = GRO_CANNOT;
640 0 : goto func_exit;
641 0 : } else if (flush) {
642 0 : ret = GRO_CANNOT;
643 0 : goto func_exit;
644 : }
645 :
646 0 : entry = rte_zmalloc("GRO_entry", sizeof(*entry), RTE_CACHE_LINE_SIZE);
647 0 : if (unlikely(entry == NULL)) {
648 0 : ret = GRO_ERROR;
649 0 : goto func_exit;
650 : }
651 0 : entry->is_ipv6 = is_ipv6;
652 0 : entry->src_vif_idx = src_vif_idx;
653 0 : if (likely(src_vif != NULL))
654 0 : entry->src_vif_gen = src_vif->vif_gen;
655 0 : entry->dst_vif_idx = nh->nh_dev->vif_idx;
656 0 : entry->dst_vif_gen = nh->nh_dev->vif_gen;
657 0 : entry->nh_id = nh_id;
658 0 : entry->mtime = lcore->lcore_fwd_loops;
659 0 : entry->p_len = ip_pkt_len;
660 0 : entry->next_seq = rte_be_to_cpu_32(tcph->tcp_seq) + tcp_data_len;
661 0 : entry->ack_seq = rte_be_to_cpu_32(tcph->tcp_ack);
662 0 : entry->window = rte_be_to_cpu_16(tcph->tcp_win);
663 0 : entry->seg_sz = tcp_data_len;
664 0 : if (!is_ipv6)
665 0 : entry->le_ip4 = nw_hdr;
666 : else
667 0 : entry->le_ip6 = nw_hdr;
668 0 : entry->mbuf_cnt++;
669 0 : if (unlikely(l != 0)) {
670 0 : entry->timestamp = 1;
671 0 : entry->tsval = rte_be_to_cpu_32(*(ts_ptr + 1));
672 0 : entry->tsecr = rte_be_to_cpu_32(*(ts_ptr + 2));
673 : }
674 0 : entry->mbuf_head = m;
675 0 : entry->mbuf_tail = rte_pktmbuf_lastseg(m);
676 :
677 0 : entry->ulp_csum = dpdk_gro_rx_csum_fixup(entry, is_ipv6, nw_hdr, tcph, tcp_data_len, ~csum);
678 0 : tcph->tcp_csum = rte_cpu_to_be_16(csum);/* Restore checksum on first packet. */
679 0 : lcore->gro.gro_flows++;
680 :
681 0 : if (!is_ipv6) {
682 0 : ret = (rte_hash_add_key_data(lcore->gro.gro_tbl_v4_handle, &flow4, entry) < 0)?
683 0 : GRO_ERROR : GRO_MERGED;
684 : } else {
685 0 : ret = (rte_hash_add_key_data(lcore->gro.gro_tbl_v6_handle, &flow6, entry) < 0)?
686 0 : GRO_ERROR : GRO_MERGED;
687 : }
688 :
689 0 : if (ret == GRO_MERGED) {
690 : /* point mbuf to network header */
691 0 : rte_pktmbuf_adj(m, pkt_get_network_header_off(pkt)- pkt_head_space(pkt));
692 0 : pkt_pull(pkt, pkt_get_network_header_off(pkt)- pkt_head_space(pkt));
693 0 : } else if (ret == GRO_ERROR) {
694 0 : rte_free(entry);
695 : }
696 : }
697 :
698 0 : func_exit:
699 0 : if (ret != GRO_MERGED) {
700 0 : pkt_push(pkt, sizeof(struct vr_gro));
701 0 : rte_pktmbuf_prepend(m, sizeof(struct vr_gro));
702 : }
703 0 : return (ret == GRO_MERGED)?1:0;
704 : }
705 :
706 :
|
