Line data Source code
1 : /*
2 : * Copyright (C) 2017 Juniper Networks.
3 : *
4 : * This program is free software; you can redistribute it and/or
5 : * modify it under the terms of the GNU General Public License as
6 : * published by the Free Software Foundation version 2.
7 : *
8 : * This program is distributed "as is" WITHOUT ANY WARRANTY of any
9 : * kind, whether express or implied; without even the implied warranty
10 : * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 : * GNU General Public License for more details.
12 : *
13 : * vr_dpdk_gro.c -- TCP Offloads on the receiver
14 : *
15 : * This is an adaptation of FreeBSD's tcp_lro.c.
16 : * Copyright (c) 2007, Myricom Inc.
17 : * Copyright (c) 2008, Intel Corporation.
18 : * Copyright (c) 2012 The FreeBSD Foundation
19 : * All rights reserved.
20 : * BSD LICENSE
21 : *
22 : */
23 :
24 : #include "vr_dpdk.h"
25 : #include "vr_dpdk_netlink.h"
26 : #include "vr_dpdk_usocket.h"
27 : #include "vr_dpdk_virtio.h"
28 : #include "vr_dpdk_gro.h"
29 : #include "vr_packet.h"
30 : #include "vr_datapath.h"
31 :
32 : #include <netinet/ip.h>
33 : #include <netinet/tcp.h>
34 :
35 : #include <rte_errno.h>
36 : #include <rte_ethdev.h>
37 : #include <rte_ip_frag.h>
38 : #include <rte_ip.h>
39 : #include <rte_port_ethdev.h>
40 :
41 : #include <rte_hash.h>
42 : #include <rte_cycles.h>
43 : #include <rte_malloc.h>
44 : #include <rte_tcp.h>
45 : #include <rte_eth_bond.h>
46 :
47 : #define GRO_TABLE_NAME_LEN 45
48 :
49 : int
50 105 : vr_dpdk_gro_init(unsigned lcore_id, struct vr_dpdk_lcore *lcore)
51 : {
52 : /* Parameters used for hash tables. Name is set later. */
53 105 : struct rte_hash_parameters gro_tbl_params_v4 = {
54 : .entries = 1<<3,
55 : .key_len = sizeof(struct vr_dpdk_gro_flow_key_v4),
56 : .hash_func = rte_jhash,
57 : .hash_func_init_val = 0,
58 : .socket_id = 0, /* TODO */
59 : };
60 105 : struct rte_hash_parameters gro_tbl_params_v6 = {
61 : .entries = 1<<3,
62 : .key_len = sizeof(struct vr_dpdk_gro_flow_key_v6),
63 : .hash_func = rte_jhash,
64 : .hash_func_init_val = 0,
65 : .socket_id = 0, /* TODO */
66 : };
67 :
68 : char gro_v4_tbl_name[GRO_TABLE_NAME_LEN], gro_v6_tbl_name[GRO_TABLE_NAME_LEN];
69 :
70 105 : snprintf(gro_v4_tbl_name, sizeof(gro_v4_tbl_name), "GRO_Table_v4_lcore_%d", lcore_id);
71 105 : gro_tbl_params_v4.name = gro_v4_tbl_name;
72 105 : lcore->gro.gro_tbl_v4_handle = rte_hash_create(&gro_tbl_params_v4);
73 106 : if (lcore->gro.gro_tbl_v4_handle == NULL) {
74 0 : RTE_LOG(ERR, VROUTER, "Warning! %s: lcore:%d, Unable to allocate memory "
75 : "for IPv4 GRO table\n", __func__, lcore_id);
76 0 : return -ENOMEM;
77 : }
78 106 : snprintf(gro_v6_tbl_name, sizeof(gro_v6_tbl_name), "GRO_Table_v6_lcore_%d", lcore_id);
79 106 : gro_tbl_params_v6.name = gro_v6_tbl_name;
80 106 : lcore->gro.gro_tbl_v6_handle = rte_hash_create(&gro_tbl_params_v6);
81 106 : if (lcore->gro.gro_tbl_v6_handle == NULL) {
82 0 : RTE_LOG(ERR, VROUTER, "Warning! %s: lcore:%d, Unable to allocate memory "
83 : "for IPv6 GRO table\n", __func__, lcore_id);
84 0 : return -ENOMEM;
85 : }
86 106 : return 0;
87 : }
88 :
89 :
90 : static uint16_t
91 0 : dpdk_gro_csum_tcph(struct vr_tcp *tcph)
92 : {
93 : uint32_t ch;
94 : uint16_t *p, l;
95 :
96 0 : ch = tcph->tcp_csum = 0x0000;
97 0 : l = VR_TCP_OFFSET(tcph->tcp_offset_r_flags);
98 0 : p = (uint16_t *)tcph;
99 0 : while (l > 0) {
100 0 : ch += (*p);
101 0 : p++;
102 0 : ch += (*p);
103 0 : p++;
104 0 : l--;
105 : }
106 0 : while (ch > 0xffff)
107 0 : ch = (ch >> 16) + (ch & 0xffff);
108 :
109 0 : return (ch & 0xffff);
110 : }
111 :
112 : static uint16_t
113 0 : dpdk_gro_rx_csum_fixup(struct gro_entry *entry, uint8_t is_ipv6, void *nw_hdr,
114 : struct vr_tcp *tcph, uint16_t tcp_data_len, uint16_t csum)
115 : {
116 : uint32_t c;
117 : uint16_t cs;
118 :
119 0 : c = csum;
120 :
121 : /* For first packet, remove just the length from checksum
122 : * For subsequent packets, remove the checksum of full TCP header
123 : */
124 0 : if (!is_ipv6) {
125 0 : struct vr_ip* ip4 = nw_hdr;
126 0 : if (entry->mbuf_cnt == 1)
127 0 : cs = rte_be_to_cpu_16(ip4->ip_len) - sizeof(*ip4);
128 : else
129 0 : cs = rte_be_to_cpu_16(rte_ipv4_phdr_cksum((const struct rte_ipv4_hdr*)ip4,0));
130 : } else {
131 0 : struct vr_ip6* ip6 = nw_hdr;
132 0 : if (entry->mbuf_cnt == 1)
133 0 : cs = rte_be_to_cpu_16(ip6->ip6_plen);
134 : else
135 0 : cs = rte_be_to_cpu_16(rte_ipv6_phdr_cksum((const struct rte_ipv6_hdr*)ip6,0));
136 : }
137 :
138 0 : cs = ~cs;
139 0 : c += cs;
140 :
141 : /* Remove TCP header csum. */
142 0 : cs = rte_be_to_cpu_16(~dpdk_gro_csum_tcph(tcph));
143 0 : c += cs;
144 0 : while (c > 0xffff)
145 0 : c = (c >> 16) + (c & 0xffff);
146 :
147 0 : return (c & 0xffff);
148 : }
149 :
150 : static int
151 0 : dpdk_gro_flush(struct gro_ctrl *gro, void *key, struct gro_entry *entry)
152 : {
153 : int ret;
154 0 : struct vr_tcp *tcph = NULL;
155 0 : struct vr_interface *vif = NULL;
156 0 : struct vrouter *router = vrouter_get(0);
157 0 : struct rte_mbuf *m = entry->mbuf_head;
158 0 : struct vr_packet *pkt = vr_dpdk_mbuf_to_pkt(m);
159 : struct vr_forwarding_md fmd;
160 : uint32_t cl;
161 : uint16_t c;
162 : unsigned short drop_reason;
163 : struct vr_nexthop *nh;
164 :
165 0 : if (entry->mbuf_cnt > 1) {
166 :
167 0 : if (!entry->is_ipv6) {
168 : /* Fix IP header checksum for new length. */
169 0 : struct vr_ip *ip4 = entry->le_ip4;
170 0 : c = rte_be_to_cpu_16(~ip4->ip_csum);
171 0 : cl = c;
172 0 : c = rte_be_to_cpu_16(~ip4->ip_len);
173 0 : cl += c + entry->p_len;
174 0 : while (cl > 0xffff)
175 0 : cl = (cl >> 16) + (cl & 0xffff);
176 0 : c = cl;
177 0 : ip4->ip_csum = rte_cpu_to_be_16(~c);
178 :
179 0 : entry->le_ip4->ip_len = rte_cpu_to_be_16(entry->p_len);
180 0 : tcph = (struct vr_tcp *)(entry->le_ip4 + 1);
181 0 : m->ol_flags |= PKT_RX_GSO_TCP4;
182 : } else {
183 0 : struct vr_ip6 *ip6 = entry->le_ip6;
184 0 : ip6->ip6_plen = rte_cpu_to_be_16(entry->p_len - sizeof(struct vr_ip6));
185 0 : tcph = (struct vr_tcp *)(entry->le_ip6 + 1);
186 0 : m->ol_flags |= PKT_RX_GSO_TCP6;
187 : }
188 0 : m->pkt_len =
189 0 : entry->p_len + pkt_get_network_header_off(pkt) - pkt_head_space(pkt);
190 0 : m->nb_segs = entry->mbuf_cnt;
191 0 : m->tso_segsz = entry->seg_sz;
192 :
193 : /* Incorporate the latest ACK into the TCP header. */
194 0 : tcph->tcp_ack = rte_cpu_to_be_32(entry->ack_seq);
195 0 : tcph->tcp_win = rte_cpu_to_be_16(entry->window);
196 : /* Incorporate latest timestamp into the TCP header. */
197 0 : if (entry->timestamp != 0) {
198 : uint32_t *ts_ptr;
199 :
200 0 : ts_ptr = (uint32_t *)(tcph + 1);
201 0 : ts_ptr[1] = rte_cpu_to_be_32(entry->tsval);
202 0 : ts_ptr[2] = rte_cpu_to_be_32(entry->tsecr);
203 : }
204 :
205 : /* Update the TCP header checksum. */
206 0 : if (!entry->is_ipv6)
207 0 : entry->ulp_csum += entry->p_len - sizeof(struct vr_ip);
208 : else
209 0 : entry->ulp_csum += entry->p_len - sizeof(struct vr_ip6);
210 :
211 0 : entry->ulp_csum += rte_cpu_to_be_16(dpdk_gro_csum_tcph(tcph));
212 0 : while (entry->ulp_csum > 0xffff)
213 0 : entry->ulp_csum = (entry->ulp_csum >> 16) +
214 0 : (entry->ulp_csum & 0xffff);
215 0 : tcph->tcp_csum = entry->ulp_csum & 0xffff;
216 0 : tcph->tcp_csum = rte_cpu_to_be_16(~tcph->tcp_csum);
217 : }
218 :
219 0 : gro->gro_queued += entry->mbuf_cnt;
220 0 : gro->gro_flushed++;
221 :
222 : /* Delete flow entry from hash table */
223 0 : if (!entry->is_ipv6)
224 0 : ret = rte_hash_del_key(gro->gro_tbl_v4_handle, key);
225 : else
226 0 : ret = rte_hash_del_key(gro->gro_tbl_v6_handle, key);
227 0 : ASSERT(ret >= 0);
228 :
229 0 : nh = __vrouter_get_nexthop(router, entry->nh_id);
230 0 : if (!nh) {
231 0 : drop_reason = VP_DROP_INVALID_NH;
232 0 : goto drop;
233 : }
234 0 : vif = nh->nh_dev;
235 0 : if ((vif == NULL) || (!vif_is_virtual(vif))) {
236 0 : drop_reason = VP_DROP_INVALID_IF;
237 0 : goto drop;
238 : }
239 :
240 : /* Validate the gen_id of the vif */
241 0 : if (unlikely((vif->vif_idx != entry->dst_vif_idx) ||
242 : (vif->vif_gen != entry->dst_vif_gen))) {
243 0 : drop_reason = VP_DROP_INVALID_IF;
244 0 : goto drop;
245 : }
246 :
247 0 : if (nh->nh_family == AF_BRIDGE) {
248 0 : if (!rte_pktmbuf_prepend(m, VR_ETHER_HLEN)) {
249 0 : drop_reason = VP_DROP_INVALID_PACKET;
250 0 : goto drop;
251 : }
252 : }
253 :
254 0 : pkt = vr_dpdk_packet_get(m, NULL);
255 0 : if (!pkt) {
256 0 : drop_reason = VP_DROP_INVALID_IF;
257 0 : goto drop;
258 : }
259 :
260 : /*
261 : * since vif was not available when we did vr_dpdk_packet_get, set vif
262 : * manually here
263 : */
264 0 : vif = __vrouter_get_interface(router, entry->src_vif_idx);
265 0 : if (!vif) {
266 0 : drop_reason = VP_DROP_INVALID_IF;
267 0 : goto drop;
268 : }
269 :
270 : /* Validate the gen_id of the vif */
271 0 : if (unlikely(vif->vif_gen != entry->src_vif_gen)) {
272 0 : drop_reason = VP_DROP_INVALID_IF;
273 0 : goto drop;
274 : }
275 :
276 0 : pkt->vp_if = vif;
277 :
278 0 : vr_init_forwarding_md(&fmd);
279 0 : fmd.fmd_dvrf = nh->nh_dev->vif_vrf;
280 :
281 0 : if (nh->nh_family == AF_BRIDGE) {
282 0 : if (vr_pkt_type(pkt, 0, &fmd)) {
283 0 : drop_reason = VP_DROP_INVALID_PACKET;
284 0 : goto drop;
285 : }
286 : } else {
287 0 : if (vr_ip_is_ip4((struct vr_ip *)pkt_data(pkt))) {
288 0 : pkt->vp_type = VP_TYPE_IP;
289 0 : } else if (vr_ip_is_ip6((struct vr_ip *)pkt_data(pkt))) {
290 0 : pkt->vp_type = VP_TYPE_IP6;
291 : } else {
292 0 : drop_reason = VP_DROP_INVALID_PROTOCOL;
293 0 : goto drop;
294 : }
295 :
296 0 : pkt_set_network_header(pkt, pkt->vp_data);
297 0 : pkt_set_inner_network_header(pkt, pkt->vp_data);
298 : }
299 :
300 : /* For GRO packet, count will be incremented once */
301 0 : if (vif) {
302 : struct vr_interface_stats *gro_vif_stats;
303 0 : gro_vif_stats = vif_get_stats(vif, vr_get_cpu());
304 0 : if (gro_vif_stats) {
305 0 : gro_vif_stats->vis_opackets++;
306 0 : gro_vif_stats->vis_obytes += pkt_len(pkt);
307 : }
308 : }
309 :
310 0 : pkt->vp_flags |= VP_FLAG_FLOW_SET | VP_FLAG_GROED;
311 0 : nh_output(pkt, nh, &fmd);
312 0 : rte_free(entry);
313 0 : return GRO_MERGED;
314 :
315 0 : drop:
316 0 : vr_dpdk_pfree(m, vif, drop_reason);
317 0 : rte_free(entry);
318 0 : return 0;
319 : }
320 :
321 : void
322 53 : dpdk_gro_free_all_flows(struct vr_dpdk_lcore *lcore)
323 : {
324 : struct gro_entry *entry;
325 53 : uint32_t iter = 0;
326 : void *next_key;
327 53 : struct gro_ctrl *gro = &lcore->gro;
328 53 : if (gro->gro_tbl_v4_handle != NULL) {
329 0 : while (rte_hash_iterate(gro->gro_tbl_v4_handle, (const void**)&next_key,
330 0 : (void**)&entry, &iter) >= 0) {
331 0 : vr_dpdk_pfree(entry->mbuf_head, NULL, VP_DROP_DISCARD);
332 0 : rte_free(entry);
333 : }
334 0 : rte_hash_reset(gro->gro_tbl_v4_handle);
335 : }
336 53 : iter = 0;
337 53 : if (gro->gro_tbl_v6_handle != NULL) {
338 0 : while (rte_hash_iterate(gro->gro_tbl_v6_handle, (const void**)&next_key,
339 0 : (void**)&entry, &iter) >= 0) {
340 0 : vr_dpdk_pfree(entry->mbuf_head, NULL, VP_DROP_DISCARD);
341 0 : rte_free(entry);
342 : }
343 0 : rte_hash_reset(gro->gro_tbl_v6_handle);
344 : }
345 53 : }
346 :
347 : void
348 631292 : dpdk_gro_flush_all_inactive(struct vr_dpdk_lcore *lcore)
349 : {
350 631292 : uint32_t iter = 0;
351 : void *next_key;
352 : struct gro_entry *entry;
353 631292 : struct gro_ctrl *gro = &lcore->gro;
354 631292 : uint64_t cur_cycles = 0, diff_cycles;
355 631292 : const uint64_t gro_flush_cycles = 100 * VR_DPDK_TX_FLUSH_LOOPS;
356 :
357 631292 : if (gro->gro_tbl_v4_handle != NULL) {
358 631292 : cur_cycles = lcore->lcore_fwd_loops;
359 : /* Iterate through the hash table */
360 631292 : while (rte_hash_iterate(gro->gro_tbl_v4_handle, (const void**)&next_key,
361 631296 : (void**)&entry, &iter) >= 0) {
362 0 : diff_cycles = cur_cycles - entry->mtime;
363 0 : if (unlikely(gro_flush_cycles < diff_cycles)) {
364 : /* Flush flow */
365 0 : dpdk_gro_flush(gro, next_key, entry);
366 0 : lcore->gro.gro_flush_inactive_flows++;
367 : }
368 : }
369 : }
370 :
371 631296 : iter = 0;
372 631296 : if (gro->gro_tbl_v6_handle != NULL) {
373 631295 : cur_cycles = lcore->lcore_fwd_loops;
374 : /* Iterate through the hash table */
375 631295 : while (rte_hash_iterate(gro->gro_tbl_v6_handle, (const void**)&next_key,
376 631297 : (void**)&entry, &iter) >= 0) {
377 0 : diff_cycles = cur_cycles - entry->mtime;
378 0 : if (unlikely(gro_flush_cycles < diff_cycles)) {
379 : /* Flush flow */
380 0 : dpdk_gro_flush(gro, next_key, entry);
381 0 : lcore->gro.gro_flush_inactive_flows++;
382 : }
383 : }
384 : }
385 631298 : }
386 :
387 : int
388 4 : dpdk_gro_process(struct vr_packet *pkt, struct vr_interface *vif, bool l2_pkt)
389 : {
390 : struct vr_dpdk_gro_flow_key_v4 flow4;
391 : struct vr_dpdk_gro_flow_key_v6 flow6;
392 : struct gro_entry *entry;
393 : struct rte_mbuf *m;
394 : void *nw_hdr;
395 4 : struct vr_tcp *tcph = NULL;
396 : uint16_t tcp_data_len, ip_pkt_len, l;
397 : uint32_t *ts_ptr;
398 : int32_t ret;
399 : unsigned short csum;
400 4 : unsigned short src_vif_idx = 0;
401 4 : uint32_t nh_id = 0;
402 : unsigned lcore_id;
403 : struct vr_dpdk_lcore *lcore;
404 4 : uint8_t is_ipv6 = 0, flush = 0;
405 4 : void *key = NULL;
406 : struct vrouter *router;
407 : struct vr_interface *src_vif;
408 : struct vr_nexthop *nh;
409 : struct vr_gro *gro;
410 :
411 :
412 : /* Normal processing for VMs if -
413 : * => They dont require GRO (like DPDK VM's)
414 : * => They dont support mergeable buffers
415 : */
416 4 : if ((vif->vif_flags &
417 : (VIF_FLAG_GRO_NEEDED | VIF_FLAG_MRG_RXBUF)) !=
418 : (VIF_FLAG_GRO_NEEDED | VIF_FLAG_MRG_RXBUF)) {
419 4 : return 0;
420 : }
421 :
422 0 : lcore_id = rte_lcore_id();
423 0 : lcore = vr_dpdk.lcores[lcore_id];
424 :
425 : /* Packets arriving through non-Fwd cores - Normal processing */
426 0 : if (unlikely((lcore->gro.gro_tbl_v4_handle == NULL) ||
427 : (lcore->gro.gro_tbl_v6_handle == NULL))){
428 0 : return 0;
429 : }
430 :
431 0 : m = vr_dpdk_pkt_to_mbuf(pkt);
432 0 : nw_hdr = m->buf_addr + pkt_get_network_header_off(pkt);
433 0 : router = vrouter_get(0);
434 : /* Adjust mbuf */
435 0 : m->data_off = pkt_head_space(pkt);
436 0 : m->pkt_len = pkt_len(pkt);
437 0 : m->data_len = pkt_head_len(pkt);
438 :
439 : /* get the vif_idx and nh_id */
440 0 : gro = rte_pktmbuf_mtod(m, struct vr_gro *);
441 0 : src_vif_idx = gro->vg_vif_id;
442 0 : nh_id = gro->vg_nh_id;
443 0 : rte_pktmbuf_adj(m, sizeof(struct vr_gro));
444 0 : pkt_pull(pkt, sizeof(struct vr_gro));
445 :
446 0 : src_vif = __vrouter_get_interface(router, src_vif_idx);
447 :
448 : /* Make sure nh points to a virtual vif */
449 0 : nh = __vrouter_get_nexthop(router, nh_id);
450 0 : if ((nh == NULL) || (nh->nh_dev == NULL)
451 0 : || (!vif_is_virtual(nh->nh_dev))) {
452 0 : ret = GRO_NORMAL;
453 0 : goto func_exit;
454 : }
455 :
456 : /* Can't handle chained mbufs */
457 0 : if (m->nb_segs > 1) {
458 0 : flush = 1;
459 : }
460 :
461 : /* Parse IP header */
462 0 : if (vr_ip_is_ip6(nw_hdr)) {
463 0 : struct vr_ip6 *ip6 = nw_hdr;
464 0 : is_ipv6 = 1;
465 0 : flow6.proto = ip6->ip6_nxt;
466 : /* For non TCP packets, no GRO */
467 0 : if (flow6.proto != VR_IP_PROTO_TCP) {
468 0 : ret = GRO_NOT_APPLICABLE;
469 0 : goto func_exit;
470 : }
471 0 : rte_memcpy(&flow6.ip6_src, ip6->ip6_src, sizeof(flow6.ip6_src));
472 0 : rte_memcpy(&flow6.ip6_dst, ip6->ip6_dst, sizeof(flow6.ip6_dst));
473 0 : tcph = (struct vr_tcp *)(nw_hdr + sizeof(*ip6));
474 0 : flow6.port_src = rte_be_to_cpu_16(tcph->tcp_sport);
475 0 : flow6.port_dst = rte_be_to_cpu_16(tcph->tcp_dport);
476 0 : flow6.vif_idx = vif->vif_idx;
477 0 : ip_pkt_len = rte_be_to_cpu_16(ip6->ip6_plen) + sizeof(struct vr_ip6);
478 0 : tcp_data_len = rte_be_to_cpu_16(ip6->ip6_plen);
479 0 : key = &flow6;
480 0 : } else if (vr_ip_is_ip4(nw_hdr)) {
481 0 : struct vr_ip *ip4 = nw_hdr;
482 0 : unsigned int hlen = 0;
483 0 : hlen = ip4->ip_hl * 4;
484 0 : flow4.proto = ip4->ip_proto;
485 : /* For non TCP packets, no GRO */
486 0 : if (flow4.proto != VR_IP_PROTO_TCP) {
487 0 : ret = GRO_NOT_APPLICABLE;
488 0 : goto func_exit;
489 : }
490 : /* Ensure the packet is not fragmented. */
491 0 : else if (ip4->ip_frag_off & rte_cpu_to_be_16(IP_MF|IP_OFFMASK)) {
492 0 : flush = 1;
493 : }
494 : /* .. and there are no options. */
495 0 : else if (hlen != sizeof (*ip4)) {
496 0 : flush = 1;
497 : }
498 0 : flow4.ip_src = ip4->ip_saddr;
499 0 : flow4.ip_dst = ip4->ip_daddr;
500 0 : tcph = (struct vr_tcp *)(nw_hdr + hlen);
501 0 : flow4.port_src = rte_be_to_cpu_16(tcph->tcp_sport);
502 0 : flow4.port_dst = rte_be_to_cpu_16(tcph->tcp_dport);
503 0 : flow4.vif_idx = vif->vif_idx;
504 0 : ip_pkt_len = rte_be_to_cpu_16(ip4->ip_len);
505 0 : tcp_data_len = ip_pkt_len - (ip4->ip_hl << 2);
506 0 : key = &flow4;
507 : } else {
508 0 : ret = GRO_NOT_APPLICABLE;
509 0 : goto func_exit;
510 : }
511 0 : csum = rte_be_to_cpu_16(tcph->tcp_csum);
512 :
513 : /* Check TCP header constraints */
514 :
515 : /* Ensure no bits set besides ACK. */
516 0 : if (VR_TCP_FLAGS(tcph->tcp_offset_r_flags) & (~VR_TCP_FLAG_ACK)) {
517 0 : flush = 1;
518 : }
519 :
520 : /*
521 : * Check for timestamps.
522 : * Since the only option we handle are timestamps, we only have to
523 : * handle the simple case of aligned timestamps.
524 : */
525 0 : l = VR_TCP_OFFSET(tcph->tcp_offset_r_flags) << 2;
526 0 : tcp_data_len -= l;
527 0 : l -= sizeof(*tcph);
528 0 : ts_ptr = (uint32_t *)(tcph + 1);
529 0 : if (l != 0 && (unlikely(l != TCPOLEN_TSTAMP_APPA) ||
530 0 : (*ts_ptr != rte_cpu_to_be_32(TCPOPT_NOP<<24|TCPOPT_NOP<<16|
531 : TCPOPT_TIMESTAMP<<8|TCPOLEN_TIMESTAMP)))) {
532 0 : flush = 1;
533 : }
534 :
535 : /* Cannot handle odd size segments */
536 0 : if (tcp_data_len & 1) {
537 0 : flush = 1;
538 : }
539 :
540 0 : if (likely(flush == 0)) {
541 : /* Check TCP checksum */
542 0 : tcph->tcp_csum = 0;
543 0 : if (!is_ipv6)
544 0 : tcph->tcp_csum = rte_ipv4_udptcp_cksum(nw_hdr, tcph);
545 : else
546 0 : tcph->tcp_csum = rte_ipv6_udptcp_cksum(nw_hdr, tcph);
547 :
548 0 : if (tcph->tcp_csum == 0xffff)
549 0 : tcph->tcp_csum = 0;
550 :
551 0 : if (csum != rte_be_to_cpu_16(tcph->tcp_csum)) {
552 0 : vr_dpdk_pfree(m, vif, VP_DROP_CKSUM_ERR);
553 0 : ret = GRO_MERGED;
554 0 : goto func_exit;
555 : }
556 : }
557 :
558 0 : if (!is_ipv6)
559 0 : ret = rte_hash_lookup_data(lcore->gro.gro_tbl_v4_handle, &flow4, (void*)&entry);
560 : else
561 0 : ret = rte_hash_lookup_data(lcore->gro.gro_tbl_v6_handle, &flow6, (void*)&entry);
562 :
563 0 : if (ret >= 0) {
564 : /* Update flow */
565 :
566 : /* Check if packet needs to be flushed */
567 0 : if (flush) {
568 0 : dpdk_gro_flush(&lcore->gro, key, entry);
569 0 : ret = GRO_CANNOT;
570 0 : goto func_exit;
571 : }
572 :
573 : /* Flush now if appending will result in overflow. */
574 0 : if (entry->p_len > (65535 - tcp_data_len)) {
575 0 : dpdk_gro_flush(&lcore->gro, key, entry);
576 0 : goto create;
577 : }
578 :
579 : /* Try to append the new segment. */
580 0 : if (unlikely(rte_be_to_cpu_32(tcph->tcp_seq) != entry->next_seq ||
581 : (tcp_data_len == 0))) {
582 : /* Out of order packet or duplicate ACK. */
583 0 : dpdk_gro_flush(&lcore->gro, key, entry);
584 0 : ret = GRO_CANNOT;
585 0 : goto func_exit;
586 : }
587 :
588 0 : if (l != 0) {
589 0 : uint32_t tsval = rte_be_to_cpu_32(*(ts_ptr + 1));
590 : /* Make sure timestamp values are increasing.
591 : * Also make sure the timestamp echo reply value is valid.
592 : * Note: timestamp echo reply is valid if its value is not 0.
593 : */
594 0 : if (unlikely(entry->tsval > tsval ||
595 : rte_be_to_cpu_32(*(ts_ptr + 2)) == 0)) {
596 0 : dpdk_gro_flush(&lcore->gro, key, entry);
597 0 : ret = GRO_CANNOT;
598 0 : goto func_exit;
599 : }
600 0 : entry->tsval = tsval;
601 0 : entry->tsecr = rte_be_to_cpu_32(*(ts_ptr + 2));
602 : }
603 :
604 0 : entry->next_seq += tcp_data_len;
605 0 : entry->ack_seq = rte_be_to_cpu_32(tcph->tcp_ack);
606 0 : entry->window = rte_be_to_cpu_16(tcph->tcp_win);
607 0 : entry->mbuf_cnt++;
608 0 : entry->ulp_csum += dpdk_gro_rx_csum_fixup(entry, is_ipv6, nw_hdr, tcph, tcp_data_len, ~csum);
609 :
610 0 : entry->p_len += tcp_data_len;
611 :
612 : /*
613 : * Adjust the mbuf so that rte_pktmbuf_mtod(m) points to the first byte of
614 : * the ULP payload. Adjust the mbuf to avoid complications and
615 : * append new segment to existing mbuf chain.
616 : */
617 0 : rte_pktmbuf_adj(m, rte_pktmbuf_data_len(m) - tcp_data_len);
618 :
619 0 : entry->mbuf_tail->next = m;
620 0 : entry->mbuf_tail = rte_pktmbuf_lastseg(m);
621 :
622 : /* If segment size is different, flush */
623 0 : if (entry->seg_sz != tcp_data_len) {
624 0 : dpdk_gro_flush(&lcore->gro, key, entry);
625 0 : ret = GRO_MERGED;
626 0 : goto func_exit;
627 : }
628 :
629 : /* Save cycles */
630 0 : entry->mtime = lcore->lcore_fwd_loops;
631 :
632 0 : ret = GRO_MERGED;
633 : } else {
634 : /* Create new flow */
635 : struct gro_entry *entry;
636 0 : create:
637 0 : if (tcp_data_len == 0) {
638 0 : ret = GRO_CANNOT;
639 0 : goto func_exit;
640 0 : } else if (flush) {
641 0 : ret = GRO_CANNOT;
642 0 : goto func_exit;
643 : }
644 :
645 0 : entry = rte_zmalloc("GRO_entry", sizeof(*entry), RTE_CACHE_LINE_SIZE);
646 0 : if (unlikely(entry == NULL)) {
647 0 : ret = GRO_ERROR;
648 0 : goto func_exit;
649 : }
650 0 : entry->is_ipv6 = is_ipv6;
651 0 : entry->src_vif_idx = src_vif_idx;
652 0 : if (likely(src_vif != NULL))
653 0 : entry->src_vif_gen = src_vif->vif_gen;
654 0 : entry->dst_vif_idx = nh->nh_dev->vif_idx;
655 0 : entry->dst_vif_gen = nh->nh_dev->vif_gen;
656 0 : entry->nh_id = nh_id;
657 0 : entry->mtime = lcore->lcore_fwd_loops;
658 0 : entry->p_len = ip_pkt_len;
659 0 : entry->next_seq = rte_be_to_cpu_32(tcph->tcp_seq) + tcp_data_len;
660 0 : entry->ack_seq = rte_be_to_cpu_32(tcph->tcp_ack);
661 0 : entry->window = rte_be_to_cpu_16(tcph->tcp_win);
662 0 : entry->seg_sz = tcp_data_len;
663 0 : if (!is_ipv6)
664 0 : entry->le_ip4 = nw_hdr;
665 : else
666 0 : entry->le_ip6 = nw_hdr;
667 0 : entry->mbuf_cnt++;
668 0 : if (unlikely(l != 0)) {
669 0 : entry->timestamp = 1;
670 0 : entry->tsval = rte_be_to_cpu_32(*(ts_ptr + 1));
671 0 : entry->tsecr = rte_be_to_cpu_32(*(ts_ptr + 2));
672 : }
673 0 : entry->mbuf_head = m;
674 0 : entry->mbuf_tail = rte_pktmbuf_lastseg(m);
675 :
676 0 : entry->ulp_csum = dpdk_gro_rx_csum_fixup(entry, is_ipv6, nw_hdr, tcph, tcp_data_len, ~csum);
677 0 : tcph->tcp_csum = rte_cpu_to_be_16(csum);/* Restore checksum on first packet. */
678 0 : lcore->gro.gro_flows++;
679 :
680 0 : if (!is_ipv6) {
681 0 : ret = (rte_hash_add_key_data(lcore->gro.gro_tbl_v4_handle, &flow4, entry) < 0)?
682 0 : GRO_ERROR : GRO_MERGED;
683 : } else {
684 0 : ret = (rte_hash_add_key_data(lcore->gro.gro_tbl_v6_handle, &flow6, entry) < 0)?
685 0 : GRO_ERROR : GRO_MERGED;
686 : }
687 :
688 0 : if (ret == GRO_MERGED) {
689 : /* point mbuf to network header */
690 0 : rte_pktmbuf_adj(m, pkt_get_network_header_off(pkt)- pkt_head_space(pkt));
691 0 : pkt_pull(pkt, pkt_get_network_header_off(pkt)- pkt_head_space(pkt));
692 0 : } else if (ret == GRO_ERROR) {
693 0 : rte_free(entry);
694 : }
695 : }
696 :
697 0 : func_exit:
698 0 : if (ret != GRO_MERGED) {
699 0 : pkt_push(pkt, sizeof(struct vr_gro));
700 0 : rte_pktmbuf_prepend(m, sizeof(struct vr_gro));
701 : }
702 0 : return (ret == GRO_MERGED)?1:0;
703 : }
704 :
705 :
|
