Line data Source code
1 : /*
2 : * vr_flow.c -- flow handling
3 : *
4 : * Copyright (c) 2013 Juniper Networks, Inc. All rights reserved.
5 : * Copyright Contributors to the OpenSDN project
6 : */
7 : #include <vr_os.h>
8 : #include <vr_types.h>
9 : #include <vrouter.h>
10 : #include <vr_packet.h>
11 : #include <vr_htable.h>
12 : #include <vr_flow.h>
13 : #include <vr_mirror.h>
14 : #include "vr_interface.h"
15 : #include "vr_sandesh.h"
16 : #include "vr_message.h"
17 : #include "vr_btable.h"
18 : #include "vr_fragment.h"
19 : #include "vr_datapath.h"
20 : #include "vr_hash.h"
21 : #include "vr_ip_mtrie.h"
22 : #include "vr_bridge.h"
23 : #include "vr_vrf_table.h"
24 : #include "vr_nexthop.h"
25 :
26 : #include "vr_offloads_dp.h"
27 :
28 : #define VR_NUM_FLOW_TABLES 1
29 :
30 : #define VR_NUM_OFLOW_TABLES 1
31 :
32 : #define VR_DEF_MAX_FLOW_TABLE_HOLD_COUNT 8192
33 :
34 : unsigned int vr_flow_entries = VR_DEF_FLOW_ENTRIES;
35 : unsigned int vr_oflow_entries = 0;
36 : /*
37 : * Knob to unconditionally close flow on TCP RST;
38 : * If this knob is set, the flow would be closed
39 : * on receiving a TCP RST without doing any seqnum
40 : * validation (for backward compatibility with older
41 : * implementation);
42 : * If this knob is off, TCP RST seqnum validation
43 : * as per RFC 5961 sec 3.2 will be done. If the
44 : * validation fails, the RST will be ignored.
45 : * By default the knob is off;
46 : */
47 : unsigned int vr_uncond_close_flow_on_tcp_rst = 0;
48 :
49 : /*
50 : * host can provide its own memory . Point in case is the DPDK. In DPDK,
51 : * we allocate the table from hugepages and just ask the flow module to
52 : * use those tables
53 : */
54 : void *vr_flow_table = NULL;
55 : void *vr_oflow_table = NULL;
56 : /*
57 : * The flow table memory can also be a file that could be mapped. The path
58 : * is set by somebody and passed to agent for it to map
59 : */
60 : unsigned char *vr_flow_path;
61 : unsigned int vr_flow_hold_limit = VR_DEF_MAX_FLOW_TABLE_HOLD_COUNT;
62 :
63 : #if defined(__linux__) && defined(__KERNEL__)
64 : extern short vr_flow_major;
65 : #endif
66 :
67 : void vr_flow_defer_cb(struct vrouter *router, void *arg);
68 :
69 : uint32_t vr_hashrnd = 0;
70 : int hashrnd_inited = 0;
71 :
72 109 : static struct vr_flow_entry *vr_flow_bucket_first_entry(struct vrouter *router, struct vr_flow *key) {
73 109 : return (struct vr_flow_entry*)
74 109 : vr_htable_get_bucket(router->vr_flow_table, key, key->flow_key_len);
75 : }
76 :
77 : /*
78 : * The vr_flow_bucket_lock and vr_flow_bucket_unlock callbacks may not be
79 : * implemented for all instances of host_os. Those callback are needed only for
80 : * implementations that require strict race condition controll for
81 : * hardware offloading (like DPDK).
82 : * Those helper functions call the locks only when they are available.
83 : */
84 109 : static void vr_flow_bucket_may_lock(struct vr_flow_entry *fe) {
85 109 : if (vr_flow_bucket_lock && vr_flow_bucket_unlock)
86 109 : vr_flow_bucket_lock(fe);
87 109 : }
88 :
89 109 : static void vr_flow_bucket_may_unlock(struct vr_flow_entry *fe) {
90 109 : if (vr_flow_bucket_lock && vr_flow_bucket_unlock)
91 109 : vr_flow_bucket_unlock(fe);
92 109 : }
93 :
94 : static void vr_flush_entry(struct vrouter *, struct vr_flow_entry *,
95 : struct vr_flow_md *, struct vr_forwarding_md *);
96 : static void __vr_flow_flush_hold_queue(struct vrouter *, struct vr_flow_entry *,
97 : struct vr_forwarding_md *, struct vr_flow_queue *);
98 : static void vr_flow_set_forwarding_md(struct vrouter *, struct vr_flow_entry *,
99 : unsigned int, struct vr_forwarding_md *);
100 : static int
101 : __vr_flow_schedule_transition(struct vrouter *, struct vr_flow_entry *,
102 : unsigned int, unsigned short);
103 : static bool vr_flow_is_fat_flow(struct vrouter *, struct vr_packet *,
104 : struct vr_flow_entry *);
105 :
106 : struct vr_flow_entry *vr_find_flow(struct vrouter *, struct vr_flow *,
107 : uint8_t, unsigned int *);
108 : unsigned int vr_trap_flow(struct vrouter *, struct vr_flow_entry *,
109 : struct vr_packet *, unsigned int, struct vr_flow_stats *,
110 : struct vr_packet_node *);
111 : extern struct vr_nexthop *vr_inet_ip_lookup(unsigned short, uint32_t);
112 : extern struct vr_nexthop *vr_inet6_ip_lookup(unsigned short, uint8_t *);
113 :
114 : bool
115 1 : vr_valid_link_local_port(struct vrouter *router, int family,
116 : int proto, int port)
117 : {
118 : unsigned char data;
119 : unsigned int tmp;
120 :
121 1 : if (!router->vr_link_local_ports)
122 0 : return false;
123 :
124 1 : if ((family != AF_INET) ||
125 1 : ((proto != VR_IP_PROTO_TCP) && (proto != VR_IP_PROTO_UDP) &&
126 : (proto != VR_IP_PROTO_ICMP)))
127 0 : return false;
128 :
129 1 : if ((port < VR_DYNAMIC_PORT_START) || (port > VR_DYNAMIC_PORT_END))
130 0 : return false;
131 :
132 1 : tmp = port - VR_DYNAMIC_PORT_START;
133 1 : if (proto == VR_IP_PROTO_UDP)
134 1 : tmp += (router->vr_link_local_ports_size * 8 / VR_LL_RP_MAX);
135 1 : if (proto == VR_IP_PROTO_ICMP)
136 0 : tmp += (router->vr_link_local_ports_size * 8 * VR_LL_RP_ICMP_INDEX /
137 : VR_LL_RP_MAX);
138 :
139 1 : data = router->vr_link_local_ports[(tmp / 8)];
140 1 : if (data & (1 << (tmp % 8)))
141 1 : return true;
142 :
143 0 : return false;
144 : }
145 :
146 : static void
147 3 : vr_clear_link_local_port(struct vrouter *router, int family,
148 : int proto, int port)
149 : {
150 : unsigned char *data;
151 : unsigned int tmp;
152 :
153 3 : if (!router->vr_link_local_ports)
154 0 : return;
155 :
156 3 : if ((family != AF_INET) ||
157 3 : ((proto != VR_IP_PROTO_TCP) && (proto != VR_IP_PROTO_UDP) &&
158 : (proto != VR_IP_PROTO_ICMP)))
159 0 : return;
160 :
161 3 : if ((port < VR_DYNAMIC_PORT_START) || (port > VR_DYNAMIC_PORT_END))
162 0 : return;
163 :
164 3 : tmp = port - VR_DYNAMIC_PORT_START;
165 3 : if (proto == VR_IP_PROTO_UDP)
166 2 : tmp += (router->vr_link_local_ports_size * 8 / VR_LL_RP_MAX);
167 3 : if (proto == VR_IP_PROTO_ICMP)
168 1 : tmp += ((router->vr_link_local_ports_size * 8 * VR_LL_RP_ICMP_INDEX)/
169 : VR_LL_RP_MAX);
170 :
171 3 : data = &router->vr_link_local_ports[(tmp / 8)];
172 3 : *data &= (~(1 << (tmp % 8)));
173 :
174 3 : return;
175 : }
176 :
177 : static void
178 2 : vr_set_link_local_port(struct vrouter *router, int family,
179 : int proto, int port)
180 : {
181 : unsigned char *data;
182 : unsigned int tmp;
183 :
184 2 : if (!router->vr_link_local_ports)
185 0 : return;
186 :
187 2 : if ((family != AF_INET) ||
188 2 : ((proto != VR_IP_PROTO_TCP) && (proto != VR_IP_PROTO_UDP) &&
189 : (proto != VR_IP_PROTO_ICMP)))
190 0 : return;
191 :
192 2 : if ((port < VR_DYNAMIC_PORT_START) || (port > VR_DYNAMIC_PORT_END))
193 0 : return;
194 :
195 2 : tmp = port - VR_DYNAMIC_PORT_START;
196 2 : if (proto == VR_IP_PROTO_UDP)
197 1 : tmp += (router->vr_link_local_ports_size * 8 / VR_LL_RP_MAX);
198 2 : if (proto == VR_IP_PROTO_ICMP)
199 1 : tmp += ((router->vr_link_local_ports_size * 8 * VR_LL_RP_ICMP_INDEX)/
200 : VR_LL_RP_MAX);
201 :
202 2 : data = &router->vr_link_local_ports[tmp / 8];
203 2 : *data |= (1 << (tmp % 8));
204 :
205 2 : return;
206 : }
207 :
208 : static void
209 215 : vr_flow_reset_mirror(struct vrouter *router, struct vr_flow_entry *fe,
210 : unsigned int index)
211 : {
212 215 : if (fe->fe_flags & VR_FLOW_FLAG_MIRROR) {
213 2 : fe->fe_mirror_id = VR_MAX_MIRROR_INDICES;
214 2 : fe->fe_sec_mirror_id = VR_MAX_MIRROR_INDICES;
215 2 : if (fe->fe_mme) {
216 0 : vr_mirror_meta_entry_del(router, fe->fe_mme);
217 0 : fe->fe_mme = NULL;
218 0 : vr_offload_flow_meta_data_set(index, 0, 0, 0);
219 : }
220 : }
221 215 : fe->fe_flags &= ~VR_FLOW_FLAG_MIRROR;
222 215 : fe->fe_mirror_id = VR_MAX_MIRROR_INDICES;
223 215 : fe->fe_sec_mirror_id = VR_MAX_MIRROR_INDICES;
224 :
225 215 : return;
226 : }
227 :
228 : static void
229 109 : vr_init_flow_entry(struct vr_flow_entry *fe)
230 : {
231 109 : fe->fe_rflow = -1;
232 109 : fe->fe_mirror_id = VR_MAX_MIRROR_INDICES;
233 109 : fe->fe_sec_mirror_id = VR_MAX_MIRROR_INDICES;
234 109 : fe->fe_ecmp_nh_index = -1;
235 :
236 109 : return;
237 : }
238 :
239 :
240 : static void
241 117 : __vr_flow_reset_entry(struct vrouter *router, struct vr_flow_entry *fe)
242 : {
243 117 : if (fe->fe_hold_list) {
244 0 : vr_printf("vrouter: Potential memory leak @ %s:%d\n",
245 : __FILE__, __LINE__);
246 : }
247 117 : fe->fe_hold_list = NULL;
248 117 : fe->fe_key.flow_key_len = 0;
249 :
250 117 : (void)vr_offload_flow_del(fe);
251 :
252 117 : vr_flow_reset_mirror(router, fe, fe->fe_hentry.hentry_index);
253 117 : fe->fe_ecmp_nh_index = -1;
254 117 : fe->fe_src_nh_index = NH_DISCARD_ID;
255 117 : fe->fe_rflow = -1;
256 117 : fe->fe_action = VR_FLOW_ACTION_DROP;
257 117 : fe->fe_udp_src_port = 0;
258 117 : fe->fe_tcp_flags = 0;
259 117 : fe->fe_flags &=
260 : (VR_FLOW_FLAG_ACTIVE | VR_FLOW_FLAG_EVICTED |
261 : VR_FLOW_FLAG_NEW_FLOW | VR_FLOW_FLAG_DELETE_MARKED);
262 117 : fe->fe_flags1 &=
263 : ~(VR_FLOW_FLAG1_HBS_LEFT | VR_FLOW_FLAG1_HBS_RIGHT);
264 117 : fe->fe_ttl = 0;
265 117 : fe->fe_underlay_ecmp_index = -1;
266 117 : fe->fe_src_info = 0;
267 :
268 117 : return;
269 : }
270 :
271 : static void
272 109 : vr_flow_reset_entry(struct vrouter *router, struct vr_flow_entry *fe)
273 : {
274 109 : __vr_flow_reset_entry(router, fe);
275 109 : memset(&fe->fe_stats, 0, sizeof(fe->fe_stats));
276 109 : fe->fe_type = VP_TYPE_NULL;
277 109 : fe->fe_flags = 0;
278 :
279 109 : vr_htable_release_hentry(router->vr_flow_table, &fe->fe_hentry);
280 109 : return;
281 : }
282 :
283 : static void
284 8 : vr_flow_reset_active_entry(struct vrouter *router, struct vr_flow_entry *fe)
285 : {
286 8 : __vr_flow_reset_entry(router, fe);
287 8 : vr_htable_release_hentry(router->vr_flow_table, &fe->fe_hentry);
288 8 : return;
289 : }
290 :
291 :
292 : static vr_hentry_key
293 62 : vr_flow_get_key(vr_htable_t flow_table, vr_hentry_t *entry,
294 : unsigned int *key_len)
295 : {
296 62 : struct vr_flow_entry *fe = CONTAINER_OF(fe_hentry,
297 : struct vr_flow_entry, entry);
298 :
299 62 : if ((fe->fe_flags & VR_FLOW_FLAG_DELETE_MARKED) ||
300 62 : !(fe->fe_flags & VR_FLOW_FLAG_ACTIVE))
301 0 : return NULL;
302 :
303 62 : if (key_len)
304 62 : *key_len = fe->fe_key.flow_key_len;
305 :
306 62 : return &fe->fe_key;
307 : }
308 :
309 : uint32_t
310 1 : vr_flow_get_rflow_src_info(struct vrouter *router,
311 : struct vr_flow_entry *fe)
312 : {
313 : struct vr_flow_entry *rfe;
314 :
315 1 : if ((!fe) || !(fe->fe_flags & VR_RFLOW_VALID))
316 0 : return (unsigned int)-1;
317 :
318 1 : rfe = vr_flow_get_entry(router, fe->fe_rflow);
319 1 : if (!rfe)
320 0 : return (unsigned int)-1;
321 :
322 1 : return rfe->fe_src_info;
323 : }
324 :
325 : static inline bool
326 109 : vr_flow_set_active(struct vr_flow_entry *fe)
327 : {
328 109 : return vr_sync_bool_compare_and_swap_16u(&fe->fe_flags,
329 : fe->fe_flags & ~VR_FLOW_FLAG_ACTIVE,
330 : VR_FLOW_FLAG_ACTIVE | VR_FLOW_FLAG_NEW_FLOW);
331 : }
332 :
333 : /*
334 : * This api is called to get flow table size
335 : * NOTE: This api is also called from agent (via sandesh)
336 : * very early during init even before flow table is
337 : * initialized to calculate huge page table size.
338 : * Hence this api should not use vr_htable structures
339 : */
340 : unsigned int
341 13 : vr_flow_table_size(struct vrouter *router)
342 : {
343 : // set the overlflow flow table entries
344 13 : vr_compute_size_oflow_table();
345 13 : return (VR_FLOW_TABLE_SIZE + VR_OFLOW_TABLE_SIZE);
346 : }
347 :
348 : unsigned int
349 13 : vr_flow_table_used_oflow_entries(struct vrouter *router)
350 : {
351 13 : return vr_htable_used_oflow_entries(router->vr_flow_table);
352 : }
353 :
354 : unsigned int
355 13 : vr_flow_table_used_total_entries(struct vrouter *router)
356 : {
357 13 : return vr_htable_used_total_entries(router->vr_flow_table);
358 : }
359 : /*
360 : * this is used by the mmap code. mmap sees the whole flow table
361 : * (including the overflow table) as one large table. so, given
362 : * an offset into that large memory, we should return the correct
363 : * virtual address
364 : */
365 : void *
366 0 : vr_flow_get_va(struct vrouter *router, uint64_t offset)
367 : {
368 0 : return vr_htable_get_address(router->vr_flow_table, offset);
369 : }
370 :
371 : struct vr_flow_entry *
372 1236 : vr_flow_get_entry(struct vrouter *router, int index)
373 : {
374 1236 : if (index < 0)
375 104 : return NULL;
376 :
377 1132 : return (struct vr_flow_entry *)
378 1132 : vr_htable_get_hentry_by_index(router->vr_flow_table, index);
379 : }
380 :
381 : static inline void
382 120 : vr_flow_stop_modify(struct vrouter *router, struct vr_flow_entry *fe)
383 : {
384 120 : if (!fe)
385 0 : return;
386 :
387 120 : (void)vr_sync_and_and_fetch_16u(&fe->fe_flags, ~VR_FLOW_FLAG_MODIFIED);
388 120 : return;
389 : }
390 :
391 : static inline bool
392 222 : vr_flow_start_modify(struct vrouter *router, struct vr_flow_entry *fe)
393 : {
394 : unsigned short flags;
395 :
396 222 : flags = fe->fe_flags;
397 222 : if (!(flags & (VR_FLOW_FLAG_MODIFIED | VR_FLOW_FLAG_EVICTED |
398 : VR_FLOW_FLAG_NEW_FLOW))) {
399 218 : if (vr_sync_bool_compare_and_swap_16u(&fe->fe_flags, flags,
400 : flags | VR_FLOW_FLAG_MODIFIED)) {
401 218 : return true;
402 : }
403 : }
404 :
405 4 : return false;
406 : }
407 :
408 :
409 : /* Non-static due to RCU callback pointer comparison in vRouter/DPDK */
410 : static void
411 2 : vr_flow_flush_hold_queue(struct vrouter *router, struct vr_flow_entry *fe,
412 : struct vr_flow_queue *vfq)
413 : {
414 : struct vr_forwarding_md fmd;
415 :
416 2 : if (vfq) {
417 2 : vr_init_forwarding_md(&fmd);
418 2 : vr_flow_set_forwarding_md(router, fe, vfq->vfq_index, &fmd);
419 2 : __vr_flow_flush_hold_queue(router, fe, &fmd, vfq);
420 : }
421 :
422 2 : return;
423 : }
424 :
425 : static void
426 8 : vr_flow_evict_flow(struct vrouter *router, struct vr_flow_entry *fe)
427 : {
428 : unsigned short flags;
429 :
430 8 : if (!fe)
431 0 : return;
432 :
433 8 : if ((fe->fe_flags & VR_FLOW_FLAG_ACTIVE) &&
434 8 : (fe->fe_flags & VR_FLOW_FLAG_EVICT_CANDIDATE)) {
435 8 : flags = fe->fe_flags | VR_FLOW_FLAG_ACTIVE |
436 : VR_FLOW_FLAG_EVICT_CANDIDATE;
437 8 : if (vr_sync_bool_compare_and_swap_16u(&fe->fe_flags, flags,
438 : (flags ^ VR_FLOW_FLAG_EVICT_CANDIDATE) |
439 : VR_FLOW_FLAG_EVICTED)) {
440 8 : vr_flow_stop_modify(router, fe);
441 8 : vr_flow_reset_active_entry(router, fe);
442 : }
443 : }
444 :
445 8 : return;
446 : }
447 :
448 : void
449 6 : vr_flow_defer_cb(struct vrouter *router, void *arg)
450 : {
451 : struct vr_defer_data *defer;
452 : struct vr_flow_entry *fe, *rfe;
453 : struct vr_flow_queue *vfq;
454 : struct vr_flow_defer_data *vfdd;
455 :
456 6 : defer = (struct vr_defer_data *)arg;
457 6 : if (!defer)
458 0 : return;
459 :
460 6 : vfdd = (struct vr_flow_defer_data *)defer->vdd_data;
461 6 : if (!vfdd)
462 0 : return;
463 6 : fe = vfdd->vfdd_fe;
464 :
465 6 : vfq = (struct vr_flow_queue *)vfdd->vfdd_flow_queue;
466 6 : if (vfq) {
467 2 : vr_flow_flush_hold_queue(router, fe, vfq);
468 2 : vr_free(vfq, VR_FLOW_QUEUE_OBJECT);
469 2 : vfdd->vfdd_flow_queue = NULL;
470 : }
471 :
472 6 : if (vfdd->vfdd_delete) {
473 0 : vr_flow_reset_entry(router, fe);
474 6 : } else if (vfdd->vfdd_evict_flow) {
475 4 : rfe = vr_flow_get_entry(router, fe->fe_rflow);
476 4 : vr_flow_evict_flow(router, fe);
477 4 : if (rfe)
478 4 : vr_flow_evict_flow(router, rfe);
479 : }
480 :
481 6 : vr_free(vfdd, VR_FLOW_DEFER_DATA_OBJECT);
482 :
483 6 : return;
484 : }
485 :
486 : static void
487 0 : vr_flow_reset_evict(struct vrouter *router, struct vr_flow_entry *fe)
488 : {
489 : unsigned short flags;
490 :
491 0 : if (!fe)
492 0 : return;
493 :
494 0 : flags = fe->fe_flags;
495 0 : if (flags & VR_FLOW_FLAG_EVICT_CANDIDATE) {
496 0 : (void)vr_sync_bool_compare_and_swap_16u(&fe->fe_flags, flags,
497 : (flags ^ VR_FLOW_FLAG_EVICT_CANDIDATE));
498 : }
499 :
500 0 : vr_flow_stop_modify(router, fe);
501 :
502 0 : return;
503 : }
504 :
505 : static void
506 318 : vr_flow_defer(struct vr_flow_md *flmd, struct vr_flow_entry *fe)
507 : {
508 : struct vr_flow_entry *rfe;
509 318 : struct vr_defer_data *vdd = flmd->flmd_defer_data;
510 : struct vr_flow_defer_data *vfdd;
511 :
512 318 : if (!vdd || !vdd->vdd_data) {
513 312 : if (flmd->flmd_flags & VR_FLOW_FLAG_EVICT_CANDIDATE) {
514 0 : if (fe->fe_rflow) {
515 0 : rfe = vr_flow_get_entry(flmd->flmd_router, fe->fe_rflow);
516 0 : vr_flow_reset_evict(flmd->flmd_router, rfe);
517 : }
518 0 : vr_flow_reset_evict(flmd->flmd_router, fe);
519 : }
520 :
521 312 : if (!(flmd->flmd_flags & VR_FLOW_FLAG_ACTIVE)) {
522 98 : vr_flow_reset_entry(flmd->flmd_router, fe);
523 : }
524 :
525 312 : return;
526 : }
527 :
528 6 : vfdd = (struct vr_flow_defer_data *)vdd->vdd_data;
529 6 : vfdd->vfdd_fe = fe;
530 :
531 6 : vr_defer(flmd->flmd_router, vr_flow_defer_cb, (void *)vdd);
532 6 : flmd->flmd_defer_data = NULL;
533 :
534 6 : return;
535 : }
536 :
537 : static struct vr_flow_entry *
538 109 : vr_flow_table_get_free_entry(struct vrouter *router, struct vr_flow *key,
539 : unsigned int *free_index)
540 : {
541 : unsigned short flags;
542 : struct vr_flow_entry *fe;
543 :
544 : fe = (struct vr_flow_entry *)
545 109 : vr_htable_find_free_hentry(router->vr_flow_table, key,
546 109 : key->flow_key_len);
547 109 : if (fe) {
548 109 : flags = fe->fe_flags;
549 109 : if (!(flags & VR_FLOW_FLAG_ACTIVE)) {
550 109 : if (vr_flow_set_active(fe)) {
551 109 : vr_init_flow_entry(fe);
552 : }
553 0 : } else if (flags & VR_FLOW_FLAG_EVICTED) {
554 0 : fe->fe_flags = ((flags & ~VR_FLOW_FLAG_EVICTED) |
555 : VR_FLOW_FLAG_NEW_FLOW);
556 : }
557 :
558 109 : fe->fe_underlay_ecmp_index = -1;
559 :
560 109 : fe->fe_gen_id = (fe->fe_gen_id + 1) %
561 : (1 << (8 * sizeof(fe->fe_gen_id)));
562 109 : *free_index = fe->fe_hentry.hentry_index;
563 : }
564 :
565 109 : return fe;
566 : }
567 :
568 :
569 : static struct vr_flow_entry *
570 109 : vr_flow_get_free_entry(struct vrouter *router, struct vr_flow *key, uint8_t type,
571 : bool need_hold, unsigned int *fe_index)
572 : {
573 109 : struct vr_flow_entry *fe = NULL;
574 :
575 109 : fe = vr_flow_table_get_free_entry(router, key, fe_index);
576 109 : if (fe) {
577 109 : if (need_hold) {
578 5 : fe->fe_hold_list = vr_zalloc(sizeof(struct vr_flow_queue),
579 : VR_FLOW_QUEUE_OBJECT);
580 5 : if (!fe->fe_hold_list) {
581 0 : vr_flow_reset_entry(router, fe);
582 0 : fe = NULL;
583 0 : vr_printf("%s:%d flow reset\n", __func__, __LINE__);
584 0 : return fe;
585 : } else {
586 5 : fe->fe_hold_list->vfq_index = *fe_index;
587 : }
588 : }
589 :
590 109 : fe->fe_type = type;
591 109 : memcpy(&fe->fe_key, key, key->flow_key_len);
592 109 : fe->fe_key.flow_key_len = key->flow_key_len;
593 : }
594 :
595 109 : return fe;
596 : }
597 :
598 :
599 : struct vr_flow_entry *
600 176 : vr_find_flow(struct vrouter *router, struct vr_flow *key,
601 : uint8_t type, unsigned int *fe_index)
602 : {
603 : struct vr_flow_entry *fe;
604 :
605 176 : fe = (struct vr_flow_entry *)vr_htable_find_hentry(router->vr_flow_table,
606 176 : key, key->flow_key_len);
607 176 : if (fe) {
608 62 : if (fe_index)
609 62 : *fe_index = fe->fe_hentry.hentry_index;
610 : }
611 :
612 176 : return fe;
613 : }
614 :
615 :
616 : void
617 28 : vr_flow_fill_pnode(struct vr_packet_node *pnode, struct vr_packet *pkt,
618 : struct vr_forwarding_md *fmd)
619 : {
620 28 : struct vr_ip *ip = (struct vr_ip *)pkt_inner_network_header(pkt);
621 :
622 : /*
623 : * we cannot cache nexthop here. to cache, we need to hold reference
624 : * to the nexthop. to hold a reference, we will have to hold a lock,
625 : * which we cannot. the only known case of misbehavior if we do not
626 : * cache is ECMP. when the packet comes from the fabric, the nexthop
627 : * actually points to a local composite, whereas a route lookup actually
628 : * returns a different nexthop, in which case the ecmp index will return
629 : * a bad nexthop. to avoid that, we will cache the label, and reuse it
630 : */
631 28 : pkt->vp_nh = NULL;
632 :
633 28 : pnode->pl_flags = 0;
634 28 : pnode->pl_vif_idx = pkt->vp_if->vif_idx;
635 :
636 28 : if (fmd) {
637 28 : pnode->pl_outer_src_ip = fmd->fmd_outer_src_ip;
638 28 : pnode->pl_label = fmd->fmd_label;
639 28 : if (vr_fmd_label_is_vxlan_id(fmd))
640 0 : pnode->pl_flags |= PN_FLAG_LABEL_IS_VXLAN_ID;
641 28 : if (fmd->fmd_to_me)
642 0 : pnode->pl_flags |= PN_FLAG_TO_ME;
643 : }
644 :
645 28 : if (ip) {
646 28 : if (vr_ip_is_ip4(ip)) {
647 : /*
648 : * Source IP & Dest IP can change while the packet is in the queue
649 : * (NAT). For e.g.: when the cloned head of a fragment is enqueued
650 : * to the assembler and subsequently dequeued by the assembler, the
651 : * original packet might have undergone a NAT, resulting in wrong
652 : * hash and thus a wrong search for other fragments of the packet.
653 : * Hence, store them here for others interested in the original IPs
654 : */
655 28 : pnode->pl_inner_src_ip = ip->ip_saddr;
656 28 : pnode->pl_inner_dst_ip = ip->ip_daddr;
657 28 : if (vr_ip_fragment_head(ip))
658 5 : pnode->pl_flags |= PN_FLAG_FRAGMENT_HEAD;
659 0 : } else if (vr_ip_is_ip6(ip)) {
660 0 : if (vr_ip6_fragment_head((struct vr_ip6 *)ip))
661 0 : pnode->pl_flags |= PN_FLAG_FRAGMENT_HEAD;
662 : }
663 : }
664 :
665 28 : pnode->pl_dscp = fmd->fmd_dscp;
666 28 : pnode->pl_dotonep = fmd->fmd_dotonep;
667 28 : pnode->pl_vrf = fmd->fmd_dvrf;
668 28 : pnode->pl_vlan = fmd->fmd_vlan;
669 28 : pnode->pl_mirror_vlan = fmd->fmd_mirror_data;
670 :
671 28 : vr_sync_synchronize();
672 28 : pnode->pl_packet = pkt;
673 :
674 28 : return;
675 : }
676 :
677 : static int
678 7 : vr_enqueue_flow(struct vrouter *router, struct vr_flow_entry *fe,
679 : struct vr_packet *pkt, unsigned int index,
680 : struct vr_flow_stats *stats, struct vr_forwarding_md *fmd)
681 : {
682 7 : int ret = 0;
683 : unsigned int i;
684 7 : unsigned short drop_reason = 0;
685 7 : struct vr_flow_queue *vfq = fe->fe_hold_list;
686 : struct vr_packet_node *pnode;
687 :
688 7 : if (!vfq) {
689 0 : drop_reason = VP_DROP_FLOW_UNUSABLE;
690 0 : PKT_LOG(drop_reason, pkt, 0, VR_FLOW_C, __LINE__);
691 0 : goto drop;
692 : }
693 :
694 7 : i = vr_sync_fetch_and_add_32u(&vfq->vfq_entries, 1);
695 7 : if (i >= VR_MAX_FLOW_QUEUE_ENTRIES) {
696 0 : drop_reason = VP_DROP_FLOW_QUEUE_LIMIT_EXCEEDED;
697 0 : PKT_LOG(drop_reason, pkt, 0, VR_FLOW_C, __LINE__);
698 0 : goto drop;
699 : }
700 :
701 7 : pnode = &vfq->vfq_pnodes[i];
702 7 : vr_flow_fill_pnode(pnode, pkt, fmd);
703 7 : if (!i)
704 5 : ret = vr_trap_flow(router, fe, pkt, index, stats, pnode);
705 :
706 7 : return ret;
707 0 : drop:
708 0 : vr_pfree(pkt, drop_reason);
709 0 : return 0;
710 : }
711 :
712 : static flow_result_t
713 6 : vr_flow_nat(struct vr_flow_entry *fe,
714 : struct vr_packet *pkt, struct vr_forwarding_md *fmd)
715 : {
716 6 : if (pkt->vp_type == VP_TYPE_IP)
717 4 : return vr_inet_flow_nat(fe, pkt, fmd);
718 :
719 2 : if (pkt->vp_type == VP_TYPE_IP6)
720 2 : return vr_inet6_flow_nat(fe, pkt, fmd);
721 :
722 0 : PKT_LOG(VP_DROP_FLOW_ACTION_INVALID, pkt, 0, VR_FLOW_C, __LINE__);
723 0 : vr_pfree(pkt, VP_DROP_FLOW_ACTION_INVALID);
724 0 : return FLOW_CONSUMED;
725 : }
726 :
727 : static void
728 395 : vr_flow_set_forwarding_md(struct vrouter *router, struct vr_flow_entry *fe,
729 : unsigned int index, struct vr_forwarding_md *md)
730 : {
731 : struct vr_flow_entry *rfe;
732 :
733 395 : md->fmd_flow_index = index;
734 395 : md->fmd_ecmp_nh_index = fe->fe_ecmp_nh_index;
735 395 : md->fmd_underlay_ecmp_index = fe->fe_underlay_ecmp_index;
736 395 : md->fmd_udp_src_port = fe->fe_udp_src_port;
737 395 : if (fe->fe_flags & VR_RFLOW_VALID) {
738 320 : rfe = vr_flow_get_entry(router, fe->fe_rflow);
739 320 : if (rfe)
740 275 : md->fmd_ecmp_src_nh_index = rfe->fe_ecmp_nh_index;
741 : }
742 :
743 395 : return;
744 : }
745 :
746 : static bool
747 8 : __vr_flow_mark_evict(struct vrouter *router, struct vr_flow_entry *fe)
748 : {
749 : unsigned short flags;
750 :
751 8 : flags = fe->fe_flags;
752 8 : if (flags & VR_FLOW_FLAG_ACTIVE) {
753 8 : flags = vr_sync_fetch_and_or_16u(&fe->fe_flags,
754 : VR_FLOW_FLAG_EVICT_CANDIDATE);
755 8 : if (!(flags & VR_FLOW_FLAG_EVICT_CANDIDATE)) {
756 8 : return true;
757 : }
758 : }
759 :
760 0 : return false;
761 : }
762 :
763 : static void
764 8 : vr_flow_mark_evict(struct vrouter *router, struct vr_flow_entry *fe,
765 : unsigned int index)
766 : {
767 8 : bool evict_forward_flow = true;
768 :
769 8 : struct vr_flow_entry *rfe = NULL;
770 :
771 : /* start modifying the entry */
772 8 : if (!vr_flow_start_modify(router, fe)) {
773 4 : return;
774 : }
775 :
776 4 : if (fe->fe_rflow >= 0) {
777 4 : rfe = vr_flow_get_entry(router, fe->fe_rflow);
778 4 : if (rfe) {
779 4 : evict_forward_flow = false;
780 4 : if (rfe->fe_tcp_flags & VR_FLOW_TCP_DEAD) {
781 4 : if (!vr_flow_start_modify(router, rfe)) {
782 : /* no modification. hence...*/
783 0 : rfe = NULL;
784 : } else {
785 : /* we do not want hold flows to be evicted, just yet */
786 4 : if (((rfe->fe_rflow == index) || (rfe->fe_rflow < 0)) &&
787 4 : (rfe->fe_action != VR_FLOW_ACTION_HOLD)) {
788 4 : evict_forward_flow = __vr_flow_mark_evict(router, rfe);
789 : }
790 : }
791 : } else {
792 : /* no modification. hence...*/
793 0 : rfe = NULL;
794 : }
795 : }
796 : }
797 :
798 : /*
799 : * presence of rfe means that we might need to reset the evict bit
800 : * or at the minimum reset the modified bit under failure conditions
801 : *
802 : * CEM-18166: In case rfe has already been deleted, under special
803 : * circumstances, fe might still have fe_rflow pointing to it.
804 : * In this case (and other cases where rfe is not active anymore),
805 : * evict_forward_flow will remain true, which makes sense because we still
806 : * want to evict this flow, because of TCP FIN or RST.
807 : */
808 4 : if (evict_forward_flow) {
809 4 : if (__vr_flow_mark_evict(router, fe)) {
810 4 : if (!__vr_flow_schedule_transition(router, fe,
811 4 : index, fe->fe_flags)) {
812 4 : return;
813 : } else {
814 0 : goto reset_evict;
815 : }
816 : }
817 : }
818 :
819 : /* stop modifying the forward and the reverse */
820 0 : if (rfe)
821 0 : vr_flow_stop_modify(router, rfe);
822 0 : vr_flow_stop_modify(router, fe);
823 :
824 0 : return;
825 :
826 0 : reset_evict:
827 0 : if (rfe)
828 0 : vr_flow_reset_evict(router, rfe);
829 0 : vr_flow_reset_evict(router, fe);
830 :
831 0 : return;
832 : }
833 :
834 : int16_t
835 21 : vr_flow_get_qos(struct vrouter *router, struct vr_packet *pkt,
836 : struct vr_forwarding_md *fmd)
837 : {
838 : struct vr_flow_entry *fe;
839 :
840 21 : if (fmd->fmd_flow_index >= 0) {
841 21 : fe = vr_flow_get_entry(router, fmd->fmd_flow_index);
842 21 : if (fe)
843 21 : return fe->fe_qos_id;
844 : }
845 :
846 0 : return -1;
847 : }
848 :
849 : static int
850 0 : vr_rflow_update_ecmp_index(struct vrouter *router, struct vr_flow_entry *fe,
851 : unsigned int new_ecmp_index, struct vr_forwarding_md *fmd)
852 : {
853 : struct vr_flow_entry *rfe;
854 :
855 0 : if (new_ecmp_index == -1)
856 0 : return -1;
857 :
858 0 : rfe = vr_flow_get_entry(router, fe->fe_rflow);
859 0 : if ((!rfe) || (rfe->fe_flags & VR_FLOW_FLAG_DELETE_MARKED))
860 0 : return -1;
861 :
862 0 : rfe->fe_ecmp_nh_index = new_ecmp_index;
863 :
864 : /* Update hardware reverse flow. */
865 0 : (void)vr_offload_flow_set(rfe, fe->fe_rflow, NULL);
866 :
867 0 : fmd->fmd_ecmp_src_nh_index = new_ecmp_index;
868 :
869 0 : return 0;
870 : }
871 :
872 :
873 : int
874 1 : vr_flow_update_ecmp_index(struct vrouter *router, struct vr_flow_entry *fe,
875 : unsigned int new_ecmp_index, struct vr_forwarding_md *fmd)
876 : {
877 :
878 1 : if (new_ecmp_index == -1)
879 0 : return -1;
880 :
881 1 : if ((!fe) || (fe->fe_flags & VR_FLOW_FLAG_DELETE_MARKED))
882 0 : return -1;
883 :
884 : /* If RPF verification is manipulating this flow, let it succeed */
885 1 : (void)vr_sync_bool_compare_and_swap_8s(&fe->fe_ecmp_nh_index,
886 : fmd->fmd_ecmp_nh_index, new_ecmp_index);
887 :
888 1 : fmd->fmd_ecmp_nh_index = fe->fe_ecmp_nh_index;
889 :
890 1 : return 0;
891 : }
892 :
893 : static flow_result_t
894 61 : vr_flow_action_default(struct vrouter *router, struct vr_flow_entry *fe,
895 : unsigned int index, struct vr_packet *pkt,
896 : struct vr_forwarding_md *fmd)
897 : {
898 61 : unsigned int ip_inc_diff_cksum = 0;
899 : struct vr_ip *ip;
900 61 : flow_result_t result = FLOW_CONSUMED;
901 :
902 : struct vr_forwarding_md mirror_fmd;
903 :
904 61 : fmd->fmd_dvrf = fe->fe_vrf;
905 : /*
906 : * for now, we will not use dvrf if VRFT is set, because the RPF
907 : * check needs to happen in the source vrf
908 : */
909 :
910 61 : if (fe->fe_flags & VR_FLOW_FLAG_VRFT) {
911 5 : if (fmd->fmd_dvrf != fe->fe_dvrf) {
912 1 : fmd->fmd_dvrf = fe->fe_dvrf;
913 1 : fmd->fmd_to_me = 1;
914 : }
915 : }
916 :
917 61 : if (fe->fe_flags & VR_FLOW_FLAG_MIRROR) {
918 2 : if (fe->fe_mirror_id < VR_MAX_MIRROR_INDICES) {
919 2 : mirror_fmd = *fmd;
920 2 : mirror_fmd.fmd_ecmp_nh_index = -1;
921 2 : mirror_fmd.fmd_underlay_ecmp_index = -1;
922 2 : vr_mirror(router, fe->fe_mirror_id, pkt, &mirror_fmd,
923 : MIRROR_TYPE_ACL);
924 2 : fmd->fmd_mirror_data = mirror_fmd.fmd_mirror_data;
925 : }
926 :
927 2 : if (fe->fe_sec_mirror_id < VR_MAX_MIRROR_INDICES) {
928 0 : mirror_fmd = *fmd;
929 0 : mirror_fmd.fmd_ecmp_nh_index = -1;
930 0 : mirror_fmd.fmd_underlay_ecmp_index = -1;
931 0 : vr_mirror(router, fe->fe_sec_mirror_id, pkt, &mirror_fmd,
932 : MIRROR_TYPE_ACL);
933 0 : fmd->fmd_mirror_data = mirror_fmd.fmd_mirror_data;
934 : }
935 : }
936 :
937 61 : switch (fe->fe_action) {
938 0 : case VR_FLOW_ACTION_DROP:
939 0 : PKT_LOG(VP_DROP_FLOW_ACTION_DROP, pkt, 0, VR_FLOW_C, __LINE__);
940 0 : vr_pfree(pkt, VP_DROP_FLOW_ACTION_DROP);
941 0 : result = FLOW_CONSUMED;
942 0 : break;
943 :
944 55 : case VR_FLOW_ACTION_FORWARD:
945 55 : result = FLOW_FORWARD;
946 55 : break;
947 :
948 6 : case VR_FLOW_ACTION_NAT:
949 6 : result = vr_flow_nat(fe, pkt, fmd);
950 6 : break;
951 :
952 0 : default:
953 0 : PKT_LOG(VP_DROP_FLOW_ACTION_INVALID, pkt, 0, VR_FLOW_C, __LINE__);
954 0 : vr_pfree(pkt, VP_DROP_FLOW_ACTION_INVALID);
955 0 : result = FLOW_CONSUMED;
956 0 : break;
957 : }
958 :
959 61 : if (result == FLOW_FORWARD) {
960 61 : if (pkt->vp_type == VP_TYPE_IP) {
961 57 : ip = (struct vr_ip *)pkt_network_header(pkt);
962 57 : if (ip) {
963 57 : if (fe->fe_ttl && (fe->fe_ttl != ip->ip_ttl)) {
964 0 : vr_incremental_diff(ip->ip_ttl, fe->fe_ttl, &ip_inc_diff_cksum);
965 0 : ip->ip_ttl = fe->fe_ttl;
966 :
967 0 : if (ip_inc_diff_cksum)
968 0 : vr_ip_incremental_csum(ip, ip_inc_diff_cksum);
969 : }
970 : }
971 : }
972 : }
973 :
974 61 : if (fe->fe_tcp_flags & VR_FLOW_TCP_DEAD)
975 4 : vr_flow_mark_evict(router, fe, index);
976 :
977 61 : return result;
978 : }
979 :
980 : static flow_result_t
981 62 : vr_flow_action_hbs(struct vrouter *router, struct vr_flow_entry *fe,
982 : unsigned int index, struct vr_packet *pkt,
983 : struct vr_forwarding_md *fmd)
984 : {
985 62 : struct vr_vrf_table_entry *vrf_entry = NULL;
986 : uint16_t dvrf;
987 :
988 62 : dvrf = (fmd->fmd_dvrf == -1)? fe->fe_vrf: fmd->fmd_dvrf;
989 :
990 : /* If not HBS flow, return */
991 62 : if (!(fe->fe_flags1 & VR_FLOW_FLAG1_HBS_MASK))
992 53 : return FLOW_FORWARD;
993 :
994 9 : vrf_entry = vrouter_get_vrf_table(router, dvrf);
995 :
996 : /* Packet entering vrouter and going to hbs-l or hbs-r
997 : * for hbs-flows
998 : */
999 9 : if (fe->fe_flags1 & VR_FLOW_FLAG1_HBS_LEFT) {
1000 : struct vr_interface *hbs_l;
1001 :
1002 : /* Flow is marked as HBS, but there is no HBS instance,
1003 : * so drop the packet
1004 : */
1005 4 : if (!vrf_entry) {
1006 0 : PKT_LOG(VP_DROP_INVALID_HBS_PKT, pkt, 0, VR_FLOW_C, __LINE__);
1007 0 : goto drop_pkt;
1008 : }
1009 :
1010 4 : hbs_l = vrf_entry->hbs_l_vif;
1011 4 : if (hbs_l) {
1012 : struct vr_eth_hbs_md *eth_hbs;
1013 :
1014 4 : if (vif_is_virtual(pkt->vp_if)) {
1015 : /* Packet entering vrouter from vmi and going to hbs-l,
1016 : * encode flow_id in src mac
1017 : */
1018 4 : eth_hbs = (struct vr_eth_hbs_md*)pkt_data(pkt);
1019 4 : eth_hbs->flow_id_smac = htonl(index);
1020 4 : eth_hbs->magic_smac = htons(VR_HBS_SMAC_MAGIC | VR_HBS_FROM_VMI);
1021 0 : } else if (vif_is_fabric(pkt->vp_if)) {
1022 : /* Packet entering vrouter from fabric and going to hbs-l
1023 : * (service chaining case) encode flow_id in dst mac
1024 : */
1025 :
1026 : /* Add ethernet header if there is none */
1027 0 : if (pkt_data(pkt) == pkt_network_header(pkt)) {
1028 0 : struct vr_eth *eth = (struct vr_eth*)pkt_data(pkt);
1029 0 : eth = (struct vr_eth *)pkt_push(pkt, VR_ETHER_HLEN);
1030 0 : memcpy(eth->eth_dmac, hbs_l->vif_mac, VR_ETHER_ALEN);
1031 0 : memcpy(eth->eth_smac, hbs_l->vif_mac, VR_ETHER_ALEN);
1032 0 : if (pkt->vp_type == VP_TYPE_IP) {
1033 0 : eth->eth_proto = htons(VR_ETH_PROTO_IP);
1034 0 : } else if (pkt->vp_type == VP_TYPE_IP6) {
1035 0 : eth->eth_proto = htons(VR_ETH_PROTO_IP6);
1036 : }
1037 : }
1038 :
1039 0 : eth_hbs = (struct vr_eth_hbs_md*)pkt_data(pkt);
1040 0 : eth_hbs->flow_id_dmac = htonl(index);
1041 0 : eth_hbs->magic_dmac = htons(VR_HBS_DMAC_MAGIC | VR_HBS_FROM_FABRIC);
1042 : }
1043 4 : pkt->vp_if = hbs_l;
1044 : /*
1045 : * If for some reason, we have GRO flag set and we have not invoked
1046 : * the GRO, we need to unset
1047 : */
1048 4 : vr_pkt_unset_gro(pkt);
1049 4 : pkt->vp_nh = NULL;
1050 4 : hbs_l->vif_tx(hbs_l, pkt, fmd);
1051 4 : return FLOW_HELD;
1052 : }
1053 0 : return FLOW_FORWARD;
1054 5 : } else if (fe->fe_flags1 & VR_FLOW_FLAG1_HBS_RIGHT) {
1055 : struct vr_interface *hbs_r;
1056 :
1057 : /* Flow is marked as HBS, but there is no HBS instance,
1058 : * so drop the packet
1059 : */
1060 5 : if (!vrf_entry) {
1061 0 : PKT_LOG(VP_DROP_INVALID_HBS_PKT, pkt, 0, VR_FLOW_C, __LINE__);
1062 0 : goto drop_pkt;
1063 : }
1064 :
1065 5 : hbs_r = vrf_entry->hbs_r_vif;
1066 5 : if (hbs_r) {
1067 : /* Packet entering vrouter from fabric and going to hbs-r,
1068 : * encode flow_id in dst mac
1069 : *
1070 : * Note: packet can also enter vrouter and goto hbs-r
1071 : * from vmi (instead of fabric) in case of intra-compute
1072 : */
1073 : struct vr_eth *eth;
1074 : struct vr_eth_hbs_md *eth_hbs;
1075 5 : bool pkt_is_l3 = false;
1076 :
1077 : /* Add ethernet header if there is none */
1078 5 : if (pkt_data(pkt) == pkt_network_header(pkt)) {
1079 1 : eth = (struct vr_eth *)pkt_push(pkt, VR_ETHER_HLEN);
1080 1 : memcpy(eth->eth_dmac, hbs_r->vif_mac, VR_ETHER_ALEN);
1081 1 : memcpy(eth->eth_smac, hbs_r->vif_mac, VR_ETHER_ALEN);
1082 1 : if (pkt->vp_type == VP_TYPE_IP) {
1083 1 : eth->eth_proto = htons(VR_ETH_PROTO_IP);
1084 0 : } else if (pkt->vp_type == VP_TYPE_IP6) {
1085 0 : eth->eth_proto = htons(VR_ETH_PROTO_IP6);
1086 : }
1087 1 : pkt_is_l3 = true;
1088 : }
1089 :
1090 5 : eth_hbs = (struct vr_eth_hbs_md*)pkt_data(pkt);
1091 :
1092 5 : eth = (struct vr_eth*)pkt_data(pkt);
1093 5 : if (!pkt_is_l3 && !memcmp(eth->eth_dmac, hbs_r->vif_mac, VR_ETHER_ALEN))
1094 1 : pkt_is_l3 = true;
1095 :
1096 5 : eth_hbs->flow_id_dmac = htonl(index);
1097 :
1098 : /* Encode packet source in the header */
1099 5 : if (vif_is_virtual(pkt->vp_if))
1100 2 : eth_hbs->magic_dmac = htons(VR_HBS_DMAC_MAGIC | VR_HBS_FROM_VMI);
1101 3 : else if (vif_is_fabric(pkt->vp_if))
1102 3 : eth_hbs->magic_dmac = htons(VR_HBS_DMAC_MAGIC | VR_HBS_FROM_FABRIC);
1103 : /* If it's L3 packet, encode it */
1104 5 : if (pkt_is_l3)
1105 2 : eth_hbs->magic_dmac |= htons(VR_HBS_L3_PKT);
1106 :
1107 5 : pkt->vp_if = hbs_r;
1108 : /*
1109 : * If for some reason, we have GRO flag set and we have not invoked
1110 : * the GRO, we need to unset
1111 : */
1112 5 : vr_pkt_unset_gro(pkt);
1113 5 : pkt->vp_nh = NULL;
1114 5 : hbs_r->vif_tx(hbs_r, pkt, fmd);
1115 5 : return FLOW_HELD;
1116 : }
1117 0 : return FLOW_FORWARD;
1118 : }
1119 0 : return FLOW_FORWARD;
1120 :
1121 0 : drop_pkt:
1122 0 : vr_pfree(pkt, VP_DROP_INVALID_HBS_PKT);
1123 0 : return FLOW_HELD;
1124 : }
1125 :
1126 : static flow_result_t
1127 62 : vr_flow_action(struct vrouter *router, struct vr_flow_entry *fe,
1128 : unsigned int index, struct vr_packet *pkt,
1129 : struct vr_forwarding_md *fmd)
1130 : {
1131 62 : flow_result_t ret = FLOW_CONSUMED;
1132 : struct vr_nexthop *src_nh;
1133 62 : int valid_src, modified_index = -1;
1134 :
1135 62 : src_nh = __vrouter_get_nexthop(router, fe->fe_src_nh_index);
1136 62 : if (!src_nh) {
1137 0 : PKT_LOG(VP_DROP_INVALID_NH, pkt, 0, VR_FLOW_C, __LINE__);
1138 0 : vr_pfree(pkt, VP_DROP_INVALID_NH);
1139 0 : goto res;
1140 : }
1141 :
1142 62 : if (src_nh->nh_validate_src) {
1143 60 : valid_src = src_nh->nh_validate_src(pkt, src_nh, fmd, &modified_index);
1144 60 : if (valid_src == NH_SOURCE_INVALID) {
1145 0 : PKT_LOG(VP_DROP_INVALID_SOURCE, pkt, 0, VR_FLOW_C, __LINE__);
1146 0 : vr_pfree(pkt, VP_DROP_INVALID_SOURCE);
1147 0 : goto res;
1148 : }
1149 :
1150 60 : if (valid_src == NH_SOURCE_MISMATCH) {
1151 0 : valid_src = vr_rflow_update_ecmp_index(router, fe,
1152 : modified_index, fmd);
1153 0 : if (valid_src == -1) {
1154 0 : PKT_LOG(VP_DROP_INVALID_SOURCE, pkt, 0, VR_FLOW_C, __LINE__);
1155 0 : vr_pfree(pkt, VP_DROP_INVALID_SOURCE);
1156 0 : goto res;
1157 : }
1158 : }
1159 : }
1160 :
1161 :
1162 124 : if ((fe->fe_action == VR_FLOW_ACTION_DROP) ||
1163 62 : ((ret = vr_flow_action_hbs(router, fe, index, pkt, fmd)) != FLOW_HELD))
1164 53 : ret = vr_flow_action_default(router, fe, index, pkt, fmd);
1165 :
1166 9 : res:
1167 62 : if (fe->fe_tcp_flags & VR_FLOW_TCP_DEAD)
1168 4 : vr_flow_mark_evict(router, fe, index);
1169 :
1170 62 : return ret;
1171 : }
1172 :
1173 : unsigned int
1174 5 : vr_trap_flow(struct vrouter *router, struct vr_flow_entry *fe,
1175 : struct vr_packet *pkt, unsigned int index,
1176 : struct vr_flow_stats *stats, struct vr_packet_node *pnode)
1177 : {
1178 : unsigned int trap_reason;
1179 :
1180 : struct vr_packet *npkt;
1181 : struct vr_flow_trap_arg ta;
1182 :
1183 5 : npkt = vr_pclone(pkt);
1184 5 : if (!npkt) {
1185 : /* Lets manipulate the stats */
1186 0 : pkt_drop_stats(pkt->vp_if, VP_DROP_TRAP_ORIGINAL, pkt->vp_cpu);
1187 0 : if (pnode)
1188 0 : pnode->pl_packet = NULL;
1189 0 : npkt = pkt;
1190 : }
1191 :
1192 5 : vr_preset(npkt);
1193 :
1194 5 : switch (fe->fe_flags & VR_FLOW_FLAG_TRAP_MASK) {
1195 : default:
1196 : /*
1197 : * agent needs a method to identify new flows from existing flows.
1198 : * existing flows can be reused (evicted) or the action of such flows
1199 : * can become hold. If existing flows are reused and packet is trapped,
1200 : * agent will not re-evaluate the flow. Hence, agent has to be told
1201 : * that this is a new flow, which we indicate by the trap reason.
1202 : */
1203 5 : if (fe->fe_flags & VR_FLOW_FLAG_NEW_FLOW) {
1204 5 : trap_reason = AGENT_TRAP_FLOW_MISS;
1205 5 : fe->fe_flags ^= VR_FLOW_FLAG_NEW_FLOW;
1206 : } else {
1207 0 : trap_reason = AGENT_TRAP_FLOW_ACTION_HOLD;
1208 : }
1209 :
1210 5 : ta.vfta_index = index;
1211 5 : if ((fe->fe_type == VP_TYPE_IP) || (fe->fe_type == VP_TYPE_IP6))
1212 5 : ta.vfta_nh_index = fe->fe_key.flow_nh_id;
1213 5 : if (stats) {
1214 5 : ta.vfta_stats = *stats;
1215 : } else {
1216 0 : ta.vfta_stats = fe->fe_stats;
1217 : }
1218 :
1219 5 : ta.vfta_gen_id = fe->fe_gen_id;
1220 :
1221 5 : break;
1222 : }
1223 :
1224 5 : return vr_trap(npkt, fe->fe_vrf, trap_reason, &ta);
1225 : }
1226 :
1227 : static flow_result_t
1228 67 : vr_do_flow_action(struct vrouter *router, struct vr_flow_entry *fe,
1229 : unsigned int index, struct vr_packet *pkt,
1230 : struct vr_forwarding_md *fmd)
1231 : {
1232 : uint32_t new_stats;
1233 67 : struct vr_flow_stats stats, *stats_p = NULL;
1234 :
1235 67 : if (fe->fe_flags & VR_FLOW_FLAG_NEW_FLOW) {
1236 5 : memcpy(&stats, &fe->fe_stats, sizeof(fe->fe_stats));
1237 5 : memset(&fe->fe_stats, 0, sizeof(fe->fe_stats));
1238 5 : stats_p = &stats;
1239 : }
1240 :
1241 67 : new_stats = vr_sync_add_and_fetch_32u(&fe->fe_stats.flow_bytes, pkt_len(pkt));
1242 67 : if (new_stats < pkt_len(pkt))
1243 0 : fe->fe_stats.flow_bytes_oflow++;
1244 :
1245 67 : new_stats = vr_sync_add_and_fetch_32u(&fe->fe_stats.flow_packets, 1);
1246 67 : if (!new_stats)
1247 0 : fe->fe_stats.flow_packets_oflow++;
1248 :
1249 67 : if (fe->fe_action == VR_FLOW_ACTION_HOLD) {
1250 7 : vr_enqueue_flow(router, fe, pkt, index, stats_p, fmd);
1251 7 : return FLOW_HELD;
1252 : }
1253 :
1254 60 : return vr_flow_action(router, fe, index, pkt, fmd);
1255 : }
1256 :
1257 : static unsigned int
1258 18 : vr_flow_table_hold_count(struct vrouter *router)
1259 : {
1260 : unsigned int i, num_cpus;
1261 18 : uint64_t hcount = 0, act_count;
1262 18 : struct vr_flow_table_info *infop = router->vr_flow_table_info;
1263 :
1264 18 : num_cpus = vr_num_cpus;
1265 234 : for (i = 0; i < num_cpus; i++)
1266 216 : hcount += infop->vfti_hold_count[i];
1267 :
1268 18 : act_count = infop->vfti_action_count;
1269 18 : if (hcount >= act_count)
1270 18 : return hcount - act_count;
1271 :
1272 0 : return 0;
1273 : }
1274 :
1275 : static void
1276 0 : vr_flow_burst_timeout(void *arg)
1277 : {
1278 : int tokens;
1279 0 : struct vrouter *router = (struct vrouter *)arg;
1280 0 : struct vr_flow_table_info *infop = router->vr_flow_table_info;
1281 :
1282 0 : tokens = infop->vfti_burst_tokens - infop->vfti_burst_used;
1283 0 : if (tokens > 0) {
1284 :
1285 0 : tokens = infop->vfti_burst_tokens_configured - tokens;
1286 0 : if (tokens <= 0) {
1287 0 : infop->vfti_timer->vt_stop_timer = 1;
1288 0 : return;
1289 : }
1290 :
1291 0 : if (tokens > infop->vfti_burst_step_configured)
1292 0 : tokens = infop->vfti_burst_step_configured;
1293 : } else {
1294 0 : tokens = infop->vfti_burst_step_configured;
1295 : }
1296 :
1297 0 : infop->vfti_burst_tokens += tokens;
1298 :
1299 0 : return;
1300 : }
1301 :
1302 : static void
1303 0 : vr_flow_start_burst_processing(struct vrouter *router)
1304 : {
1305 : struct vr_timer *vtimer;
1306 0 : struct vr_flow_table_info *infop = router->vr_flow_table_info;
1307 :
1308 0 : if (!infop->vfti_burst_tokens_configured ||
1309 0 : !infop->vfti_burst_interval_configured ||
1310 0 : !infop->vfti_burst_step_configured) {
1311 0 : return;
1312 : }
1313 :
1314 0 : if (!infop->vfti_timer) {
1315 0 : vtimer = vr_zalloc(sizeof(*vtimer), VR_TIMER_OBJECT);
1316 0 : if (!vtimer) {
1317 0 : vr_module_error(-ENOMEM, __FUNCTION__, __LINE__, sizeof(*vtimer));
1318 0 : return;
1319 : }
1320 :
1321 0 : vtimer->vt_timer = vr_flow_burst_timeout;
1322 0 : vtimer->vt_vr_arg = router;
1323 0 : vtimer->vt_msecs = infop->vfti_burst_interval_configured;
1324 :
1325 0 : if (vr_create_timer(vtimer)) {
1326 0 : vr_free(vtimer, VR_TIMER_OBJECT);
1327 0 : return;
1328 : }
1329 :
1330 0 : infop->vfti_timer = vtimer;
1331 : } else {
1332 0 : if (!infop->vfti_timer->vt_stop_timer)
1333 0 : return;
1334 :
1335 0 : if (vr_sync_bool_compare_and_swap_32u(
1336 : &infop->vfti_timer->vt_stop_timer, 1, 0)) {
1337 0 : infop->vfti_timer->vt_msecs = infop->vfti_burst_interval_configured;
1338 0 : vr_restart_timer(infop->vfti_timer);
1339 : }
1340 : }
1341 :
1342 0 : return;
1343 : }
1344 :
1345 : static void
1346 5 : vr_flow_entry_set_hold(struct vrouter *router, struct vr_flow_entry
1347 : *flow_e, bool burst)
1348 : {
1349 : unsigned int cpu;
1350 : uint64_t act_count;
1351 5 : struct vr_flow_table_info *infop = router->vr_flow_table_info;
1352 :
1353 5 : cpu = vr_get_cpu();
1354 5 : if (cpu >= vr_num_cpus) {
1355 0 : vr_printf("vrouter: Set HOLD failed (cpu %u num_cpus %u)\n",
1356 : cpu, vr_num_cpus);
1357 0 : return;
1358 : }
1359 :
1360 5 : flow_e->fe_action = VR_FLOW_ACTION_HOLD;
1361 :
1362 5 : if (infop->vfti_hold_count[cpu] + 1 < infop->vfti_hold_count[cpu]) {
1363 0 : (void)vr_sync_add_and_fetch_32u(&infop->vfti_oflows, 1);
1364 0 : act_count = infop->vfti_action_count;
1365 0 : if (act_count > infop->vfti_hold_count[cpu]) {
1366 0 : (void)vr_sync_sub_and_fetch_64u(&infop->vfti_action_count,
1367 : infop->vfti_hold_count[cpu]);
1368 0 : infop->vfti_hold_count[cpu] = 0;
1369 : } else {
1370 0 : infop->vfti_hold_count[cpu] -= act_count;
1371 0 : (void)vr_sync_sub_and_fetch_64u(&infop->vfti_action_count,
1372 : act_count);
1373 : }
1374 : }
1375 :
1376 5 : infop->vfti_hold_count[cpu]++;
1377 :
1378 5 : if (burst == true) {
1379 0 : (void)vr_sync_add_and_fetch_64u(&infop->vfti_burst_used, 1);
1380 0 : vr_flow_start_burst_processing(router);
1381 : }
1382 :
1383 5 : return;
1384 : }
1385 :
1386 : static void
1387 4 : vr_flow_init_close(struct vrouter *router, struct vr_flow_entry *flow_e,
1388 : struct vr_packet *pkt, struct vr_forwarding_md *fmd)
1389 : {
1390 : struct vr_flow_entry *rfe;
1391 :
1392 4 : (void)vr_sync_fetch_and_or_16u(&flow_e->fe_tcp_flags, VR_FLOW_TCP_DEAD);
1393 4 : rfe = vr_flow_get_entry(router, flow_e->fe_rflow);
1394 4 : if (rfe) {
1395 4 : (void)vr_sync_fetch_and_or_16u(&rfe->fe_tcp_flags, VR_FLOW_TCP_DEAD);
1396 : }
1397 :
1398 4 : return;
1399 : }
1400 :
1401 : static void
1402 49 : vr_flow_tcp_rflow_set(struct vrouter *router, struct vr_flow_entry *fe,
1403 : struct vr_flow_entry *rfe)
1404 : {
1405 49 : uint16_t flags = 0;
1406 :
1407 49 : if (!fe || !rfe)
1408 0 : return;
1409 :
1410 49 : if (rfe->fe_tcp_flags & VR_FLOW_TCP_SYN) {
1411 1 : flags |= VR_FLOW_TCP_SYN_R;
1412 : }
1413 :
1414 49 : if (rfe->fe_tcp_flags & VR_FLOW_TCP_RST) {
1415 0 : flags |= VR_FLOW_TCP_RST;
1416 : }
1417 :
1418 49 : if (rfe->fe_tcp_flags & VR_FLOW_TCP_DEAD) {
1419 0 : flags |= VR_FLOW_TCP_DEAD;
1420 : }
1421 :
1422 49 : if (rfe->fe_tcp_flags & VR_FLOW_TCP_FIN) {
1423 0 : flags |= VR_FLOW_TCP_FIN_R;
1424 : }
1425 :
1426 49 : if (rfe->fe_tcp_flags & VR_FLOW_TCP_ESTABLISHED) {
1427 0 : flags |= (VR_FLOW_TCP_ESTABLISHED | VR_FLOW_TCP_ESTABLISHED_R);
1428 : }
1429 :
1430 49 : (void)vr_sync_fetch_and_or_16u(&fe->fe_tcp_flags, flags);
1431 49 : return;
1432 : }
1433 :
1434 : static void
1435 67 : vr_flow_tcp_digest(struct vrouter *router, struct vr_flow_entry *flow_e,
1436 : struct vr_packet *pkt, struct vr_forwarding_md *fmd)
1437 : {
1438 67 : uint8_t proto = 0, hlen = 0;
1439 : uint16_t tcp_offset_flags;
1440 67 : unsigned int length = 0;
1441 : uint16_t flow_tcp_flags;
1442 :
1443 : struct vr_ip *iph;
1444 : struct vr_ip6 *ip6h;
1445 : struct vr_tcp *tcph;
1446 : struct vr_ip6_frag *v6_frag;
1447 67 : struct vr_flow_entry *rflow_e = NULL;
1448 : unsigned int rflow_ack;
1449 :
1450 67 : if (pkt->vp_type == VP_TYPE_IP) {
1451 63 : iph = (struct vr_ip *)pkt_network_header(pkt);
1452 63 : if (!vr_ip_transport_header_valid(iph))
1453 12 : return;
1454 51 : proto = iph->ip_proto;
1455 :
1456 51 : length = ntohs(iph->ip_len) - (iph->ip_hl * 4);
1457 51 : hlen = iph->ip_hl * 4;
1458 4 : } else if (pkt->vp_type == VP_TYPE_IP6) {
1459 4 : ip6h = (struct vr_ip6 *)pkt_network_header(pkt);
1460 4 : if (!vr_ip6_transport_header_valid(ip6h))
1461 0 : return;
1462 4 : proto = ip6h->ip6_nxt;
1463 4 : length = ntohs(ip6h->ip6_plen);
1464 4 : hlen = sizeof(struct vr_ip6);
1465 4 : if (proto == VR_IP6_PROTO_FRAG) {
1466 0 : v6_frag = (struct vr_ip6_frag *)(ip6h + 1);
1467 0 : proto = v6_frag->ip6_frag_nxt;
1468 0 : length -= sizeof(struct vr_ip6_frag);
1469 0 : hlen += sizeof(struct vr_ip6_frag);
1470 : }
1471 : }
1472 :
1473 55 : if (proto != VR_IP_PROTO_TCP)
1474 35 : return;
1475 :
1476 20 : tcph = (struct vr_tcp *)(pkt_network_header(pkt) + hlen);
1477 :
1478 20 : if (tcph) {
1479 20 : if (vr_flow_is_fat_flow(router, pkt, flow_e))
1480 0 : return;
1481 :
1482 : /*
1483 : * there are some optimizations here that makes the code slightly
1484 : * not so frugal. For e.g.: the *_R flags are used to make sure that
1485 : * for a packet that contains ACK, we will not need to fetch the
1486 : * reverse flow if we are not interested, thus saving some execution
1487 : * time.
1488 : */
1489 20 : tcp_offset_flags = ntohs(tcph->tcp_offset_r_flags);
1490 :
1491 : /*
1492 : * If this is an ack, set the last acked seqnum
1493 : */
1494 20 : if (tcp_offset_flags & VR_TCP_FLAG_ACK) {
1495 9 : flow_e->fe_tcp_ack = ntohl(tcph->tcp_ack);
1496 : }
1497 :
1498 : /*
1499 : * if we get a reset, TCP session will be closed if the
1500 : * vr_uncond_close_flow_on_tcp_rst flag is enabled or the TCP RST
1501 : * seqnum matches with the seqnum acked by the receiver
1502 : * - as per RFC 5961 sec 3.2
1503 : */
1504 20 : if (tcp_offset_flags & VR_TCP_FLAG_RST) {
1505 6 : if (!vr_uncond_close_flow_on_tcp_rst) {
1506 : /* get the reverse flow ack seq num if valid */
1507 4 : if (flow_e->fe_flags & VR_RFLOW_VALID) {
1508 4 : rflow_e = vr_flow_get_entry(router, flow_e->fe_rflow);
1509 : }
1510 4 : if (rflow_e) {
1511 4 : rflow_ack = rflow_e->fe_tcp_ack;
1512 : /* Implementing rfc-5961 (section 3.2),
1513 : * In the SYN-SENT state (a RST received in response to an initial SYN),
1514 : * the RST is acceptable if the ACK field acknowledges the SYN. In all
1515 : * other cases the receiver MUST silently discard the segment. */
1516 4 : if ((tcp_offset_flags & VR_TCP_FLAG_ACK)
1517 2 : && (rflow_e->fe_tcp_flags & VR_FLOW_TCP_SYN)) {
1518 2 : if (ntohl(tcph->tcp_ack) != (rflow_e->fe_tcp_seq + 1)) {
1519 : /* Ignore the RST */
1520 1 : return;
1521 : }
1522 : }
1523 2 : else if (ntohl(tcph->tcp_seq) != rflow_ack) {
1524 : /* Ignore the RST */
1525 1 : return;
1526 : }
1527 : }
1528 : /* If Reverse flow not valid, go ahead and close this flow */
1529 : }
1530 :
1531 4 : (void)vr_sync_fetch_and_or_16u(&flow_e->fe_tcp_flags,
1532 : VR_FLOW_TCP_RST);
1533 4 : if (flow_e->fe_flags & VR_RFLOW_VALID) {
1534 4 : rflow_e = vr_flow_get_entry(router, flow_e->fe_rflow);
1535 4 : if (rflow_e) {
1536 4 : (void)vr_sync_fetch_and_or_16u(&rflow_e->fe_tcp_flags,
1537 : VR_FLOW_TCP_RST);
1538 : }
1539 : }
1540 4 : vr_flow_init_close(router, flow_e, pkt, fmd);
1541 4 : return;
1542 14 : } else if (tcp_offset_flags & VR_TCP_FLAG_SYN) {
1543 : /* if only a SYN... */
1544 11 : flow_e->fe_tcp_seq = ntohl(tcph->tcp_seq);
1545 11 : (void)vr_sync_fetch_and_or_16u(&flow_e->fe_tcp_flags, VR_FLOW_TCP_SYN);
1546 11 : if (flow_e->fe_flags & VR_RFLOW_VALID) {
1547 9 : rflow_e = vr_flow_get_entry(router, flow_e->fe_rflow);
1548 9 : if (rflow_e) {
1549 9 : (void)vr_sync_fetch_and_or_16u(&rflow_e->fe_tcp_flags,
1550 : VR_FLOW_TCP_SYN_R);
1551 9 : if ((flow_e->fe_tcp_flags & VR_FLOW_TCP_SYN_R) &&
1552 3 : (tcp_offset_flags & VR_TCP_FLAG_ACK)) {
1553 3 : if (ntohl(tcph->tcp_ack) == (rflow_e->fe_tcp_seq + 1)) {
1554 3 : (void)vr_sync_fetch_and_or_16u(&rflow_e->fe_tcp_flags,
1555 : VR_FLOW_TCP_ESTABLISHED);
1556 3 : flow_tcp_flags = vr_sync_fetch_and_or_16u(
1557 : &flow_e->fe_tcp_flags, VR_FLOW_TCP_ESTABLISHED_R);
1558 3 : if (!(flow_tcp_flags & VR_FLOW_TCP_ESTABLISHED_R) &&
1559 3 : !(flow_tcp_flags & VR_FLOW_TCP_ESTABLISHED)) {
1560 : // If it wasn't established or reverse established,
1561 : // it is being established now. Offload both.
1562 3 : vr_offload_flow_set(flow_e,
1563 : flow_e->fe_hentry.hentry_index, rflow_e);
1564 3 : vr_offload_flow_set(rflow_e,
1565 : rflow_e->fe_hentry.hentry_index, flow_e);
1566 : }
1567 : }
1568 : }
1569 : }
1570 : }
1571 3 : } else if (tcp_offset_flags & VR_TCP_FLAG_FIN) {
1572 : /*
1573 : * when a FIN is received, update the sequence of the FIN and set
1574 : * the flow FIN flag. It is possible that the FIN packet came with
1575 : * some data, in which case the sequence number of the FIN is one
1576 : * more than the last data byte in the sequence
1577 : */
1578 0 : length -= (((tcp_offset_flags) >> 12) * 4);
1579 0 : flow_e->fe_tcp_seq = ntohl(tcph->tcp_seq) + length;
1580 0 : (void)vr_sync_fetch_and_or_16u(&flow_e->fe_tcp_flags, VR_FLOW_TCP_FIN);
1581 : /*
1582 : * when an ack for a FIN is sent, we need to take some actions
1583 : * on the reverse flow (since FIN came in the reverse flow). to
1584 : * avoid looking up the reverse flow for all acks, we mark the
1585 : * reverse flow's reverse flow with a flag (FIN_R). we will
1586 : * lookup the reverse flow only if this flag is set and the
1587 : * tcp header has an ack bit set
1588 : */
1589 0 : if (flow_e->fe_flags & VR_RFLOW_VALID) {
1590 0 : rflow_e = vr_flow_get_entry(router, flow_e->fe_rflow);
1591 : /* Delete offloaded flow on TCP FIN */
1592 0 : (void)vr_offload_flow_del(flow_e);
1593 0 : if (rflow_e) {
1594 0 : (void)vr_sync_fetch_and_or_16u(&rflow_e->fe_tcp_flags,
1595 : VR_FLOW_TCP_FIN_R);
1596 : /* Delete offloaded reverse flow on TCP FIN */
1597 0 : (void)vr_offload_flow_del(rflow_e);
1598 : }
1599 : }
1600 : }
1601 :
1602 : /*
1603 : * if FIN_R is set in the flow and if the ACK bit is set in the
1604 : * tcp header, then we need to mark the reverse flow as dead.
1605 : *
1606 : * OR
1607 : *
1608 : * if the SYN_R is set and ESTABLISHED_R is not set and if this
1609 : * is an ack packet, if this ack completes the connection, we
1610 : * need to set ESTABLISHED
1611 : */
1612 14 : if (((flow_e->fe_tcp_flags & VR_FLOW_TCP_FIN_R) ||
1613 14 : (!(flow_e->fe_tcp_flags & VR_FLOW_TCP_ESTABLISHED_R) &&
1614 11 : (flow_e->fe_tcp_flags & VR_FLOW_TCP_SYN_R))) &&
1615 3 : (tcp_offset_flags & VR_TCP_FLAG_ACK)) {
1616 3 : if (flow_e->fe_flags & VR_RFLOW_VALID) {
1617 3 : if (!rflow_e) {
1618 3 : rflow_e = vr_flow_get_entry(router, flow_e->fe_rflow);
1619 : }
1620 :
1621 3 : if (rflow_e) {
1622 3 : if ((ntohl(tcph->tcp_ack) == (rflow_e->fe_tcp_seq + 1)) &&
1623 3 : (flow_e->fe_tcp_flags & VR_FLOW_TCP_FIN_R)) {
1624 0 : (void)vr_sync_fetch_and_or_16u(&rflow_e->fe_tcp_flags,
1625 : VR_FLOW_TCP_HALF_CLOSE);
1626 : /*
1627 : * both the forward and the reverse flows are
1628 : * now dead
1629 : */
1630 0 : if (flow_e->fe_tcp_flags & VR_FLOW_TCP_HALF_CLOSE) {
1631 0 : vr_flow_init_close(router, flow_e, pkt, fmd);
1632 : }
1633 3 : } else if (ntohl(tcph->tcp_ack) != rflow_e->fe_tcp_seq) {
1634 3 : if (!(flow_e->fe_tcp_flags &
1635 : VR_FLOW_TCP_ESTABLISHED_R)) {
1636 3 : (void)vr_sync_fetch_and_or_16u(&rflow_e->fe_tcp_flags,
1637 : VR_FLOW_TCP_ESTABLISHED);
1638 3 : flow_tcp_flags = vr_sync_fetch_and_or_16u(
1639 : &flow_e->fe_tcp_flags, VR_FLOW_TCP_ESTABLISHED_R);
1640 3 : if (!(flow_tcp_flags & VR_FLOW_TCP_ESTABLISHED_R) &&
1641 3 : !(flow_tcp_flags & VR_FLOW_TCP_ESTABLISHED)) {
1642 : // If it wasn't established or reverse established,
1643 : // it is being established now. Offload both.
1644 0 : vr_offload_flow_set(flow_e,
1645 : flow_e->fe_hentry.hentry_index, rflow_e);
1646 0 : vr_offload_flow_set(rflow_e,
1647 : rflow_e->fe_hentry.hentry_index, flow_e);
1648 : }
1649 : }
1650 : }
1651 : }
1652 : }
1653 : }
1654 : }
1655 :
1656 14 : return;
1657 : }
1658 :
1659 : static inline bool
1660 5 : vr_flow_vif_allow_new_flow(struct vrouter *router, struct vr_packet *pkt,
1661 : unsigned short *drop_reason)
1662 : {
1663 5 : struct vr_interface *vif_l = NULL;
1664 5 : struct vr_nexthop *nh = NULL;
1665 :
1666 5 : if (vif_is_virtual(pkt->vp_if)) {
1667 3 : vif_l = pkt->vp_if;
1668 2 : } else if (vif_is_fabric(pkt->vp_if)) {
1669 1 : nh = pkt->vp_nh;
1670 1 : if ((nh != NULL) && (nh->nh_flags & NH_FLAG_VALID)) {
1671 1 : vif_l = nh->nh_dev;
1672 : }
1673 : }
1674 :
1675 5 : if (vif_l && vif_drop_new_flows(vif_l)) {
1676 0 : PKT_LOG(VP_DROP_NEW_FLOWS, pkt, 0, VR_FLOW_C, __LINE__);
1677 0 : *drop_reason = VP_DROP_NEW_FLOWS;
1678 0 : return false;
1679 : }
1680 :
1681 5 : return true;
1682 : }
1683 :
1684 : void
1685 0 : vr_flow_get_burst_params(struct vrouter *router, int *burst_tokens,
1686 : int *burst_interval, int *burst_step)
1687 : {
1688 : struct vr_flow_table_info *infop;
1689 :
1690 0 : if (!router || !router->vr_flow_table_info)
1691 0 : return;
1692 :
1693 0 : infop = router->vr_flow_table_info;
1694 :
1695 0 : if (burst_tokens)
1696 0 : *burst_tokens = infop->vfti_burst_tokens_configured;
1697 0 : if (burst_interval)
1698 0 : *burst_interval = infop->vfti_burst_interval_configured;
1699 0 : if (burst_step)
1700 0 : *burst_step = infop->vfti_burst_step_configured;
1701 :
1702 0 : return;
1703 : }
1704 :
1705 : void
1706 0 : vr_flow_set_burst_params(struct vrouter *router, int burst_tokens,
1707 : int burst_interval, int burst_step)
1708 : {
1709 : struct vr_flow_table_info *infop;
1710 :
1711 0 : if (!router || !router->vr_flow_table_info)
1712 0 : return;
1713 :
1714 0 : infop = router->vr_flow_table_info;
1715 :
1716 0 : if (burst_tokens != -1)
1717 0 : infop->vfti_burst_tokens_configured = burst_tokens;
1718 :
1719 0 : if (burst_interval != -1)
1720 0 : infop->vfti_burst_interval_configured = burst_interval;
1721 :
1722 0 : if (burst_step != -1)
1723 0 : infop->vfti_burst_step_configured = burst_step;
1724 :
1725 :
1726 0 : vr_flow_start_burst_processing(router);
1727 0 : return;
1728 : }
1729 :
1730 : static inline unsigned int
1731 0 : vr_flow_burst_count(struct vrouter *router)
1732 : {
1733 0 : struct vr_flow_table_info *infop = router->vr_flow_table_info;
1734 :
1735 0 : return infop->vfti_burst_tokens;
1736 : }
1737 :
1738 : static inline bool
1739 5 : vr_flow_allow_new_flow(struct vrouter *router, struct vr_packet *pkt,
1740 : unsigned short *drop_reason, bool *burst)
1741 : {
1742 : unsigned int hold_count;
1743 5 : struct vr_flow_table_info *infop = router->vr_flow_table_info;
1744 :
1745 5 : *drop_reason = VP_DROP_FLOW_UNUSABLE;
1746 5 : if (burst)
1747 5 : *burst = false;
1748 :
1749 5 : if (pkt->vp_type == VP_TYPE_IP) {
1750 5 : if (!vr_inet_flow_allow_new_flow(router, pkt)) {
1751 0 : PKT_LOG(VP_DROP_FLOW_UNUSABLE, pkt, 0, VR_FLOW_C, __LINE__);
1752 0 : *drop_reason = VP_DROP_FLOW_UNUSABLE;
1753 0 : return false;
1754 : }
1755 : }
1756 :
1757 5 : if (vr_flow_hold_limit) {
1758 5 : hold_count = vr_flow_table_hold_count(router);
1759 5 : if (hold_count > vr_flow_hold_limit) {
1760 0 : if (infop->vfti_burst_used >= vr_flow_burst_count(router)) {
1761 0 : PKT_LOG(VP_DROP_FLOW_UNUSABLE, pkt, 0, VR_FLOW_C, __LINE__);
1762 0 : *drop_reason = VP_DROP_FLOW_UNUSABLE;
1763 0 : return false;
1764 : }
1765 0 : if (burst) {
1766 0 : *burst = true;
1767 : }
1768 : }
1769 : }
1770 :
1771 5 : return vr_flow_vif_allow_new_flow(router, pkt, drop_reason);
1772 : }
1773 :
1774 : static inline struct vr_flow_entry *
1775 5 : vr_flow_new_hold_flow(struct vrouter *router, struct vr_flow *key,
1776 : struct vr_packet *pkt, unsigned int *fe_index,
1777 : struct vr_forwarding_md *fmd) {
1778 : struct vr_flow_entry *bucket_fe, *flow_e;
1779 5 : unsigned short drop_reason = 0;
1780 5 : bool burst = false;
1781 : /* Slow path: lock, to make find+create atomic thus avoid races
1782 : * with another vr_flow_lookup or vr_add_flow). We also need need
1783 : * to retry the find, as previous one was done without locking */
1784 5 : bucket_fe = vr_flow_bucket_first_entry(router, key);
1785 5 : vr_flow_bucket_may_lock(bucket_fe);
1786 5 : flow_e = vr_find_flow(router, key, pkt->vp_type, fe_index);
1787 5 : if (!flow_e) {
1788 5 : if (!vr_flow_allow_new_flow(router, pkt, &drop_reason, &burst)) {
1789 0 : vr_flow_bucket_may_unlock(bucket_fe);
1790 0 : PKT_LOG(drop_reason, pkt, key , VR_FLOW_C, __LINE__);
1791 0 : vr_pfree(pkt, drop_reason);
1792 0 : return flow_e;
1793 : }
1794 :
1795 5 : flow_e = vr_flow_get_free_entry(router, key, pkt->vp_type,
1796 : true, fe_index);
1797 5 : if (!flow_e) {
1798 0 : vr_flow_bucket_may_unlock(bucket_fe);
1799 0 : PKT_LOG(VP_DROP_FLOW_TABLE_FULL, pkt, key, VR_FLOW_C, __LINE__);
1800 0 : vr_pfree(pkt, VP_DROP_FLOW_TABLE_FULL);
1801 0 : return flow_e;
1802 : }
1803 :
1804 5 : flow_e->fe_vrf = fmd->fmd_dvrf;
1805 : /* mark as hold */
1806 5 : vr_flow_entry_set_hold(router, flow_e, burst);
1807 : }
1808 5 : vr_flow_bucket_may_unlock(bucket_fe);
1809 5 : return flow_e;
1810 : }
1811 :
1812 : flow_result_t
1813 67 : vr_flow_lookup(struct vrouter *router, struct vr_flow *key,
1814 : struct vr_packet *pkt, struct vr_forwarding_md *fmd)
1815 : {
1816 : unsigned int fe_index;
1817 : struct vr_flow_entry *flow_e;
1818 67 : pkt->vp_flags |= VP_FLAG_FLOW_SET;
1819 :
1820 67 : if (!fmd->fmd_fe) {
1821 : /* Happy path: without locking */
1822 67 : flow_e = vr_find_flow(router, key, pkt->vp_type, &fe_index);
1823 67 : if (!flow_e) {
1824 5 : if (pkt->vp_nh &&
1825 2 : (pkt->vp_nh->nh_flags &
1826 : (NH_FLAG_RELAXED_POLICY | NH_FLAG_FLOW_LOOKUP)))
1827 0 : return FLOW_FORWARD;
1828 :
1829 5 : flow_e = vr_flow_new_hold_flow(router, key, pkt, &fe_index, fmd);
1830 5 : if (!flow_e)
1831 0 : return FLOW_CONSUMED;
1832 : }
1833 : } else {
1834 0 : flow_e = fmd->fmd_fe;
1835 0 : fe_index = fmd->fmd_flow_index;
1836 : }
1837 :
1838 67 : if (flow_e->fe_flags & VR_FLOW_FLAG_EVICT_CANDIDATE)
1839 0 : return FLOW_EVICT_DROP;
1840 :
1841 : /*
1842 : * Store the source of the packet which gets used incase of Ecmp
1843 : * Source
1844 : */
1845 67 : if (vif_is_fabric(pkt->vp_if))
1846 8 : flow_e->fe_src_info = fmd->fmd_outer_src_ip;
1847 59 : else if (vif_is_virtual(pkt->vp_if))
1848 58 : flow_e->fe_src_info = pkt->vp_if->vif_idx;
1849 :
1850 67 : vr_flow_set_forwarding_md(router, flow_e, fe_index, fmd);
1851 67 : vr_flow_tcp_digest(router, flow_e, pkt, fmd);
1852 :
1853 67 : return vr_do_flow_action(router, flow_e, fe_index, pkt, fmd);
1854 : }
1855 :
1856 : static bool
1857 188 : __vr_flow_forward(flow_result_t result, struct vr_packet *pkt,
1858 : struct vr_forwarding_md *fmd)
1859 : {
1860 188 : bool forward = false;
1861 :
1862 188 : switch (result) {
1863 164 : case FLOW_FORWARD:
1864 164 : forward = true;
1865 164 : break;
1866 :
1867 0 : case FLOW_TRAP:
1868 0 : vr_trap(pkt, fmd->fmd_dvrf, AGENT_TRAP_L3_PROTOCOLS, NULL);
1869 0 : break;
1870 :
1871 0 : case FLOW_EVICT_DROP:
1872 0 : PKT_LOG(VP_DROP_FLOW_EVICT, pkt, 0, VR_FLOW_C, __LINE__);
1873 0 : vr_pfree(pkt, VP_DROP_FLOW_EVICT);
1874 0 : break;
1875 :
1876 24 : case FLOW_HELD:
1877 : case FLOW_CONSUMED:
1878 24 : break;
1879 :
1880 0 : case FLOW_DROP:
1881 : default:
1882 0 : PKT_LOG(VP_DROP_FLOW_UNUSABLE, pkt, 0, VR_FLOW_C, __LINE__);
1883 0 : vr_pfree(pkt, VP_DROP_FLOW_UNUSABLE);
1884 0 : break;
1885 : }
1886 :
1887 188 : return forward;
1888 : }
1889 :
1890 : static bool
1891 20 : vr_flow_is_fat_flow(struct vrouter *router, struct vr_packet *pkt,
1892 : struct vr_flow_entry *fe)
1893 : {
1894 20 : if (pkt->vp_type == VP_TYPE_IP) {
1895 20 : return vr_inet_flow_is_fat_flow(router, pkt, fe);
1896 0 : } else if (pkt->vp_type == VP_TYPE_IP6) {
1897 0 : return vr_inet6_flow_is_fat_flow(router, pkt, fe);
1898 : }
1899 :
1900 0 : return false;
1901 : }
1902 :
1903 : uint16_t
1904 59 : vr_flow_fat_flow_lookup(struct vrouter *router, struct vr_packet *pkt,
1905 : uint16_t l4_proto, uint16_t sport, uint16_t dport,
1906 : unsigned int *saddr, unsigned int *daddr,
1907 : unsigned char *ip6_src, unsigned char *ip6_dst)
1908 : {
1909 59 : uint8_t fat_flow_mask, tmp_mask = 0;
1910 : struct vr_nexthop *nh;
1911 59 : struct vr_interface *vif_l = NULL;
1912 :
1913 59 : if (vif_is_virtual(pkt->vp_if)) {
1914 49 : vif_l = pkt->vp_if;
1915 10 : } else if (vif_is_fabric(pkt->vp_if)) {
1916 8 : if ((nh = pkt->vp_nh) && (nh->nh_flags & NH_FLAG_VALID)) {
1917 : /* In case of ECMP, check the vif within the ECMP member NH */
1918 8 : if (vr_is_local_ecmp_nh(nh)) {
1919 0 : vif_l = vr_get_ecmp_first_member_dev(nh);
1920 : } else {
1921 8 : vif_l = nh->nh_dev;
1922 : }
1923 : }
1924 : }
1925 :
1926 59 : if (!vif_l)
1927 2 : return VR_FAT_FLOW_NO_MASK;
1928 :
1929 57 : fat_flow_mask = vif_fat_flow_lookup((pkt->vp_if == vif_l), vif_l, l4_proto, sport, dport,
1930 : saddr, daddr, ip6_src, ip6_dst);
1931 57 : if (pkt->vp_if != vif_l) {
1932 :
1933 8 : if (fat_flow_mask & VR_FAT_FLOW_SRC_IP_MASK)
1934 0 : tmp_mask |= VR_FAT_FLOW_DST_IP_MASK;
1935 :
1936 8 : if (fat_flow_mask & VR_FAT_FLOW_DST_IP_MASK)
1937 0 : tmp_mask |= VR_FAT_FLOW_SRC_IP_MASK;
1938 :
1939 8 : fat_flow_mask &= ~(VR_FAT_FLOW_DST_IP_MASK |
1940 : VR_FAT_FLOW_SRC_IP_MASK);
1941 :
1942 :
1943 8 : fat_flow_mask |= tmp_mask;
1944 :
1945 : }
1946 :
1947 57 : return fat_flow_mask;
1948 : }
1949 :
1950 : static flow_result_t
1951 170 : vr_do_flow_lookup(struct vrouter *router, struct vr_packet *pkt,
1952 : struct vr_forwarding_md *fmd)
1953 : {
1954 170 : flow_result_t result = FLOW_FORWARD;
1955 :
1956 : /* Flow processing is only for untagged unicast IP packets */
1957 170 : if (pkt->vp_type == VP_TYPE_IP)
1958 156 : result = vr_inet_flow_lookup(router, pkt, fmd);
1959 14 : else if (pkt->vp_type == VP_TYPE_IP6)
1960 10 : result = vr_inet6_flow_lookup(router, pkt, fmd);
1961 :
1962 170 : return result;
1963 : }
1964 :
1965 : static void
1966 8 : vr_reinit_forwarding_md(struct vrouter *router, struct vr_packet *pkt,
1967 : struct vr_flow_entry *fe, uint32_t flow_index,
1968 : struct vr_nexthop *nh, struct vr_forwarding_md *fmd)
1969 : {
1970 : int i;
1971 8 : struct vr_nexthop *src_nh = __vrouter_get_nexthop(router, fe->fe_src_nh_index);
1972 :
1973 8 : for (i = 0; i < VR_MAX_PHY_INF; i++) {
1974 8 : if (nh->nh_dev_arr[i] != NULL) {
1975 8 : fmd->fmd_dvrf = nh->nh_dev_arr[i]->vif_vrf;
1976 8 : break;
1977 : }
1978 : }
1979 :
1980 8 : fmd->fmd_vlan = 0;
1981 8 : fmd->fmd_dotonep = -1;
1982 8 : fmd->fmd_outer_src_ip = src_nh->nh_udp_tun_dip;
1983 8 : vr_flow_set_forwarding_md(router, fe, flow_index, fmd);
1984 8 : }
1985 :
1986 : bool
1987 186 : vr_flow_forward(struct vrouter *router, struct vr_packet *pkt,
1988 : struct vr_forwarding_md *fmd)
1989 : {
1990 186 : flow_result_t result = FLOW_FORWARD;
1991 :
1992 186 : if (vif_is_hbs_right(pkt->vp_if)) {
1993 : /* Pkt entering vrouter from hbs-r
1994 : * - If SMAC has magic, Restore actual SMAC from flow_index and continue
1995 : * - If DMAC has magic, Restore actual DMAC from flow_index and continue
1996 : */
1997 4 : struct vr_eth_hbs_md *eth_hbs = (struct vr_eth_hbs_md*)pkt_data(pkt);
1998 4 : struct vr_eth *eth = (struct vr_eth*)pkt_data(pkt);
1999 : uint16_t magic;
2000 : unsigned int flow_index;
2001 : uint32_t nh_id;
2002 4 : struct vr_flow_entry *fe = NULL;
2003 : unsigned char* mac;
2004 4 : if ((ntohs(eth_hbs->magic_smac) & VR_HBS_MAGIC_MASK) ==
2005 : VR_HBS_SMAC_MAGIC) {
2006 4 : magic = VR_HBS_SMAC_MAGIC;
2007 4 : flow_index = ntohl(eth_hbs->flow_id_smac);
2008 4 : mac = eth->eth_smac;
2009 0 : } else if ((ntohs(eth_hbs->magic_dmac) & VR_HBS_MAGIC_MASK) ==
2010 : VR_HBS_DMAC_MAGIC) {
2011 0 : magic = VR_HBS_DMAC_MAGIC;
2012 0 : flow_index = ntohl(eth_hbs->flow_id_dmac);
2013 0 : mac = eth->eth_dmac;
2014 : } else {
2015 0 : PKT_LOG(VP_DROP_INVALID_HBS_PKT, pkt, 0, VR_FLOW_C, __LINE__);
2016 0 : goto drop_pkt;
2017 : }
2018 :
2019 4 : if (vr_htable_get_hentry_by_index(router->vr_flow_table, flow_index)) {
2020 : struct vr_nexthop *nh;
2021 4 : fe = CONTAINER_OF(
2022 : fe_hentry,
2023 : struct vr_flow_entry,
2024 : vr_htable_get_hentry_by_index(
2025 : router->vr_flow_table, flow_index)
2026 : );
2027 4 : nh_id = fe->fe_key.flow_nh_id;
2028 4 : nh = vrouter_get_nexthop(0, nh_id);
2029 4 : vr_reinit_forwarding_md(router, pkt, fe, flow_index, nh, fmd);
2030 :
2031 4 : if(fe->fe_underlay_ecmp_index >= 0) {
2032 0 : if(nh->nh_type == NH_TUNNEL) {
2033 0 : pkt->vp_if = __vrouter_get_interface(router,
2034 0 : fe->fe_underlay_ecmp_index);
2035 : } else
2036 0 : pkt->vp_if = nh->nh_dev;
2037 : } else
2038 4 : pkt->vp_if = nh->nh_dev;
2039 :
2040 4 : memcpy(mac, nh->nh_data, VR_ETHER_ALEN);
2041 4 : result = vr_flow_action_default(router, fe, flow_index, pkt, fmd);
2042 4 : return __vr_flow_forward(result, pkt, fmd);
2043 : }
2044 182 : } else if (vif_is_hbs_left(pkt->vp_if)) {
2045 : /* Pkt entering vrouter from hbs-l
2046 : * - DMAC of the packet contains flow_index
2047 : * - Restore actual DMAC from flow_index and continue
2048 : * flow action
2049 : * If VR_HBS_FROM_VMI is set in the DMAC -
2050 : * - The packet originated from VMI (instead of fabric)
2051 : * - Happens if both src and dst tenant VMs are in
2052 : * the same compute (intra-compute case)
2053 : * - Restore actual DMAC from "reverse flow_index"
2054 : */
2055 4 : struct vr_eth *eth = (struct vr_eth*)pkt_data(pkt);
2056 4 : struct vr_eth_hbs_md *eth_hbs = (struct vr_eth_hbs_md*)pkt_data(pkt);
2057 4 : uint16_t magic = ntohs(eth_hbs->magic_dmac);
2058 4 : unsigned int flow_index = ntohl(eth_hbs->flow_id_dmac);
2059 : uint32_t nh_id;
2060 4 : struct vr_flow_entry *fe = NULL;
2061 4 : if ((magic & VR_HBS_MAGIC_MASK) != VR_HBS_DMAC_MAGIC) {
2062 0 : PKT_LOG(VP_DROP_INVALID_HBS_PKT, pkt, 0, VR_FLOW_C, __LINE__);
2063 0 : goto drop_pkt;
2064 : }
2065 :
2066 4 : if (vr_htable_get_hentry_by_index(router->vr_flow_table, flow_index)) {
2067 : struct vr_nexthop *nh;
2068 4 : fe = CONTAINER_OF(
2069 : fe_hentry,
2070 : struct vr_flow_entry,
2071 : vr_htable_get_hentry_by_index(
2072 : router->vr_flow_table, flow_index)
2073 : );
2074 : /* If packet is coming from VMI instead of fabric,
2075 : * restore DMAC using reverse flow
2076 : */
2077 4 : if (magic & VR_HBS_FROM_VMI) {
2078 2 : flow_index = fe->fe_rflow;
2079 2 : fe = vr_flow_get_entry(router, fe->fe_rflow);
2080 2 : if (!fe) {
2081 0 : PKT_LOG(VP_DROP_INVALID_HBS_PKT, pkt, 0, VR_FLOW_C, __LINE__);
2082 0 : goto drop_pkt;
2083 : }
2084 : }
2085 :
2086 4 : nh_id = fe->fe_key.flow_nh_id;
2087 4 : nh = vrouter_get_nexthop(0, nh_id);
2088 4 : vr_reinit_forwarding_md(router, pkt, fe, flow_index, nh, fmd);
2089 :
2090 4 : if(fe->fe_underlay_ecmp_index >= 0) {
2091 0 : if(nh->nh_type == NH_TUNNEL) {
2092 0 : pkt->vp_if = __vrouter_get_interface(router,
2093 0 : fe->fe_underlay_ecmp_index);
2094 : } else
2095 0 : pkt->vp_if = nh->nh_dev;
2096 : } else
2097 4 : pkt->vp_if = nh->nh_dev;
2098 :
2099 4 : if (magic & VR_HBS_L3_PKT)
2100 0 : memcpy(eth->eth_dmac, pkt->vp_if->vif_mac, VR_ETHER_ALEN);
2101 : else
2102 4 : memcpy(eth->eth_dmac, nh->nh_data, VR_ETHER_ALEN);
2103 4 : result = vr_flow_action_default(router, fe, flow_index, pkt, fmd);
2104 4 : return __vr_flow_forward(result, pkt, fmd);
2105 : }
2106 178 : } else if ((!(pkt->vp_flags & VP_FLAG_MULTICAST))
2107 178 : && ((fmd->fmd_vlan == VLAN_ID_INVALID) || vif_is_service(pkt->vp_if)))
2108 170 : result = vr_do_flow_lookup(router, pkt, fmd);
2109 :
2110 178 : return __vr_flow_forward(result, pkt, fmd);
2111 :
2112 0 : drop_pkt:
2113 0 : vr_pfree(pkt, VP_DROP_INVALID_HBS_PKT);
2114 0 : return false;
2115 : }
2116 :
2117 : int
2118 29 : vr_flow_flush_pnode(struct vrouter *router, struct vr_packet_node *pnode,
2119 : struct vr_flow_entry *fe, struct vr_forwarding_md *fmd)
2120 : {
2121 : bool forward;
2122 :
2123 : struct vr_interface *vif;
2124 : struct vr_packet *pkt;
2125 : struct vr_ip *ip;
2126 : struct vr_ip6 *ip6;
2127 : flow_result_t result;
2128 :
2129 29 : fmd->fmd_outer_src_ip = pnode->pl_outer_src_ip;
2130 29 : if (pnode->pl_flags & PN_FLAG_LABEL_IS_VXLAN_ID) {
2131 0 : vr_fmd_set_label(fmd, pnode->pl_label,
2132 : VR_LABEL_TYPE_VXLAN_ID);
2133 : } else {
2134 29 : vr_fmd_set_label(fmd, pnode->pl_label,
2135 : VR_LABEL_TYPE_MPLS);
2136 : }
2137 :
2138 29 : if (pnode->pl_flags & PN_FLAG_TO_ME)
2139 0 : fmd->fmd_to_me = 1;
2140 :
2141 29 : pkt = pnode->pl_packet;
2142 29 : if (!pkt)
2143 14 : return -EINVAL;
2144 :
2145 15 : fmd->fmd_dscp = pnode->pl_dscp;
2146 15 : fmd->fmd_dotonep = pnode->pl_dotonep;
2147 15 : fmd->fmd_vlan = pnode->pl_vlan;
2148 15 : fmd->fmd_mirror_data = pnode->pl_mirror_vlan;
2149 :
2150 15 : pnode->pl_packet = NULL;
2151 : /*
2152 : * this is only a security check and not a catch all check. one note
2153 : * of caution. please do not access pkt->vp_if till the if block is
2154 : * succesfully bypassed
2155 : */
2156 15 : vif = __vrouter_get_interface(router, pnode->pl_vif_idx);
2157 15 : if (!vif || (pkt->vp_if != vif)) {
2158 5 : pkt->vp_if = NULL;
2159 5 : PKT_LOG(VP_DROP_INVALID_IF, pkt, 0, VR_FLOW_C, __LINE__);
2160 5 : vr_pfree(pkt, VP_DROP_INVALID_IF);
2161 5 : return -ENODEV;
2162 : }
2163 :
2164 10 : if (!pkt->vp_nh) {
2165 10 : if (vif_is_fabric(pkt->vp_if) && fmd &&
2166 1 : (fmd->fmd_label >= 0)) {
2167 1 : if (!vr_fmd_label_is_vxlan_id(fmd)) {
2168 1 : pkt->vp_nh = __vrouter_get_label(router, fmd->fmd_label);
2169 : } else {
2170 0 : pkt->vp_nh = __vrouter_bridge_lookup(fmd->fmd_dvrf,
2171 : pkt_data(pkt));
2172 : }
2173 : }
2174 10 : if(vif_is_vhost(vif) && !(vif->vif_flags & VIF_FLAG_POLICY_ENABLED)) {
2175 1 : if (pkt->vp_type == VP_TYPE_IP) {
2176 1 : ip = (struct vr_ip *) pkt_network_header(pkt);
2177 1 : pkt->vp_nh = vr_inet_ip_lookup(pnode->pl_vrf, ip->ip_daddr);
2178 0 : } else if (pkt->vp_type == VP_TYPE_IP6) {
2179 0 : ip6 = (struct vr_ip6 *) pkt_network_header(pkt);
2180 0 : pkt->vp_nh = vr_inet6_ip_lookup(pnode->pl_vrf, ip6->ip6_dst);
2181 : }
2182 : }
2183 : }
2184 :
2185 10 : if (fe) {
2186 2 : result = vr_flow_action(router, fe, fmd->fmd_flow_index, pkt, fmd);
2187 2 : forward = __vr_flow_forward(result, pkt, fmd);
2188 : } else {
2189 8 : forward = vr_flow_forward(router, pkt, fmd);
2190 : }
2191 :
2192 10 : if (forward)
2193 9 : vr_reinject_packet(pkt, fmd);
2194 :
2195 10 : return 0;
2196 : }
2197 :
2198 : static void
2199 7 : __vr_flow_flush_hold_queue(struct vrouter *router, struct vr_flow_entry *fe,
2200 : struct vr_forwarding_md *fmd, struct vr_flow_queue *vfq)
2201 : {
2202 : unsigned int i;
2203 : struct vr_packet_node *pnode;
2204 :
2205 28 : for (i = 0; i < VR_MAX_FLOW_QUEUE_ENTRIES; i++) {
2206 21 : pnode = &vfq->vfq_pnodes[i];
2207 21 : vr_flow_flush_pnode(router, pnode, fe, fmd);
2208 : }
2209 :
2210 7 : return;
2211 : }
2212 :
2213 : static void
2214 329 : vr_flush_entry(struct vrouter *router, struct vr_flow_entry *fe,
2215 : struct vr_flow_md *flmd, struct vr_forwarding_md *fmd)
2216 : {
2217 : bool swapped;
2218 :
2219 : struct vr_flow_queue *vfq;
2220 329 : struct vr_defer_data *vdd = flmd->flmd_defer_data;
2221 : struct vr_flow_defer_data *vfdd;
2222 :
2223 329 : vfq = fe->fe_hold_list;
2224 329 : if (vfq) {
2225 5 : if (fe->fe_action == VR_FLOW_ACTION_HOLD)
2226 0 : return;
2227 :
2228 5 : swapped = vr_sync_bool_compare_and_swap_p(&fe->fe_hold_list, vfq, NULL);
2229 5 : if (swapped) {
2230 5 : __vr_flow_flush_hold_queue(router, fe, fmd, vfq);
2231 5 : if (!vdd || !vdd->vdd_data)
2232 3 : goto free_flush_queue;
2233 :
2234 2 : vfdd = (struct vr_flow_defer_data *)vdd->vdd_data;
2235 2 : vfdd->vfdd_flow_queue = vfq;
2236 : }
2237 : }
2238 :
2239 326 : return;
2240 :
2241 3 : free_flush_queue:
2242 3 : if (vfq)
2243 3 : vr_free(vfq, VR_FLOW_QUEUE_OBJECT);
2244 3 : return;
2245 : }
2246 :
2247 : static void
2248 318 : __vr_flow_work(struct vrouter *router, struct vr_flow_entry *fe,
2249 : struct vr_flow_md *flmd)
2250 : {
2251 : struct vr_forwarding_md fmd;
2252 :
2253 318 : vr_init_forwarding_md(&fmd);
2254 318 : vr_flow_set_forwarding_md(router, fe, flmd->flmd_index, &fmd);
2255 318 : vr_flush_entry(router, fe, flmd, &fmd);
2256 :
2257 318 : vr_flow_defer(flmd, fe);
2258 318 : return;
2259 : }
2260 :
2261 :
2262 : static void
2263 318 : vr_flow_work(void *arg)
2264 : {
2265 : struct vrouter *router;
2266 : struct vr_flow_entry *fe;
2267 318 : struct vr_flow_md *flmd =
2268 : (struct vr_flow_md *)arg;
2269 :
2270 318 : router = flmd->flmd_router;
2271 318 : if (!router)
2272 0 : goto exit_flush;
2273 :
2274 318 : fe = vr_flow_get_entry(router, flmd->flmd_index);
2275 318 : if (!fe)
2276 0 : goto exit_flush;
2277 :
2278 318 : __vr_flow_work(router, fe, flmd);
2279 :
2280 318 : exit_flush:
2281 318 : if (flmd->flmd_defer_data) {
2282 0 : if (flmd->flmd_defer_data->vdd_data) {
2283 0 : vr_free(flmd->flmd_defer_data->vdd_data,
2284 : VR_FLOW_DEFER_DATA_OBJECT);
2285 : }
2286 0 : vr_put_defer_data(flmd->flmd_defer_data);
2287 0 : flmd->flmd_defer_data = NULL;
2288 : }
2289 :
2290 318 : vr_free(flmd, VR_FLOW_METADATA_OBJECT);
2291 :
2292 318 : return;
2293 : }
2294 :
2295 : static void
2296 216 : vr_flow_set_mirror(struct vrouter *router, vr_flow_req *req,
2297 : struct vr_flow_entry *fe)
2298 : {
2299 216 : struct vr_mirror_entry *mirror = NULL, *sec_mirror = NULL;
2300 :
2301 216 : if (!(req->fr_flags & VR_FLOW_FLAG_MIRROR) &&
2302 214 : (fe->fe_flags & VR_FLOW_FLAG_MIRROR)) {
2303 0 : vr_flow_reset_mirror(router, fe, req->fr_index);
2304 0 : return;
2305 : }
2306 :
2307 216 : if (!(req->fr_flags & VR_FLOW_FLAG_MIRROR))
2308 214 : return;
2309 :
2310 2 : if (fe->fe_mirror_id != req->fr_mir_id) {
2311 2 : if (fe->fe_mirror_id < router->vr_max_mirror_indices) {
2312 0 : fe->fe_mirror_id = router->vr_max_mirror_indices;
2313 : }
2314 :
2315 2 : if ((unsigned int)req->fr_mir_id < router->vr_max_mirror_indices) {
2316 2 : mirror = vrouter_get_mirror(req->fr_rid, req->fr_mir_id);
2317 2 : if (mirror)
2318 2 : fe->fe_mirror_id = req->fr_mir_id;
2319 :
2320 : /* when we reached this point, we had already done all the
2321 : * sanity checks we could do. failing here will add only
2322 : * complexity to code here. so !mirror case, we will not
2323 : * handle
2324 : */
2325 : }
2326 : }
2327 :
2328 2 : if (fe->fe_sec_mirror_id != req->fr_sec_mir_id) {
2329 2 : if (fe->fe_sec_mirror_id < router->vr_max_mirror_indices) {
2330 0 : fe->fe_sec_mirror_id = router->vr_max_mirror_indices;
2331 : }
2332 :
2333 2 : if ((unsigned int)req->fr_sec_mir_id < router->vr_max_mirror_indices) {
2334 2 : sec_mirror = vrouter_get_mirror(req->fr_rid, req->fr_sec_mir_id);
2335 2 : if (sec_mirror)
2336 0 : fe->fe_sec_mirror_id = req->fr_sec_mir_id;
2337 : }
2338 : }
2339 :
2340 2 : if (req->fr_pcap_meta_data_size && req->fr_pcap_meta_data) {
2341 0 : if (fe->fe_mme) {
2342 0 : vr_mirror_meta_entry_del(router, fe->fe_mme);
2343 0 : fe->fe_mme = NULL;
2344 : }
2345 :
2346 0 : fe->fe_mme = vr_mirror_meta_entry_set(router, req->fr_index,
2347 0 : req->fr_mir_sip, req->fr_mir_sport,
2348 0 : req->fr_pcap_meta_data, req->fr_pcap_meta_data_size,
2349 0 : req->fr_mir_vrf);
2350 :
2351 0 : if (fe->fe_mme) {
2352 0 : vr_offload_flow_meta_data_set(req->fr_index,
2353 : req->fr_pcap_meta_data_size,
2354 0 : req->fr_pcap_meta_data,
2355 0 : req->fr_mir_vrf);
2356 : }
2357 : }
2358 :
2359 2 : return;
2360 : }
2361 :
2362 : void
2363 184 : vr_fill_flow_common(struct vr_flow *flowp, unsigned int nh_id,
2364 : uint8_t proto, uint16_t sport, uint16_t dport, uint8_t family,
2365 : uint8_t valid_fkey_params)
2366 : {
2367 184 : flowp->flow_nh_id = nh_id;
2368 184 : flowp->flow_family = family;
2369 184 : if (family == AF_INET)
2370 174 : flowp->flow_key_len = VR_FLOW_IPV4_HASH_SIZE;
2371 : else
2372 10 : flowp->flow_key_len = VR_FLOW_IPV6_HASH_SIZE;
2373 184 : flowp->flow_unused = 0;
2374 :
2375 184 : if (valid_fkey_params & VR_FLOW_KEY_PROTO)
2376 184 : flowp->flow_proto = proto;
2377 :
2378 184 : if (valid_fkey_params & VR_FLOW_KEY_SRC_PORT)
2379 184 : flowp->flow_sport = sport;
2380 :
2381 184 : if (valid_fkey_params & VR_FLOW_KEY_DST_PORT)
2382 184 : flowp->flow_dport = dport;
2383 :
2384 184 : return;
2385 : }
2386 :
2387 : static struct vr_flow_entry *
2388 104 : vr_add_flow(unsigned int rid, struct vr_flow *key, uint8_t type,
2389 : bool need_hold_queue, unsigned int *fe_index,
2390 : uint8_t *fe_gen_id)
2391 : {
2392 : struct vr_flow_entry *flow_e;
2393 104 : struct vrouter *router = vrouter_get(rid);
2394 :
2395 104 : struct vr_flow_entry *bucket_fe = vr_flow_bucket_first_entry(router, key);
2396 104 : vr_flow_bucket_may_lock(bucket_fe);
2397 104 : flow_e = vr_find_flow(router, key, type, fe_index);
2398 104 : if (flow_e) {
2399 0 : vr_flow_bucket_may_unlock(bucket_fe);
2400 0 : *fe_gen_id = flow_e->fe_gen_id;
2401 : /* a race between agent and dp. allow agent to handle this error */
2402 0 : return NULL;
2403 : } else {
2404 104 : flow_e = vr_flow_get_free_entry(router, key, type,
2405 : need_hold_queue, fe_index);
2406 104 : vr_flow_bucket_may_unlock(bucket_fe);
2407 : }
2408 :
2409 104 : return flow_e;
2410 : }
2411 :
2412 : static struct vr_flow_entry *
2413 104 : vr_add_flow_req(vr_flow_req *req, unsigned int *fe_index, uint8_t *fe_gen_id)
2414 : {
2415 : uint8_t type;
2416 104 : bool need_hold_queue = false;
2417 :
2418 : struct vr_flow key;
2419 : struct vr_flow_entry *fe;
2420 :
2421 104 : switch (req->fr_family) {
2422 6 : case AF_INET6:
2423 6 : type = VP_TYPE_IP6;
2424 6 : vr_inet6_fill_flow_from_req(&key, req);
2425 6 : break;
2426 :
2427 98 : case AF_INET:
2428 98 : type = VP_TYPE_IP;
2429 98 : vr_inet_fill_flow(&key, req->fr_flow_nh_id,
2430 98 : (uint32_t)req->fr_flow_sip_l, (uint32_t)req->fr_flow_dip_l,
2431 98 : req->fr_flow_proto, req->fr_flow_sport, req->fr_flow_dport,
2432 : VR_FLOW_KEY_ALL);
2433 98 : break;
2434 :
2435 0 : default:
2436 0 : return NULL;
2437 : }
2438 :
2439 104 : if (req->fr_action == VR_FLOW_ACTION_HOLD)
2440 0 : need_hold_queue = true;
2441 :
2442 104 : fe = vr_add_flow(req->fr_rid, &key, type, need_hold_queue, fe_index,
2443 : fe_gen_id);
2444 104 : if (fe)
2445 104 : req->fr_index = *fe_index;
2446 :
2447 104 : return fe;
2448 : }
2449 :
2450 : /*
2451 : * can be called with 'fe' as null (specifically when flow is added from
2452 : * agent), in which case we should be checking only the request
2453 : */
2454 : static int
2455 322 : vr_flow_set_req_is_invalid(struct vrouter *router, vr_flow_req *req,
2456 : struct vr_flow_entry *fe)
2457 : {
2458 322 : int error = 0, key_type;
2459 : struct vr_flow_entry *rfe;
2460 : struct vr_flow key;
2461 : uint64_t *ip;
2462 :
2463 322 : if (fe) {
2464 :
2465 : /* If Delete marked, dont allow any other change */
2466 210 : if (fe->fe_flags & VR_FLOW_FLAG_DELETE_MARKED)
2467 0 : return -EINVAL;
2468 :
2469 210 : if ((fe->fe_type == VP_TYPE_IP) || (fe->fe_type == VP_TYPE_IP6)) {
2470 210 : if ((uint8_t)req->fr_gen_id != fe->fe_gen_id) {
2471 0 : error = -EBADF;
2472 0 : goto invalid_req;
2473 : }
2474 :
2475 : /*
2476 : * when gen id is same flow keys should not mis-match
2477 : * send EFAULT if such incident happens
2478 : */
2479 210 : if((unsigned short)req->fr_flow_sport != fe->fe_key.flow_sport ||
2480 210 : (unsigned short)req->fr_flow_dport != fe->fe_key.flow_dport||
2481 210 : (uint32_t) req->fr_flow_nh_id != (uint32_t) fe->fe_key.flow_nh_id ||
2482 210 : (unsigned char)req->fr_flow_proto != fe->fe_key.flow_proto) {
2483 0 : error = -EFAULT;
2484 0 : goto invalid_req;
2485 : }
2486 :
2487 210 : if (fe->fe_type == VP_TYPE_IP) {
2488 200 : if ((fe->fe_key.flow4_sip != (uint32_t)req->fr_flow_sip_l) ||
2489 200 : (fe->fe_key.flow4_dip != (uint32_t)req->fr_flow_dip_l)) {
2490 0 : error = -EFAULT;
2491 0 : goto invalid_req;
2492 : }
2493 : } else {
2494 10 : ip = (uint64_t *)fe->fe_key.flow6_sip;
2495 10 : if ((*ip != req->fr_flow_sip_u) ||
2496 10 : (*(ip+1) != req->fr_flow_sip_l) ||
2497 10 : (*(ip+2) != req->fr_flow_dip_u) ||
2498 10 : (*(ip+3) != req->fr_flow_dip_l)) {
2499 0 : error = -EFAULT;
2500 0 : goto invalid_req;
2501 : }
2502 : }
2503 : }
2504 : } else {
2505 : /*
2506 : * flow set request received with an index which is
2507 : * not active anymore, return ENOENT error
2508 : */
2509 112 : if ((req->fr_flags & VR_FLOW_FLAG_ACTIVE) && !(req->fr_index < 0)) {
2510 0 : error = -ENOENT;
2511 0 : goto invalid_req;
2512 : }
2513 : }
2514 :
2515 322 : if (req->fr_flags & VR_FLOW_FLAG_VRFT) {
2516 12 : if ((unsigned short)req->fr_flow_dvrf >= router->vr_max_vrfs) {
2517 0 : error = -EINVAL;
2518 0 : goto invalid_req;
2519 : }
2520 : }
2521 :
2522 322 : if (req->fr_flags & VR_FLOW_FLAG_MIRROR) {
2523 2 : if (((unsigned int)req->fr_mir_id >= router->vr_max_mirror_indices) &&
2524 0 : (unsigned int)req->fr_sec_mir_id >= router->vr_max_mirror_indices) {
2525 0 : error = -EINVAL;
2526 0 : goto invalid_req;
2527 : }
2528 : }
2529 :
2530 322 : if (req->fr_flags & VR_RFLOW_VALID) {
2531 160 : if (req->fr_rindex != -1) {
2532 160 : rfe = vr_flow_get_entry(router, req->fr_rindex);
2533 : } else {
2534 0 : if (req->fr_family == AF_INET) {
2535 0 : vr_inet_fill_flow(&key, req->fr_rflow_nh_id,
2536 0 : (uint32_t)req->fr_rflow_sip_l, (uint32_t)req->fr_rflow_dip_l,
2537 0 : req->fr_flow_proto, req->fr_rflow_sport,
2538 0 : req->fr_rflow_dport, VR_FLOW_KEY_ALL);
2539 :
2540 0 : key_type = VP_TYPE_IP;
2541 : } else {
2542 0 : vr_inet6_fill_rflow_from_req(&key, req);
2543 0 : key_type = VP_TYPE_IP6;
2544 : }
2545 :
2546 0 : rfe = vr_find_flow(router, &key, key_type, &req->fr_rindex);
2547 : }
2548 :
2549 160 : if (!rfe) {
2550 0 : error = -EINVAL;
2551 0 : goto invalid_req;
2552 : }
2553 : }
2554 :
2555 322 : return 0;
2556 :
2557 0 : invalid_req:
2558 0 : return error;
2559 : }
2560 :
2561 : static int
2562 318 : __vr_flow_schedule_transition(struct vrouter *router, struct vr_flow_entry *fe,
2563 : unsigned int index, unsigned short flags)
2564 : {
2565 : struct vr_flow_md *flmd;
2566 318 : struct vr_defer_data *defer = NULL;
2567 318 : struct vr_flow_entry *rfe = NULL;
2568 :
2569 318 : flmd = (struct vr_flow_md *)vr_malloc(sizeof(*flmd),
2570 : VR_FLOW_METADATA_OBJECT);
2571 318 : if (!flmd)
2572 0 : return -ENOMEM;
2573 :
2574 318 : flmd->flmd_router = router;
2575 318 : flmd->flmd_index = index;
2576 318 : flmd->flmd_flags = flags;
2577 318 : if (fe->fe_hold_list || (flags & VR_FLOW_FLAG_EVICT_CANDIDATE)) {
2578 6 : defer = vr_get_defer_data(sizeof(*defer));
2579 6 : if (defer) {
2580 6 : defer->vdd_data = (void *)vr_zalloc(sizeof(struct vr_flow_defer_data),
2581 : VR_FLOW_DEFER_DATA_OBJECT);
2582 6 : if (!(flmd->flmd_flags & VR_FLOW_FLAG_ACTIVE)) {
2583 0 : ((struct vr_flow_defer_data *)defer->vdd_data)->vfdd_delete =
2584 : true;
2585 : }
2586 : /*
2587 : * Set vfdd_evict_flow to true only if there is only one flow
2588 : * or both flows have EVICT_CANDIDATE flag set;
2589 : * This is to avoid a race condition which can lead to non
2590 : * eviction of one of the flows. See CEM-4275 for more details.
2591 : *
2592 : * CEM-18166: Handle a corner case for BGPaaS where two different
2593 : * flows are trapped to agent (due to NAT translation to the same
2594 : * port), combined with TCP RST/FIN pkts received on the reverse
2595 : * flow. The sequence of events permits deletion of the FF, while
2596 : * RF still pointing to it. When the Controller tries to reset the
2597 : * connection, thus causing eviction the flow, vrouter will fail
2598 : * because rfe for the RF is NULL.
2599 : */
2600 6 : if ((fe->fe_flags & VR_FLOW_FLAG_EVICT_CANDIDATE)) {
2601 8 : if ((fe->fe_rflow < 0) ||
2602 4 : ((rfe = vr_flow_get_entry(router, fe->fe_rflow)) &&
2603 4 : (rfe->fe_flags & VR_FLOW_FLAG_EVICT_CANDIDATE)) || !rfe) {
2604 4 : ((struct vr_flow_defer_data *)defer->vdd_data)->vfdd_evict_flow =
2605 : true;
2606 : }
2607 : }
2608 : }
2609 : }
2610 318 : flmd->flmd_defer_data = defer;
2611 :
2612 318 : return vr_schedule_work(vr_get_cpu(), vr_flow_work, (void *)flmd);
2613 : }
2614 :
2615 : static int
2616 314 : vr_flow_schedule_transition(struct vrouter *router, vr_flow_req *req,
2617 : struct vr_flow_entry *fe)
2618 : {
2619 314 : return __vr_flow_schedule_transition(router, fe, req->fr_index, req->fr_flags);
2620 : }
2621 :
2622 : static int
2623 98 : vr_flow_delete(struct vrouter *router, vr_flow_req *req,
2624 : struct vr_flow_entry *fe)
2625 : {
2626 98 : int port = 0;
2627 : /* Delete Mark it */
2628 98 : fe->fe_flags |= VR_FLOW_FLAG_DELETE_MARKED;
2629 :
2630 :
2631 98 : if (fe->fe_flags & VR_FLOW_FLAG_LINK_LOCAL) {
2632 3 : if (fe->fe_key.flow_proto == VR_IP_PROTO_ICMP) {
2633 : /*
2634 : * ICMP id passed as source port would be
2635 : * used for relaxed policy flow lookup
2636 : */
2637 1 : port = ntohs(fe->fe_key.flow_sport);
2638 : } else {
2639 2 : port = ntohs(fe->fe_key.flow_dport);
2640 : }
2641 3 : vr_clear_link_local_port(router, AF_INET, fe->fe_key.flow_proto, port);
2642 : }
2643 :
2644 98 : fe->fe_action = VR_FLOW_ACTION_DROP;
2645 98 : vr_flow_reset_mirror(router, fe, req->fr_index);
2646 98 : return vr_flow_schedule_transition(router, req, fe);
2647 : }
2648 :
2649 : static void
2650 216 : vr_flow_udp_src_port (struct vrouter *router, struct vr_flow_entry *fe)
2651 : {
2652 : uint32_t hash_key[10], hashval, port_range, hash_len;
2653 : uint16_t port;
2654 :
2655 216 : if (fe->fe_udp_src_port)
2656 110 : return;
2657 :
2658 106 : if (hashrnd_inited == 0) {
2659 33 : get_random_bytes(&vr_hashrnd, sizeof(vr_hashrnd));
2660 33 : hashrnd_inited = 1;
2661 : }
2662 :
2663 106 : hash_key[0] = fe->fe_vrf;
2664 106 : hash_key[1] = (fe->fe_key.flow_sport << 16) | fe->fe_key.flow_dport;
2665 106 : memcpy(&hash_key[2], fe->fe_key.flow_ip, 2 * VR_IP_ADDR_SIZE(fe->fe_type));
2666 106 : hash_len = VR_FLOW_HASH_SIZE(fe->fe_type);
2667 :
2668 106 : hashval = vr_hash(hash_key, hash_len, vr_hashrnd);
2669 106 : port_range = VR_MUDP_PORT_RANGE_END - VR_MUDP_PORT_RANGE_START;
2670 106 : port = (uint16_t ) (((uint64_t ) hashval * port_range) >> 32);
2671 :
2672 106 : if (port > port_range) {
2673 : /*
2674 : * Shouldn't happen...
2675 : */
2676 0 : port = 0;
2677 : }
2678 106 : fe->fe_udp_src_port = port + VR_MUDP_PORT_RANGE_START;
2679 : }
2680 :
2681 : static void
2682 216 : vr_flow_update_link_local_port(struct vrouter *router, vr_flow_req *req,
2683 : struct vr_flow_entry *fe)
2684 : {
2685 216 : bool set_port = false;
2686 216 : int port = 0;
2687 :
2688 216 : if (!req || !fe)
2689 0 : return;
2690 :
2691 216 : if (fe->fe_type != VP_TYPE_IP)
2692 10 : return;
2693 :
2694 206 : if (fe->fe_key.flow_proto == VR_IP_PROTO_ICMP) {
2695 : /*
2696 : * ICMP id passed as source port would be
2697 : * used for relaxed policy flow lookup
2698 : */
2699 173 : port = ntohs(fe->fe_key.flow_sport);
2700 : } else {
2701 33 : port = ntohs(fe->fe_key.flow_dport);
2702 : }
2703 :
2704 206 : if (req->fr_flags & VR_FLOW_FLAG_LINK_LOCAL) {
2705 4 : if (!(fe->fe_flags & VR_FLOW_FLAG_LINK_LOCAL))
2706 2 : set_port = true;
2707 202 : } else if (fe->fe_flags & VR_FLOW_FLAG_LINK_LOCAL) {
2708 0 : vr_clear_link_local_port(router, AF_INET, fe->fe_key.flow_proto,
2709 : port);
2710 : }
2711 :
2712 206 : if (req->fr_flags & VR_FLOW_BGP_SERVICE) {
2713 0 : if (!(fe->fe_flags & VR_FLOW_BGP_SERVICE))
2714 0 : set_port = true;
2715 : }
2716 :
2717 206 : if (set_port) {
2718 2 : vr_set_link_local_port(router, AF_INET, fe->fe_key.flow_proto, port);
2719 : }
2720 :
2721 206 : return;
2722 : }
2723 :
2724 : static int
2725 0 : vr_flow_force_evict (struct vrouter *router, vr_flow_req *req)
2726 : {
2727 0 : struct vr_flow_entry *fe = NULL;
2728 :
2729 0 : fe = vr_flow_get_entry(router, req->fr_index);
2730 0 : if (fe) {
2731 : /* Do force eviction only for TCP flows with dead flag set */
2732 0 : if (fe->fe_key.flow_proto != VR_IP_PROTO_TCP) {
2733 0 : return -EINVAL;
2734 : }
2735 0 : if (!(fe->fe_tcp_flags & VR_FLOW_TCP_DEAD)) {
2736 0 : return -EINVAL;
2737 : }
2738 0 : if (!(fe->fe_flags & VR_FLOW_FLAG_ACTIVE)) {
2739 0 : return -EINVAL;
2740 : }
2741 : /*
2742 : * We have a TCP flow which is dead,
2743 : * Evict Candidate may or may not be set
2744 : */
2745 0 : vr_flow_start_modify(router, fe);
2746 : /*
2747 : * Reset flow's reverse flow index as we don't want to
2748 : * touch reverse flow
2749 : */
2750 0 : fe->fe_flags &= ~VR_RFLOW_VALID;
2751 0 : fe->fe_rflow = -1;
2752 0 : if (!(fe->fe_flags & VR_FLOW_FLAG_EVICT_CANDIDATE)) {
2753 0 : __vr_flow_mark_evict(router, fe);
2754 : }
2755 0 : __vr_flow_schedule_transition(router, fe, req->fr_index, fe->fe_flags);
2756 : } else {
2757 0 : return -EINVAL;
2758 : }
2759 0 : return 0;
2760 : }
2761 :
2762 :
2763 : /* command from agent */
2764 : static int
2765 322 : vr_flow_set(struct vrouter *router, vr_flow_req *req,
2766 : vr_flow_response *flow_resp)
2767 0 : {
2768 : int ret;
2769 : int8_t cur_fe_underlay_ecmp_index;
2770 322 : unsigned int fe_index = (unsigned int)-1;
2771 322 : uint8_t fe_gen_id = 0;
2772 322 : bool new_flow = false, modified = false;
2773 :
2774 322 : struct vr_flow_entry *fe = NULL, *rfe = NULL;
2775 322 : struct vr_flow_table_info *infop = router->vr_flow_table_info;
2776 :
2777 322 : router = vrouter_get(req->fr_rid);
2778 322 : if (!router)
2779 0 : return -EINVAL;
2780 :
2781 322 : flow_resp->fresp_index = req->fr_index;
2782 :
2783 322 : if (req->fr_extflags & VR_FLOW_EXT_FLAG_FORCE_EVICT) {
2784 0 : return vr_flow_force_evict(router, req);
2785 : }
2786 :
2787 322 : fe = vr_flow_get_entry(router, req->fr_index);
2788 322 : if (fe) {
2789 210 : if (!(modified = vr_flow_start_modify(router, fe)))
2790 0 : return -EBUSY;
2791 210 : fe_index = (unsigned int)(req->fr_index);
2792 : }
2793 :
2794 322 : if ((ret = vr_flow_set_req_is_invalid(router, req, fe)))
2795 0 : goto exit_set;
2796 :
2797 322 : if (fe) {
2798 210 : if ((fe->fe_action == VR_FLOW_ACTION_HOLD) &&
2799 2 : ((req->fr_action != fe->fe_action) ||
2800 0 : !(req->fr_flags & VR_FLOW_FLAG_ACTIVE))) {
2801 2 : vr_sync_fetch_and_add_64u(&infop->vfti_action_count, 1);
2802 : } else {
2803 208 : infop->vfti_changed++;
2804 : }
2805 :
2806 : }
2807 : /*
2808 : * for delete, absence of the requested flow entry is caustic. so
2809 : * handle that case first
2810 : */
2811 322 : if (!(req->fr_flags & VR_FLOW_FLAG_ACTIVE)) {
2812 106 : if (!fe)
2813 8 : return -ENOENT;
2814 :
2815 98 : infop->vfti_deleted++;
2816 98 : flow_resp->fresp_flags |= VR_FLOW_RESP_FLAG_DELETED;
2817 98 : return vr_flow_delete(router, req, fe);
2818 : }
2819 :
2820 :
2821 : /*
2822 : * for non-delete cases, absence of flow entry means addition of a
2823 : * new flow entry with the key specified in the request
2824 : */
2825 216 : if (!fe) {
2826 104 : fe = vr_add_flow_req(req, &fe_index, &fe_gen_id);
2827 104 : if (!fe) {
2828 0 : if (fe_index != (unsigned int)-1) {
2829 : /*
2830 : * add flow req failed to allocate an entry due to race
2831 : * between agent and datapath, where flow entry at fe_index
2832 : * was already created due to packet trap, return EEXIST
2833 : * error and allow agent to wait and handle flow add due to
2834 : * packet trap
2835 : */
2836 0 : flow_resp->fresp_index = fe_index;
2837 0 : flow_resp->fresp_gen_id = fe_gen_id;
2838 0 : return -EEXIST;
2839 : }
2840 0 : return -ENOSPC;
2841 : }
2842 :
2843 104 : new_flow = true;
2844 104 : infop->vfti_added++;
2845 : } else {
2846 112 : if ((req->fr_action == VR_FLOW_ACTION_HOLD) &&
2847 0 : (fe->fe_action != req->fr_action)) {
2848 0 : if (!fe->fe_hold_list) {
2849 0 : fe->fe_hold_list = vr_zalloc(sizeof(struct vr_flow_queue),
2850 : VR_FLOW_QUEUE_OBJECT);
2851 0 : if (!fe->fe_hold_list) {
2852 0 : ret = -ENOMEM;
2853 0 : goto exit_set;
2854 : }
2855 : }
2856 : }
2857 : }
2858 :
2859 216 : flow_resp->fresp_gen_id = fe->fe_gen_id;
2860 216 : flow_resp->fresp_index = fe->fe_hentry.hentry_index;
2861 :
2862 216 : vr_flow_set_mirror(router, req, fe);
2863 :
2864 216 : if (req->fr_flags & VR_RFLOW_VALID) {
2865 160 : fe->fe_rflow = req->fr_rindex;
2866 : } else {
2867 56 : if (fe->fe_rflow >= 0)
2868 0 : fe->fe_rflow = -1;
2869 : }
2870 :
2871 216 : fe->fe_vrf = req->fr_flow_vrf;
2872 216 : if (req->fr_flags & VR_FLOW_FLAG_VRFT)
2873 12 : fe->fe_dvrf = req->fr_flow_dvrf;
2874 :
2875 216 : vr_flow_update_link_local_port(router, req, fe);
2876 :
2877 216 : if (fe->fe_ecmp_nh_index == -1)
2878 216 : (void)vr_sync_bool_compare_and_swap_8s(&fe->fe_ecmp_nh_index, -1,
2879 : req->fr_ecmp_nh_index);
2880 :
2881 216 : fe->fe_src_nh_index = req->fr_src_nh_index;
2882 216 : fe->fe_qos_id = req->fr_qos_id;
2883 :
2884 216 : if ((req->fr_action == VR_FLOW_ACTION_HOLD) &&
2885 0 : (fe->fe_action != VR_FLOW_ACTION_HOLD)) {
2886 0 : vr_flow_entry_set_hold(router, fe, false);
2887 : } else {
2888 216 : fe->fe_action = req->fr_action;
2889 : }
2890 :
2891 216 : fe->fe_ttl = req->fr_ttl;
2892 :
2893 216 : cur_fe_underlay_ecmp_index = req->fr_underlay_ecmp_index;
2894 : /* repeats until the comapre and swap operation is successful */
2895 216 : while(!vr_sync_bool_compare_and_swap_8s(&fe->fe_underlay_ecmp_index,
2896 : fe->fe_underlay_ecmp_index, cur_fe_underlay_ecmp_index)) {
2897 0 : continue;
2898 : }
2899 :
2900 216 : if (fe->fe_action == VR_FLOW_ACTION_DROP)
2901 0 : fe->fe_drop_reason = (uint8_t)req->fr_drop_reason;
2902 :
2903 216 : fe->fe_flags = VR_FLOW_FLAG_DP_BITS(fe) |
2904 216 : VR_FLOW_FLAG_MASK(req->fr_flags);
2905 216 : fe->fe_flags1 = req->fr_flags1;
2906 216 : if (new_flow) {
2907 :
2908 104 : flow_resp->fresp_bytes = fe->fe_stats.flow_bytes;
2909 104 : flow_resp->fresp_packets = fe->fe_stats.flow_packets;
2910 104 : flow_resp->fresp_stats_oflow = (fe->fe_stats.flow_bytes_oflow |
2911 104 : (fe->fe_stats.flow_packets_oflow << 16));
2912 :
2913 104 : if (fe->fe_flags & VR_FLOW_FLAG_NEW_FLOW) {
2914 104 : if (fe->fe_stats.flow_packets || fe->fe_stats.flow_packets_oflow)
2915 0 : memset(&fe->fe_stats, 0, sizeof(fe->fe_stats));
2916 : }
2917 :
2918 104 : if (fe->fe_flags & VR_RFLOW_VALID) {
2919 49 : rfe = vr_flow_get_entry(router, fe->fe_rflow);
2920 49 : if (rfe) {
2921 49 : vr_flow_tcp_rflow_set(router, fe, rfe);
2922 : }
2923 : }
2924 :
2925 : }
2926 :
2927 216 : vr_flow_udp_src_port(router, fe);
2928 :
2929 : /* Mock Src UDP port used for vrouter simulation - vtest */
2930 216 : if(req->fr_extflags & VR_FLOW_EXT_FLAG_MOCK_SRC_UDP)
2931 2 : fe->fe_udp_src_port = VR_FLOW_MOCK_SRC_UDP_PORT;
2932 :
2933 216 : if (fe->fe_flags & VR_FLOW_FLAG_NEW_FLOW)
2934 104 : fe->fe_flags &= ~VR_FLOW_FLAG_NEW_FLOW;
2935 :
2936 :
2937 :
2938 216 : ret = vr_flow_schedule_transition(router, req, fe);
2939 :
2940 : /*
2941 : * offload, no need to differentiate between add and modify. Pass the
2942 : * reverse flow as well if present.
2943 : */
2944 216 : if (!ret) {
2945 216 : vr_offload_flow_set(fe, fe_index, rfe);
2946 : }
2947 :
2948 0 : exit_set:
2949 216 : if (modified && fe) {
2950 112 : vr_flow_stop_modify(router, fe);
2951 : }
2952 :
2953 216 : return ret;
2954 : }
2955 :
2956 : static void
2957 13 : vr_flow_table_data_destroy(vr_flow_table_data *ftable)
2958 : {
2959 13 : if (!ftable)
2960 0 : return;
2961 :
2962 13 : if (ftable->ftable_file_path) {
2963 13 : vr_free(ftable->ftable_file_path, VR_FLOW_REQ_PATH_OBJECT);
2964 13 : ftable->ftable_file_path = NULL;
2965 : }
2966 :
2967 13 : if (ftable->ftable_hold_stat && ftable->ftable_hold_stat_size) {
2968 13 : vr_free(ftable->ftable_hold_stat, VR_FLOW_HOLD_STAT_OBJECT);
2969 13 : ftable->ftable_hold_stat = NULL;
2970 13 : ftable->ftable_hold_stat_size = 0;
2971 : }
2972 :
2973 13 : vr_free(ftable, VR_FLOW_TABLE_DATA_OBJECT);
2974 :
2975 13 : return;
2976 : }
2977 :
2978 : static vr_flow_table_data *
2979 13 : vr_flow_table_data_get(vr_flow_table_data *ref)
2980 : {
2981 : unsigned int hold_stat_size;
2982 13 : unsigned int num_cpus = vr_num_cpus;
2983 13 : vr_flow_table_data *ftable = vr_zalloc(sizeof(*ref),
2984 : VR_FLOW_TABLE_DATA_OBJECT);
2985 :
2986 13 : if (!ftable)
2987 0 : return NULL;
2988 :
2989 13 : if (vr_flow_path) {
2990 13 : ftable->ftable_file_path = vr_zalloc(VR_UNIX_PATH_MAX,
2991 : VR_FLOW_REQ_PATH_OBJECT);
2992 13 : if (!ftable->ftable_file_path) {
2993 0 : vr_free(ftable, VR_FLOW_TABLE_DATA_OBJECT);
2994 0 : return NULL;
2995 : }
2996 : }
2997 :
2998 13 : if (num_cpus > VR_FLOW_MAX_CPUS)
2999 0 : num_cpus = VR_FLOW_MAX_CPUS;
3000 :
3001 13 : hold_stat_size = num_cpus * sizeof(uint32_t);
3002 13 : ftable->ftable_hold_stat = vr_zalloc(hold_stat_size, VR_FLOW_HOLD_STAT_OBJECT);
3003 13 : if (!ftable->ftable_hold_stat) {
3004 0 : if (ftable->ftable_file_path) {
3005 0 : vr_free(ftable->ftable_file_path, VR_FLOW_REQ_PATH_OBJECT);
3006 0 : ftable->ftable_file_path = NULL;
3007 : }
3008 :
3009 0 : vr_free(ftable, VR_FLOW_TABLE_DATA_OBJECT);
3010 0 : return NULL;
3011 : }
3012 13 : ftable->ftable_hold_stat_size = num_cpus;
3013 :
3014 13 : return ftable;
3015 : }
3016 :
3017 : void
3018 8186880 : update_flow_entry(vr_htable_t table __attribute__unused__, vr_hentry_t *ent ,
3019 : unsigned int index, void *data __attribute__unused__)
3020 : {
3021 8186880 : struct vr_flow_entry *fe = (struct vr_flow_entry *)ent;
3022 :
3023 8186880 : if (fe == NULL)
3024 8186865 : return;
3025 :
3026 15 : vr_offload_flow_stats_update(fe);
3027 : }
3028 :
3029 : /*
3030 : * sandesh handler for vr_flow_table_data
3031 : */
3032 : void
3033 13 : vr_flow_table_data_process(void *s_req)
3034 : {
3035 13 : int i, ret = 0;
3036 13 : uint64_t hold_count = 0;
3037 : struct vrouter *router;
3038 : struct vr_flow_table_info *infop;
3039 13 : vr_flow_table_data *resp = NULL, *ftable = (vr_flow_table_data *)s_req;
3040 :
3041 13 : if (!ftable) {
3042 0 : ret = -ENOMEM;
3043 0 : goto send_response;
3044 : }
3045 13 : router = vrouter_get(ftable->ftable_rid);
3046 13 : if (! router) {
3047 0 : ret = -ENOMEM;
3048 0 : goto send_response;
3049 : }
3050 :
3051 13 : if(EINVAL == vr_htable_trav(router->vr_flow_table, 0, update_flow_entry, NULL))
3052 : {
3053 0 : ret = -ENOMEM;
3054 0 : goto send_response;
3055 : }
3056 13 : resp = vr_flow_table_data_get(ftable);
3057 13 : if (!resp) {
3058 0 : ret = -ENOMEM;
3059 0 : goto send_response;
3060 : }
3061 :
3062 13 : infop = router->vr_flow_table_info;
3063 13 : resp->ftable_op = ftable->ftable_op;
3064 13 : resp->ftable_size = vr_flow_table_size(router);
3065 : #if defined(__linux__) && defined(__KERNEL__)
3066 : resp->ftable_dev = vr_flow_major;
3067 : #endif
3068 13 : if (vr_flow_path) {
3069 13 : strncpy(resp->ftable_file_path, vr_flow_path, VR_UNIX_PATH_MAX - 1);
3070 : }
3071 :
3072 13 : if (!infop)
3073 0 : goto send_response;
3074 :
3075 13 : resp->ftable_used_entries = vr_flow_table_used_total_entries(router);
3076 13 : resp->ftable_deleted = infop->vfti_deleted;
3077 13 : resp->ftable_changed = infop->vfti_changed;
3078 13 : resp->ftable_processed = infop->vfti_action_count;
3079 13 : resp->ftable_hold_oflows = infop->vfti_oflows;
3080 13 : resp->ftable_added = infop->vfti_added;
3081 13 : resp->ftable_cpus = vr_num_cpus;
3082 : /* we only have space for 64 stats block max when encoding */
3083 169 : for (i = 0; ((i < vr_num_cpus) && (i < VR_FLOW_MAX_CPUS)); i++) {
3084 156 : resp->ftable_hold_stat[i] = infop->vfti_hold_count[i];
3085 156 : hold_count += resp->ftable_hold_stat[i];
3086 : }
3087 :
3088 13 : resp->ftable_created = hold_count;
3089 13 : resp->ftable_oflow_entries = vr_flow_table_used_oflow_entries(router);
3090 13 : resp->ftable_burst_free_tokens = infop->vfti_burst_tokens - infop->vfti_burst_used;
3091 13 : resp->ftable_hold_entries = vr_flow_table_hold_count(router);
3092 :
3093 13 : send_response:
3094 13 : vr_message_response(VR_FLOW_TABLE_DATA_OBJECT_ID, resp, ret, false);
3095 13 : if (resp)
3096 13 : vr_flow_table_data_destroy(resp);
3097 :
3098 13 : return;
3099 : }
3100 :
3101 : /*
3102 : * sandesh handler for vr_flow_req
3103 : */
3104 : void
3105 322 : vr_flow_req_process(void *s_req)
3106 : {
3107 322 : int ret = 0;
3108 : struct vrouter *router;
3109 322 : vr_flow_req *req = (vr_flow_req *)s_req;
3110 : vr_flow_response flow_resp;
3111 :
3112 322 : router = vrouter_get(req->fr_rid);
3113 322 : switch (req->fr_op) {
3114 322 : case FLOW_OP_FLOW_SET:
3115 :
3116 322 : flow_resp.fresp_rid = 0;
3117 322 : flow_resp.fresp_op = req->fr_op;
3118 :
3119 322 : ret = vr_flow_set(router, req, &flow_resp);
3120 322 : break;
3121 :
3122 0 : default:
3123 0 : ret = -EINVAL;
3124 : }
3125 :
3126 322 : vr_message_response(VR_FLOW_RESPONSE_OBJECT_ID, &flow_resp, ret, false);
3127 :
3128 322 : return;
3129 : }
3130 :
3131 : void
3132 0 : vr_flow_response_process(void *s_req)
3133 : {
3134 0 : return;
3135 : }
3136 :
3137 : static void
3138 53 : vr_flow_table_info_destroy(struct vrouter *router)
3139 : {
3140 53 : if (!router->vr_flow_table_info)
3141 0 : return;
3142 :
3143 53 : vr_free(router->vr_flow_table_info, VR_FLOW_TABLE_INFO_OBJECT);
3144 53 : router->vr_flow_table_info = NULL;
3145 53 : router->vr_flow_table_info_size = 0;
3146 :
3147 53 : return;
3148 : }
3149 :
3150 : static void
3151 53 : vr_flow_table_info_reset(struct vrouter *router)
3152 : {
3153 53 : if (!router->vr_flow_table_info)
3154 0 : return;
3155 :
3156 53 : if (router->vr_flow_table_info->vfti_timer) {
3157 0 : vr_delete_timer(router->vr_flow_table_info->vfti_timer);
3158 0 : vr_free(router->vr_flow_table_info->vfti_timer, VR_TIMER_OBJECT);
3159 0 : router->vr_flow_table_info->vfti_timer = NULL;
3160 : }
3161 :
3162 53 : memset(router->vr_flow_table_info, 0, router->vr_flow_table_info_size);
3163 :
3164 53 : return;
3165 : }
3166 :
3167 : static int
3168 53 : vr_flow_table_info_init(struct vrouter *router)
3169 : {
3170 : unsigned int size;
3171 : struct vr_flow_table_info *infop;
3172 :
3173 53 : if (router->vr_flow_table_info)
3174 0 : return 0;
3175 :
3176 53 : size = sizeof(struct vr_flow_table_info) + sizeof(uint32_t) * vr_num_cpus;
3177 53 : infop = (struct vr_flow_table_info *)vr_zalloc(size,
3178 : VR_FLOW_TABLE_INFO_OBJECT);
3179 53 : if (!infop)
3180 0 : return vr_module_error(-ENOMEM, __FUNCTION__, __LINE__, size);
3181 :
3182 53 : router->vr_flow_table_info = infop;
3183 53 : router->vr_flow_table_info_size = size;
3184 :
3185 53 : return 0;
3186 : }
3187 :
3188 : static void
3189 53 : vr_flow_table_destroy(struct vrouter *router)
3190 : {
3191 53 : if (router->vr_flow_table) {
3192 53 : vr_htable_delete(router->vr_flow_table);
3193 53 : router->vr_flow_table = NULL;
3194 : }
3195 :
3196 53 : vr_flow_table_info_destroy(router);
3197 :
3198 53 : return;
3199 : }
3200 :
3201 : static void
3202 33377280 : vr_flow_invalidate_entry(vr_htable_t htable, vr_hentry_t *ent,
3203 : unsigned int index, void *data)
3204 : {
3205 : struct vr_flow_entry *fe;
3206 : struct vr_forwarding_md fmd;
3207 : struct vr_flow_md flmd;
3208 33377280 : struct vrouter *router = (struct vrouter *)data;
3209 :
3210 33377280 : if (!ent || !data)
3211 33377269 : return;
3212 :
3213 33377280 : fe = CONTAINER_OF(fe_hentry, struct vr_flow_entry, ent);
3214 33377280 : if (!(fe->fe_flags & VR_FLOW_FLAG_ACTIVE))
3215 33377269 : return;
3216 :
3217 11 : flmd.flmd_defer_data = NULL;
3218 11 : flmd.flmd_index = index;
3219 11 : flmd.flmd_flags = fe->fe_flags;
3220 :
3221 11 : vr_init_forwarding_md(&fmd);
3222 :
3223 11 : fe->fe_action = VR_FLOW_ACTION_DROP;
3224 11 : vr_flush_entry(router, fe, &flmd, &fmd);
3225 11 : vr_flow_reset_entry(router, fe);
3226 : }
3227 :
3228 : static void
3229 53 : vr_flow_table_reset(struct vrouter *router)
3230 : {
3231 53 : vr_htable_reset(router->vr_flow_table,
3232 : vr_flow_invalidate_entry, router);
3233 53 : vr_flow_table_info_reset(router);
3234 :
3235 53 : return;
3236 : }
3237 :
3238 : void
3239 66 : vr_compute_size_oflow_table(void)
3240 : {
3241 : /*
3242 : * Overflow entries is 20% of the main flow table
3243 : * adjusted to next 1k
3244 : */
3245 66 : if (!vr_oflow_entries) {
3246 0 : vr_oflow_entries = ((vr_flow_entries / 5) + 1023) & ~1023;
3247 : }
3248 66 : }
3249 :
3250 : static int
3251 53 : vr_flow_table_init(struct vrouter *router)
3252 : {
3253 53 : if (!router->vr_flow_table) {
3254 :
3255 53 : vr_compute_size_oflow_table();
3256 :
3257 53 : if (!vr_flow_table && vr_huge_page_mem_get) {
3258 :
3259 0 : vr_flow_table = vr_huge_page_mem_get(VR_FLOW_TABLE_SIZE +
3260 : VR_OFLOW_TABLE_SIZE, &vr_flow_path);
3261 0 : if (vr_flow_table)
3262 0 : vr_oflow_table = (char*)vr_flow_table + VR_FLOW_TABLE_SIZE;
3263 : }
3264 :
3265 53 : router->vr_flow_table = vr_htable_attach(router, vr_flow_entries,
3266 : vr_flow_table, vr_oflow_entries, vr_oflow_table,
3267 : sizeof(struct vr_flow_entry), 0, 0, vr_flow_get_key);
3268 :
3269 53 : if (!router->vr_flow_table) {
3270 0 : return vr_module_error(-ENOMEM, __FUNCTION__,
3271 0 : __LINE__, vr_flow_entries + vr_oflow_entries);
3272 : }
3273 : }
3274 :
3275 53 : return vr_flow_table_info_init(router);
3276 : }
3277 :
3278 : static void
3279 53 : vr_link_local_ports_reset(struct vrouter *router)
3280 : {
3281 53 : if (router->vr_link_local_ports) {
3282 53 : memset(router->vr_link_local_ports,
3283 53 : 0, router->vr_link_local_ports_size);
3284 : }
3285 :
3286 53 : return;
3287 : }
3288 :
3289 : static void
3290 53 : vr_link_local_ports_exit(struct vrouter *router)
3291 : {
3292 53 : if (router->vr_link_local_ports) {
3293 53 : vr_free(router->vr_link_local_ports, VR_FLOW_LINK_LOCAL_OBJECT);
3294 53 : router->vr_link_local_ports = NULL;
3295 53 : router->vr_link_local_ports_size = 0;
3296 : }
3297 :
3298 53 : return;
3299 : }
3300 :
3301 : static int
3302 53 : vr_link_local_ports_init(struct vrouter *router)
3303 : {
3304 : unsigned int port_range, bytes;
3305 :
3306 53 : if (router->vr_link_local_ports)
3307 0 : return 0;
3308 :
3309 : /* ICMP, Udp and TCP inclusive of low and high limits*/
3310 53 : port_range = VR_LL_RP_MAX *
3311 : ((VR_DYNAMIC_PORT_END - VR_DYNAMIC_PORT_START) + 1);
3312 : /* Make it 16 bit boundary */
3313 53 : bytes = (port_range + 15) & ~15;
3314 : /* Bits to Bytes */
3315 53 : bytes /= 8;
3316 :
3317 53 : router->vr_link_local_ports = vr_zalloc(bytes, VR_FLOW_LINK_LOCAL_OBJECT);
3318 53 : if (!router->vr_link_local_ports)
3319 0 : return -1;
3320 53 : router->vr_link_local_ports_size = bytes;
3321 :
3322 53 : return 0;
3323 : }
3324 :
3325 : /* flow module exit and init */
3326 : void
3327 53 : vr_flow_exit(struct vrouter *router, bool soft_reset)
3328 : {
3329 53 : vr_flow_table_reset(router);
3330 53 : vr_link_local_ports_reset(router);
3331 53 : if (!soft_reset) {
3332 53 : vr_flow_table_destroy(router);
3333 53 : vr_fragment_table_exit(router);
3334 53 : vr_link_local_ports_exit(router);
3335 : }
3336 :
3337 53 : return;
3338 : }
3339 :
3340 : int
3341 53 : vr_flow_mem(struct vrouter *router)
3342 : {
3343 : int ret;
3344 :
3345 53 : if ((ret = vr_fragment_table_init(router)) < 0)
3346 0 : return ret;
3347 :
3348 53 : if ((ret = vr_flow_table_init(router)))
3349 0 : return ret;
3350 :
3351 53 : if ((ret = vr_link_local_ports_init(router)))
3352 0 : return ret;
3353 :
3354 53 : return 0;
3355 : }
3356 :
3357 : int
3358 53 : vr_flow_init(struct vrouter *router)
3359 : {
3360 53 : return 0;
3361 : }
3362 :
3363 : /*
3364 : * Called by offload module to update flow stats with packets which have been
3365 : * offloaded. over_flow_bytes and over_flow_packets account for overflows which
3366 : * happen in firmware between updates using this function.
3367 : */
3368 : int
3369 0 : vr_flow_incr_stats(int fe_index, uint32_t flow_bytes, uint16_t over_flow_bytes,
3370 : uint32_t flow_packets, uint8_t over_flow_packets)
3371 : {
3372 0 : struct vrouter *router = vrouter_get(0);
3373 : struct vr_flow_entry *fe;
3374 : uint32_t new_stats;
3375 :
3376 0 : if (router == NULL)
3377 0 : return -EINVAL;
3378 :
3379 0 : fe = vr_flow_get_entry(router, fe_index);
3380 0 : if (fe == NULL)
3381 0 : return -ENOENT;
3382 :
3383 0 : if (!(fe->fe_flags & VR_FLOW_FLAG_ACTIVE))
3384 0 : return -ENOENT;
3385 :
3386 0 : new_stats = vr_sync_add_and_fetch_32u(&fe->fe_stats.flow_bytes, flow_bytes);
3387 0 : if (new_stats < flow_bytes)
3388 0 : ++fe->fe_stats.flow_bytes_oflow;
3389 0 : fe->fe_stats.flow_bytes_oflow += over_flow_bytes;
3390 :
3391 0 : new_stats = vr_sync_add_and_fetch_32u(&fe->fe_stats.flow_packets,
3392 : flow_packets);
3393 0 : if (new_stats < flow_packets)
3394 0 : ++fe->fe_stats.flow_packets_oflow;
3395 0 : fe->fe_stats.flow_packets_oflow += over_flow_packets;
3396 :
3397 0 : return 0;
3398 : }
|
