Line data Source code
1 : /*
2 : * vr_flow.c -- flow handling
3 : *
4 : * Copyright (c) 2013 Juniper Networks, Inc. All rights reserved.
5 : */
6 : #include <vr_os.h>
7 : #include <vr_types.h>
8 : #include <vrouter.h>
9 : #include <vr_packet.h>
10 : #include <vr_htable.h>
11 : #include <vr_flow.h>
12 : #include <vr_mirror.h>
13 : #include "vr_interface.h"
14 : #include "vr_sandesh.h"
15 : #include "vr_message.h"
16 : #include "vr_btable.h"
17 : #include "vr_fragment.h"
18 : #include "vr_datapath.h"
19 : #include "vr_hash.h"
20 : #include "vr_ip_mtrie.h"
21 : #include "vr_bridge.h"
22 : #include "vr_vrf_table.h"
23 : #include "vr_nexthop.h"
24 :
25 : #include "vr_offloads_dp.h"
26 :
27 : #define VR_NUM_FLOW_TABLES 1
28 :
29 : #define VR_NUM_OFLOW_TABLES 1
30 :
31 : #define VR_DEF_MAX_FLOW_TABLE_HOLD_COUNT 8192
32 :
33 : unsigned int vr_flow_entries = VR_DEF_FLOW_ENTRIES;
34 : unsigned int vr_oflow_entries = 0;
35 : /*
36 : * Knob to unconditionally close flow on TCP RST;
37 : * If this knob is set, the flow would be closed
38 : * on receiving a TCP RST without doing any seqnum
39 : * validation (for backward compatibility with older
40 : * implementation);
41 : * If this knob is off, TCP RST seqnum validation
42 : * as per RFC 5961 sec 3.2 will be done. If the
43 : * validation fails, the RST will be ignored.
44 : * By default the knob is off;
45 : */
46 : unsigned int vr_uncond_close_flow_on_tcp_rst = 0;
47 :
48 : /*
49 : * host can provide its own memory . Point in case is the DPDK. In DPDK,
50 : * we allocate the table from hugepages and just ask the flow module to
51 : * use those tables
52 : */
53 : void *vr_flow_table = NULL;
54 : void *vr_oflow_table = NULL;
55 : /*
56 : * The flow table memory can also be a file that could be mapped. The path
57 : * is set by somebody and passed to agent for it to map
58 : */
59 : unsigned char *vr_flow_path;
60 : unsigned int vr_flow_hold_limit = VR_DEF_MAX_FLOW_TABLE_HOLD_COUNT;
61 :
62 : #if defined(__linux__) && defined(__KERNEL__)
63 : extern short vr_flow_major;
64 : #endif
65 :
66 : void vr_flow_defer_cb(struct vrouter *router, void *arg);
67 :
68 : uint32_t vr_hashrnd = 0;
69 : int hashrnd_inited = 0;
70 :
71 109 : static struct vr_flow_entry *vr_flow_bucket_first_entry(struct vrouter *router, struct vr_flow *key) {
72 109 : return (struct vr_flow_entry*)
73 109 : vr_htable_get_bucket(router->vr_flow_table, key, key->flow_key_len);
74 : }
75 :
76 : /*
77 : * The vr_flow_bucket_lock and vr_flow_bucket_unlock callbacks may not be
78 : * implemented for all instances of host_os. Those callback are needed only for
79 : * implementations that require strict race condition controll for
80 : * hardware offloading (like DPDK).
81 : * Those helper functions call the locks only when they are available.
82 : */
83 109 : static void vr_flow_bucket_may_lock(struct vr_flow_entry *fe) {
84 109 : if (vr_flow_bucket_lock && vr_flow_bucket_unlock)
85 109 : vr_flow_bucket_lock(fe);
86 109 : }
87 :
88 109 : static void vr_flow_bucket_may_unlock(struct vr_flow_entry *fe) {
89 109 : if (vr_flow_bucket_lock && vr_flow_bucket_unlock)
90 109 : vr_flow_bucket_unlock(fe);
91 109 : }
92 :
93 : static void vr_flush_entry(struct vrouter *, struct vr_flow_entry *,
94 : struct vr_flow_md *, struct vr_forwarding_md *);
95 : static void __vr_flow_flush_hold_queue(struct vrouter *, struct vr_flow_entry *,
96 : struct vr_forwarding_md *, struct vr_flow_queue *);
97 : static void vr_flow_set_forwarding_md(struct vrouter *, struct vr_flow_entry *,
98 : unsigned int, struct vr_forwarding_md *);
99 : static int
100 : __vr_flow_schedule_transition(struct vrouter *, struct vr_flow_entry *,
101 : unsigned int, unsigned short);
102 : static bool vr_flow_is_fat_flow(struct vrouter *, struct vr_packet *,
103 : struct vr_flow_entry *);
104 :
105 : struct vr_flow_entry *vr_find_flow(struct vrouter *, struct vr_flow *,
106 : uint8_t, unsigned int *);
107 : unsigned int vr_trap_flow(struct vrouter *, struct vr_flow_entry *,
108 : struct vr_packet *, unsigned int, struct vr_flow_stats *,
109 : struct vr_packet_node *);
110 : extern struct vr_nexthop *vr_inet_ip_lookup(unsigned short, uint32_t);
111 : extern struct vr_nexthop *vr_inet6_ip_lookup(unsigned short, uint8_t *);
112 :
113 : bool
114 1 : vr_valid_link_local_port(struct vrouter *router, int family,
115 : int proto, int port)
116 : {
117 : unsigned char data;
118 : unsigned int tmp;
119 :
120 1 : if (!router->vr_link_local_ports)
121 0 : return false;
122 :
123 1 : if ((family != AF_INET) ||
124 1 : ((proto != VR_IP_PROTO_TCP) && (proto != VR_IP_PROTO_UDP) &&
125 : (proto != VR_IP_PROTO_ICMP)))
126 0 : return false;
127 :
128 1 : if ((port < VR_DYNAMIC_PORT_START) || (port > VR_DYNAMIC_PORT_END))
129 0 : return false;
130 :
131 1 : tmp = port - VR_DYNAMIC_PORT_START;
132 1 : if (proto == VR_IP_PROTO_UDP)
133 1 : tmp += (router->vr_link_local_ports_size * 8 / VR_LL_RP_MAX);
134 1 : if (proto == VR_IP_PROTO_ICMP)
135 0 : tmp += (router->vr_link_local_ports_size * 8 * VR_LL_RP_ICMP_INDEX /
136 : VR_LL_RP_MAX);
137 :
138 1 : data = router->vr_link_local_ports[(tmp / 8)];
139 1 : if (data & (1 << (tmp % 8)))
140 1 : return true;
141 :
142 0 : return false;
143 : }
144 :
145 : static void
146 3 : vr_clear_link_local_port(struct vrouter *router, int family,
147 : int proto, int port)
148 : {
149 : unsigned char *data;
150 : unsigned int tmp;
151 :
152 3 : if (!router->vr_link_local_ports)
153 0 : return;
154 :
155 3 : if ((family != AF_INET) ||
156 3 : ((proto != VR_IP_PROTO_TCP) && (proto != VR_IP_PROTO_UDP) &&
157 : (proto != VR_IP_PROTO_ICMP)))
158 0 : return;
159 :
160 3 : if ((port < VR_DYNAMIC_PORT_START) || (port > VR_DYNAMIC_PORT_END))
161 0 : return;
162 :
163 3 : tmp = port - VR_DYNAMIC_PORT_START;
164 3 : if (proto == VR_IP_PROTO_UDP)
165 2 : tmp += (router->vr_link_local_ports_size * 8 / VR_LL_RP_MAX);
166 3 : if (proto == VR_IP_PROTO_ICMP)
167 1 : tmp += ((router->vr_link_local_ports_size * 8 * VR_LL_RP_ICMP_INDEX)/
168 : VR_LL_RP_MAX);
169 :
170 3 : data = &router->vr_link_local_ports[(tmp / 8)];
171 3 : *data &= (~(1 << (tmp % 8)));
172 :
173 3 : return;
174 : }
175 :
176 : static void
177 2 : vr_set_link_local_port(struct vrouter *router, int family,
178 : int proto, int port)
179 : {
180 : unsigned char *data;
181 : unsigned int tmp;
182 :
183 2 : if (!router->vr_link_local_ports)
184 0 : return;
185 :
186 2 : if ((family != AF_INET) ||
187 2 : ((proto != VR_IP_PROTO_TCP) && (proto != VR_IP_PROTO_UDP) &&
188 : (proto != VR_IP_PROTO_ICMP)))
189 0 : return;
190 :
191 2 : if ((port < VR_DYNAMIC_PORT_START) || (port > VR_DYNAMIC_PORT_END))
192 0 : return;
193 :
194 2 : tmp = port - VR_DYNAMIC_PORT_START;
195 2 : if (proto == VR_IP_PROTO_UDP)
196 1 : tmp += (router->vr_link_local_ports_size * 8 / VR_LL_RP_MAX);
197 2 : if (proto == VR_IP_PROTO_ICMP)
198 1 : tmp += ((router->vr_link_local_ports_size * 8 * VR_LL_RP_ICMP_INDEX)/
199 : VR_LL_RP_MAX);
200 :
201 2 : data = &router->vr_link_local_ports[tmp / 8];
202 2 : *data |= (1 << (tmp % 8));
203 :
204 2 : return;
205 : }
206 :
207 : static void
208 215 : vr_flow_reset_mirror(struct vrouter *router, struct vr_flow_entry *fe,
209 : unsigned int index)
210 : {
211 215 : if (fe->fe_flags & VR_FLOW_FLAG_MIRROR) {
212 2 : fe->fe_mirror_id = VR_MAX_MIRROR_INDICES;
213 2 : fe->fe_sec_mirror_id = VR_MAX_MIRROR_INDICES;
214 2 : if (fe->fe_mme) {
215 0 : vr_mirror_meta_entry_del(router, fe->fe_mme);
216 0 : fe->fe_mme = NULL;
217 0 : vr_offload_flow_meta_data_set(index, 0, 0, 0);
218 : }
219 : }
220 215 : fe->fe_flags &= ~VR_FLOW_FLAG_MIRROR;
221 215 : fe->fe_mirror_id = VR_MAX_MIRROR_INDICES;
222 215 : fe->fe_sec_mirror_id = VR_MAX_MIRROR_INDICES;
223 :
224 215 : return;
225 : }
226 :
227 : static void
228 109 : vr_init_flow_entry(struct vr_flow_entry *fe)
229 : {
230 109 : fe->fe_rflow = -1;
231 109 : fe->fe_mirror_id = VR_MAX_MIRROR_INDICES;
232 109 : fe->fe_sec_mirror_id = VR_MAX_MIRROR_INDICES;
233 109 : fe->fe_ecmp_nh_index = -1;
234 :
235 109 : return;
236 : }
237 :
238 :
239 : static void
240 117 : __vr_flow_reset_entry(struct vrouter *router, struct vr_flow_entry *fe)
241 : {
242 117 : if (fe->fe_hold_list) {
243 0 : vr_printf("vrouter: Potential memory leak @ %s:%d\n",
244 : __FILE__, __LINE__);
245 : }
246 117 : fe->fe_hold_list = NULL;
247 117 : fe->fe_key.flow_key_len = 0;
248 :
249 117 : (void)vr_offload_flow_del(fe);
250 :
251 117 : vr_flow_reset_mirror(router, fe, fe->fe_hentry.hentry_index);
252 117 : fe->fe_ecmp_nh_index = -1;
253 117 : fe->fe_src_nh_index = NH_DISCARD_ID;
254 117 : fe->fe_rflow = -1;
255 117 : fe->fe_action = VR_FLOW_ACTION_DROP;
256 117 : fe->fe_udp_src_port = 0;
257 117 : fe->fe_tcp_flags = 0;
258 117 : fe->fe_flags &=
259 : (VR_FLOW_FLAG_ACTIVE | VR_FLOW_FLAG_EVICTED |
260 : VR_FLOW_FLAG_NEW_FLOW | VR_FLOW_FLAG_DELETE_MARKED);
261 117 : fe->fe_flags1 &=
262 : ~(VR_FLOW_FLAG1_HBS_LEFT | VR_FLOW_FLAG1_HBS_RIGHT);
263 117 : fe->fe_ttl = 0;
264 117 : fe->fe_underlay_ecmp_index = -1;
265 117 : fe->fe_src_info = 0;
266 :
267 117 : return;
268 : }
269 :
270 : static void
271 109 : vr_flow_reset_entry(struct vrouter *router, struct vr_flow_entry *fe)
272 : {
273 109 : __vr_flow_reset_entry(router, fe);
274 109 : memset(&fe->fe_stats, 0, sizeof(fe->fe_stats));
275 109 : fe->fe_type = VP_TYPE_NULL;
276 109 : fe->fe_flags = 0;
277 :
278 109 : vr_htable_release_hentry(router->vr_flow_table, &fe->fe_hentry);
279 109 : return;
280 : }
281 :
282 : static void
283 8 : vr_flow_reset_active_entry(struct vrouter *router, struct vr_flow_entry *fe)
284 : {
285 8 : __vr_flow_reset_entry(router, fe);
286 8 : vr_htable_release_hentry(router->vr_flow_table, &fe->fe_hentry);
287 8 : return;
288 : }
289 :
290 :
291 : static vr_hentry_key
292 62 : vr_flow_get_key(vr_htable_t flow_table, vr_hentry_t *entry,
293 : unsigned int *key_len)
294 : {
295 62 : struct vr_flow_entry *fe = CONTAINER_OF(fe_hentry,
296 : struct vr_flow_entry, entry);
297 :
298 62 : if ((fe->fe_flags & VR_FLOW_FLAG_DELETE_MARKED) ||
299 62 : !(fe->fe_flags & VR_FLOW_FLAG_ACTIVE))
300 0 : return NULL;
301 :
302 62 : if (key_len)
303 62 : *key_len = fe->fe_key.flow_key_len;
304 :
305 62 : return &fe->fe_key;
306 : }
307 :
308 : uint32_t
309 1 : vr_flow_get_rflow_src_info(struct vrouter *router,
310 : struct vr_flow_entry *fe)
311 : {
312 : struct vr_flow_entry *rfe;
313 :
314 1 : if ((!fe) || !(fe->fe_flags & VR_RFLOW_VALID))
315 0 : return (unsigned int)-1;
316 :
317 1 : rfe = vr_flow_get_entry(router, fe->fe_rflow);
318 1 : if (!rfe)
319 0 : return (unsigned int)-1;
320 :
321 1 : return rfe->fe_src_info;
322 : }
323 :
324 : static inline bool
325 109 : vr_flow_set_active(struct vr_flow_entry *fe)
326 : {
327 109 : return vr_sync_bool_compare_and_swap_16u(&fe->fe_flags,
328 : fe->fe_flags & ~VR_FLOW_FLAG_ACTIVE,
329 : VR_FLOW_FLAG_ACTIVE | VR_FLOW_FLAG_NEW_FLOW);
330 : }
331 :
332 : /*
333 : * This api is called to get flow table size
334 : * NOTE: This api is also called from agent (via sandesh)
335 : * very early during init even before flow table is
336 : * initialized to calculate huge page table size.
337 : * Hence this api should not use vr_htable structures
338 : */
339 : unsigned int
340 13 : vr_flow_table_size(struct vrouter *router)
341 : {
342 : // set the overlflow flow table entries
343 13 : vr_compute_size_oflow_table();
344 13 : return (VR_FLOW_TABLE_SIZE + VR_OFLOW_TABLE_SIZE);
345 : }
346 :
347 : unsigned int
348 13 : vr_flow_table_used_oflow_entries(struct vrouter *router)
349 : {
350 13 : return vr_htable_used_oflow_entries(router->vr_flow_table);
351 : }
352 :
353 : unsigned int
354 13 : vr_flow_table_used_total_entries(struct vrouter *router)
355 : {
356 13 : return vr_htable_used_total_entries(router->vr_flow_table);
357 : }
358 : /*
359 : * this is used by the mmap code. mmap sees the whole flow table
360 : * (including the overflow table) as one large table. so, given
361 : * an offset into that large memory, we should return the correct
362 : * virtual address
363 : */
364 : void *
365 0 : vr_flow_get_va(struct vrouter *router, uint64_t offset)
366 : {
367 0 : return vr_htable_get_address(router->vr_flow_table, offset);
368 : }
369 :
370 : struct vr_flow_entry *
371 1236 : vr_flow_get_entry(struct vrouter *router, int index)
372 : {
373 1236 : if (index < 0)
374 104 : return NULL;
375 :
376 1132 : return (struct vr_flow_entry *)
377 1132 : vr_htable_get_hentry_by_index(router->vr_flow_table, index);
378 : }
379 :
380 : static inline void
381 120 : vr_flow_stop_modify(struct vrouter *router, struct vr_flow_entry *fe)
382 : {
383 120 : if (!fe)
384 0 : return;
385 :
386 120 : (void)vr_sync_and_and_fetch_16u(&fe->fe_flags, ~VR_FLOW_FLAG_MODIFIED);
387 120 : return;
388 : }
389 :
390 : static inline bool
391 222 : vr_flow_start_modify(struct vrouter *router, struct vr_flow_entry *fe)
392 : {
393 : unsigned short flags;
394 :
395 222 : flags = fe->fe_flags;
396 222 : if (!(flags & (VR_FLOW_FLAG_MODIFIED | VR_FLOW_FLAG_EVICTED |
397 : VR_FLOW_FLAG_NEW_FLOW))) {
398 218 : if (vr_sync_bool_compare_and_swap_16u(&fe->fe_flags, flags,
399 : flags | VR_FLOW_FLAG_MODIFIED)) {
400 218 : return true;
401 : }
402 : }
403 :
404 4 : return false;
405 : }
406 :
407 :
408 : /* Non-static due to RCU callback pointer comparison in vRouter/DPDK */
409 : static void
410 2 : vr_flow_flush_hold_queue(struct vrouter *router, struct vr_flow_entry *fe,
411 : struct vr_flow_queue *vfq)
412 : {
413 : struct vr_forwarding_md fmd;
414 :
415 2 : if (vfq) {
416 2 : vr_init_forwarding_md(&fmd);
417 2 : vr_flow_set_forwarding_md(router, fe, vfq->vfq_index, &fmd);
418 2 : __vr_flow_flush_hold_queue(router, fe, &fmd, vfq);
419 : }
420 :
421 2 : return;
422 : }
423 :
424 : static void
425 8 : vr_flow_evict_flow(struct vrouter *router, struct vr_flow_entry *fe)
426 : {
427 : unsigned short flags;
428 :
429 8 : if (!fe)
430 0 : return;
431 :
432 8 : if ((fe->fe_flags & VR_FLOW_FLAG_ACTIVE) &&
433 8 : (fe->fe_flags & VR_FLOW_FLAG_EVICT_CANDIDATE)) {
434 8 : flags = fe->fe_flags | VR_FLOW_FLAG_ACTIVE |
435 : VR_FLOW_FLAG_EVICT_CANDIDATE;
436 8 : if (vr_sync_bool_compare_and_swap_16u(&fe->fe_flags, flags,
437 : (flags ^ VR_FLOW_FLAG_EVICT_CANDIDATE) |
438 : VR_FLOW_FLAG_EVICTED)) {
439 8 : vr_flow_stop_modify(router, fe);
440 8 : vr_flow_reset_active_entry(router, fe);
441 : }
442 : }
443 :
444 8 : return;
445 : }
446 :
447 : void
448 6 : vr_flow_defer_cb(struct vrouter *router, void *arg)
449 : {
450 : struct vr_defer_data *defer;
451 : struct vr_flow_entry *fe, *rfe;
452 : struct vr_flow_queue *vfq;
453 : struct vr_flow_defer_data *vfdd;
454 :
455 6 : defer = (struct vr_defer_data *)arg;
456 6 : if (!defer)
457 0 : return;
458 :
459 6 : vfdd = (struct vr_flow_defer_data *)defer->vdd_data;
460 6 : if (!vfdd)
461 0 : return;
462 6 : fe = vfdd->vfdd_fe;
463 :
464 6 : vfq = (struct vr_flow_queue *)vfdd->vfdd_flow_queue;
465 6 : if (vfq) {
466 2 : vr_flow_flush_hold_queue(router, fe, vfq);
467 2 : vr_free(vfq, VR_FLOW_QUEUE_OBJECT);
468 2 : vfdd->vfdd_flow_queue = NULL;
469 : }
470 :
471 6 : if (vfdd->vfdd_delete) {
472 0 : vr_flow_reset_entry(router, fe);
473 6 : } else if (vfdd->vfdd_evict_flow) {
474 4 : rfe = vr_flow_get_entry(router, fe->fe_rflow);
475 4 : vr_flow_evict_flow(router, fe);
476 4 : if (rfe)
477 4 : vr_flow_evict_flow(router, rfe);
478 : }
479 :
480 6 : vr_free(vfdd, VR_FLOW_DEFER_DATA_OBJECT);
481 :
482 6 : return;
483 : }
484 :
485 : static void
486 0 : vr_flow_reset_evict(struct vrouter *router, struct vr_flow_entry *fe)
487 : {
488 : unsigned short flags;
489 :
490 0 : if (!fe)
491 0 : return;
492 :
493 0 : flags = fe->fe_flags;
494 0 : if (flags & VR_FLOW_FLAG_EVICT_CANDIDATE) {
495 0 : (void)vr_sync_bool_compare_and_swap_16u(&fe->fe_flags, flags,
496 : (flags ^ VR_FLOW_FLAG_EVICT_CANDIDATE));
497 : }
498 :
499 0 : vr_flow_stop_modify(router, fe);
500 :
501 0 : return;
502 : }
503 :
504 : static void
505 318 : vr_flow_defer(struct vr_flow_md *flmd, struct vr_flow_entry *fe)
506 : {
507 : struct vr_flow_entry *rfe;
508 318 : struct vr_defer_data *vdd = flmd->flmd_defer_data;
509 : struct vr_flow_defer_data *vfdd;
510 :
511 318 : if (!vdd || !vdd->vdd_data) {
512 312 : if (flmd->flmd_flags & VR_FLOW_FLAG_EVICT_CANDIDATE) {
513 0 : if (fe->fe_rflow) {
514 0 : rfe = vr_flow_get_entry(flmd->flmd_router, fe->fe_rflow);
515 0 : vr_flow_reset_evict(flmd->flmd_router, rfe);
516 : }
517 0 : vr_flow_reset_evict(flmd->flmd_router, fe);
518 : }
519 :
520 312 : if (!(flmd->flmd_flags & VR_FLOW_FLAG_ACTIVE)) {
521 98 : vr_flow_reset_entry(flmd->flmd_router, fe);
522 : }
523 :
524 312 : return;
525 : }
526 :
527 6 : vfdd = (struct vr_flow_defer_data *)vdd->vdd_data;
528 6 : vfdd->vfdd_fe = fe;
529 :
530 6 : vr_defer(flmd->flmd_router, vr_flow_defer_cb, (void *)vdd);
531 6 : flmd->flmd_defer_data = NULL;
532 :
533 6 : return;
534 : }
535 :
536 : static struct vr_flow_entry *
537 109 : vr_flow_table_get_free_entry(struct vrouter *router, struct vr_flow *key,
538 : unsigned int *free_index)
539 : {
540 : unsigned short flags;
541 : struct vr_flow_entry *fe;
542 :
543 : fe = (struct vr_flow_entry *)
544 109 : vr_htable_find_free_hentry(router->vr_flow_table, key,
545 109 : key->flow_key_len);
546 109 : if (fe) {
547 109 : flags = fe->fe_flags;
548 109 : if (!(flags & VR_FLOW_FLAG_ACTIVE)) {
549 109 : if (vr_flow_set_active(fe)) {
550 109 : vr_init_flow_entry(fe);
551 : }
552 0 : } else if (flags & VR_FLOW_FLAG_EVICTED) {
553 0 : fe->fe_flags = ((flags & ~VR_FLOW_FLAG_EVICTED) |
554 : VR_FLOW_FLAG_NEW_FLOW);
555 : }
556 :
557 109 : fe->fe_underlay_ecmp_index = -1;
558 :
559 109 : fe->fe_gen_id = (fe->fe_gen_id + 1) %
560 : (1 << (8 * sizeof(fe->fe_gen_id)));
561 109 : *free_index = fe->fe_hentry.hentry_index;
562 : }
563 :
564 109 : return fe;
565 : }
566 :
567 :
568 : static struct vr_flow_entry *
569 109 : vr_flow_get_free_entry(struct vrouter *router, struct vr_flow *key, uint8_t type,
570 : bool need_hold, unsigned int *fe_index)
571 : {
572 109 : struct vr_flow_entry *fe = NULL;
573 :
574 109 : fe = vr_flow_table_get_free_entry(router, key, fe_index);
575 109 : if (fe) {
576 109 : if (need_hold) {
577 5 : fe->fe_hold_list = vr_zalloc(sizeof(struct vr_flow_queue),
578 : VR_FLOW_QUEUE_OBJECT);
579 5 : if (!fe->fe_hold_list) {
580 0 : vr_flow_reset_entry(router, fe);
581 0 : fe = NULL;
582 0 : vr_printf("%s:%d flow reset\n", __func__, __LINE__);
583 0 : return fe;
584 : } else {
585 5 : fe->fe_hold_list->vfq_index = *fe_index;
586 : }
587 : }
588 :
589 109 : fe->fe_type = type;
590 109 : memcpy(&fe->fe_key, key, key->flow_key_len);
591 109 : fe->fe_key.flow_key_len = key->flow_key_len;
592 : }
593 :
594 109 : return fe;
595 : }
596 :
597 :
598 : struct vr_flow_entry *
599 176 : vr_find_flow(struct vrouter *router, struct vr_flow *key,
600 : uint8_t type, unsigned int *fe_index)
601 : {
602 : struct vr_flow_entry *fe;
603 :
604 176 : fe = (struct vr_flow_entry *)vr_htable_find_hentry(router->vr_flow_table,
605 176 : key, key->flow_key_len);
606 176 : if (fe) {
607 62 : if (fe_index)
608 62 : *fe_index = fe->fe_hentry.hentry_index;
609 : }
610 :
611 176 : return fe;
612 : }
613 :
614 :
615 : void
616 28 : vr_flow_fill_pnode(struct vr_packet_node *pnode, struct vr_packet *pkt,
617 : struct vr_forwarding_md *fmd)
618 : {
619 28 : struct vr_ip *ip = (struct vr_ip *)pkt_inner_network_header(pkt);
620 :
621 : /*
622 : * we cannot cache nexthop here. to cache, we need to hold reference
623 : * to the nexthop. to hold a reference, we will have to hold a lock,
624 : * which we cannot. the only known case of misbehavior if we do not
625 : * cache is ECMP. when the packet comes from the fabric, the nexthop
626 : * actually points to a local composite, whereas a route lookup actually
627 : * returns a different nexthop, in which case the ecmp index will return
628 : * a bad nexthop. to avoid that, we will cache the label, and reuse it
629 : */
630 28 : pkt->vp_nh = NULL;
631 :
632 28 : pnode->pl_flags = 0;
633 28 : pnode->pl_vif_idx = pkt->vp_if->vif_idx;
634 :
635 28 : if (fmd) {
636 28 : pnode->pl_outer_src_ip = fmd->fmd_outer_src_ip;
637 28 : pnode->pl_label = fmd->fmd_label;
638 28 : if (vr_fmd_label_is_vxlan_id(fmd))
639 0 : pnode->pl_flags |= PN_FLAG_LABEL_IS_VXLAN_ID;
640 28 : if (fmd->fmd_to_me)
641 0 : pnode->pl_flags |= PN_FLAG_TO_ME;
642 : }
643 :
644 28 : if (ip) {
645 28 : if (vr_ip_is_ip4(ip)) {
646 : /*
647 : * Source IP & Dest IP can change while the packet is in the queue
648 : * (NAT). For e.g.: when the cloned head of a fragment is enqueued
649 : * to the assembler and subsequently dequeued by the assembler, the
650 : * original packet might have undergone a NAT, resulting in wrong
651 : * hash and thus a wrong search for other fragments of the packet.
652 : * Hence, store them here for others interested in the original IPs
653 : */
654 28 : pnode->pl_inner_src_ip = ip->ip_saddr;
655 28 : pnode->pl_inner_dst_ip = ip->ip_daddr;
656 28 : if (vr_ip_fragment_head(ip))
657 5 : pnode->pl_flags |= PN_FLAG_FRAGMENT_HEAD;
658 0 : } else if (vr_ip_is_ip6(ip)) {
659 0 : if (vr_ip6_fragment_head((struct vr_ip6 *)ip))
660 0 : pnode->pl_flags |= PN_FLAG_FRAGMENT_HEAD;
661 : }
662 : }
663 :
664 28 : pnode->pl_dscp = fmd->fmd_dscp;
665 28 : pnode->pl_dotonep = fmd->fmd_dotonep;
666 28 : pnode->pl_vrf = fmd->fmd_dvrf;
667 28 : pnode->pl_vlan = fmd->fmd_vlan;
668 28 : pnode->pl_mirror_vlan = fmd->fmd_mirror_data;
669 :
670 28 : vr_sync_synchronize();
671 28 : pnode->pl_packet = pkt;
672 :
673 28 : return;
674 : }
675 :
676 : static int
677 7 : vr_enqueue_flow(struct vrouter *router, struct vr_flow_entry *fe,
678 : struct vr_packet *pkt, unsigned int index,
679 : struct vr_flow_stats *stats, struct vr_forwarding_md *fmd)
680 : {
681 7 : int ret = 0;
682 : unsigned int i;
683 7 : unsigned short drop_reason = 0;
684 7 : struct vr_flow_queue *vfq = fe->fe_hold_list;
685 : struct vr_packet_node *pnode;
686 :
687 7 : if (!vfq) {
688 0 : drop_reason = VP_DROP_FLOW_UNUSABLE;
689 0 : PKT_LOG(drop_reason, pkt, 0, VR_FLOW_C, __LINE__);
690 0 : goto drop;
691 : }
692 :
693 7 : i = vr_sync_fetch_and_add_32u(&vfq->vfq_entries, 1);
694 7 : if (i >= VR_MAX_FLOW_QUEUE_ENTRIES) {
695 0 : drop_reason = VP_DROP_FLOW_QUEUE_LIMIT_EXCEEDED;
696 0 : PKT_LOG(drop_reason, pkt, 0, VR_FLOW_C, __LINE__);
697 0 : goto drop;
698 : }
699 :
700 7 : pnode = &vfq->vfq_pnodes[i];
701 7 : vr_flow_fill_pnode(pnode, pkt, fmd);
702 7 : if (!i)
703 5 : ret = vr_trap_flow(router, fe, pkt, index, stats, pnode);
704 :
705 7 : return ret;
706 0 : drop:
707 0 : vr_pfree(pkt, drop_reason);
708 0 : return 0;
709 : }
710 :
711 : static flow_result_t
712 6 : vr_flow_nat(struct vr_flow_entry *fe,
713 : struct vr_packet *pkt, struct vr_forwarding_md *fmd)
714 : {
715 6 : if (pkt->vp_type == VP_TYPE_IP)
716 4 : return vr_inet_flow_nat(fe, pkt, fmd);
717 :
718 2 : if (pkt->vp_type == VP_TYPE_IP6)
719 2 : return vr_inet6_flow_nat(fe, pkt, fmd);
720 :
721 0 : PKT_LOG(VP_DROP_FLOW_ACTION_INVALID, pkt, 0, VR_FLOW_C, __LINE__);
722 0 : vr_pfree(pkt, VP_DROP_FLOW_ACTION_INVALID);
723 0 : return FLOW_CONSUMED;
724 : }
725 :
726 : static void
727 395 : vr_flow_set_forwarding_md(struct vrouter *router, struct vr_flow_entry *fe,
728 : unsigned int index, struct vr_forwarding_md *md)
729 : {
730 : struct vr_flow_entry *rfe;
731 :
732 395 : md->fmd_flow_index = index;
733 395 : md->fmd_ecmp_nh_index = fe->fe_ecmp_nh_index;
734 395 : md->fmd_underlay_ecmp_index = fe->fe_underlay_ecmp_index;
735 395 : md->fmd_udp_src_port = fe->fe_udp_src_port;
736 395 : if (fe->fe_flags & VR_RFLOW_VALID) {
737 320 : rfe = vr_flow_get_entry(router, fe->fe_rflow);
738 320 : if (rfe)
739 275 : md->fmd_ecmp_src_nh_index = rfe->fe_ecmp_nh_index;
740 : }
741 :
742 395 : return;
743 : }
744 :
745 : static bool
746 8 : __vr_flow_mark_evict(struct vrouter *router, struct vr_flow_entry *fe)
747 : {
748 : unsigned short flags;
749 :
750 8 : flags = fe->fe_flags;
751 8 : if (flags & VR_FLOW_FLAG_ACTIVE) {
752 8 : flags = vr_sync_fetch_and_or_16u(&fe->fe_flags,
753 : VR_FLOW_FLAG_EVICT_CANDIDATE);
754 8 : if (!(flags & VR_FLOW_FLAG_EVICT_CANDIDATE)) {
755 8 : return true;
756 : }
757 : }
758 :
759 0 : return false;
760 : }
761 :
762 : static void
763 8 : vr_flow_mark_evict(struct vrouter *router, struct vr_flow_entry *fe,
764 : unsigned int index)
765 : {
766 8 : bool evict_forward_flow = true;
767 :
768 8 : struct vr_flow_entry *rfe = NULL;
769 :
770 : /* start modifying the entry */
771 8 : if (!vr_flow_start_modify(router, fe)) {
772 4 : return;
773 : }
774 :
775 4 : if (fe->fe_rflow >= 0) {
776 4 : rfe = vr_flow_get_entry(router, fe->fe_rflow);
777 4 : if (rfe) {
778 4 : evict_forward_flow = false;
779 4 : if (rfe->fe_tcp_flags & VR_FLOW_TCP_DEAD) {
780 4 : if (!vr_flow_start_modify(router, rfe)) {
781 : /* no modification. hence...*/
782 0 : rfe = NULL;
783 : } else {
784 : /* we do not want hold flows to be evicted, just yet */
785 4 : if (((rfe->fe_rflow == index) || (rfe->fe_rflow < 0)) &&
786 4 : (rfe->fe_action != VR_FLOW_ACTION_HOLD)) {
787 4 : evict_forward_flow = __vr_flow_mark_evict(router, rfe);
788 : }
789 : }
790 : } else {
791 : /* no modification. hence...*/
792 0 : rfe = NULL;
793 : }
794 : }
795 : }
796 :
797 : /*
798 : * presence of rfe means that we might need to reset the evict bit
799 : * or at the minimum reset the modified bit under failure conditions
800 : *
801 : * CEM-18166: In case rfe has already been deleted, under special
802 : * circumstances, fe might still have fe_rflow pointing to it.
803 : * In this case (and other cases where rfe is not active anymore),
804 : * evict_forward_flow will remain true, which makes sense because we still
805 : * want to evict this flow, because of TCP FIN or RST.
806 : */
807 4 : if (evict_forward_flow) {
808 4 : if (__vr_flow_mark_evict(router, fe)) {
809 4 : if (!__vr_flow_schedule_transition(router, fe,
810 4 : index, fe->fe_flags)) {
811 4 : return;
812 : } else {
813 0 : goto reset_evict;
814 : }
815 : }
816 : }
817 :
818 : /* stop modifying the forward and the reverse */
819 0 : if (rfe)
820 0 : vr_flow_stop_modify(router, rfe);
821 0 : vr_flow_stop_modify(router, fe);
822 :
823 0 : return;
824 :
825 0 : reset_evict:
826 0 : if (rfe)
827 0 : vr_flow_reset_evict(router, rfe);
828 0 : vr_flow_reset_evict(router, fe);
829 :
830 0 : return;
831 : }
832 :
833 : int16_t
834 21 : vr_flow_get_qos(struct vrouter *router, struct vr_packet *pkt,
835 : struct vr_forwarding_md *fmd)
836 : {
837 : struct vr_flow_entry *fe;
838 :
839 21 : if (fmd->fmd_flow_index >= 0) {
840 21 : fe = vr_flow_get_entry(router, fmd->fmd_flow_index);
841 21 : if (fe)
842 21 : return fe->fe_qos_id;
843 : }
844 :
845 0 : return -1;
846 : }
847 :
848 : static int
849 0 : vr_rflow_update_ecmp_index(struct vrouter *router, struct vr_flow_entry *fe,
850 : unsigned int new_ecmp_index, struct vr_forwarding_md *fmd)
851 : {
852 : struct vr_flow_entry *rfe;
853 :
854 0 : if (new_ecmp_index == -1)
855 0 : return -1;
856 :
857 0 : rfe = vr_flow_get_entry(router, fe->fe_rflow);
858 0 : if ((!rfe) || (rfe->fe_flags & VR_FLOW_FLAG_DELETE_MARKED))
859 0 : return -1;
860 :
861 0 : rfe->fe_ecmp_nh_index = new_ecmp_index;
862 :
863 : /* Update hardware reverse flow. */
864 0 : (void)vr_offload_flow_set(rfe, fe->fe_rflow, NULL);
865 :
866 0 : fmd->fmd_ecmp_src_nh_index = new_ecmp_index;
867 :
868 0 : return 0;
869 : }
870 :
871 :
872 : int
873 1 : vr_flow_update_ecmp_index(struct vrouter *router, struct vr_flow_entry *fe,
874 : unsigned int new_ecmp_index, struct vr_forwarding_md *fmd)
875 : {
876 :
877 1 : if (new_ecmp_index == -1)
878 0 : return -1;
879 :
880 1 : if ((!fe) || (fe->fe_flags & VR_FLOW_FLAG_DELETE_MARKED))
881 0 : return -1;
882 :
883 : /* If RPF verification is manipulating this flow, let it succeed */
884 1 : (void)vr_sync_bool_compare_and_swap_8s(&fe->fe_ecmp_nh_index,
885 : fmd->fmd_ecmp_nh_index, new_ecmp_index);
886 :
887 1 : fmd->fmd_ecmp_nh_index = fe->fe_ecmp_nh_index;
888 :
889 1 : return 0;
890 : }
891 :
892 : static flow_result_t
893 61 : vr_flow_action_default(struct vrouter *router, struct vr_flow_entry *fe,
894 : unsigned int index, struct vr_packet *pkt,
895 : struct vr_forwarding_md *fmd)
896 : {
897 61 : unsigned int ip_inc_diff_cksum = 0;
898 : struct vr_ip *ip;
899 61 : flow_result_t result = FLOW_CONSUMED;
900 :
901 : struct vr_forwarding_md mirror_fmd;
902 :
903 61 : fmd->fmd_dvrf = fe->fe_vrf;
904 : /*
905 : * for now, we will not use dvrf if VRFT is set, because the RPF
906 : * check needs to happen in the source vrf
907 : */
908 :
909 61 : if (fe->fe_flags & VR_FLOW_FLAG_VRFT) {
910 5 : if (fmd->fmd_dvrf != fe->fe_dvrf) {
911 1 : fmd->fmd_dvrf = fe->fe_dvrf;
912 1 : fmd->fmd_to_me = 1;
913 : }
914 : }
915 :
916 61 : if (fe->fe_flags & VR_FLOW_FLAG_MIRROR) {
917 2 : if (fe->fe_mirror_id < VR_MAX_MIRROR_INDICES) {
918 2 : mirror_fmd = *fmd;
919 2 : mirror_fmd.fmd_ecmp_nh_index = -1;
920 2 : mirror_fmd.fmd_underlay_ecmp_index = -1;
921 2 : vr_mirror(router, fe->fe_mirror_id, pkt, &mirror_fmd,
922 : MIRROR_TYPE_ACL);
923 2 : fmd->fmd_mirror_data = mirror_fmd.fmd_mirror_data;
924 : }
925 :
926 2 : if (fe->fe_sec_mirror_id < VR_MAX_MIRROR_INDICES) {
927 0 : mirror_fmd = *fmd;
928 0 : mirror_fmd.fmd_ecmp_nh_index = -1;
929 0 : mirror_fmd.fmd_underlay_ecmp_index = -1;
930 0 : vr_mirror(router, fe->fe_sec_mirror_id, pkt, &mirror_fmd,
931 : MIRROR_TYPE_ACL);
932 0 : fmd->fmd_mirror_data = mirror_fmd.fmd_mirror_data;
933 : }
934 : }
935 :
936 61 : switch (fe->fe_action) {
937 0 : case VR_FLOW_ACTION_DROP:
938 0 : PKT_LOG(VP_DROP_FLOW_ACTION_DROP, pkt, 0, VR_FLOW_C, __LINE__);
939 0 : vr_pfree(pkt, VP_DROP_FLOW_ACTION_DROP);
940 0 : result = FLOW_CONSUMED;
941 0 : break;
942 :
943 55 : case VR_FLOW_ACTION_FORWARD:
944 55 : result = FLOW_FORWARD;
945 55 : break;
946 :
947 6 : case VR_FLOW_ACTION_NAT:
948 6 : result = vr_flow_nat(fe, pkt, fmd);
949 6 : break;
950 :
951 0 : default:
952 0 : PKT_LOG(VP_DROP_FLOW_ACTION_INVALID, pkt, 0, VR_FLOW_C, __LINE__);
953 0 : vr_pfree(pkt, VP_DROP_FLOW_ACTION_INVALID);
954 0 : result = FLOW_CONSUMED;
955 0 : break;
956 : }
957 :
958 61 : if (result == FLOW_FORWARD) {
959 61 : if (pkt->vp_type == VP_TYPE_IP) {
960 57 : ip = (struct vr_ip *)pkt_network_header(pkt);
961 57 : if (ip) {
962 57 : if (fe->fe_ttl && (fe->fe_ttl != ip->ip_ttl)) {
963 0 : vr_incremental_diff(ip->ip_ttl, fe->fe_ttl, &ip_inc_diff_cksum);
964 0 : ip->ip_ttl = fe->fe_ttl;
965 :
966 0 : if (ip_inc_diff_cksum)
967 0 : vr_ip_incremental_csum(ip, ip_inc_diff_cksum);
968 : }
969 : }
970 : }
971 : }
972 :
973 61 : if (fe->fe_tcp_flags & VR_FLOW_TCP_DEAD)
974 4 : vr_flow_mark_evict(router, fe, index);
975 :
976 61 : return result;
977 : }
978 :
979 : static flow_result_t
980 62 : vr_flow_action_hbs(struct vrouter *router, struct vr_flow_entry *fe,
981 : unsigned int index, struct vr_packet *pkt,
982 : struct vr_forwarding_md *fmd)
983 : {
984 62 : struct vr_vrf_table_entry *vrf_entry = NULL;
985 : uint16_t dvrf;
986 :
987 62 : dvrf = (fmd->fmd_dvrf == -1)? fe->fe_vrf: fmd->fmd_dvrf;
988 :
989 : /* If not HBS flow, return */
990 62 : if (!(fe->fe_flags1 & VR_FLOW_FLAG1_HBS_MASK))
991 53 : return FLOW_FORWARD;
992 :
993 9 : vrf_entry = vrouter_get_vrf_table(router, dvrf);
994 :
995 : /* Packet entering vrouter and going to hbs-l or hbs-r
996 : * for hbs-flows
997 : */
998 9 : if (fe->fe_flags1 & VR_FLOW_FLAG1_HBS_LEFT) {
999 : struct vr_interface *hbs_l;
1000 :
1001 : /* Flow is marked as HBS, but there is no HBS instance,
1002 : * so drop the packet
1003 : */
1004 4 : if (!vrf_entry) {
1005 0 : PKT_LOG(VP_DROP_INVALID_HBS_PKT, pkt, 0, VR_FLOW_C, __LINE__);
1006 0 : goto drop_pkt;
1007 : }
1008 :
1009 4 : hbs_l = vrf_entry->hbs_l_vif;
1010 4 : if (hbs_l) {
1011 : struct vr_eth_hbs_md *eth_hbs;
1012 :
1013 4 : if (vif_is_virtual(pkt->vp_if)) {
1014 : /* Packet entering vrouter from vmi and going to hbs-l,
1015 : * encode flow_id in src mac
1016 : */
1017 4 : eth_hbs = (struct vr_eth_hbs_md*)pkt_data(pkt);
1018 4 : eth_hbs->flow_id_smac = htonl(index);
1019 4 : eth_hbs->magic_smac = htons(VR_HBS_SMAC_MAGIC | VR_HBS_FROM_VMI);
1020 0 : } else if (vif_is_fabric(pkt->vp_if)) {
1021 : /* Packet entering vrouter from fabric and going to hbs-l
1022 : * (service chaining case) encode flow_id in dst mac
1023 : */
1024 :
1025 : /* Add ethernet header if there is none */
1026 0 : if (pkt_data(pkt) == pkt_network_header(pkt)) {
1027 0 : struct vr_eth *eth = (struct vr_eth*)pkt_data(pkt);
1028 0 : eth = (struct vr_eth *)pkt_push(pkt, VR_ETHER_HLEN);
1029 0 : memcpy(eth->eth_dmac, hbs_l->vif_mac, VR_ETHER_ALEN);
1030 0 : memcpy(eth->eth_smac, hbs_l->vif_mac, VR_ETHER_ALEN);
1031 0 : if (pkt->vp_type == VP_TYPE_IP) {
1032 0 : eth->eth_proto = htons(VR_ETH_PROTO_IP);
1033 0 : } else if (pkt->vp_type == VP_TYPE_IP6) {
1034 0 : eth->eth_proto = htons(VR_ETH_PROTO_IP6);
1035 : }
1036 : }
1037 :
1038 0 : eth_hbs = (struct vr_eth_hbs_md*)pkt_data(pkt);
1039 0 : eth_hbs->flow_id_dmac = htonl(index);
1040 0 : eth_hbs->magic_dmac = htons(VR_HBS_DMAC_MAGIC | VR_HBS_FROM_FABRIC);
1041 : }
1042 4 : pkt->vp_if = hbs_l;
1043 : /*
1044 : * If for some reason, we have GRO flag set and we have not invoked
1045 : * the GRO, we need to unset
1046 : */
1047 4 : vr_pkt_unset_gro(pkt);
1048 4 : pkt->vp_nh = NULL;
1049 4 : hbs_l->vif_tx(hbs_l, pkt, fmd);
1050 4 : return FLOW_HELD;
1051 : }
1052 0 : return FLOW_FORWARD;
1053 5 : } else if (fe->fe_flags1 & VR_FLOW_FLAG1_HBS_RIGHT) {
1054 : struct vr_interface *hbs_r;
1055 :
1056 : /* Flow is marked as HBS, but there is no HBS instance,
1057 : * so drop the packet
1058 : */
1059 5 : if (!vrf_entry) {
1060 0 : PKT_LOG(VP_DROP_INVALID_HBS_PKT, pkt, 0, VR_FLOW_C, __LINE__);
1061 0 : goto drop_pkt;
1062 : }
1063 :
1064 5 : hbs_r = vrf_entry->hbs_r_vif;
1065 5 : if (hbs_r) {
1066 : /* Packet entering vrouter from fabric and going to hbs-r,
1067 : * encode flow_id in dst mac
1068 : *
1069 : * Note: packet can also enter vrouter and goto hbs-r
1070 : * from vmi (instead of fabric) in case of intra-compute
1071 : */
1072 : struct vr_eth *eth;
1073 : struct vr_eth_hbs_md *eth_hbs;
1074 5 : bool pkt_is_l3 = false;
1075 :
1076 : /* Add ethernet header if there is none */
1077 5 : if (pkt_data(pkt) == pkt_network_header(pkt)) {
1078 1 : eth = (struct vr_eth *)pkt_push(pkt, VR_ETHER_HLEN);
1079 1 : memcpy(eth->eth_dmac, hbs_r->vif_mac, VR_ETHER_ALEN);
1080 1 : memcpy(eth->eth_smac, hbs_r->vif_mac, VR_ETHER_ALEN);
1081 1 : if (pkt->vp_type == VP_TYPE_IP) {
1082 1 : eth->eth_proto = htons(VR_ETH_PROTO_IP);
1083 0 : } else if (pkt->vp_type == VP_TYPE_IP6) {
1084 0 : eth->eth_proto = htons(VR_ETH_PROTO_IP6);
1085 : }
1086 1 : pkt_is_l3 = true;
1087 : }
1088 :
1089 5 : eth_hbs = (struct vr_eth_hbs_md*)pkt_data(pkt);
1090 :
1091 5 : eth = (struct vr_eth*)pkt_data(pkt);
1092 5 : if (!pkt_is_l3 && !memcmp(eth->eth_dmac, hbs_r->vif_mac, VR_ETHER_ALEN))
1093 1 : pkt_is_l3 = true;
1094 :
1095 5 : eth_hbs->flow_id_dmac = htonl(index);
1096 :
1097 : /* Encode packet source in the header */
1098 5 : if (vif_is_virtual(pkt->vp_if))
1099 2 : eth_hbs->magic_dmac = htons(VR_HBS_DMAC_MAGIC | VR_HBS_FROM_VMI);
1100 3 : else if (vif_is_fabric(pkt->vp_if))
1101 3 : eth_hbs->magic_dmac = htons(VR_HBS_DMAC_MAGIC | VR_HBS_FROM_FABRIC);
1102 : /* If it's L3 packet, encode it */
1103 5 : if (pkt_is_l3)
1104 2 : eth_hbs->magic_dmac |= htons(VR_HBS_L3_PKT);
1105 :
1106 5 : pkt->vp_if = hbs_r;
1107 : /*
1108 : * If for some reason, we have GRO flag set and we have not invoked
1109 : * the GRO, we need to unset
1110 : */
1111 5 : vr_pkt_unset_gro(pkt);
1112 5 : pkt->vp_nh = NULL;
1113 5 : hbs_r->vif_tx(hbs_r, pkt, fmd);
1114 5 : return FLOW_HELD;
1115 : }
1116 0 : return FLOW_FORWARD;
1117 : }
1118 0 : return FLOW_FORWARD;
1119 :
1120 0 : drop_pkt:
1121 0 : vr_pfree(pkt, VP_DROP_INVALID_HBS_PKT);
1122 0 : return FLOW_HELD;
1123 : }
1124 :
1125 : static flow_result_t
1126 62 : vr_flow_action(struct vrouter *router, struct vr_flow_entry *fe,
1127 : unsigned int index, struct vr_packet *pkt,
1128 : struct vr_forwarding_md *fmd)
1129 : {
1130 62 : flow_result_t ret = FLOW_CONSUMED;
1131 : struct vr_nexthop *src_nh;
1132 62 : int valid_src, modified_index = -1;
1133 :
1134 62 : src_nh = __vrouter_get_nexthop(router, fe->fe_src_nh_index);
1135 62 : if (!src_nh) {
1136 0 : PKT_LOG(VP_DROP_INVALID_NH, pkt, 0, VR_FLOW_C, __LINE__);
1137 0 : vr_pfree(pkt, VP_DROP_INVALID_NH);
1138 0 : goto res;
1139 : }
1140 :
1141 62 : if (src_nh->nh_validate_src) {
1142 60 : valid_src = src_nh->nh_validate_src(pkt, src_nh, fmd, &modified_index);
1143 60 : if (valid_src == NH_SOURCE_INVALID) {
1144 0 : PKT_LOG(VP_DROP_INVALID_SOURCE, pkt, 0, VR_FLOW_C, __LINE__);
1145 0 : vr_pfree(pkt, VP_DROP_INVALID_SOURCE);
1146 0 : goto res;
1147 : }
1148 :
1149 60 : if (valid_src == NH_SOURCE_MISMATCH) {
1150 0 : valid_src = vr_rflow_update_ecmp_index(router, fe,
1151 : modified_index, fmd);
1152 0 : if (valid_src == -1) {
1153 0 : PKT_LOG(VP_DROP_INVALID_SOURCE, pkt, 0, VR_FLOW_C, __LINE__);
1154 0 : vr_pfree(pkt, VP_DROP_INVALID_SOURCE);
1155 0 : goto res;
1156 : }
1157 : }
1158 : }
1159 :
1160 :
1161 124 : if ((fe->fe_action == VR_FLOW_ACTION_DROP) ||
1162 62 : ((ret = vr_flow_action_hbs(router, fe, index, pkt, fmd)) != FLOW_HELD))
1163 53 : ret = vr_flow_action_default(router, fe, index, pkt, fmd);
1164 :
1165 9 : res:
1166 62 : if (fe->fe_tcp_flags & VR_FLOW_TCP_DEAD)
1167 4 : vr_flow_mark_evict(router, fe, index);
1168 :
1169 62 : return ret;
1170 : }
1171 :
1172 : unsigned int
1173 5 : vr_trap_flow(struct vrouter *router, struct vr_flow_entry *fe,
1174 : struct vr_packet *pkt, unsigned int index,
1175 : struct vr_flow_stats *stats, struct vr_packet_node *pnode)
1176 : {
1177 : unsigned int trap_reason;
1178 :
1179 : struct vr_packet *npkt;
1180 : struct vr_flow_trap_arg ta;
1181 :
1182 5 : npkt = vr_pclone(pkt);
1183 5 : if (!npkt) {
1184 : /* Lets manipulate the stats */
1185 0 : pkt_drop_stats(pkt->vp_if, VP_DROP_TRAP_ORIGINAL, pkt->vp_cpu);
1186 0 : if (pnode)
1187 0 : pnode->pl_packet = NULL;
1188 0 : npkt = pkt;
1189 : }
1190 :
1191 5 : vr_preset(npkt);
1192 :
1193 5 : switch (fe->fe_flags & VR_FLOW_FLAG_TRAP_MASK) {
1194 : default:
1195 : /*
1196 : * agent needs a method to identify new flows from existing flows.
1197 : * existing flows can be reused (evicted) or the action of such flows
1198 : * can become hold. If existing flows are reused and packet is trapped,
1199 : * agent will not re-evaluate the flow. Hence, agent has to be told
1200 : * that this is a new flow, which we indicate by the trap reason.
1201 : */
1202 5 : if (fe->fe_flags & VR_FLOW_FLAG_NEW_FLOW) {
1203 5 : trap_reason = AGENT_TRAP_FLOW_MISS;
1204 5 : fe->fe_flags ^= VR_FLOW_FLAG_NEW_FLOW;
1205 : } else {
1206 0 : trap_reason = AGENT_TRAP_FLOW_ACTION_HOLD;
1207 : }
1208 :
1209 5 : ta.vfta_index = index;
1210 5 : if ((fe->fe_type == VP_TYPE_IP) || (fe->fe_type == VP_TYPE_IP6))
1211 5 : ta.vfta_nh_index = fe->fe_key.flow_nh_id;
1212 5 : if (stats) {
1213 5 : ta.vfta_stats = *stats;
1214 : } else {
1215 0 : ta.vfta_stats = fe->fe_stats;
1216 : }
1217 :
1218 5 : ta.vfta_gen_id = fe->fe_gen_id;
1219 :
1220 5 : break;
1221 : }
1222 :
1223 5 : return vr_trap(npkt, fe->fe_vrf, trap_reason, &ta);
1224 : }
1225 :
1226 : static flow_result_t
1227 67 : vr_do_flow_action(struct vrouter *router, struct vr_flow_entry *fe,
1228 : unsigned int index, struct vr_packet *pkt,
1229 : struct vr_forwarding_md *fmd)
1230 : {
1231 : uint32_t new_stats;
1232 67 : struct vr_flow_stats stats, *stats_p = NULL;
1233 :
1234 67 : if (fe->fe_flags & VR_FLOW_FLAG_NEW_FLOW) {
1235 5 : memcpy(&stats, &fe->fe_stats, sizeof(fe->fe_stats));
1236 5 : memset(&fe->fe_stats, 0, sizeof(fe->fe_stats));
1237 5 : stats_p = &stats;
1238 : }
1239 :
1240 67 : new_stats = vr_sync_add_and_fetch_32u(&fe->fe_stats.flow_bytes, pkt_len(pkt));
1241 67 : if (new_stats < pkt_len(pkt))
1242 0 : fe->fe_stats.flow_bytes_oflow++;
1243 :
1244 67 : new_stats = vr_sync_add_and_fetch_32u(&fe->fe_stats.flow_packets, 1);
1245 67 : if (!new_stats)
1246 0 : fe->fe_stats.flow_packets_oflow++;
1247 :
1248 67 : if (fe->fe_action == VR_FLOW_ACTION_HOLD) {
1249 7 : vr_enqueue_flow(router, fe, pkt, index, stats_p, fmd);
1250 7 : return FLOW_HELD;
1251 : }
1252 :
1253 60 : return vr_flow_action(router, fe, index, pkt, fmd);
1254 : }
1255 :
1256 : static unsigned int
1257 18 : vr_flow_table_hold_count(struct vrouter *router)
1258 : {
1259 : unsigned int i, num_cpus;
1260 18 : uint64_t hcount = 0, act_count;
1261 18 : struct vr_flow_table_info *infop = router->vr_flow_table_info;
1262 :
1263 18 : num_cpus = vr_num_cpus;
1264 234 : for (i = 0; i < num_cpus; i++)
1265 216 : hcount += infop->vfti_hold_count[i];
1266 :
1267 18 : act_count = infop->vfti_action_count;
1268 18 : if (hcount >= act_count)
1269 18 : return hcount - act_count;
1270 :
1271 0 : return 0;
1272 : }
1273 :
1274 : static void
1275 0 : vr_flow_burst_timeout(void *arg)
1276 : {
1277 : int tokens;
1278 0 : struct vrouter *router = (struct vrouter *)arg;
1279 0 : struct vr_flow_table_info *infop = router->vr_flow_table_info;
1280 :
1281 0 : tokens = infop->vfti_burst_tokens - infop->vfti_burst_used;
1282 0 : if (tokens > 0) {
1283 :
1284 0 : tokens = infop->vfti_burst_tokens_configured - tokens;
1285 0 : if (tokens <= 0) {
1286 0 : infop->vfti_timer->vt_stop_timer = 1;
1287 0 : return;
1288 : }
1289 :
1290 0 : if (tokens > infop->vfti_burst_step_configured)
1291 0 : tokens = infop->vfti_burst_step_configured;
1292 : } else {
1293 0 : tokens = infop->vfti_burst_step_configured;
1294 : }
1295 :
1296 0 : infop->vfti_burst_tokens += tokens;
1297 :
1298 0 : return;
1299 : }
1300 :
1301 : static void
1302 0 : vr_flow_start_burst_processing(struct vrouter *router)
1303 : {
1304 : struct vr_timer *vtimer;
1305 0 : struct vr_flow_table_info *infop = router->vr_flow_table_info;
1306 :
1307 0 : if (!infop->vfti_burst_tokens_configured ||
1308 0 : !infop->vfti_burst_interval_configured ||
1309 0 : !infop->vfti_burst_step_configured) {
1310 0 : return;
1311 : }
1312 :
1313 0 : if (!infop->vfti_timer) {
1314 0 : vtimer = vr_zalloc(sizeof(*vtimer), VR_TIMER_OBJECT);
1315 0 : if (!vtimer) {
1316 0 : vr_module_error(-ENOMEM, __FUNCTION__, __LINE__, sizeof(*vtimer));
1317 0 : return;
1318 : }
1319 :
1320 0 : vtimer->vt_timer = vr_flow_burst_timeout;
1321 0 : vtimer->vt_vr_arg = router;
1322 0 : vtimer->vt_msecs = infop->vfti_burst_interval_configured;
1323 :
1324 0 : if (vr_create_timer(vtimer)) {
1325 0 : vr_free(vtimer, VR_TIMER_OBJECT);
1326 0 : return;
1327 : }
1328 :
1329 0 : infop->vfti_timer = vtimer;
1330 : } else {
1331 0 : if (!infop->vfti_timer->vt_stop_timer)
1332 0 : return;
1333 :
1334 0 : if (vr_sync_bool_compare_and_swap_32u(
1335 : &infop->vfti_timer->vt_stop_timer, 1, 0)) {
1336 0 : infop->vfti_timer->vt_msecs = infop->vfti_burst_interval_configured;
1337 0 : vr_restart_timer(infop->vfti_timer);
1338 : }
1339 : }
1340 :
1341 0 : return;
1342 : }
1343 :
1344 : static void
1345 5 : vr_flow_entry_set_hold(struct vrouter *router, struct vr_flow_entry
1346 : *flow_e, bool burst)
1347 : {
1348 : unsigned int cpu;
1349 : uint64_t act_count;
1350 5 : struct vr_flow_table_info *infop = router->vr_flow_table_info;
1351 :
1352 5 : cpu = vr_get_cpu();
1353 5 : if (cpu >= vr_num_cpus) {
1354 0 : vr_printf("vrouter: Set HOLD failed (cpu %u num_cpus %u)\n",
1355 : cpu, vr_num_cpus);
1356 0 : return;
1357 : }
1358 :
1359 5 : flow_e->fe_action = VR_FLOW_ACTION_HOLD;
1360 :
1361 5 : if (infop->vfti_hold_count[cpu] + 1 < infop->vfti_hold_count[cpu]) {
1362 0 : (void)vr_sync_add_and_fetch_32u(&infop->vfti_oflows, 1);
1363 0 : act_count = infop->vfti_action_count;
1364 0 : if (act_count > infop->vfti_hold_count[cpu]) {
1365 0 : (void)vr_sync_sub_and_fetch_64u(&infop->vfti_action_count,
1366 : infop->vfti_hold_count[cpu]);
1367 0 : infop->vfti_hold_count[cpu] = 0;
1368 : } else {
1369 0 : infop->vfti_hold_count[cpu] -= act_count;
1370 0 : (void)vr_sync_sub_and_fetch_64u(&infop->vfti_action_count,
1371 : act_count);
1372 : }
1373 : }
1374 :
1375 5 : infop->vfti_hold_count[cpu]++;
1376 :
1377 5 : if (burst == true) {
1378 0 : (void)vr_sync_add_and_fetch_64u(&infop->vfti_burst_used, 1);
1379 0 : vr_flow_start_burst_processing(router);
1380 : }
1381 :
1382 5 : return;
1383 : }
1384 :
1385 : static void
1386 4 : vr_flow_init_close(struct vrouter *router, struct vr_flow_entry *flow_e,
1387 : struct vr_packet *pkt, struct vr_forwarding_md *fmd)
1388 : {
1389 : struct vr_flow_entry *rfe;
1390 :
1391 4 : (void)vr_sync_fetch_and_or_16u(&flow_e->fe_tcp_flags, VR_FLOW_TCP_DEAD);
1392 4 : rfe = vr_flow_get_entry(router, flow_e->fe_rflow);
1393 4 : if (rfe) {
1394 4 : (void)vr_sync_fetch_and_or_16u(&rfe->fe_tcp_flags, VR_FLOW_TCP_DEAD);
1395 : }
1396 :
1397 4 : return;
1398 : }
1399 :
1400 : static void
1401 49 : vr_flow_tcp_rflow_set(struct vrouter *router, struct vr_flow_entry *fe,
1402 : struct vr_flow_entry *rfe)
1403 : {
1404 49 : uint16_t flags = 0;
1405 :
1406 49 : if (!fe || !rfe)
1407 0 : return;
1408 :
1409 49 : if (rfe->fe_tcp_flags & VR_FLOW_TCP_SYN) {
1410 1 : flags |= VR_FLOW_TCP_SYN_R;
1411 : }
1412 :
1413 49 : if (rfe->fe_tcp_flags & VR_FLOW_TCP_RST) {
1414 0 : flags |= VR_FLOW_TCP_RST;
1415 : }
1416 :
1417 49 : if (rfe->fe_tcp_flags & VR_FLOW_TCP_DEAD) {
1418 0 : flags |= VR_FLOW_TCP_DEAD;
1419 : }
1420 :
1421 49 : if (rfe->fe_tcp_flags & VR_FLOW_TCP_FIN) {
1422 0 : flags |= VR_FLOW_TCP_FIN_R;
1423 : }
1424 :
1425 49 : if (rfe->fe_tcp_flags & VR_FLOW_TCP_ESTABLISHED) {
1426 0 : flags |= (VR_FLOW_TCP_ESTABLISHED | VR_FLOW_TCP_ESTABLISHED_R);
1427 : }
1428 :
1429 49 : (void)vr_sync_fetch_and_or_16u(&fe->fe_tcp_flags, flags);
1430 49 : return;
1431 : }
1432 :
1433 : static void
1434 67 : vr_flow_tcp_digest(struct vrouter *router, struct vr_flow_entry *flow_e,
1435 : struct vr_packet *pkt, struct vr_forwarding_md *fmd)
1436 : {
1437 67 : uint8_t proto = 0, hlen = 0;
1438 : uint16_t tcp_offset_flags;
1439 67 : unsigned int length = 0;
1440 : uint16_t flow_tcp_flags;
1441 :
1442 : struct vr_ip *iph;
1443 : struct vr_ip6 *ip6h;
1444 : struct vr_tcp *tcph;
1445 : struct vr_ip6_frag *v6_frag;
1446 67 : struct vr_flow_entry *rflow_e = NULL;
1447 : unsigned int rflow_ack;
1448 :
1449 67 : if (pkt->vp_type == VP_TYPE_IP) {
1450 63 : iph = (struct vr_ip *)pkt_network_header(pkt);
1451 63 : if (!vr_ip_transport_header_valid(iph))
1452 12 : return;
1453 51 : proto = iph->ip_proto;
1454 :
1455 51 : length = ntohs(iph->ip_len) - (iph->ip_hl * 4);
1456 51 : hlen = iph->ip_hl * 4;
1457 4 : } else if (pkt->vp_type == VP_TYPE_IP6) {
1458 4 : ip6h = (struct vr_ip6 *)pkt_network_header(pkt);
1459 4 : if (!vr_ip6_transport_header_valid(ip6h))
1460 0 : return;
1461 4 : proto = ip6h->ip6_nxt;
1462 4 : length = ntohs(ip6h->ip6_plen);
1463 4 : hlen = sizeof(struct vr_ip6);
1464 4 : if (proto == VR_IP6_PROTO_FRAG) {
1465 0 : v6_frag = (struct vr_ip6_frag *)(ip6h + 1);
1466 0 : proto = v6_frag->ip6_frag_nxt;
1467 0 : length -= sizeof(struct vr_ip6_frag);
1468 0 : hlen += sizeof(struct vr_ip6_frag);
1469 : }
1470 : }
1471 :
1472 55 : if (proto != VR_IP_PROTO_TCP)
1473 35 : return;
1474 :
1475 20 : tcph = (struct vr_tcp *)(pkt_network_header(pkt) + hlen);
1476 :
1477 20 : if (tcph) {
1478 20 : if (vr_flow_is_fat_flow(router, pkt, flow_e))
1479 0 : return;
1480 :
1481 : /*
1482 : * there are some optimizations here that makes the code slightly
1483 : * not so frugal. For e.g.: the *_R flags are used to make sure that
1484 : * for a packet that contains ACK, we will not need to fetch the
1485 : * reverse flow if we are not interested, thus saving some execution
1486 : * time.
1487 : */
1488 20 : tcp_offset_flags = ntohs(tcph->tcp_offset_r_flags);
1489 :
1490 : /*
1491 : * If this is an ack, set the last acked seqnum
1492 : */
1493 20 : if (tcp_offset_flags & VR_TCP_FLAG_ACK) {
1494 9 : flow_e->fe_tcp_ack = ntohl(tcph->tcp_ack);
1495 : }
1496 :
1497 : /*
1498 : * if we get a reset, TCP session will be closed if the
1499 : * vr_uncond_close_flow_on_tcp_rst flag is enabled or the TCP RST
1500 : * seqnum matches with the seqnum acked by the receiver
1501 : * - as per RFC 5961 sec 3.2
1502 : */
1503 20 : if (tcp_offset_flags & VR_TCP_FLAG_RST) {
1504 6 : if (!vr_uncond_close_flow_on_tcp_rst) {
1505 : /* get the reverse flow ack seq num if valid */
1506 4 : if (flow_e->fe_flags & VR_RFLOW_VALID) {
1507 4 : rflow_e = vr_flow_get_entry(router, flow_e->fe_rflow);
1508 : }
1509 4 : if (rflow_e) {
1510 4 : rflow_ack = rflow_e->fe_tcp_ack;
1511 : /* Implementing rfc-5961 (section 3.2),
1512 : * In the SYN-SENT state (a RST received in response to an initial SYN),
1513 : * the RST is acceptable if the ACK field acknowledges the SYN. In all
1514 : * other cases the receiver MUST silently discard the segment. */
1515 4 : if ((tcp_offset_flags & VR_TCP_FLAG_ACK)
1516 2 : && (rflow_e->fe_tcp_flags & VR_FLOW_TCP_SYN)) {
1517 2 : if (ntohl(tcph->tcp_ack) != (rflow_e->fe_tcp_seq + 1)) {
1518 : /* Ignore the RST */
1519 1 : return;
1520 : }
1521 : }
1522 2 : else if (ntohl(tcph->tcp_seq) != rflow_ack) {
1523 : /* Ignore the RST */
1524 1 : return;
1525 : }
1526 : }
1527 : /* If Reverse flow not valid, go ahead and close this flow */
1528 : }
1529 :
1530 4 : (void)vr_sync_fetch_and_or_16u(&flow_e->fe_tcp_flags,
1531 : VR_FLOW_TCP_RST);
1532 4 : if (flow_e->fe_flags & VR_RFLOW_VALID) {
1533 4 : rflow_e = vr_flow_get_entry(router, flow_e->fe_rflow);
1534 4 : if (rflow_e) {
1535 4 : (void)vr_sync_fetch_and_or_16u(&rflow_e->fe_tcp_flags,
1536 : VR_FLOW_TCP_RST);
1537 : }
1538 : }
1539 4 : vr_flow_init_close(router, flow_e, pkt, fmd);
1540 4 : return;
1541 14 : } else if (tcp_offset_flags & VR_TCP_FLAG_SYN) {
1542 : /* if only a SYN... */
1543 11 : flow_e->fe_tcp_seq = ntohl(tcph->tcp_seq);
1544 11 : (void)vr_sync_fetch_and_or_16u(&flow_e->fe_tcp_flags, VR_FLOW_TCP_SYN);
1545 11 : if (flow_e->fe_flags & VR_RFLOW_VALID) {
1546 9 : rflow_e = vr_flow_get_entry(router, flow_e->fe_rflow);
1547 9 : if (rflow_e) {
1548 9 : (void)vr_sync_fetch_and_or_16u(&rflow_e->fe_tcp_flags,
1549 : VR_FLOW_TCP_SYN_R);
1550 9 : if ((flow_e->fe_tcp_flags & VR_FLOW_TCP_SYN_R) &&
1551 3 : (tcp_offset_flags & VR_TCP_FLAG_ACK)) {
1552 3 : if (ntohl(tcph->tcp_ack) == (rflow_e->fe_tcp_seq + 1)) {
1553 3 : (void)vr_sync_fetch_and_or_16u(&rflow_e->fe_tcp_flags,
1554 : VR_FLOW_TCP_ESTABLISHED);
1555 3 : flow_tcp_flags = vr_sync_fetch_and_or_16u(
1556 : &flow_e->fe_tcp_flags, VR_FLOW_TCP_ESTABLISHED_R);
1557 3 : if (!(flow_tcp_flags & VR_FLOW_TCP_ESTABLISHED_R) &&
1558 3 : !(flow_tcp_flags & VR_FLOW_TCP_ESTABLISHED)) {
1559 : // If it wasn't established or reverse established,
1560 : // it is being established now. Offload both.
1561 3 : vr_offload_flow_set(flow_e,
1562 : flow_e->fe_hentry.hentry_index, rflow_e);
1563 3 : vr_offload_flow_set(rflow_e,
1564 : rflow_e->fe_hentry.hentry_index, flow_e);
1565 : }
1566 : }
1567 : }
1568 : }
1569 : }
1570 3 : } else if (tcp_offset_flags & VR_TCP_FLAG_FIN) {
1571 : /*
1572 : * when a FIN is received, update the sequence of the FIN and set
1573 : * the flow FIN flag. It is possible that the FIN packet came with
1574 : * some data, in which case the sequence number of the FIN is one
1575 : * more than the last data byte in the sequence
1576 : */
1577 0 : length -= (((tcp_offset_flags) >> 12) * 4);
1578 0 : flow_e->fe_tcp_seq = ntohl(tcph->tcp_seq) + length;
1579 0 : (void)vr_sync_fetch_and_or_16u(&flow_e->fe_tcp_flags, VR_FLOW_TCP_FIN);
1580 : /*
1581 : * when an ack for a FIN is sent, we need to take some actions
1582 : * on the reverse flow (since FIN came in the reverse flow). to
1583 : * avoid looking up the reverse flow for all acks, we mark the
1584 : * reverse flow's reverse flow with a flag (FIN_R). we will
1585 : * lookup the reverse flow only if this flag is set and the
1586 : * tcp header has an ack bit set
1587 : */
1588 0 : if (flow_e->fe_flags & VR_RFLOW_VALID) {
1589 0 : rflow_e = vr_flow_get_entry(router, flow_e->fe_rflow);
1590 : /* Delete offloaded flow on TCP FIN */
1591 0 : (void)vr_offload_flow_del(flow_e);
1592 0 : if (rflow_e) {
1593 0 : (void)vr_sync_fetch_and_or_16u(&rflow_e->fe_tcp_flags,
1594 : VR_FLOW_TCP_FIN_R);
1595 : /* Delete offloaded reverse flow on TCP FIN */
1596 0 : (void)vr_offload_flow_del(rflow_e);
1597 : }
1598 : }
1599 : }
1600 :
1601 : /*
1602 : * if FIN_R is set in the flow and if the ACK bit is set in the
1603 : * tcp header, then we need to mark the reverse flow as dead.
1604 : *
1605 : * OR
1606 : *
1607 : * if the SYN_R is set and ESTABLISHED_R is not set and if this
1608 : * is an ack packet, if this ack completes the connection, we
1609 : * need to set ESTABLISHED
1610 : */
1611 14 : if (((flow_e->fe_tcp_flags & VR_FLOW_TCP_FIN_R) ||
1612 14 : (!(flow_e->fe_tcp_flags & VR_FLOW_TCP_ESTABLISHED_R) &&
1613 11 : (flow_e->fe_tcp_flags & VR_FLOW_TCP_SYN_R))) &&
1614 3 : (tcp_offset_flags & VR_TCP_FLAG_ACK)) {
1615 3 : if (flow_e->fe_flags & VR_RFLOW_VALID) {
1616 3 : if (!rflow_e) {
1617 3 : rflow_e = vr_flow_get_entry(router, flow_e->fe_rflow);
1618 : }
1619 :
1620 3 : if (rflow_e) {
1621 3 : if ((ntohl(tcph->tcp_ack) == (rflow_e->fe_tcp_seq + 1)) &&
1622 3 : (flow_e->fe_tcp_flags & VR_FLOW_TCP_FIN_R)) {
1623 0 : (void)vr_sync_fetch_and_or_16u(&rflow_e->fe_tcp_flags,
1624 : VR_FLOW_TCP_HALF_CLOSE);
1625 : /*
1626 : * both the forward and the reverse flows are
1627 : * now dead
1628 : */
1629 0 : if (flow_e->fe_tcp_flags & VR_FLOW_TCP_HALF_CLOSE) {
1630 0 : vr_flow_init_close(router, flow_e, pkt, fmd);
1631 : }
1632 3 : } else if (ntohl(tcph->tcp_ack) != rflow_e->fe_tcp_seq) {
1633 3 : if (!(flow_e->fe_tcp_flags &
1634 : VR_FLOW_TCP_ESTABLISHED_R)) {
1635 3 : (void)vr_sync_fetch_and_or_16u(&rflow_e->fe_tcp_flags,
1636 : VR_FLOW_TCP_ESTABLISHED);
1637 3 : flow_tcp_flags = vr_sync_fetch_and_or_16u(
1638 : &flow_e->fe_tcp_flags, VR_FLOW_TCP_ESTABLISHED_R);
1639 3 : if (!(flow_tcp_flags & VR_FLOW_TCP_ESTABLISHED_R) &&
1640 3 : !(flow_tcp_flags & VR_FLOW_TCP_ESTABLISHED)) {
1641 : // If it wasn't established or reverse established,
1642 : // it is being established now. Offload both.
1643 0 : vr_offload_flow_set(flow_e,
1644 : flow_e->fe_hentry.hentry_index, rflow_e);
1645 0 : vr_offload_flow_set(rflow_e,
1646 : rflow_e->fe_hentry.hentry_index, flow_e);
1647 : }
1648 : }
1649 : }
1650 : }
1651 : }
1652 : }
1653 : }
1654 :
1655 14 : return;
1656 : }
1657 :
1658 : static inline bool
1659 5 : vr_flow_vif_allow_new_flow(struct vrouter *router, struct vr_packet *pkt,
1660 : unsigned short *drop_reason)
1661 : {
1662 5 : struct vr_interface *vif_l = NULL;
1663 5 : struct vr_nexthop *nh = NULL;
1664 :
1665 5 : if (vif_is_virtual(pkt->vp_if)) {
1666 3 : vif_l = pkt->vp_if;
1667 2 : } else if (vif_is_fabric(pkt->vp_if)) {
1668 1 : nh = pkt->vp_nh;
1669 1 : if ((nh != NULL) && (nh->nh_flags & NH_FLAG_VALID)) {
1670 1 : vif_l = nh->nh_dev;
1671 : }
1672 : }
1673 :
1674 5 : if (vif_l && vif_drop_new_flows(vif_l)) {
1675 0 : PKT_LOG(VP_DROP_NEW_FLOWS, pkt, 0, VR_FLOW_C, __LINE__);
1676 0 : *drop_reason = VP_DROP_NEW_FLOWS;
1677 0 : return false;
1678 : }
1679 :
1680 5 : return true;
1681 : }
1682 :
1683 : void
1684 0 : vr_flow_get_burst_params(struct vrouter *router, int *burst_tokens,
1685 : int *burst_interval, int *burst_step)
1686 : {
1687 : struct vr_flow_table_info *infop;
1688 :
1689 0 : if (!router || !router->vr_flow_table_info)
1690 0 : return;
1691 :
1692 0 : infop = router->vr_flow_table_info;
1693 :
1694 0 : if (burst_tokens)
1695 0 : *burst_tokens = infop->vfti_burst_tokens_configured;
1696 0 : if (burst_interval)
1697 0 : *burst_interval = infop->vfti_burst_interval_configured;
1698 0 : if (burst_step)
1699 0 : *burst_step = infop->vfti_burst_step_configured;
1700 :
1701 0 : return;
1702 : }
1703 :
1704 : void
1705 0 : vr_flow_set_burst_params(struct vrouter *router, int burst_tokens,
1706 : int burst_interval, int burst_step)
1707 : {
1708 : struct vr_flow_table_info *infop;
1709 :
1710 0 : if (!router || !router->vr_flow_table_info)
1711 0 : return;
1712 :
1713 0 : infop = router->vr_flow_table_info;
1714 :
1715 0 : if (burst_tokens != -1)
1716 0 : infop->vfti_burst_tokens_configured = burst_tokens;
1717 :
1718 0 : if (burst_interval != -1)
1719 0 : infop->vfti_burst_interval_configured = burst_interval;
1720 :
1721 0 : if (burst_step != -1)
1722 0 : infop->vfti_burst_step_configured = burst_step;
1723 :
1724 :
1725 0 : vr_flow_start_burst_processing(router);
1726 0 : return;
1727 : }
1728 :
1729 : static inline unsigned int
1730 0 : vr_flow_burst_count(struct vrouter *router)
1731 : {
1732 0 : struct vr_flow_table_info *infop = router->vr_flow_table_info;
1733 :
1734 0 : return infop->vfti_burst_tokens;
1735 : }
1736 :
1737 : static inline bool
1738 5 : vr_flow_allow_new_flow(struct vrouter *router, struct vr_packet *pkt,
1739 : unsigned short *drop_reason, bool *burst)
1740 : {
1741 : unsigned int hold_count;
1742 5 : struct vr_flow_table_info *infop = router->vr_flow_table_info;
1743 :
1744 5 : *drop_reason = VP_DROP_FLOW_UNUSABLE;
1745 5 : if (burst)
1746 5 : *burst = false;
1747 :
1748 5 : if (pkt->vp_type == VP_TYPE_IP) {
1749 5 : if (!vr_inet_flow_allow_new_flow(router, pkt)) {
1750 0 : PKT_LOG(VP_DROP_FLOW_UNUSABLE, pkt, 0, VR_FLOW_C, __LINE__);
1751 0 : *drop_reason = VP_DROP_FLOW_UNUSABLE;
1752 0 : return false;
1753 : }
1754 : }
1755 :
1756 5 : if (vr_flow_hold_limit) {
1757 5 : hold_count = vr_flow_table_hold_count(router);
1758 5 : if (hold_count > vr_flow_hold_limit) {
1759 0 : if (infop->vfti_burst_used >= vr_flow_burst_count(router)) {
1760 0 : PKT_LOG(VP_DROP_FLOW_UNUSABLE, pkt, 0, VR_FLOW_C, __LINE__);
1761 0 : *drop_reason = VP_DROP_FLOW_UNUSABLE;
1762 0 : return false;
1763 : }
1764 0 : if (burst) {
1765 0 : *burst = true;
1766 : }
1767 : }
1768 : }
1769 :
1770 5 : return vr_flow_vif_allow_new_flow(router, pkt, drop_reason);
1771 : }
1772 :
1773 : static inline struct vr_flow_entry *
1774 5 : vr_flow_new_hold_flow(struct vrouter *router, struct vr_flow *key,
1775 : struct vr_packet *pkt, unsigned int *fe_index,
1776 : struct vr_forwarding_md *fmd) {
1777 : struct vr_flow_entry *bucket_fe, *flow_e;
1778 5 : unsigned short drop_reason = 0;
1779 5 : bool burst = false;
1780 : /* Slow path: lock, to make find+create atomic thus avoid races
1781 : * with another vr_flow_lookup or vr_add_flow). We also need need
1782 : * to retry the find, as previous one was done without locking */
1783 5 : bucket_fe = vr_flow_bucket_first_entry(router, key);
1784 5 : vr_flow_bucket_may_lock(bucket_fe);
1785 5 : flow_e = vr_find_flow(router, key, pkt->vp_type, fe_index);
1786 5 : if (!flow_e) {
1787 5 : if (!vr_flow_allow_new_flow(router, pkt, &drop_reason, &burst)) {
1788 0 : vr_flow_bucket_may_unlock(bucket_fe);
1789 0 : PKT_LOG(drop_reason, pkt, key , VR_FLOW_C, __LINE__);
1790 0 : vr_pfree(pkt, drop_reason);
1791 0 : return flow_e;
1792 : }
1793 :
1794 5 : flow_e = vr_flow_get_free_entry(router, key, pkt->vp_type,
1795 : true, fe_index);
1796 5 : if (!flow_e) {
1797 0 : vr_flow_bucket_may_unlock(bucket_fe);
1798 0 : PKT_LOG(VP_DROP_FLOW_TABLE_FULL, pkt, key, VR_FLOW_C, __LINE__);
1799 0 : vr_pfree(pkt, VP_DROP_FLOW_TABLE_FULL);
1800 0 : return flow_e;
1801 : }
1802 :
1803 5 : flow_e->fe_vrf = fmd->fmd_dvrf;
1804 : /* mark as hold */
1805 5 : vr_flow_entry_set_hold(router, flow_e, burst);
1806 : }
1807 5 : vr_flow_bucket_may_unlock(bucket_fe);
1808 5 : return flow_e;
1809 : }
1810 :
1811 : flow_result_t
1812 67 : vr_flow_lookup(struct vrouter *router, struct vr_flow *key,
1813 : struct vr_packet *pkt, struct vr_forwarding_md *fmd)
1814 : {
1815 : unsigned int fe_index;
1816 : struct vr_flow_entry *flow_e;
1817 67 : pkt->vp_flags |= VP_FLAG_FLOW_SET;
1818 :
1819 67 : if (!fmd->fmd_fe) {
1820 : /* Happy path: without locking */
1821 67 : flow_e = vr_find_flow(router, key, pkt->vp_type, &fe_index);
1822 67 : if (!flow_e) {
1823 5 : if (pkt->vp_nh &&
1824 2 : (pkt->vp_nh->nh_flags &
1825 : (NH_FLAG_RELAXED_POLICY | NH_FLAG_FLOW_LOOKUP)))
1826 0 : return FLOW_FORWARD;
1827 :
1828 5 : flow_e = vr_flow_new_hold_flow(router, key, pkt, &fe_index, fmd);
1829 5 : if (!flow_e)
1830 0 : return FLOW_CONSUMED;
1831 : }
1832 : } else {
1833 0 : flow_e = fmd->fmd_fe;
1834 0 : fe_index = fmd->fmd_flow_index;
1835 : }
1836 :
1837 67 : if (flow_e->fe_flags & VR_FLOW_FLAG_EVICT_CANDIDATE)
1838 0 : return FLOW_EVICT_DROP;
1839 :
1840 : /*
1841 : * Store the source of the packet which gets used incase of Ecmp
1842 : * Source
1843 : */
1844 67 : if (vif_is_fabric(pkt->vp_if))
1845 8 : flow_e->fe_src_info = fmd->fmd_outer_src_ip;
1846 59 : else if (vif_is_virtual(pkt->vp_if))
1847 58 : flow_e->fe_src_info = pkt->vp_if->vif_idx;
1848 :
1849 67 : vr_flow_set_forwarding_md(router, flow_e, fe_index, fmd);
1850 67 : vr_flow_tcp_digest(router, flow_e, pkt, fmd);
1851 :
1852 67 : return vr_do_flow_action(router, flow_e, fe_index, pkt, fmd);
1853 : }
1854 :
1855 : static bool
1856 188 : __vr_flow_forward(flow_result_t result, struct vr_packet *pkt,
1857 : struct vr_forwarding_md *fmd)
1858 : {
1859 188 : bool forward = false;
1860 :
1861 188 : switch (result) {
1862 164 : case FLOW_FORWARD:
1863 164 : forward = true;
1864 164 : break;
1865 :
1866 0 : case FLOW_TRAP:
1867 0 : vr_trap(pkt, fmd->fmd_dvrf, AGENT_TRAP_L3_PROTOCOLS, NULL);
1868 0 : break;
1869 :
1870 0 : case FLOW_EVICT_DROP:
1871 0 : PKT_LOG(VP_DROP_FLOW_EVICT, pkt, 0, VR_FLOW_C, __LINE__);
1872 0 : vr_pfree(pkt, VP_DROP_FLOW_EVICT);
1873 0 : break;
1874 :
1875 24 : case FLOW_HELD:
1876 : case FLOW_CONSUMED:
1877 24 : break;
1878 :
1879 0 : case FLOW_DROP:
1880 : default:
1881 0 : PKT_LOG(VP_DROP_FLOW_UNUSABLE, pkt, 0, VR_FLOW_C, __LINE__);
1882 0 : vr_pfree(pkt, VP_DROP_FLOW_UNUSABLE);
1883 0 : break;
1884 : }
1885 :
1886 188 : return forward;
1887 : }
1888 :
1889 : static bool
1890 20 : vr_flow_is_fat_flow(struct vrouter *router, struct vr_packet *pkt,
1891 : struct vr_flow_entry *fe)
1892 : {
1893 20 : if (pkt->vp_type == VP_TYPE_IP) {
1894 20 : return vr_inet_flow_is_fat_flow(router, pkt, fe);
1895 0 : } else if (pkt->vp_type == VP_TYPE_IP6) {
1896 0 : return vr_inet6_flow_is_fat_flow(router, pkt, fe);
1897 : }
1898 :
1899 0 : return false;
1900 : }
1901 :
1902 : uint16_t
1903 59 : vr_flow_fat_flow_lookup(struct vrouter *router, struct vr_packet *pkt,
1904 : uint16_t l4_proto, uint16_t sport, uint16_t dport,
1905 : unsigned int *saddr, unsigned int *daddr,
1906 : unsigned char *ip6_src, unsigned char *ip6_dst)
1907 : {
1908 59 : uint8_t fat_flow_mask, tmp_mask = 0;
1909 : struct vr_nexthop *nh;
1910 59 : struct vr_interface *vif_l = NULL;
1911 :
1912 59 : if (vif_is_virtual(pkt->vp_if)) {
1913 49 : vif_l = pkt->vp_if;
1914 10 : } else if (vif_is_fabric(pkt->vp_if)) {
1915 8 : if ((nh = pkt->vp_nh) && (nh->nh_flags & NH_FLAG_VALID)) {
1916 : /* In case of ECMP, check the vif within the ECMP member NH */
1917 8 : if (vr_is_local_ecmp_nh(nh)) {
1918 0 : vif_l = vr_get_ecmp_first_member_dev(nh);
1919 : } else {
1920 8 : vif_l = nh->nh_dev;
1921 : }
1922 : }
1923 : }
1924 :
1925 59 : if (!vif_l)
1926 2 : return VR_FAT_FLOW_NO_MASK;
1927 :
1928 57 : fat_flow_mask = vif_fat_flow_lookup((pkt->vp_if == vif_l), vif_l, l4_proto, sport, dport,
1929 : saddr, daddr, ip6_src, ip6_dst);
1930 57 : if (pkt->vp_if != vif_l) {
1931 :
1932 8 : if (fat_flow_mask & VR_FAT_FLOW_SRC_IP_MASK)
1933 0 : tmp_mask |= VR_FAT_FLOW_DST_IP_MASK;
1934 :
1935 8 : if (fat_flow_mask & VR_FAT_FLOW_DST_IP_MASK)
1936 0 : tmp_mask |= VR_FAT_FLOW_SRC_IP_MASK;
1937 :
1938 8 : fat_flow_mask &= ~(VR_FAT_FLOW_DST_IP_MASK |
1939 : VR_FAT_FLOW_SRC_IP_MASK);
1940 :
1941 :
1942 8 : fat_flow_mask |= tmp_mask;
1943 :
1944 : }
1945 :
1946 57 : return fat_flow_mask;
1947 : }
1948 :
1949 : static flow_result_t
1950 170 : vr_do_flow_lookup(struct vrouter *router, struct vr_packet *pkt,
1951 : struct vr_forwarding_md *fmd)
1952 : {
1953 170 : flow_result_t result = FLOW_FORWARD;
1954 :
1955 : /* Flow processing is only for untagged unicast IP packets */
1956 170 : if (pkt->vp_type == VP_TYPE_IP)
1957 156 : result = vr_inet_flow_lookup(router, pkt, fmd);
1958 14 : else if (pkt->vp_type == VP_TYPE_IP6)
1959 10 : result = vr_inet6_flow_lookup(router, pkt, fmd);
1960 :
1961 170 : return result;
1962 : }
1963 :
1964 : static void
1965 8 : vr_reinit_forwarding_md(struct vrouter *router, struct vr_packet *pkt,
1966 : struct vr_flow_entry *fe, uint32_t flow_index,
1967 : struct vr_nexthop *nh, struct vr_forwarding_md *fmd)
1968 : {
1969 : int i;
1970 8 : struct vr_nexthop *src_nh = __vrouter_get_nexthop(router, fe->fe_src_nh_index);
1971 :
1972 8 : for (i = 0; i < VR_MAX_PHY_INF; i++) {
1973 8 : if (nh->nh_dev_arr[i] != NULL) {
1974 8 : fmd->fmd_dvrf = nh->nh_dev_arr[i]->vif_vrf;
1975 8 : break;
1976 : }
1977 : }
1978 :
1979 8 : fmd->fmd_vlan = 0;
1980 8 : fmd->fmd_dotonep = -1;
1981 8 : fmd->fmd_outer_src_ip = src_nh->nh_udp_tun_dip;
1982 8 : vr_flow_set_forwarding_md(router, fe, flow_index, fmd);
1983 8 : }
1984 :
1985 : bool
1986 186 : vr_flow_forward(struct vrouter *router, struct vr_packet *pkt,
1987 : struct vr_forwarding_md *fmd)
1988 : {
1989 186 : flow_result_t result = FLOW_FORWARD;
1990 :
1991 186 : if (vif_is_hbs_right(pkt->vp_if)) {
1992 : /* Pkt entering vrouter from hbs-r
1993 : * - If SMAC has magic, Restore actual SMAC from flow_index and continue
1994 : * - If DMAC has magic, Restore actual DMAC from flow_index and continue
1995 : */
1996 4 : struct vr_eth_hbs_md *eth_hbs = (struct vr_eth_hbs_md*)pkt_data(pkt);
1997 4 : struct vr_eth *eth = (struct vr_eth*)pkt_data(pkt);
1998 : uint16_t magic;
1999 : unsigned int flow_index;
2000 : uint32_t nh_id;
2001 4 : struct vr_flow_entry *fe = NULL;
2002 : unsigned char* mac;
2003 4 : if ((ntohs(eth_hbs->magic_smac) & VR_HBS_MAGIC_MASK) ==
2004 : VR_HBS_SMAC_MAGIC) {
2005 4 : magic = VR_HBS_SMAC_MAGIC;
2006 4 : flow_index = ntohl(eth_hbs->flow_id_smac);
2007 4 : mac = eth->eth_smac;
2008 0 : } else if ((ntohs(eth_hbs->magic_dmac) & VR_HBS_MAGIC_MASK) ==
2009 : VR_HBS_DMAC_MAGIC) {
2010 0 : magic = VR_HBS_DMAC_MAGIC;
2011 0 : flow_index = ntohl(eth_hbs->flow_id_dmac);
2012 0 : mac = eth->eth_dmac;
2013 : } else {
2014 0 : PKT_LOG(VP_DROP_INVALID_HBS_PKT, pkt, 0, VR_FLOW_C, __LINE__);
2015 0 : goto drop_pkt;
2016 : }
2017 :
2018 4 : if (vr_htable_get_hentry_by_index(router->vr_flow_table, flow_index)) {
2019 : struct vr_nexthop *nh;
2020 4 : fe = CONTAINER_OF(
2021 : fe_hentry,
2022 : struct vr_flow_entry,
2023 : vr_htable_get_hentry_by_index(
2024 : router->vr_flow_table, flow_index)
2025 : );
2026 4 : nh_id = fe->fe_key.flow_nh_id;
2027 4 : nh = vrouter_get_nexthop(0, nh_id);
2028 4 : vr_reinit_forwarding_md(router, pkt, fe, flow_index, nh, fmd);
2029 :
2030 4 : if(fe->fe_underlay_ecmp_index >= 0) {
2031 0 : if(nh->nh_type == NH_TUNNEL) {
2032 0 : pkt->vp_if = __vrouter_get_interface(router,
2033 0 : fe->fe_underlay_ecmp_index);
2034 : } else
2035 0 : pkt->vp_if = nh->nh_dev;
2036 : } else
2037 4 : pkt->vp_if = nh->nh_dev;
2038 :
2039 4 : memcpy(mac, nh->nh_data, VR_ETHER_ALEN);
2040 4 : result = vr_flow_action_default(router, fe, flow_index, pkt, fmd);
2041 4 : return __vr_flow_forward(result, pkt, fmd);
2042 : }
2043 182 : } else if (vif_is_hbs_left(pkt->vp_if)) {
2044 : /* Pkt entering vrouter from hbs-l
2045 : * - DMAC of the packet contains flow_index
2046 : * - Restore actual DMAC from flow_index and continue
2047 : * flow action
2048 : * If VR_HBS_FROM_VMI is set in the DMAC -
2049 : * - The packet originated from VMI (instead of fabric)
2050 : * - Happens if both src and dst tenant VMs are in
2051 : * the same compute (intra-compute case)
2052 : * - Restore actual DMAC from "reverse flow_index"
2053 : */
2054 4 : struct vr_eth *eth = (struct vr_eth*)pkt_data(pkt);
2055 4 : struct vr_eth_hbs_md *eth_hbs = (struct vr_eth_hbs_md*)pkt_data(pkt);
2056 4 : uint16_t magic = ntohs(eth_hbs->magic_dmac);
2057 4 : unsigned int flow_index = ntohl(eth_hbs->flow_id_dmac);
2058 : uint32_t nh_id;
2059 4 : struct vr_flow_entry *fe = NULL;
2060 4 : if ((magic & VR_HBS_MAGIC_MASK) != VR_HBS_DMAC_MAGIC) {
2061 0 : PKT_LOG(VP_DROP_INVALID_HBS_PKT, pkt, 0, VR_FLOW_C, __LINE__);
2062 0 : goto drop_pkt;
2063 : }
2064 :
2065 4 : if (vr_htable_get_hentry_by_index(router->vr_flow_table, flow_index)) {
2066 : struct vr_nexthop *nh;
2067 4 : fe = CONTAINER_OF(
2068 : fe_hentry,
2069 : struct vr_flow_entry,
2070 : vr_htable_get_hentry_by_index(
2071 : router->vr_flow_table, flow_index)
2072 : );
2073 : /* If packet is coming from VMI instead of fabric,
2074 : * restore DMAC using reverse flow
2075 : */
2076 4 : if (magic & VR_HBS_FROM_VMI) {
2077 2 : flow_index = fe->fe_rflow;
2078 2 : fe = vr_flow_get_entry(router, fe->fe_rflow);
2079 2 : if (!fe) {
2080 0 : PKT_LOG(VP_DROP_INVALID_HBS_PKT, pkt, 0, VR_FLOW_C, __LINE__);
2081 0 : goto drop_pkt;
2082 : }
2083 : }
2084 :
2085 4 : nh_id = fe->fe_key.flow_nh_id;
2086 4 : nh = vrouter_get_nexthop(0, nh_id);
2087 4 : vr_reinit_forwarding_md(router, pkt, fe, flow_index, nh, fmd);
2088 :
2089 4 : if(fe->fe_underlay_ecmp_index >= 0) {
2090 0 : if(nh->nh_type == NH_TUNNEL) {
2091 0 : pkt->vp_if = __vrouter_get_interface(router,
2092 0 : fe->fe_underlay_ecmp_index);
2093 : } else
2094 0 : pkt->vp_if = nh->nh_dev;
2095 : } else
2096 4 : pkt->vp_if = nh->nh_dev;
2097 :
2098 4 : if (magic & VR_HBS_L3_PKT)
2099 0 : memcpy(eth->eth_dmac, pkt->vp_if->vif_mac, VR_ETHER_ALEN);
2100 : else
2101 4 : memcpy(eth->eth_dmac, nh->nh_data, VR_ETHER_ALEN);
2102 4 : result = vr_flow_action_default(router, fe, flow_index, pkt, fmd);
2103 4 : return __vr_flow_forward(result, pkt, fmd);
2104 : }
2105 178 : } else if ((!(pkt->vp_flags & VP_FLAG_MULTICAST))
2106 178 : && ((fmd->fmd_vlan == VLAN_ID_INVALID) || vif_is_service(pkt->vp_if)))
2107 170 : result = vr_do_flow_lookup(router, pkt, fmd);
2108 :
2109 178 : return __vr_flow_forward(result, pkt, fmd);
2110 :
2111 0 : drop_pkt:
2112 0 : vr_pfree(pkt, VP_DROP_INVALID_HBS_PKT);
2113 0 : return false;
2114 : }
2115 :
2116 : int
2117 29 : vr_flow_flush_pnode(struct vrouter *router, struct vr_packet_node *pnode,
2118 : struct vr_flow_entry *fe, struct vr_forwarding_md *fmd)
2119 : {
2120 : bool forward;
2121 :
2122 : struct vr_interface *vif;
2123 : struct vr_packet *pkt;
2124 : struct vr_ip *ip;
2125 : struct vr_ip6 *ip6;
2126 : flow_result_t result;
2127 :
2128 29 : fmd->fmd_outer_src_ip = pnode->pl_outer_src_ip;
2129 29 : if (pnode->pl_flags & PN_FLAG_LABEL_IS_VXLAN_ID) {
2130 0 : vr_fmd_set_label(fmd, pnode->pl_label,
2131 : VR_LABEL_TYPE_VXLAN_ID);
2132 : } else {
2133 29 : vr_fmd_set_label(fmd, pnode->pl_label,
2134 : VR_LABEL_TYPE_MPLS);
2135 : }
2136 :
2137 29 : if (pnode->pl_flags & PN_FLAG_TO_ME)
2138 0 : fmd->fmd_to_me = 1;
2139 :
2140 29 : pkt = pnode->pl_packet;
2141 29 : if (!pkt)
2142 14 : return -EINVAL;
2143 :
2144 15 : fmd->fmd_dscp = pnode->pl_dscp;
2145 15 : fmd->fmd_dotonep = pnode->pl_dotonep;
2146 15 : fmd->fmd_vlan = pnode->pl_vlan;
2147 15 : fmd->fmd_mirror_data = pnode->pl_mirror_vlan;
2148 :
2149 15 : pnode->pl_packet = NULL;
2150 : /*
2151 : * this is only a security check and not a catch all check. one note
2152 : * of caution. please do not access pkt->vp_if till the if block is
2153 : * succesfully bypassed
2154 : */
2155 15 : vif = __vrouter_get_interface(router, pnode->pl_vif_idx);
2156 15 : if (!vif || (pkt->vp_if != vif)) {
2157 5 : pkt->vp_if = NULL;
2158 5 : PKT_LOG(VP_DROP_INVALID_IF, pkt, 0, VR_FLOW_C, __LINE__);
2159 5 : vr_pfree(pkt, VP_DROP_INVALID_IF);
2160 5 : return -ENODEV;
2161 : }
2162 :
2163 10 : if (!pkt->vp_nh) {
2164 10 : if (vif_is_fabric(pkt->vp_if) && fmd &&
2165 1 : (fmd->fmd_label >= 0)) {
2166 1 : if (!vr_fmd_label_is_vxlan_id(fmd)) {
2167 1 : pkt->vp_nh = __vrouter_get_label(router, fmd->fmd_label);
2168 : } else {
2169 0 : pkt->vp_nh = __vrouter_bridge_lookup(fmd->fmd_dvrf,
2170 : pkt_data(pkt));
2171 : }
2172 : }
2173 10 : if(vif_is_vhost(vif) && !(vif->vif_flags & VIF_FLAG_POLICY_ENABLED)) {
2174 1 : if (pkt->vp_type == VP_TYPE_IP) {
2175 1 : ip = (struct vr_ip *) pkt_network_header(pkt);
2176 1 : pkt->vp_nh = vr_inet_ip_lookup(pnode->pl_vrf, ip->ip_daddr);
2177 0 : } else if (pkt->vp_type == VP_TYPE_IP6) {
2178 0 : ip6 = (struct vr_ip6 *) pkt_network_header(pkt);
2179 0 : pkt->vp_nh = vr_inet6_ip_lookup(pnode->pl_vrf, ip6->ip6_dst);
2180 : }
2181 : }
2182 : }
2183 :
2184 10 : if (fe) {
2185 2 : result = vr_flow_action(router, fe, fmd->fmd_flow_index, pkt, fmd);
2186 2 : forward = __vr_flow_forward(result, pkt, fmd);
2187 : } else {
2188 8 : forward = vr_flow_forward(router, pkt, fmd);
2189 : }
2190 :
2191 10 : if (forward)
2192 9 : vr_reinject_packet(pkt, fmd);
2193 :
2194 10 : return 0;
2195 : }
2196 :
2197 : static void
2198 7 : __vr_flow_flush_hold_queue(struct vrouter *router, struct vr_flow_entry *fe,
2199 : struct vr_forwarding_md *fmd, struct vr_flow_queue *vfq)
2200 : {
2201 : unsigned int i;
2202 : struct vr_packet_node *pnode;
2203 :
2204 28 : for (i = 0; i < VR_MAX_FLOW_QUEUE_ENTRIES; i++) {
2205 21 : pnode = &vfq->vfq_pnodes[i];
2206 21 : vr_flow_flush_pnode(router, pnode, fe, fmd);
2207 : }
2208 :
2209 7 : return;
2210 : }
2211 :
2212 : static void
2213 329 : vr_flush_entry(struct vrouter *router, struct vr_flow_entry *fe,
2214 : struct vr_flow_md *flmd, struct vr_forwarding_md *fmd)
2215 : {
2216 : bool swapped;
2217 :
2218 : struct vr_flow_queue *vfq;
2219 329 : struct vr_defer_data *vdd = flmd->flmd_defer_data;
2220 : struct vr_flow_defer_data *vfdd;
2221 :
2222 329 : vfq = fe->fe_hold_list;
2223 329 : if (vfq) {
2224 5 : if (fe->fe_action == VR_FLOW_ACTION_HOLD)
2225 0 : return;
2226 :
2227 5 : swapped = vr_sync_bool_compare_and_swap_p(&fe->fe_hold_list, vfq, NULL);
2228 5 : if (swapped) {
2229 5 : __vr_flow_flush_hold_queue(router, fe, fmd, vfq);
2230 5 : if (!vdd || !vdd->vdd_data)
2231 3 : goto free_flush_queue;
2232 :
2233 2 : vfdd = (struct vr_flow_defer_data *)vdd->vdd_data;
2234 2 : vfdd->vfdd_flow_queue = vfq;
2235 : }
2236 : }
2237 :
2238 326 : return;
2239 :
2240 3 : free_flush_queue:
2241 3 : if (vfq)
2242 3 : vr_free(vfq, VR_FLOW_QUEUE_OBJECT);
2243 3 : return;
2244 : }
2245 :
2246 : static void
2247 318 : __vr_flow_work(struct vrouter *router, struct vr_flow_entry *fe,
2248 : struct vr_flow_md *flmd)
2249 : {
2250 : struct vr_forwarding_md fmd;
2251 :
2252 318 : vr_init_forwarding_md(&fmd);
2253 318 : vr_flow_set_forwarding_md(router, fe, flmd->flmd_index, &fmd);
2254 318 : vr_flush_entry(router, fe, flmd, &fmd);
2255 :
2256 318 : vr_flow_defer(flmd, fe);
2257 318 : return;
2258 : }
2259 :
2260 :
2261 : static void
2262 318 : vr_flow_work(void *arg)
2263 : {
2264 : struct vrouter *router;
2265 : struct vr_flow_entry *fe;
2266 318 : struct vr_flow_md *flmd =
2267 : (struct vr_flow_md *)arg;
2268 :
2269 318 : router = flmd->flmd_router;
2270 318 : if (!router)
2271 0 : goto exit_flush;
2272 :
2273 318 : fe = vr_flow_get_entry(router, flmd->flmd_index);
2274 318 : if (!fe)
2275 0 : goto exit_flush;
2276 :
2277 318 : __vr_flow_work(router, fe, flmd);
2278 :
2279 318 : exit_flush:
2280 318 : if (flmd->flmd_defer_data) {
2281 0 : if (flmd->flmd_defer_data->vdd_data) {
2282 0 : vr_free(flmd->flmd_defer_data->vdd_data,
2283 : VR_FLOW_DEFER_DATA_OBJECT);
2284 : }
2285 0 : vr_put_defer_data(flmd->flmd_defer_data);
2286 0 : flmd->flmd_defer_data = NULL;
2287 : }
2288 :
2289 318 : vr_free(flmd, VR_FLOW_METADATA_OBJECT);
2290 :
2291 318 : return;
2292 : }
2293 :
2294 : static void
2295 216 : vr_flow_set_mirror(struct vrouter *router, vr_flow_req *req,
2296 : struct vr_flow_entry *fe)
2297 : {
2298 216 : struct vr_mirror_entry *mirror = NULL, *sec_mirror = NULL;
2299 :
2300 216 : if (!(req->fr_flags & VR_FLOW_FLAG_MIRROR) &&
2301 214 : (fe->fe_flags & VR_FLOW_FLAG_MIRROR)) {
2302 0 : vr_flow_reset_mirror(router, fe, req->fr_index);
2303 0 : return;
2304 : }
2305 :
2306 216 : if (!(req->fr_flags & VR_FLOW_FLAG_MIRROR))
2307 214 : return;
2308 :
2309 2 : if (fe->fe_mirror_id != req->fr_mir_id) {
2310 2 : if (fe->fe_mirror_id < router->vr_max_mirror_indices) {
2311 0 : fe->fe_mirror_id = router->vr_max_mirror_indices;
2312 : }
2313 :
2314 2 : if ((unsigned int)req->fr_mir_id < router->vr_max_mirror_indices) {
2315 2 : mirror = vrouter_get_mirror(req->fr_rid, req->fr_mir_id);
2316 2 : if (mirror)
2317 2 : fe->fe_mirror_id = req->fr_mir_id;
2318 :
2319 : /* when we reached this point, we had already done all the
2320 : * sanity checks we could do. failing here will add only
2321 : * complexity to code here. so !mirror case, we will not
2322 : * handle
2323 : */
2324 : }
2325 : }
2326 :
2327 2 : if (fe->fe_sec_mirror_id != req->fr_sec_mir_id) {
2328 2 : if (fe->fe_sec_mirror_id < router->vr_max_mirror_indices) {
2329 0 : fe->fe_sec_mirror_id = router->vr_max_mirror_indices;
2330 : }
2331 :
2332 2 : if ((unsigned int)req->fr_sec_mir_id < router->vr_max_mirror_indices) {
2333 2 : sec_mirror = vrouter_get_mirror(req->fr_rid, req->fr_sec_mir_id);
2334 2 : if (sec_mirror)
2335 0 : fe->fe_sec_mirror_id = req->fr_sec_mir_id;
2336 : }
2337 : }
2338 :
2339 2 : if (req->fr_pcap_meta_data_size && req->fr_pcap_meta_data) {
2340 0 : if (fe->fe_mme) {
2341 0 : vr_mirror_meta_entry_del(router, fe->fe_mme);
2342 0 : fe->fe_mme = NULL;
2343 : }
2344 :
2345 0 : fe->fe_mme = vr_mirror_meta_entry_set(router, req->fr_index,
2346 0 : req->fr_mir_sip, req->fr_mir_sport,
2347 0 : req->fr_pcap_meta_data, req->fr_pcap_meta_data_size,
2348 0 : req->fr_mir_vrf);
2349 :
2350 0 : if (fe->fe_mme) {
2351 0 : vr_offload_flow_meta_data_set(req->fr_index,
2352 : req->fr_pcap_meta_data_size,
2353 0 : req->fr_pcap_meta_data,
2354 0 : req->fr_mir_vrf);
2355 : }
2356 : }
2357 :
2358 2 : return;
2359 : }
2360 :
2361 : void
2362 184 : vr_fill_flow_common(struct vr_flow *flowp, unsigned int nh_id,
2363 : uint8_t proto, uint16_t sport, uint16_t dport, uint8_t family,
2364 : uint8_t valid_fkey_params)
2365 : {
2366 184 : flowp->flow_nh_id = nh_id;
2367 184 : flowp->flow_family = family;
2368 184 : if (family == AF_INET)
2369 174 : flowp->flow_key_len = VR_FLOW_IPV4_HASH_SIZE;
2370 : else
2371 10 : flowp->flow_key_len = VR_FLOW_IPV6_HASH_SIZE;
2372 184 : flowp->flow_unused = 0;
2373 :
2374 184 : if (valid_fkey_params & VR_FLOW_KEY_PROTO)
2375 184 : flowp->flow_proto = proto;
2376 :
2377 184 : if (valid_fkey_params & VR_FLOW_KEY_SRC_PORT)
2378 184 : flowp->flow_sport = sport;
2379 :
2380 184 : if (valid_fkey_params & VR_FLOW_KEY_DST_PORT)
2381 184 : flowp->flow_dport = dport;
2382 :
2383 184 : return;
2384 : }
2385 :
2386 : static struct vr_flow_entry *
2387 104 : vr_add_flow(unsigned int rid, struct vr_flow *key, uint8_t type,
2388 : bool need_hold_queue, unsigned int *fe_index,
2389 : uint8_t *fe_gen_id)
2390 : {
2391 : struct vr_flow_entry *flow_e;
2392 104 : struct vrouter *router = vrouter_get(rid);
2393 :
2394 104 : struct vr_flow_entry *bucket_fe = vr_flow_bucket_first_entry(router, key);
2395 104 : vr_flow_bucket_may_lock(bucket_fe);
2396 104 : flow_e = vr_find_flow(router, key, type, fe_index);
2397 104 : if (flow_e) {
2398 0 : vr_flow_bucket_may_unlock(bucket_fe);
2399 0 : *fe_gen_id = flow_e->fe_gen_id;
2400 : /* a race between agent and dp. allow agent to handle this error */
2401 0 : return NULL;
2402 : } else {
2403 104 : flow_e = vr_flow_get_free_entry(router, key, type,
2404 : need_hold_queue, fe_index);
2405 104 : vr_flow_bucket_may_unlock(bucket_fe);
2406 : }
2407 :
2408 104 : return flow_e;
2409 : }
2410 :
2411 : static struct vr_flow_entry *
2412 104 : vr_add_flow_req(vr_flow_req *req, unsigned int *fe_index, uint8_t *fe_gen_id)
2413 : {
2414 : uint8_t type;
2415 104 : bool need_hold_queue = false;
2416 :
2417 : struct vr_flow key;
2418 : struct vr_flow_entry *fe;
2419 :
2420 104 : switch (req->fr_family) {
2421 6 : case AF_INET6:
2422 6 : type = VP_TYPE_IP6;
2423 6 : vr_inet6_fill_flow_from_req(&key, req);
2424 6 : break;
2425 :
2426 98 : case AF_INET:
2427 98 : type = VP_TYPE_IP;
2428 98 : vr_inet_fill_flow(&key, req->fr_flow_nh_id,
2429 98 : (uint32_t)req->fr_flow_sip_l, (uint32_t)req->fr_flow_dip_l,
2430 98 : req->fr_flow_proto, req->fr_flow_sport, req->fr_flow_dport,
2431 : VR_FLOW_KEY_ALL);
2432 98 : break;
2433 :
2434 0 : default:
2435 0 : return NULL;
2436 : }
2437 :
2438 104 : if (req->fr_action == VR_FLOW_ACTION_HOLD)
2439 0 : need_hold_queue = true;
2440 :
2441 104 : fe = vr_add_flow(req->fr_rid, &key, type, need_hold_queue, fe_index,
2442 : fe_gen_id);
2443 104 : if (fe)
2444 104 : req->fr_index = *fe_index;
2445 :
2446 104 : return fe;
2447 : }
2448 :
2449 : /*
2450 : * can be called with 'fe' as null (specifically when flow is added from
2451 : * agent), in which case we should be checking only the request
2452 : */
2453 : static int
2454 322 : vr_flow_set_req_is_invalid(struct vrouter *router, vr_flow_req *req,
2455 : struct vr_flow_entry *fe)
2456 : {
2457 322 : int error = 0, key_type;
2458 : struct vr_flow_entry *rfe;
2459 : struct vr_flow key;
2460 : uint64_t *ip;
2461 :
2462 322 : if (fe) {
2463 :
2464 : /* If Delete marked, dont allow any other change */
2465 210 : if (fe->fe_flags & VR_FLOW_FLAG_DELETE_MARKED)
2466 0 : return -EINVAL;
2467 :
2468 210 : if ((fe->fe_type == VP_TYPE_IP) || (fe->fe_type == VP_TYPE_IP6)) {
2469 210 : if ((uint8_t)req->fr_gen_id != fe->fe_gen_id) {
2470 0 : error = -EBADF;
2471 0 : goto invalid_req;
2472 : }
2473 :
2474 : /*
2475 : * when gen id is same flow keys should not mis-match
2476 : * send EFAULT if such incident happens
2477 : */
2478 210 : if((unsigned short)req->fr_flow_sport != fe->fe_key.flow_sport ||
2479 210 : (unsigned short)req->fr_flow_dport != fe->fe_key.flow_dport||
2480 210 : (uint32_t) req->fr_flow_nh_id != (uint32_t) fe->fe_key.flow_nh_id ||
2481 210 : (unsigned char)req->fr_flow_proto != fe->fe_key.flow_proto) {
2482 0 : error = -EFAULT;
2483 0 : goto invalid_req;
2484 : }
2485 :
2486 210 : if (fe->fe_type == VP_TYPE_IP) {
2487 200 : if ((fe->fe_key.flow4_sip != (uint32_t)req->fr_flow_sip_l) ||
2488 200 : (fe->fe_key.flow4_dip != (uint32_t)req->fr_flow_dip_l)) {
2489 0 : error = -EFAULT;
2490 0 : goto invalid_req;
2491 : }
2492 : } else {
2493 10 : ip = (uint64_t *)fe->fe_key.flow6_sip;
2494 10 : if ((*ip != req->fr_flow_sip_u) ||
2495 10 : (*(ip+1) != req->fr_flow_sip_l) ||
2496 10 : (*(ip+2) != req->fr_flow_dip_u) ||
2497 10 : (*(ip+3) != req->fr_flow_dip_l)) {
2498 0 : error = -EFAULT;
2499 0 : goto invalid_req;
2500 : }
2501 : }
2502 : }
2503 : } else {
2504 : /*
2505 : * flow set request received with an index which is
2506 : * not active anymore, return ENOENT error
2507 : */
2508 112 : if ((req->fr_flags & VR_FLOW_FLAG_ACTIVE) && !(req->fr_index < 0)) {
2509 0 : error = -ENOENT;
2510 0 : goto invalid_req;
2511 : }
2512 : }
2513 :
2514 322 : if (req->fr_flags & VR_FLOW_FLAG_VRFT) {
2515 12 : if ((unsigned short)req->fr_flow_dvrf >= router->vr_max_vrfs) {
2516 0 : error = -EINVAL;
2517 0 : goto invalid_req;
2518 : }
2519 : }
2520 :
2521 322 : if (req->fr_flags & VR_FLOW_FLAG_MIRROR) {
2522 2 : if (((unsigned int)req->fr_mir_id >= router->vr_max_mirror_indices) &&
2523 0 : (unsigned int)req->fr_sec_mir_id >= router->vr_max_mirror_indices) {
2524 0 : error = -EINVAL;
2525 0 : goto invalid_req;
2526 : }
2527 : }
2528 :
2529 322 : if (req->fr_flags & VR_RFLOW_VALID) {
2530 160 : if (req->fr_rindex != -1) {
2531 160 : rfe = vr_flow_get_entry(router, req->fr_rindex);
2532 : } else {
2533 0 : if (req->fr_family == AF_INET) {
2534 0 : vr_inet_fill_flow(&key, req->fr_rflow_nh_id,
2535 0 : (uint32_t)req->fr_rflow_sip_l, (uint32_t)req->fr_rflow_dip_l,
2536 0 : req->fr_flow_proto, req->fr_rflow_sport,
2537 0 : req->fr_rflow_dport, VR_FLOW_KEY_ALL);
2538 :
2539 0 : key_type = VP_TYPE_IP;
2540 : } else {
2541 0 : vr_inet6_fill_rflow_from_req(&key, req);
2542 0 : key_type = VP_TYPE_IP6;
2543 : }
2544 :
2545 0 : rfe = vr_find_flow(router, &key, key_type, &req->fr_rindex);
2546 : }
2547 :
2548 160 : if (!rfe) {
2549 0 : error = -EINVAL;
2550 0 : goto invalid_req;
2551 : }
2552 : }
2553 :
2554 322 : return 0;
2555 :
2556 0 : invalid_req:
2557 0 : return error;
2558 : }
2559 :
2560 : static int
2561 318 : __vr_flow_schedule_transition(struct vrouter *router, struct vr_flow_entry *fe,
2562 : unsigned int index, unsigned short flags)
2563 : {
2564 : struct vr_flow_md *flmd;
2565 318 : struct vr_defer_data *defer = NULL;
2566 318 : struct vr_flow_entry *rfe = NULL;
2567 :
2568 318 : flmd = (struct vr_flow_md *)vr_malloc(sizeof(*flmd),
2569 : VR_FLOW_METADATA_OBJECT);
2570 318 : if (!flmd)
2571 0 : return -ENOMEM;
2572 :
2573 318 : flmd->flmd_router = router;
2574 318 : flmd->flmd_index = index;
2575 318 : flmd->flmd_flags = flags;
2576 318 : if (fe->fe_hold_list || (flags & VR_FLOW_FLAG_EVICT_CANDIDATE)) {
2577 6 : defer = vr_get_defer_data(sizeof(*defer));
2578 6 : if (defer) {
2579 6 : defer->vdd_data = (void *)vr_zalloc(sizeof(struct vr_flow_defer_data),
2580 : VR_FLOW_DEFER_DATA_OBJECT);
2581 6 : if (!(flmd->flmd_flags & VR_FLOW_FLAG_ACTIVE)) {
2582 0 : ((struct vr_flow_defer_data *)defer->vdd_data)->vfdd_delete =
2583 : true;
2584 : }
2585 : /*
2586 : * Set vfdd_evict_flow to true only if there is only one flow
2587 : * or both flows have EVICT_CANDIDATE flag set;
2588 : * This is to avoid a race condition which can lead to non
2589 : * eviction of one of the flows. See CEM-4275 for more details.
2590 : *
2591 : * CEM-18166: Handle a corner case for BGPaaS where two different
2592 : * flows are trapped to agent (due to NAT translation to the same
2593 : * port), combined with TCP RST/FIN pkts received on the reverse
2594 : * flow. The sequence of events permits deletion of the FF, while
2595 : * RF still pointing to it. When the Controller tries to reset the
2596 : * connection, thus causing eviction the flow, vrouter will fail
2597 : * because rfe for the RF is NULL.
2598 : */
2599 6 : if ((fe->fe_flags & VR_FLOW_FLAG_EVICT_CANDIDATE)) {
2600 8 : if ((fe->fe_rflow < 0) ||
2601 4 : ((rfe = vr_flow_get_entry(router, fe->fe_rflow)) &&
2602 4 : (rfe->fe_flags & VR_FLOW_FLAG_EVICT_CANDIDATE)) || !rfe) {
2603 4 : ((struct vr_flow_defer_data *)defer->vdd_data)->vfdd_evict_flow =
2604 : true;
2605 : }
2606 : }
2607 : }
2608 : }
2609 318 : flmd->flmd_defer_data = defer;
2610 :
2611 318 : return vr_schedule_work(vr_get_cpu(), vr_flow_work, (void *)flmd);
2612 : }
2613 :
2614 : static int
2615 314 : vr_flow_schedule_transition(struct vrouter *router, vr_flow_req *req,
2616 : struct vr_flow_entry *fe)
2617 : {
2618 314 : return __vr_flow_schedule_transition(router, fe, req->fr_index, req->fr_flags);
2619 : }
2620 :
2621 : static int
2622 98 : vr_flow_delete(struct vrouter *router, vr_flow_req *req,
2623 : struct vr_flow_entry *fe)
2624 : {
2625 98 : int port = 0;
2626 : /* Delete Mark it */
2627 98 : fe->fe_flags |= VR_FLOW_FLAG_DELETE_MARKED;
2628 :
2629 :
2630 98 : if (fe->fe_flags & VR_FLOW_FLAG_LINK_LOCAL) {
2631 3 : if (fe->fe_key.flow_proto == VR_IP_PROTO_ICMP) {
2632 : /*
2633 : * ICMP id passed as source port would be
2634 : * used for relaxed policy flow lookup
2635 : */
2636 1 : port = ntohs(fe->fe_key.flow_sport);
2637 : } else {
2638 2 : port = ntohs(fe->fe_key.flow_dport);
2639 : }
2640 3 : vr_clear_link_local_port(router, AF_INET, fe->fe_key.flow_proto, port);
2641 : }
2642 :
2643 98 : fe->fe_action = VR_FLOW_ACTION_DROP;
2644 98 : vr_flow_reset_mirror(router, fe, req->fr_index);
2645 98 : return vr_flow_schedule_transition(router, req, fe);
2646 : }
2647 :
2648 : static void
2649 216 : vr_flow_udp_src_port (struct vrouter *router, struct vr_flow_entry *fe)
2650 : {
2651 : uint32_t hash_key[10], hashval, port_range, hash_len;
2652 : uint16_t port;
2653 :
2654 216 : if (fe->fe_udp_src_port)
2655 110 : return;
2656 :
2657 106 : if (hashrnd_inited == 0) {
2658 33 : get_random_bytes(&vr_hashrnd, sizeof(vr_hashrnd));
2659 33 : hashrnd_inited = 1;
2660 : }
2661 :
2662 106 : hash_key[0] = fe->fe_vrf;
2663 106 : hash_key[1] = (fe->fe_key.flow_sport << 16) | fe->fe_key.flow_dport;
2664 106 : memcpy(&hash_key[2], fe->fe_key.flow_ip, 2 * VR_IP_ADDR_SIZE(fe->fe_type));
2665 106 : hash_len = VR_FLOW_HASH_SIZE(fe->fe_type);
2666 :
2667 106 : hashval = vr_hash(hash_key, hash_len, vr_hashrnd);
2668 106 : port_range = VR_MUDP_PORT_RANGE_END - VR_MUDP_PORT_RANGE_START;
2669 106 : port = (uint16_t ) (((uint64_t ) hashval * port_range) >> 32);
2670 :
2671 106 : if (port > port_range) {
2672 : /*
2673 : * Shouldn't happen...
2674 : */
2675 0 : port = 0;
2676 : }
2677 106 : fe->fe_udp_src_port = port + VR_MUDP_PORT_RANGE_START;
2678 : }
2679 :
2680 : static void
2681 216 : vr_flow_update_link_local_port(struct vrouter *router, vr_flow_req *req,
2682 : struct vr_flow_entry *fe)
2683 : {
2684 216 : bool set_port = false;
2685 216 : int port = 0;
2686 :
2687 216 : if (!req || !fe)
2688 0 : return;
2689 :
2690 216 : if (fe->fe_type != VP_TYPE_IP)
2691 10 : return;
2692 :
2693 206 : if (fe->fe_key.flow_proto == VR_IP_PROTO_ICMP) {
2694 : /*
2695 : * ICMP id passed as source port would be
2696 : * used for relaxed policy flow lookup
2697 : */
2698 173 : port = ntohs(fe->fe_key.flow_sport);
2699 : } else {
2700 33 : port = ntohs(fe->fe_key.flow_dport);
2701 : }
2702 :
2703 206 : if (req->fr_flags & VR_FLOW_FLAG_LINK_LOCAL) {
2704 4 : if (!(fe->fe_flags & VR_FLOW_FLAG_LINK_LOCAL))
2705 2 : set_port = true;
2706 202 : } else if (fe->fe_flags & VR_FLOW_FLAG_LINK_LOCAL) {
2707 0 : vr_clear_link_local_port(router, AF_INET, fe->fe_key.flow_proto,
2708 : port);
2709 : }
2710 :
2711 206 : if (req->fr_flags & VR_FLOW_BGP_SERVICE) {
2712 0 : if (!(fe->fe_flags & VR_FLOW_BGP_SERVICE))
2713 0 : set_port = true;
2714 : }
2715 :
2716 206 : if (set_port) {
2717 2 : vr_set_link_local_port(router, AF_INET, fe->fe_key.flow_proto, port);
2718 : }
2719 :
2720 206 : return;
2721 : }
2722 :
2723 : static int
2724 0 : vr_flow_force_evict (struct vrouter *router, vr_flow_req *req)
2725 : {
2726 0 : struct vr_flow_entry *fe = NULL;
2727 :
2728 0 : fe = vr_flow_get_entry(router, req->fr_index);
2729 0 : if (fe) {
2730 : /* Do force eviction only for TCP flows with dead flag set */
2731 0 : if (fe->fe_key.flow_proto != VR_IP_PROTO_TCP) {
2732 0 : return -EINVAL;
2733 : }
2734 0 : if (!(fe->fe_tcp_flags & VR_FLOW_TCP_DEAD)) {
2735 0 : return -EINVAL;
2736 : }
2737 0 : if (!(fe->fe_flags & VR_FLOW_FLAG_ACTIVE)) {
2738 0 : return -EINVAL;
2739 : }
2740 : /*
2741 : * We have a TCP flow which is dead,
2742 : * Evict Candidate may or may not be set
2743 : */
2744 0 : vr_flow_start_modify(router, fe);
2745 : /*
2746 : * Reset flow's reverse flow index as we don't want to
2747 : * touch reverse flow
2748 : */
2749 0 : fe->fe_flags &= ~VR_RFLOW_VALID;
2750 0 : fe->fe_rflow = -1;
2751 0 : if (!(fe->fe_flags & VR_FLOW_FLAG_EVICT_CANDIDATE)) {
2752 0 : __vr_flow_mark_evict(router, fe);
2753 : }
2754 0 : __vr_flow_schedule_transition(router, fe, req->fr_index, fe->fe_flags);
2755 : } else {
2756 0 : return -EINVAL;
2757 : }
2758 0 : return 0;
2759 : }
2760 :
2761 :
2762 : /* command from agent */
2763 : static int
2764 322 : vr_flow_set(struct vrouter *router, vr_flow_req *req,
2765 : vr_flow_response *flow_resp)
2766 0 : {
2767 : int ret;
2768 : int8_t cur_fe_underlay_ecmp_index;
2769 322 : unsigned int fe_index = (unsigned int)-1;
2770 322 : uint8_t fe_gen_id = 0;
2771 322 : bool new_flow = false, modified = false;
2772 :
2773 322 : struct vr_flow_entry *fe = NULL, *rfe = NULL;
2774 322 : struct vr_flow_table_info *infop = router->vr_flow_table_info;
2775 :
2776 322 : router = vrouter_get(req->fr_rid);
2777 322 : if (!router)
2778 0 : return -EINVAL;
2779 :
2780 322 : flow_resp->fresp_index = req->fr_index;
2781 :
2782 322 : if (req->fr_extflags & VR_FLOW_EXT_FLAG_FORCE_EVICT) {
2783 0 : return vr_flow_force_evict(router, req);
2784 : }
2785 :
2786 322 : fe = vr_flow_get_entry(router, req->fr_index);
2787 322 : if (fe) {
2788 210 : if (!(modified = vr_flow_start_modify(router, fe)))
2789 0 : return -EBUSY;
2790 210 : fe_index = (unsigned int)(req->fr_index);
2791 : }
2792 :
2793 322 : if ((ret = vr_flow_set_req_is_invalid(router, req, fe)))
2794 0 : goto exit_set;
2795 :
2796 322 : if (fe) {
2797 210 : if ((fe->fe_action == VR_FLOW_ACTION_HOLD) &&
2798 2 : ((req->fr_action != fe->fe_action) ||
2799 0 : !(req->fr_flags & VR_FLOW_FLAG_ACTIVE))) {
2800 2 : vr_sync_fetch_and_add_64u(&infop->vfti_action_count, 1);
2801 : } else {
2802 208 : infop->vfti_changed++;
2803 : }
2804 :
2805 : }
2806 : /*
2807 : * for delete, absence of the requested flow entry is caustic. so
2808 : * handle that case first
2809 : */
2810 322 : if (!(req->fr_flags & VR_FLOW_FLAG_ACTIVE)) {
2811 106 : if (!fe)
2812 8 : return -ENOENT;
2813 :
2814 98 : infop->vfti_deleted++;
2815 98 : flow_resp->fresp_flags |= VR_FLOW_RESP_FLAG_DELETED;
2816 98 : return vr_flow_delete(router, req, fe);
2817 : }
2818 :
2819 :
2820 : /*
2821 : * for non-delete cases, absence of flow entry means addition of a
2822 : * new flow entry with the key specified in the request
2823 : */
2824 216 : if (!fe) {
2825 104 : fe = vr_add_flow_req(req, &fe_index, &fe_gen_id);
2826 104 : if (!fe) {
2827 0 : if (fe_index != (unsigned int)-1) {
2828 : /*
2829 : * add flow req failed to allocate an entry due to race
2830 : * between agent and datapath, where flow entry at fe_index
2831 : * was already created due to packet trap, return EEXIST
2832 : * error and allow agent to wait and handle flow add due to
2833 : * packet trap
2834 : */
2835 0 : flow_resp->fresp_index = fe_index;
2836 0 : flow_resp->fresp_gen_id = fe_gen_id;
2837 0 : return -EEXIST;
2838 : }
2839 0 : return -ENOSPC;
2840 : }
2841 :
2842 104 : new_flow = true;
2843 104 : infop->vfti_added++;
2844 : } else {
2845 112 : if ((req->fr_action == VR_FLOW_ACTION_HOLD) &&
2846 0 : (fe->fe_action != req->fr_action)) {
2847 0 : if (!fe->fe_hold_list) {
2848 0 : fe->fe_hold_list = vr_zalloc(sizeof(struct vr_flow_queue),
2849 : VR_FLOW_QUEUE_OBJECT);
2850 0 : if (!fe->fe_hold_list) {
2851 0 : ret = -ENOMEM;
2852 0 : goto exit_set;
2853 : }
2854 : }
2855 : }
2856 : }
2857 :
2858 216 : flow_resp->fresp_gen_id = fe->fe_gen_id;
2859 216 : flow_resp->fresp_index = fe->fe_hentry.hentry_index;
2860 :
2861 216 : vr_flow_set_mirror(router, req, fe);
2862 :
2863 216 : if (req->fr_flags & VR_RFLOW_VALID) {
2864 160 : fe->fe_rflow = req->fr_rindex;
2865 : } else {
2866 56 : if (fe->fe_rflow >= 0)
2867 0 : fe->fe_rflow = -1;
2868 : }
2869 :
2870 216 : fe->fe_vrf = req->fr_flow_vrf;
2871 216 : if (req->fr_flags & VR_FLOW_FLAG_VRFT)
2872 12 : fe->fe_dvrf = req->fr_flow_dvrf;
2873 :
2874 216 : vr_flow_update_link_local_port(router, req, fe);
2875 :
2876 216 : if (fe->fe_ecmp_nh_index == -1)
2877 216 : (void)vr_sync_bool_compare_and_swap_8s(&fe->fe_ecmp_nh_index, -1,
2878 : req->fr_ecmp_nh_index);
2879 :
2880 216 : fe->fe_src_nh_index = req->fr_src_nh_index;
2881 216 : fe->fe_qos_id = req->fr_qos_id;
2882 :
2883 216 : if ((req->fr_action == VR_FLOW_ACTION_HOLD) &&
2884 0 : (fe->fe_action != VR_FLOW_ACTION_HOLD)) {
2885 0 : vr_flow_entry_set_hold(router, fe, false);
2886 : } else {
2887 216 : fe->fe_action = req->fr_action;
2888 : }
2889 :
2890 216 : fe->fe_ttl = req->fr_ttl;
2891 :
2892 216 : cur_fe_underlay_ecmp_index = req->fr_underlay_ecmp_index;
2893 : /* repeats until the comapre and swap operation is successful */
2894 216 : while(!vr_sync_bool_compare_and_swap_8s(&fe->fe_underlay_ecmp_index,
2895 : fe->fe_underlay_ecmp_index, cur_fe_underlay_ecmp_index)) {
2896 0 : continue;
2897 : }
2898 :
2899 216 : if (fe->fe_action == VR_FLOW_ACTION_DROP)
2900 0 : fe->fe_drop_reason = (uint8_t)req->fr_drop_reason;
2901 :
2902 216 : fe->fe_flags = VR_FLOW_FLAG_DP_BITS(fe) |
2903 216 : VR_FLOW_FLAG_MASK(req->fr_flags);
2904 216 : fe->fe_flags1 = req->fr_flags1;
2905 216 : if (new_flow) {
2906 :
2907 104 : flow_resp->fresp_bytes = fe->fe_stats.flow_bytes;
2908 104 : flow_resp->fresp_packets = fe->fe_stats.flow_packets;
2909 104 : flow_resp->fresp_stats_oflow = (fe->fe_stats.flow_bytes_oflow |
2910 104 : (fe->fe_stats.flow_packets_oflow << 16));
2911 :
2912 104 : if (fe->fe_flags & VR_FLOW_FLAG_NEW_FLOW) {
2913 104 : if (fe->fe_stats.flow_packets || fe->fe_stats.flow_packets_oflow)
2914 0 : memset(&fe->fe_stats, 0, sizeof(fe->fe_stats));
2915 : }
2916 :
2917 104 : if (fe->fe_flags & VR_RFLOW_VALID) {
2918 49 : rfe = vr_flow_get_entry(router, fe->fe_rflow);
2919 49 : if (rfe) {
2920 49 : vr_flow_tcp_rflow_set(router, fe, rfe);
2921 : }
2922 : }
2923 :
2924 : }
2925 :
2926 216 : vr_flow_udp_src_port(router, fe);
2927 :
2928 : /* Mock Src UDP port used for vrouter simulation - vtest */
2929 216 : if(req->fr_extflags & VR_FLOW_EXT_FLAG_MOCK_SRC_UDP)
2930 2 : fe->fe_udp_src_port = VR_FLOW_MOCK_SRC_UDP_PORT;
2931 :
2932 216 : if (fe->fe_flags & VR_FLOW_FLAG_NEW_FLOW)
2933 104 : fe->fe_flags &= ~VR_FLOW_FLAG_NEW_FLOW;
2934 :
2935 :
2936 :
2937 216 : ret = vr_flow_schedule_transition(router, req, fe);
2938 :
2939 : /*
2940 : * offload, no need to differentiate between add and modify. Pass the
2941 : * reverse flow as well if present.
2942 : */
2943 216 : if (!ret) {
2944 216 : vr_offload_flow_set(fe, fe_index, rfe);
2945 : }
2946 :
2947 0 : exit_set:
2948 216 : if (modified && fe) {
2949 112 : vr_flow_stop_modify(router, fe);
2950 : }
2951 :
2952 216 : return ret;
2953 : }
2954 :
2955 : static void
2956 13 : vr_flow_table_data_destroy(vr_flow_table_data *ftable)
2957 : {
2958 13 : if (!ftable)
2959 0 : return;
2960 :
2961 13 : if (ftable->ftable_file_path) {
2962 13 : vr_free(ftable->ftable_file_path, VR_FLOW_REQ_PATH_OBJECT);
2963 13 : ftable->ftable_file_path = NULL;
2964 : }
2965 :
2966 13 : if (ftable->ftable_hold_stat && ftable->ftable_hold_stat_size) {
2967 13 : vr_free(ftable->ftable_hold_stat, VR_FLOW_HOLD_STAT_OBJECT);
2968 13 : ftable->ftable_hold_stat = NULL;
2969 13 : ftable->ftable_hold_stat_size = 0;
2970 : }
2971 :
2972 13 : vr_free(ftable, VR_FLOW_TABLE_DATA_OBJECT);
2973 :
2974 13 : return;
2975 : }
2976 :
2977 : static vr_flow_table_data *
2978 13 : vr_flow_table_data_get(vr_flow_table_data *ref)
2979 : {
2980 : unsigned int hold_stat_size;
2981 13 : unsigned int num_cpus = vr_num_cpus;
2982 13 : vr_flow_table_data *ftable = vr_zalloc(sizeof(*ref),
2983 : VR_FLOW_TABLE_DATA_OBJECT);
2984 :
2985 13 : if (!ftable)
2986 0 : return NULL;
2987 :
2988 13 : if (vr_flow_path) {
2989 13 : ftable->ftable_file_path = vr_zalloc(VR_UNIX_PATH_MAX,
2990 : VR_FLOW_REQ_PATH_OBJECT);
2991 13 : if (!ftable->ftable_file_path) {
2992 0 : vr_free(ftable, VR_FLOW_TABLE_DATA_OBJECT);
2993 0 : return NULL;
2994 : }
2995 : }
2996 :
2997 13 : if (num_cpus > VR_FLOW_MAX_CPUS)
2998 0 : num_cpus = VR_FLOW_MAX_CPUS;
2999 :
3000 13 : hold_stat_size = num_cpus * sizeof(uint32_t);
3001 13 : ftable->ftable_hold_stat = vr_zalloc(hold_stat_size, VR_FLOW_HOLD_STAT_OBJECT);
3002 13 : if (!ftable->ftable_hold_stat) {
3003 0 : if (ftable->ftable_file_path) {
3004 0 : vr_free(ftable->ftable_file_path, VR_FLOW_REQ_PATH_OBJECT);
3005 0 : ftable->ftable_file_path = NULL;
3006 : }
3007 :
3008 0 : vr_free(ftable, VR_FLOW_TABLE_DATA_OBJECT);
3009 0 : return NULL;
3010 : }
3011 13 : ftable->ftable_hold_stat_size = num_cpus;
3012 :
3013 13 : return ftable;
3014 : }
3015 :
3016 : void
3017 8186880 : update_flow_entry(vr_htable_t table __attribute__unused__, vr_hentry_t *ent ,
3018 : unsigned int index, void *data __attribute__unused__)
3019 : {
3020 8186880 : struct vr_flow_entry *fe = (struct vr_flow_entry *)ent;
3021 :
3022 8186880 : if (fe == NULL)
3023 8186865 : return;
3024 :
3025 15 : vr_offload_flow_stats_update(fe);
3026 : }
3027 :
3028 : /*
3029 : * sandesh handler for vr_flow_table_data
3030 : */
3031 : void
3032 13 : vr_flow_table_data_process(void *s_req)
3033 : {
3034 13 : int i, ret = 0;
3035 13 : uint64_t hold_count = 0;
3036 : struct vrouter *router;
3037 : struct vr_flow_table_info *infop;
3038 13 : vr_flow_table_data *resp = NULL, *ftable = (vr_flow_table_data *)s_req;
3039 :
3040 13 : if (!ftable) {
3041 0 : ret = -ENOMEM;
3042 0 : goto send_response;
3043 : }
3044 13 : router = vrouter_get(ftable->ftable_rid);
3045 13 : if (! router) {
3046 0 : ret = -ENOMEM;
3047 0 : goto send_response;
3048 : }
3049 :
3050 13 : if(EINVAL == vr_htable_trav(router->vr_flow_table, 0, update_flow_entry, NULL))
3051 : {
3052 0 : ret = -ENOMEM;
3053 0 : goto send_response;
3054 : }
3055 13 : resp = vr_flow_table_data_get(ftable);
3056 13 : if (!resp) {
3057 0 : ret = -ENOMEM;
3058 0 : goto send_response;
3059 : }
3060 :
3061 13 : infop = router->vr_flow_table_info;
3062 13 : resp->ftable_op = ftable->ftable_op;
3063 13 : resp->ftable_size = vr_flow_table_size(router);
3064 : #if defined(__linux__) && defined(__KERNEL__)
3065 : resp->ftable_dev = vr_flow_major;
3066 : #endif
3067 13 : if (vr_flow_path) {
3068 13 : strncpy(resp->ftable_file_path, vr_flow_path, VR_UNIX_PATH_MAX - 1);
3069 : }
3070 :
3071 13 : if (!infop)
3072 0 : goto send_response;
3073 :
3074 13 : resp->ftable_used_entries = vr_flow_table_used_total_entries(router);
3075 13 : resp->ftable_deleted = infop->vfti_deleted;
3076 13 : resp->ftable_changed = infop->vfti_changed;
3077 13 : resp->ftable_processed = infop->vfti_action_count;
3078 13 : resp->ftable_hold_oflows = infop->vfti_oflows;
3079 13 : resp->ftable_added = infop->vfti_added;
3080 13 : resp->ftable_cpus = vr_num_cpus;
3081 : /* we only have space for 64 stats block max when encoding */
3082 169 : for (i = 0; ((i < vr_num_cpus) && (i < VR_FLOW_MAX_CPUS)); i++) {
3083 156 : resp->ftable_hold_stat[i] = infop->vfti_hold_count[i];
3084 156 : hold_count += resp->ftable_hold_stat[i];
3085 : }
3086 :
3087 13 : resp->ftable_created = hold_count;
3088 13 : resp->ftable_oflow_entries = vr_flow_table_used_oflow_entries(router);
3089 13 : resp->ftable_burst_free_tokens = infop->vfti_burst_tokens - infop->vfti_burst_used;
3090 13 : resp->ftable_hold_entries = vr_flow_table_hold_count(router);
3091 :
3092 13 : send_response:
3093 13 : vr_message_response(VR_FLOW_TABLE_DATA_OBJECT_ID, resp, ret, false);
3094 13 : if (resp)
3095 13 : vr_flow_table_data_destroy(resp);
3096 :
3097 13 : return;
3098 : }
3099 :
3100 : /*
3101 : * sandesh handler for vr_flow_req
3102 : */
3103 : void
3104 322 : vr_flow_req_process(void *s_req)
3105 : {
3106 322 : int ret = 0;
3107 : struct vrouter *router;
3108 322 : vr_flow_req *req = (vr_flow_req *)s_req;
3109 : vr_flow_response flow_resp;
3110 :
3111 322 : router = vrouter_get(req->fr_rid);
3112 322 : switch (req->fr_op) {
3113 322 : case FLOW_OP_FLOW_SET:
3114 :
3115 322 : flow_resp.fresp_rid = 0;
3116 322 : flow_resp.fresp_op = req->fr_op;
3117 :
3118 322 : ret = vr_flow_set(router, req, &flow_resp);
3119 322 : break;
3120 :
3121 0 : default:
3122 0 : ret = -EINVAL;
3123 : }
3124 :
3125 322 : vr_message_response(VR_FLOW_RESPONSE_OBJECT_ID, &flow_resp, ret, false);
3126 :
3127 322 : return;
3128 : }
3129 :
3130 : void
3131 0 : vr_flow_response_process(void *s_req)
3132 : {
3133 0 : return;
3134 : }
3135 :
3136 : static void
3137 53 : vr_flow_table_info_destroy(struct vrouter *router)
3138 : {
3139 53 : if (!router->vr_flow_table_info)
3140 0 : return;
3141 :
3142 53 : vr_free(router->vr_flow_table_info, VR_FLOW_TABLE_INFO_OBJECT);
3143 53 : router->vr_flow_table_info = NULL;
3144 53 : router->vr_flow_table_info_size = 0;
3145 :
3146 53 : return;
3147 : }
3148 :
3149 : static void
3150 53 : vr_flow_table_info_reset(struct vrouter *router)
3151 : {
3152 53 : if (!router->vr_flow_table_info)
3153 0 : return;
3154 :
3155 53 : if (router->vr_flow_table_info->vfti_timer) {
3156 0 : vr_delete_timer(router->vr_flow_table_info->vfti_timer);
3157 0 : vr_free(router->vr_flow_table_info->vfti_timer, VR_TIMER_OBJECT);
3158 0 : router->vr_flow_table_info->vfti_timer = NULL;
3159 : }
3160 :
3161 53 : memset(router->vr_flow_table_info, 0, router->vr_flow_table_info_size);
3162 :
3163 53 : return;
3164 : }
3165 :
3166 : static int
3167 53 : vr_flow_table_info_init(struct vrouter *router)
3168 : {
3169 : unsigned int size;
3170 : struct vr_flow_table_info *infop;
3171 :
3172 53 : if (router->vr_flow_table_info)
3173 0 : return 0;
3174 :
3175 53 : size = sizeof(struct vr_flow_table_info) + sizeof(uint32_t) * vr_num_cpus;
3176 53 : infop = (struct vr_flow_table_info *)vr_zalloc(size,
3177 : VR_FLOW_TABLE_INFO_OBJECT);
3178 53 : if (!infop)
3179 0 : return vr_module_error(-ENOMEM, __FUNCTION__, __LINE__, size);
3180 :
3181 53 : router->vr_flow_table_info = infop;
3182 53 : router->vr_flow_table_info_size = size;
3183 :
3184 53 : return 0;
3185 : }
3186 :
3187 : static void
3188 53 : vr_flow_table_destroy(struct vrouter *router)
3189 : {
3190 53 : if (router->vr_flow_table) {
3191 53 : vr_htable_delete(router->vr_flow_table);
3192 53 : router->vr_flow_table = NULL;
3193 : }
3194 :
3195 53 : vr_flow_table_info_destroy(router);
3196 :
3197 53 : return;
3198 : }
3199 :
3200 : static void
3201 33377280 : vr_flow_invalidate_entry(vr_htable_t htable, vr_hentry_t *ent,
3202 : unsigned int index, void *data)
3203 : {
3204 : struct vr_flow_entry *fe;
3205 : struct vr_forwarding_md fmd;
3206 : struct vr_flow_md flmd;
3207 33377280 : struct vrouter *router = (struct vrouter *)data;
3208 :
3209 33377280 : if (!ent || !data)
3210 33377269 : return;
3211 :
3212 33377280 : fe = CONTAINER_OF(fe_hentry, struct vr_flow_entry, ent);
3213 33377280 : if (!(fe->fe_flags & VR_FLOW_FLAG_ACTIVE))
3214 33377269 : return;
3215 :
3216 11 : flmd.flmd_defer_data = NULL;
3217 11 : flmd.flmd_index = index;
3218 11 : flmd.flmd_flags = fe->fe_flags;
3219 :
3220 11 : vr_init_forwarding_md(&fmd);
3221 :
3222 11 : fe->fe_action = VR_FLOW_ACTION_DROP;
3223 11 : vr_flush_entry(router, fe, &flmd, &fmd);
3224 11 : vr_flow_reset_entry(router, fe);
3225 : }
3226 :
3227 : static void
3228 53 : vr_flow_table_reset(struct vrouter *router)
3229 : {
3230 53 : vr_htable_reset(router->vr_flow_table,
3231 : vr_flow_invalidate_entry, router);
3232 53 : vr_flow_table_info_reset(router);
3233 :
3234 53 : return;
3235 : }
3236 :
3237 : void
3238 66 : vr_compute_size_oflow_table(void)
3239 : {
3240 : /*
3241 : * Overflow entries is 20% of the main flow table
3242 : * adjusted to next 1k
3243 : */
3244 66 : if (!vr_oflow_entries) {
3245 0 : vr_oflow_entries = ((vr_flow_entries / 5) + 1023) & ~1023;
3246 : }
3247 66 : }
3248 :
3249 : static int
3250 53 : vr_flow_table_init(struct vrouter *router)
3251 : {
3252 53 : if (!router->vr_flow_table) {
3253 :
3254 53 : vr_compute_size_oflow_table();
3255 :
3256 53 : if (!vr_flow_table && vr_huge_page_mem_get) {
3257 :
3258 0 : vr_flow_table = vr_huge_page_mem_get(VR_FLOW_TABLE_SIZE +
3259 : VR_OFLOW_TABLE_SIZE, &vr_flow_path);
3260 0 : if (vr_flow_table)
3261 0 : vr_oflow_table = (char*)vr_flow_table + VR_FLOW_TABLE_SIZE;
3262 : }
3263 :
3264 53 : router->vr_flow_table = vr_htable_attach(router, vr_flow_entries,
3265 : vr_flow_table, vr_oflow_entries, vr_oflow_table,
3266 : sizeof(struct vr_flow_entry), 0, 0, vr_flow_get_key);
3267 :
3268 53 : if (!router->vr_flow_table) {
3269 0 : return vr_module_error(-ENOMEM, __FUNCTION__,
3270 0 : __LINE__, vr_flow_entries + vr_oflow_entries);
3271 : }
3272 : }
3273 :
3274 53 : return vr_flow_table_info_init(router);
3275 : }
3276 :
3277 : static void
3278 53 : vr_link_local_ports_reset(struct vrouter *router)
3279 : {
3280 53 : if (router->vr_link_local_ports) {
3281 53 : memset(router->vr_link_local_ports,
3282 53 : 0, router->vr_link_local_ports_size);
3283 : }
3284 :
3285 53 : return;
3286 : }
3287 :
3288 : static void
3289 53 : vr_link_local_ports_exit(struct vrouter *router)
3290 : {
3291 53 : if (router->vr_link_local_ports) {
3292 53 : vr_free(router->vr_link_local_ports, VR_FLOW_LINK_LOCAL_OBJECT);
3293 53 : router->vr_link_local_ports = NULL;
3294 53 : router->vr_link_local_ports_size = 0;
3295 : }
3296 :
3297 53 : return;
3298 : }
3299 :
3300 : static int
3301 53 : vr_link_local_ports_init(struct vrouter *router)
3302 : {
3303 : unsigned int port_range, bytes;
3304 :
3305 53 : if (router->vr_link_local_ports)
3306 0 : return 0;
3307 :
3308 : /* ICMP, Udp and TCP inclusive of low and high limits*/
3309 53 : port_range = VR_LL_RP_MAX *
3310 : ((VR_DYNAMIC_PORT_END - VR_DYNAMIC_PORT_START) + 1);
3311 : /* Make it 16 bit boundary */
3312 53 : bytes = (port_range + 15) & ~15;
3313 : /* Bits to Bytes */
3314 53 : bytes /= 8;
3315 :
3316 53 : router->vr_link_local_ports = vr_zalloc(bytes, VR_FLOW_LINK_LOCAL_OBJECT);
3317 53 : if (!router->vr_link_local_ports)
3318 0 : return -1;
3319 53 : router->vr_link_local_ports_size = bytes;
3320 :
3321 53 : return 0;
3322 : }
3323 :
3324 : /* flow module exit and init */
3325 : void
3326 53 : vr_flow_exit(struct vrouter *router, bool soft_reset)
3327 : {
3328 53 : vr_flow_table_reset(router);
3329 53 : vr_link_local_ports_reset(router);
3330 53 : if (!soft_reset) {
3331 53 : vr_flow_table_destroy(router);
3332 53 : vr_fragment_table_exit(router);
3333 53 : vr_link_local_ports_exit(router);
3334 : }
3335 :
3336 53 : return;
3337 : }
3338 :
3339 : int
3340 53 : vr_flow_mem(struct vrouter *router)
3341 : {
3342 : int ret;
3343 :
3344 53 : if ((ret = vr_fragment_table_init(router)) < 0)
3345 0 : return ret;
3346 :
3347 53 : if ((ret = vr_flow_table_init(router)))
3348 0 : return ret;
3349 :
3350 53 : if ((ret = vr_link_local_ports_init(router)))
3351 0 : return ret;
3352 :
3353 53 : return 0;
3354 : }
3355 :
3356 : int
3357 53 : vr_flow_init(struct vrouter *router)
3358 : {
3359 53 : return 0;
3360 : }
3361 :
3362 : /*
3363 : * Called by offload module to update flow stats with packets which have been
3364 : * offloaded. over_flow_bytes and over_flow_packets account for overflows which
3365 : * happen in firmware between updates using this function.
3366 : */
3367 : int
3368 0 : vr_flow_incr_stats(int fe_index, uint32_t flow_bytes, uint16_t over_flow_bytes,
3369 : uint32_t flow_packets, uint8_t over_flow_packets)
3370 : {
3371 0 : struct vrouter *router = vrouter_get(0);
3372 : struct vr_flow_entry *fe;
3373 : uint32_t new_stats;
3374 :
3375 0 : if (router == NULL)
3376 0 : return -EINVAL;
3377 :
3378 0 : fe = vr_flow_get_entry(router, fe_index);
3379 0 : if (fe == NULL)
3380 0 : return -ENOENT;
3381 :
3382 0 : if (!(fe->fe_flags & VR_FLOW_FLAG_ACTIVE))
3383 0 : return -ENOENT;
3384 :
3385 0 : new_stats = vr_sync_add_and_fetch_32u(&fe->fe_stats.flow_bytes, flow_bytes);
3386 0 : if (new_stats < flow_bytes)
3387 0 : ++fe->fe_stats.flow_bytes_oflow;
3388 0 : fe->fe_stats.flow_bytes_oflow += over_flow_bytes;
3389 :
3390 0 : new_stats = vr_sync_add_and_fetch_32u(&fe->fe_stats.flow_packets,
3391 : flow_packets);
3392 0 : if (new_stats < flow_packets)
3393 0 : ++fe->fe_stats.flow_packets_oflow;
3394 0 : fe->fe_stats.flow_packets_oflow += over_flow_packets;
3395 :
3396 0 : return 0;
3397 : }
|
