Line data Source code
1 : /*
2 : * vr_fragment.c -- basic fragment handling code
3 : *
4 : * Copyright (c) 2013, Juniper Networks Private Limited
5 : * All rights reserved
6 : */
7 : #include <vr_os.h>
8 : #include <vr_packet.h>
9 : #include "vr_interface.h"
10 : #include "vr_btable.h"
11 : #include "vr_fragment.h"
12 : #include "vr_hash.h"
13 :
14 : /*
15 : * Handling out of order fragment arrival:
16 : *
17 : * Every fragment that does not have a corresponding entry in the fragment
18 : * metadata table (which is the primary table that the datapath will rely on),
19 : * will be enqueued to the assembler. The head of an ip fragment is a special
20 : * case. It will be both forwarded to the destination and enqueued to the
21 : * assembler. The assembler just extracts the required data from the packet
22 : * and will not forward the packet . The assembler will use the head of the
23 : * fragment to search and then to forward the other fragments of the packet,
24 : * while not forwarding the head itself.
25 :
26 : * Enqueue to per-cpu queue
27 : * ------------------------
28 :
29 : * Its important that locks are avoided as far as possible in datapath. To avoid
30 : * contention between multiple threads running datapath, we will have a per-cpu
31 : * queue to the assembler. Since both the assembler and the datapath will be
32 : * dequeueing from and enqueueing to the queue and frees will be involved, its
33 : * much easier and safer to enqueue the packet to the head of the queue (which
34 : * is a memory that is never freed and involves only updating the next pointer,
35 : * which can be a stale memory, but a safe operation nevertheless) rather than
36 : * at the tail (which is an element whose life cycle is not easy to determine).
37 :
38 : * Dequeue from the per-cpu queue
39 : * ------------------------------
40 :
41 : * The assembler will do an atomic read and update of head to NULL. There is a
42 : * small consistency issue to be taken care of while updating the head. The
43 : * enqueuer will first update the next pointer of the queued element and then
44 : * update the head. Updation of the head will be atomic read and update. Post
45 : * update, if the old head is not the same as the next, next will be updated to
46 : * NULL.
47 :
48 : * The Assembler
49 : * -------------
50 :
51 : * The assembler will be (a) kernel thread(s) which will wakeup when there is
52 : * work to do and go to sleep when there is nothing to do. The infrastructure to
53 : * wake up a thread when there is work to do is highly OS specific. Hence, to
54 : * accommodate that need, we will introduce a new host os entry point called
55 : * 'enqueue_to_assembler'. This entry point will enqueue to the per-cpu queue
56 : * as discussed above and will wakeup the kthread that does the assembly. The
57 : * enqueue code can be in platform independent part, so that all platforms can
58 : * reuse the code, with the infrastructure to wakeup the assembler being in
59 : * tail of the 'enqueue_to_assembler'.
60 :
61 : * For Linux kernel, we will use the workqueue infrastructure. We will create
62 : * a new workqueue for assembling the packets. The workqueue will have a dedicated
63 : * thread for each processor in the system(kernel infrastructure). So, once you
64 : * queue work to the queue, the thread for that processor will wakeup and do the
65 : * work of dequeuing it from the percpu queue and enqueuing it to the assembler
66 : * table.
67 :
68 : * When the assembler is woken up, it goes through the per-cpu queue and
69 : * dequeues all packets that have been enqueued and inserts them into the hash
70 : * list. The work area of the assembler is the hash table, where each bucket will
71 : * be a pointer to list of fragment metadata. Each such bucket will be protected
72 : * by a spinlock. The spinlock is mainly meant for exclusion between the aging
73 : * timer and the assembler.
74 :
75 : * Since a spinlock/mutex structure is needed and such structures are OS specific,
76 : * the assembler's origin will be in the OS specific part. The assembler will
77 : * define the hash buckets (that has the spinlock), with the individual and list
78 : * definitions coming from the OS independent part. While going through each
79 : * bucket, assembler will hold the bucket lock and pass control to the os
80 : * independent part.
81 :
82 : * For the linux kernel implementation, we will use spinlock rather than a mutex,
83 : * since the aging timer is invoked in an atomic context and hence can't block.
84 :
85 : * The assembler will dequeue packets from per-cpu queue and queue it in the
86 : * hash list. When the head of the fragment arrives, every fragment is dequeued
87 : * and flushed out of that entry, while still maintaining the metadata.
88 : */
89 :
90 : struct vr_timer *vr_assembler_table_scan_timer;
91 :
92 : static inline void
93 99 : __fragment_key(struct vr_fragment_key *key, unsigned short vrf,
94 : uint64_t sip_u, uint64_t sip_l, uint64_t dip_u, uint64_t dip_l,
95 : uint32_t id, unsigned short custom)
96 : {
97 99 : key->fk_sip_u = sip_u;
98 99 : key->fk_sip_l = sip_l;
99 99 : key->fk_dip_u = dip_u;
100 99 : key->fk_dip_l = dip_l;
101 99 : key->fk_id = id;
102 99 : key->fk_vrf = vrf;
103 99 : key->fk_custom = custom;
104 :
105 99 : return;
106 : }
107 :
108 : #define VR_FRAGMENT_FROM_HENTRY(entry) \
109 : (struct vr_fragment *)((entry) ?\
110 : CONTAINER_OF(f_hentry, struct vr_fragment, entry) :\
111 : NULL)
112 :
113 : #define VR_HENTRY_FROM_FRAGMENT(fe) \
114 : (vr_hentry_t *)((fe) ? &fe->f_hentry : NULL)
115 :
116 : static vr_hentry_key
117 20 : vr_fragment_get_entry_key(vr_htable_t table, vr_hentry_t *entry,
118 : unsigned int *key_len)
119 : {
120 20 : struct vr_fragment *fe =
121 : (struct vr_fragment *)CONTAINER_OF(f_hentry, struct vr_fragment, entry);
122 :
123 20 : if (key_len) {
124 20 : *key_len = sizeof(struct vr_fragment_key);
125 : }
126 :
127 20 : return &fe->f_key;
128 : }
129 :
130 : static inline void
131 5 : fragment_entry_set(struct vr_fragment *fe, struct vr_fragment_key *key,
132 : unsigned short sport, unsigned short dport)
133 : {
134 : uint64_t sec, nsec;
135 :
136 5 : fe->f_sip_u = key->fk_sip_u;
137 5 : fe->f_sip_l = key->fk_sip_l;
138 5 : fe->f_dip_u = key->fk_dip_u;
139 5 : fe->f_dip_l = key->fk_dip_l;
140 5 : fe->f_id = key->fk_id;
141 5 : fe->f_vrf = key->fk_vrf;
142 5 : fe->f_custom = key->fk_custom;
143 5 : fe->f_sport = sport;
144 5 : fe->f_dport = dport;
145 5 : vr_get_mono_time(&sec, &nsec);
146 5 : fe->f_time = sec;
147 5 : fe->f_expected = 0;
148 5 : fe->f_received = 0;
149 :
150 5 : return;
151 : }
152 :
153 : void
154 0 : vr_fragment_queue_free(struct vr_fragment_queue *queue)
155 : {
156 : struct vr_fragment_queue_element *vfqe, *next;
157 :
158 0 : vfqe = queue->vfq_tail;
159 0 : queue->vfq_tail = NULL;
160 0 : while (vfqe) {
161 0 : next = vfqe->fqe_next;
162 0 : if (vfqe->fqe_pnode.pl_packet) {
163 0 : PKT_LOG(VP_DROP_MISC, 0, 0, VR_FRAGMENT_C, __LINE__);
164 0 : vr_pfree(vfqe->fqe_pnode.pl_packet, VP_DROP_MISC);
165 0 : vfqe->fqe_pnode.pl_packet = NULL;
166 : }
167 0 : vr_free(vfqe, VR_FRAGMENT_QUEUE_ELEMENT_OBJECT);
168 0 : vfqe = next;
169 : }
170 :
171 0 : return;
172 : }
173 :
174 : static void
175 21 : vr_fragment_queue_element_free(struct vr_fragment_queue_element *vfqe,
176 : unsigned int drop_reason)
177 : {
178 21 : if (vfqe->fqe_pnode.pl_packet) {
179 13 : PKT_LOG(drop_reason, vfqe->fqe_pnode.pl_packet, 0, VR_FRAGMENT_C, __LINE__);
180 13 : vr_pfree(vfqe->fqe_pnode.pl_packet, drop_reason);
181 : }
182 :
183 21 : vr_free(vfqe, VR_FRAGMENT_QUEUE_ELEMENT_OBJECT);
184 21 : return;
185 : }
186 :
187 : static void
188 4 : fragment_free_frag(struct vr_fragment *frag)
189 : {
190 : struct vr_fragment_queue_element *fqe;
191 :
192 12 : while ((fqe = frag->f_qe)) {
193 8 : frag->f_qe = fqe->fqe_next;
194 8 : vr_fragment_queue_element_free(fqe, VP_DROP_FRAGMENTS);
195 : }
196 :
197 4 : vr_free(frag, VR_FRAGMENT_OBJECT);
198 4 : return;
199 : }
200 :
201 : static void
202 4 : fragment_unlink_frag(struct vr_fragment **prev, struct vr_fragment *frag)
203 : {
204 4 : *prev = frag->f_next;
205 4 : return;
206 : }
207 :
208 : /* Scan assembler_table[][] and free up stale entries
209 : * Arguments:
210 : * - head: Bucket entry in assembler_table[][] for the
211 : * forwarding core which executes it
212 : * It has all fragment entries which hash to that bucket
213 : */
214 : unsigned int
215 2369 : vr_assembler_table_scan(struct vr_fragment **head)
216 : {
217 2369 : unsigned int scanned = 0;
218 : uint64_t sec, nsec, dest;
219 2369 : struct vr_fragment *frag = *head, *next, **prev;
220 :
221 2369 : prev = head;
222 4738 : while (frag) {
223 2369 : next = frag->f_next;
224 :
225 2369 : vr_get_mono_time(&sec, &nsec);
226 2369 : dest = frag->f_time + VR_ASSEMBLER_TIMEOUT_SECS;
227 2369 : if (dest < frag->f_time) {
228 0 : if ((sec < frag->f_time) && (dest < sec)) {
229 0 : fragment_unlink_frag(prev, frag);
230 0 : fragment_free_frag(frag);
231 : } else {
232 0 : prev = &frag->f_next;
233 : }
234 : } else {
235 2369 : if ((sec > dest) || (sec < frag->f_time)) {
236 2 : fragment_unlink_frag(prev, frag);
237 2 : fragment_free_frag(frag);
238 : } else {
239 2367 : prev = &frag->f_next;
240 : }
241 : }
242 2369 : scanned++;
243 2369 : frag = next;
244 : }
245 :
246 2369 : return scanned;
247 : }
248 :
249 :
250 : void
251 53 : vr_assembler_table_scan_exit(void)
252 : {
253 53 : if (vr_assembler_table_scan_timer) {
254 0 : vr_delete_timer(vr_assembler_table_scan_timer);
255 0 : vr_free(vr_assembler_table_scan_timer, VR_TIMER_OBJECT);
256 0 : vr_assembler_table_scan_timer = NULL;
257 : }
258 :
259 53 : return;
260 : }
261 :
262 : /* Initialize scanning of assembler_table[][] */
263 : int
264 0 : vr_assembler_table_scan_init(void (*scanner)(void *))
265 : {
266 : struct vr_timer *vtimer;
267 :
268 0 : vr_assembler_table_scan_timer = vr_zalloc(sizeof(*vtimer), VR_TIMER_OBJECT);
269 0 : if (!vr_assembler_table_scan_timer)
270 0 : return -ENOMEM;
271 :
272 0 : vtimer = vr_assembler_table_scan_timer;
273 0 : vtimer->vt_timer = scanner;
274 0 : vtimer->vt_vr_arg = NULL;
275 0 : vtimer->vt_msecs =
276 : (VR_ASSEMBLER_TIMEOUT_SECS * 1000) / VR_ASSEMBLER_BUCKET_COUNT;
277 0 : if (vr_create_timer(vtimer)) {
278 0 : vr_free(vtimer, VR_TIMER_OBJECT);
279 0 : vr_assembler_table_scan_timer = NULL;
280 0 : return -ENOMEM;
281 : }
282 :
283 0 : return 0;
284 : }
285 :
286 : /* Flush a fragment queue entry and re-inject for packet processing */
287 : static void
288 8 : vr_fragment_flush_queue_element(struct vr_fragment_queue_element *vfqe)
289 : {
290 : struct vrouter *router;
291 : struct vr_packet *pkt;
292 :
293 : struct vr_forwarding_md fmd;
294 : struct vr_packet_node *pnode;
295 :
296 8 : if (!vfqe) {
297 0 : PKT_LOG(VP_DROP_CLONED_ORIGINAL, 0, 0, VR_FRAGMENT_C, __LINE__);
298 0 : return;
299 : }
300 :
301 8 : router = vfqe->fqe_router;
302 8 : pnode = &vfqe->fqe_pnode;
303 8 : pkt = pnode->pl_packet;
304 8 : if (!pkt) {
305 0 : PKT_LOG(VP_DROP_CLONED_ORIGINAL, 0, 0, VR_FRAGMENT_C, __LINE__);
306 0 : goto exit_flush;
307 : }
308 :
309 8 : vr_init_forwarding_md(&fmd);
310 8 : fmd.fmd_vlan = pnode->pl_vlan;
311 8 : fmd.fmd_dvrf = pnode->pl_vrf;
312 8 : vr_flow_flush_pnode(router, pnode, NULL, &fmd);
313 :
314 8 : exit_flush:
315 8 : vr_fragment_queue_element_free(vfqe, VP_DROP_CLONED_ORIGINAL);
316 8 : return;
317 : }
318 :
319 : /* Main assembler function to process fragments from per-cpu queue to
320 : * assembler_table
321 : * - Create a head fragment entry if head fragment arrives
322 : * - If a non-head fragment arrives and a corresponding head-fragment entry
323 : * is already present, flush it
324 : * - Flush non-head fragments which are already enqueued if their head fragment arrives
325 : * - Enqueue non-head fragment if their head fragment has not yet arrived
326 : *
327 : * Arguments:
328 : * - head_p: bucket entry in assembler_table
329 : * - vfqe : fragment present in the per-cpu queue
330 : */
331 : int
332 21 : vr_fragment_assemble(struct vr_fragment **head_p,
333 : struct vr_fragment_queue_element *vfqe)
334 : {
335 21 : int ret = 0;
336 : uint64_t sec, nsec;
337 21 : unsigned int list_length = 0, drop_reason;
338 21 : bool found = false, frag_head = false;
339 : uint64_t *v6_addr;
340 :
341 : struct vrouter *router;
342 : struct vr_ip *ip;
343 : struct vr_ip6 *ip6;
344 : struct vr_ip6_frag *v6_frag;
345 : struct vr_packet *pkt;
346 : struct vr_packet_node *pnode;
347 21 : struct vr_fragment *frag, *frag_flow, **prev = NULL;
348 : struct vr_fragment_queue_element *fqe;
349 : struct vr_fragment_key vfk;
350 :
351 :
352 21 : router = vfqe->fqe_router;
353 21 : pnode = &vfqe->fqe_pnode;
354 :
355 : /* Is it head fragment? */
356 21 : if (pnode->pl_flags & PN_FLAG_FRAGMENT_HEAD)
357 5 : frag_head = true;
358 :
359 21 : pkt = pnode->pl_packet;
360 21 : ip = (struct vr_ip *)pkt_network_header(pkt);
361 :
362 : /* Get hash of the fragment key */
363 21 : if (vr_ip_is_ip6(ip)) {
364 0 : ip6 = (struct vr_ip6 *)ip;
365 0 : v6_frag = (struct vr_ip6_frag *)(ip6 + 1);
366 0 : v6_addr = (uint64_t *)(ip6->ip6_src);
367 0 : __fragment_key(&vfk, pnode->pl_vrf, *v6_addr, *(v6_addr + 1),
368 0 : *(v6_addr +2 ), *(v6_addr + 3), v6_frag->ip6_frag_id,
369 0 : pnode->pl_custom);
370 : } else {
371 21 : __fragment_key(&vfk, pnode->pl_vrf, 0, pnode->pl_inner_src_ip,
372 21 : 0, pnode->pl_inner_dst_ip, ip->ip_id, pnode->pl_custom);
373 : }
374 :
375 : /* Check if the fragment with the same key is found
376 : * in the assembler bucket
377 : */
378 21 : frag = *head_p;
379 21 : prev = head_p;
380 21 : while (frag) {
381 14 : list_length++;
382 14 : if (!memcmp(&frag->f_key, &vfk, sizeof(vfk))) {
383 14 : found = true;
384 14 : break;
385 : }
386 :
387 0 : prev = &frag->f_next;
388 0 : frag = frag->f_next;
389 : }
390 :
391 : /* If its a non-head fragment and the head fragment has
392 : * already arrived, just flush (re-inject) it for regular
393 : * packet processing
394 : */
395 21 : if (!frag_head) {
396 16 : frag_flow = vr_fragment_get(router, pnode->pl_vrf, ip, pnode->pl_custom);
397 16 : if (frag_flow) {
398 0 : vr_fragment_flush_queue_element(vfqe);
399 0 : return 0;
400 : }
401 : }
402 :
403 : /* If fragment entry is not found */
404 21 : if (!found) {
405 : /* If its a fragment head, it's a cloned packet
406 : * Nothing to be done, just exit
407 : */
408 7 : if (frag_head) {
409 3 : drop_reason = VP_DROP_CLONED_ORIGINAL;
410 3 : PKT_LOG(drop_reason, pkt, 0, VR_FRAGMENT_C, __LINE__);
411 3 : goto exit_assembly;
412 : }
413 :
414 : /* Check if max length of assembler bucket is exceeded */
415 4 : if (list_length > VR_MAX_FRAGMENTS_PER_ASSEMBLER_QUEUE) {
416 0 : drop_reason = VP_DROP_FRAGMENT_QUEUE_FAIL;
417 0 : PKT_LOG(drop_reason, pkt, 0, VR_FRAGMENT_C, __LINE__);
418 0 : goto exit_assembly;
419 : }
420 :
421 : /* If it's a non-head fragment, allocate a new entry */
422 4 : frag = vr_zalloc(sizeof(*frag), VR_FRAGMENT_OBJECT);
423 4 : if (!frag) {
424 0 : ret = -ENOMEM;
425 0 : drop_reason = VP_DROP_NO_MEMORY;
426 0 : PKT_LOG(drop_reason, pkt, 0, VR_FRAGMENT_C, __LINE__);
427 0 : goto exit_assembly;
428 : }
429 :
430 4 : memcpy(&frag->f_key, &vfk, sizeof(vfk));
431 : /* This is a non-head fragment, so the port info
432 : * is not valid since head fragment has not arrived
433 : */
434 4 : frag->f_port_info_valid = false;
435 : }
436 :
437 : /* Timestamp the fragment */
438 18 : vr_get_mono_time(&sec, &nsec);
439 18 : frag->f_time = sec;
440 :
441 : /* If frag entry is not found, create a new entry
442 : * at the head of the assembler bucket
443 : */
444 18 : if (!found) {
445 4 : prev = head_p;
446 4 : frag->f_next = *head_p;
447 4 : *head_p = frag;
448 : }
449 :
450 : /* If the packet is non a fragment head,
451 : * append the packet to the end of the fragment
452 : * queue entry of the assembler bucket
453 : */
454 18 : if (!frag_head) {
455 16 : vfqe->fqe_next = NULL;
456 16 : fqe = frag->f_qe;
457 16 : if (!fqe) {
458 4 : frag->f_qe = vfqe;
459 : } else {
460 24 : while (fqe) {
461 24 : if (fqe->fqe_next) {
462 12 : fqe = fqe->fqe_next;
463 : } else {
464 12 : break;
465 : }
466 : }
467 :
468 12 : fqe->fqe_next = vfqe;
469 : }
470 : } else {
471 : /* If the packet is a fragment head, make the port
472 : * info as valid since it contains a valid transport
473 : * header. Also, free the cloned head-fragment
474 : */
475 2 : frag->f_port_info_valid = true;
476 2 : PKT_LOG(VP_DROP_CLONED_ORIGINAL, pkt, 0, VR_FRAGMENT_C, __LINE__);
477 2 : vr_fragment_queue_element_free(vfqe, VP_DROP_CLONED_ORIGINAL);
478 : }
479 :
480 : /* If a head fragment arrives after the non-head
481 : * fragments, it means that we have the valid port info.
482 : * In this case, flush the non-head fragments and
483 : * re-inject them for normal packet processing
484 : */
485 18 : if (frag->f_port_info_valid) {
486 10 : while ((fqe = frag->f_qe)) {
487 8 : frag->f_qe = fqe->fqe_next;
488 8 : vr_fragment_flush_queue_element(fqe);
489 : }
490 :
491 : /* After flushing, remove the fragment entries from assembler table */
492 2 : fragment_unlink_frag(prev, frag);
493 2 : fragment_free_frag(frag);
494 : }
495 :
496 18 : return 0;
497 :
498 3 : exit_assembly:
499 3 : vr_fragment_queue_element_free(vfqe, drop_reason);
500 3 : return ret;
501 : }
502 :
503 : void
504 21 : vr_fragment_assemble_queue(struct vr_fragment_queue *vfq)
505 : {
506 : struct vr_packet_node *pnode;
507 : struct vr_fragment_queue_element *tail, *tail_n, *tail_p, *tail_pn;
508 :
509 21 : tail = vr_sync_lock_test_and_set_p(&vfq->vfq_tail, NULL);
510 21 : if (!tail) {
511 0 : return;
512 : }
513 :
514 : /*
515 : * first, reverse the list, since packets that came later are at the
516 : * head of the list
517 : */
518 21 : tail_p = tail->fqe_next;
519 21 : tail->fqe_next = NULL;
520 21 : while (tail_p) {
521 0 : tail_pn = tail_p->fqe_next;
522 0 : tail_p->fqe_next = tail;
523 0 : tail = tail_p;
524 0 : tail_p = tail_pn;
525 : }
526 :
527 : /* go through the list and insert it in the assembler work area */
528 42 : while (tail) {
529 21 : tail_n = tail->fqe_next;
530 21 : tail->fqe_next = NULL;
531 :
532 21 : pnode = &tail->fqe_pnode;
533 21 : if (pnode->pl_packet) {
534 21 : vr_fragment_sync_assemble(tail);
535 : }
536 :
537 21 : tail = tail_n;
538 : }
539 : }
540 :
541 : static uint32_t
542 21 : __vr_fragment_get_hash(unsigned int vrf, uint64_t sip_u, uint64_t sip_l,
543 : uint64_t dip_u, uint64_t dip_l, uint32_t id, unsigned short custom)
544 : {
545 : struct vr_fragment_key vfk;
546 :
547 21 : __fragment_key(&vfk, vrf, sip_u, sip_l, dip_u, dip_l, id, custom);
548 :
549 21 : return vr_hash(&vfk, sizeof(vfk), 0);
550 : }
551 :
552 : uint32_t
553 21 : vr_fragment_get_hash(struct vr_packet_node *pnode)
554 : {
555 : uint64_t *v6_addr;
556 : struct vr_ip *ip;
557 : struct vr_ip6 *ip6;
558 : struct vr_ip6_frag *v6_frag;
559 : struct vr_packet *pkt;
560 :
561 21 : if (!pnode || !pnode->pl_packet)
562 0 : return (uint32_t)-1;
563 :
564 21 : pkt = pnode->pl_packet;
565 :
566 21 : ip = (struct vr_ip *)pkt_network_header(pkt);
567 21 : if (vr_ip_is_ip6(ip)) {
568 0 : ip6 = (struct vr_ip6 *)pkt_network_header(pkt);
569 0 : v6_frag = (struct vr_ip6_frag *)(ip6 + 1);
570 0 : v6_addr = (uint64_t *)(ip6->ip6_src);
571 :
572 0 : return __vr_fragment_get_hash(pnode->pl_vrf, *v6_addr, *(v6_addr+ 1),
573 0 : *(v6_addr + 2), *(v6_addr + 3), v6_frag->ip6_frag_id,
574 0 : pnode->pl_custom);
575 21 : } else if(vr_ip_is_ip4(ip)) {
576 21 : return __vr_fragment_get_hash(pnode->pl_vrf, 0, pnode->pl_inner_src_ip,
577 21 : 0, pnode->pl_inner_dst_ip, ip->ip_id, pnode->pl_custom);
578 : }
579 :
580 0 : return (uint32_t)-1;
581 : }
582 :
583 : /* Enqueue fragment in per cpu queue */
584 : int
585 21 : vr_fragment_enqueue(struct vrouter *router,
586 : struct vr_fragment_queue *vfq,
587 : struct vr_packet *pkt, struct vr_forwarding_md *fmd)
588 : {
589 21 : bool swapped = false;
590 : unsigned int i;
591 :
592 : struct vr_packet_node *pnode;
593 21 : struct vr_fragment_queue_element *fqe = NULL, *tail, **tailp;
594 :
595 21 : tailp = &vfq->vfq_tail;
596 21 : if (*tailp == NULL) {
597 21 : vfq->vfq_length = 0;
598 : } else {
599 0 : if ((vfq->vfq_length + 1) > VR_MAX_FRAGMENTS_PER_CPU_QUEUE) {
600 0 : PKT_LOG(VP_DROP_FRAGMENTS, pkt, 0, VR_FRAGMENT_C, __LINE__);
601 0 : goto fail;
602 : }
603 : }
604 :
605 : /* Check if the total number of fragmented packets across
606 : * all cores exceeded. */
607 42 : if (vrouter_host->hos_is_frag_limit_exceeded &&
608 21 : vrouter_host->hos_is_frag_limit_exceeded()) {
609 0 : PKT_LOG(VP_DROP_FRAGMENTS, pkt, 0, VR_FRAGMENT_C, __LINE__);
610 0 : goto fail;
611 : }
612 :
613 21 : fqe = vr_malloc(sizeof(*fqe), VR_FRAGMENT_QUEUE_ELEMENT_OBJECT);
614 21 : if (!fqe) {
615 0 : PKT_LOG(VP_DROP_FRAGMENTS, pkt, 0, VR_FRAGMENT_C, __LINE__);
616 0 : goto fail;
617 : }
618 21 : fqe->fqe_router = router;
619 21 : fqe->fqe_next = NULL;
620 :
621 21 : pkt->vp_flags &= ~VP_FLAG_FLOW_SET;
622 :
623 21 : pnode = &fqe->fqe_pnode;
624 21 : vr_flow_fill_pnode(pnode, pkt, fmd);
625 :
626 : /*
627 : * we are actually competing with an existing assembler work that must
628 : * be in the process of dequeueing the list from the per-cpu queue.
629 : * we try thrice to enqueue our element. It is unlikely that it will
630 : * fail more than once
631 : *
632 : * calculation of vfq_length could be erroneous. But, we will err by
633 : * maximum 1, which is fine.
634 : */
635 21 : for (i = 0; i < VR_FRAG_ENQUEUE_ATTEMPTS; i++) {
636 21 : tail = *tailp;
637 21 : fqe->fqe_next = tail;
638 21 : vfq->vfq_length++;
639 21 : swapped = vr_sync_bool_compare_and_swap_p(tailp, tail, fqe);
640 21 : if (swapped) {
641 21 : if (tail == NULL)
642 21 : vfq->vfq_length = 1;
643 21 : break;
644 : } else {
645 0 : vfq->vfq_length--;
646 0 : if (i == (VR_FRAG_ENQUEUE_ATTEMPTS - 1)) {
647 0 : PKT_LOG(VP_DROP_FRAGMENTS, pkt, 0, VR_FRAGMENT_C, __LINE__);
648 0 : goto fail;
649 : }
650 : }
651 : }
652 :
653 21 : return 0;
654 :
655 0 : fail:
656 0 : if (fqe)
657 0 : vr_free(fqe, VR_FRAGMENT_QUEUE_ELEMENT_OBJECT);
658 :
659 0 : vr_pfree(pkt, VP_DROP_FRAGMENTS);
660 0 : return -1;
661 : }
662 :
663 :
664 : /* Delete fragment from the fragment hash table */
665 : void
666 5 : vr_fragment_del(vr_htable_t table, struct vr_fragment *fe)
667 : {
668 5 : fe->f_dip_u = fe->f_dip_l = 0;
669 5 : fe->f_received = 0;
670 5 : vr_htable_release_hentry(table, VR_HENTRY_FROM_FRAGMENT(fe));
671 :
672 5 : return;
673 : }
674 :
675 : /* Add fragment from the fragment hash table */
676 : static int
677 5 : vr_fragment_add(struct vrouter *router, struct vr_fragment_key *key,
678 : unsigned short sport, unsigned short dport, unsigned short len)
679 : {
680 : void *fe_ent;
681 : struct vr_fragment *fe;
682 5 : vr_htable_t ftable = router->vr_fragment_table;
683 :
684 5 : fe_ent = (void *) vr_htable_find_hentry(ftable, key, 0);
685 5 : fe = VR_FRAGMENT_FROM_HENTRY(fe_ent);
686 5 : if (fe)
687 0 : return 0;
688 :
689 5 : fe_ent = (void *) vr_htable_find_free_hentry(ftable, key, 0);
690 5 : fe = VR_FRAGMENT_FROM_HENTRY(fe_ent);
691 5 : if (!fe)
692 0 : return -ENOMEM;
693 :
694 5 : fragment_entry_set(fe, key, sport, dport);
695 5 : fe->f_received += len;
696 :
697 5 : return 0;
698 : }
699 :
700 : /* Add v4 fragment from the fragment hash table */
701 : int
702 5 : vr_v4_fragment_add(struct vrouter *router, unsigned short vrf,
703 : struct vr_ip *iph, unsigned short sport, unsigned short dport,
704 : unsigned short custom)
705 : {
706 : struct vr_fragment_key key;
707 :
708 5 : __fragment_key(&key, vrf, 0, iph->ip_saddr, 0,
709 5 : iph->ip_daddr, iph->ip_id, custom);
710 :
711 10 : return vr_fragment_add(router, &key, sport, dport,
712 5 : (ntohs(iph->ip_len) - iph->ip_hl * 4));
713 : }
714 :
715 : /* Add v6 fragment from the fragment hash table */
716 : int
717 0 : vr_v6_fragment_add(struct vrouter *router, unsigned short vrf,
718 : struct vr_ip6 *ip6, unsigned short sport, unsigned short dport,
719 : unsigned short custom)
720 : {
721 : uint64_t *v6_addr;
722 : struct vr_fragment_key key;
723 : struct vr_ip6_frag *v6_frag;
724 :
725 0 : v6_addr = (uint64_t *)(ip6->ip6_src);
726 0 : v6_frag = (struct vr_ip6_frag *)(ip6 + 1);
727 0 : __fragment_key(&key, vrf, *v6_addr, *(v6_addr + 1), *(v6_addr + 2),
728 0 : *(v6_addr + 3), v6_frag->ip6_frag_id, custom);
729 :
730 0 : return vr_fragment_add(router, &key, sport, dport,
731 0 : (ntohs(ip6->ip6_plen) - sizeof(struct vr_ip6_frag)));
732 : }
733 :
734 : /* Check if fragment entry exists in the fragment hash table */
735 : struct vr_fragment *
736 52 : vr_fragment_get(struct vrouter *router, unsigned short vrf,
737 : struct vr_ip *ip, unsigned short custom)
738 : {
739 : uint64_t sec, nsec;
740 : uint64_t *v6_addr;
741 : struct vr_ip6 *ip6;
742 : struct vr_ip6_frag *v6_frag;
743 : struct vr_fragment *fe;
744 : struct vr_fragment_key key;
745 : vr_htable_t ftable;
746 : void *fe_ent;
747 :
748 52 : if (vr_ip_is_ip6(ip)) {
749 0 : ip6 = (struct vr_ip6 *)ip;
750 0 : if (ip6->ip6_nxt == VR_IP6_PROTO_FRAG) {
751 0 : v6_frag = (struct vr_ip6_frag *)(ip6 + 1);
752 0 : v6_addr = (uint64_t *)(ip6->ip6_src);
753 0 : __fragment_key(&key, vrf, *v6_addr, *(v6_addr+ 1),
754 0 : *(v6_addr + 2), *(v6_addr+ 3), v6_frag->ip6_frag_id,
755 : custom);
756 : }
757 52 : } else if (vr_ip_is_ip4(ip)) {
758 52 : __fragment_key(&key, vrf, 0, ip->ip_saddr, 0, ip->ip_daddr,
759 52 : ip->ip_id, custom);
760 : } else {
761 0 : return NULL;
762 : }
763 :
764 :
765 52 : ftable = router->vr_fragment_table;
766 :
767 52 : fe_ent = (void *) vr_htable_find_hentry(ftable, &key, 0);
768 52 : fe = VR_FRAGMENT_FROM_HENTRY(fe_ent);
769 52 : if (fe) {
770 20 : vr_get_mono_time(&sec, &nsec);
771 20 : fe->f_time = sec;
772 : }
773 :
774 52 : return fe;
775 : }
776 :
777 :
778 : struct scanner_params {
779 : struct vrouter *sp_router;
780 : int sp_scan_marker;
781 : };
782 :
783 : static void
784 12972032 : __fragment_reap(vr_htable_t table, vr_hentry_t *ent,
785 : unsigned int index, void *data)
786 : {
787 : uint64_t sec, nsec;
788 : struct vr_fragment *fe;
789 :
790 12972032 : fe = VR_FRAGMENT_FROM_HENTRY(ent);
791 12972032 : if (!fe || ((!fe->f_dip_u) && !(fe->f_dip_l)))
792 12972032 : return;
793 :
794 0 : vr_get_mono_time(&sec, &nsec);
795 0 : if (sec > fe->f_time + VR_FRAG_HASH_TABLE_TIMEOUT_SECS) {
796 0 : vr_fragment_del(table, fe);
797 : }
798 :
799 0 : return;
800 : }
801 :
802 : /* Reap/cleanup stale entries from fragment hash table */
803 : static int
804 6334 : fragment_reap(vr_htable_t htable, int start)
805 : {
806 6334 : return vr_htable_trav_range(htable, start,
807 : VR_FRAG_HASH_TABLE_ENTRIES_PER_SCAN,
808 : __fragment_reap, NULL);
809 : }
810 :
811 : /* Scanner callback for fragment hash table */
812 : static void
813 6334 : fragment_table_scanner(void *arg)
814 : {
815 : int ret;
816 6334 : struct scanner_params *sp = (struct scanner_params *)arg;
817 :
818 6334 : ret = fragment_reap(sp->sp_router->vr_fragment_table, sp->sp_scan_marker);
819 6334 : if (ret < 0)
820 0 : return;
821 :
822 6334 : sp->sp_scan_marker = ret;
823 6334 : return;
824 : }
825 :
826 : /* Initialize scanner for fragment hash table */
827 : static struct vr_timer *
828 53 : fragment_table_scanner_init(struct vrouter *router)
829 : {
830 : struct vr_timer *vtimer;
831 : struct scanner_params *scanner;
832 :
833 53 : scanner = vr_zalloc(sizeof(*scanner), VR_FRAGMENT_SCANNER_OBJECT);
834 53 : if (!scanner) {
835 0 : vr_module_error(-ENOMEM, __FUNCTION__, __LINE__, sizeof(*scanner));
836 0 : return NULL;
837 : }
838 53 : scanner->sp_router = router;
839 53 : scanner->sp_scan_marker = 0;
840 :
841 53 : vtimer = vr_malloc(sizeof(*vtimer), VR_TIMER_OBJECT);
842 53 : if (!vtimer) {
843 0 : vr_module_error(-ENOMEM, __FUNCTION__, __LINE__, sizeof(*vtimer));
844 0 : goto fail_init;
845 : }
846 :
847 53 : vtimer->vt_timer = fragment_table_scanner;
848 53 : vtimer->vt_vr_arg = scanner;
849 53 : vtimer->vt_msecs = VR_FRAG_HASH_TABLE_SCANNER_INTERVAL_MSEC;
850 53 : if (vr_create_timer(vtimer)) {
851 0 : vr_module_error(-ENOMEM, __FUNCTION__, __LINE__, 0);
852 0 : goto fail_init;
853 : }
854 :
855 53 : return vtimer;
856 :
857 0 : fail_init:
858 0 : if (scanner)
859 0 : vr_free(scanner, VR_FRAGMENT_SCANNER_OBJECT);
860 :
861 0 : return NULL;
862 : }
863 :
864 : static void
865 53 : vr_fragment_table_scanner_exit(struct vrouter *router)
866 : {
867 53 : if (router->vr_fragment_table_scanner) {
868 53 : vr_delete_timer(router->vr_fragment_table_scanner);
869 53 : vr_free(router->vr_fragment_table_scanner->vt_vr_arg,
870 : VR_FRAGMENT_SCANNER_OBJECT);
871 53 : vr_free(router->vr_fragment_table_scanner, VR_TIMER_OBJECT);
872 53 : router->vr_fragment_table_scanner = NULL;
873 : }
874 :
875 53 : return;
876 : }
877 :
878 : static int
879 53 : vr_fragment_table_scanner_init(struct vrouter *router)
880 : {
881 53 : if (!router->vr_fragment_table_scanner) {
882 53 : router->vr_fragment_table_scanner =
883 53 : fragment_table_scanner_init(router);
884 53 : if (!router->vr_fragment_table_scanner) {
885 0 : return -ENOMEM;
886 : }
887 : }
888 :
889 53 : return 0;
890 : }
891 :
892 : void
893 53 : vr_fragment_table_exit(struct vrouter *router)
894 : {
895 53 : vr_fragment_table_scanner_exit(router);
896 :
897 53 : if (router->vr_fragment_table) {
898 53 : vr_htable_delete(router->vr_fragment_table);
899 53 : router->vr_fragment_table = NULL;
900 : }
901 :
902 53 : return;
903 : }
904 :
905 : /* Initialize fragment hash table */
906 : int
907 53 : vr_fragment_table_init(struct vrouter *router)
908 : {
909 : int ret;
910 :
911 53 : if (!router->vr_fragment_table) {
912 53 : router->vr_fragment_table = vr_htable_create(router,
913 : VR_FRAG_HASH_TABLE_ENTRIES, VR_FRAG_HASH_OTABLE_ENTRIES,
914 : sizeof(struct vr_fragment), sizeof(struct vr_fragment_key),
915 : VR_FRAG_HASH_TABLE_BUCKETS, vr_fragment_get_entry_key);
916 53 : if (!router->vr_fragment_table) {
917 0 : return vr_module_error(-ENOMEM, __FUNCTION__, __LINE__,
918 : VR_FRAG_HASH_TABLE_ENTRIES + VR_FRAG_HASH_OTABLE_ENTRIES);
919 : }
920 : }
921 :
922 53 : if ((ret = vr_fragment_table_scanner_init(router)))
923 0 : return ret;
924 :
925 53 : return 0;
926 : }
|
