Line data Source code
1 : /*
2 : * Copyright (C) 2014 Semihalf.
3 : *
4 : * This program is free software; you can redistribute it and/or
5 : * modify it under the terms of the GNU General Public License as
6 : * published by the Free Software Foundation version 2.
7 : *
8 : * This program is distributed "as is" WITHOUT ANY WARRANTY of any
9 : * kind, whether express or implied; without even the implied warranty
10 : * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 : * GNU General Public License for more details.
12 : *
13 : * vr_dpdk_host.c -- DPDK vrouter module
14 : *
15 : */
16 :
17 : #include "vr_dpdk.h"
18 : #include "vr_fragment.h"
19 : #include "vr_hash.h"
20 : #include "vr_proto.h"
21 : #include "vr_sandesh.h"
22 : #include "vr_dpdk_offloads.h"
23 : #include "vr_cpuid.h"
24 :
25 : #include <linux/if_ether.h>
26 : #include <netinet/ip.h>
27 : #include <netinet/tcp.h>
28 : #include <sys/user.h>
29 : #include <sys/resource.h>
30 :
31 : #include <rte_cycles.h>
32 : #include <rte_errno.h>
33 : #include <rte_ethdev.h>
34 : #include <rte_jhash.h>
35 : #include <rte_malloc.h>
36 : #include <rte_timer.h>
37 : #include <rte_hash.h>
38 :
39 : struct dpdk_work_cb_data {
40 : void (*dwc_fn)(void *);
41 : void *dwc_data;
42 : };
43 :
44 : /* Max number of CPUs. We adjust it later in vr_dpdk_host_init() */
45 : unsigned int vr_num_cpus = VR_MAX_CPUS_DPDK;
46 :
47 : /* Global init flag */
48 : static bool vr_host_inited = false;
49 :
50 : extern void vr_malloc_stats(unsigned int, unsigned int);
51 : extern void vr_free_stats(unsigned int);
52 : /* RCU callback */
53 : extern void vr_flow_defer_cb(struct vrouter *router, void *arg);
54 : extern void vr_htable_hentry_scheduled_delete(void *arg);
55 :
56 :
57 : static void *
58 703 : dpdk_page_alloc(unsigned int size)
59 : {
60 703 : return rte_malloc(0, size, PAGE_SIZE);
61 : }
62 :
63 : static void
64 577 : dpdk_page_free(void *address, unsigned int size)
65 : {
66 577 : rte_free(address);
67 577 : }
68 :
69 : static int
70 85 : dpdk_printf(const char *format, ...)
71 : {
72 : va_list args;
73 :
74 85 : if (RTE_LOGTYPE_DPCORE & rte_logs.type) {
75 85 : char buf[VR_DPDK_STR_BUF_SZ] = "DPCORE: ";
76 :
77 85 : strncat(buf, format, sizeof(buf) - strlen(buf) - 1);
78 85 : buf[sizeof(buf) - 1] = '\0';
79 :
80 85 : va_start(args, format);
81 85 : rte_vlog(RTE_LOG_INFO, RTE_LOGTYPE_DPCORE, buf, args);
82 85 : va_end(args);
83 : }
84 :
85 85 : return 0;
86 : }
87 :
88 : static void *
89 6463 : dpdk_malloc(unsigned int size, unsigned int object)
90 : {
91 : struct vr_malloc_md *md;
92 : void *mem;
93 :
94 6463 : if (!size)
95 0 : return NULL;
96 :
97 6463 : if (vr_memory_alloc_checks) {
98 0 : size += sizeof(*md);
99 : }
100 :
101 6463 : mem = rte_malloc(NULL, size, 0);
102 6463 : if (likely(mem != NULL)) {
103 6463 : vr_malloc_stats(size, object);
104 :
105 6463 : if (vr_memory_alloc_checks) {
106 0 : vr_malloc_md_set(mem, object);
107 0 : mem = (uint8_t *)mem + sizeof(*md);
108 : }
109 : }
110 :
111 6463 : return mem;
112 : }
113 :
114 : static void *
115 237666 : dpdk_zalloc(unsigned int size, unsigned int object)
116 : {
117 : struct vr_malloc_md *md;
118 : void *mem;
119 :
120 237666 : if (!size)
121 0 : return NULL;
122 :
123 237666 : if (vr_memory_alloc_checks) {
124 0 : size += sizeof(*md);
125 : }
126 :
127 237666 : mem = rte_zmalloc(NULL, size, 0);
128 237666 : if (likely(mem != NULL)) {
129 237666 : vr_malloc_stats(size, object);
130 :
131 237666 : if (vr_memory_alloc_checks) {
132 0 : vr_malloc_md_set(mem, object);
133 0 : mem = (uint8_t *)mem + sizeof(*md);
134 : }
135 : }
136 :
137 237666 : return mem;
138 : }
139 :
140 : static void
141 240821 : dpdk_free(void *mem, unsigned int object)
142 : {
143 240821 : if (mem) {
144 240821 : if (vr_memory_alloc_checks) {
145 0 : vr_malloc_md_check(mem, object);
146 : }
147 :
148 240821 : vr_free_stats(object);
149 :
150 240821 : if (vr_memory_alloc_checks) {
151 0 : mem = (uint8_t *)mem - sizeof(struct vr_malloc_md);
152 : }
153 :
154 240821 : rte_free(mem);
155 : }
156 :
157 240821 : return;
158 : }
159 :
160 : static uint64_t
161 0 : dpdk_vtop(void *address)
162 : {
163 : /* TODO: not used */
164 0 : rte_panic("%s: not used in DPDK mode\n", __func__);
165 :
166 : return (uint64_t)0;
167 : }
168 :
169 : static struct vr_packet *
170 0 : dpdk_palloc(unsigned int size)
171 : {
172 : struct rte_mbuf *m;
173 :
174 : /* in DPDK we have fixed-sized mbufs only */
175 0 : RTE_VERIFY(size <= vr_packet_sz);
176 0 : m = rte_pktmbuf_alloc(vr_dpdk.rss_mempool);
177 0 : if (!m)
178 0 : return (NULL);
179 :
180 0 : return vr_dpdk_packet_get(m, NULL);
181 : }
182 :
183 : static struct vr_packet *
184 0 : dpdk_palloc_head(struct vr_packet *pkt, unsigned int size)
185 : {
186 : /* TODO: not implemented */
187 0 : RTE_LOG(ERR, VROUTER, "%s: not implemented\n", __func__);
188 0 : return NULL;
189 : }
190 :
191 : static struct vr_packet *
192 0 : dpdk_pexpand_head(struct vr_packet *pkt, unsigned int hspace)
193 : {
194 : /* TODO: not implemented */
195 0 : return pkt;
196 : }
197 :
198 : static void
199 39 : dpdk_pfree(struct vr_packet *pkt, unsigned short reason)
200 : {
201 39 : if (pkt) {
202 : /* Handle Vrouter statistics */
203 39 : pkt_drop_stats(pkt->vp_if, reason, rte_lcore_id());
204 :
205 39 : rte_pktmbuf_free(vr_dpdk_pkt_to_mbuf(pkt));
206 : }
207 39 : }
208 :
209 : void
210 0 : vr_dpdk_pfree(struct rte_mbuf *mbuf, struct vr_interface *vif, unsigned short reason)
211 : {
212 0 : struct vr_packet *pkt = vr_dpdk_mbuf_to_pkt(mbuf);
213 0 : vr_dpdk_packet_get(mbuf, vif);
214 0 : dpdk_pfree(pkt, reason);
215 0 : }
216 :
217 :
218 : static void
219 42 : dpdk_preset(struct vr_packet *pkt)
220 : {
221 : struct rte_mbuf *m;
222 :
223 42 : if (!pkt)
224 0 : rte_panic("%s: NULL pkt", __func__);
225 :
226 42 : m = vr_dpdk_pkt_to_mbuf(pkt);
227 :
228 : /* Reset packet data */
229 42 : pkt->vp_data = rte_pktmbuf_headroom(m);
230 42 : pkt->vp_tail = rte_pktmbuf_headroom(m) + rte_pktmbuf_data_len(m);
231 42 : pkt->vp_len = rte_pktmbuf_data_len(m);
232 :
233 42 : return;
234 : }
235 :
236 : /**
237 : * Copy packet mbuf data to another packet mbuf.
238 : *
239 : * @param dst
240 : * The destination packet mbuf.
241 : * @param src
242 : * The source packet mbuf.
243 : */
244 :
245 : static inline void
246 6 : dpdk_pktmbuf_data_copy(struct rte_mbuf *dst, struct rte_mbuf *src)
247 : {
248 6 : dst->data_off = src->data_off;
249 6 : dst->port = src->port;
250 6 : dst->ol_flags = src->ol_flags;
251 : #ifdef IND_ATTACHED_MBUF
252 6 : dst->ol_flags &= (~IND_ATTACHED_MBUF);
253 : #endif
254 6 : dst->packet_type = src->packet_type;
255 6 : dst->data_len = src->data_len;
256 6 : dst->pkt_len = src->pkt_len;
257 6 : dst->vlan_tci = src->vlan_tci;
258 6 : dst->hash = src->hash;
259 6 : dst->seqn = src->seqn;
260 6 : dst->userdata = src->userdata;
261 6 : dst->tx_offload = src->tx_offload;
262 :
263 : __rte_mbuf_sanity_check(dst, 1);
264 : __rte_mbuf_sanity_check(src, 0);
265 :
266 : /* copy data */
267 6 : rte_memcpy(rte_pktmbuf_mtod(dst, void *),
268 6 : rte_pktmbuf_mtod(src, void *), src->data_len);
269 6 : }
270 :
271 : /**
272 : * Creates a copy of the given packet mbuf.
273 : * TODO: remove once rte_pktmbuf_copy() is in DPDK
274 : *
275 : * Walks through all segments of the given packet mbuf, and for each of them:
276 : * - Creates a new packet mbuf from the given pool.
277 : * - Copies data to the newly created mbuf.
278 : * Then updates pkt_len and nb_segs of the "copy" packet mbuf to match values
279 : * from the original packet mbuf.
280 : *
281 : * @param md
282 : * The packet mbuf to be copied.
283 : * @param mp
284 : * The mempool from which the "copy" mbufs are allocated.
285 : * @return
286 : * - The pointer to the new "copy" mbuf on success.
287 : * - NULL if allocation fails.
288 : */
289 : inline struct rte_mbuf *
290 6 : vr_dpdk_pktmbuf_copy(struct rte_mbuf *md, struct rte_mempool *mp)
291 : {
292 : struct rte_mbuf *mc, *mi, **prev;
293 : uint32_t pktlen;
294 : uint8_t nseg;
295 :
296 6 : if (unlikely ((mc = rte_pktmbuf_alloc(mp)) == NULL))
297 0 : return (NULL);
298 :
299 6 : mi = mc;
300 6 : prev = &mi->next;
301 6 : pktlen = md->pkt_len;
302 6 : nseg = 0;
303 :
304 : do {
305 6 : nseg++;
306 6 : dpdk_pktmbuf_data_copy(mi, md);
307 6 : *prev = mi;
308 6 : prev = &mi->next;
309 6 : } while ((md = md->next) != NULL &&
310 0 : (mi = rte_pktmbuf_alloc(mp)) != NULL);
311 :
312 6 : *prev = NULL;
313 6 : mc->nb_segs = nseg;
314 6 : mc->pkt_len = pktlen;
315 :
316 : /* Allocation of new indirect segment failed */
317 6 : if (unlikely (mi == NULL)) {
318 0 : rte_pktmbuf_free(mc);
319 0 : return (NULL);
320 : }
321 :
322 : __rte_mbuf_sanity_check(mc, 1);
323 6 : return (mc);
324 : }
325 :
326 : /**
327 : * Creates a copy of the given packet mbuf.
328 : *
329 : * Creates a new packet mbuf from the given pool.
330 : * Walks through all segments of the given packet mbuf, and appends them
331 : * in the new packet upto the size of the new packet and ignores the rest.
332 : * This needs to be done because KNI doesn't handle mbuf chains.
333 : *
334 : * @param md
335 : * The packet mbuf to be copied.
336 : * @param mp
337 : * The mempool from which the "copy" mbufs are allocated.
338 : * @return
339 : * - The pointer to the new "copy" mbuf on success.
340 : * - NULL if allocation fails.
341 : */
342 :
343 : inline struct rte_mbuf *
344 0 : vr_dpdk_pktmbuf_copy_mon(struct rte_mbuf *md, struct rte_mempool *mp)
345 : {
346 : struct rte_mbuf *mc;
347 : char *append_ptr;
348 : uint32_t append_len;
349 :
350 0 : if (unlikely ((mc = rte_pktmbuf_alloc(mp)) == NULL))
351 0 : return (NULL);
352 :
353 0 : dpdk_pktmbuf_data_copy(mc, md);
354 0 : mc->pkt_len = md->data_len;
355 :
356 0 : while (md->next)
357 : {
358 0 : md = md->next;
359 0 : append_ptr = rte_pktmbuf_append(mc, md->data_len);
360 0 : append_len = (append_ptr)? md->data_len : rte_pktmbuf_tailroom(mc);
361 0 : if (append_ptr == NULL) {
362 0 : append_ptr = rte_pktmbuf_append(mc, rte_pktmbuf_tailroom(mc));
363 0 : rte_memcpy(append_ptr, rte_pktmbuf_mtod(md, void*), append_len);
364 0 : break;
365 : }
366 0 : rte_memcpy(append_ptr, rte_pktmbuf_mtod(md, void*), append_len);
367 : }
368 :
369 0 : mc->nb_segs = 1;
370 :
371 : __rte_mbuf_sanity_check(mc, 1);
372 0 : return (mc);
373 : }
374 :
375 : /* VRouter callback */
376 : static struct vr_packet *
377 16 : dpdk_pclone(struct vr_packet *pkt)
378 : {
379 : struct rte_mbuf *m, *m_clone;
380 : struct vr_packet *pkt_clone;
381 :
382 16 : m = vr_dpdk_pkt_to_mbuf(pkt);
383 :
384 16 : m_clone = rte_pktmbuf_clone(m, vr_dpdk.rss_mempool);
385 16 : if (!m_clone)
386 0 : return NULL;
387 :
388 : /* clone vr_packet data */
389 16 : pkt_clone = vr_dpdk_mbuf_to_pkt(m_clone);
390 16 : *pkt_clone = *pkt;
391 16 : pkt_clone->vp_cpu = vr_get_cpu();
392 :
393 16 : return pkt_clone;
394 : }
395 :
396 : /* Copy the specified number of bytes from the source mbuf to the
397 : * destination buffer.
398 : */
399 : static int
400 0 : dpdk_pktmbuf_copy_bits(const struct rte_mbuf *mbuf, int offset,
401 : void *to, int len)
402 : {
403 : /* how many bytes to copy in loop */
404 0 : int copy = 0;
405 : /* loop pointer to a source data */
406 : void *from;
407 :
408 : /* check total packet length */
409 0 : if (unlikely(offset > (int)rte_pktmbuf_pkt_len(mbuf) - len))
410 0 : goto fault;
411 :
412 0 : do {
413 0 : if (offset < rte_pktmbuf_data_len(mbuf)) {
414 : /* copy a piece of data */
415 0 : from = (void *)(rte_pktmbuf_mtod_offset(mbuf, uintptr_t, offset));
416 0 : copy = rte_pktmbuf_data_len(mbuf) - offset;
417 0 : if (copy > len)
418 0 : copy = len;
419 0 : rte_memcpy(to, from, copy);
420 0 : offset = 0;
421 : } else {
422 0 : offset -= rte_pktmbuf_data_len(mbuf);
423 : }
424 : /* get next mbuf */
425 0 : to += copy;
426 0 : len -= copy;
427 0 : mbuf = mbuf->next;
428 0 : } while (unlikely(len > 0 && NULL != mbuf));
429 :
430 0 : if (likely(0 == len))
431 0 : return 0;
432 :
433 0 : fault:
434 0 : return -EFAULT;
435 : }
436 :
437 : /* VRouter callback */
438 : static int
439 0 : dpdk_pcopy(unsigned char *dst, struct vr_packet *p_src,
440 : unsigned int offset, unsigned int len)
441 : {
442 : int ret;
443 : struct rte_mbuf *src;
444 :
445 0 : src = vr_dpdk_pkt_to_mbuf(p_src);
446 0 : ret = dpdk_pktmbuf_copy_bits(src, offset, dst, len);
447 0 : if (ret)
448 0 : return ret;
449 :
450 0 : return len;
451 : }
452 :
453 : static unsigned short
454 771 : dpdk_pfrag_len(struct vr_packet *pkt)
455 : {
456 : struct rte_mbuf *m;
457 :
458 771 : m = vr_dpdk_pkt_to_mbuf(pkt);
459 :
460 771 : return rte_pktmbuf_pkt_len(m) - rte_pktmbuf_data_len(m);
461 : }
462 :
463 : static unsigned short
464 3 : dpdk_phead_len(struct vr_packet *pkt)
465 : {
466 : struct rte_mbuf *m;
467 :
468 3 : m = vr_dpdk_pkt_to_mbuf(pkt);
469 :
470 3 : return rte_pktmbuf_data_len(m);
471 : }
472 :
473 : static void
474 1 : dpdk_pset_data(struct vr_packet *pkt, unsigned short offset)
475 : {
476 : struct rte_mbuf *m;
477 :
478 1 : m = vr_dpdk_pkt_to_mbuf(pkt);
479 1 : m->buf_addr = pkt->vp_head;
480 1 : m->data_off = offset;
481 :
482 1 : return;
483 : }
484 :
485 : static unsigned int
486 2107028 : dpdk_get_cpu(void)
487 : {
488 2107028 : unsigned lcore_id = rte_lcore_id();
489 :
490 : /* For the RCU thread we get LCORE_ID_ANY, so memory stats and
491 : * other functions get crashed trying to index per-cpu data.
492 : */
493 2106746 : if (lcore_id < vr_num_cpus)
494 2105142 : return lcore_id;
495 : else
496 1604 : return 0;
497 : }
498 :
499 : /* DPDK timer callback */
500 : static void
501 6334 : dpdk_timer(struct rte_timer *tim, void *arg)
502 : {
503 6334 : struct vr_timer *vtimer = (struct vr_timer*)arg;
504 :
505 6334 : vtimer->vt_timer(vtimer->vt_vr_arg);
506 6334 : }
507 :
508 : static int
509 53 : dpdk_create_timer(struct vr_timer *vtimer)
510 : {
511 : struct rte_timer *timer;
512 : uint64_t hz, ticks;
513 :
514 53 : timer = rte_zmalloc("vr_dpdk_timer", sizeof(struct rte_timer), 0);
515 :
516 53 : if (!timer) {
517 0 : RTE_LOG(ERR, VROUTER, "Error allocating RTE timer\n");
518 0 : return -1;
519 : }
520 :
521 : /* init timer */
522 53 : rte_timer_init(timer);
523 53 : vtimer->vt_os_arg = (void *)timer;
524 :
525 : /* reset timer */
526 53 : hz = rte_get_timer_hz();
527 53 : ticks = hz * vtimer->vt_msecs / 1000;
528 53 : if (rte_timer_reset(timer, ticks, PERIODICAL, VR_DPDK_TIMER_LCORE_ID,
529 : dpdk_timer, vtimer) == -1) {
530 0 : RTE_LOG(ERR, VROUTER, "Error resetting timer\n");
531 0 : rte_free(timer);
532 :
533 0 : return -1;
534 : }
535 :
536 53 : return 0;
537 : }
538 :
539 : static void
540 53 : dpdk_delete_timer(struct vr_timer *vtimer)
541 : {
542 53 : struct rte_timer *timer = (struct rte_timer*)vtimer->vt_os_arg;
543 :
544 53 : if (timer) {
545 53 : rte_timer_stop_sync(timer);
546 53 : rte_free(timer);
547 : } else {
548 0 : RTE_LOG(ERR, VROUTER, "No timer to delete\n");
549 : }
550 53 : }
551 :
552 : static void
553 42 : dpdk_get_time(uint64_t *sec, uint64_t *usec)
554 : {
555 : struct timespec ts;
556 :
557 42 : *sec = *usec = 0;
558 42 : if (-1 == clock_gettime(CLOCK_REALTIME, &ts))
559 0 : return;
560 :
561 42 : *sec = ts.tv_sec;
562 42 : *usec = ts.tv_nsec / 1000;
563 :
564 42 : return;
565 : }
566 :
567 : static void
568 2412 : dpdk_get_mono_time(uint64_t *sec, uint64_t *nsec)
569 : {
570 : struct timespec ts;
571 :
572 2412 : *sec = *nsec = 0;
573 2412 : if (-1 == clock_gettime(CLOCK_MONOTONIC, &ts))
574 0 : return;
575 :
576 2412 : *sec = ts.tv_sec;
577 2412 : *nsec = ts.tv_nsec;
578 :
579 2412 : return;
580 : }
581 :
582 : static void
583 0 : dpdk_htable_work_cb(struct vrouter *router __attribute__((unused)), void *arg)
584 : {
585 0 : struct dpdk_work_cb_data *defer = (struct dpdk_work_cb_data *)arg;
586 0 : defer->dwc_fn(defer->dwc_data);
587 :
588 0 : return;
589 : }
590 :
591 : /* Work callback called on NetLink lcore */
592 : static int
593 318 : dpdk_schedule_work(unsigned int cpu, void (*fn)(void *), void *arg)
594 : {
595 : struct dpdk_work_cb_data *defer;
596 :
597 318 : if (!fn)
598 0 : return -1;
599 :
600 : /*
601 : * Fix ME:
602 : * Use RCU to defer the work only for hash tables. Rest all, invoke
603 : * function as is
604 : * This is a temporary fix to ensure that hash table deletion does
605 : * not happen parrallely in different CPU's
606 : */
607 :
608 318 : if (fn == vr_htable_hentry_scheduled_delete) {
609 0 : defer = vr_get_defer_data(sizeof(*defer));
610 0 : if (!defer)
611 0 : return -1;
612 :
613 0 : defer->dwc_fn = fn;
614 0 : defer->dwc_data = arg;
615 0 : vr_defer(NULL, dpdk_htable_work_cb, defer);
616 :
617 0 : return 0;
618 : }
619 :
620 318 : fn(arg);
621 :
622 318 : return 0;
623 : }
624 :
625 : static void
626 394 : dpdk_delay_op(void)
627 : {
628 394 : synchronize_rcu();
629 :
630 394 : return;
631 : }
632 :
633 : /* RCU callback called on RCU thread */
634 : static void
635 257 : dpdk_rcu_cb(struct rcu_head *rh)
636 : {
637 : int i;
638 : struct vr_dpdk_rcu_cb_data *cb_data;
639 : struct vr_defer_data *defer;
640 : struct vr_flow_queue *vfq;
641 : struct vr_packet_node *pnode;
642 :
643 :
644 257 : cb_data = CONTAINER_OF(rcd_rcu, struct vr_dpdk_rcu_cb_data, rh);
645 :
646 : /* check if we need to pass the callback to packet lcore */
647 257 : if ((cb_data->rcd_user_cb == vr_flow_defer_cb) &&
648 : cb_data->rcd_user_data) {
649 6 : defer = (struct vr_defer_data *)cb_data->rcd_user_data;
650 6 : vfq = ((struct vr_flow_defer_data *)defer->vdd_data)->vfdd_flow_queue;
651 6 : if (vfq) {
652 8 : for (i = 0; i < VR_MAX_FLOW_QUEUE_ENTRIES; i++) {
653 6 : pnode = &vfq->vfq_pnodes[i];
654 6 : if (pnode->pl_packet) {
655 : RTE_LOG_DP(DEBUG, VROUTER, "%s: lcore %u passing RCU callback "
656 : "to lcore %u\n", __func__, rte_lcore_id(),
657 : VR_DPDK_PACKET_LCORE_ID);
658 0 : vr_dpdk_lcore_cmd_post(VR_DPDK_PACKET_LCORE_ID,
659 : VR_DPDK_LCORE_RCU_CMD, (uintptr_t)rh);
660 0 : return;
661 : }
662 : }
663 : RTE_LOG_DP(DEBUG, VROUTER, "%s: lcore %u passing RCU callback to lcore %u\n",
664 : __func__, rte_lcore_id(), VR_DPDK_PACKET_LCORE_ID);
665 : }
666 : }
667 : /* no need to send any packets, so just call the callback */
668 257 : cb_data->rcd_user_cb(cb_data->rcd_router, cb_data->rcd_user_data);
669 257 : vr_free(cb_data, VR_DEFER_OBJECT);
670 : }
671 :
672 : static void
673 257 : dpdk_defer(struct vrouter *router, vr_defer_cb user_cb, void *data)
674 : {
675 : struct vr_dpdk_rcu_cb_data *cb_data;
676 :
677 257 : cb_data = CONTAINER_OF(rcd_user_data, struct vr_dpdk_rcu_cb_data, data);
678 257 : cb_data->rcd_user_cb = user_cb;
679 257 : cb_data->rcd_router = router;
680 257 : call_rcu(&cb_data->rcd_rcu, dpdk_rcu_cb);
681 257 : }
682 :
683 : static void *
684 257 : dpdk_get_defer_data(unsigned int len)
685 : {
686 : struct vr_dpdk_rcu_cb_data *cb_data;
687 :
688 257 : if (!len)
689 0 : return NULL;
690 :
691 257 : cb_data = dpdk_malloc(sizeof(*cb_data) + len, VR_DEFER_OBJECT);
692 257 : if (!cb_data) {
693 0 : return NULL;
694 : }
695 :
696 257 : return cb_data->rcd_user_data;
697 : }
698 :
699 : static void
700 0 : dpdk_put_defer_data(void *data)
701 : {
702 : struct vr_dpdk_rcu_cb_data *cb_data;
703 :
704 0 : if (!data)
705 0 : return;
706 :
707 0 : cb_data = CONTAINER_OF(rcd_user_data, struct vr_dpdk_rcu_cb_data, data);
708 0 : dpdk_free(cb_data, VR_DEFER_OBJECT);
709 :
710 0 : return;
711 : }
712 :
713 : static void *
714 0 : dpdk_network_header(struct vr_packet *pkt)
715 : {
716 0 : if (pkt->vp_network_h < pkt->vp_end)
717 0 : return pkt->vp_head + pkt->vp_network_h;
718 :
719 : /* TODO: for buffer chain? */
720 0 : rte_panic("%s: buffer chain not supported\n", __func__);
721 :
722 : return NULL;
723 : }
724 :
725 : static void *
726 0 : dpdk_inner_network_header(struct vr_packet *pkt)
727 : {
728 : /* TODO: not used? */
729 0 : rte_panic("%s: not implemented\n", __func__);
730 :
731 : return NULL;
732 : }
733 :
734 : static void *
735 0 : dpdk_data_at_offset(struct vr_packet *pkt, unsigned short off)
736 : {
737 0 : if (off < pkt->vp_end)
738 0 : return pkt->vp_head + off;
739 :
740 : /* TODO: for buffer chain? */
741 0 : rte_panic("%s: buffer chain not supported\n", __func__);
742 :
743 : return NULL;
744 : }
745 :
746 : /*
747 : * dpdk_pheader_pointer
748 : * return pointer to data at pkt->vp_data offset if hdr_len bytes
749 : * in continuous memory, otherwise copy data to buf
750 : */
751 : static void *
752 9 : dpdk_pheader_pointer(struct vr_packet *pkt, unsigned short hdr_len, void *buf)
753 : {
754 : struct rte_mbuf *m;
755 : int offset;
756 :
757 9 : m = vr_dpdk_pkt_to_mbuf(pkt);
758 :
759 : /*
760 : * vp_data is offset from start of buffer,
761 : * so first calculate offset from start of mbuf payload
762 : */
763 9 : offset = pkt->vp_data - rte_pktmbuf_headroom(m);
764 9 : if ((offset + hdr_len) < rte_pktmbuf_data_len(m))
765 9 : return (void *)((uintptr_t)m->buf_addr + pkt->vp_data);
766 : else {
767 0 : int len = rte_pktmbuf_data_len(m) - offset;
768 0 : void *tmp_buf = buf;
769 :
770 0 : rte_memcpy(tmp_buf, rte_pktmbuf_mtod_offset(m, char *, offset), len);
771 0 : hdr_len -= len;
772 0 : tmp_buf = (void *)((uintptr_t)tmp_buf + len);
773 :
774 : /* iterate thru buffers chain */
775 0 : while (hdr_len) {
776 0 : m = m->next;
777 0 : if (!m)
778 0 : return (NULL);
779 0 : if (hdr_len > rte_pktmbuf_data_len(m))
780 0 : len = rte_pktmbuf_data_len(m);
781 : else
782 0 : len = hdr_len;
783 :
784 0 : rte_memcpy(tmp_buf, rte_pktmbuf_mtod(m, void *), len);
785 :
786 0 : tmp_buf = (void *)((uintptr_t)tmp_buf + len);
787 0 : hdr_len -= len;
788 : }
789 :
790 0 : return (buf);
791 : }
792 : }
793 :
794 : /* VRouter callback */
795 : static int
796 6 : dpdk_pcow(struct vr_packet **pktp, unsigned short head_room)
797 : {
798 6 : struct vr_packet *pkt = *pktp;
799 6 : struct rte_mbuf *mbuf = vr_dpdk_pkt_to_mbuf(pkt);
800 : struct rte_mbuf *m_copy;
801 : struct vr_packet *p_copy;
802 : struct rte_mbuf *mbuf_new;
803 :
804 : /*
805 : * If this is an indirect mbuf, allocate a new mbuf and copy
806 : * its data. Then free the original mbuf.
807 : */
808 6 : if (RTE_MBUF_CLONED(mbuf)) {
809 6 : m_copy = vr_dpdk_pktmbuf_copy(mbuf, mbuf->pool);
810 6 : if (!m_copy) {
811 0 : return -ENOMEM;
812 : }
813 :
814 6 : p_copy = vr_dpdk_mbuf_to_pkt(m_copy);
815 6 : *p_copy = *pkt;
816 6 : p_copy->vp_head = m_copy->buf_addr;
817 :
818 6 : rte_pktmbuf_free(mbuf);
819 6 : mbuf = m_copy;
820 6 : *pktp = p_copy;
821 : }
822 :
823 6 : if (head_room > rte_pktmbuf_headroom(mbuf)) {
824 :
825 : /* When requested headroom is higher than configured pktmuf_headroom,
826 : * Create a new memory buffer and link to the old mbuf */
827 0 : mbuf_new = rte_pktmbuf_alloc(vr_dpdk.rss_mempool);
828 0 : if (!mbuf_new) {
829 0 : return -ENOMEM;
830 : }
831 :
832 0 : pkt = vr_dpdk_packet_get(mbuf_new, pkt->vp_if);
833 0 : if(!pkt)
834 0 : return -ENOMEM;
835 :
836 0 : mbuf_new->next = mbuf;
837 0 : mbuf_new->nb_segs++;
838 0 : mbuf_new->ol_flags = mbuf->ol_flags;
839 0 : mbuf_new->data_off += head_room;
840 0 : mbuf_new->pkt_len += mbuf->data_len;
841 :
842 0 : pkt->vp_data = mbuf_new->data_off;
843 0 : *pktp = pkt;
844 : }
845 :
846 6 : return 0;
847 : }
848 :
849 : /*
850 : * dpdk_get_udp_src_port - return a source port for the outer UDP header.
851 : * The source port is based on a hash of the inner IP source/dest addresses,
852 : * vrf (and inner TCP/UDP ports in the future). The label from fmd
853 : * will be used in the future to detect whether it is a L2/L3 packet.
854 : * Returns 0 on error, valid source port otherwise.
855 : *
856 : * Based on linux/vrouter_mod.c:lh_get_udp_src_port
857 : * Copyright (c) 2013, 2014 Juniper Networks, Inc.
858 : */
859 : static uint16_t
860 38 : dpdk_get_udp_src_port(struct vr_packet *pkt, struct vr_forwarding_md *fmd,
861 : unsigned short vrf)
862 : {
863 38 : struct rte_mbuf *mbuf = vr_dpdk_pkt_to_mbuf(pkt);
864 : unsigned int pull_len;
865 : uint32_t ip_src, ip_dst, hashval, port_range;
866 : struct vr_ip *iph;
867 : uint16_t port;
868 38 : uint16_t sport = 0, dport = 0;
869 : struct vr_fragment *frag;
870 38 : struct vrouter *router = vrouter_get(0);
871 : uint32_t hash_key[5];
872 : uint16_t *l4_hdr;
873 : struct vr_flow_entry *fentry;
874 :
875 38 : if (likely(mbuf->ol_flags & PKT_RX_RSS_HASH)) {
876 36 : hashval = mbuf->hash.rss;
877 : } else {
878 2 : if (unlikely(hashrnd_inited == 0)) {
879 0 : vr_hashrnd = random();
880 0 : hashrnd_inited = 1;
881 : }
882 :
883 2 : if (pkt->vp_type == VP_TYPE_IP) {
884 : /* Ideally the below code is only for VP_TYPE_IP and not
885 : * for IP6. But having explicit check for IP only break IP6
886 : */
887 0 : pull_len = sizeof(struct iphdr);
888 0 : pull_len += pkt_get_network_header_off(pkt);
889 0 : pull_len -= rte_pktmbuf_headroom(mbuf);
890 :
891 : /* It's safe to assume the ip hdr is within this mbuf, so we skip
892 : * all the header checks.
893 : */
894 :
895 0 : iph = (struct vr_ip *)(mbuf->buf_addr + pkt_get_network_header_off(pkt));
896 0 : if (vr_ip_transport_header_valid(iph)) {
897 0 : if ((iph->ip_proto == VR_IP_PROTO_TCP) ||
898 0 : (iph->ip_proto == VR_IP_PROTO_UDP)) {
899 0 : l4_hdr = (__u16 *) (((char *) iph) + (iph->ip_hl * 4));
900 0 : sport = *l4_hdr;
901 0 : dport = *(l4_hdr+1);
902 : }
903 : } else {
904 : /*
905 : * If this fragment required flow lookup, get the source and
906 : * dst port from the frag entry. Otherwise, use 0 as the source
907 : * dst port (which could result in fragments getting a different
908 : * outer UDP source port than non-fragments in the same flow).
909 : */
910 0 : frag = vr_fragment_get(router, vrf, iph, 0);
911 0 : if (frag) {
912 0 : sport = frag->f_sport;
913 0 : dport = frag->f_dport;
914 : }
915 : }
916 :
917 0 : if (fmd && fmd->fmd_flow_index >= 0) {
918 0 : fentry = vr_flow_get_entry(router, fmd->fmd_flow_index);
919 0 : if (fentry) {
920 0 : vr_dpdk_mbuf_reset(pkt);
921 0 : return fentry->fe_udp_src_port;
922 : }
923 : }
924 :
925 0 : ip_src = iph->ip_saddr;
926 0 : ip_dst = iph->ip_daddr;
927 :
928 0 : hash_key[0] = ip_src;
929 0 : hash_key[1] = ip_dst;
930 0 : hash_key[2] = vrf;
931 0 : hash_key[3] = sport;
932 0 : hash_key[4] = dport;
933 :
934 0 : hashval = rte_jhash(hash_key, 20, vr_hashrnd);
935 0 : vr_dpdk_mbuf_reset(pkt);
936 : } else {
937 :
938 : /* We treat all non-ip packets as L2 here. For V6 we can extract
939 : * the required fieleds explicity and manipulate the src port
940 : */
941 :
942 2 : if (pkt_head_len(pkt) < ETH_HLEN)
943 0 : goto error;
944 :
945 2 : hashval = vr_hash(pkt_data(pkt), ETH_HLEN, vr_hashrnd);
946 : /* Include the VRF to calculate the hash */
947 2 : hashval = vr_hash_2words(hashval, vrf, vr_hashrnd);
948 : }
949 : } /* !PKT_RX_RSS_HASH */
950 :
951 :
952 : /*
953 : * Convert the hash value to a value in the port range that we want
954 : * for dynamic UDP ports
955 : */
956 38 : port_range = VR_MUDP_PORT_RANGE_END - VR_MUDP_PORT_RANGE_START;
957 38 : port = (uint16_t) (((uint64_t) hashval * port_range) >> 32);
958 :
959 38 : if (unlikely(port > port_range)) {
960 : /*
961 : * Shouldn't happen...
962 : */
963 0 : port = 0;
964 : }
965 :
966 38 : return (port + VR_MUDP_PORT_RANGE_START);
967 :
968 0 : error:
969 0 : vr_dpdk_mbuf_reset(pkt);
970 0 : return 0;
971 : }
972 :
973 : /**
974 : * dpdk_adjust_tcp_mss - helper adjusting TCP Maximum Segment Size, used in
975 : * dpdk_pkt_from_vm_tcp_mss_adj vRouter callback for packets from the VM and in
976 : * vr_ip_transport_parse to perform MSS adjust for packets sent to the VM.
977 : */
978 : void
979 0 : dpdk_adjust_tcp_mss(struct tcphdr *tcph, unsigned short overlay_len,
980 : unsigned char iph_len)
981 : {
982 0 : int opt_off = sizeof(struct tcphdr);
983 0 : u_int8_t *opt_ptr = (u_int8_t *) tcph;
984 : u_int16_t pkt_mss, max_mss, mtu;
985 : unsigned int csum;
986 : uint8_t port_id;
987 0 : struct vrouter *router = vrouter_get(0);
988 :
989 0 : if ((tcph == NULL) || !(tcph->syn) || (router == NULL))
990 0 : return;
991 :
992 0 : if (router->vr_eth_if[0] == NULL)
993 0 : return;
994 :
995 0 : while (opt_off < (tcph->doff * 4)) {
996 0 : switch (opt_ptr[opt_off]) {
997 0 : case TCPOPT_EOL:
998 0 : return;
999 :
1000 0 : case TCPOPT_NOP:
1001 0 : opt_off++;
1002 0 : continue;
1003 :
1004 0 : case TCPOPT_MAXSEG:
1005 0 : if ((opt_off + TCPOLEN_MAXSEG) > (tcph->doff*4))
1006 0 : return;
1007 :
1008 0 : if (opt_ptr[opt_off+1] != TCPOLEN_MAXSEG)
1009 0 : return;
1010 :
1011 0 : pkt_mss = (opt_ptr[opt_off+2] << 8) | opt_ptr[opt_off+3];
1012 0 : if (router->vr_eth_if[0] == NULL)
1013 0 : return;
1014 :
1015 0 : port_id = (((struct vr_dpdk_ethdev *)(router->vr_eth_if[0]->vif_os))->
1016 : ethdev_port_id);
1017 0 : rte_eth_dev_get_mtu(port_id, &mtu);
1018 :
1019 0 : max_mss = mtu - (overlay_len + iph_len + sizeof(struct tcphdr));
1020 :
1021 0 : if (pkt_mss > max_mss) {
1022 0 : opt_ptr[opt_off+2] = (max_mss & 0xff00) >> 8;
1023 0 : opt_ptr[opt_off+3] = max_mss & 0xff;
1024 :
1025 : /* Recalculate checksum */
1026 0 : csum = (unsigned short)(~rte_cpu_to_be_16(tcph->check));
1027 0 : csum = csum + (unsigned short)~pkt_mss;
1028 0 : csum = (csum & 0xffff) + (csum >> 16);
1029 0 : csum += max_mss;
1030 0 : csum = (csum & 0xffff) + (csum >> 16);
1031 0 : tcph->check = rte_cpu_to_be_16(~((unsigned short)csum));
1032 : }
1033 0 : return;
1034 :
1035 0 : default:
1036 0 : if ((opt_off + 1) == (tcph->doff*4))
1037 0 : return;
1038 :
1039 0 : if (opt_ptr[opt_off+1])
1040 0 : opt_off += opt_ptr[opt_off+1];
1041 : else
1042 0 : opt_off++;
1043 :
1044 0 : continue;
1045 : } /* switch */
1046 : } /* while */
1047 :
1048 0 : return;
1049 : }
1050 :
1051 : /*
1052 : * dpdk_pkt_from_vm_tcp_mss_adj - perform TCP MSS adjust, if required, for packets
1053 : * that are sent by a VM. Returns 0 on success, non-zero otherwise.
1054 : */
1055 : static int
1056 103 : dpdk_pkt_from_vm_tcp_mss_adj(struct vr_packet *pkt, unsigned short overlay_len)
1057 : {
1058 103 : struct rte_mbuf *m = vr_dpdk_pkt_to_mbuf(pkt);
1059 103 : struct vr_ip *ip4h = NULL;
1060 103 : struct vr_ip6 *ip6h = NULL;
1061 : struct tcphdr *tcph;
1062 : int offset;
1063 103 : unsigned char iph_len = 0, iph_proto = 0;
1064 :
1065 : /* check if whole ip header is in the packet */
1066 103 : if (pkt->vp_type == VP_TYPE_IP) {
1067 96 : offset = sizeof(struct vr_ip);
1068 96 : if (pkt->vp_data + offset < pkt->vp_end)
1069 96 : ip4h = (struct vr_ip *) ((uintptr_t)m->buf_addr + pkt->vp_data);
1070 : else
1071 0 : rte_panic("%s: ip header not in first buffer\n", __func__);
1072 96 : iph_proto = ip4h->ip_proto;
1073 96 : iph_len = ip4h->ip_hl * 4;
1074 :
1075 : /*
1076 : * If this is a fragment and not the first one, it can be ignored
1077 : */
1078 96 : if (ip4h->ip_frag_off & rte_cpu_to_be_16(IP_OFFMASK))
1079 36 : goto out;
1080 7 : } else if (pkt->vp_type == VP_TYPE_IP6) {
1081 7 : iph_len = offset = sizeof(struct vr_ip6);
1082 7 : if (pkt->vp_data + offset < pkt->vp_end)
1083 7 : ip6h = (struct vr_ip6 *) ((uintptr_t)m->buf_addr + pkt->vp_data);
1084 : else
1085 0 : rte_panic("%s: ip header not in first buffer\n", __func__);
1086 7 : iph_proto = ip6h->ip6_nxt;
1087 : }
1088 :
1089 67 : if (iph_proto != VR_IP_PROTO_TCP)
1090 49 : goto out;
1091 :
1092 : /*
1093 : * Now we know exact ip header length,
1094 : * check if whole tcp header is also in the packet
1095 : */
1096 18 : offset = iph_len + sizeof(struct tcphdr);
1097 :
1098 18 : if (pkt->vp_data + offset < pkt->vp_end)
1099 18 : tcph = (struct tcphdr *)pkt_data_at_offset(pkt, pkt->vp_data + iph_len);
1100 : else
1101 0 : rte_panic("%s: tcp header not in first buffer\n", __func__);
1102 :
1103 18 : if ((tcph->doff << 2) <= (sizeof(struct tcphdr))) {
1104 : /*Nothing to do if there are no TCP options */
1105 18 : goto out;
1106 : }
1107 :
1108 :
1109 0 : offset += (tcph->doff << 2) - sizeof(struct tcphdr);
1110 0 : if (pkt->vp_data + offset > pkt->vp_end)
1111 0 : rte_panic("%s: tcp header outside first buffer\n", __func__);
1112 :
1113 :
1114 0 : dpdk_adjust_tcp_mss(tcph, overlay_len, iph_len);
1115 :
1116 103 : out:
1117 103 : return 0;
1118 : }
1119 :
1120 : static unsigned int
1121 72 : dpdk_pgso_size(struct vr_packet *pkt)
1122 : {
1123 72 : struct rte_mbuf *m = vr_dpdk_pkt_to_mbuf(pkt);
1124 :
1125 72 : return m->tso_segsz;
1126 : }
1127 :
1128 : static void
1129 9 : dpdk_add_mpls(struct vrouter *router, unsigned mpls_label)
1130 : {
1131 : int ret, i;
1132 : struct vr_interface *eth_vif;
1133 :
1134 39204 : for (i = 0; i < router->vr_max_interfaces; i++) {
1135 39195 : eth_vif = __vrouter_get_interface(router, i);
1136 39195 : if (eth_vif && (eth_vif->vif_type == VIF_TYPE_PHYSICAL)
1137 10 : && (eth_vif->vif_flags & VIF_FLAG_FILTERING_OFFLOAD)) {
1138 0 : RTE_LOG(INFO, VROUTER, "Enabling hardware acceleration on vif %u for MPLS %u\n",
1139 : (unsigned)eth_vif->vif_idx, mpls_label);
1140 0 : if (!eth_vif->vif_ip) {
1141 0 : RTE_LOG(ERR, VROUTER, " error accelerating MPLS %u: no IP address set\n",
1142 : mpls_label);
1143 0 : continue;
1144 : }
1145 0 : ret = vr_dpdk_lcore_mpls_schedule(eth_vif,
1146 0 : ntohs(eth_vif->vif_ip), mpls_label);
1147 0 : if (ret != 0)
1148 0 : RTE_LOG(INFO, VROUTER, " error accelerating MPLS %u: %s (%d)\n",
1149 : mpls_label, rte_strerror(-ret), -ret);
1150 : }
1151 : }
1152 :
1153 9 : }
1154 :
1155 : static void
1156 9 : dpdk_del_mpls(struct vrouter *router, unsigned mpls_label)
1157 : {
1158 : /* TODO: not implemented */
1159 9 : }
1160 :
1161 : static int
1162 0 : dpdk_pkt_may_pull(struct vr_packet *pkt, unsigned int len)
1163 : {
1164 0 : struct rte_mbuf *mbuf = vr_dpdk_pkt_to_mbuf(pkt);
1165 :
1166 0 : if (len > rte_pktmbuf_data_len(mbuf))
1167 0 : return -1;
1168 :
1169 0 : vr_dpdk_mbuf_reset(pkt);
1170 0 : return 0;
1171 : }
1172 :
1173 : static void
1174 0 : dpdk_set_log_level(unsigned int log_level)
1175 : {
1176 : unsigned int level;
1177 :
1178 0 : switch(log_level) {
1179 0 : case VR_LOG_EMERG:
1180 0 : level = RTE_LOG_EMERG;
1181 0 : break;
1182 :
1183 0 : case VR_LOG_ALERT:
1184 0 : level = RTE_LOG_ALERT;
1185 0 : break;
1186 :
1187 0 : case VR_LOG_CRIT:
1188 0 : level = RTE_LOG_CRIT;
1189 0 : break;
1190 :
1191 0 : case VR_LOG_ERR:
1192 0 : level = RTE_LOG_ERR;
1193 0 : break;
1194 :
1195 0 : case VR_LOG_WARNING:
1196 0 : level = RTE_LOG_WARNING;
1197 0 : break;
1198 :
1199 0 : case VR_LOG_NOTICE:
1200 0 : level = RTE_LOG_NOTICE;
1201 0 : break;
1202 :
1203 0 : case VR_LOG_INFO:
1204 0 : level = RTE_LOG_INFO;
1205 0 : break;
1206 :
1207 0 : case VR_LOG_DEBUG:
1208 0 : level = RTE_LOG_DEBUG;
1209 0 : break;
1210 :
1211 0 : default:
1212 0 : level = 0;
1213 0 : break;
1214 : }
1215 :
1216 0 : if (level > 0) {
1217 0 : rte_log_set_global_level(level);
1218 : } else
1219 0 : RTE_LOG(ERR, VROUTER, "Error: wrong log level (%u) specified\n",
1220 : level);
1221 0 : }
1222 :
1223 : static unsigned int
1224 0 : dpdk_get_log_level(void)
1225 : {
1226 0 : unsigned int level = rte_log_get_global_level();
1227 :
1228 0 : switch(level) {
1229 0 : case RTE_LOG_EMERG:
1230 0 : return VR_LOG_EMERG;
1231 :
1232 0 : case RTE_LOG_ALERT:
1233 0 : return VR_LOG_ALERT;
1234 :
1235 0 : case RTE_LOG_CRIT:
1236 0 : return VR_LOG_CRIT;
1237 :
1238 0 : case RTE_LOG_ERR:
1239 0 : return VR_LOG_ERR;
1240 :
1241 0 : case RTE_LOG_WARNING:
1242 0 : return VR_LOG_WARNING;
1243 :
1244 0 : case RTE_LOG_NOTICE:
1245 0 : return VR_LOG_NOTICE;
1246 :
1247 0 : case RTE_LOG_INFO:
1248 0 : return VR_LOG_INFO;
1249 :
1250 0 : case RTE_LOG_DEBUG:
1251 0 : return VR_LOG_DEBUG;
1252 : }
1253 :
1254 : /* Should never reach here */
1255 0 : return 0;
1256 : }
1257 :
1258 : static void
1259 0 : dpdk_set_log_type(unsigned int log_type, int enable)
1260 : {
1261 : unsigned int type;
1262 :
1263 0 : switch (log_type) {
1264 0 : case VR_LOGTYPE_VROUTER:
1265 0 : type = RTE_LOGTYPE_VROUTER;
1266 0 : break;
1267 :
1268 0 : case VR_LOGTYPE_USOCK:
1269 0 : type = RTE_LOGTYPE_USOCK;
1270 0 : break;
1271 :
1272 0 : case VR_LOGTYPE_UVHOST:
1273 0 : type = RTE_LOGTYPE_UVHOST;
1274 0 : break;
1275 :
1276 0 : case VR_LOGTYPE_DPCORE:
1277 0 : type = RTE_LOGTYPE_DPCORE;
1278 0 : break;
1279 :
1280 0 : default:
1281 0 : type = 0;
1282 0 : break;
1283 : }
1284 :
1285 0 : if (type > 0) {
1286 0 : rte_log_set_level(type, enable);
1287 : } else
1288 0 : RTE_LOG(ERR, VROUTER, "Error: wrong log type (0x%x) specified\n",
1289 : type);
1290 0 : }
1291 :
1292 : static unsigned int
1293 0 : dpdk_log_type_to_vr_type(unsigned int type)
1294 : {
1295 0 : switch (type) {
1296 0 : case RTE_LOGTYPE_VROUTER:
1297 0 : return VR_LOGTYPE_VROUTER;
1298 :
1299 0 : case RTE_LOGTYPE_USOCK:
1300 0 : return VR_LOGTYPE_USOCK;
1301 :
1302 0 : case RTE_LOGTYPE_UVHOST:
1303 0 : return VR_LOGTYPE_UVHOST;
1304 :
1305 0 : case RTE_LOGTYPE_DPCORE:
1306 0 : return VR_LOGTYPE_DPCORE;
1307 : }
1308 :
1309 : /* Should never reach here */
1310 0 : return 0;
1311 : }
1312 :
1313 : static unsigned int *
1314 0 : dpdk_get_enabled_log_types(int *size)
1315 : {
1316 0 : int num = rte_logs.dynamic_types_len;
1317 :
1318 : unsigned int *enabled_array =
1319 0 : vr_malloc(sizeof(int) * num, VR_LOG_TYPES_OBJECT);
1320 : int i;
1321 :
1322 0 : RTE_VERIFY(enabled_array != NULL);
1323 0 : *size = 0;
1324 0 : for (i = 0; i < num; i++) {
1325 0 : if (rte_log_get_level(i) > 0) {
1326 0 : enabled_array[i] = dpdk_log_type_to_vr_type(i);
1327 0 : (*size)++;
1328 : }
1329 : }
1330 :
1331 0 : return enabled_array;
1332 : }
1333 :
1334 : static void
1335 53 : dpdk_soft_reset(struct vrouter *router)
1336 : {
1337 : unsigned lcore_id;
1338 53 : rcu_barrier();
1339 :
1340 : /* Reset GRO hash tables */
1341 424 : RTE_LCORE_FOREACH(lcore_id) {
1342 371 : struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id];
1343 371 : if (lcore != NULL)
1344 53 : dpdk_gro_free_all_flows(lcore);
1345 : }
1346 53 : }
1347 :
1348 : static int
1349 21 : dpdk_is_frag_limit_exceeded(void)
1350 : {
1351 21 : struct vrouter *router = vrouter_get(0);
1352 : struct vr_malloc_stats *stats;
1353 21 : uint64_t sum = 0;
1354 : unsigned int cpu;
1355 :
1356 21 : if (router->vr_malloc_stats) {
1357 273 : for (cpu = 0; cpu < vr_num_cpus; cpu++) {
1358 252 : if (router->vr_malloc_stats[cpu]) {
1359 252 : stats = &router->vr_malloc_stats[cpu][VR_FRAGMENT_QUEUE_ELEMENT_OBJECT];
1360 252 : sum += stats->ms_alloc;
1361 252 : sum -= stats->ms_free;
1362 : }
1363 : }
1364 21 : if (sum > VR_DPDK_MAX_FRAGMENT_ELEMENTS)
1365 0 : return 1;
1366 : }
1367 :
1368 21 : return 0;
1369 : }
1370 :
1371 : static void
1372 332 : dpdk_register_nic(struct vr_interface* vif __attribute__((unused)),
1373 : vr_interface_req* vifr __attribute__((unused)))
1374 : {
1375 332 : }
1376 :
1377 : static void
1378 109 : dpdk_flow_bucket_lock(struct vr_flow_entry *fe)
1379 : {
1380 109 : while (!vr_sync_bool_compare_and_swap_8u(&fe->fe_bucket_lock, 0, 1)) {
1381 0 : vr_pause();
1382 : }
1383 109 : }
1384 :
1385 : static void
1386 109 : dpdk_flow_bucket_unlock(struct vr_flow_entry *fe)
1387 : {
1388 109 : vr_sync_bool_compare_and_swap_8u(&fe->fe_bucket_lock, 1, 0);
1389 109 : }
1390 :
1391 : struct host_os dpdk_host = {
1392 : .hos_printf = dpdk_printf,
1393 : .hos_malloc = dpdk_malloc,
1394 : .hos_zalloc = dpdk_zalloc,
1395 : .hos_free = dpdk_free,
1396 : .hos_vtop = dpdk_vtop, /* not used */
1397 : .hos_page_alloc = dpdk_page_alloc,
1398 : .hos_page_free = dpdk_page_free,
1399 :
1400 : .hos_palloc = dpdk_palloc,
1401 : .hos_palloc_head = dpdk_palloc_head, /* not implemented */
1402 : .hos_pexpand_head = dpdk_pexpand_head, /* not implemented */
1403 : .hos_pfree = dpdk_pfree,
1404 : .hos_preset = dpdk_preset,
1405 : .hos_pclone = dpdk_pclone,
1406 : .hos_pcopy = dpdk_pcopy,
1407 : .hos_pfrag_len = dpdk_pfrag_len,
1408 : .hos_phead_len = dpdk_phead_len,
1409 : .hos_pset_data = dpdk_pset_data,
1410 : .hos_pgso_size = dpdk_pgso_size,
1411 : .hos_get_cpu = dpdk_get_cpu,
1412 : .hos_schedule_work = dpdk_schedule_work,
1413 : .hos_delay_op = dpdk_delay_op, /* do nothing */
1414 : .hos_defer = dpdk_defer,
1415 : .hos_get_defer_data = dpdk_get_defer_data,
1416 : .hos_put_defer_data = dpdk_put_defer_data,
1417 : .hos_get_time = dpdk_get_time,
1418 : .hos_get_mono_time = dpdk_get_mono_time,
1419 : .hos_create_timer = dpdk_create_timer,
1420 : .hos_delete_timer = dpdk_delete_timer,
1421 :
1422 : .hos_network_header = dpdk_network_header, /* for chains? */
1423 : .hos_inner_network_header = dpdk_inner_network_header, /* not used? */
1424 : .hos_data_at_offset = dpdk_data_at_offset, /* for chains? */
1425 : .hos_pheader_pointer = dpdk_pheader_pointer,
1426 : .hos_pull_inner_headers = NULL, /* not necessary */
1427 : .hos_pcow = dpdk_pcow,
1428 : .hos_pull_inner_headers_fast = NULL, /* not necessary */
1429 : #if VR_DPDK_USE_MPLS_UDP_ECMP
1430 : .hos_get_udp_src_port = dpdk_get_udp_src_port,
1431 : #endif
1432 : .hos_pkt_from_vm_tcp_mss_adj = dpdk_pkt_from_vm_tcp_mss_adj,
1433 : .hos_pkt_may_pull = dpdk_pkt_may_pull,
1434 : .hos_gro_process = dpdk_gro_process,
1435 : .hos_add_mpls = dpdk_add_mpls,
1436 : .hos_del_mpls = dpdk_del_mpls, /* not implemented */
1437 : .hos_enqueue_to_assembler = dpdk_fragment_assembler_enqueue,
1438 : .hos_fragment_sync_assemble = dpdk_fragment_sync_assemble,
1439 : .hos_set_log_level = dpdk_set_log_level,
1440 : .hos_set_log_type = dpdk_set_log_type,
1441 : .hos_get_log_level = dpdk_get_log_level,
1442 : .hos_get_enabled_log_types = dpdk_get_enabled_log_types,
1443 : .hos_soft_reset = dpdk_soft_reset,
1444 : .hos_is_frag_limit_exceeded = dpdk_is_frag_limit_exceeded,
1445 : .hos_register_nic = dpdk_register_nic, /* not used with DPDK */
1446 : .hos_nl_broadcast_supported = false,
1447 : .hos_offload_flow_create = dpdk_offload_flow_create,
1448 : .hos_offload_flow_destroy = dpdk_offload_flow_destroy,
1449 : .hos_offload_prepare = dpdk_offload_prepare,
1450 : .hos_flow_bucket_lock = dpdk_flow_bucket_lock,
1451 : .hos_flow_bucket_unlock = dpdk_flow_bucket_unlock,
1452 : /* Below macro would be expanded for each callbacks registered in vr_info.h.
1453 : * this would map the actual dpdk callback function with
1454 : * vrouter_host(.hos_<fn. name>) */
1455 : FOREACH_VR_INFO_MAP()
1456 : };
1457 :
1458 : struct host_os *
1459 106 : vrouter_get_host(void)
1460 : {
1461 106 : return &dpdk_host;
1462 : }
1463 :
1464 : /* Remove xconnect callback */
1465 : void
1466 42 : vhost_remove_xconnect(void)
1467 : {
1468 : int i;
1469 : struct vr_interface *vif;
1470 42 : struct vrouter *router = vrouter_get(0);
1471 :
1472 182952 : for (i = 0; i < router->vr_max_interfaces; i++) {
1473 182910 : vif = __vrouter_get_interface(router, i);
1474 182910 : if (vif && (vif_is_vhost(vif))) {
1475 42 : vif_remove_xconnect(vif);
1476 42 : if (vif->vif_bridge[0] != NULL)
1477 42 : vif_remove_xconnect(vif->vif_bridge[0]);
1478 : }
1479 : }
1480 42 : }
1481 :
1482 : /* Implementation of the Linux kernel function */
1483 : void
1484 33 : get_random_bytes(void *buf, int nbytes)
1485 : {
1486 : int i;
1487 :
1488 33 : if (nbytes == sizeof(uint32_t)) {
1489 33 : *(uint32_t *)buf = (uint32_t)rte_rand();
1490 0 : } else if (nbytes == sizeof(uint64_t)) {
1491 0 : *(uint64_t *)buf = rte_rand();
1492 : } else {
1493 0 : for (i = 0; i < nbytes; i++) {
1494 0 : *((uint8_t *)buf + i) = (uint8_t)rte_rand();
1495 : }
1496 : }
1497 33 : }
1498 :
1499 : uint32_t
1500 0 : jhash(void *key, uint32_t length, uint32_t initval)
1501 : {
1502 0 : return rte_jhash(key, length, initval);
1503 : }
1504 :
1505 :
1506 :
1507 : /**
1508 : * vr_dpdk_packet_get - convert DPDK mbuf to dp-core vr_packet
1509 : * Based on linux_get_packet()
1510 : *
1511 : * Return vr_packet pointer.
1512 : */
1513 : struct vr_packet *
1514 160 : vr_dpdk_packet_get(struct rte_mbuf *m, struct vr_interface *vif)
1515 : {
1516 160 : struct vr_packet *pkt = vr_dpdk_mbuf_to_pkt(m);
1517 160 : pkt->vp_cpu = rte_lcore_id();
1518 160 : pkt->vp_head = m->buf_addr;
1519 :
1520 160 : pkt->vp_tail = rte_pktmbuf_headroom(m) + rte_pktmbuf_data_len(m);
1521 160 : pkt->vp_data = rte_pktmbuf_headroom(m);
1522 : /* vp_end is set in vr_dpdk_pktmbuf_init() */
1523 :
1524 160 : pkt->vp_len = rte_pktmbuf_data_len(m);
1525 160 : pkt->vp_if = vif;
1526 160 : pkt->vp_network_h = pkt->vp_inner_network_h = 0;
1527 160 : pkt->vp_nh = NULL;
1528 160 : pkt->vp_flags = 0;
1529 160 : if (likely(m->ol_flags & PKT_RX_IP_CKSUM_BAD))
1530 160 : pkt->vp_flags |= VP_FLAG_CSUM_PARTIAL;
1531 :
1532 160 : pkt->vp_ttl = 64;
1533 160 : pkt->vp_type = VP_TYPE_NULL;
1534 160 : pkt->vp_queue = VP_QUEUE_INVALID;
1535 160 : pkt->vp_priority = VP_PRIORITY_INVALID;
1536 160 : pkt->vp_rx_pass = 0;
1537 :
1538 160 : return pkt;
1539 : }
1540 :
1541 : /* Exit vRouter */
1542 : void
1543 53 : vr_dpdk_host_exit(void)
1544 : {
1545 53 : vr_sandesh_exit();
1546 53 : vrouter_exit(false);
1547 :
1548 53 : return;
1549 : }
1550 :
1551 : /*
1552 : * vr_dpdk_set_fd_limit - set the max number of open files for the process. The
1553 : * user space vhost server requires one socket per interface. Allow a few more than
1554 : * that.
1555 : *
1556 : * Returns 0 on success, -1 otherwise.
1557 : */
1558 : static int
1559 53 : vr_dpdk_set_fd_limit(void)
1560 : {
1561 : struct rlimit rl;
1562 : int ret, old_cur;
1563 :
1564 53 : ret = getrlimit(RLIMIT_NOFILE, &rl);
1565 53 : if (ret != 0) {
1566 0 : RTE_LOG(ERR, VROUTER,
1567 : "Could not get resource limits, error %d\n", errno);
1568 0 : return -1;
1569 : }
1570 :
1571 53 : old_cur = (int) rl.rlim_cur;
1572 53 : if (rl.rlim_max < (VR_MAX_INTERFACES + VR_DPDK_NUM_FDS)) {
1573 0 : rl.rlim_cur = rl.rlim_max;
1574 : } else {
1575 53 : rl.rlim_cur = VR_MAX_INTERFACES + VR_DPDK_NUM_FDS;
1576 : }
1577 :
1578 53 : ret = setrlimit(RLIMIT_NOFILE, &rl);
1579 53 : if (ret) {
1580 0 : RTE_LOG(ERR, VROUTER,
1581 : "Could not set fd limit to %d (max %d), error %d\n",
1582 : (int) rl.rlim_cur, (int) rl.rlim_max, errno);
1583 0 : return -1;
1584 : }
1585 :
1586 53 : RTE_LOG(INFO, VROUTER,
1587 : "set fd limit to %d (prev %d, max %d)\n",
1588 : (int) rl.rlim_cur, old_cur, (int) rl.rlim_max);
1589 :
1590 53 : return 0;
1591 : }
1592 :
1593 :
1594 : /* Init vRouter */
1595 : int
1596 53 : vr_dpdk_host_init(void)
1597 : {
1598 : int ret;
1599 : unsigned lcore_id;
1600 :
1601 53 : if (vr_host_inited)
1602 0 : return 0;
1603 :
1604 53 : vr_init_cpuid = vr_dpdk_init_cpuid;
1605 :
1606 : /*
1607 : * Set number of CPUs. Note it is not just number of lcores, so we
1608 : * cannot just use rte_lcore_count() here.
1609 : */
1610 53 : vr_num_cpus = 0;
1611 424 : RTE_LCORE_FOREACH(lcore_id) {
1612 371 : vr_num_cpus = RTE_MAX(vr_num_cpus, lcore_id);
1613 : }
1614 53 : vr_num_cpus++;
1615 :
1616 53 : if (!vrouter_host) {
1617 53 : vrouter_host = vrouter_get_host();
1618 :
1619 53 : if (vr_dpdk_flow_init()) {
1620 0 : return -1;
1621 : }
1622 :
1623 53 : if (vr_dpdk_bridge_init()) {
1624 0 : return -1;
1625 : }
1626 : }
1627 :
1628 : /*
1629 : * Allow at least one file descriptor per interface (as required by the
1630 : * user space vhost server.
1631 : */
1632 53 : ret = vr_dpdk_set_fd_limit();
1633 53 : if (ret) {
1634 0 : return ret;
1635 : }
1636 :
1637 53 : ret = vrouter_init();
1638 53 : if (ret)
1639 0 : return ret;
1640 :
1641 53 : ret = vr_sandesh_init();
1642 53 : if (ret)
1643 0 : goto init_fail;
1644 :
1645 53 : vr_host_inited = true;
1646 :
1647 53 : return 0;
1648 :
1649 0 : init_fail:
1650 0 : vr_dpdk_host_exit();
1651 0 : return ret;
1652 : }
1653 :
1654 : /* Retry socket connection */
1655 : int
1656 53 : vr_dpdk_retry_connect(int sockfd, const struct sockaddr *addr,
1657 : socklen_t alen)
1658 : {
1659 : int nsec;
1660 :
1661 101 : for (nsec = 1; nsec < VR_DPDK_RETRY_CONNECT_SECS; nsec <<= 1) {
1662 101 : if (connect(sockfd, addr, alen) == 0)
1663 53 : return 0;
1664 :
1665 48 : if (nsec < VR_DPDK_RETRY_CONNECT_SECS/2) {
1666 48 : sleep(nsec);
1667 48 : RTE_LOG(INFO, VROUTER, "Retrying connection for socket %d...\n",
1668 : sockfd);
1669 : }
1670 : }
1671 :
1672 0 : return -1;
1673 : }
1674 :
1675 : /* Returns a string hash */
1676 : static inline uint32_t
1677 0 : dpdk_strhash(const char *k, uint32_t initval)
1678 : {
1679 : uint32_t a, b, c;
1680 :
1681 0 : a = b = RTE_JHASH_GOLDEN_RATIO;
1682 0 : c = initval;
1683 :
1684 : do {
1685 0 : if (*k) {
1686 0 : a += k[0];
1687 0 : k++;
1688 : }
1689 0 : if (*k) {
1690 0 : b += k[0];
1691 0 : k++;
1692 : }
1693 0 : if (*k) {
1694 0 : c += k[0];
1695 0 : k++;
1696 : }
1697 0 : __rte_jhash_mix(a, b, c);
1698 0 : } while (*k);
1699 :
1700 0 : return c;
1701 : }
1702 :
1703 : /* Generates unique log message */
1704 0 : int vr_dpdk_ulog(uint32_t level, uint32_t logtype, uint32_t *last_hash,
1705 : const char *format, ...)
1706 : {
1707 : va_list ap;
1708 0 : int ret = 0;
1709 : uint32_t hash;
1710 : char buf[VR_DPDK_STR_BUF_SZ];
1711 :
1712 : /* fallback to rte_log */
1713 0 : if (last_hash == NULL) {
1714 0 : va_start(ap, format);
1715 0 : ret = rte_log(level, logtype, "%s", buf);
1716 0 : va_end(ap);
1717 : } else {
1718 : /* calculate message hash */
1719 0 : va_start(ap, format);
1720 0 : vsnprintf(buf, sizeof(buf) - 1, format, ap);
1721 0 : va_end(ap);
1722 0 : buf[sizeof(buf) - 1] = '\0';
1723 0 : hash = dpdk_strhash(buf, level + logtype);
1724 :
1725 0 : if (hash != *last_hash) {
1726 0 : *last_hash = hash;
1727 0 : ret = rte_log(level, logtype, "%s", buf);
1728 : }
1729 : }
1730 :
1731 0 : return ret;
1732 : }
|
