Line data Source code
1 : /*
2 : * vr_dpdk_virtio.c - implements DPDK forwarding infrastructure for
3 : * virtio interfaces. The virtio data structures are setup by the user
4 : * space vhost server.
5 : *
6 : * Copyright (c) 2014 Juniper Networks, Inc. All rights reserved.
7 : * Copyright Contributors to the OpenSDN project
8 : */
9 :
10 : #include "vr_dpdk.h"
11 : #include "vr_dpdk_virtio.h"
12 : #include "vr_uvhost_client.h"
13 :
14 : #include <linux/virtio_net.h>
15 : #include <sys/eventfd.h>
16 :
17 : #include <sys/mman.h>
18 : #include <sys/stat.h>
19 : #include <sys/types.h>
20 : #include <unistd.h>
21 :
22 : #include <rte_malloc.h>
23 : #include <rte_memcpy.h>
24 :
25 : #define VIRTIO_HDR_MRG_RXBUF 1
26 :
27 : void *vr_dpdk_vif_clients[VR_MAX_INTERFACES];
28 : vr_dpdk_virtioq_t vr_dpdk_virtio_rxqs[VR_MAX_INTERFACES][VR_DPDK_VIRTIO_MAX_QUEUES];
29 : vr_dpdk_virtioq_t vr_dpdk_virtio_txqs[VR_MAX_INTERFACES][VR_DPDK_VIRTIO_MAX_QUEUES];
30 :
31 : static int dpdk_virtio_from_vm_rx(void *port, struct rte_mbuf **pkts,
32 : uint32_t max_pkts);
33 : static int dpdk_virtio_to_vm_tx(void *port, struct rte_mbuf *pkt);
34 : static int dpdk_virtio_to_vm_flush(void *port);
35 : static int dpdk_virtio_writer_stats_read(void *port,
36 : struct rte_port_out_stats *stats,
37 : int clear);
38 : static int dpdk_virtio_reader_stats_read(void *port,
39 : struct rte_port_in_stats *stats,
40 : int clear);
41 :
42 : /*
43 : * Virtio writer
44 : */
45 : struct dpdk_virtio_writer {
46 : struct rte_port_out_stats stats;
47 : /* extra statistics */
48 : uint64_t nb_syscalls;
49 : /* last packet TX */
50 : uint64_t last_pkt_tx;
51 : /* last TX flush */
52 : uint64_t last_pkt_tx_flush;
53 :
54 : vr_dpdk_virtioq_t *tx_virtioq;
55 : struct rte_mbuf *tx_buf[VR_DPDK_VIRTIO_TX_BURST_SZ];
56 : /* Total number of mbuf chains
57 : * Say if a mbuf chain contains 10 segments, it is counted as 1
58 : */
59 : uint32_t tx_buf_count;
60 : /* Total number of mbufs in all the chains */
61 : uint32_t tx_mbufs;
62 : };
63 :
64 : struct dpdk_virtio_writer_params {
65 : /* virtio TX queue pointer */
66 : vr_dpdk_virtioq_t *tx_virtioq;
67 : };
68 :
69 : /*
70 : * vr_dpdk_virtio_stop - stop the virtio interface.
71 : *
72 : * Returns 0 on success, -1 otherwise.
73 : */
74 : int
75 554 : vr_dpdk_virtio_stop(unsigned int vif_idx)
76 : {
77 : int i;
78 : vr_dpdk_virtioq_t *vq;
79 :
80 554 : if (vif_idx >= VR_MAX_INTERFACES) {
81 0 : return -1;
82 : }
83 :
84 : /* Disable and reset all the virtio queues. */
85 18282 : for (i = 0; i < VR_DPDK_VIRTIO_MAX_QUEUES*2; i++) {
86 17728 : if (i & 1) {
87 8864 : vq = &vr_dpdk_virtio_rxqs[vif_idx][i/2];
88 : } else {
89 8864 : vq = &vr_dpdk_virtio_txqs[vif_idx][i/2];
90 : }
91 :
92 17728 : if (vq->vdv_ready_state != VQ_NOT_READY) {
93 388 : vr_dpdk_set_virtq_ready(vif_idx, i, VQ_NOT_READY);
94 : rte_wmb();
95 388 : synchronize_rcu();
96 : /*
97 : * TODO: code duplication to minimize the changes.
98 : * See vr_dpdk_virtio_get_vring_base().
99 : */
100 388 : vq->vdv_desc = NULL;
101 388 : if (vq->vdv_callfd) {
102 388 : close(vq->vdv_callfd);
103 388 : vq->vdv_callfd = 0;
104 : }
105 : }
106 : }
107 :
108 554 : return 0;
109 : }
110 :
111 : static void *
112 1328 : dpdk_virtio_writer_create(void *params, int socket_id)
113 : {
114 1328 : struct dpdk_virtio_writer_params *conf =
115 : (struct dpdk_virtio_writer_params *) params;
116 : struct dpdk_virtio_writer *port;
117 :
118 : /* Check input parameters */
119 1328 : if (conf == NULL) {
120 0 : RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__);
121 0 : return NULL;
122 : }
123 :
124 : /* Memory allocation */
125 1328 : port = rte_zmalloc_socket("PORT", sizeof(*port),
126 : RTE_CACHE_LINE_SIZE, socket_id);
127 1328 : if (port == NULL) {
128 0 : RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__);
129 0 : return NULL;
130 : }
131 :
132 : /* Initialization */
133 1328 : port->tx_virtioq = conf->tx_virtioq;
134 :
135 1328 : return port;
136 : }
137 :
138 : static int
139 860 : dpdk_virtio_writer_free(void *port)
140 : {
141 : vr_dpdk_virtioq_t *tx_virtioq;
142 :
143 860 : if (port == NULL) {
144 0 : RTE_LOG(ERR, PORT, "%s: port is NULL\n", __func__);
145 0 : return -EINVAL;
146 : }
147 :
148 860 : tx_virtioq = ((struct dpdk_virtio_writer *)port)->tx_virtioq;
149 :
150 : /* close FDs */
151 860 : if (tx_virtioq->vdv_callfd > 0) {
152 0 : close(tx_virtioq->vdv_callfd);
153 : }
154 860 : if (tx_virtioq->vdv_kickfd > 0) {
155 0 : close(tx_virtioq->vdv_kickfd);
156 : }
157 :
158 : /* reset the virtio */
159 860 : memset(tx_virtioq, 0, sizeof(vr_dpdk_virtioq_t));
160 :
161 860 : rte_free(port);
162 :
163 860 : return 0;
164 : }
165 :
166 : struct rte_port_out_ops vr_dpdk_virtio_writer_ops = {
167 : .f_create = dpdk_virtio_writer_create,
168 : .f_free = dpdk_virtio_writer_free,
169 : .f_tx = dpdk_virtio_to_vm_tx,
170 : .f_tx_bulk = NULL, /* TODO: not implemented */
171 : .f_flush = dpdk_virtio_to_vm_flush,
172 : .f_stats = dpdk_virtio_writer_stats_read
173 : };
174 :
175 : /*
176 : * Virtio reader
177 : */
178 : struct dpdk_virtio_reader {
179 : struct rte_port_in_stats stats;
180 : /* extra statistics */
181 : uint64_t nb_syscalls;
182 : uint64_t nb_nombufs;
183 :
184 : vr_dpdk_virtioq_t *rx_virtioq;
185 : };
186 :
187 : struct dpdk_virtio_reader_params {
188 : /* virtio RX queue pointer */
189 : vr_dpdk_virtioq_t *rx_virtioq;
190 : };
191 :
192 : static void *
193 664 : dpdk_virtio_reader_create(void *params, int socket_id)
194 : {
195 664 : struct dpdk_virtio_reader_params *conf =
196 : (struct dpdk_virtio_reader_params *) params;
197 : struct dpdk_virtio_reader *port;
198 :
199 : /* Check input parameters */
200 664 : if (conf == NULL) {
201 0 : RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__);
202 0 : return NULL;
203 : }
204 :
205 : /* Memory allocation */
206 664 : port = rte_zmalloc_socket("PORT", sizeof(*port),
207 : RTE_CACHE_LINE_SIZE, socket_id);
208 664 : if (port == NULL) {
209 0 : RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__);
210 0 : return NULL;
211 : }
212 :
213 : /* Initialization */
214 664 : port->rx_virtioq = conf->rx_virtioq;
215 :
216 664 : return port;
217 : }
218 :
219 : static int
220 430 : dpdk_virtio_reader_free(void *port)
221 : {
222 : vr_dpdk_virtioq_t *rx_virtioq;
223 :
224 430 : if (port == NULL) {
225 0 : RTE_LOG(ERR, PORT, "%s: port is NULL\n", __func__);
226 0 : return -EINVAL;
227 : }
228 :
229 430 : rx_virtioq = ((struct dpdk_virtio_reader *)port)->rx_virtioq;
230 :
231 : /* close FDs */
232 430 : if (rx_virtioq->vdv_callfd > 0) {
233 0 : close(rx_virtioq->vdv_callfd);
234 : }
235 430 : if (rx_virtioq->vdv_kickfd > 0) {
236 0 : close(rx_virtioq->vdv_kickfd);
237 : }
238 :
239 : /* reset the virtio */
240 430 : memset(rx_virtioq, 0, sizeof(vr_dpdk_virtioq_t));
241 :
242 430 : rte_free(port);
243 :
244 430 : return 0;
245 : }
246 :
247 :
248 : struct rte_port_in_ops vr_dpdk_virtio_reader_ops = {
249 : .f_create = dpdk_virtio_reader_create,
250 : .f_free = dpdk_virtio_reader_free,
251 : .f_rx = dpdk_virtio_from_vm_rx,
252 : .f_stats = dpdk_virtio_reader_stats_read
253 : };
254 :
255 : /*
256 : * vr_dpdk_vrtio_uvh_get_blk_size - set the block size of fd.
257 : * On error -1 is returned, otherwise 0.
258 : */
259 : int
260 388 : vr_dpdk_virtio_uvh_get_blk_size(int fd, uint64_t *const blksize)
261 : {
262 : struct stat fd_stat;
263 : int ret;
264 388 : memset(&fd_stat, 0, sizeof(stat));
265 :
266 388 : ret = fstat(fd, &fd_stat);
267 388 : if (!ret){
268 388 : *blksize = (uint64_t)fd_stat.st_blksize;
269 : } else {
270 : RTE_LOG_DP(DEBUG, UVHOST, "Error getting file status for FD %d: %s (%d)\n",
271 : fd, strerror(errno), errno);
272 : }
273 :
274 388 : return ret;
275 : }
276 :
277 : /*
278 : * vr_dpdk_virtio_nrxqs - returns the number of receives queues for a virtio
279 : * interface.
280 : */
281 : uint16_t
282 996 : vr_dpdk_virtio_nrxqs(struct vr_interface *vif)
283 : {
284 996 : return vr_dpdk.nb_fwd_lcores;
285 : }
286 :
287 : /*
288 : * vr_dpdk_virtio_ntxqs - returns the number of transmit queues for a virtio
289 : * interface.
290 : */
291 : uint16_t
292 1328 : vr_dpdk_virtio_ntxqs(struct vr_interface *vif)
293 : {
294 1328 : return vr_dpdk.nb_fwd_lcores;
295 : }
296 :
297 : static unsigned int vif_rx_queue_lcore[VR_MAX_INTERFACES][VR_MAX_INTERFACES];
298 :
299 : /*
300 : * dpdk_virtio_rx_queue_release - releases a virtio RX queue.
301 : *
302 : * Returns nothing.
303 : */
304 : static void
305 430 : dpdk_virtio_rx_queue_release(unsigned lcore_id,
306 : unsigned queue_index __attribute__((unused)),
307 : struct vr_interface *vif)
308 : {
309 430 : struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id];
310 430 : struct vr_dpdk_queue *rx_queue = &lcore->lcore_rx_queues[vif->vif_idx];
311 430 : struct vr_dpdk_queue_params *rx_queue_params
312 430 : = &lcore->lcore_rx_queue_params[vif->vif_idx];
313 : /* free the queue */
314 430 : if (rx_queue->rxq_ops.f_free(rx_queue->q_queue_h)) {
315 0 : RTE_LOG(ERR, VROUTER, " error freeing lcore %u virtio device RX queue\n",
316 : lcore_id);
317 : }
318 :
319 : /* reset the queue */
320 430 : vrouter_put_interface(rx_queue->q_vif);
321 430 : memset(rx_queue, 0, sizeof(*rx_queue));
322 430 : memset(rx_queue_params, 0, sizeof(*rx_queue_params));
323 430 : }
324 :
325 : /*
326 : * vr_dpdk_virtio_rx_queue_init - initializes a virtio RX queue.
327 : *
328 : * Returns a pointer to the RX queue on success, NULL otherwise.
329 : */
330 : struct vr_dpdk_queue *
331 664 : vr_dpdk_virtio_rx_queue_init(unsigned int lcore_id, struct vr_interface *vif,
332 : unsigned int queue_or_lcore_id)
333 : {
334 664 : uint16_t queue_id = queue_or_lcore_id;
335 664 : struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id];
336 664 : const unsigned int socket_id = rte_lcore_to_socket_id(lcore_id);
337 664 : unsigned int vif_idx = vif->vif_idx;
338 664 : struct vr_dpdk_queue *rx_queue = &lcore->lcore_rx_queues[vif_idx];
339 664 : struct vr_dpdk_queue_params *rx_queue_params =
340 : &lcore->lcore_rx_queue_params[vif_idx];
341 :
342 : /* Check input parameters */
343 664 : if (queue_id >= vr_dpdk_virtio_nrxqs(vif)) {
344 0 : RTE_LOG(ERR, VROUTER, " error creating virtio device %s RX queue %"
345 : PRIu16 "\n", vif->vif_name, queue_id);
346 0 : return NULL;
347 : }
348 :
349 : /* init queue */
350 664 : rx_queue->rxq_ops = vr_dpdk_virtio_reader_ops;
351 664 : rx_queue->q_vif = vrouter_get_interface(vif->vif_rid, vif_idx);
352 :
353 : /* init virtio queue */
354 664 : vr_dpdk_virtio_rxqs[vif_idx][queue_id].vdv_ready_state = VQ_NOT_READY;
355 664 : vr_dpdk_virtio_rxqs[vif_idx][queue_id].vdv_last_used_idx = 0;
356 664 : vr_dpdk_virtio_rxqs[vif_idx][queue_id].vdv_last_used_idx_res = 0;
357 664 : vr_dpdk_virtio_rxqs[vif_idx][queue_id].vdv_vif_idx = vif->vif_idx;
358 :
359 : /* create the queue */
360 664 : struct dpdk_virtio_reader_params reader_params = {
361 664 : .rx_virtioq = &vr_dpdk_virtio_rxqs[vif_idx][queue_id],
362 : };
363 664 : rx_queue->q_queue_h = rx_queue->rxq_ops.f_create(&reader_params, socket_id);
364 664 : if (rx_queue->q_queue_h == NULL) {
365 0 : RTE_LOG(ERR, VROUTER, " error creating virtio device %s RX queue %"
366 : PRIu16 "\n", vif->vif_name, queue_id);
367 0 : return NULL;
368 : }
369 664 : rx_queue->vring_queue_id = queue_id;
370 : /* store queue params */
371 664 : rx_queue_params->qp_release_op = &dpdk_virtio_rx_queue_release;
372 :
373 : /* save the lcore serving the queue for later enabling/disabling */
374 664 : vif_rx_queue_lcore[vif_idx][queue_id] = lcore_id;
375 :
376 664 : return rx_queue;
377 : }
378 :
379 : /*
380 : * dpdk_virtio_tx_queue_release - releases a virtio TX queue.
381 : *
382 : * Returns nothing.
383 : */
384 : static void
385 860 : dpdk_virtio_tx_queue_release(unsigned lcore_id, unsigned queue_index,
386 : struct vr_interface *vif)
387 : {
388 860 : struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id];
389 860 : struct vr_dpdk_queue *tx_queue =
390 860 : &lcore->lcore_tx_queues[vif->vif_idx][queue_index];
391 860 : struct vr_dpdk_queue_params *tx_queue_params
392 860 : = &lcore->lcore_tx_queue_params[vif->vif_idx][queue_index];
393 :
394 860 : tx_queue->txq_ops.f_tx = NULL;
395 : rte_wmb();
396 :
397 : /* flush and free the queue */
398 860 : if (tx_queue->txq_ops.f_free(tx_queue->q_queue_h)) {
399 0 : RTE_LOG(ERR, VROUTER, " error freeing lcore %u virtio device TX queue\n",
400 : lcore_id);
401 : }
402 :
403 : /* reset the queue */
404 860 : vrouter_put_interface(tx_queue->q_vif);
405 860 : memset(tx_queue, 0, sizeof(*tx_queue));
406 860 : memset(tx_queue_params, 0, sizeof(*tx_queue_params));
407 860 : }
408 :
409 : /*
410 : * vr_dpdk_virtio_tx_queue_init - initializes a virtio TX queue.
411 : *
412 : * Returns a pointer to the TX queue on success, NULL otherwise.
413 : */
414 : struct vr_dpdk_queue *
415 1328 : vr_dpdk_virtio_tx_queue_init(unsigned int lcore_id, struct vr_interface *vif,
416 : unsigned int queue_or_lcore_id)
417 : {
418 1328 : uint16_t queue_id = queue_or_lcore_id;
419 1328 : struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id];
420 1328 : const unsigned int socket_id = rte_lcore_to_socket_id(lcore_id);
421 1328 : unsigned int vif_idx = vif->vif_idx;
422 1328 : struct vr_dpdk_queue *tx_queue = &lcore->lcore_tx_queues[vif_idx][0];
423 1328 : struct vr_dpdk_queue_params *tx_queue_params
424 : = &lcore->lcore_tx_queue_params[vif_idx][0];
425 :
426 : /* Check input parameters */
427 : /* virtio TX is thread safe, so just use one of the rings */
428 1328 : queue_id = queue_id % vr_dpdk_virtio_ntxqs(vif);
429 :
430 : /* init queue */
431 1328 : tx_queue->txq_ops = vr_dpdk_virtio_writer_ops;
432 1328 : tx_queue->q_vif = vrouter_get_interface(vif->vif_rid, vif_idx);
433 :
434 : /* init virtio queue */
435 1328 : vr_dpdk_virtio_txqs[vif_idx][queue_id].vdv_ready_state = VQ_NOT_READY;
436 1328 : vr_dpdk_virtio_txqs[vif_idx][queue_id].vdv_last_used_idx = 0;
437 1328 : vr_dpdk_virtio_txqs[vif_idx][queue_id].vdv_last_used_idx_res = 0;
438 1328 : vr_dpdk_virtio_txqs[vif_idx][queue_id].vdv_vif_idx = vif->vif_idx;
439 :
440 : /* create the queue */
441 1328 : struct dpdk_virtio_writer_params writer_params = {
442 : /*
443 : * Always initialize each lcore's tx_queue with virtio queue number 0.
444 : * If there are more queues, they will be enabled later via
445 : * VHOST_USER_SET_VRING_ENABLE message.
446 : */
447 1328 : .tx_virtioq = &vr_dpdk_virtio_txqs[vif_idx][0],
448 : };
449 1328 : tx_queue->q_queue_h = tx_queue->txq_ops.f_create(&writer_params, socket_id);
450 1328 : if (tx_queue->q_queue_h == NULL) {
451 0 : RTE_LOG(ERR, VROUTER, " error creating virtio device %s TX queue %"
452 : PRIu16 "\n", vif->vif_name, queue_id);
453 0 : return NULL;
454 : }
455 :
456 : /* store queue params */
457 1328 : tx_queue_params->qp_release_op = &dpdk_virtio_tx_queue_release;
458 :
459 1328 : return tx_queue;
460 : }
461 :
462 : struct dpdk_virtio_tx_queue_set_params {
463 : unsigned int vif_id;
464 : unsigned int vif_gen;
465 : unsigned int queue_id;
466 : };
467 :
468 : static unsigned int vif_lcore_tx_queue[VR_MAX_INTERFACES][VR_MAX_CPUS_DPDK];
469 : static unsigned int vif_tx_queues_enabled[VR_MAX_INTERFACES];
470 :
471 : /*
472 : * Enable or disable given queue for a vif.
473 : *
474 : * In current vRouter design, every lcore that can send packets has to have a
475 : * TX queue available for every existing vif. It is because we do not know
476 : * which lcore wil eventually send the packet, and thus each has to have a
477 : * queue to use.
478 : *
479 : * If VM requests more than one virtio queue, then we distribute them among the
480 : * forwarding lcores as evenly as possible.
481 : *
482 : * The entire process (this function, which sends commands to other lcores and
483 : * then vr_dpdk_virtio_tx_queue_set(), which is called from the destination
484 : * lcores' main loop) works fine as long as the QEMU enables/disables each
485 : * queues in ascending order. For example, if the maximal number of queues is
486 : * 4, and inside a VM ethtool -L eth0 combined 2 is issued, the QEMU will send
487 : * the following messages:
488 : * 1. Enable queue 0.
489 : * 2. Enable queue 1.
490 : * 3. Disable queue 2.
491 : * 4. Disable queue 3.
492 : *
493 : * TODO: Remove the above assumption as there is no guarantee that QEMU will
494 : * always work as described.
495 : */
496 : void
497 0 : vr_dpdk_virtio_tx_queue_enable_disable(unsigned int vif_id,
498 : unsigned int vif_gen,
499 : unsigned int queue_id,
500 : bool enable)
501 : {
502 : unsigned int lcore_id;
503 : unsigned int starting_lcore;
504 : struct dpdk_virtio_tx_queue_set_params *arg;
505 : unsigned int qid;
506 : unsigned int queue_num;
507 :
508 : /* If command is 'disable', we enable all lower numbered queues */
509 0 : if (!enable)
510 0 : queue_num = queue_id - 1;
511 : else
512 0 : queue_num = queue_id;
513 :
514 : /*
515 : * Subsequent 'disable' commands are ignored. For example if we enabled
516 : * queues 0 and 1, then all higher queues (2, 3, ..) had already been
517 : * disabled. Thus we ignore the 'disable' request for them
518 : */
519 0 : if (!enable && queue_num > vif_tx_queues_enabled[vif_id])
520 0 : return;
521 :
522 : /*
523 : * Each lcore that does tx has to have a queue assigned for every
524 : * interface. We assign queue 0 for pkt and netlink lcores. All
525 : * other queues (including queue 0) are distributed among forwarding
526 : * lcores.
527 : */
528 0 : if (queue_id == 0)
529 0 : starting_lcore = VR_DPDK_PACKET_LCORE_ID;
530 : else
531 0 : starting_lcore = VR_DPDK_FWD_LCORE_ID;
532 :
533 0 : for (lcore_id = starting_lcore, qid = 0; lcore_id < vr_dpdk.nb_fwd_lcores +
534 0 : VR_DPDK_FWD_LCORE_ID; ++lcore_id) {
535 :
536 : /*
537 : * Send cmd to destination lcore only if it has different queue enabled
538 : * curently.
539 : */
540 0 : if (vif_lcore_tx_queue[vif_id][lcore_id - VR_DPDK_PACKET_LCORE_ID] !=
541 : qid) {
542 0 : vif_lcore_tx_queue[vif_id][lcore_id - VR_DPDK_PACKET_LCORE_ID] =
543 : qid;
544 :
545 0 : arg = rte_malloc("virtio_tx_queue_set", sizeof(*arg), 0);
546 :
547 0 : arg->vif_id = vif_id;
548 0 : arg->queue_id = qid;
549 0 : arg->vif_gen = vif_gen;
550 :
551 0 : vr_dpdk_lcore_cmd_post(lcore_id, VR_DPDK_LCORE_TX_QUEUE_SET_CMD,
552 : (uint64_t)arg);
553 : }
554 :
555 0 : ++qid;
556 0 : qid %= queue_num + 1;
557 : }
558 :
559 : /* Save current number of TX queues enabled for vif */
560 0 : vif_tx_queues_enabled[vif_id] = queue_num;
561 : }
562 :
563 : /*
564 : * Assign given virtio queue to vRouter's dpdk (per lcore) tx queue.
565 : *
566 : * The assignment is done by setting correct virtio queue pointer in the
567 : * lcore's tx queue handler.
568 : *
569 : * This function is called only from the main loops of the lcores that have TX
570 : * queues (packet lcore, netlink lcore, forwarding lcores).
571 : */
572 : void
573 0 : vr_dpdk_virtio_tx_queue_set(void *arg)
574 : {
575 0 : struct dpdk_virtio_tx_queue_set_params *p = arg;
576 : struct vr_dpdk_queue *tx_queue;
577 : struct dpdk_virtio_writer *port;
578 : struct vr_dpdk_lcore *lcore;
579 : struct vr_interface *vif;
580 :
581 : /* Check if vif is still valid */
582 0 : vif = __vrouter_get_interface(vrouter_get(0), p->vif_id);
583 0 : if (!vif || vif->vif_gen != p->vif_gen) {
584 0 : rte_free(arg);
585 0 : return;
586 : }
587 :
588 0 : lcore = vr_dpdk.lcores[rte_lcore_id()];
589 0 : tx_queue = &lcore->lcore_tx_queues[p->vif_id][0];
590 0 : port = (struct dpdk_virtio_writer *)tx_queue->q_queue_h;
591 :
592 : /* Assign new queue to the lcore's tx_queue handler */
593 0 : port->tx_virtioq = &vr_dpdk_virtio_txqs[p->vif_id][p->queue_id];
594 :
595 : /*
596 : * Each tx_queue has to have a f_flush method, but we do not need to crash
597 : * in other case.
598 : */
599 0 : if (tx_queue->txq_ops.f_flush)
600 0 : tx_queue->txq_ops.f_flush(tx_queue->q_queue_h);
601 : else
602 0 : RTE_LOG(ERR, VROUTER, "%s: Flush function for tx_queue(%p) unavailable\n",
603 : __func__, tx_queue);
604 :
605 0 : rte_free(arg);
606 : }
607 :
608 : struct dpdk_virtio_rx_queue_set_params {
609 : bool enable;
610 : unsigned int vif_id;
611 : unsigned int vif_gen;
612 : unsigned int queue_id;
613 : };
614 :
615 :
616 : void
617 : dpdk_lcore_queue_add(unsigned lcore_id, struct vr_dpdk_q_slist *q_head,
618 : struct vr_dpdk_queue *queue);
619 : void
620 : dpdk_lcore_rx_queue_remove(struct vr_dpdk_lcore *lcore,
621 : struct vr_dpdk_queue *rx_queue,
622 : bool clear_f_rx);
623 :
624 : /*
625 : * Called on uvhost lcore only.
626 : */
627 : void
628 0 : vr_dpdk_virtio_rx_queue_enable_disable(unsigned int vif_id,
629 : unsigned int vif_gen,
630 : unsigned int queue_id,
631 : bool enable)
632 : {
633 : struct dpdk_virtio_rx_queue_set_params *arg;
634 :
635 : /*
636 : * Ignore requests for queue number 0. It has already been added to lcore's
637 : * list of queues and can never be disabled (qemu never sends the 'disable'
638 : * command for queue 0). Doing otherwise would result in double adding the
639 : * virtio queue to lcore's list of rx queues.
640 : */
641 0 : if (queue_id == 0)
642 0 : return;
643 :
644 0 : arg = rte_malloc("virtio_rx_queue_set", sizeof(*arg), 0);
645 :
646 0 : arg->vif_id = vif_id;
647 0 : arg->vif_gen = vif_gen;
648 0 : arg->queue_id = queue_id;
649 0 : arg->enable = enable;
650 :
651 0 : vr_dpdk_lcore_cmd_post(VR_DPDK_NETLINK_LCORE_ID,
652 : VR_DPDK_LCORE_RX_QUEUE_SET_CMD, (uint64_t)arg);
653 : }
654 :
655 : /*
656 : * Called only on netlink lcore.
657 : */
658 : void
659 0 : vr_dpdk_virtio_rx_queue_set(void *arg)
660 : {
661 0 : struct dpdk_virtio_rx_queue_set_params *p = arg;
662 : struct vr_interface *vif;
663 : struct vr_dpdk_queue *rx_queue;
664 : struct vr_dpdk_lcore *lcore;
665 : unsigned int lcore_id;
666 : struct vr_dpdk_lcore_rx_queue_remove_arg *rx_rm_arg;
667 :
668 : /* Check if vif is still valid */
669 0 : vif = __vrouter_get_interface(vrouter_get(0), p->vif_id);
670 0 : if (!vif || vif->vif_gen != p->vif_gen) {
671 0 : rte_free(arg);
672 0 : return;
673 : }
674 :
675 0 : if (p->enable) {
676 0 : lcore_id = vif_rx_queue_lcore[p->vif_id][p->queue_id];
677 0 : lcore = vr_dpdk.lcores[lcore_id];
678 0 : rx_queue = &lcore->lcore_rx_queues[p->vif_id];
679 0 : lcore->lcore_rx_queues[p->vif_id].vring_queue_id = p->queue_id;
680 0 : dpdk_lcore_queue_add(lcore_id, &lcore->lcore_rx_head, rx_queue);
681 :
682 : } else {
683 0 : lcore_id = vif_rx_queue_lcore[p->vif_id][p->queue_id];
684 0 : lcore = vr_dpdk.lcores[lcore_id];
685 0 : rx_queue = &lcore->lcore_rx_queues[p->vif_id];
686 0 : if (rx_queue->enabled) {
687 0 : rx_rm_arg = rte_malloc("lcore_rx_queue_rm_cmd", sizeof(*rx_rm_arg),
688 : 0);
689 0 : rx_rm_arg->vif_id = vif->vif_idx;
690 0 : rx_rm_arg->clear_f_rx = false;
691 0 : rx_rm_arg->free_arg = true;
692 0 : vr_dpdk_lcore_cmd_post(lcore_id, VR_DPDK_LCORE_RX_RM_CMD,
693 : (uint64_t)rx_rm_arg);
694 : }
695 : }
696 :
697 0 : rte_free(arg);
698 : }
699 :
700 : /*
701 : * vr_dpdk_guest_phys_to_host_virt - convert a guest physical address
702 : * to a host virtual address. Uses the guest memory map stored in the
703 : * vhost client for the guest interface.
704 : *
705 : * Returns address on success, NULL otherwise.
706 : */
707 : static char *
708 243 : vr_dpdk_guest_phys_to_host_virt(vr_uvh_client_t *vru_cl, uint64_t paddr)
709 : {
710 : int i;
711 : vr_uvh_client_mem_region_t *reg;
712 :
713 403 : for (i = 0; i < vru_cl->vruc_num_mem_regions; i++) {
714 403 : reg = &vru_cl->vruc_mem_regions[i];
715 :
716 403 : if ((paddr >= reg->vrucmr_phys_addr) &&
717 243 : (paddr <= (reg->vrucmr_phys_addr + reg->vrucmr_size))) {
718 243 : return ((char *) reg->vrucmr_mmap_addr) +
719 243 : (paddr - reg->vrucmr_phys_addr);
720 : }
721 : }
722 :
723 0 : return NULL;
724 : }
725 :
726 : #ifdef RTE_PORT_STATS_COLLECT
727 :
728 : #define DPDK_VIRTIO_READER_STATS_PKTS_IN_ADD(port, val) \
729 : port->stats.n_pkts_in += val
730 : #define DPDK_VIRTIO_READER_STATS_PKTS_DROP_ADD(port, val) \
731 : port->stats.n_pkts_drop += val
732 :
733 : #else
734 :
735 : /* keep compiler happy, for unused variables */
736 : #define DPDK_VIRTIO_READER_STATS_PKTS_IN_ADD(port, val) \
737 : (void)(val)
738 : #define DPDK_VIRTIO_READER_STATS_PKTS_DROP_ADD(port, val) \
739 : (void)(val)
740 :
741 : #endif
742 :
743 : static inline uint32_t
744 0 : dpdk_virtio_get_ip_tcp_hdr_len(char *pkt_addr, uint32_t pkt_len)
745 : {
746 0 : struct vr_eth *eth_hdr = (struct vr_eth*)pkt_addr;
747 0 : struct vr_ip6 *ipv6_hdr = NULL;
748 0 : struct vr_tcp *tcp_hdr = NULL;
749 0 : unsigned int pull_len = VR_ETHER_HLEN;
750 : unsigned short eth_proto;
751 :
752 0 : if (unlikely(pkt_len < pull_len))
753 0 : return 0;
754 :
755 0 : eth_proto = eth_hdr->eth_proto;
756 :
757 : /* Skip VLAN tag which may be present if VM sends tagged pkts */
758 0 : while (eth_proto == rte_cpu_to_be_16(VR_ETH_PROTO_VLAN)) {
759 0 : if (unlikely(pkt_len < pull_len + VR_VLAN_HLEN))
760 0 : return 0;
761 0 : eth_proto = ((struct vr_vlan_hdr *)((uintptr_t)eth_hdr + pull_len))->vlan_proto;
762 0 : pull_len += VR_VLAN_HLEN;
763 : }
764 :
765 0 : if (likely(eth_proto == rte_cpu_to_be_16(VR_ETH_PROTO_IP))) {
766 0 : struct vr_ip *ipv4_hdr = NULL;
767 : uint32_t ipv4_hlen;
768 0 : ipv4_hdr = (struct vr_ip *)((uintptr_t)eth_hdr + pull_len);
769 :
770 0 : if (unlikely(pkt_len < pull_len + sizeof(struct vr_ip)))
771 0 : return 0;
772 :
773 0 : ipv4_hlen = ((ipv4_hdr->ip_hl) * RTE_IPV4_IHL_MULTIPLIER);
774 0 : pull_len += ipv4_hlen;
775 0 : tcp_hdr = (struct vr_tcp*)((uint8_t*)ipv4_hdr + ipv4_hlen);
776 0 : } else if (eth_proto == rte_cpu_to_be_16(VR_ETH_PROTO_IP6)) {
777 0 : ipv6_hdr = (struct vr_ip6 *)((uintptr_t)eth_hdr + pull_len);
778 :
779 0 : if (unlikely(pkt_len < pull_len + sizeof(struct vr_ip6)))
780 0 : return 0;
781 :
782 0 : pull_len += sizeof(*ipv6_hdr);
783 0 : tcp_hdr = (struct vr_tcp*)((uint8_t*)ipv6_hdr + sizeof(*ipv6_hdr));
784 : }
785 0 : if (likely(tcp_hdr != NULL)) {
786 0 : pull_len += (VR_TCP_OFFSET(tcp_hdr->tcp_offset_r_flags) << 2);
787 : }
788 :
789 0 : return pull_len;
790 : }
791 :
792 0 : static inline char *dpdk_pktmbuf_append(struct rte_mbuf *m, struct rte_mbuf *last, uint16_t len)
793 : {
794 : void *tail;
795 : struct rte_mbuf *m_last;
796 :
797 : __rte_mbuf_sanity_check(m, 1);
798 : __rte_mbuf_sanity_check(last, 1);
799 :
800 0 : m_last = rte_pktmbuf_lastseg(last);
801 0 : if (unlikely(len > rte_pktmbuf_tailroom(m_last)))
802 0 : return NULL;
803 :
804 0 : tail = (char *)m_last->buf_addr + m_last->data_off + m_last->data_len;
805 0 : m_last->data_len = (uint16_t)(m_last->data_len + len);
806 0 : m->pkt_len = (m->pkt_len + len);
807 0 : return (char*) tail;
808 : }
809 :
810 : /*
811 : * dpdk_virtio_create_mss_sized_mbuf_chain - Create a chained mbuf where each segment
812 : * in the chain is of length 'mss' and copy the data pointed to by pkt_addr
813 : *
814 : * @input -
815 : * mbuf: pointer to the mbuf where the chain needs to be created
816 : * mss: lenght of each segment in the chain
817 : * pkt_addr: pointer to the data which has to be copied to mbuf
818 : * pkt_len: length of the data which has to be copied
819 : * header_len: first segment of the chain will have a length of mss + this value
820 : * to account for the headers
821 : *
822 : * @output -
823 : * 0: success
824 : * -1: failure
825 : */
826 : static int
827 0 : dpdk_virtio_create_mss_sized_mbuf_chain(struct rte_mbuf *mbuf,
828 : uint32_t mss, char* pkt_addr, uint32_t pkt_len, uint32_t header_len)
829 : {
830 0 : char *tail_addr, *append_addr = pkt_addr;
831 0 : uint32_t pktlen_to_copy = pkt_len, copy_len;
832 0 : struct rte_mbuf *new_mbuf, *last_mbuf = rte_pktmbuf_lastseg(mbuf);
833 :
834 : /* header is only applicable for first segment */
835 0 : if (mbuf->nb_segs > 1)
836 0 : header_len = 0;
837 :
838 : /* Cannot compute checksum of odd sized mbufs in chain,
839 : * so make it even sized
840 : */
841 0 : if (mss & 1)
842 0 : mbuf->tso_segsz = mss -=1;
843 :
844 0 : while (pktlen_to_copy > 0) {
845 0 : copy_len = mss + header_len - last_mbuf->data_len;
846 0 : header_len = 0;
847 0 : if (pktlen_to_copy > copy_len) {
848 0 : tail_addr = dpdk_pktmbuf_append(mbuf, last_mbuf, copy_len);
849 0 : if (unlikely(tail_addr == NULL))
850 0 : return -1;
851 0 : rte_memcpy(tail_addr, append_addr, copy_len);
852 0 : pktlen_to_copy -= copy_len;
853 0 : append_addr += copy_len;
854 0 : new_mbuf = rte_pktmbuf_alloc(vr_dpdk.rss_mempool);
855 0 : if (unlikely(new_mbuf == NULL)) {
856 : RTE_LOG_DP(DEBUG, VROUTER, "%s: mbuf alloc failed\n",__func__);
857 0 : return -1;
858 : }
859 0 : last_mbuf->next = new_mbuf;
860 0 : last_mbuf = new_mbuf;
861 0 : mbuf->nb_segs += 1;
862 : } else {
863 : /* for last segment */
864 0 : tail_addr = dpdk_pktmbuf_append(mbuf, last_mbuf, pktlen_to_copy);
865 0 : if (unlikely(tail_addr == NULL))
866 0 : return -1;
867 0 : rte_memcpy(tail_addr, append_addr, pktlen_to_copy);
868 0 : pktlen_to_copy = 0;
869 : }
870 : }
871 0 : return 0;
872 : }
873 :
874 : /*
875 : * dpdk_virtio_create_chained_mbuf - Create a chained mbuf and copy the data pointed
876 : * to by pkt_addr of len pkt_len
877 : */
878 : static int
879 1 : dpdk_virtio_create_chained_mbuf(struct rte_mbuf *mbuf, char* pkt_addr, uint32_t pkt_len)
880 : {
881 1 : char *tail_addr, *append_addr = pkt_addr;
882 1 : uint32_t append_len = pkt_len;
883 : struct rte_mbuf *new_mbuf;
884 :
885 5 : while((tail_addr = rte_pktmbuf_append(mbuf, append_len)) == NULL) {
886 4 : uint32_t pkt_tailroom = rte_pktmbuf_tailroom(rte_pktmbuf_lastseg(mbuf));
887 4 : tail_addr = rte_pktmbuf_append(mbuf, pkt_tailroom);
888 4 : if (unlikely(tail_addr == NULL))
889 0 : return -1;
890 4 : rte_memcpy(tail_addr, append_addr, pkt_tailroom);
891 4 : append_len -= pkt_tailroom;
892 4 : append_addr += pkt_tailroom;
893 4 : new_mbuf = rte_pktmbuf_alloc(vr_dpdk.rss_mempool);
894 4 : if (unlikely(new_mbuf == NULL)) {
895 : RTE_LOG_DP(DEBUG, VROUTER, "%s: mbuf alloc failed\n",__func__);
896 0 : return -1;
897 : }
898 4 : rte_pktmbuf_lastseg(mbuf)->next = new_mbuf;
899 4 : mbuf->nb_segs += 1;
900 : }
901 1 : rte_memcpy(tail_addr, append_addr, append_len);
902 1 : return 0;
903 : }
904 :
905 : /*
906 : * dpdk_virtio_from_vm_rx - receive packets from a virtio client so that
907 : * the packets can be handed to vrouter for forwarding. the virtio client is
908 : * usually a VM.
909 : *
910 : * Returns the number of packets received from the virtio.
911 : */
912 : static int
913 2585218483 : dpdk_virtio_from_vm_rx(void *port, struct rte_mbuf **pkts, uint32_t max_pkts)
914 : {
915 2585218483 : struct dpdk_virtio_reader *p = (struct dpdk_virtio_reader *)port;
916 2585218483 : vr_dpdk_virtioq_t *vq = p->rx_virtioq;
917 : uint16_t vq_hard_avail_idx, i;
918 : uint16_t avail_pkts, next_desc_idx, next_avail_idx;
919 : struct vring_desc *desc;
920 : char *pkt_addr, *tail_addr;
921 : struct rte_mbuf *mbuf;
922 2585218483 : uint32_t pkt_len, nb_pkts = 0;
923 : vr_uvh_client_t *vru_cl;
924 :
925 2585218483 : if (unlikely(vq->vdv_ready_state == VQ_NOT_READY)) {
926 2384431448 : DPDK_UDEBUG(VROUTER, &vq->vdv_hash, "%s: queue %p is not ready\n",
927 : __func__, vq);
928 2382942966 : return 0;
929 : }
930 :
931 200787035 : vru_cl = vr_dpdk_virtio_get_vif_client(vq->vdv_vif_idx);
932 248760381 : if (unlikely(vru_cl == NULL))
933 0 : return 0;
934 :
935 248760381 : vq_hard_avail_idx = (*((volatile uint16_t *)&vq->vdv_avail->idx));
936 :
937 : /* Unsigned subtraction gives the right result even with wrap around. */
938 248760381 : avail_pkts = vq_hard_avail_idx - vq->vdv_last_used_idx;
939 248760381 : avail_pkts = RTE_MIN(avail_pkts, max_pkts);
940 248760381 : if (unlikely(avail_pkts == 0)) {
941 248700979 : DPDK_UDEBUG(VROUTER, &vq->vdv_hash, "%s: queue %p has no packets\n",
942 : __func__, vq);
943 248720289 : return 0;
944 : }
945 :
946 59402 : DPDK_UDEBUG(VROUTER, &vq->vdv_hash, "%s: queue %p AVAILABLE %u packets\n",
947 : __func__, vq, avail_pkts);
948 320 : for (i = 0; i < avail_pkts; i++) {
949 160 : uint32_t header_len = 0;
950 : /* Allocate a mbuf. */
951 160 : mbuf = rte_pktmbuf_alloc(vr_dpdk.rss_mempool);
952 160 : if (unlikely(mbuf == NULL)) {
953 0 : p->nb_nombufs++;
954 0 : DPDK_UDEBUG(VROUTER, &vq->vdv_hash, "%s: queue %p no_mbufs=%"PRIu64"\n",
955 : __func__, vq, p->nb_nombufs);
956 0 : break;
957 : }
958 :
959 160 : next_avail_idx = (vq->vdv_last_used_idx + i) & (vq->vdv_size - 1);
960 160 : next_desc_idx = vq->vdv_avail->ring[next_avail_idx];
961 : /*
962 : * Move the (chain of) descriptors to the vdv_used list. The used
963 : * index will, however, only be updated at the end of the loop.
964 : */
965 160 : vq->vdv_used->ring[next_avail_idx].id = next_desc_idx;
966 160 : vq->vdv_used->ring[next_avail_idx].len = 0;
967 :
968 160 : desc = &vq->vdv_desc[next_desc_idx];
969 160 : pkt_len = desc->len;
970 160 : pkt_addr = vr_dpdk_guest_phys_to_host_virt(vru_cl, desc->addr);
971 : /* Check the descriptor is sane. */
972 160 : if (unlikely(desc->len < vq->vdv_hlen ||
973 : desc->addr == 0 || pkt_addr == NULL)) {
974 0 : goto free_mbuf;
975 : }
976 160 : mbuf->tso_segsz = 0;
977 : /* Now pkt_addr points to the virtio_net_hdr. */
978 160 : if (((struct virtio_net_hdr *)pkt_addr)->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
979 160 : mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD;
980 160 : if (((struct virtio_net_hdr *)pkt_addr)->gso_type == VIRTIO_NET_HDR_GSO_TCPV4) {
981 0 : mbuf->ol_flags |= PKT_RX_GSO_TCP4;
982 0 : mbuf->tso_segsz = ((struct virtio_net_hdr *)pkt_addr)->gso_size;
983 160 : } else if (((struct virtio_net_hdr *)pkt_addr)->gso_type == VIRTIO_NET_HDR_GSO_TCPV6) {
984 0 : mbuf->ol_flags |= PKT_RX_GSO_TCP6;
985 0 : mbuf->tso_segsz = ((struct virtio_net_hdr *)pkt_addr)->gso_size;
986 : }
987 :
988 : /* Skip virtio_net_hdr */
989 160 : if (likely(desc->flags & VRING_DESC_F_NEXT &&
990 : pkt_len == vq->vdv_hlen)) {
991 0 : DPDK_UDEBUG(VROUTER, &vq->vdv_hash, "%s: queue %p pkt %u F_NEXT\n",
992 : __func__, vq, i);
993 0 : desc = &vq->vdv_desc[desc->next];
994 0 : pkt_len = desc->len;
995 0 : pkt_addr = vr_dpdk_guest_phys_to_host_virt(vru_cl, desc->addr);
996 : } else {
997 160 : DPDK_UDEBUG(VROUTER, &vq->vdv_hash, "%s: queue %p pkt %u no F_NEXT\n",
998 : __func__, vq, i);
999 160 : pkt_addr += vq->vdv_hlen;
1000 160 : pkt_len -= vq->vdv_hlen;
1001 : }
1002 : /* Now pkt_addr points to the packet data. */
1003 160 : if (mbuf->tso_segsz == 0) {
1004 160 : tail_addr = rte_pktmbuf_append(mbuf, pkt_len);
1005 : /* Check we ready to copy the data. */
1006 160 : if (unlikely(desc->addr == 0 || pkt_addr == NULL)) {
1007 0 : goto free_mbuf;
1008 160 : } else if (unlikely(tail_addr == NULL)) {
1009 : /* If insufficient tailroom, create a chained mbuf and copy the data */
1010 1 : if (unlikely(dpdk_virtio_create_chained_mbuf(mbuf, pkt_addr, pkt_len) < 0)) {
1011 0 : goto free_mbuf;
1012 : }
1013 : } else {
1014 : /* No chaining - Just Copy first descriptor data. */
1015 159 : rte_memcpy(tail_addr, pkt_addr, pkt_len);
1016 : }
1017 : } else {
1018 0 : header_len = dpdk_virtio_get_ip_tcp_hdr_len(pkt_addr, pkt_len);
1019 0 : if (unlikely(dpdk_virtio_create_mss_sized_mbuf_chain(mbuf,
1020 : mbuf->tso_segsz, pkt_addr, pkt_len, header_len) < 0)) {
1021 0 : goto free_mbuf;
1022 : }
1023 : }
1024 :
1025 : /*
1026 : * Gather mbuf from several virtio buffers.
1027 : */
1028 160 : while (unlikely(desc->flags & VRING_DESC_F_NEXT)) {
1029 0 : desc = &vq->vdv_desc[desc->next];
1030 0 : pkt_len = desc->len;
1031 0 : pkt_addr = vr_dpdk_guest_phys_to_host_virt(vru_cl, desc->addr);
1032 0 : if (mbuf->tso_segsz == 0) {
1033 0 : tail_addr = rte_pktmbuf_append(mbuf, pkt_len);
1034 : /* Check we ready to copy the data. */
1035 0 : if (unlikely(desc->addr == 0 || pkt_addr == NULL)) {
1036 0 : goto free_mbuf;
1037 0 : } else if (unlikely(tail_addr == NULL)) {
1038 : /* If insufficient tailroom, create a chained mbuf and copy the data */
1039 0 : if (unlikely(dpdk_virtio_create_chained_mbuf(mbuf, pkt_addr, pkt_len) < 0)) {
1040 0 : goto free_mbuf;
1041 : }
1042 : } else {
1043 : /* No chaining - Just append next descriptor(s) data. */
1044 0 : rte_memcpy(tail_addr, pkt_addr, pkt_len);
1045 : }
1046 : } else {
1047 0 : if (unlikely(dpdk_virtio_create_mss_sized_mbuf_chain(mbuf,
1048 : mbuf->tso_segsz, pkt_addr, pkt_len, header_len) < 0)) {
1049 0 : goto free_mbuf;
1050 : }
1051 :
1052 : }
1053 : }
1054 :
1055 160 : pkts[nb_pkts] = mbuf;
1056 160 : nb_pkts++;
1057 160 : continue;
1058 :
1059 0 : free_mbuf:
1060 0 : DPDK_UDEBUG(VROUTER, &vq->vdv_hash, "%s: queue %p DROP desc->addr %p "
1061 : "pkt_addr %p tail_addr %p len %d\n",
1062 : __func__, vq, desc->addr, pkt_addr, tail_addr, pkt_len);
1063 0 : DPDK_VIRTIO_READER_STATS_PKTS_DROP_ADD(p, 1);
1064 0 : rte_pktmbuf_free(mbuf);
1065 : }
1066 :
1067 : /*
1068 : * Do not call the guest if there are no descriptors processed.
1069 : *
1070 : * If there are no free mbufs on host, the TX queue in guest gets
1071 : * filled up. This makes the guest kernel to switch to interrupt mode
1072 : * and clear the VRING_AVAIL_F_NO_INTERRUPT flag.
1073 : *
1074 : * Meanwhile the host polls the virtio queue, sees the available
1075 : * descriptors and interrupts the guest. Those interrupts get unhandled by
1076 : * the guest virtio driver, so after 100K of the interrupts the IRQ get
1077 : * reported and disabled by the guest kernel.
1078 : */
1079 160 : if (likely(i > 0)) {
1080 160 : vq->vdv_last_used_idx += i;
1081 : rte_wmb();
1082 160 : vq->vdv_used->idx += i;
1083 : RTE_LOG_DP(DEBUG, VROUTER,
1084 : "%s: vif %d vq %p vdv_last_used_idx %d vdv_used->idx %u vdv_avail->idx %u\n",
1085 : __func__, vq->vdv_vif_idx, vq, vq->vdv_last_used_idx,
1086 : vq->vdv_used->idx, vq->vdv_avail->idx);
1087 :
1088 : /* Call guest if required. */
1089 160 : if (unlikely(!(vq->vdv_avail->flags & VRING_AVAIL_F_NO_INTERRUPT))) {
1090 160 : p->nb_syscalls++;
1091 160 : eventfd_write(vq->vdv_callfd, 1);
1092 : }
1093 : }
1094 :
1095 160 : DPDK_UDEBUG(VROUTER, &vq->vdv_hash, "%s: queue %p RETURNS %u pkts\n",
1096 : __func__, vq, nb_pkts);
1097 :
1098 160 : DPDK_VIRTIO_READER_STATS_PKTS_IN_ADD(p, nb_pkts);
1099 :
1100 160 : return nb_pkts;
1101 : }
1102 :
1103 : #ifdef RTE_PORT_STATS_COLLECT
1104 :
1105 : #define DPDK_VIRTIO_WRITER_STATS_PKTS_IN_ADD(port, val) \
1106 : port->stats.n_pkts_in += val
1107 : #define DPDK_VIRTIO_WRITER_STATS_PKTS_DROP_ADD(port, val) \
1108 : port->stats.n_pkts_drop += val
1109 :
1110 : #else
1111 :
1112 : /* keep compiler happy, for unused variables */
1113 : #define DPDK_VIRTIO_WRITER_STATS_PKTS_IN_ADD(port, val) \
1114 : (void)(val)
1115 : #define DPDK_VIRTIO_WRITER_STATS_PKTS_DROP_ADD(port, val) \
1116 : (void)(val)
1117 :
1118 : #endif
1119 :
1120 : static inline int32_t __attribute__((always_inline))
1121 : dpdk_virtio_dev_to_vm_tx_burst_simple(struct dpdk_virtio_writer *p,
1122 : vr_dpdk_virtioq_t *vq, uint16_t res_base_idx, uint16_t res_end_idx,
1123 : struct rte_mbuf **pkts, uint32_t count, uint8_t mrg_hdr)
1124 : {
1125 : struct vring_desc *desc;
1126 : struct rte_mbuf *buff;
1127 : /* The virtio_hdr is initialised to 0. */
1128 83 : struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {{0, 0, 0, 0, 0, 0}, 1};
1129 83 : uint64_t buff_addr = 0;
1130 83 : uint64_t buff_hdr_addr = 0;
1131 : uint32_t head[VR_DPDK_VIRTIO_TX_BURST_SZ];
1132 83 : uint32_t head_idx, packet_success = 0;
1133 : uint16_t res_cur_idx;
1134 : uint8_t virtio_hdr_len;
1135 : vr_uvh_client_t *vru_cl;
1136 :
1137 83 : vru_cl = vr_dpdk_virtio_get_vif_client(vq->vdv_vif_idx);
1138 83 : if (unlikely(vru_cl == NULL))
1139 0 : return 0;
1140 :
1141 83 : res_cur_idx = res_base_idx;
1142 : RTE_LOG_DP(DEBUG, VROUTER, "%s: Current Index %d| End Index %d\n",
1143 : __func__, res_cur_idx, res_end_idx);
1144 :
1145 : /* Prefetch available ring to retrieve indexes. */
1146 83 : rte_prefetch0(&vq->vdv_avail->ring[res_cur_idx & (vq->vdv_size - 1)]);
1147 :
1148 : /* Retrieve all of the head indexes first to avoid caching issues. */
1149 166 : for (head_idx = 0; head_idx < count; head_idx++)
1150 83 : head[head_idx] = vq->vdv_avail->ring[(res_cur_idx + head_idx) &
1151 83 : (vq->vdv_size - 1)];
1152 :
1153 83 : virtio_hdr_len = (mrg_hdr)? sizeof(struct virtio_net_hdr_mrg_rxbuf):
1154 : sizeof(struct virtio_net_hdr);
1155 :
1156 : /* Prefetch descriptor index. */
1157 83 : rte_prefetch0(&vq->vdv_desc[head[packet_success]]);
1158 :
1159 166 : while (res_cur_idx != res_end_idx) {
1160 83 : uint32_t offset = 0, vb_offset = 0;
1161 83 : uint32_t pkt_len, len_to_cpy, data_len, total_copied = 0;
1162 83 : uint8_t hdr = 0, uncompleted_pkt = 0;
1163 :
1164 : /* Get descriptor from available ring */
1165 83 : desc = &vq->vdv_desc[head[packet_success]];
1166 :
1167 83 : buff = pkts[packet_success];
1168 :
1169 : /* Convert from gpa to vva (guest physical addr -> vhost virtual addr) */
1170 83 : buff_addr = (uintptr_t)vr_dpdk_guest_phys_to_host_virt(vru_cl, desc->addr);
1171 :
1172 : /* Copy virtio_hdr to packet and increment buffer address */
1173 83 : buff_hdr_addr = buff_addr;
1174 :
1175 83 : if (unlikely(buff_addr == (uint64_t)NULL)) {
1176 : /* Retry with next descriptor */
1177 0 : uncompleted_pkt = 1;
1178 0 : goto next_descr;
1179 : }
1180 :
1181 : /* Prefetch buffer address. */
1182 83 : rte_prefetch0((void *)(uintptr_t)buff_addr);
1183 :
1184 : /*
1185 : * If the descriptors are chained the header and data are
1186 : * placed in separate buffers.
1187 : */
1188 83 : if (likely(desc->flags & VRING_DESC_F_NEXT)
1189 0 : && !mrg_hdr && (desc->len == sizeof(struct virtio_net_hdr))) {
1190 : /*
1191 : * TODO: verify that desc->next is sane below.
1192 : */
1193 0 : desc = &vq->vdv_desc[desc->next];
1194 : /* Buffer address translation. */
1195 0 : buff_addr = (uintptr_t)vr_dpdk_guest_phys_to_host_virt(vru_cl, desc->addr);
1196 0 : if (unlikely(buff_addr == (uint64_t)NULL)) {
1197 : /* Retry with next descriptor */
1198 0 : uncompleted_pkt = 1;
1199 0 : goto next_descr;
1200 : }
1201 : } else {
1202 83 : vb_offset += virtio_hdr_len;
1203 83 : hdr = 1;
1204 : }
1205 :
1206 83 : pkt_len = rte_pktmbuf_pkt_len(buff);
1207 83 : data_len = rte_pktmbuf_data_len(buff);
1208 83 : len_to_cpy = RTE_MIN(data_len,
1209 : hdr ? desc->len - virtio_hdr_len : desc->len);
1210 87 : while (total_copied < pkt_len) {
1211 : /* Copy mbuf data to buffer */
1212 87 : rte_memcpy((void *)(uintptr_t)(buff_addr + vb_offset),
1213 87 : rte_pktmbuf_mtod_offset(buff, const void *, offset),
1214 : len_to_cpy);
1215 :
1216 87 : offset += len_to_cpy;
1217 87 : vb_offset += len_to_cpy;
1218 87 : total_copied += len_to_cpy;
1219 :
1220 : /* The whole packet completes */
1221 87 : if (likely(total_copied == pkt_len))
1222 83 : break;
1223 :
1224 : /* The current segment completes */
1225 4 : if (offset == data_len) {
1226 4 : buff = buff->next;
1227 4 : offset = 0;
1228 4 : data_len = rte_pktmbuf_data_len(buff);
1229 : }
1230 :
1231 : /* The current vring descriptor done */
1232 4 : if (vb_offset == desc->len) {
1233 0 : if (desc->flags & VRING_DESC_F_NEXT) {
1234 0 : desc = &vq->vdv_desc[desc->next];
1235 0 : buff_addr = (uintptr_t)vr_dpdk_guest_phys_to_host_virt(vru_cl, desc->addr);
1236 0 : if (unlikely(buff_addr == (uint64_t)NULL)) {
1237 : /* Retry with next descriptor */
1238 0 : uncompleted_pkt = 1;
1239 0 : goto next_descr;
1240 : }
1241 0 : vb_offset = 0;
1242 : } else {
1243 : /* Room in vring buffer is not enough */
1244 0 : uncompleted_pkt = 1;
1245 0 : break;
1246 : }
1247 : }
1248 4 : len_to_cpy = RTE_MIN(data_len - offset, desc->len - vb_offset);
1249 : };
1250 :
1251 0 : next_descr:
1252 : /* Update used ring with desc information */
1253 83 : vq->vdv_used->ring[res_cur_idx & (vq->vdv_size - 1)].id =
1254 83 : head[packet_success];
1255 :
1256 : /* Drop the packet if it is uncompleted */
1257 83 : if (unlikely(uncompleted_pkt == 1))
1258 0 : vq->vdv_used->ring[res_cur_idx & (vq->vdv_size - 1)].len =
1259 : virtio_hdr_len;
1260 : else
1261 83 : vq->vdv_used->ring[res_cur_idx & (vq->vdv_size - 1)].len =
1262 83 : pkt_len + virtio_hdr_len;
1263 :
1264 83 : res_cur_idx++;
1265 83 : packet_success++;
1266 :
1267 : /* TODO: in DPDK 2.1 we do not copy the header
1268 : if (unlikely(uncompleted_pkt == 1))
1269 : continue;
1270 : */
1271 83 : if (buff_hdr_addr) {
1272 83 : rte_memcpy((void *)(uintptr_t)buff_hdr_addr,
1273 : (const void *)&virtio_hdr, virtio_hdr_len);
1274 : }
1275 :
1276 83 : if (likely(res_cur_idx < res_end_idx)) {
1277 : /* Prefetch descriptor index. */
1278 0 : rte_prefetch0(&vq->vdv_desc[head[packet_success]]);
1279 : }
1280 : }
1281 :
1282 83 : rte_compiler_barrier();
1283 :
1284 : /* Wait until it's our turn to add our buffer to the used ring. */
1285 83 : while (unlikely(vq->vdv_last_used_idx != res_base_idx))
1286 0 : rte_pause();
1287 :
1288 83 : *(volatile uint16_t *)&vq->vdv_used->idx += count;
1289 83 : vq->vdv_last_used_idx = res_end_idx;
1290 : RTE_LOG_DP(DEBUG, VROUTER, "%s: vif %d vq %p last_used_idx %d used->idx %d\n",
1291 : __func__, vq->vdv_vif_idx, vq, vq->vdv_last_used_idx, vq->vdv_used->idx);
1292 :
1293 : /* flush used->idx update before we read avail->flags. */
1294 : rte_mb();
1295 :
1296 : /* Kick the guest if necessary. */
1297 83 : if (unlikely(!(vq->vdv_avail->flags & VRING_AVAIL_F_NO_INTERRUPT))) {
1298 83 : p->nb_syscalls++;
1299 83 : eventfd_write(vq->vdv_callfd, 1);
1300 : }
1301 83 : return count;
1302 : }
1303 :
1304 : /**
1305 : * This function adds buffers to the virtio devices RX virtqueue. Buffers can
1306 : * be received from the physical port or from another virtio device. A packet
1307 : * count is returned to indicate the number of packets that are succesfully
1308 : * added to the RX queue. This function works when mergeable is disabled.
1309 : *
1310 : * This is an adaptation of DPDK virtio_dev_rx() function.
1311 : * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
1312 : * BSD LICENSE
1313 : */
1314 : static inline uint32_t __attribute__((always_inline))
1315 105 : dpdk_virtio_dev_to_vm_tx_burst(struct dpdk_virtio_writer *p,
1316 : vr_dpdk_virtioq_t *vq, struct rte_mbuf **pkts, uint32_t count)
1317 : {
1318 : uint16_t res_base_idx, res_end_idx, avail_idx, free_entries;
1319 105 : uint8_t success = 0;
1320 :
1321 105 : if (unlikely(vq->vdv_ready_state == VQ_NOT_READY))
1322 22 : return 0;
1323 :
1324 : /*
1325 : * As many data cores may want access to available buffers,
1326 : * they need to be reserved.
1327 : */
1328 : do {
1329 83 : res_base_idx = vq->vdv_last_used_idx_res;
1330 83 : avail_idx = *((volatile uint16_t *)&vq->vdv_avail->idx);
1331 :
1332 83 : free_entries = (avail_idx - res_base_idx);
1333 : /*check that we have enough buffers*/
1334 83 : if (unlikely(count > free_entries))
1335 0 : count = free_entries;
1336 :
1337 83 : if (unlikely(count == 0))
1338 0 : return 0;
1339 :
1340 83 : res_end_idx = res_base_idx + count;
1341 : /* vq->vdv_last_used_idx_res is atomically updated. */
1342 : /* TODO: Allow to disable cmpset if no concurrency in application. */
1343 83 : success = rte_atomic16_cmpset(&vq->vdv_last_used_idx_res,
1344 : res_base_idx, res_end_idx);
1345 83 : } while (unlikely(success == 0));
1346 :
1347 166 : return dpdk_virtio_dev_to_vm_tx_burst_simple(p, vq,
1348 : res_base_idx, res_end_idx,
1349 : pkts, count, !VIRTIO_HDR_MRG_RXBUF);
1350 : }
1351 :
1352 : static inline uint32_t __attribute__((always_inline))
1353 : copy_from_mbuf_to_vring(vr_dpdk_virtioq_t *vq, vr_uvh_client_t *vru_cl, uint16_t res_base_idx,
1354 : uint16_t res_end_idx, struct vq_buf_vector *buf_vec,
1355 : struct virtio_net_hdr_mrg_rxbuf* virtio_hdr,
1356 : struct rte_mbuf *pkt)
1357 : {
1358 0 : uint32_t vec_idx = 0;
1359 0 : uint32_t entry_success = 0;
1360 0 : uint16_t cur_idx = res_base_idx;
1361 0 : uint64_t vb_addr = 0;
1362 0 : uint64_t vb_hdr_addr = 0;
1363 0 : uint32_t seg_offset = 0;
1364 0 : uint32_t vb_offset = 0;
1365 : uint32_t seg_avail;
1366 : uint32_t vb_avail;
1367 : uint32_t cpy_len, entry_len;
1368 :
1369 0 : if (pkt == NULL)
1370 0 : return 0;
1371 :
1372 : RTE_LOG_DP(DEBUG, VROUTER, "%s: Current Index %d| "
1373 : "End Index %d\n",
1374 : __func__, cur_idx, res_end_idx);
1375 :
1376 : /*
1377 : * Convert from gpa to vva
1378 : * (guest physical addr -> vhost virtual addr)
1379 : */
1380 0 : vb_addr = (uintptr_t)vr_dpdk_guest_phys_to_host_virt(vru_cl,
1381 0 : buf_vec[vec_idx].buf_addr);
1382 0 : vb_hdr_addr = vb_addr;
1383 :
1384 : /* Prefetch buffer address. */
1385 0 : rte_prefetch0((void *)(uintptr_t)vb_addr);
1386 :
1387 : RTE_LOG_DP(DEBUG, VROUTER, "%s RX: Num merge buffers %d\n",
1388 : __func__, virtio_hdr->num_buffers);
1389 :
1390 0 : rte_memcpy((void *)(uintptr_t)vb_hdr_addr,
1391 0 : (const void *)virtio_hdr, vq->vdv_hlen);
1392 :
1393 0 : seg_avail = rte_pktmbuf_data_len(pkt);
1394 0 : vb_offset = vq->vdv_hlen;
1395 0 : vb_avail = buf_vec[vec_idx].buf_len - vq->vdv_hlen;
1396 :
1397 0 : entry_len = vq->vdv_hlen;
1398 :
1399 0 : if (vb_avail == 0) {
1400 0 : uint32_t desc_idx =
1401 0 : buf_vec[vec_idx].desc_idx;
1402 :
1403 0 : if ((vq->vdv_desc[desc_idx].flags
1404 0 : & VRING_DESC_F_NEXT) == 0) {
1405 : /* Update vdv_used ring with vdv_desc information */
1406 0 : vq->vdv_used->ring[cur_idx & (vq->vdv_size - 1)].id
1407 0 : = buf_vec[vec_idx].desc_idx;
1408 0 : vq->vdv_used->ring[cur_idx & (vq->vdv_size - 1)].len
1409 0 : = entry_len;
1410 :
1411 0 : entry_len = 0;
1412 0 : cur_idx++;
1413 0 : entry_success++;
1414 : }
1415 :
1416 0 : vec_idx++;
1417 0 : vb_addr = (uintptr_t)vr_dpdk_guest_phys_to_host_virt(vru_cl,
1418 0 : buf_vec[vec_idx].buf_addr);
1419 :
1420 : /* Prefetch buffer address. */
1421 0 : rte_prefetch0((void *)(uintptr_t)vb_addr);
1422 0 : vb_offset = 0;
1423 0 : vb_avail = buf_vec[vec_idx].buf_len;
1424 : }
1425 :
1426 0 : cpy_len = RTE_MIN(vb_avail, seg_avail);
1427 :
1428 0 : while (cpy_len > 0) {
1429 : /* Copy mbuf data to vring buffer */
1430 0 : rte_memcpy((void *)(uintptr_t)(vb_addr + vb_offset),
1431 0 : rte_pktmbuf_mtod_offset(pkt, const void *, seg_offset),
1432 : cpy_len);
1433 :
1434 0 : seg_offset += cpy_len;
1435 0 : vb_offset += cpy_len;
1436 0 : seg_avail -= cpy_len;
1437 0 : vb_avail -= cpy_len;
1438 0 : entry_len += cpy_len;
1439 :
1440 0 : if (seg_avail != 0) {
1441 : /*
1442 : * The virtio buffer in this vring
1443 : * entry reach to its end.
1444 : * But the segment doesn't complete.
1445 : */
1446 0 : if ((vq->vdv_desc[buf_vec[vec_idx].desc_idx].flags &
1447 : VRING_DESC_F_NEXT) == 0) {
1448 : /* Update vdv_used ring with vdv_desc information */
1449 0 : vq->vdv_used->ring[cur_idx & (vq->vdv_size - 1)].id
1450 0 : = buf_vec[vec_idx].desc_idx;
1451 0 : vq->vdv_used->ring[cur_idx & (vq->vdv_size - 1)].len
1452 0 : = entry_len;
1453 0 : entry_len = 0;
1454 0 : cur_idx++;
1455 0 : entry_success++;
1456 : }
1457 :
1458 0 : vec_idx++;
1459 0 : vb_addr = (uintptr_t)vr_dpdk_guest_phys_to_host_virt(vru_cl,
1460 0 : buf_vec[vec_idx].buf_addr);
1461 0 : vb_offset = 0;
1462 0 : vb_avail = buf_vec[vec_idx].buf_len;
1463 0 : cpy_len = RTE_MIN(vb_avail, seg_avail);
1464 : } else {
1465 : /*
1466 : * This current segment complete, need continue to
1467 : * check if the whole packet complete or not.
1468 : */
1469 0 : pkt = pkt->next;
1470 0 : if (pkt != NULL) {
1471 : /*
1472 : * There are more segments.
1473 : */
1474 0 : if (vb_avail == 0) {
1475 : /*
1476 : * This current buffer from vring is
1477 : * vdv_used up, need fetch next buffer
1478 : * from buf_vec.
1479 : */
1480 0 : uint32_t desc_idx =
1481 0 : buf_vec[vec_idx].desc_idx;
1482 :
1483 0 : if ((vq->vdv_desc[desc_idx].flags &
1484 : VRING_DESC_F_NEXT) == 0) {
1485 0 : uint16_t wrapped_idx =
1486 0 : cur_idx & (vq->vdv_size - 1);
1487 : /*
1488 : * Update vdv_used ring with the
1489 : * descriptor information
1490 : */
1491 0 : vq->vdv_used->ring[wrapped_idx].id
1492 0 : = desc_idx;
1493 0 : vq->vdv_used->ring[wrapped_idx].len
1494 0 : = entry_len;
1495 0 : entry_success++;
1496 0 : entry_len = 0;
1497 0 : cur_idx++;
1498 : }
1499 :
1500 : /* Get next buffer from buf_vec. */
1501 0 : vec_idx++;
1502 0 : vb_addr = (uintptr_t)vr_dpdk_guest_phys_to_host_virt(vru_cl,
1503 0 : buf_vec[vec_idx].buf_addr);
1504 0 : vb_avail =
1505 0 : buf_vec[vec_idx].buf_len;
1506 0 : vb_offset = 0;
1507 : }
1508 :
1509 0 : seg_offset = 0;
1510 0 : seg_avail = rte_pktmbuf_data_len(pkt);
1511 0 : cpy_len = RTE_MIN(vb_avail, seg_avail);
1512 : } else {
1513 : /*
1514 : * This whole packet completes.
1515 : */
1516 : /* Update vdv_used ring with vdv_desc information */
1517 0 : vq->vdv_used->ring[cur_idx & (vq->vdv_size - 1)].id
1518 0 : = buf_vec[vec_idx].desc_idx;
1519 0 : vq->vdv_used->ring[cur_idx & (vq->vdv_size - 1)].len
1520 0 : = entry_len;
1521 0 : entry_success++;
1522 0 : break;
1523 : }
1524 : }
1525 : }
1526 :
1527 0 : return entry_success;
1528 : }
1529 :
1530 : static inline void __attribute__((always_inline))
1531 : update_secure_len(vr_dpdk_virtioq_t *vq, uint32_t id,
1532 : uint32_t *secure_len, struct vq_buf_vector *buf_vec, uint32_t *vec_idx)
1533 : {
1534 0 : uint16_t wrapped_idx = id & (vq->vdv_size - 1);
1535 0 : uint32_t idx = vq->vdv_avail->ring[wrapped_idx];
1536 : uint8_t next_desc;
1537 0 : uint32_t len = *secure_len;
1538 0 : uint32_t vec_id = *vec_idx;
1539 :
1540 : do {
1541 0 : if (vec_id >= VR_BUF_VECTOR_MAX)
1542 0 : break;
1543 0 : next_desc = 0;
1544 0 : len += vq->vdv_desc[idx].len;
1545 0 : buf_vec[vec_id].buf_addr = vq->vdv_desc[idx].addr;
1546 0 : buf_vec[vec_id].buf_len = vq->vdv_desc[idx].len;
1547 0 : buf_vec[vec_id].desc_idx = idx;
1548 0 : vec_id++;
1549 :
1550 0 : if (vq->vdv_desc[idx].flags & VRING_DESC_F_NEXT) {
1551 0 : idx = vq->vdv_desc[idx].next;
1552 0 : next_desc = 1;
1553 : }
1554 0 : } while (next_desc);
1555 :
1556 0 : *secure_len = len;
1557 0 : *vec_idx = vec_id;
1558 0 : }
1559 :
1560 : static inline uint32_t __attribute__((always_inline))
1561 0 : dpdk_virtio_dev_to_vm_tx_burst_mergeable(struct dpdk_virtio_writer *p,
1562 : vr_dpdk_virtioq_t *vq, struct rte_mbuf **pkts, uint32_t count)
1563 : {
1564 0 : uint32_t pkt_idx = 0, start_idx = 0, entry_success = 0, simple_count;
1565 : uint16_t avail_idx;
1566 : uint16_t res_base_idx, res_cur_idx;
1567 0 : uint8_t success = 0;
1568 : vr_uvh_client_t *vru_cl;
1569 : struct vq_buf_vector buf_vec[VR_BUF_VECTOR_MAX];
1570 :
1571 0 : if (unlikely(vq->vdv_ready_state == VQ_NOT_READY))
1572 0 : return 0;
1573 :
1574 0 : vru_cl = vr_dpdk_virtio_get_vif_client(vq->vdv_vif_idx);
1575 0 : if (unlikely(vru_cl == NULL))
1576 0 : return 0;
1577 :
1578 0 : count = RTE_MIN((uint32_t)VR_DPDK_VIRTIO_TX_BURST_SZ, count);
1579 :
1580 0 : if (count == 0)
1581 0 : return 0;
1582 :
1583 : /* Check if we can call -
1584 : * dpdk_virtio_dev_to_vm_tx_burst_simple() for some/all pkts
1585 : */
1586 : do {
1587 0 : res_base_idx = vq->vdv_last_used_idx_res;
1588 0 : res_cur_idx = res_base_idx;
1589 0 : avail_idx = *((volatile uint16_t *)&vq->vdv_avail->idx);
1590 0 : for (pkt_idx = 0; pkt_idx < count; pkt_idx++) {
1591 0 : uint32_t pkt_len = pkts[pkt_idx]->pkt_len + vq->vdv_hlen;
1592 0 : if (unlikely(res_cur_idx == avail_idx)) {
1593 : RTE_LOG_DP(DEBUG, VROUTER,
1594 : "Failed "
1595 : "to get enough vdv_desc from "
1596 : "vring\n");
1597 0 : count = pkt_idx;
1598 0 : break;
1599 : } else {
1600 : uint8_t next_desc;
1601 0 : uint16_t wrapped_idx = res_cur_idx & (vq->vdv_size - 1);
1602 0 : uint32_t len = 0, idx = vq->vdv_avail->ring[wrapped_idx];
1603 : do {
1604 0 : next_desc = 0;
1605 0 : len += vq->vdv_desc[idx].len;
1606 0 : if (vq->vdv_desc[idx].flags & VRING_DESC_F_NEXT) {
1607 0 : idx = vq->vdv_desc[idx].next;
1608 0 : next_desc = 1;
1609 : }
1610 0 : if (len > pkt_len)
1611 0 : break;
1612 0 : }while (next_desc);
1613 0 : if (len < pkt_len)
1614 0 : break;
1615 0 : res_cur_idx++;
1616 : }
1617 : }
1618 :
1619 : /* If there are no packets to pass to
1620 : * dpdk_virtio_dev_to_vm_tx_burst_simple() function, break
1621 : */
1622 0 : if (pkt_idx == 0)
1623 0 : break;
1624 :
1625 0 : success = rte_atomic16_cmpset(&vq->vdv_last_used_idx_res,
1626 : res_base_idx, res_cur_idx);
1627 0 : } while (unlikely(success == 0));
1628 :
1629 0 : if (pkt_idx) {
1630 0 : simple_count = dpdk_virtio_dev_to_vm_tx_burst_simple(p, vq,
1631 : res_base_idx, res_cur_idx,
1632 : pkts, pkt_idx, VIRTIO_HDR_MRG_RXBUF);
1633 0 : if (simple_count < pkt_idx)
1634 0 : return simple_count;
1635 : }
1636 :
1637 0 : start_idx = pkt_idx;
1638 0 : for (pkt_idx = start_idx; pkt_idx < count; pkt_idx++) {
1639 0 : struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {
1640 : {0, 0, 0, 0, 0, 0}, 0};
1641 0 : uint32_t pkt_len = pkts[pkt_idx]->pkt_len + vq->vdv_hlen;
1642 :
1643 : do {
1644 : /*
1645 : * As many data cores may want access to available
1646 : * buffers, they need to be reserved.
1647 : */
1648 0 : uint32_t secure_len = 0;
1649 0 : uint32_t vec_idx = 0;
1650 :
1651 0 : res_base_idx = vq->vdv_last_used_idx_res;
1652 0 : res_cur_idx = res_base_idx;
1653 :
1654 : do {
1655 0 : avail_idx = *((volatile uint16_t *)&vq->vdv_avail->idx);
1656 0 : if (unlikely(res_cur_idx == avail_idx)) {
1657 : RTE_LOG_DP(DEBUG, VROUTER,
1658 : "Failed "
1659 : "to get enough vdv_desc from "
1660 : "vring\n");
1661 0 : return pkt_idx;
1662 : } else {
1663 0 : update_secure_len(vq, res_cur_idx, &secure_len, buf_vec, &vec_idx);
1664 0 : res_cur_idx++;
1665 : }
1666 0 : } while (pkt_len > secure_len);
1667 :
1668 : /* vq->vdv_last_used_idx_res is atomically updated. */
1669 0 : success = rte_atomic16_cmpset(&vq->vdv_last_used_idx_res,
1670 : res_base_idx,
1671 : res_cur_idx);
1672 0 : } while (success == 0);
1673 :
1674 : /* Fill the virtio hdr */
1675 0 : virtio_hdr.num_buffers = res_cur_idx - res_base_idx;
1676 0 : if (pkts[pkt_idx]->ol_flags & PKT_RX_GSO_TCP4) {
1677 0 : virtio_hdr.hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
1678 0 : virtio_hdr.hdr.gso_size = pkts[pkt_idx]->tso_segsz;
1679 0 : } else if (pkts[pkt_idx]->ol_flags & PKT_RX_GSO_TCP6) {
1680 0 : virtio_hdr.hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
1681 0 : virtio_hdr.hdr.gso_size = pkts[pkt_idx]->tso_segsz;
1682 : }
1683 :
1684 0 : entry_success = copy_from_mbuf_to_vring(vq, vru_cl, res_base_idx,
1685 0 : res_cur_idx, buf_vec, &virtio_hdr, pkts[pkt_idx]);
1686 :
1687 0 : rte_compiler_barrier();
1688 :
1689 : /*
1690 : * Wait until it's our turn to add our buffer
1691 : * to the vdv_used ring.
1692 : */
1693 0 : while (unlikely(vq->vdv_last_used_idx != res_base_idx))
1694 0 : rte_pause();
1695 :
1696 0 : *(volatile uint16_t *)&vq->vdv_used->idx += entry_success;
1697 0 : vq->vdv_last_used_idx = res_cur_idx;
1698 :
1699 : /* flush vdv_used->idx update before we read vdv_avail->flags. */
1700 : rte_mb();
1701 :
1702 : /* Kick the guest if necessary. */
1703 0 : if (unlikely(!(vq->vdv_avail->flags & VRING_AVAIL_F_NO_INTERRUPT))) {
1704 0 : p->nb_syscalls++;
1705 0 : eventfd_write(vq->vdv_callfd, 1);
1706 : }
1707 : }
1708 :
1709 0 : return count;
1710 : }
1711 :
1712 : void
1713 0 : vr_dpdk_set_vhost_send_func(unsigned int vif_idx, uint32_t mrg)
1714 : {
1715 : int i;
1716 : vr_dpdk_virtioq_t *vq;
1717 :
1718 0 : if (vif_idx >= VR_MAX_INTERFACES) {
1719 0 : return;
1720 : }
1721 :
1722 0 : for (i = 0; i < VR_DPDK_VIRTIO_MAX_QUEUES*2; i++) {
1723 0 : if (i & 1) {
1724 0 : vq = &vr_dpdk_virtio_rxqs[vif_idx][i/2];
1725 : } else {
1726 0 : vq = &vr_dpdk_virtio_txqs[vif_idx][i/2];
1727 : }
1728 :
1729 0 : if (mrg) {
1730 0 : vq->vdv_send_func = dpdk_virtio_dev_to_vm_tx_burst_mergeable;
1731 0 : vq->vdv_hlen = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1732 : } else {
1733 0 : vq->vdv_send_func = dpdk_virtio_dev_to_vm_tx_burst;
1734 0 : vq->vdv_hlen = sizeof(struct virtio_net_hdr);
1735 : }
1736 : }
1737 : }
1738 :
1739 : static inline void
1740 130 : dpdk_virtio_send_burst(struct dpdk_virtio_writer *p)
1741 : {
1742 130 : uint32_t nb_tx = 0;
1743 : int i;
1744 :
1745 130 : if (likely(p->tx_buf_count)) {
1746 : /*
1747 : * prefetch the tx buffer to be sent
1748 : * This will avoid large cpu cycles in the
1749 : * dpdk_virtio_dev_to_vm_tx_burst_mergeable()
1750 : */
1751 266 : for (i=0;i<p->tx_buf_count;i++)
1752 136 : rte_prefetch0((void *)p->tx_buf[i]);
1753 130 : if (likely(p->tx_virtioq->vdv_send_func != NULL)) {
1754 105 : nb_tx = p->tx_virtioq->vdv_send_func(p, p->tx_virtioq,
1755 105 : p->tx_buf, p->tx_buf_count);
1756 : }
1757 :
1758 130 : DPDK_VIRTIO_WRITER_STATS_PKTS_DROP_ADD(p, p->tx_buf_count - nb_tx);
1759 : /* dpdk_virtio_dev_to_vm_tx_burst() does not free any mbufs */
1760 266 : while (likely(p->tx_buf_count)) {
1761 136 : p->tx_buf_count--;
1762 136 : p->tx_mbufs -= p->tx_buf[p->tx_buf_count]->nb_segs;
1763 136 : rte_pktmbuf_free(p->tx_buf[p->tx_buf_count]);
1764 : }
1765 : }
1766 130 : }
1767 :
1768 : /*
1769 : * dpdk_virtio_to_vm_tx - sends a packet from vrouter to a virtio client. The
1770 : * virtio client is usually a VM.
1771 : *
1772 : * Returns nothing.
1773 : */
1774 : static int
1775 136 : dpdk_virtio_to_vm_tx(void *port, struct rte_mbuf *pkt)
1776 : {
1777 136 : struct dpdk_virtio_writer *p = (struct dpdk_virtio_writer *)port;
1778 136 : const unsigned lcore_id = rte_lcore_id();
1779 136 : struct vr_dpdk_lcore *lcore = NULL;
1780 :
1781 136 : if (lcore_id >= VR_DPDK_FWD_LCORE_ID) {
1782 134 : lcore = vr_dpdk.lcores[lcore_id];
1783 134 : p->last_pkt_tx = lcore->lcore_fwd_loops;
1784 : }
1785 :
1786 136 : p->tx_buf[p->tx_buf_count++] = pkt;
1787 136 : p->tx_mbufs += pkt->nb_segs;
1788 136 : DPDK_VIRTIO_WRITER_STATS_PKTS_IN_ADD(p, 1);
1789 :
1790 136 : if (unlikely(p->tx_mbufs >= VR_DPDK_VIRTIO_TX_BURST_SZ)) {
1791 0 : dpdk_virtio_send_burst(p);
1792 0 : if (lcore) {
1793 0 : p->last_pkt_tx_flush = lcore->lcore_fwd_loops;
1794 : }
1795 : }
1796 :
1797 136 : return 0;
1798 : }
1799 :
1800 : /*
1801 : * dpdk_virtio_to_vm_flush - flushes packets from vrouter to a virtio client.
1802 : * The virtio client is usually a VM.
1803 : *
1804 : * Returns nothing.
1805 : */
1806 : static int
1807 429324395 : dpdk_virtio_to_vm_flush(void *port)
1808 : {
1809 429324395 : struct dpdk_virtio_writer *p = (struct dpdk_virtio_writer *)port;
1810 : unsigned lcore_id;
1811 429324395 : struct vr_dpdk_lcore *lcore = NULL;
1812 :
1813 429324395 : if (p->tx_buf_count == 0) {
1814 429308078 : return 0;
1815 : }
1816 :
1817 16317 : lcore_id = rte_lcore_id();
1818 130 : if (lcore_id >= VR_DPDK_FWD_LCORE_ID) {
1819 128 : lcore = vr_dpdk.lcores[lcore_id];
1820 : }
1821 :
1822 130 : if (lcore) {
1823 : /*
1824 : * Flush the TX queue if it has been a while since it was last done OR
1825 : * if there are packets in the queue and no packets have been enqueued
1826 : * for a short while. The latter condition helps to reduce latency in
1827 : * case there isn't a lot of traffic on the queue.
1828 : */
1829 128 : if ((lcore->lcore_fwd_loops - p->last_pkt_tx_flush) <
1830 : VR_DPDK_TX_FLUSH_LOOPS) {
1831 0 : if ((lcore->lcore_fwd_loops - p->last_pkt_tx) <
1832 : VR_DPDK_TX_IDLE_LOOPS) {
1833 0 : return 0;
1834 : }
1835 : }
1836 : }
1837 :
1838 130 : dpdk_virtio_send_burst(p);
1839 130 : if (lcore) {
1840 128 : p->last_pkt_tx_flush = lcore->lcore_fwd_loops;
1841 : }
1842 :
1843 130 : return 0;
1844 : }
1845 :
1846 : /*
1847 : * vr_dpdk_virtio_set_vring_base - sets the vring base using data sent by
1848 : * vhost client.
1849 : *
1850 : * Returns 0 on success, -1 otherwise.
1851 : */
1852 : int
1853 388 : vr_dpdk_virtio_set_vring_base(unsigned int vif_idx, unsigned int vring_idx,
1854 : unsigned int vring_base)
1855 : {
1856 : vr_dpdk_virtioq_t *vq;
1857 :
1858 388 : if ((vif_idx >= VR_MAX_INTERFACES)
1859 388 : || (vring_idx >= (2 * VR_DPDK_VIRTIO_MAX_QUEUES))) {
1860 0 : return -1;
1861 : }
1862 :
1863 : /*
1864 : * RX rings are even numbered and TX rings are odd numbered from the
1865 : * VM's point of view. From vrouter's point of view, VM's TX ring is
1866 : * vrouter's RX ring and vice versa.
1867 : */
1868 388 : if (vring_idx & 1) {
1869 194 : vq = &vr_dpdk_virtio_rxqs[vif_idx][vring_idx/2];
1870 : } else {
1871 194 : vq = &vr_dpdk_virtio_txqs[vif_idx][vring_idx/2];
1872 : }
1873 :
1874 388 : vq->vdv_last_used_idx = vring_base;
1875 388 : vq->vdv_last_used_idx_res = vring_base;
1876 388 : return 0;
1877 : }
1878 :
1879 : /*
1880 : * vr_dpdk_virtio_get_vring_base - gets the vring base for the specified vring
1881 : * sent by the vhost client.
1882 : *
1883 : * Returns 0 on success, -1 otherwise.
1884 : */
1885 : int
1886 0 : vr_dpdk_virtio_get_vring_base(unsigned int vif_idx, unsigned int vring_idx,
1887 : unsigned int *vring_basep)
1888 : {
1889 : vr_dpdk_virtioq_t *vq;
1890 :
1891 0 : if ((vif_idx >= VR_MAX_INTERFACES)
1892 0 : || (vring_idx >= (2 * VR_DPDK_VIRTIO_MAX_QUEUES))) {
1893 0 : return -1;
1894 : }
1895 :
1896 : /*
1897 : * RX rings are even numbered and TX rings are odd numbered from the
1898 : * VM's point of view. From vrouter's point of view, VM's TX ring is
1899 : * vrouter's RX ring and vice versa.
1900 : */
1901 0 : if (vring_idx & 1) {
1902 0 : vq = &vr_dpdk_virtio_rxqs[vif_idx][vring_idx/2];
1903 : } else {
1904 0 : vq = &vr_dpdk_virtio_txqs[vif_idx][vring_idx/2];
1905 : }
1906 :
1907 0 : *vring_basep = vq->vdv_last_used_idx;
1908 :
1909 : /*
1910 : * This is usually called when qemu shuts down a virtio queue. Set the
1911 : * state to indicate that this queue should not be used any more.
1912 : */
1913 0 : vq->vdv_ready_state = VQ_NOT_READY;
1914 : rte_wmb();
1915 0 : synchronize_rcu();
1916 :
1917 : /* Reset the queue. We reset only those values we analyze in
1918 : * uvhm_check_vring_ready()
1919 : */
1920 0 : vq->vdv_desc = NULL;
1921 0 : if (vq->vdv_callfd) {
1922 0 : close(vq->vdv_callfd);
1923 0 : vq->vdv_callfd = 0;
1924 : }
1925 :
1926 0 : return 0;
1927 : }
1928 :
1929 : /*
1930 : * vr_dpdk_virtio_recover_vring_base - recovers the vring base from the shared
1931 : * memory after vRouter crash.
1932 : *
1933 : * Returns 0 on success, -1 otherwise.
1934 : */
1935 : int
1936 388 : vr_dpdk_virtio_recover_vring_base(unsigned int vif_idx, unsigned int vring_idx)
1937 : {
1938 : vr_dpdk_virtioq_t *vq;
1939 :
1940 388 : if ((vif_idx >= VR_MAX_INTERFACES)
1941 388 : || (vring_idx >= (2 * VR_DPDK_VIRTIO_MAX_QUEUES))) {
1942 0 : return -1;
1943 : }
1944 :
1945 388 : if (vring_idx & 1) {
1946 194 : vq = &vr_dpdk_virtio_rxqs[vif_idx][vring_idx/2];
1947 : } else {
1948 194 : vq = &vr_dpdk_virtio_txqs[vif_idx][vring_idx/2];
1949 : }
1950 :
1951 388 : if (vq->vdv_used) {
1952 : /* Reading base index from the shared memory. */
1953 388 : if (vq->vdv_last_used_idx != vq->vdv_used->idx) {
1954 0 : RTE_LOG(INFO, UVHOST, " recovering vring base %d -> %d\n",
1955 : vq->vdv_last_used_idx, vq->vdv_used->idx);
1956 0 : vr_dpdk_virtio_set_vring_base(vif_idx, vring_idx, vq->vdv_used->idx);
1957 : }
1958 : }
1959 :
1960 388 : return 0;
1961 : }
1962 :
1963 : /*
1964 : * vr_dpdk_set_vring_addr - Sets the address of the virtio descriptor and
1965 : * available/used rings based on messages sent by the vhost client.
1966 : *
1967 : * Returns 0 on success, -1 otherwise.
1968 : */
1969 : int
1970 388 : vr_dpdk_set_vring_addr(unsigned int vif_idx, unsigned int vring_idx,
1971 : struct vring_desc *vrucv_desc,
1972 : struct vring_avail *vrucv_avail,
1973 : struct vring_used *vrucv_used)
1974 : {
1975 : vr_dpdk_virtioq_t *vq;
1976 :
1977 388 : if ((vif_idx >= VR_MAX_INTERFACES)
1978 388 : || (vring_idx >= (2 * VR_DPDK_VIRTIO_MAX_QUEUES))) {
1979 0 : return -1;
1980 : }
1981 :
1982 : /*
1983 : * RX rings are even numbered and TX rings are odd numbered from the
1984 : * VM's point of view. From vrouter's point of view, VM's TX ring is
1985 : * vrouter's RX ring and vice versa.
1986 : */
1987 388 : if (vring_idx & 1) {
1988 194 : vq = &vr_dpdk_virtio_rxqs[vif_idx][vring_idx/2];
1989 : } else {
1990 194 : vq = &vr_dpdk_virtio_txqs[vif_idx][vring_idx/2];
1991 : }
1992 :
1993 388 : vq->vdv_desc = vrucv_desc;
1994 388 : vq->vdv_avail = vrucv_avail;
1995 388 : vq->vdv_used = vrucv_used;
1996 :
1997 : /*
1998 : * Tell the guest that it need not interrupt vrouter when it updates the
1999 : * available ring (as vrouter is polling it).
2000 : */
2001 388 : vq->vdv_used->flags |= VRING_USED_F_NO_NOTIFY;
2002 :
2003 388 : return 0;
2004 : }
2005 :
2006 : /*
2007 : * vr_dpdk_set_ring_num_desc - sets the number of descriptors in a vring
2008 : * based on messages from the vhost client.
2009 : *
2010 : * Returns 0 on success, -1 otherwise.
2011 : */
2012 : int
2013 388 : vr_dpdk_set_ring_num_desc(unsigned int vif_idx, unsigned int vring_idx,
2014 : unsigned int num_desc)
2015 : {
2016 : vr_dpdk_virtioq_t *vq;
2017 :
2018 388 : if ((vif_idx >= VR_MAX_INTERFACES) || (vring_idx > 2 * VR_DPDK_VIRTIO_MAX_QUEUES)) {
2019 0 : return -1;
2020 : }
2021 :
2022 : /*
2023 : * RX rings are even numbered and TX rings are odd numbered from the
2024 : * VM's point of view. From vrouter's point of view, VM's TX ring is
2025 : * vrouter's RX ring and vice versa.
2026 : */
2027 388 : if (vring_idx & 1) {
2028 194 : vq = &vr_dpdk_virtio_rxqs[vif_idx][vring_idx/2];
2029 : } else {
2030 194 : vq = &vr_dpdk_virtio_txqs[vif_idx][vring_idx/2];
2031 : }
2032 :
2033 388 : vq->vdv_size = num_desc;
2034 :
2035 388 : return 0;
2036 : }
2037 :
2038 : /*
2039 : * vr_dpdk_set_ring_callfd - set the eventd used to raise interrupts in
2040 : * the guest (if required). Returns 0 on success, -1 otherwise.
2041 : */
2042 : int
2043 388 : vr_dpdk_set_ring_callfd(unsigned int vif_idx, unsigned int vring_idx,
2044 : int callfd)
2045 : {
2046 : vr_dpdk_virtioq_t *vq;
2047 :
2048 388 : if ((vif_idx >= VR_MAX_INTERFACES)
2049 388 : || (vring_idx >= (2 * VR_DPDK_VIRTIO_MAX_QUEUES))) {
2050 0 : return -1;
2051 : }
2052 :
2053 : /*
2054 : * RX rings are even numbered and TX rings are odd numbered from the
2055 : * VM's point of view. From vrouter's point of view, VM's TX ring is
2056 : * vrouter's RX ring and vice versa.
2057 : */
2058 388 : if (vring_idx & 1) {
2059 194 : vq = &vr_dpdk_virtio_rxqs[vif_idx][vring_idx/2];
2060 : } else {
2061 194 : vq = &vr_dpdk_virtio_txqs[vif_idx][vring_idx/2];
2062 : }
2063 :
2064 388 : if (vq->vdv_callfd > 0) {
2065 0 : close(vq->vdv_callfd);
2066 : }
2067 388 : vq->vdv_callfd = callfd;
2068 :
2069 388 : return 0;
2070 : }
2071 :
2072 : /*
2073 : * vr_dpdk_set_virtq_ready - sets the virtio queue ready state to indicate
2074 : * whether forwarding can start on the virtio queue or not.
2075 : *
2076 : * Returns 0 on success, -1 otherwise.
2077 : */
2078 : int
2079 776 : vr_dpdk_set_virtq_ready(unsigned int vif_idx, unsigned int vring_idx,
2080 : vq_ready_state_t ready)
2081 : {
2082 : vr_dpdk_virtioq_t *vq;
2083 :
2084 776 : if ((vif_idx >= VR_MAX_INTERFACES)
2085 776 : || (vring_idx >= (2 * VR_DPDK_VIRTIO_MAX_QUEUES))) {
2086 0 : return -1;
2087 : }
2088 :
2089 : /*
2090 : * RX rings are even numbered and TX rings are odd numbered from the
2091 : * VM's point of view. From vrouter's point of view, VM's TX ring is
2092 : * vrouter's RX ring and vice versa.
2093 : */
2094 776 : if (vring_idx & 1) {
2095 388 : vq = &vr_dpdk_virtio_rxqs[vif_idx][vring_idx/2];
2096 : } else {
2097 388 : vq = &vr_dpdk_virtio_txqs[vif_idx][vring_idx/2];
2098 : }
2099 :
2100 776 : if (vq->vdv_hlen == 0) {
2101 : struct vr_interface *vif;
2102 266 : vif = __vrouter_get_interface(vrouter_get(0), vq->vdv_vif_idx);
2103 266 : if (vif && (vif->vif_flags & VIF_FLAG_MRG_RXBUF)) {
2104 0 : vq->vdv_send_func = dpdk_virtio_dev_to_vm_tx_burst_mergeable;
2105 0 : vq->vdv_hlen = sizeof(struct virtio_net_hdr_mrg_rxbuf);
2106 : } else {
2107 266 : vq->vdv_send_func = dpdk_virtio_dev_to_vm_tx_burst;
2108 266 : vq->vdv_hlen = sizeof(struct virtio_net_hdr);
2109 : }
2110 : }
2111 :
2112 776 : vq->vdv_ready_state = ready;
2113 :
2114 776 : return 0;
2115 : }
2116 :
2117 : /*
2118 : * vr_dpdk_virtio_set_vif_client - sets a pointer to per vif state. Currently
2119 : * used to store a pointer to the vhost client structure.
2120 : *
2121 : * Returns nothing.
2122 : */
2123 : void
2124 401 : vr_dpdk_virtio_set_vif_client(unsigned int idx, void *client)
2125 : {
2126 401 : if (idx >= VR_MAX_INTERFACES) {
2127 0 : return;
2128 : }
2129 :
2130 401 : vr_dpdk_vif_clients[idx] = client;
2131 :
2132 401 : return;
2133 : }
2134 :
2135 : /*
2136 : * vr_dpdk_virtio_get_vif_client - returns a pointer to per vif state if it
2137 : * exists, NULL otherwise.
2138 : */
2139 : void *
2140 248895961 : vr_dpdk_virtio_get_vif_client(unsigned int idx)
2141 : {
2142 248895961 : if (idx >= VR_MAX_INTERFACES) {
2143 0 : return NULL;
2144 : }
2145 :
2146 248895961 : return vr_dpdk_vif_clients[idx];
2147 : }
2148 :
2149 : static int
2150 336 : dpdk_virtio_reader_stats_read(void *port,
2151 : struct rte_port_in_stats *stats, int clear)
2152 : {
2153 336 : struct dpdk_virtio_reader *p = (struct dpdk_virtio_reader *)port;
2154 :
2155 336 : if (stats != NULL)
2156 336 : memcpy(stats, &p->stats, sizeof(p->stats));
2157 :
2158 336 : if (clear)
2159 20 : memset(&p->stats, 0, sizeof(p->stats));
2160 :
2161 336 : return 0;
2162 : }
2163 :
2164 : static int
2165 672 : dpdk_virtio_writer_stats_read(void *port,
2166 : struct rte_port_out_stats *stats, int clear)
2167 : {
2168 672 : struct dpdk_virtio_reader *p = (struct dpdk_virtio_reader *)port;
2169 :
2170 672 : if (stats != NULL)
2171 672 : memcpy(stats, &p->stats, sizeof(p->stats));
2172 :
2173 672 : if (clear)
2174 40 : memset(&p->stats, 0, sizeof(p->stats));
2175 :
2176 672 : return 0;
2177 : }
2178 :
2179 : /* Update extra statistics for virtio queue */
2180 : void
2181 948 : vr_dpdk_virtio_xstats_update(struct vr_interface_stats *stats,
2182 : struct vr_dpdk_queue *queue)
2183 : {
2184 : struct dpdk_virtio_reader *reader;
2185 : struct dpdk_virtio_writer *writer;
2186 :
2187 948 : if (queue->rxq_ops.f_rx == vr_dpdk_virtio_reader_ops.f_rx) {
2188 316 : reader = (struct dpdk_virtio_reader *)queue->q_queue_h;
2189 316 : stats->vis_port_isyscalls = reader->nb_syscalls;
2190 316 : stats->vis_port_inombufs = reader->nb_nombufs;
2191 632 : } else if (queue->txq_ops.f_tx == vr_dpdk_virtio_writer_ops.f_tx) {
2192 632 : writer = (struct dpdk_virtio_writer *)queue->q_queue_h;
2193 632 : stats->vis_port_osyscalls = writer->nb_syscalls;
2194 : }
2195 948 : }
|
