Line data Source code
1 : /*
2 : * Copyright (C) 2014 Semihalf.
3 : *
4 : * This program is free software; you can redistribute it and/or
5 : * modify it under the terms of the GNU General Public License as
6 : * published by the Free Software Foundation version 2.
7 : *
8 : * This program is distributed "as is" WITHOUT ANY WARRANTY of any
9 : * kind, whether express or implied; without even the implied warranty
10 : * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 : * GNU General Public License for more details.
12 : *
13 : * vr_dpdk_interface.c -- vRouter interface callbacks
14 : *
15 : */
16 :
17 : #include "vr_dpdk.h"
18 : #include "vr_dpdk_netlink.h"
19 : #include "vr_dpdk_usocket.h"
20 : #include "vr_dpdk_virtio.h"
21 : #include "vr_dpdk_representor.h"
22 :
23 : #include <rte_errno.h>
24 : #include <rte_ethdev_pci.h>
25 : #include <rte_ethdev_vdev.h>
26 : #include <rte_ethdev.h>
27 : #include <rte_ip_frag.h>
28 : #include <rte_ip.h>
29 : #include <rte_port_ethdev.h>
30 : #include <rte_pci.h>
31 :
32 : #include <linux/if_tun.h>
33 : #include <net/if_arp.h>
34 : #include <sys/ioctl.h>
35 :
36 : void
37 215 : vr_dpdk_interface_queue_free(struct vr_interface *vif)
38 : {
39 : unsigned int lcore, i;
40 : struct vr_dpdk_lcore *lcore_p;
41 :
42 215 : i = lcore = VR_DPDK_FWD_LCORE_ID;
43 : do {
44 1290 : lcore_p = vr_dpdk.lcores[i];
45 1290 : if (lcore_p->lcore_tx_queues[vif->vif_idx]) {
46 860 : vr_free(lcore_p->lcore_tx_queues[vif->vif_idx],
47 : VR_INTERFACE_QUEUE_OBJECT);
48 860 : lcore_p->lcore_tx_queues[vif->vif_idx] = NULL;
49 : }
50 :
51 1290 : if (lcore_p->lcore_tx_queue_params[vif->vif_idx]) {
52 860 : vr_free(lcore_p->lcore_tx_queue_params[vif->vif_idx],
53 : VR_INTERFACE_QUEUE_OBJECT);
54 860 : lcore_p->lcore_tx_queue_params[vif->vif_idx] = NULL;
55 : }
56 :
57 1290 : if (lcore_p->lcore_hw_queue_to_dpdk_index[vif->vif_idx]) {
58 0 : vr_free(lcore_p->lcore_hw_queue_to_dpdk_index[vif->vif_idx],
59 : VR_INTERFACE_QUEUE_OBJECT);
60 0 : lcore_p->lcore_hw_queue_to_dpdk_index[vif->vif_idx] = NULL;
61 : }
62 :
63 1290 : lcore_p->num_tx_queues_per_lcore[vif->vif_idx] = 0;
64 :
65 1290 : i = rte_get_next_lcore(i, 1, 1);
66 1290 : } while (i != lcore);
67 :
68 215 : if (vif->vif_queue_host_data) {
69 0 : vr_free(vif->vif_queue_host_data, VR_INTERFACE_QUEUE_OBJECT);
70 0 : vif->vif_queue_host_data = NULL;
71 : }
72 :
73 215 : return;
74 : }
75 :
76 : int
77 332 : vr_dpdk_interface_queue_setup(struct vr_interface *vif)
78 : {
79 332 : int16_t vif_max_queue = -1;
80 : uint16_t num_tx_queues_per_lcore;
81 : unsigned int lcore, i;
82 :
83 : struct vr_dpdk_lcore *lcore_p;
84 : struct vif_queue_dpdk_data *q_data;
85 :
86 332 : if (vif->vif_num_hw_queues) {
87 0 : num_tx_queues_per_lcore = vif->vif_num_hw_queues;
88 0 : for (i = 0; i < vif->vif_num_hw_queues; i++) {
89 0 : if (vif->vif_hw_queues[i] > vif_max_queue) {
90 0 : vif_max_queue = vif->vif_hw_queues[i];
91 : }
92 : }
93 :
94 0 : q_data = vif->vif_queue_host_data;
95 0 : if (!q_data) {
96 0 : vif->vif_queue_host_data = vr_malloc(sizeof(*q_data),
97 : VR_INTERFACE_QUEUE_OBJECT);
98 0 : if (!vif->vif_queue_host_data) {
99 0 : goto unwind;
100 : }
101 :
102 0 : q_data = (struct vif_queue_dpdk_data *)vif->vif_queue_host_data;
103 0 : memset(q_data->vqdd_queue_to_lcore, -1,
104 : VR_DPDK_MAX_NB_TX_QUEUES * sizeof(int16_t));
105 : }
106 : } else {
107 332 : num_tx_queues_per_lcore = 1;
108 : }
109 :
110 332 : lcore = VR_DPDK_FWD_LCORE_ID;
111 : do {
112 1992 : if (lcore >= VR_DPDK_PACKET_LCORE_ID) {
113 1328 : lcore_p = vr_dpdk.lcores[lcore];
114 1328 : lcore_p->lcore_hw_queue[vif->vif_idx] = -1;
115 2656 : lcore_p->lcore_tx_queues[vif->vif_idx] =
116 1328 : vr_zalloc(num_tx_queues_per_lcore * sizeof(struct vr_dpdk_queue),
117 : VR_INTERFACE_QUEUE_OBJECT);
118 1328 : if (!lcore_p->lcore_tx_queues) {
119 0 : goto unwind;
120 : }
121 :
122 2656 : lcore_p->lcore_tx_queue_params[vif->vif_idx] =
123 1328 : vr_zalloc(num_tx_queues_per_lcore *
124 : sizeof(struct vr_dpdk_queue_params),
125 : VR_INTERFACE_QUEUE_OBJECT);
126 1328 : if (!lcore_p->lcore_tx_queue_params) {
127 0 : goto unwind;
128 : }
129 1328 : lcore_p->num_tx_queues_per_lcore[vif->vif_idx] =
130 : num_tx_queues_per_lcore;
131 :
132 1328 : if (vif_max_queue > 0) {
133 0 : lcore_p->lcore_hw_queue_to_dpdk_index[vif->vif_idx] =
134 0 : (int16_t *)vr_zalloc(sizeof(int16_t) * (vif_max_queue + 1),
135 : VR_INTERFACE_QUEUE_OBJECT);
136 0 : if (!lcore_p->lcore_hw_queue_to_dpdk_index[vif->vif_idx]) {
137 0 : goto unwind;
138 : }
139 : }
140 : }
141 :
142 1992 : lcore = rte_get_next_lcore(lcore, 1, 1);
143 1992 : } while (lcore != VR_DPDK_FWD_LCORE_ID);
144 :
145 332 : return 0;
146 :
147 0 : unwind:
148 0 : vr_dpdk_interface_queue_free(vif);
149 0 : return -ENOMEM;
150 : }
151 :
152 : /*
153 : * dpdk_virtual_if_add - add a virtual (virtio) interface to vrouter.
154 : * Returns 0 on success, < 0 otherwise.
155 : */
156 : static int
157 332 : dpdk_virtual_if_add(struct vr_interface *vif)
158 : {
159 : int ret;
160 : uint16_t nrxqs, ntxqs;
161 :
162 332 : RTE_LOG(INFO, VROUTER, "Adding vif %u (gen. %u) virtual device %s\n",
163 : vif->vif_idx, vif->vif_gen, vif->vif_name);
164 :
165 332 : nrxqs = vr_dpdk_virtio_nrxqs(vif);
166 : /* virtio TX is thread safe, so we assign TX queue to each lcore */
167 332 : ntxqs = (uint16_t)-1;
168 :
169 332 : ret = vr_dpdk_interface_queue_setup(vif);
170 332 : if (ret)
171 0 : return ret;
172 :
173 332 : ret = vr_dpdk_lcore_if_schedule(vif, vr_dpdk_lcore_least_used_get(),
174 : nrxqs, &vr_dpdk_virtio_rx_queue_init,
175 : ntxqs, &vr_dpdk_virtio_tx_queue_init);
176 332 : if (ret) {
177 0 : return ret;
178 : }
179 :
180 : /*
181 : * When something goes wrong, vr_netlink_uvhost_vif_add() returns
182 : * non-zero value. Then we return this value here. It is handled by
183 : * dp-core and dpdk_virtual_if_del() is called, so there is no need
184 : * to do it manually here.
185 : *
186 : * Check dp-core/vf_interface.c:eth_drv_add() for reference.
187 : */
188 332 : return vr_netlink_uvhost_vif_add(vif->vif_name, vif->vif_idx, vif->vif_gen,
189 332 : nrxqs, ntxqs, vif->vif_vhostuser_mode);
190 : }
191 :
192 : /* Creating mock physical/vhost/agent device used for simulating
193 : * interfaces through vtest */
194 : static int
195 164 : dpdk_mock_vif_add(struct vr_interface *vif)
196 : {
197 164 : int ret = 0;
198 :
199 : /* A virtual interface is being added for mock physical/vhost device,
200 : * For upper layers, it looks like packet being sent/receive on
201 : * physical/vhost interface*/
202 164 : vif->vif_flags |= VIF_FLAG_MOCK_DEVICE;
203 164 : ret = dpdk_virtual_if_add(vif);
204 :
205 164 : if(ret != 0)
206 0 : RTE_LOG(ERR, VROUTER, "Error adding mock device vif: %u for vif_name: %s\n",
207 : vif->vif_idx, vif->vif_name);
208 : else
209 164 : RTE_LOG(INFO, VROUTER, "Added Mock device vif: %u for vif_name: %s\n",
210 : vif->vif_idx, vif->vif_name);
211 :
212 164 : return ret;
213 : }
214 :
215 : /*
216 : * dpdk_virtual_vlan_if_add - add a virtual VLAN interface to vRouter.
217 : * Returns 0 on success, < 0 otherwise.
218 : */
219 : static int
220 0 : dpdk_virtual_vlan_if_add(struct vr_interface *vif)
221 : {
222 :
223 0 : RTE_LOG(INFO, VROUTER, "Adding vif %u (gen. %u) virtual VLAN(o/i) %u/%u device %s\n",
224 : vif->vif_idx, vif->vif_gen, vif->vif_ovlan_id, vif->vif_vlan_id,
225 : vif->vif_name);
226 0 : RTE_LOG(INFO, VROUTER, " parent vif %u device %s\n",
227 : vif->vif_parent->vif_idx, vif->vif_parent->vif_name);
228 :
229 0 : return 0;
230 : }
231 :
232 : /*
233 : * dpdk_virtual_if_del - deletes a virtual (virtio) interface from vrouter.
234 : * Returns 0 on success, -1 otherwise.
235 : */
236 : static int
237 166 : dpdk_virtual_if_del(struct vr_interface *vif)
238 : {
239 : int ret;
240 :
241 166 : RTE_LOG(INFO, VROUTER, "Deleting vif %u virtual device\n",
242 : vif->vif_idx);
243 :
244 166 : vr_dpdk_lcore_if_unschedule(vif);
245 166 : ret = vr_netlink_uvhost_vif_del(vif->vif_idx);
246 166 : if (ret) {
247 0 : RTE_LOG(ERR, VROUTER, "Error deleting vif %u virtual device %s\n",
248 : vif->vif_idx, vif->vif_name);
249 : }
250 :
251 166 : vr_dpdk_interface_queue_free(vif);
252 :
253 166 : return ret;
254 : }
255 :
256 : /*
257 : * dpdk_virtual_vlan_if_del - deletes a virtual VLAN interface from vRouter.
258 : * Returns 0 on success, < 0 otherwise.
259 : */
260 : static int
261 0 : dpdk_virtual_vlan_if_del(struct vr_interface *vif)
262 : {
263 0 : RTE_LOG(INFO, VROUTER, "Deleting vif %u virtual VLAN(o/i) %u/%u device\n",
264 : vif->vif_idx, vif->vif_ovlan_id, vif->vif_vlan_id);
265 :
266 0 : return 0;
267 : }
268 :
269 : static inline void
270 : dpdk_dbdf_to_pci(unsigned int dbdf,
271 : struct rte_pci_addr *address)
272 : {
273 : address->domain = (dbdf >> 16);
274 : address->bus = (dbdf >> 8) & 0xff;
275 : address->devid = (dbdf >> 3) & 0x1f;
276 : address->function = (dbdf & 0x7);
277 :
278 : return;
279 : }
280 :
281 : static inline unsigned
282 : dpdk_pci_to_dbdf(struct rte_pci_addr *address)
283 : {
284 : return address->domain << 16
285 : | address->bus << 8
286 : | address->devid
287 : | address->function;
288 : }
289 :
290 : /* mirrors the function used in bonding */
291 : static inline uint8_t
292 : dpdk_find_port_id_by_pci_addr(const struct rte_pci_addr *addr)
293 : {
294 : uint8_t i;
295 : struct rte_pci_addr *eth_pci_addr;
296 :
297 : VR_DPDK_RTE_ETH_FOREACH_DEV(i) {
298 : if (rte_eth_devices[i].data == NULL)
299 : continue;
300 :
301 : if (strcmp(rte_eth_devices[i].device->driver->name, "net_bonding") == 0)
302 : return i;
303 :
304 : if (rte_eth_devices[i].device != NULL) {
305 : eth_pci_addr = &(RTE_DEV_TO_PCI(rte_eth_devices[i].device)->addr);
306 : RTE_LOG_DP(DEBUG, VROUTER, "count %d eth_pci_addr %x %x %x %x \n",
307 : i, eth_pci_addr->bus, eth_pci_addr->devid,
308 : eth_pci_addr->domain, eth_pci_addr->function);
309 : RTE_LOG_DP(DEBUG, VROUTER, "count %d addr %x %x %x %x \n",
310 : i, addr->bus, addr->devid,
311 : addr->domain, addr->function);
312 : if (addr->bus == eth_pci_addr->bus &&
313 : addr->devid == eth_pci_addr->devid &&
314 : addr->domain == eth_pci_addr->domain &&
315 : addr->function == eth_pci_addr->function) {
316 : return i;
317 : }
318 : }
319 : }
320 :
321 : return VR_DPDK_INVALID_PORT_ID;
322 : }
323 :
324 : /* Function to check if sub string is found.
325 : In case of bond with vlan vif name will have (bond0.101)
326 : and the eth dev will have name (eth_bond_bond0), so
327 : need to compore bond0 instead of bond0.101 with eth_bond_bond0
328 : to get device port id 0
329 : */
330 : uint8_t
331 0 : find_sub_str(const char *dest, const char *src)
332 : {
333 0 : char str[10] = {0};
334 0 : memcpy(str, src, 5);
335 :
336 0 : if(strstr(dest, str))
337 0 : return 1;
338 :
339 0 : return 0;
340 : }
341 :
342 : uint8_t
343 0 : dpdk_find_port_id_by_vif_name(struct vr_interface *vif)
344 : {
345 : uint8_t i;
346 : struct rte_ether_addr *mac_addr;
347 :
348 0 : VR_DPDK_RTE_ETH_FOREACH_DEV(i) {
349 0 : if (rte_eth_devices[i].device == NULL)
350 0 : continue;
351 :
352 0 : if (!rte_eth_dev_is_valid_port(i))
353 0 : continue;
354 :
355 0 : if (rte_eth_devices[i].data == NULL)
356 0 : continue;
357 :
358 0 : if (strncmp((const char *)vif->vif_name, "bond", 4) == 0) {
359 0 : if (find_sub_str(rte_eth_devices[i].data->name, (const char *)vif->vif_name)) {
360 0 : return i;
361 : }
362 0 : } else if (rte_eth_devices[i].data->mac_addrs != NULL) {
363 0 : mac_addr = rte_eth_devices[i].data->mac_addrs;
364 0 : if (memcmp(vif->vif_mac, mac_addr->addr_bytes, RTE_ETHER_ADDR_LEN) == 0) {
365 0 : return i;
366 : }
367 : }
368 : }
369 :
370 0 : RTE_LOG(ERR, VROUTER, "%s: Returning id:%d if_name:%s\n", __func__, i, vif->vif_name);
371 0 : return VR_DPDK_INVALID_PORT_ID;
372 : }
373 :
374 : /* function will return the bond port ids created.
375 : for L3MH we will have more then one bond ports
376 : */
377 : bool
378 0 : dpdk_find_bond_port_id_list(struct vr_dpdk_bond_port_list *bond_list)
379 : {
380 : uint8_t i;
381 0 : bool bond_port_id_found = false;
382 :
383 0 : memset(bond_list, 0, sizeof(struct vr_dpdk_bond_port_list));
384 0 : VR_DPDK_RTE_ETH_FOREACH_DEV(i) {
385 0 : if (rte_eth_devices[i].device == NULL)
386 0 : continue;
387 :
388 0 : if (!rte_eth_dev_is_valid_port(i))
389 0 : continue;
390 :
391 0 : if (rte_eth_devices[i].device->driver == NULL ||
392 0 : rte_eth_devices[i].device->driver->name == NULL)
393 0 : continue;
394 :
395 0 : if (strcmp(rte_eth_devices[i].device->driver->name, "net_bonding") == 0) {
396 0 : bond_list->intf_list[bond_list->intf_count++] = i;
397 0 : bond_port_id_found = true;
398 : }
399 : }
400 :
401 0 : return bond_port_id_found;
402 : }
403 :
404 : static inline void
405 0 : dpdk_set_addr_vlan_filter_strip(uint32_t port_id, struct vr_interface *vif)
406 : {
407 : uint32_t i, ret;
408 : uint16_t *port_id_ptr;
409 0 : int port_num = 0;
410 0 : struct vr_dpdk_ethdev *ethdev = &vr_dpdk.ethdevs[port_id];
411 :
412 0 : port_id_ptr = (ethdev->ethdev_nb_slaves == -1)?
413 0 : ðdev->ethdev_port_id:ethdev->ethdev_slaves;
414 :
415 : do {
416 : /*
417 : * TODO: vf_lcore_id check for SR-IOV VF should be a per-interface
418 : * check to handle the case where a bond has a VF and a PF in it.
419 : */
420 :
421 : /*
422 : * Set the MAC address of slave interfaces. Doing it from the bond driver in
423 : * DPDK doesn't seem to work on SR-IOV VFs.
424 : */
425 0 : if ((ethdev->ethdev_nb_slaves != -1) && vr_dpdk.vf_lcore_id) {
426 0 : ret = rte_eth_dev_default_mac_addr_set(*port_id_ptr,
427 0 : (struct rte_ether_addr *)vif->vif_mac);
428 0 : if (ret == 0) {
429 0 : RTE_LOG(INFO, VROUTER, "Bond slave port %d now uses vif MAC "
430 : MAC_FORMAT "\n",
431 : *port_id_ptr, MAC_VALUE(vif->vif_mac));
432 : } else {
433 0 : RTE_LOG(ERR, VROUTER, "Error setting vif MAC to bond slave port %d: "
434 : "%s (%d)\n",
435 : *port_id_ptr, rte_strerror(-ret), -ret);
436 : }
437 : }
438 :
439 0 : if ((vr_dpdk.vlan_tag != VLAN_ID_INVALID) && vr_dpdk.vf_lcore_id) {
440 0 : ret = rte_eth_dev_set_vlan_offload(*port_id_ptr, ETH_VLAN_FILTER_OFFLOAD);
441 0 : if (ret) {
442 0 : RTE_LOG(INFO, VROUTER, "Error %d enabling vlan offload on port %d\n",
443 : ret, *port_id_ptr);
444 : } else {
445 0 : RTE_LOG(INFO, VROUTER, "Enabled vlan offload on port %d\n",
446 : *port_id_ptr);
447 : }
448 :
449 0 : ret = rte_eth_dev_vlan_filter(*port_id_ptr, vr_dpdk.vlan_tag, 1);
450 0 : if (ret) {
451 0 : RTE_LOG(INFO, VROUTER, "Error %d enabling vlan %d on port %d\n",
452 : ret, vr_dpdk.vlan_tag, *port_id_ptr);
453 : } else {
454 0 : RTE_LOG(INFO, VROUTER, "Enabled vlan %d on port %d\n",
455 : vr_dpdk.vlan_tag, *port_id_ptr);
456 : }
457 : }
458 :
459 0 : for (i=0; i< rte_eth_devices[*port_id_ptr].data->nb_rx_queues; i++)
460 : {
461 0 : if (vif->vif_flags & VIF_FLAG_VLAN_OFFLOAD) {
462 0 : rte_eth_dev_set_vlan_strip_on_queue(*port_id_ptr, i, 1);
463 : }
464 : }
465 0 : port_num++;
466 0 : port_id_ptr++;
467 0 : } while (port_num < ethdev->ethdev_nb_slaves);
468 0 : }
469 :
470 : /*
471 : * vr_ethdev_inner_cksum_capable - check if the NIC is capable of calculating the
472 : * inner checksum in an overlay packet. ixgbe and i40e (and their VFs)support it,
473 : * so handle the cases where the physical interface is one of these or a bond with
474 : * these NICs.
475 : *
476 : * Returns 1 if capable and 9 if not.
477 : */
478 : static int
479 0 : vr_ethdev_inner_cksum_capable(struct vr_dpdk_ethdev *ethdev)
480 : {
481 : struct rte_eth_dev_info dev_info;
482 : uint16_t *port_id_ptr;
483 0 : int port_num = 0;
484 :
485 0 : port_id_ptr = (ethdev->ethdev_nb_slaves == -1)?
486 0 : ðdev->ethdev_port_id:ethdev->ethdev_slaves;
487 :
488 : do {
489 0 : rte_eth_dev_info_get(*port_id_ptr, &dev_info);
490 0 : if (dev_info.driver_name) {
491 0 : if ((strncmp(dev_info.driver_name, "net_ixgbe",
492 0 : strlen("net_ixgbe")) != 0) &&
493 0 : (strncmp(dev_info.driver_name, "net_i40e",
494 0 : strlen("net_i40e")) != 0) &&
495 0 : (strncmp(dev_info.driver_name, "net_ice",
496 : strlen("net_ice")) != 0)) {
497 0 : return 0;
498 : }
499 : } else {
500 0 : return 0;
501 : }
502 :
503 0 : port_num++;
504 0 : port_id_ptr++;
505 0 : } while (port_num < ethdev->ethdev_nb_slaves);
506 :
507 0 : return 1;
508 : }
509 :
510 : void
511 0 : dpdk_vif_attach_ethdev(struct vr_interface *vif,
512 : struct vr_dpdk_ethdev *ethdev)
513 : {
514 : struct rte_ether_addr mac_addr;
515 : struct rte_eth_dev_info dev_info;
516 : int ret;
517 :
518 0 : vif->vif_os = (void *)ethdev;
519 :
520 0 : rte_eth_dev_info_get(ethdev->ethdev_port_id, &dev_info);
521 0 : if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM
522 0 : && dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM
523 0 : && dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM
524 0 : && vr_ethdev_inner_cksum_capable(ethdev)) {
525 0 : vif->vif_flags |= VIF_FLAG_TX_CSUM_OFFLOAD;
526 : } else {
527 0 : vif->vif_flags &= ~VIF_FLAG_TX_CSUM_OFFLOAD;
528 : }
529 :
530 0 : if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_VLAN_INSERT
531 0 : && dev_info.rx_offload_capa & DEV_RX_OFFLOAD_VLAN_STRIP) {
532 0 : vif->vif_flags |= VIF_FLAG_VLAN_OFFLOAD;
533 : } else {
534 0 : vif->vif_flags &= ~VIF_FLAG_VLAN_OFFLOAD;
535 : }
536 :
537 : /*
538 : * Do not want to overwrite what agent had sent.
539 : * Set only if the address is null.
540 : */
541 0 : memset(&mac_addr, 0, sizeof(mac_addr));
542 0 : if (memcmp(vif->vif_mac, mac_addr.addr_bytes, RTE_ETHER_ADDR_LEN) == 0) {
543 0 : rte_eth_macaddr_get(ethdev->ethdev_port_id, &mac_addr);
544 0 : memcpy(vif->vif_mac, mac_addr.addr_bytes, RTE_ETHER_ADDR_LEN);
545 : } else {
546 : /*
547 : * On some hardware (e100e, virtual functions, etc) the MAC is random,
548 : * so we check if vif and NIC MACs are match and set the NIC MAC.
549 : */
550 0 : rte_eth_macaddr_get(ethdev->ethdev_port_id, &mac_addr);
551 0 : if (memcmp(vif->vif_mac, mac_addr.addr_bytes, RTE_ETHER_ADDR_LEN) != 0) {
552 : /* No match, so set vif MAC to NIC. */
553 0 : ret = rte_eth_dev_default_mac_addr_set(ethdev->ethdev_port_id,
554 0 : (struct rte_ether_addr *)vif->vif_mac);
555 0 : if (ret == 0) {
556 0 : RTE_LOG(INFO, VROUTER, " eth dev %s now use vif MAC "
557 : MAC_FORMAT "\n",
558 : vif->vif_name, MAC_VALUE(vif->vif_mac));
559 : } else {
560 0 : RTE_LOG(ERR, VROUTER, " error setting vif MAC to eth dev %s: "
561 : "%s (%d)\n",
562 : vif->vif_name, rte_strerror(-ret), -ret);
563 : }
564 : }
565 : }
566 0 : }
567 :
568 : /*
569 : * dpdk_vlan_forwarding_if_add - add VLAN forwarding interface
570 : * Returns 0 on success, < 0 otherwise.
571 : */
572 : int
573 0 : dpdk_vlan_forwarding_if_add(void)
574 : {
575 : int ret;
576 :
577 0 : RTE_LOG(INFO, VROUTER, "Adding VLAN forwarding interface %s\n",
578 : vr_dpdk.vlan_name);
579 :
580 : /* Allocate vlan vif. */
581 0 : vr_dpdk.vlan_vif = vr_zalloc(sizeof(struct vr_interface),
582 : VR_INTERFACE_OBJECT);
583 0 : if (!vr_dpdk.vlan_vif) {
584 0 : RTE_LOG(ERR, VROUTER, "Error allocating interface object\n");
585 0 : return -ENOMEM;
586 : }
587 :
588 0 : vr_dpdk.vlan_vif->vif_stats = vr_zalloc(vr_num_cpus *
589 : sizeof(struct vr_interface_stats), VR_INTERFACE_STATS_OBJECT);
590 0 : if (!vr_dpdk.vlan_vif->vif_stats) {
591 0 : RTE_LOG(ERR, VROUTER, "Error allocating interface stats object\n");
592 0 : return -ENOMEM;
593 : }
594 :
595 0 : strncpy((char *)vr_dpdk.vlan_vif->vif_name, vr_dpdk.vlan_name,
596 : sizeof(vr_dpdk.vlan_vif->vif_name));
597 0 : vr_dpdk.vlan_vif->vif_type = VIF_TYPE_VLAN;
598 :
599 0 : ret = vr_dpdk_tapdev_init(vr_dpdk.vlan_vif);
600 :
601 0 : if (ret != 0) {
602 0 : RTE_LOG(ERR, VROUTER,
603 : "Error initializing device for VLAN forwarding interface: %s (%d)\n",
604 : rte_strerror(-ret), -ret);
605 0 : return ret;
606 : }
607 :
608 : /* Save device pointer needed to send packets to the interface. */
609 0 : vr_dpdk.vlan_dev = vr_dpdk.vlan_vif->vif_os;
610 :
611 : /*
612 : * Allocate a multi-producer single-consumer ring - a buffer for packets
613 : * waiting to be send to the forwarding interface.
614 : */
615 0 : vr_dpdk.vlan_ring = vr_dpdk_ring_allocate(VR_DPDK_FWD_LCORE_ID,
616 : vr_dpdk.vlan_name, vr_dpdk_tx_ring_sz, RING_F_SC_DEQ);
617 0 : if (!vr_dpdk.vlan_ring) {
618 0 : RTE_LOG(ERR, VROUTER, "Error allocating ring for VLAN forwarding interface\n");
619 0 : vr_dpdk.vlan_dev = NULL;
620 0 : vr_dpdk_tapdev_release(vr_dpdk.vlan_vif);
621 0 : return -1;
622 : }
623 :
624 0 : return 0;
625 : }
626 :
627 : /* customize the ethdev_conf for af_packet devices */
628 : static void
629 0 : vr_af_ethdev_conf_update(struct rte_eth_conf *dev_conf)
630 : {
631 0 : dev_conf->intr_conf.lsc = 0;
632 0 : }
633 :
634 : /*
635 : * Add af_packet virtual device to communicate with veth namespace devices.
636 : * The device is removed with dpdk_fabric_af_packet_if_del().
637 : */
638 : static int
639 0 : dpdk_af_packet_if_add(struct vr_interface *vif)
640 : {
641 : int ret;
642 : char params[VR_DPDK_STR_BUF_SZ];
643 : char name[VR_INTERFACE_NAME_LEN];
644 : struct vr_dpdk_ethdev *ethdev;
645 : uint8_t port_id;
646 : int frame_size;
647 : struct rte_eth_conf af_ethdev_conf;
648 :
649 0 : RTE_LOG(INFO, VROUTER,
650 : "Adding vif %u (gen. %u) af_packet device %s\n",
651 : vif->vif_idx, vif->vif_gen, vif->vif_name);
652 :
653 0 : ret = snprintf(name, sizeof(name), "eth_af_packet_%d", vif->vif_idx);
654 0 : if (ret >= sizeof(name)) {
655 0 : RTE_LOG(ERR, VROUTER,
656 : " error creating name for af_packet device %s\n", name);
657 0 : return ret;
658 : }
659 :
660 : /* Frame size should be a multiple of page size. */
661 0 : frame_size = (vr_packet_sz / getpagesize()) * getpagesize();
662 :
663 0 : ret = snprintf(params, sizeof(params),
664 : /* TODO: Optional af_packet mmap parameters
665 : * "qpairs=%d,framecnt=%d", 16, 512);
666 : */
667 : "iface=%s,framesz=%d,blocksz=%d",
668 0 : vif->vif_name, frame_size, frame_size);
669 0 : if (ret >= sizeof(params)) {
670 0 : RTE_LOG(ERR, VROUTER,
671 : " error creating config for af_packet device %s\n", name);
672 0 : return ret;
673 : }
674 :
675 0 : ret = rte_vdev_init(name, params);
676 0 : if (ret < 0) {
677 0 : RTE_LOG(ERR, VROUTER,
678 : " error initializing af_packet device %s\n", name);
679 0 : return ret;
680 : }
681 0 : port_id = (uint8_t)(rte_eth_dev_allocated(name) - rte_eth_devices);
682 :
683 0 : ethdev = &vr_dpdk.ethdevs[port_id];
684 0 : if (ethdev->ethdev_ptr != NULL) {
685 0 : RTE_LOG(ERR, VROUTER,
686 : " error adding af_packet device %s: eth device %"PRIu8" already added\n",
687 : name, port_id);
688 0 : return -EEXIST;
689 : }
690 0 : ethdev->ethdev_port_id = port_id;
691 0 : ethdev->ethdev_vif_idx = vif->vif_idx;
692 :
693 0 : af_ethdev_conf = ethdev_conf;
694 0 : vr_af_ethdev_conf_update(&af_ethdev_conf);
695 :
696 : /* init af_packet device */
697 0 : ret = vr_dpdk_ethdev_init(ethdev, &af_ethdev_conf, NULL, NULL);
698 0 : if (ret != 0)
699 0 : return ret;
700 :
701 0 : dpdk_vif_attach_ethdev(vif, ethdev);
702 :
703 0 : ret = vr_dpdk_interface_queue_setup(vif);
704 0 : if (ret < 0)
705 0 : return ret;
706 :
707 0 : ret = rte_eth_dev_start(port_id);
708 0 : if (ret < 0) {
709 0 : RTE_LOG(ERR, VROUTER,
710 : " error starting eth device %" PRIu8": %s (%d)\n",
711 : port_id, rte_strerror(-ret), -ret);
712 0 : return ret;
713 : }
714 :
715 : /* schedule RX/TX queues */
716 0 : return vr_dpdk_lcore_if_schedule(vif, vr_dpdk_lcore_least_used_get(),
717 0 : ethdev->ethdev_nb_rss_queues, &vr_dpdk_ethdev_rx_queue_init,
718 0 : ethdev->ethdev_nb_tx_queues, &vr_dpdk_ethdev_tx_queue_init);
719 : }
720 :
721 : /*
722 : * vr_ethdev_conf_update - adjust the config for fabric interfaces
723 : * depending on the NIC. Broadcom 25G interfaces and enic only support
724 : * 9022 byte jumbo frames. ixgbe VFs do not allow setting max packet
725 : * length higher than 1518 if the PF hasn't been configured similarly,
726 : * so the default is set to 1518 for VFs.
727 : */
728 : static void
729 0 : vr_ethdev_conf_update(struct rte_eth_conf *dev_conf)
730 : {
731 : int i;
732 : struct rte_eth_dev_info dev_info;
733 :
734 0 : if (vr_dpdk.vf_lcore_id) {
735 0 : if (dev_conf->rxmode.max_rx_pkt_len > RTE_ETHER_MAX_LEN) {
736 0 : dev_conf->rxmode.max_rx_pkt_len = RTE_ETHER_MAX_LEN;
737 : }
738 : }
739 :
740 0 : VR_DPDK_RTE_ETH_FOREACH_DEV(i)
741 : {
742 0 : rte_eth_dev_info_get(i, &dev_info);
743 :
744 0 : if (dev_info.driver_name) {
745 0 : if ((strncmp(dev_info.driver_name, "net_bnxt",
746 0 : strlen("net_bnxt") + 1) == 0) ||
747 0 : (strncmp(dev_info.driver_name, "net_enic",
748 : strlen("net_enic") + 1) == 0)) {
749 0 : if (dev_conf->rxmode.max_rx_pkt_len > VT_DPDK_MAX_RX_PKT_LEN_9022) {
750 0 : dev_conf->rxmode.max_rx_pkt_len = VT_DPDK_MAX_RX_PKT_LEN_9022;
751 : }
752 : }
753 : }
754 : }
755 :
756 0 : return;
757 : }
758 :
759 : /* Add fabric interface */
760 : static int
761 72 : dpdk_fabric_if_add(struct vr_interface *vif)
762 : {
763 : int ret;
764 : uint16_t port_id, ports_num;
765 : uint16_t mtu;
766 : struct rte_pci_addr pci_address;
767 : struct vr_dpdk_ethdev *ethdev;
768 : struct rte_ether_addr mac_addr;
769 : struct rte_eth_conf fabric_ethdev_conf;
770 :
771 72 : ports_num = rte_eth_dev_count_avail();
772 :
773 : /* When there is no PCI ports for DPDK, considered to be running on
774 : * vtest(Vrouter Unit Test simulation framework) and create virtual port instead
775 : * of physical port */
776 72 : if(ports_num == 0)
777 72 : return dpdk_mock_vif_add(vif);
778 :
779 0 : memset(&pci_address, 0, sizeof(pci_address));
780 0 : memset(&mac_addr, 0, sizeof(mac_addr));
781 0 : if (vif->vif_flags & VIF_FLAG_PMD) {
782 0 : if (vif->vif_os_idx >= ports_num) {
783 0 : RTE_LOG(ERR, VROUTER, "Error adding vif %u eth device %s: invalid PMD %u"
784 : " (must be less than %u)\n", vif->vif_idx, vif->vif_name,
785 : vif->vif_os_idx, ports_num);
786 0 : return -ENOENT;
787 : }
788 :
789 0 : port_id = dpdk_find_port_id_by_vif_name(vif);
790 0 : if (port_id == VR_DPDK_INVALID_PORT_ID) {
791 0 : RTE_LOG(ERR, VROUTER, "%s: Port Id is invalid\n", __func__);
792 0 : return -ENOENT;
793 : }
794 :
795 0 : rte_eth_macaddr_get(port_id, &mac_addr);
796 0 : RTE_LOG(INFO, VROUTER, "Adding vif %u (gen. %u) eth device %" PRIu8
797 : " (PMD) MAC " MAC_FORMAT " (vif MAC "MAC_FORMAT")\n",
798 : vif->vif_idx, vif->vif_gen, port_id,
799 : MAC_VALUE(mac_addr.addr_bytes), MAC_VALUE(vif->vif_mac));
800 : } else {
801 0 : port_id = dpdk_find_port_id_by_vif_name(vif);
802 0 : if (port_id == VR_DPDK_INVALID_PORT_ID) {
803 0 : RTE_LOG(ERR, VROUTER, "Error adding vif %u eth device %s:"
804 : " no port ID found for PCI " PCI_PRI_FMT "\n",
805 : vif->vif_idx, vif->vif_name,
806 : pci_address.domain, pci_address.bus,
807 : pci_address.devid, pci_address.function);
808 0 : return -ENOENT;
809 : }
810 :
811 0 : rte_eth_macaddr_get(port_id, &mac_addr);
812 0 : RTE_LOG(INFO, VROUTER, "Adding vif %u (gen. %u) eth device %" PRIu8
813 : " PCI " PCI_PRI_FMT " MAC " MAC_FORMAT " (vif MAC "MAC_FORMAT")\n",
814 : vif->vif_idx, vif->vif_gen, port_id,
815 : pci_address.domain, pci_address.bus,
816 : pci_address.devid, pci_address.function,
817 : MAC_VALUE(mac_addr.addr_bytes), MAC_VALUE(vif->vif_mac));
818 : }
819 :
820 0 : if (rte_eth_dev_get_mtu(port_id, &mtu) == 0 && mtu > 0) {
821 : /* Ethernet header size */
822 0 : mtu += sizeof(struct vr_eth);
823 0 : if (vr_dpdk.vlan_tag != VLAN_ID_INVALID) {
824 : /* 802.1q header size */
825 0 : mtu += sizeof(uint32_t);
826 : }
827 0 : vif->vif_mtu = mtu;
828 0 : if (vif->vif_bridge[0])
829 0 : vif->vif_bridge[0]->vif_mtu = mtu;
830 : }
831 :
832 0 : ethdev = &vr_dpdk.ethdevs[port_id];
833 0 : if (ethdev->ethdev_ptr != NULL) {
834 0 : RTE_LOG(ERR, VROUTER, " error adding eth dev %s: already added\n",
835 : vif->vif_name);
836 0 : return -EEXIST;
837 : }
838 0 : ethdev->ethdev_port_id = port_id;
839 0 : ethdev->ethdev_vif_idx = vif->vif_idx;
840 :
841 0 : fabric_ethdev_conf = ethdev_conf;
842 0 : vr_ethdev_conf_update(&fabric_ethdev_conf);
843 :
844 : /* init eth device */
845 0 : ret = vr_dpdk_ethdev_init(ethdev, &fabric_ethdev_conf, NULL, NULL);
846 0 : if (ret != 0)
847 0 : return ret;
848 :
849 0 : dpdk_vif_attach_ethdev(vif, ethdev);
850 :
851 0 : ret = vr_dpdk_interface_queue_setup(vif);
852 0 : if (ret < 0)
853 0 : return ret;
854 :
855 0 : ret = rte_eth_dev_start(port_id);
856 0 : if (ret < 0) {
857 0 : RTE_LOG(ERR, VROUTER, " error starting eth device %" PRIu8
858 : ": %s (%d)\n", port_id, rte_strerror(-ret), -ret);
859 0 : return ret;
860 : }
861 :
862 0 : ret = vr_dpdk_ethdev_rss_init(ethdev);
863 0 : if (ret < 0)
864 0 : return ret;
865 :
866 : /* we need to init Flow Director after the device has started */
867 : #if VR_DPDK_USE_HW_FILTERING
868 : /* init hardware filtering */
869 : ret = vr_dpdk_ethdev_filtering_init(vif, ethdev);
870 : if (ret < 0)
871 : return ret;
872 : #endif
873 :
874 : /* Set hardware VLAN stripping */
875 0 : dpdk_set_addr_vlan_filter_strip(port_id, vif);
876 :
877 : /* schedule RX/TX queues */
878 0 : return vr_dpdk_lcore_if_schedule(vif, vr_dpdk_lcore_least_used_get(),
879 0 : ethdev->ethdev_nb_rss_queues, &vr_dpdk_ethdev_rx_queue_init,
880 0 : ethdev->ethdev_nb_tx_queues, &vr_dpdk_ethdev_tx_queue_init);
881 : }
882 :
883 : /* Delete fabric or af_packet interface */
884 : static int
885 72 : dpdk_fabric_af_packet_if_del(struct vr_interface *vif)
886 : {
887 : uint8_t port_id;
888 : struct vr_dpdk_ethdev *ethdev;
889 : char name[VR_INTERFACE_NAME_LEN];
890 : int ret;
891 :
892 72 : ret = snprintf(name, sizeof(name), "eth_af_packet_%d", vif->vif_idx);
893 72 : if (ret >= sizeof(name)) {
894 0 : RTE_LOG(ERR, VROUTER,
895 : " error creating name for af_packet device %s\n", name);
896 0 : return ret;
897 : }
898 :
899 :
900 72 : RTE_LOG(INFO, VROUTER, "Deleting vif %u %s device\n", vif->vif_idx,
901 : vif_is_fabric(vif) ? "eth" : "af_packet");
902 :
903 : /*
904 : * If dpdk_fabric_if_add() failed before dpdk_vif_attach_ethdev,
905 : * then vif->vif_os will be NULL.
906 : */
907 72 : if (vif->vif_os == NULL) {
908 72 : RTE_LOG(ERR, VROUTER, " error deleting %s dev: already removed\n",
909 : vif_is_fabric(vif) ? "eth" : "af_packet");
910 72 : return -EEXIST;
911 : }
912 :
913 0 : ethdev = (struct vr_dpdk_ethdev *)(vif->vif_os);
914 0 : port_id = ethdev->ethdev_port_id;
915 :
916 : /* unschedule RX/TX queues */
917 0 : vr_dpdk_lcore_if_unschedule(vif);
918 :
919 0 : rte_eth_dev_stop(port_id);
920 :
921 : /* af_packet release */
922 0 : if (vif_is_namespace(vif)) {
923 : /**
924 : * af_packet does not implement rte_driver.uninit() that should
925 : * free memory and call rte_eth_dev_release_port(). If we ever wanted
926 : * to move to the pcap driver, we should call rte_eth_dev_close(),
927 : * then rte_eth_dev_detach(). _detach() will call .uninit(), that is
928 : * implemented in pcap. .uninit() will free memory and call
929 : * _release_port().
930 : */
931 0 : rte_eth_dev_close(port_id);
932 : }
933 :
934 0 : vr_dpdk_interface_queue_free(vif);
935 :
936 : /* release eth device */
937 0 : vr_dpdk_ethdev_release(ethdev);
938 0 : return rte_vdev_uninit(name);
939 : }
940 :
941 : /* Add vhost interface */
942 : static int
943 50 : dpdk_vhost_if_add(struct vr_interface *vif)
944 : {
945 : int ret;
946 : uint16_t nb_txqs, ports_num;
947 :
948 50 : ports_num = rte_eth_dev_count_avail();
949 :
950 : /* When there is no PCI ports for DPDK, considered to be running on
951 : * vtest(Vrouter Unit Test simulation framework) and create virtual port instead
952 : * of physical port */
953 50 : if(ports_num == 0)
954 50 : return dpdk_mock_vif_add(vif);
955 :
956 : /* If there is a tapdev VLAN device, assign vhost0 MAC address to it */
957 0 : if (vr_dpdk.vlan_dev) {
958 : struct ifreq ifr;
959 0 : struct vr_dpdk_tapdev *tapdev = vr_dpdk.vlan_dev;
960 0 : memset(&ifr, 0, sizeof(ifr));
961 0 : memcpy(ifr.ifr_name, vr_dpdk.vlan_name, sizeof(ifr.ifr_name) - 1);
962 0 : rte_memcpy(ifr.ifr_hwaddr.sa_data, vif->vif_mac, RTE_ETHER_ADDR_LEN);
963 0 : ifr.ifr_hwaddr.sa_family = ARPHRD_ETHER;
964 0 : if (ioctl(tapdev->tapdev_fd, SIOCSIFHWADDR, &ifr) < 0) {
965 0 : RTE_LOG(ERR, VROUTER, " error assigning MAC address to %s: %s(%d)\n",
966 : vr_dpdk.vlan_name, rte_strerror(errno), errno);
967 : } else {
968 0 : RTE_LOG(INFO, VROUTER, " Adding MAC " MAC_FORMAT " to %s\n",
969 : MAC_VALUE(vif->vif_mac), vr_dpdk.vlan_name);
970 : }
971 : }
972 :
973 0 : ret = vr_dpdk_interface_queue_setup(vif);
974 0 : if (ret < 0)
975 0 : return ret;
976 :
977 0 : ret = vr_dpdk_tapdev_init(vif);
978 0 : if (ret != 0)
979 0 : return ret;
980 :
981 : /* We use few single-producer rings, so we assign TX queue to each lcore */
982 0 : nb_txqs = (uint16_t)-1;
983 :
984 : /* Schedule the TAP interface with 1 RX queue and unlimited TX queues. */
985 0 : ret = vr_dpdk_lcore_if_schedule(vif, vr_dpdk_lcore_least_used_get(),
986 : 1, &vr_dpdk_tapdev_rx_queue_init,
987 : nb_txqs, &vr_dpdk_tapdev_tx_queue_init);
988 :
989 0 : return ret;
990 : }
991 :
992 : /* Delete vhost interface */
993 : static int
994 49 : dpdk_vhost_if_del(struct vr_interface *vif)
995 : {
996 49 : RTE_LOG(INFO, VROUTER, "Deleting vif %u device %s\n",
997 : vif->vif_idx, vif->vif_name);
998 :
999 49 : vr_dpdk_lcore_if_unschedule(vif);
1000 :
1001 49 : vr_dpdk_interface_queue_free(vif);
1002 :
1003 49 : return vr_dpdk_tapdev_release(vif);
1004 : }
1005 :
1006 : /* Start interface monitoring */
1007 : static void
1008 0 : dpdk_monitoring_start(struct vr_interface *monitored_vif,
1009 : struct vr_interface *monitoring_vif)
1010 : {
1011 : /* set monitoring redirection */
1012 0 : vr_dpdk.monitorings[monitored_vif->vif_idx] = monitoring_vif->vif_idx;
1013 :
1014 : /* set vif flag */
1015 : rte_wmb();
1016 0 : monitored_vif->vif_flags |= VIF_FLAG_MONITORED;
1017 0 : }
1018 :
1019 : /* Stop interface monitoring */
1020 : static void
1021 0 : dpdk_monitoring_stop(struct vr_interface *monitored_vif,
1022 : struct vr_interface *monitoring_vif)
1023 : {
1024 : /* check if the monitored vif was reused */
1025 0 : if (vr_dpdk.monitorings[monitored_vif->vif_idx] != monitoring_vif->vif_idx)
1026 0 : return;
1027 :
1028 : /* clear vif flag */
1029 0 : monitored_vif->vif_flags &= ~((unsigned int)VIF_FLAG_MONITORED);
1030 : rte_wmb();
1031 :
1032 : /* clear monitoring redirection */
1033 0 : vr_dpdk.monitorings[monitored_vif->vif_idx] = VR_MAX_INTERFACES;
1034 : }
1035 :
1036 : /* Add monitoring interface */
1037 : static int
1038 0 : dpdk_monitoring_if_add(struct vr_interface *vif)
1039 : {
1040 : int ret;
1041 0 : unsigned short monitored_vif_id = vif->vif_os_idx;
1042 : struct vr_interface *monitored_vif;
1043 0 : struct vrouter *router = vrouter_get(vif->vif_rid);
1044 : uint16_t nb_txqs;
1045 :
1046 0 : RTE_LOG(INFO, VROUTER, "Adding monitoring vif %u (gen. %u) device %s"
1047 : " to monitor vif %u\n",
1048 : vif->vif_idx, vif->vif_gen, vif->vif_name, monitored_vif_id);
1049 :
1050 : /* Check if vif exist.
1051 : * We don't need vif reference in order to monitor it.
1052 : * We use the VIF_FLAG_MONITORED to copy in/out packet to the
1053 : * monitoring interface. If the monitored vif get deleted, we simply
1054 : * get no more packets.
1055 : */
1056 0 : monitored_vif = __vrouter_get_interface(router, monitored_vif_id);
1057 0 : if (!monitored_vif) {
1058 0 : RTE_LOG(ERR, VROUTER, " error getting vif to monitor:"
1059 : " vif %u does not exist\n", monitored_vif_id);
1060 0 : return -EINVAL;
1061 : }
1062 :
1063 0 : ret = vr_dpdk_interface_queue_setup(vif);
1064 0 : if (ret)
1065 0 : return ret;
1066 :
1067 : /*
1068 : * TODO: we always use DPDK port 0 for monitoring KNI
1069 : * DPDK numerates all the detected Ethernet devices starting from 0.
1070 : * So we might only get into an issue if we have no eth devices at all
1071 : * or we have few eth ports and don't want to use the first one.
1072 : */
1073 :
1074 0 : ret = vr_dpdk_tapdev_init(vif);
1075 0 : if (ret != 0)
1076 0 : return ret;
1077 :
1078 : /* We use few single-producer rings, so we assign TX queue to each lcore */
1079 0 : nb_txqs = (uint16_t)-1;
1080 :
1081 : /* Schedule the TAP interface with 1 RX queue and unlimited TX queues. */
1082 : /* Write-only interface. */
1083 0 : ret = vr_dpdk_lcore_if_schedule(vif, vr_dpdk_lcore_least_used_get(),
1084 : 0, NULL,
1085 : nb_txqs, &vr_dpdk_tapdev_tx_queue_init);
1086 :
1087 0 : if (ret == 0) {
1088 : /* Start monitoring. */
1089 0 : dpdk_monitoring_start(monitored_vif, vif);
1090 : }
1091 :
1092 0 : return ret;
1093 : }
1094 :
1095 : /* Delete monitoring interface */
1096 : static int
1097 0 : dpdk_monitoring_if_del(struct vr_interface *vif)
1098 : {
1099 0 : unsigned short monitored_vif_id = vif->vif_os_idx;
1100 : struct vr_interface *monitored_vif;
1101 :
1102 0 : RTE_LOG(INFO, VROUTER, "Deleting monitoring vif %u device"
1103 : " to monitor vif %u\n",
1104 : vif->vif_idx, monitored_vif_id);
1105 :
1106 : /* check if vif exist */
1107 0 : monitored_vif = __vrouter_get_interface(vrouter_get(vif->vif_rid),
1108 : monitored_vif_id);
1109 0 : if (!monitored_vif) {
1110 0 : RTE_LOG(ERR, VROUTER, " error getting vif to monitor:"
1111 : " vif %u does not exist\n", monitored_vif_id);
1112 : } else {
1113 : /* stop monitoring */
1114 0 : dpdk_monitoring_stop(monitored_vif, vif);
1115 : }
1116 :
1117 0 : vr_dpdk_lcore_if_unschedule(vif);
1118 :
1119 0 : vr_dpdk_interface_queue_free(vif);
1120 :
1121 : /* Release TAP device. */
1122 0 : return vr_dpdk_tapdev_release(vif);
1123 : }
1124 :
1125 : /* Add agent interface */
1126 : static int
1127 42 : dpdk_agent_if_add(struct vr_interface *vif)
1128 : {
1129 : int ret;
1130 : uint16_t ports_num;
1131 :
1132 42 : ports_num = rte_eth_dev_count_avail();
1133 :
1134 42 : RTE_LOG(INFO, VROUTER, "Adding vif %u (gen. %u) packet device %s\n",
1135 : vif->vif_idx, vif->vif_gen, vif->vif_name);
1136 :
1137 : /* When there is no PCI ports for DPDK, considered to be running on
1138 : * vtest(Vrouter Unit Test simulation framework) and create virtual port instead
1139 : * of physical port */
1140 42 : if(ports_num == 0)
1141 42 : return dpdk_mock_vif_add(vif);
1142 :
1143 : /* check if packet device is already added */
1144 0 : if (vr_dpdk.packet_transport != NULL) {
1145 0 : RTE_LOG(ERR, VROUTER, " error adding packet device %s: already exist\n",
1146 : vif->vif_name);
1147 0 : return -EEXIST;
1148 : }
1149 :
1150 : /* init packet device */
1151 0 : ret = dpdk_packet_socket_init();
1152 0 : if (ret < 0) {
1153 0 : RTE_LOG(ERR, VROUTER, " error initializing packet socket: %s (%d)\n",
1154 : rte_strerror(errno), errno);
1155 0 : return ret;
1156 : }
1157 :
1158 0 : vr_usocket_attach_vif(vr_dpdk.packet_transport, vif);
1159 :
1160 : /* No need to schedule the pkt0 at the moment, since we RX from the
1161 : * socket and TX to the global packet_ring.
1162 : */
1163 0 : return 0;
1164 : }
1165 :
1166 : /* Delete agent interface */
1167 : static int
1168 42 : dpdk_agent_if_del(struct vr_interface *vif)
1169 : {
1170 42 : RTE_LOG(INFO, VROUTER, "Deleting vif %u packet device\n",
1171 : vif->vif_idx);
1172 :
1173 42 : dpdk_packet_socket_close();
1174 :
1175 42 : return 0;
1176 : }
1177 :
1178 : extern void vhost_remove_xconnect(void);
1179 :
1180 : /* vRouter callback */
1181 : static int
1182 332 : dpdk_if_add(struct vr_interface *vif)
1183 : {
1184 332 : if (vr_dpdk_is_stop_flag_set())
1185 0 : return -EINPROGRESS;
1186 :
1187 332 : int ret = vr_dpdk_representor_add(vif);
1188 332 : if (ret != VR_DPDK_REPRESENTOR_OP_RES_NOT_HANDLED) {
1189 0 : return ret == VR_DPDK_REPRESENTOR_OP_RES_HANDLED_OK ? 0 : -EFAULT;
1190 : }
1191 :
1192 332 : if (vif_is_fabric(vif)) {
1193 72 : return dpdk_fabric_if_add(vif);
1194 260 : } else if (vif_is_vm(vif)) {
1195 168 : return dpdk_virtual_if_add(vif);
1196 92 : } else if (vif_is_vlan(vif)) {
1197 0 : return dpdk_virtual_vlan_if_add(vif);
1198 92 : } else if (vif_is_namespace(vif)) {
1199 0 : return dpdk_af_packet_if_add(vif);
1200 92 : } else if (vif_is_vhost(vif)) {
1201 50 : return dpdk_vhost_if_add(vif);
1202 42 : } else if (vif_is_agent(vif)) {
1203 42 : return dpdk_agent_if_add(vif);
1204 0 : } else if (vif_is_monitoring(vif)) {
1205 0 : return dpdk_monitoring_if_add(vif);
1206 : }
1207 :
1208 0 : RTE_LOG(ERR, VROUTER,
1209 : "Error adding vif %d (%s): unsupported interface type %d transport %d\n",
1210 : vif->vif_idx, vif->vif_name, vif->vif_type, vif->vif_transport);
1211 :
1212 0 : return -EFAULT;
1213 : }
1214 :
1215 : static int
1216 332 : dpdk_if_del(struct vr_interface *vif)
1217 : {
1218 332 : if (vr_dpdk_is_stop_flag_set())
1219 3 : return -EINPROGRESS;
1220 :
1221 329 : int ret = vr_dpdk_representor_del(vif);
1222 329 : if (ret != VR_DPDK_REPRESENTOR_OP_RES_NOT_HANDLED) {
1223 0 : return ret == VR_DPDK_REPRESENTOR_OP_RES_HANDLED_OK ? 0 : -EFAULT;
1224 : }
1225 :
1226 329 : if (vif_is_fabric(vif) || vif_is_namespace(vif)) {
1227 72 : return dpdk_fabric_af_packet_if_del(vif);
1228 257 : } else if (vif_is_vm(vif)) {
1229 166 : return dpdk_virtual_if_del(vif);
1230 91 : } else if (vif_is_vlan(vif)) {
1231 0 : return dpdk_virtual_vlan_if_del(vif);
1232 91 : } else if (vif_is_vhost(vif)) {
1233 49 : return dpdk_vhost_if_del(vif);
1234 42 : } else if (vif_is_agent(vif)) {
1235 42 : return dpdk_agent_if_del(vif);
1236 0 : } else if (vif_is_monitoring(vif)) {
1237 0 : return dpdk_monitoring_if_del(vif);
1238 : }
1239 :
1240 0 : RTE_LOG(ERR, VROUTER,
1241 : "Error deleting vif %d: unsupported interface type %d transport %d\n",
1242 : vif->vif_idx, vif->vif_type, vif->vif_transport);
1243 :
1244 0 : return -EFAULT;
1245 : }
1246 :
1247 : /*
1248 : *This function is used to delete a tuntap vif corresponding
1249 : *to a physical interface in the case of l3mh
1250 : */
1251 : static int
1252 0 : dpdk_if_del_tun_tap(struct vr_interface *vif)
1253 : {
1254 0 : if (vif) {
1255 0 : if (vif->vif_type == VIF_TYPE_HOST)
1256 0 : vif_delete(vif);
1257 : }
1258 :
1259 0 : return 0;
1260 : }
1261 :
1262 : /*
1263 : * This function is used to create a tuntap vif corresponding
1264 : * to a physical interface in the case of l3mh
1265 : */
1266 : static int
1267 10 : dpdk_if_add_tun_tap(struct vr_interface *vif, vr_interface_req *vifr)
1268 : {
1269 10 : int ret = 0;
1270 10 : vr_interface_req req = *vifr;
1271 10 : struct vrouter *router = vrouter_get(0);
1272 : uint16_t ports_num;
1273 :
1274 10 : ports_num = rte_eth_dev_count_avail();
1275 :
1276 : /*
1277 : * When there is no PCI ports for DPDK, it considered to be running on
1278 : * vtest(Vrouter Unit Test simulation framework). Even in this case,
1279 : * the tun tap interfaces need to be created in case of l3mh.
1280 : */
1281 10 : if((vifr->vifr_idx < VR_TOTAL_INTERFACES) &&
1282 10 : ((vr_dpdk.tapdevs[0].tapdev_vhost_fd > 0) || (!ports_num))) {
1283 10 : req.vifr_idx = VR_TOTAL_INTERFACES + vif->vif_idx;
1284 10 : memcpy(req.vifr_mac, vif->vif_mac, sizeof(vif->vif_mac));
1285 10 : snprintf(req.vifr_name, VR_INTERFACE_NAME_LEN, "tap%d", vif->vif_idx);
1286 10 : RTE_LOG(INFO, VROUTER,
1287 : "%s : L3MH - Phy Intf: %s Corresponding tuntap intf %s \n",
1288 : __func__, vif->vif_name, req.vifr_name);
1289 10 : ret = vr_interface_add(&req, 0);
1290 10 : if(ret) {
1291 0 : RTE_LOG(ERR, VROUTER,
1292 : "%s: Tuntap interface creation failed for %s id %d\n",
1293 : __func__, req.vifr_name, req.vifr_idx);
1294 : } else {
1295 10 : struct vr_interface *tap_if = __vrouter_get_interface(router,
1296 10 : (VR_TOTAL_INTERFACES + vif->vif_idx));
1297 10 : tap_if->vif_bridge[0] = vif;
1298 10 : vif->vif_bridge[0] = tap_if;
1299 : }
1300 : }
1301 :
1302 10 : return ret;
1303 : }
1304 :
1305 : /* vRouter callback */
1306 : static int
1307 282 : dpdk_if_del_tap(struct vr_interface *vif)
1308 : {
1309 282 : return 0;
1310 : }
1311 :
1312 : /* vRouter callback */
1313 : static int
1314 264 : dpdk_if_add_tap(struct vr_interface *vif, vr_interface_req *vifr)
1315 : {
1316 264 : return 0;
1317 : }
1318 :
1319 : static int
1320 61 : dpdk_pkt_is_gso(struct rte_mbuf *m)
1321 : {
1322 : /*
1323 : * This is only for TCP4/TCP6. UDP frag offload goes through the
1324 : * regular dpdk_fragment_packet() path
1325 : */
1326 61 : return (m->ol_flags & (PKT_RX_GSO_TCP4| PKT_RX_GSO_TCP6));
1327 : }
1328 :
1329 : static inline void
1330 0 : dpdk_hw_checksum_at_offset(struct vr_packet *pkt, unsigned offset)
1331 : {
1332 0 : struct rte_mbuf *m = vr_dpdk_pkt_to_mbuf(pkt);
1333 0 : struct vr_ip *iph = NULL;
1334 0 : struct vr_ip6 *ip6h = NULL;
1335 0 : unsigned char iph_len = 0, iph_proto = 0;
1336 : struct vr_tcp *tcph;
1337 : struct vr_udp *udph;
1338 :
1339 0 : RTE_VERIFY(0 < offset);
1340 :
1341 0 : if (pkt->vp_type == VP_TYPE_IP || pkt->vp_type == VP_TYPE_IPOIP) {
1342 0 : iph = (struct vr_ip *)pkt_data_at_offset(pkt, offset);
1343 0 : iph_len = iph->ip_hl * 4;
1344 0 : iph_proto = iph->ip_proto;
1345 0 : m->ol_flags |= PKT_TX_IP_CKSUM | PKT_TX_IPV4;
1346 0 : iph->ip_csum = 0;
1347 0 : } else if (pkt->vp_type == VP_TYPE_IP6 || pkt->vp_type == VP_TYPE_IP6OIP) {
1348 0 : ip6h = (struct vr_ip6 *)pkt_data_at_offset(pkt, offset);
1349 0 : iph_len = sizeof(struct vr_ip6);
1350 0 : iph_proto = ip6h->ip6_nxt;
1351 0 : m->ol_flags |= PKT_TX_IPV6;
1352 : } else {
1353 : /* Nothing to do if the packet is neither IPv4 nor IPv6. */
1354 0 : return;
1355 : }
1356 :
1357 : /* Note: Intel NICs need the checksum set to zero
1358 : * and proper l2/l3 lens to be set.
1359 : */
1360 0 : m->l3_len = iph_len;
1361 0 : m->l2_len = offset - rte_pktmbuf_headroom(m);
1362 :
1363 : /* calculate TCP/UDP checksum */
1364 0 : if (likely(iph_proto == VR_IP_PROTO_UDP)) {
1365 0 : m->ol_flags |= PKT_TX_UDP_CKSUM;
1366 0 : udph = (struct vr_udp *)pkt_data_at_offset(pkt, offset + iph_len);
1367 0 : udph->udp_csum = 0;
1368 0 : if (iph)
1369 0 : udph->udp_csum = rte_ipv4_phdr_cksum((struct rte_ipv4_hdr *)iph, m->ol_flags);
1370 0 : else if (ip6h)
1371 0 : udph->udp_csum = rte_ipv6_phdr_cksum((struct rte_ipv6_hdr *)ip6h, m->ol_flags);
1372 0 : } else if (likely(iph_proto == VR_IP_PROTO_TCP)) {
1373 0 : m->ol_flags |= PKT_TX_TCP_CKSUM;
1374 0 : tcph = (struct vr_tcp *)pkt_data_at_offset(pkt, offset + iph_len);
1375 0 : tcph->tcp_csum = 0;
1376 0 : if (iph)
1377 0 : tcph->tcp_csum = rte_ipv4_phdr_cksum((struct rte_ipv4_hdr *)iph, m->ol_flags);
1378 0 : else if (ip6h)
1379 0 : tcph->tcp_csum = rte_ipv6_phdr_cksum((struct rte_ipv6_hdr *)ip6h, m->ol_flags);
1380 : }
1381 : }
1382 :
1383 : static inline void
1384 60 : dpdk_ipv4_sw_iphdr_checksum_at_offset(struct vr_packet *pkt, unsigned offset)
1385 : {
1386 : struct vr_ip *iph;
1387 :
1388 60 : RTE_VERIFY(0 < offset);
1389 :
1390 60 : iph = (struct vr_ip *)pkt_data_at_offset(pkt, offset);
1391 60 : iph->ip_csum = vr_ip_csum(iph);
1392 60 : }
1393 :
1394 : static inline void
1395 123 : dpdk_sw_checksum_at_offset(struct vr_packet *pkt, unsigned offset)
1396 : {
1397 123 : struct vr_ip *iph = NULL;
1398 123 : struct vr_ip6 *ip6h = NULL;
1399 123 : unsigned char iph_len = 0, iph_proto = 0;
1400 : struct vr_udp *udph;
1401 : struct vr_tcp *tcph;
1402 123 : struct rte_mbuf *m = vr_dpdk_pkt_to_mbuf(pkt);
1403 :
1404 123 : RTE_VERIFY(0 < offset);
1405 :
1406 123 : if (pkt->vp_type == VP_TYPE_IP || pkt->vp_type == VP_TYPE_IPOIP) {
1407 105 : iph = (struct vr_ip *)pkt_data_at_offset(pkt, offset);
1408 105 : iph->ip_csum = vr_ip_csum(iph);
1409 105 : iph_len = iph->ip_hl * 4;
1410 105 : iph_proto = iph->ip_proto;
1411 18 : } else if (pkt->vp_type == VP_TYPE_IP6 || pkt->vp_type == VP_TYPE_IP6OIP) {
1412 18 : ip6h = (struct vr_ip6 *)pkt_data_at_offset(pkt, offset);
1413 18 : iph_len = sizeof(struct vr_ip6);
1414 18 : iph_proto = ip6h->ip6_nxt;
1415 : } else {
1416 : /* Nothing to do if the packet is neither IPv4 nor IPv6. */
1417 0 : return;
1418 : }
1419 :
1420 123 : if (iph_proto == VR_IP_PROTO_UDP) {
1421 24 : udph = (struct vr_udp *)pkt_data_at_offset(pkt, offset + iph_len);
1422 24 : udph->udp_csum = 0;
1423 24 : if (iph)
1424 17 : udph->udp_csum = dpdk_ipv4_udptcp_cksum(m, (struct rte_ipv4_hdr *)iph, (uint8_t*)udph);
1425 7 : else if (ip6h)
1426 7 : udph->udp_csum = dpdk_ipv6_udptcp_cksum(m, (struct rte_ipv6_hdr *)ip6h, (uint8_t*)udph);
1427 99 : } else if (iph_proto == VR_IP_PROTO_TCP) {
1428 20 : tcph = (struct vr_tcp *)pkt_data_at_offset(pkt, offset + iph_len);
1429 20 : tcph->tcp_csum = 0;
1430 20 : if (iph)
1431 19 : tcph->tcp_csum = dpdk_ipv4_udptcp_cksum(m, (struct rte_ipv4_hdr *)iph, (uint8_t*)tcph);
1432 1 : else if (ip6h)
1433 1 : tcph->tcp_csum = dpdk_ipv6_udptcp_cksum(m, (struct rte_ipv6_hdr *)ip6h, (uint8_t*)tcph);
1434 : }
1435 : }
1436 :
1437 : static inline void
1438 0 : dpdk_ipv4_outer_tunnel_hw_checksum(struct vr_packet *pkt)
1439 : {
1440 0 : struct rte_mbuf *m = vr_dpdk_pkt_to_mbuf(pkt);
1441 0 : unsigned offset = pkt->vp_data + dpdk_get_ether_header_len(
1442 0 : pkt_data_at_offset(pkt, pkt->vp_data));
1443 0 : struct vr_ip *iph = (struct vr_ip *)pkt_data_at_offset(pkt, offset);
1444 0 : unsigned iph_len = iph->ip_hl * 4;
1445 :
1446 0 : m->ol_flags |= PKT_TX_IP_CKSUM | PKT_TX_IPV4;
1447 0 : iph->ip_csum = 0;
1448 0 : m->l3_len = iph_len;
1449 0 : m->l2_len = offset - rte_pktmbuf_headroom(m);
1450 0 : }
1451 :
1452 : static inline void
1453 0 : dpdk_ipv4_outer_tunnel_sw_checksum(struct vr_packet *pkt)
1454 : {
1455 0 : unsigned offset = pkt->vp_data + dpdk_get_ether_header_len(
1456 0 : pkt_data_at_offset(pkt, pkt->vp_data));
1457 0 : struct vr_ip *iph = (struct vr_ip *)pkt_data_at_offset(pkt, offset);
1458 :
1459 0 : iph->ip_csum = vr_ip_csum(iph);
1460 0 : }
1461 :
1462 : static inline void
1463 0 : dpdk_hw_checksum(struct vr_packet *pkt)
1464 : {
1465 : /* if a tunnel */
1466 0 : if (vr_pkt_type_is_overlay(pkt->vp_type)) {
1467 : /* calculate outer checksum in soft */
1468 0 : dpdk_ipv4_sw_iphdr_checksum_at_offset(pkt,
1469 0 : pkt->vp_data + dpdk_get_ether_header_len(
1470 0 : pkt_data_at_offset(pkt, pkt->vp_data)));
1471 : /* calculate inner checksum in hardware */
1472 0 : dpdk_hw_checksum_at_offset(pkt, pkt_get_inner_network_header_off(pkt));
1473 0 : } else if (VP_TYPE_IP == pkt->vp_type || VP_TYPE_IP6 == pkt->vp_type) {
1474 : /* normal IPv4 or IPv6 packet */
1475 0 : dpdk_hw_checksum_at_offset(pkt, pkt->vp_data +
1476 0 : dpdk_get_ether_header_len(
1477 0 : pkt_data_at_offset(pkt, pkt->vp_data)));
1478 : }
1479 0 : }
1480 :
1481 : static inline void
1482 126 : dpdk_sw_checksum(struct vr_packet *pkt, bool will_fragment)
1483 : {
1484 : /* if a tunnel */
1485 126 : if (vr_pkt_type_is_overlay(pkt->vp_type)) {
1486 : /* calculate outer checksum */
1487 61 : if (!will_fragment)
1488 60 : dpdk_ipv4_sw_iphdr_checksum_at_offset(pkt,
1489 60 : pkt->vp_data + dpdk_get_ether_header_len(
1490 60 : pkt_data_at_offset(pkt, pkt->vp_data)));
1491 : /* calculate inner checksum */
1492 61 : dpdk_sw_checksum_at_offset(pkt, pkt_get_inner_network_header_off(pkt));
1493 65 : } else if (VP_TYPE_IP == pkt->vp_type || VP_TYPE_IP6 == pkt->vp_type) {
1494 : /* normal IPv4 or IPv6 packet */
1495 122 : dpdk_sw_checksum_at_offset(pkt, pkt->vp_data +
1496 61 : dpdk_get_ether_header_len(
1497 61 : pkt_data_at_offset(pkt, pkt->vp_data)));
1498 : }
1499 126 : }
1500 :
1501 : /**
1502 : * Fragment the input packet
1503 : *
1504 : * Please take note that the caller is responsible for freeing the input
1505 : * packet. All output fragments are hold in mbuf chains. Since we do not
1506 : * support the mbuf chains at the moment, there is no vr_packet structure
1507 : * attached to the mbufs and none of the functions using the struct can not be
1508 : * used.
1509 : *
1510 : * @param pkt The packet to be fragmented
1511 : * @param mbuf_in An mbuf of the inpun packet
1512 : * @param mbuf_out An array of mbuf pointers to hold output packets' mbufs
1513 : * @param out_num Size of the mbuf_out array
1514 : * @param mtu_size MTU size
1515 : * @param do_outer_ip_csum Whether calculate the outer IP checksum (in
1516 : * software)
1517 : * @param lcore_id An ID of the lcore executing this function
1518 : *
1519 : * @return Number of output fragments (packets)
1520 : */
1521 : static int
1522 1 : dpdk_fragment_packet(struct vr_packet *pkt, struct rte_mbuf *mbuf_in,
1523 : struct rte_mbuf **mbuf_out, const unsigned short out_num,
1524 : const unsigned short mtu_size, bool do_outer_ip_csum,
1525 : const unsigned lcore_id)
1526 : {
1527 : int number_of_packets;
1528 : uint16_t outer_header_len;
1529 : struct rte_mempool *pool_direct, *pool_indirect;
1530 : struct rte_mbuf *m;
1531 : int i;
1532 : unsigned char *original_header_ptr;
1533 : uint16_t max_frag_size;
1534 :
1535 1 : outer_header_len = pkt_get_inner_network_header_off(pkt) -
1536 1 : pkt_head_space(pkt);
1537 1 : original_header_ptr = pkt_data(pkt);
1538 :
1539 : /* Get into the inner IP header */
1540 1 : rte_pktmbuf_adj(mbuf_in, outer_header_len);
1541 :
1542 : /* Fragment the packet */
1543 1 : pool_direct = vr_dpdk.frag_direct_mempool;
1544 1 : pool_indirect = vr_dpdk.frag_indirect_mempool;
1545 :
1546 : /* Fragment with the maximum size of (MTU - outer_header_length) to leave a
1547 : * space for the header prepended later. In addition DPDK requires that the
1548 : * (max frag size - IP header) length is a multiple of 8, therefore the
1549 : * calculations below. */
1550 1 : max_frag_size = mtu_size - outer_header_len - sizeof(struct vr_ip);
1551 1 : max_frag_size &= ~7U;
1552 1 : max_frag_size += sizeof(struct vr_ip);
1553 :
1554 1 : number_of_packets = rte_ipv4_fragment_packet(mbuf_in, mbuf_out, out_num,
1555 : max_frag_size, pool_direct, pool_indirect);
1556 1 : if (number_of_packets < 0)
1557 0 : return number_of_packets;
1558 :
1559 : /* Adjust outer and inner IP headers for each fragmented packets */
1560 6 : for (i = 0; i < number_of_packets; ++i) {
1561 5 : m = mbuf_out[i];
1562 :
1563 : /* Inner header operations */
1564 5 : struct vr_ip *inner_ip = rte_pktmbuf_mtod(m, struct vr_ip *);
1565 5 : inner_ip->ip_csum = 0;
1566 5 : inner_ip->ip_csum = vr_ip_csum(inner_ip);
1567 :
1568 : /* Outer header operations */
1569 5 : char *outer_header_ptr = rte_pktmbuf_prepend(m, outer_header_len);
1570 5 : rte_memcpy(outer_header_ptr, original_header_ptr, outer_header_len);
1571 :
1572 5 : uint16_t eth_hlen = dpdk_get_ether_header_len(outer_header_ptr);
1573 5 : struct vr_ip *outer_ip = (struct vr_ip *)(outer_header_ptr + eth_hlen);
1574 5 : outer_ip->ip_len = rte_cpu_to_be_16(rte_pktmbuf_pkt_len(m) - eth_hlen);
1575 5 : m->l2_len = mbuf_in->l2_len;
1576 5 : m->l3_len = mbuf_in->l3_len;
1577 5 : m->ol_flags = mbuf_in->ol_flags;
1578 5 : m->vlan_tci = mbuf_in->vlan_tci;
1579 :
1580 : /* Copy inner IP id to outer. Currently, the Agent diagnostics depends
1581 : * on that. */
1582 5 : outer_ip->ip_id = inner_ip->ip_id;
1583 :
1584 : /* Adjust UDP length to match IP frament size */
1585 5 : if (outer_ip->ip_proto == VR_IP_PROTO_UDP) {
1586 5 : unsigned header_len = outer_ip->ip_hl * 4;
1587 5 : struct vr_udp *udp = (struct vr_udp *)((char *)outer_ip +
1588 : header_len);
1589 5 : udp->udp_length = rte_cpu_to_be_16(
1590 : rte_be_to_cpu_16(outer_ip->ip_len) - header_len);
1591 : }
1592 :
1593 : /* If it is necessary to calculate (in software) IP header checksum.
1594 : * TODO: This would not be needed if:
1595 : * 1. We would support mbuf chains. The functions that calculate the
1596 : * checksums, which uses vr_pkt struct could be used after fragmentation
1597 : * 2. We would rewrite the checksumming functions to use mbufs and not
1598 : * the vr_pkt struct, and use them after fragmentation. */
1599 5 : if (do_outer_ip_csum) {
1600 5 : outer_ip->ip_csum = vr_ip_csum(outer_ip);
1601 5 : m->ol_flags &= ~PKT_TX_IP_CKSUM;
1602 : }
1603 : }
1604 :
1605 1 : return number_of_packets;
1606 : }
1607 :
1608 : /* TX packet callback */
1609 : static int
1610 126 : dpdk_if_tx(struct vr_interface *vif, struct vr_packet *pkt)
1611 : {
1612 : uint8_t queue_index;
1613 : int ret, i;
1614 126 : unsigned int vif_idx = vif->vif_idx, dpdk_queue_index;
1615 126 : const unsigned int lcore_id = rte_lcore_id();
1616 :
1617 126 : struct vr_dpdk_lcore * const lcore = vr_dpdk.lcores[lcore_id];
1618 126 : struct rte_mbuf *m = vr_dpdk_pkt_to_mbuf(pkt);
1619 : struct vr_dpdk_queue *tx_queue;
1620 : struct vr_dpdk_queue *monitoring_tx_queue;
1621 : struct rte_mbuf *p_copy;
1622 : struct vr_interface_stats *stats;
1623 : struct rte_mbuf *mbufs_frags_out[VR_DPDK_FRAG_MAX_IP_FRAGS];
1624 : struct rte_mbuf *mbufs_segs_out[VR_DPDK_FRAG_MAX_IP_SEGS];
1625 126 : int num_of_frags = 1, num_of_segs = 1;
1626 : bool will_fragment, will_segment;
1627 :
1628 : RTE_LOG_DP(DEBUG, VROUTER,"%s: TX packet to interface %s\n", __func__,
1629 : vif->vif_name);
1630 :
1631 126 : if (pkt->vp_queue != VP_QUEUE_INVALID) {
1632 0 : queue_index = pkt->vp_queue;
1633 : } else {
1634 126 : if (lcore->lcore_hw_queue[vif_idx] >= 0) {
1635 126 : queue_index = lcore->lcore_hw_queue[vif_idx];
1636 : } else {
1637 0 : queue_index = 0;
1638 : }
1639 : }
1640 :
1641 126 : if (lcore->lcore_hw_queue_to_dpdk_index[vif->vif_idx]) {
1642 0 : dpdk_queue_index =
1643 0 : lcore->lcore_hw_queue_to_dpdk_index[vif->vif_idx][queue_index];
1644 : } else {
1645 126 : dpdk_queue_index = 0;
1646 : }
1647 :
1648 126 : tx_queue = &lcore->lcore_tx_queues[vif_idx][dpdk_queue_index];
1649 126 : stats = vif_get_stats(vif, lcore_id);
1650 :
1651 : /* reset mbuf data pointer and length */
1652 126 : m->data_off = pkt_head_space(pkt);
1653 126 : m->pkt_len = pkt_len(pkt);
1654 126 : m->data_len = pkt_head_len(pkt);
1655 :
1656 126 : if (unlikely(vif->vif_type == VIF_TYPE_AGENT)) {
1657 0 : ret = rte_ring_mp_enqueue(vr_dpdk.packet_ring, m);
1658 0 : if (likely(ret == 0)) {
1659 0 : stats->vis_queue_opackets++;
1660 : } else {
1661 : /* TODO: a separate counter for this drop */
1662 0 : vr_dpdk_pfree(m, pkt->vp_if, VP_DROP_INTERFACE_DROP);
1663 0 : stats->vis_queue_oerrors++;
1664 : /* return 0 so we do not increment vif error counter */
1665 0 : return 0;
1666 : }
1667 : #ifdef VR_DPDK_TX_PKT_DUMP
1668 : #ifdef VR_DPDK_PKT_DUMP_VIF_FILTER
1669 : if (VR_DPDK_PKT_DUMP_VIF_FILTER(vif))
1670 : #endif
1671 : rte_pktmbuf_dump(stdout, m, 0x60);
1672 : #endif
1673 0 : vr_dpdk_packet_wakeup(vif);
1674 0 : return 0;
1675 : }
1676 :
1677 126 : if (tx_queue == NULL) {
1678 0 : vr_dpdk_pfree(m, pkt->vp_if, VP_DROP_INTERFACE_DROP);
1679 0 : return 0;
1680 : }
1681 :
1682 : /* Set a flag indicating that the packet being processed is going to be
1683 : * fragmented as after prepending outer header it exceeds the MTU size of
1684 : * an interface. */
1685 187 : will_fragment = (vr_pkt_type_is_overlay(pkt->vp_type) &&
1686 61 : vif->vif_mtu < rte_pktmbuf_pkt_len(m));
1687 :
1688 : /* Segmentation is applicable if packet has GSO enabled and the
1689 : * packet length is greater than the tso segment size as given by the VM
1690 : */
1691 126 : will_segment = vr_pkt_type_is_overlay(pkt->vp_type) && dpdk_pkt_is_gso(m) &&
1692 0 : (rte_pktmbuf_pkt_len(m) > m->tso_segsz);
1693 :
1694 : /*
1695 : * With DPDK pktmbufs we don't know if the checksum is incomplete,
1696 : * i.e. there is no direct equivalent of skb->ip_summed field.
1697 : *
1698 : * So we just rely on VP_FLAG_CSUM_PARTIAL flag here, assuming
1699 : * the flag is set when we need to calculate inner or outer packet
1700 : * checksum.
1701 : *
1702 : * This is not elegant and need to be addressed.
1703 : * See dpdk/app/test-pmd/csumonly.c for more checksum examples
1704 : */
1705 126 : if (!will_segment) {
1706 126 : if (unlikely(pkt->vp_flags & VP_FLAG_CSUM_PARTIAL)) {
1707 : /* if NIC supports checksum offload */
1708 126 : if (likely((vif->vif_flags & VIF_FLAG_TX_CSUM_OFFLOAD) &&
1709 : !will_fragment))
1710 : /* Can not do hardware checksumming for fragmented packets */
1711 0 : dpdk_hw_checksum(pkt);
1712 : else {
1713 126 : dpdk_sw_checksum(pkt, will_fragment);
1714 :
1715 : /* We could not calculate the inner checkums in hardware, but we
1716 : * still can do outer header in hardware. */
1717 126 : if (unlikely(will_fragment &&
1718 : (vif->vif_flags & VIF_FLAG_TX_CSUM_OFFLOAD)))
1719 0 : dpdk_ipv4_outer_tunnel_hw_checksum(pkt);
1720 : }
1721 :
1722 0 : } else if (likely(vr_pkt_type_is_overlay(pkt->vp_type))) {
1723 : /* If NIC supports checksum offload.
1724 : * Inner checksum is already done. Compute outer IPv4 checksum,
1725 : * set UDP length, and zero UDP checksum.
1726 : */
1727 0 : if (likely(vif->vif_flags & VIF_FLAG_TX_CSUM_OFFLOAD)) {
1728 0 : dpdk_ipv4_outer_tunnel_hw_checksum(pkt);
1729 :
1730 0 : } else if (likely(!will_fragment)) {
1731 : /* if wont fragment it later */
1732 0 : dpdk_ipv4_outer_tunnel_sw_checksum(pkt);
1733 : }
1734 : }
1735 : }
1736 :
1737 : /* Inject ethertype and VLAN tag.
1738 : *
1739 : * Tag only packets that are going to be send to the physical interface,
1740 : * to allow data transfer between compute nodes in the specified VLAN.
1741 : *
1742 : * VLAN tag is adjustable by user with a command line --vlan_tci parameter:
1743 : * see dpdk_vrouter.c. If vRouter is not supposed to work in VLAN
1744 : * (parameter was not specified), packets should not be tagged.
1745 : *
1746 : * --vtest_vlan parameter changes behaviour - vRouter inject packets for
1747 : * non fabric interfaces too (Emulates physical interface for some vlan test cases).
1748 : *
1749 : */
1750 126 : if (unlikely(vr_dpdk.vlan_tag != VLAN_ID_INVALID && vif_is_fabric(vif)) ||
1751 126 : vr_dpdk.vtest_vlan) {
1752 : /* set 3 PCP bits and 12 VLAN ID bits */
1753 0 : m->vlan_tci = vr_dpdk.vlan_tag;
1754 0 : if (pkt->vp_priority != VP_PRIORITY_INVALID)
1755 0 : m->vlan_tci |= pkt->vp_priority << VR_VLAN_PRIORITY_SHIFT;
1756 :
1757 0 : if (unlikely((vif->vif_flags & VIF_FLAG_VLAN_OFFLOAD) == 0)) {
1758 : /* Software VLAN TCI insert. */
1759 0 : if (unlikely(pkt_push(pkt, sizeof(struct rte_vlan_hdr)) == NULL)) {
1760 : RTE_LOG_DP(DEBUG, VROUTER,"%s: Error inserting VLAN tag\n", __func__);
1761 0 : vr_dpdk_pfree(m, pkt->vp_if, VP_DROP_INTERFACE_DROP);
1762 0 : return -1;
1763 : }
1764 0 : m->l2_len += sizeof(struct rte_vlan_hdr);
1765 0 : if (unlikely(rte_vlan_insert(&m))) {
1766 : RTE_LOG_DP(DEBUG, VROUTER,"%s: Error inserting VLAN tag\n", __func__);
1767 0 : vr_dpdk_pfree(m, pkt->vp_if, VP_DROP_INTERFACE_DROP);
1768 0 : return -1;
1769 : }
1770 : } else {
1771 : /* Hardware VLAN TCI insert. */
1772 0 : m->ol_flags |= PKT_TX_VLAN_PKT;
1773 : }
1774 : }
1775 :
1776 : #ifdef VR_DPDK_TX_PKT_DUMP
1777 : #ifdef VR_DPDK_PKT_DUMP_VIF_FILTER
1778 : if (VR_DPDK_PKT_DUMP_VIF_FILTER(vif))
1779 : #endif
1780 : rte_pktmbuf_dump(stdout, m, 0x60);
1781 : #endif
1782 :
1783 126 : if (unlikely(will_segment)) {
1784 0 : num_of_segs = dpdk_segment_packet(pkt, m, mbufs_segs_out,
1785 0 : VR_DPDK_FRAG_MAX_IP_SEGS, m->tso_segsz,
1786 0 : (vif->vif_flags & VIF_FLAG_TX_CSUM_OFFLOAD));
1787 0 : if (num_of_segs < 0) {
1788 : RTE_LOG_DP(DEBUG, VROUTER, "%s: error %d during GSO of an "
1789 : "IP packet for interface %s on lcore %u\n", __func__,
1790 : num_of_segs, vif->vif_name, lcore_id);
1791 0 : vr_dpdk_pfree(m, pkt->vp_if, VP_DROP_INTERFACE_DROP);
1792 0 : return -1;
1793 : }
1794 126 : } else if (unlikely(will_fragment)) {
1795 1 : num_of_frags = dpdk_fragment_packet(pkt, m, mbufs_frags_out,
1796 1 : VR_DPDK_FRAG_MAX_IP_FRAGS, vif->vif_mtu,
1797 1 : !(vif->vif_flags & VIF_FLAG_TX_CSUM_OFFLOAD), lcore_id);
1798 1 : if (num_of_frags < 0) {
1799 : RTE_LOG_DP(DEBUG, VROUTER, "%s: error %d during fragmentation of an "
1800 : "IP packet for interface %s on lcore %u\n", __func__,
1801 : num_of_frags, vif->vif_name, lcore_id);
1802 0 : vr_dpdk_pfree(m, pkt->vp_if, VP_DROP_INTERFACE_DROP);
1803 0 : return -1;
1804 : }
1805 : }
1806 :
1807 126 : if (unlikely(vif->vif_flags & VIF_FLAG_MONITORED)) {
1808 0 : monitoring_tx_queue = &lcore->lcore_tx_queues[vr_dpdk.monitorings[vif_idx]][0];
1809 0 : if (likely(monitoring_tx_queue && monitoring_tx_queue->txq_ops.f_tx)) {
1810 0 : if (num_of_frags > 1) {
1811 : int i;
1812 0 : for (i=0; i < num_of_frags; i++) {
1813 0 : p_copy = vr_dpdk_pktmbuf_copy_mon(mbufs_frags_out[i], vr_dpdk.rss_mempool);
1814 0 : if (likely(p_copy != NULL)) {
1815 0 : monitoring_tx_queue->txq_ops.f_tx(monitoring_tx_queue->q_queue_h,
1816 : p_copy);
1817 : }
1818 : }
1819 0 : } else if (num_of_segs > 1) {
1820 : int i;
1821 0 : for (i=0; i < num_of_segs; i++) {
1822 0 : p_copy = vr_dpdk_pktmbuf_copy_mon(mbufs_segs_out[i], vr_dpdk.rss_mempool);
1823 0 : if (likely(p_copy != NULL)) {
1824 0 : monitoring_tx_queue->txq_ops.f_tx(monitoring_tx_queue->q_queue_h,
1825 : p_copy);
1826 : }
1827 : }
1828 : } else {
1829 0 : p_copy = vr_dpdk_pktmbuf_copy_mon(m, vr_dpdk.rss_mempool);
1830 0 : if (likely(p_copy != NULL)) {
1831 0 : monitoring_tx_queue->txq_ops.f_tx(monitoring_tx_queue->q_queue_h,
1832 : p_copy);
1833 : }
1834 : }
1835 : }
1836 : }
1837 :
1838 : /* It is not safe to access the vr_packet structure of the original packet
1839 : * after this point. It can be used only by drop function. The fragments
1840 : * have no vr_packet structure attached at all so it can not be used (see
1841 : * description for the dpdk_fragment_packet() function.
1842 : */
1843 126 : if (unlikely(num_of_frags > 1)) {
1844 1 : unsigned mask = (1 << num_of_frags) - 1;
1845 :
1846 1 : if (likely(tx_queue->txq_ops.f_tx_bulk != NULL)) {
1847 0 : tx_queue->txq_ops.f_tx_bulk(tx_queue->q_queue_h, mbufs_frags_out, mask);
1848 0 : if (unlikely(lcore_id < VR_DPDK_FWD_LCORE_ID))
1849 0 : tx_queue->txq_ops.f_flush(tx_queue->q_queue_h);
1850 :
1851 : /* Free the mbuf of the original packet (the one that has been
1852 : * fragmented) */
1853 0 : rte_pktmbuf_free(m);
1854 : } else {
1855 : RTE_LOG_DP(DEBUG, VROUTER,"%s: error TXing to interface %s: no queue "
1856 : "for lcore %u\n", __func__, vif->vif_name, lcore_id);
1857 : /* Can not do vif_drop_pkt() on fragments as mbufs after IP
1858 : * fragmentation does not have pkt structure. It is because we do
1859 : * not support chained mbufs that are results of fragmentation. */
1860 6 : for (i = 0; i < num_of_frags; ++i)
1861 5 : rte_pktmbuf_free(mbufs_frags_out[i]);
1862 1 : return -1;
1863 : }
1864 125 : } else if (unlikely(num_of_segs > 1)) {
1865 0 : if (likely(tx_queue->txq_ops.f_tx_bulk != NULL)) {
1866 0 : uint64_t segs_sent = 0;
1867 : uint64_t segs_to_send;
1868 : uint64_t mask;
1869 : /* Pkts mask has a limit for sending 64 packets.
1870 : * and the burst size is VR_DPDK_TX_BURST_SZ.
1871 : * Send only max of VR_DPDK_TX_BURST_SZ.
1872 : */
1873 0 : while (segs_sent < num_of_segs) {
1874 0 : if ((num_of_segs - segs_sent) > VR_DPDK_TX_BURST_SZ) {
1875 0 : segs_to_send = VR_DPDK_TX_BURST_SZ;
1876 : } else {
1877 0 : segs_to_send = num_of_segs - segs_sent;
1878 : }
1879 0 : mask = (uint64_t)((uint64_t) (1ULL << (uint64_t)segs_to_send) - 1);
1880 :
1881 0 : tx_queue->txq_ops.f_tx_bulk(tx_queue->q_queue_h, &mbufs_segs_out[segs_sent], mask);
1882 0 : if (unlikely(lcore_id < VR_DPDK_FWD_LCORE_ID))
1883 0 : tx_queue->txq_ops.f_flush(tx_queue->q_queue_h);
1884 0 : segs_sent += segs_to_send;
1885 : }
1886 :
1887 : } else {
1888 : RTE_LOG_DP(DEBUG, VROUTER,"%s: error TXing to interface %s: no queue "
1889 : "for lcore %u\n", __func__, vif->vif_name, lcore_id);
1890 : /* Can not do vif_drop_pkt() on segments as mbufs after
1891 : * segmentation does not have pkt structure */
1892 0 : for (i = 0; i < num_of_segs; ++i)
1893 0 : rte_pktmbuf_free(mbufs_segs_out[i]);
1894 :
1895 0 : vr_dpdk_pfree(m, pkt->vp_if, VP_DROP_INTERFACE_DROP);
1896 0 : return -1;
1897 : }
1898 : } else {
1899 125 : if (likely(tx_queue && tx_queue->txq_ops.f_tx != NULL)) {
1900 125 : tx_queue->txq_ops.f_tx(tx_queue->q_queue_h, m);
1901 125 : if (unlikely(lcore_id < VR_DPDK_FWD_LCORE_ID))
1902 2 : tx_queue->txq_ops.f_flush(tx_queue->q_queue_h);
1903 : } else {
1904 : RTE_LOG_DP(DEBUG, VROUTER,"%s: error TXing to interface %s: no queue "
1905 : "for lcore %u\n", __func__, vif->vif_name, lcore_id);
1906 0 : vr_dpdk_pfree(m, pkt->vp_if, VP_DROP_INTERFACE_DROP);
1907 0 : return -1;
1908 : }
1909 : }
1910 :
1911 125 : return 0;
1912 : }
1913 :
1914 : static int
1915 11 : dpdk_if_rx(struct vr_interface *vif, struct vr_packet *pkt)
1916 : {
1917 11 : const unsigned lcore_id = rte_lcore_id();
1918 11 : struct vr_dpdk_lcore * const lcore = vr_dpdk.lcores[lcore_id];
1919 11 : struct rte_mbuf *m = vr_dpdk_pkt_to_mbuf(pkt);
1920 11 : unsigned vif_idx = vif->vif_idx;
1921 : struct vr_dpdk_queue *tx_queue;
1922 : struct vr_dpdk_queue *monitoring_tx_queue;
1923 : struct rte_mbuf *p_copy;
1924 :
1925 : RTE_LOG_DP(DEBUG, VROUTER,"%s: TX packet to interface %s\n", __func__,
1926 : vif->vif_name);
1927 :
1928 11 : tx_queue = &lcore->lcore_tx_queues[vif_idx][0];
1929 : /* reset mbuf data pointer and length */
1930 11 : m->data_off = pkt_head_space(pkt);
1931 11 : m->data_len = pkt_head_len(pkt);
1932 :
1933 11 : m->pkt_len = pkt_len(pkt);
1934 :
1935 11 : if (unlikely(vif->vif_flags & VIF_FLAG_MONITORED)) {
1936 0 : monitoring_tx_queue =
1937 0 : &lcore->lcore_tx_queues[vr_dpdk.monitorings[vif_idx]][0];
1938 0 : if (likely(monitoring_tx_queue && monitoring_tx_queue->txq_ops.f_tx)) {
1939 0 : p_copy = vr_dpdk_pktmbuf_copy_mon(m, vr_dpdk.rss_mempool);;
1940 0 : if (likely(p_copy != NULL)) {
1941 0 : monitoring_tx_queue->txq_ops.f_tx(monitoring_tx_queue->q_queue_h,
1942 : p_copy);
1943 : }
1944 : }
1945 : }
1946 :
1947 : /* For tapdev, compute the checksum of packets originating in the
1948 : * VM and destined to the host. If offloads are enabled in the VM
1949 : * it would not compute the checksum and the host would drop it
1950 : */
1951 11 : if (vif_is_virtual(pkt->vp_if) && vif_is_vhost(vif))
1952 1 : dpdk_sw_checksum_at_offset(pkt, pkt_get_network_header_off(pkt));
1953 :
1954 : #ifdef VR_DPDK_TX_PKT_DUMP
1955 : #ifdef VR_DPDK_PKT_DUMP_VIF_FILTER
1956 : if (VR_DPDK_PKT_DUMP_VIF_FILTER(vif))
1957 : #endif
1958 : rte_pktmbuf_dump(stdout, m, 0x60);
1959 : #endif
1960 :
1961 11 : if (likely(tx_queue->txq_ops.f_tx != NULL)) {
1962 11 : tx_queue->txq_ops.f_tx(tx_queue->q_queue_h, m);
1963 11 : if (unlikely(lcore_id < VR_DPDK_FWD_LCORE_ID))
1964 0 : tx_queue->txq_ops.f_flush(tx_queue->q_queue_h);
1965 : } else {
1966 : RTE_LOG_DP(DEBUG, VROUTER,"%s: error TXing to interface %s: no queue for lcore %u\n",
1967 : __func__, vif->vif_name, lcore_id);
1968 0 : vr_dpdk_pfree(m, pkt->vp_if, VP_DROP_INTERFACE_DROP);
1969 0 : return -1;
1970 : }
1971 :
1972 11 : return 0;
1973 : }
1974 :
1975 : static int
1976 42 : dpdk_if_get_vlan_info(struct vr_interface *vif,
1977 : struct vr_interface_vlan_info *vlan_info)
1978 : {
1979 42 : if(vif->vif_flags & VIF_FLAG_MOCK_DEVICE) {
1980 42 : return -1;
1981 : }
1982 :
1983 0 : memset(vlan_info, 0, sizeof(*vlan_info));
1984 0 : if(vr_dpdk.vlan_tag != VLAN_ID_INVALID && vr_dpdk.vlan_name != NULL) {
1985 0 : vlan_info->vlan_id = vr_dpdk.vlan_tag;
1986 : /* Ignoring copy of default vlan fwd name vfw0 */
1987 0 : if (strncmp(vr_dpdk.vlan_name, VR_DPDK_VLAN_FWD_DEF_NAME,
1988 : strlen(VR_DPDK_VLAN_FWD_DEF_NAME))) {
1989 0 : strcpy(vlan_info->vlan_name, vr_dpdk.vlan_name);
1990 : }
1991 : } else
1992 0 : return -1;
1993 :
1994 0 : return 0;
1995 : }
1996 :
1997 : static void
1998 10 : dpdk_if_clear_dev_stats(struct vr_interface *vif)
1999 : {
2000 : uint16_t port_id;
2001 :
2002 10 : if(vif->vif_os) {
2003 0 : port_id = ((struct vr_dpdk_ethdev *)(vif->vif_os))->ethdev_port_id;
2004 :
2005 : /* This will internally reset for slave ports incase of bond */
2006 0 : rte_eth_stats_reset(port_id);
2007 : }
2008 :
2009 10 : return;
2010 : }
2011 :
2012 : static void
2013 120 : dpdk_if_clear_port_stats(struct vr_interface *vif, uint16_t lcore_id)
2014 : {
2015 : unsigned int i;
2016 :
2017 : struct vr_dpdk_lcore *lcore;
2018 : struct vr_dpdk_queue *queue;
2019 : struct rte_port_in_stats rx_stats;
2020 : struct rte_port_out_stats tx_stats;
2021 :
2022 120 : lcore = vr_dpdk.lcores[lcore_id];
2023 :
2024 120 : if (lcore == NULL)
2025 50 : return;
2026 :
2027 : /* RX queue */
2028 70 : queue = &lcore->lcore_rx_queues[vif->vif_idx];
2029 70 : if (queue->q_vif == vif) {
2030 : /* reset stats */
2031 20 : if (queue->rxq_ops.f_stats != NULL) {
2032 20 : queue->rxq_ops.f_stats(queue->q_queue_h, &rx_stats, 1);
2033 : }
2034 : }
2035 :
2036 : /* TX queue */
2037 110 : for (i = 0; i < lcore->num_tx_queues_per_lcore[vif->vif_idx]; i++) {
2038 40 : queue = &lcore->lcore_tx_queues[vif->vif_idx][i];
2039 40 : if (queue && (queue->q_vif == vif)) {
2040 : /* reset stats */
2041 40 : if (queue->txq_ops.f_stats != NULL)
2042 40 : queue->txq_ops.f_stats(queue->q_queue_h, &tx_stats, 1);
2043 : }
2044 : }
2045 : }
2046 :
2047 : static int
2048 10 : dpdk_if_clear_stats(struct vr_interface *vif)
2049 : {
2050 : uint16_t i;
2051 :
2052 10 : dpdk_if_clear_dev_stats(vif);
2053 130 : for (i = 0; i < vr_num_cpus; i++)
2054 : {
2055 120 : dpdk_if_clear_port_stats(vif, i);
2056 : }
2057 10 : return 0;
2058 :
2059 : }
2060 :
2061 : static int
2062 1 : dpdk_get_host_ip_mask(struct vr_interface *vif, unsigned int *ip,
2063 : unsigned int *mask)
2064 : {
2065 1 : if(!vif) {
2066 0 : RTE_LOG(ERR, VROUTER, "%s: Vif is NULL\n", __func__);
2067 0 : return -1;
2068 : }
2069 1 : *ip = vif->vif_ip;
2070 1 : *mask = vif->vif_ip_mask;
2071 1 : return 0;
2072 : }
2073 : static int
2074 1 : dpdk_get_host_mac_addr(struct vr_interface *vif, unsigned char **mac)
2075 : {
2076 1 : if(!vif) {
2077 0 : RTE_LOG(ERR, VROUTER, "%s : vif is NULL\n", __func__);
2078 0 : return -1;
2079 : }
2080 1 : *mac = vif->vif_mac;
2081 :
2082 1 : return 0;
2083 : }
2084 :
2085 : static int
2086 42 : dpdk_if_get_bond_info(struct vr_interface *vif,
2087 : struct vr_interface_bond_info *bond_info)
2088 : {
2089 :
2090 42 : uint8_t port_id = 0, i = 0;
2091 42 : struct vr_dpdk_ethdev *ethdev = ((struct vr_dpdk_ethdev*)(vif->vif_os));
2092 : struct rte_eth_link link;
2093 :
2094 42 : if(vif->vif_flags & VIF_FLAG_MOCK_DEVICE) {
2095 42 : return -1;
2096 : }
2097 :
2098 0 : memset(bond_info, 0, sizeof(*bond_info));
2099 :
2100 : /* To get fabric info */
2101 0 : bond_info->vif_fab_drv_name = rte_eth_devices[ethdev->ethdev_port_id].device->driver->name;
2102 0 : bond_info->vif_fab_name = rte_eth_devices[ethdev->ethdev_port_id].data->name;
2103 0 : rte_eth_link_get_nowait(ethdev->ethdev_port_id, &link);
2104 0 : bond_info->vif_intf_link_status = link.link_status;
2105 :
2106 0 : for (i = 0; i < ethdev->ethdev_nb_slaves; i++) {
2107 0 : port_id = ethdev->ethdev_slaves[i];
2108 :
2109 0 : bond_info->vif_slave_drv_name[i] = rte_eth_devices[port_id].device->driver->name;
2110 0 : bond_info->vif_slave_name[i] = rte_eth_devices[port_id].data->name;
2111 :
2112 : /* Get link status of bond slave ports */
2113 0 : rte_eth_link_get_nowait(port_id, &link);
2114 0 : bond_info->vif_intf_link_status |= (link.link_status << (i + 1));
2115 : }
2116 0 : bond_info->vif_num_slave = ethdev->ethdev_nb_slaves;
2117 :
2118 0 : return 0;
2119 : }
2120 :
2121 : static int
2122 42 : dpdk_if_get_settings(struct vr_interface *vif,
2123 : struct vr_interface_settings *settings)
2124 : {
2125 : uint8_t port_id;
2126 : struct rte_eth_link link;
2127 :
2128 42 : if(vif->vif_flags & VIF_FLAG_MOCK_DEVICE)
2129 42 : return -1;
2130 :
2131 0 : port_id = ((struct vr_dpdk_ethdev*)(vif->vif_os))->ethdev_port_id;
2132 0 : memset(&link, 0, sizeof(link));
2133 0 : rte_eth_link_get_nowait(port_id, &link);
2134 0 : if (link.link_speed != 0) {
2135 0 : settings->vis_speed = link.link_speed;
2136 0 : settings->vis_duplex = link.link_duplex == ETH_LINK_FULL_DUPLEX?
2137 0 : 1 : 0;
2138 : } else {
2139 : /* default values */
2140 0 : settings->vis_speed = 1000;
2141 0 : settings->vis_duplex = 1;
2142 : }
2143 0 : return 0;
2144 : }
2145 :
2146 : static unsigned int
2147 61 : dpdk_if_get_mtu(struct vr_interface *vif)
2148 : {
2149 : uint8_t port_id;
2150 : uint16_t mtu;
2151 : unsigned l3_mtu;
2152 :
2153 61 : l3_mtu = vif->vif_mtu;
2154 :
2155 61 : if(vif->vif_flags & VIF_FLAG_MOCK_DEVICE) {
2156 : /* If Mock physical flag enabled, running on
2157 : * vtest(Vrouter Unit Test simulation framework) and directly return mtu size */
2158 53 : return l3_mtu?l3_mtu:VR_DPDK_VHOST_DEFAULT_MTU_SIZE;
2159 : }
2160 8 : if (vif->vif_type == VIF_TYPE_PHYSICAL) {
2161 :
2162 0 : port_id = (((struct vr_dpdk_ethdev *)(vif->vif_os))->ethdev_port_id);
2163 : /* TODO: DPDK bond interfaces does not provide MTU (MTU is 0) */
2164 0 : if (rte_eth_dev_get_mtu(port_id, &mtu) == 0 && mtu > 0)
2165 0 : return mtu;
2166 :
2167 : /* Decrement Ethernet header size. */
2168 0 : l3_mtu -= sizeof(struct vr_eth);
2169 0 : if (vr_dpdk.vlan_tag != VLAN_ID_INVALID) {
2170 : /* Decrement 802.1q header size. */
2171 0 : l3_mtu -= sizeof(uint32_t);
2172 : }
2173 : }
2174 :
2175 :
2176 8 : return l3_mtu;
2177 : }
2178 :
2179 : static void
2180 664 : dpdk_if_unlock(void)
2181 : {
2182 664 : vr_dpdk_if_unlock();
2183 664 : }
2184 :
2185 : static void
2186 664 : dpdk_if_lock(void)
2187 : {
2188 664 : vr_dpdk_if_lock();
2189 664 : }
2190 :
2191 : static unsigned short
2192 234 : dpdk_if_get_encap(struct vr_interface *vif)
2193 : {
2194 234 : return VIF_ENCAP_TYPE_ETHER;
2195 : }
2196 :
2197 : /* Update port statistics */
2198 : static void
2199 1896 : dpdk_port_stats_update(struct vr_interface *vif, unsigned lcore_id)
2200 : {
2201 : unsigned int i;
2202 :
2203 : struct vr_interface_stats *stats;
2204 : struct vr_dpdk_lcore *lcore;
2205 : struct vr_dpdk_queue *queue;
2206 : struct rte_port_in_stats rx_stats;
2207 : struct rte_port_out_stats tx_stats;
2208 :
2209 1896 : stats = vif_get_stats(vif, lcore_id);
2210 1896 : lcore = vr_dpdk.lcores[lcore_id];
2211 :
2212 1896 : if (lcore == NULL)
2213 790 : return;
2214 :
2215 : /* RX queue */
2216 1106 : queue = &lcore->lcore_rx_queues[vif->vif_idx];
2217 1106 : if (queue->q_vif == vif) {
2218 : /* update stats */
2219 316 : if (queue->rxq_ops.f_stats != NULL) {
2220 316 : if (queue->rxq_ops.f_stats(queue->q_queue_h,
2221 : &rx_stats, 0) == 0) {
2222 316 : if (queue->rxq_ops.f_rx == rte_port_ring_reader_ops.f_rx) {
2223 : /* DPDK ports count dropped packets twice */
2224 0 : stats->vis_queue_ipackets = rx_stats.n_pkts_in - rx_stats.n_pkts_drop;
2225 0 : stats->vis_queue_ierrors = rx_stats.n_pkts_drop;
2226 : } else {
2227 : /* DPDK ports count dropped packets twice */
2228 316 : stats->vis_port_ipackets = rx_stats.n_pkts_in - rx_stats.n_pkts_drop;
2229 316 : stats->vis_port_ierrors = rx_stats.n_pkts_drop;
2230 : }
2231 : }
2232 : }
2233 :
2234 : /* update virtio syscalls and no mbufs counters */
2235 316 : vr_dpdk_virtio_xstats_update(stats, queue);
2236 : }
2237 :
2238 1106 : stats->vis_queue_opackets = stats->vis_queue_oerrors = 0;
2239 1106 : stats->vis_port_opackets = stats->vis_port_oerrors = 0;
2240 : /* TX queue */
2241 1738 : for (i = 0; i < lcore->num_tx_queues_per_lcore[vif->vif_idx]; i++) {
2242 632 : queue = &lcore->lcore_tx_queues[vif->vif_idx][i];
2243 632 : if (queue && (queue->q_vif == vif)) {
2244 : /* update stats */
2245 632 : if (queue->txq_ops.f_stats != NULL) {
2246 632 : if (queue->txq_ops.f_stats(queue->q_queue_h,
2247 : &tx_stats, 0) == 0) {
2248 632 : if (queue->txq_ops.f_tx == rte_port_ring_writer_ops.f_tx) {
2249 : /* DPDK ports count dropped packets twice */
2250 0 : stats->vis_queue_opackets += tx_stats.n_pkts_in -
2251 0 : tx_stats.n_pkts_drop;
2252 0 : stats->vis_queue_oerrors += tx_stats.n_pkts_drop;
2253 : } else {
2254 : /* DPDK ports count dropped packets twice */
2255 632 : stats->vis_port_opackets += tx_stats.n_pkts_in -
2256 632 : tx_stats.n_pkts_drop;
2257 632 : stats->vis_port_oerrors += tx_stats.n_pkts_drop;
2258 : }
2259 : }
2260 : }
2261 :
2262 : /* update virtio syscalls counters */
2263 632 : vr_dpdk_virtio_xstats_update(stats, queue);
2264 : }
2265 : }
2266 : }
2267 :
2268 : /* For non-bonded interfaces, use the xstats from the rte_eth_xstats_get() API.
2269 : * For bonded interfaces, this API is not available, so instead use xstats from
2270 : * it's individual slave interfaces
2271 : */
2272 : static void
2273 0 : vr_dpdk_eth_xstats_get(uint32_t port_id, struct rte_eth_stats *eth_stats)
2274 : {
2275 : /*
2276 : * TODO: In DPDK 2.1 ierrors includes XEC (l3_l4_xsum_error) counter.
2277 : * The counter seems to include no check sum UDP packets. As a workaround
2278 : * we count out the XEC from ierrors using rte_eth_xstats_get()
2279 : */
2280 :
2281 : uint16_t *port_id_ptr;
2282 0 : int port_num = 0;
2283 0 : struct vr_dpdk_ethdev *ethdev = &vr_dpdk.ethdevs[port_id];
2284 0 : port_id_ptr = (ethdev->ethdev_nb_slaves == -1)?
2285 0 : ðdev->ethdev_port_id:ethdev->ethdev_slaves;
2286 : do {
2287 0 : struct rte_eth_xstat *eth_xstats = NULL;
2288 : struct rte_eth_xstat_name *xstats_names;
2289 : int nb_xstats, i;
2290 :
2291 0 : nb_xstats = rte_eth_xstats_get(*port_id_ptr, eth_xstats, 0);
2292 0 : if (nb_xstats > 0) {
2293 0 : xstats_names = rte_malloc("stats_name", sizeof(struct rte_eth_xstat_name) * nb_xstats, 0);
2294 0 : if (xstats_names != NULL) {
2295 0 : if (nb_xstats != rte_eth_xstats_get_names(*port_id_ptr,
2296 : xstats_names, nb_xstats)) {
2297 :
2298 0 : rte_free(xstats_names);
2299 0 : return;
2300 : }
2301 : } else
2302 0 : return;
2303 :
2304 0 : eth_xstats = rte_malloc("xstats",
2305 : sizeof(struct rte_eth_xstat)*nb_xstats, 0);
2306 0 : if (eth_xstats != NULL) {
2307 0 : if (rte_eth_xstats_get(*port_id_ptr, eth_xstats, nb_xstats)
2308 : == nb_xstats) {
2309 : /* look for XEC counter */
2310 0 : for (i = 0; i < nb_xstats; i++) {
2311 0 : if (strncmp(xstats_names[i].name, "l3_l4_xsum_error",
2312 : sizeof(xstats_names[i].name)) == 0) {
2313 0 : eth_stats->ierrors -= eth_xstats[i].value;
2314 0 : break;
2315 : }
2316 : }
2317 : }
2318 0 : rte_free(eth_xstats);
2319 : }
2320 0 : rte_free(xstats_names);
2321 : }
2322 0 : port_num++;
2323 0 : port_id_ptr++;
2324 0 : } while (port_num < ethdev->ethdev_nb_slaves);
2325 :
2326 : /* Stats cannot go negative */
2327 0 : if ((int64_t)eth_stats->ierrors < 0)
2328 0 : eth_stats->ierrors = 0;
2329 : }
2330 :
2331 : /* Update device statistics */
2332 : static void
2333 1896 : dpdk_dev_stats_update(struct vr_interface *vif, unsigned lcore_id)
2334 : {
2335 : uint8_t port_id;
2336 : uint16_t queue_id, dpdk_queue_index, num_queues, i;
2337 :
2338 : struct vr_interface_stats *stats;
2339 : struct vr_dpdk_lcore *lcore;
2340 : struct vr_dpdk_queue *queue;
2341 : struct vr_dpdk_queue_params *queue_params;
2342 1896 : struct vif_queue_dpdk_data *q_data =
2343 : (struct vif_queue_dpdk_data *)vif->vif_queue_host_data;
2344 : struct rte_eth_stats eth_stats;
2345 1896 : bool is_representor =
2346 1896 : vr_dpdk_representor_stats_update(vif) ==
2347 : VR_DPDK_REPRESENTOR_OP_RES_HANDLED_OK;
2348 :
2349 : /* check if vif is a PMD or a representor */
2350 1896 : if ((!is_representor && !vif_is_fabric(vif)) || vif->vif_os == NULL)
2351 1896 : return;
2352 :
2353 0 : port_id = ((struct vr_dpdk_ethdev *)(vif->vif_os))->ethdev_port_id;
2354 0 : if (rte_eth_stats_get(port_id, ð_stats) != 0)
2355 0 : return;
2356 :
2357 0 : vr_dpdk_eth_xstats_get(port_id, ð_stats);
2358 :
2359 : /* per-lcore device counters */
2360 0 : lcore = vr_dpdk.lcores[lcore_id];
2361 0 : if (lcore == NULL)
2362 0 : return;
2363 :
2364 0 : stats = vif_get_stats(vif, lcore_id);
2365 :
2366 : /* get lcore RX queue index */
2367 0 : queue = &lcore->lcore_rx_queues[vif->vif_idx];
2368 0 : if (queue->rxq_ops.f_rx == rte_port_ethdev_reader_ops.f_rx) {
2369 0 : queue_params = &lcore->lcore_rx_queue_params[vif->vif_idx];
2370 0 : queue_id = queue_params->qp_ethdev.queue_id;
2371 0 : if (queue_id < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
2372 0 : stats->vis_dev_ibytes = eth_stats.q_ibytes[queue_id];
2373 0 : stats->vis_dev_ipackets = eth_stats.q_ipackets[queue_id];
2374 0 : stats->vis_dev_ierrors = eth_stats.q_errors[queue_id];
2375 : }
2376 : }
2377 :
2378 : /* get lcore TX queue index */
2379 0 : if (vif->vif_hw_queues) {
2380 0 : num_queues = vif->vif_num_hw_queues;
2381 0 : if (!q_data)
2382 0 : num_queues = 1;
2383 : } else {
2384 0 : num_queues = 1;
2385 : }
2386 :
2387 0 : stats->vis_dev_obytes = stats->vis_dev_opackets = 0;
2388 0 : for (i = 0; i < num_queues; i++) {
2389 0 : if (vif->vif_hw_queues) {
2390 0 : queue_id = vif->vif_hw_queues[i];
2391 0 : if (q_data->vqdd_queue_to_lcore[queue_id] != lcore_id) {
2392 0 : continue;
2393 : }
2394 : } else {
2395 0 : queue_id = i;
2396 : }
2397 :
2398 0 : if (lcore->lcore_tx_queues[vif->vif_idx]) {
2399 0 : if (lcore->lcore_hw_queue_to_dpdk_index[vif->vif_idx]) {
2400 0 : dpdk_queue_index =
2401 0 : lcore->lcore_hw_queue_to_dpdk_index[vif->vif_idx][queue_id];
2402 : } else {
2403 0 : dpdk_queue_index = 0;
2404 : }
2405 :
2406 0 : queue = &lcore->lcore_tx_queues[vif->vif_idx][dpdk_queue_index];
2407 0 : if (queue && (queue->txq_ops.f_tx == rte_port_ethdev_writer_ops.f_tx)) {
2408 0 : queue_params = &lcore->lcore_tx_queue_params[vif->vif_idx][queue_id];
2409 0 : queue_id = queue_params->qp_ethdev.queue_id;
2410 0 : if (queue_id < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
2411 0 : stats->vis_dev_obytes += eth_stats.q_obytes[queue_id];
2412 0 : stats->vis_dev_opackets += eth_stats.q_opackets[queue_id];
2413 : }
2414 : }
2415 : }
2416 : }
2417 :
2418 : /* For DPDK, forwarding core starts from core 10.
2419 : * Starting from core 10 respective queue and core store RX and TX packets.
2420 : * so, no need of storing global counter in lcore 0.
2421 : * for locore 0 storing only vis_dev_inombufs and
2422 : * ignoring other counters.
2423 : */
2424 0 : if (lcore_id == 0) {
2425 : /* use lcore 0 to store global device counters */
2426 0 : stats->vis_dev_inombufs = eth_stats.rx_nombuf;
2427 : }
2428 : }
2429 :
2430 : /* Update interface statistics */
2431 : static void
2432 158 : dpdk_if_stats_update(struct vr_interface *vif, unsigned core)
2433 : {
2434 : int i;
2435 :
2436 158 : if (core == (unsigned)-1) {
2437 : /* update counters for all cores */
2438 2054 : for (i = 0; i < vr_num_cpus; i++) {
2439 1896 : dpdk_dev_stats_update(vif, i);
2440 1896 : dpdk_port_stats_update(vif, i);
2441 : }
2442 0 : } else if (core < vr_num_cpus) {
2443 : /* update counters for a specific core */
2444 0 : dpdk_dev_stats_update(vif, core);
2445 0 : dpdk_port_stats_update(vif, core);
2446 : }
2447 : /* otherwise there is nothing to update */
2448 158 : }
2449 :
2450 : struct vr_host_interface_ops dpdk_interface_ops = {
2451 : .hif_lock = dpdk_if_lock,
2452 : .hif_unlock = dpdk_if_unlock,
2453 : .hif_add = dpdk_if_add,
2454 : .hif_del = dpdk_if_del,
2455 : .hif_add_tap = dpdk_if_add_tap, /* not implemented */
2456 : .hif_del_tap = dpdk_if_del_tap, /* not implemented */
2457 : .hif_add_tun_tap = dpdk_if_add_tun_tap,
2458 : .hif_del_tun_tap = dpdk_if_del_tun_tap,
2459 : .hif_tx = dpdk_if_tx,
2460 : .hif_rx = dpdk_if_rx,
2461 : .hif_get_settings = dpdk_if_get_settings,
2462 : .hif_get_mtu = dpdk_if_get_mtu,
2463 : .hif_get_encap = dpdk_if_get_encap, /* always returns VIF_ENCAP_TYPE_ETHER */
2464 : .hif_stats_update = dpdk_if_stats_update,
2465 : .hif_get_bond_info = dpdk_if_get_bond_info,
2466 : .hif_get_vlan_info = dpdk_if_get_vlan_info,
2467 : .hif_clear_stats = dpdk_if_clear_stats,
2468 : .hif_get_host_ip_mask = dpdk_get_host_ip_mask,
2469 : .hif_get_host_mac_addr = dpdk_get_host_mac_addr,
2470 : .hif_rx_pass = NULL,
2471 : };
2472 :
2473 : void
2474 53 : vr_host_vif_init(struct vrouter *router)
2475 : {
2476 53 : return;
2477 : }
2478 :
2479 : struct vr_host_interface_ops *
2480 53 : vr_host_interface_init(void)
2481 : {
2482 53 : return &dpdk_interface_ops;
2483 : }
2484 :
2485 : void
2486 53 : vr_host_interface_exit(void)
2487 : {
2488 53 : return;
2489 : }
|
