Line data    Source code

       1             : /*
       2             :  * vr_dpdk_offloads.c -- dpdk callbacks for datapath flow offloads management
       3             :  *
       4             :  * Copyright 2018 Mellanox Technologies, Ltd
       5             :  */
       6             : #include "vr_dpdk_offloads.h"
       7             : #include "vr_vxlan.h"
       8             : #include <rte_flow.h>
       9             : 
      10             : int
      11           0 : dpdk_offload_flow_destroy(struct vr_offload_flow *oflow)
      12             : {
      13           0 :     struct vr_dpdk_ethdev *ethdev = (struct vr_dpdk_ethdev *)oflow->pvif->vif_os;
      14             :     struct rte_flow_error error;
      15           0 :     int ret = 0;
      16             : 
      17           0 :     RTE_LOG(DEBUG, VROUTER, "Destroy dpdk offload flow:\n"
      18             :             "\t\tflow index = %u\n"
      19             :             "\t\tflow interface ID = %u\n"
      20             :             "\t\tflow host ip = 0x%X\n"
      21             :             "\t\tflow tunnel type = %u\n"
      22             :             "\t\tflow tunnel tag = %u\n"
      23             :             "\t\tflow nexthop ID = %u\n", oflow->fe_index, oflow->pvif->vif_idx,
      24             :             oflow->ip, oflow->tunnel_type, oflow->tunnel_tag, oflow->nh->nh_id);
      25             : 
      26           0 :     if (likely(oflow->flow_handle != NULL)) {
      27           0 :         rte_spinlock_lock(&ethdev->ethdev_lock);
      28           0 :         ret = rte_flow_destroy(ethdev->ethdev_port_id, oflow->flow_handle, &error);
      29           0 :         rte_spinlock_unlock(&ethdev->ethdev_lock);
      30           0 :         if (unlikely(ret != 0))
      31           0 :             RTE_LOG(ERR, VROUTER, "Failed to destroy flow - %s\n", error.message ?
      32             :                     error.message : "no error message");
      33             :     } else
      34           0 :         RTE_LOG(WARNING, VROUTER, "try to destroy an empty offload flow\n");
      35             : 
      36           0 :     return ret;
      37             : }
      38             : 
      39             : int
      40           0 : dpdk_offload_flow_create(struct vr_offload_flow *oflow)
      41             : {
      42             :     static struct rte_flow_attr attr = {
      43             :         .ingress = 1,
      44             :     };
      45             :     static struct rte_flow_item_eth eth_spec = {
      46             :         .type = RTE_BE16(VR_ETH_PROTO_IP),
      47             :     };
      48             :     static struct rte_flow_item_eth eth_mask = {
      49             :         .type = -1,
      50             :     };
      51             :     static struct rte_flow_item_ipv4 ipv4_spec;
      52             :     static struct rte_flow_item_ipv4 ipv4_mask = {
      53             :         .hdr = {
      54             :             .dst_addr = -1,
      55             :             .next_proto_id = -1,
      56             :         },
      57             :     };
      58             :     static struct rte_flow_item_gre gre_spec = {
      59             :         .protocol = RTE_BE16(VR_GRE_PROTO_MPLS),
      60             :     };
      61             :     static struct rte_flow_item_gre gre_mask = {
      62             :         .protocol = -1,
      63             :     };
      64             :     static struct rte_flow_item_udp udp_spec;
      65             :     static struct rte_flow_item_udp udp_mask = {
      66             :         .hdr = {
      67             :             .dst_port = -1,
      68             :         },
      69             :     };
      70             :     static struct rte_flow_item_mpls mpls_spec;
      71             :     static struct rte_flow_item_mpls mpls_mask = {
      72             :         .label_tc_s = "\xff\xff\xf0",
      73             :     };
      74             :     static struct rte_flow_item_vxlan vxlan_spec;
      75             :     static struct rte_flow_item_vxlan vxlan_mask = {
      76             :         .vni = "\xff\xff\xff",
      77             :     };
      78             : 
      79             :     static struct rte_flow_item_eth ieth_spec;
      80             :     static struct rte_flow_item_eth ieth_mask = {
      81             :         .type = -1,
      82             :     };
      83             :     static struct rte_flow_item_ipv4 iipv4_spec;
      84             :     static struct rte_flow_item_ipv4 iipv4_mask = {
      85             :         .hdr = {
      86             :             .src_addr = -1,
      87             :             .dst_addr = -1,
      88             :             .next_proto_id = -1,
      89             :         },
      90             :     };
      91             :     static struct rte_flow_item_ipv6 iipv6_spec;
      92             :     static struct rte_flow_item_ipv6 iipv6_mask = {
      93             :         .hdr = {
      94             :             .src_addr = {
      95             :                 0xff, 0xff, 0xff, 0xff,
      96             :                 0xff, 0xff, 0xff, 0xff,
      97             :                 0xff, 0xff, 0xff, 0xff,
      98             :                 0xff, 0xff, 0xff, 0xff,
      99             :             },
     100             :             .dst_addr = {
     101             :                 0xff, 0xff, 0xff, 0xff,
     102             :                 0xff, 0xff, 0xff, 0xff,
     103             :                 0xff, 0xff, 0xff, 0xff,
     104             :                 0xff, 0xff, 0xff, 0xff,
     105             :             },
     106             :             .proto = -1,
     107             :         },
     108             : 
     109             :     };
     110             :     static struct rte_flow_item_tcp itcpudp_spec;
     111             :     static struct rte_flow_item_tcp itcpudp_mask;
     112             :     enum {ETH, IPV4, GRE, UDP, MPLS, VXLAN, IETH, IIPV4, IIPV6, ITCPUDP, IEND};
     113             :     static struct rte_flow_item pattern[] = {
     114             :         [ETH] = {
     115             :             .type = RTE_FLOW_ITEM_TYPE_ETH,
     116             :             .spec = &eth_spec,
     117             :             .mask = &eth_mask,
     118             :             .last = NULL,
     119             :         },
     120             :         [IPV4] = {
     121             :             .type = RTE_FLOW_ITEM_TYPE_IPV4,
     122             :             .spec = &ipv4_spec,
     123             :             .mask = &ipv4_mask,
     124             :             .last = NULL,
     125             :         },
     126             :         [GRE] = {
     127             :             .type = RTE_FLOW_ITEM_TYPE_GRE,
     128             :             .spec = &gre_spec,
     129             :             .mask = &gre_mask,
     130             :             .last = NULL,
     131             :         },
     132             :         [UDP] = {
     133             :             .type = RTE_FLOW_ITEM_TYPE_UDP,
     134             :             .spec = &udp_spec,
     135             :             .mask = &udp_mask,
     136             :             .last = NULL,
     137             :         },
     138             :         [MPLS] = {
     139             :             .type = RTE_FLOW_ITEM_TYPE_MPLS,
     140             :             .spec = &mpls_spec,
     141             :             .mask = &mpls_mask,
     142             :             .last = NULL,
     143             :         },
     144             :         [VXLAN] = {
     145             :             .type = RTE_FLOW_ITEM_TYPE_VXLAN,
     146             :             .spec = &vxlan_spec,
     147             :             .mask = &vxlan_mask,
     148             :             .last = NULL,
     149             :         },
     150             :         [IETH] = {
     151             :             .type = RTE_FLOW_ITEM_TYPE_ETH,
     152             :             .spec = &ieth_spec,
     153             :             .mask = &ieth_mask,
     154             :             .last = NULL,
     155             :         },
     156             :         [IIPV4] = {
     157             :             .type = RTE_FLOW_ITEM_TYPE_IPV4,
     158             :             .spec = &iipv4_spec,
     159             :             .mask = &iipv4_mask,
     160             :             .last = NULL,
     161             :         },
     162             :         [IIPV6] = {
     163             :             .type = RTE_FLOW_ITEM_TYPE_IPV6,
     164             :             .spec = &iipv6_spec,
     165             :             .mask = &iipv6_mask,
     166             :             .last = NULL,
     167             :         },
     168             :         [ITCPUDP] = {
     169             :             .type = RTE_FLOW_ITEM_TYPE_TCP,
     170             :             .spec = &itcpudp_spec,
     171             :             .mask = &itcpudp_mask,
     172             :             .last = NULL,
     173             :         },
     174             :         [IEND] = {
     175             :             .type = RTE_FLOW_ITEM_TYPE_END,
     176             :             .spec = NULL,
     177             :             .mask = NULL,
     178             :             .last = NULL,
     179             :         },
     180             :     };
     181             :     static uint16_t rss_queues[VR_DPDK_MAX_NB_RX_QUEUES];
     182             :     static struct rte_flow_action_rss rss_action = {
     183             :         .queue = &rss_queues[0],
     184             :     };
     185             :     static struct rte_flow_action_mark mark;
     186             :     enum {RSS, MARK, ACTION_END};
     187             :     static struct rte_flow_action actions[] = {
     188             :         [RSS] = {
     189             :             .type = RTE_FLOW_ACTION_TYPE_RSS,
     190             :             .conf = &rss_action,
     191             :         },
     192             :         [MARK] = {
     193             :             .type = RTE_FLOW_ACTION_TYPE_MARK,
     194             :             .conf = &mark,
     195             :         },
     196             :         [ACTION_END] = {
     197             :             .type = RTE_FLOW_ACTION_TYPE_END,
     198             :         },
     199             :     };
     200           0 :     struct vr_dpdk_ethdev *ethdev = (struct vr_dpdk_ethdev *)oflow->pvif->vif_os;
     201             :     struct rte_flow_error error;
     202             :     uint32_t header;
     203             :     int i;
     204             : 
     205           0 :     RTE_LOG(DEBUG, VROUTER, "Create dpdk offload flow:\n"
     206             :             "\t\tflow index = %u\n"
     207             :             "\t\tflow interface ID = %u\n"
     208             :             "\t\tflow host ip = 0x%X\n"
     209             :             "\t\tflow tunnel type = %u\n"
     210             :             "\t\tflow tunnel tag = %u\n"
     211             :             "\t\tflow nexthop ID = %u\n", oflow->fe_index, oflow->pvif->vif_idx,
     212             :             oflow->ip, oflow->tunnel_type, oflow->tunnel_tag, oflow->nh->nh_id);
     213             : 
     214           0 :     if (unlikely(oflow->flow_handle != NULL)) {
     215           0 :         RTE_LOG(ERR, VROUTER, "Failed to create flow - oflow %u already exists\n",
     216             :                 oflow->fe_index);
     217           0 :         return -EEXIST;
     218             :     }
     219             : 
     220             :     /* Fill actions */
     221           0 :     rss_action.queue_num = 0;
     222           0 :     for (i = 0; i < VR_DPDK_MAX_NB_RX_QUEUES; ++i)
     223           0 :         if (ethdev->ethdev_queue_states[i] == VR_DPDK_QUEUE_RSS_STATE)
     224           0 :             rss_queues[rss_action.queue_num++] = i;
     225           0 :     rss_action.types = ETH_RSS_IP | ETH_RSS_UDP | ETH_RSS_TCP;
     226           0 :     mark.id = oflow->fe_index;
     227             : 
     228             :     /* Fill inner L3 item */
     229           0 :     switch (oflow->fe->fe_type) {
     230           0 :     case VP_TYPE_IP:
     231           0 :         pattern[IIPV4].type = RTE_FLOW_ITEM_TYPE_IPV4;
     232           0 :         pattern[IIPV6].type = RTE_FLOW_ITEM_TYPE_VOID;
     233           0 :         ieth_spec.type = RTE_BE16(VR_ETH_PROTO_IP);
     234           0 :         iipv4_spec.hdr.src_addr = oflow->fe->fe_key.flow4_sip;
     235           0 :         iipv4_spec.hdr.dst_addr = oflow->fe->fe_key.flow4_dip;
     236           0 :         iipv4_spec.hdr.next_proto_id = oflow->fe->fe_key.flow4_proto;
     237           0 :         rss_action.types &= ~(ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6 |
     238             :                               ETH_RSS_NONFRAG_IPV6_OTHER | ETH_RSS_IPV6_EX);
     239           0 :         RTE_LOG(DEBUG, VROUTER, "\t\tflow L3 tuples: ipv4 src = %u dst = %u\n",
     240             :                 ntohl(oflow->fe->fe_key.flow4_sip),
     241             :                 ntohl(oflow->fe->fe_key.flow4_dip));
     242           0 :         break;
     243           0 :     case VP_TYPE_IP6:
     244           0 :         pattern[IIPV4].type = RTE_FLOW_ITEM_TYPE_VOID;
     245           0 :         pattern[IIPV6].type = RTE_FLOW_ITEM_TYPE_IPV6;
     246           0 :         ieth_spec.type = RTE_BE16(VR_ETH_PROTO_IP6);
     247           0 :         memcpy(&iipv6_spec.hdr.src_addr, oflow->fe->fe_key.flow6_sip,
     248             :                VR_IP6_ADDRESS_LEN);
     249           0 :         memcpy(&iipv6_spec.hdr.dst_addr, oflow->fe->fe_key.flow6_dip,
     250             :                VR_IP6_ADDRESS_LEN);
     251           0 :         iipv6_spec.hdr.proto = oflow->fe->fe_key.flow6_proto;
     252           0 :         rss_action.types &= ~(ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 |
     253             :                               ETH_RSS_NONFRAG_IPV4_OTHER);
     254           0 :         RTE_LOG(DEBUG, VROUTER, "\t\tflow L3 tuples: ipv6 src ="
     255             :                 " %02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x"
     256             :                 " dst = %02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x\n",
     257             :                 iipv6_spec.hdr.src_addr[0], iipv6_spec.hdr.src_addr[1],
     258             :                 iipv6_spec.hdr.src_addr[2], iipv6_spec.hdr.src_addr[3],
     259             :                 iipv6_spec.hdr.src_addr[4], iipv6_spec.hdr.src_addr[5],
     260             :                 iipv6_spec.hdr.src_addr[6], iipv6_spec.hdr.src_addr[7],
     261             :                 iipv6_spec.hdr.src_addr[8], iipv6_spec.hdr.src_addr[9],
     262             :                 iipv6_spec.hdr.src_addr[10], iipv6_spec.hdr.src_addr[11],
     263             :                 iipv6_spec.hdr.src_addr[12], iipv6_spec.hdr.src_addr[13],
     264             :                 iipv6_spec.hdr.src_addr[14], iipv6_spec.hdr.src_addr[15],
     265             :                 iipv6_spec.hdr.dst_addr[0], iipv6_spec.hdr.dst_addr[1],
     266             :                 iipv6_spec.hdr.dst_addr[2], iipv6_spec.hdr.dst_addr[3],
     267             :                 iipv6_spec.hdr.dst_addr[4], iipv6_spec.hdr.dst_addr[5],
     268             :                 iipv6_spec.hdr.dst_addr[6], iipv6_spec.hdr.dst_addr[7],
     269             :                 iipv6_spec.hdr.dst_addr[8], iipv6_spec.hdr.dst_addr[9],
     270             :                 iipv6_spec.hdr.dst_addr[10], iipv6_spec.hdr.dst_addr[11],
     271             :                 iipv6_spec.hdr.dst_addr[12], iipv6_spec.hdr.dst_addr[13],
     272             :                 iipv6_spec.hdr.dst_addr[14], iipv6_spec.hdr.dst_addr[15]);
     273           0 :         break;
     274           0 :     default:
     275           0 :         RTE_LOG(ERR, VROUTER, "Failed to create flow - flow %u has unsupported L3"
     276             :                 " protocol %u\n", oflow->fe_index, oflow->fe->fe_type);
     277           0 :         return -ENOTSUP;
     278             :     }
     279             : 
     280             :     /* Fill inner L4 item */
     281           0 :     switch (oflow->fe->fe_key.flow_proto) {
     282           0 :     case VR_IP_PROTO_TCP:
     283           0 :         pattern[ITCPUDP].type = RTE_FLOW_ITEM_TYPE_TCP;
     284           0 :         rss_action.types &= ~ETH_RSS_UDP;
     285           0 :         break;
     286           0 :     case VR_IP_PROTO_UDP:
     287           0 :         pattern[ITCPUDP].type = RTE_FLOW_ITEM_TYPE_UDP;
     288           0 :         rss_action.types &= ~ETH_RSS_TCP;
     289           0 :         break;
     290           0 :     default:
     291           0 :         RTE_LOG(ERR, VROUTER, "Failed to create flow - flow %u has unsupported L4"
     292             :                 " protocol %u\n", oflow->fe_index, oflow->fe->fe_key.flow_proto);
     293           0 :         return -ENOTSUP;
     294             :     }
     295           0 :     if (oflow->fe->fe_key.flow_dport) {
     296           0 :             itcpudp_mask.hdr.dst_port = -1;
     297           0 :             itcpudp_spec.hdr.dst_port = oflow->fe->fe_key.flow_dport;
     298             :     } else
     299             :         /* Handle fat flow */
     300           0 :         itcpudp_mask.hdr.dst_port = 0;
     301             : 
     302           0 :     if (oflow->fe->fe_key.flow_sport) {
     303           0 :         itcpudp_mask.hdr.src_port = -1;
     304           0 :         itcpudp_spec.hdr.src_port = oflow->fe->fe_key.flow_sport;
     305             :      } else
     306             :          /* Handle fat flow */
     307           0 :          itcpudp_mask.hdr.src_port = 0;
     308             : 
     309           0 :     RTE_LOG(DEBUG, VROUTER, "flow L4 tuples: %s src = %u dst = %u\n",
     310             :             oflow->fe->fe_key.flow_proto == VR_IP_PROTO_TCP ? "tcp" : "udp",
     311             :             ntohs(oflow->fe->fe_key.flow_sport),
     312             :             ntohs(oflow->fe->fe_key.flow_dport));
     313             : 
     314             :     /* Fill outer items */
     315           0 :     ipv4_spec.hdr.dst_addr = oflow->ip;
     316           0 :     switch (oflow->tunnel_type) {
     317           0 :     case NH_FLAG_TUNNEL_GRE:
     318             :     case NH_FLAG_TUNNEL_UDP_MPLS:
     319           0 :         pattern[MPLS].type = RTE_FLOW_ITEM_TYPE_MPLS;
     320           0 :         header = htonl(oflow->tunnel_tag << VR_MPLS_LABEL_SHIFT);
     321           0 :         mpls_spec.label_tc_s[0] = header & 0xff;
     322           0 :         mpls_spec.label_tc_s[1] = (header >> 8) & 0xff;
     323           0 :         mpls_spec.label_tc_s[2] = (header >> 16) & 0xff;
     324           0 :         pattern[VXLAN].type = RTE_FLOW_ITEM_TYPE_VOID;
     325           0 :         if (oflow->is_mpls_l2)
     326           0 :             pattern[IETH].type =  RTE_FLOW_ITEM_TYPE_ETH;
     327             :         else
     328           0 :             pattern[IETH].type = RTE_FLOW_ITEM_TYPE_VOID;
     329           0 :         if (oflow->tunnel_type == NH_FLAG_TUNNEL_GRE) {
     330           0 :             ipv4_spec.hdr.next_proto_id = VR_IP_PROTO_GRE;
     331           0 :             pattern[GRE].type = RTE_FLOW_ITEM_TYPE_GRE;
     332           0 :             pattern[UDP].type = RTE_FLOW_ITEM_TYPE_VOID;
     333           0 :             rss_action.level = 2; /* RSS on inner */
     334             :         } else {
     335           0 :             ipv4_spec.hdr.next_proto_id = VR_IP_PROTO_UDP;
     336           0 :             pattern[GRE].type = RTE_FLOW_ITEM_TYPE_VOID;
     337           0 :             pattern[UDP].type = RTE_FLOW_ITEM_TYPE_UDP;
     338           0 :             udp_spec.hdr.dst_port = RTE_BE16(VR_MPLS_OVER_UDP_NEW_DST_PORT);
     339           0 :             rss_action.level = 1; /* RSS on outer */
     340           0 :             rss_action.types = ETH_RSS_IPV4 |
     341             :                                ETH_RSS_FRAG_IPV4 |
     342             :                                ETH_RSS_NONFRAG_IPV4_OTHER |
     343             :                                ETH_RSS_NONFRAG_IPV4_UDP;
     344             :         }
     345           0 :         break;
     346           0 :     case NH_FLAG_TUNNEL_VXLAN:
     347           0 :         ipv4_spec.hdr.next_proto_id = VR_IP_PROTO_UDP;
     348           0 :         pattern[GRE].type = RTE_FLOW_ITEM_TYPE_VOID;
     349           0 :         pattern[UDP].type = RTE_FLOW_ITEM_TYPE_UDP;
     350           0 :         udp_spec.hdr.dst_port = RTE_BE16(VR_VXLAN_UDP_DST_PORT);
     351           0 :         pattern[MPLS].type = RTE_FLOW_ITEM_TYPE_VOID;
     352           0 :         pattern[VXLAN].type = RTE_FLOW_ITEM_TYPE_VXLAN;
     353           0 :         header = htonl(oflow->tunnel_tag << VR_VXLAN_VNID_SHIFT);
     354           0 :         vxlan_spec.vni[0] = header & 0xff;
     355           0 :         vxlan_spec.vni[1] = (header >> 8) & 0xff;
     356           0 :         vxlan_spec.vni[2] = (header >> 16) & 0xff;
     357           0 :         pattern[IETH].type = RTE_FLOW_ITEM_TYPE_ETH;
     358           0 :         rss_action.level = 1; /* RSS on outer */
     359           0 :         rss_action.types = ETH_RSS_IPV4 |
     360             :                            ETH_RSS_FRAG_IPV4 |
     361             :                            ETH_RSS_NONFRAG_IPV4_OTHER |
     362             :                            ETH_RSS_NONFRAG_IPV4_UDP;
     363           0 :         break;
     364           0 :     default:
     365           0 :         RTE_LOG(ERR, VROUTER, "Failed to create flow - flow %u has unsupported tunnel"
     366             :                 " protocol %u\n", oflow->fe_index, oflow->tunnel_type);
     367           0 :         return -ENOTSUP;
     368             :     }
     369             : 
     370             :     /* Create dpdk flow */
     371           0 :     rte_spinlock_lock(&ethdev->ethdev_lock);
     372           0 :     oflow->flow_handle = rte_flow_create(ethdev->ethdev_port_id, &attr, pattern,
     373             :                                          actions, &error);
     374           0 :     rte_spinlock_unlock(&ethdev->ethdev_lock);
     375           0 :     if (unlikely(oflow->flow_handle == NULL)) {
     376           0 :         RTE_LOG(ERR, VROUTER, "Failed to create flow - %s (%d)\n", error.message ?
     377             :                 error.message : "no error message", error.type);
     378           0 :         return -rte_errno;
     379             :     }
     380           0 :     return 0;
     381             : }
     382             : 
     383             : void
     384          30 : dpdk_offload_prepare(struct vr_packet *pkt, struct vr_forwarding_md *fmd)
     385             : {
     386          30 :     struct rte_mbuf *m = vr_dpdk_pkt_to_mbuf(pkt);
     387             :     struct vr_offload_flow *oflow;
     388             : 
     389          30 :     if (m->ol_flags & PKT_RX_FDIR_ID) {
     390           0 :         oflow = vr_offloads_flow_get(m->hash.fdir.hi);
     391           0 :         if (oflow) {
     392           0 :             pkt->vp_nh = oflow->nh;
     393           0 :             pkt->vp_flags |= VP_FLAG_TO_ME;
     394           0 :             fmd->fmd_fe = oflow->fe;
     395           0 :             fmd->fmd_flow_index = oflow->fe_index;
     396           0 :             fmd->fmd_label = oflow->tunnel_tag;
     397             :         }
     398             :     }
     399          30 : }