dpdk: Add support for Mellanox ConnectX-4 devices

Due to external library dependency support for Mellanox
devices is disabled. To enable it uncoment following line:

vpp_uses_dpdk_mlx5_pmd = yes

in build-data/platforms/vpp.mk and install OFED libraries.

Change-Id: I131d52b5d449a958349f31f9cc04311948f78b71
Signed-off-by: Damjan Marion <[email protected]>

build-data/packages/dpdk.mk

@@ -27,6 +27,11 @@ ifneq ($(DPDK_CRYPTO_PMD),)
 DPDK_MAKE_ARGS += DPDK_CRYPTO_PMD=y
 endif
 
+DPDK_MLX5_PMD=$(strip $($(PLATFORM)_uses_dpdk_mlx5_pmd))
+ifneq ($(DPDK_MLX5_PMD),)
+DPDK_MAKE_ARGS += DPDK_MLX5_PMD=y
+endif
+
 DPDK_PLATFORM_TARGET=$(strip $($(PLATFORM)_dpdk_target))
 ifneq ($(DPDK_PLATFORM_TARGET),)
 DPDK_MAKE_ARGS += DPDK_TARGET=$(DPDK_PLATFORM_TARGET)

build-data/packages/vnet.mk

@@ -41,4 +41,7 @@ endif
 ifeq ($($(PLATFORM)_uses_dpdk_cryptodev),yes)
 vnet_configure_args += --with-dpdk-crypto
 endif
+ifeq ($($(PLATFORM)_uses_dpdk_mlx5_pmd),yes)
+vnet_configure_args += --with-dpdk-mlx5-pmd
+endif
 endif

build-data/packages/vpp.mk

@@ -48,4 +48,7 @@ endif
 ifeq ($($(PLATFORM)_uses_dpdk_cryptodev),yes)
 vpp_configure_args += --with-dpdk-crypto
 endif
+ifeq ($($(PLATFORM)_uses_dpdk_mlx5_pmd),yes)
+vpp_configure_args += --with-dpdk-mlx5-pmd
+endif
 endif

build-data/platforms/vpp.mk

@@ -42,6 +42,7 @@ plugins_configure_args_vpp = --with-dpdk
 
 # DPDK configuration parameters
 # vpp_uses_dpdk_cryptodev = yes
+# vpp_uses_dpdk_mlx5_pmd = yes
 # vpp_uses_external_dpdk = yes
 # vpp_dpdk_inc_dir = /usr/include/dpdk
 # vpp_dpdk_lib_dir = /usr/lib

dpdk/Makefile

@@ -21,7 +21,8 @@ DPDK_DOWNLOAD_DIR     ?= $(HOME)/Downloads
 DPDK_MARCH            ?= native
 DPDK_TUNE             ?= generic
 DPDK_DEBUG            ?= n
-DPDK_CRYPTO_PMD        ?= n
+DPDK_CRYPTO_PMD       ?= n
+DPDK_MLX5_PMD         ?= n
 
 B := $(DPDK_BUILD_DIR)
 I := $(DPDK_INSTALL_DIR)
@@ -122,6 +123,7 @@ $(B)/custom-config: $(B)/.patch.ok Makefile
 	$(call set,RTE_LIBRTE_PMD_AESNI_MB,$(DPDK_CRYPTO_PMD))
 	$(call set,RTE_LIBRTE_PMD_AESNI_GCM,$(DPDK_CRYPTO_PMD))
 	$(call set,RTE_LIBRTE_PMD_QAT,$(DPDK_CRYPTO_PMD))
+	$(call set,RTE_LIBRTE_MLX5_PMD,$(DPDK_MLX5_PMD))
 	@# not needed
 	$(call set,RTE_LIBRTE_TIMER,n)
 	$(call set,RTE_LIBRTE_CFGFILE,n)

vnet/vnet/devices/dpdk/dpdk.h

@@ -83,6 +83,7 @@ extern vlib_node_registration_t handoff_dispatch_node;
   _ ("rte_bond_pmd", BOND)        \
   _ ("net_fm10k", FM10K)          \
   _ ("net_cxgbe", CXGBE)          \
+  _ ("net_mlx5", MLX5)            \
   _ ("net_dpaa2", DPAA2)
 #else
 #define foreach_dpdk_pmd          \
@@ -118,6 +119,7 @@ typedef enum
   VNET_DPDK_PORT_TYPE_ETH_1G,
   VNET_DPDK_PORT_TYPE_ETH_10G,
   VNET_DPDK_PORT_TYPE_ETH_40G,
+  VNET_DPDK_PORT_TYPE_ETH_100G,
   VNET_DPDK_PORT_TYPE_ETH_BOND,
   VNET_DPDK_PORT_TYPE_ETH_SWITCH,
   VNET_DPDK_PORT_TYPE_AF_PACKET,

vnet/vnet/devices/dpdk/dpdk_priv.h

@@ -13,15 +13,10 @@
  * limitations under the License.
  */
 
-#define DPDK_NB_RX_DESC_DEFAULT   512
-#define DPDK_NB_TX_DESC_DEFAULT   512
+#define DPDK_NB_RX_DESC_DEFAULT   1024
+#define DPDK_NB_TX_DESC_DEFAULT   1024
 #define DPDK_NB_RX_DESC_VIRTIO    256
 #define DPDK_NB_TX_DESC_VIRTIO    256
-#define DPDK_NB_RX_DESC_10GE    1024
-#define DPDK_NB_TX_DESC_10GE    1024
-#define DPDK_NB_RX_DESC_40GE    1024
-#define DPDK_NB_TX_DESC_40GE    1024
-#define DPDK_NB_RX_DESC_ENIC    1024
 
 #define I40E_DEV_ID_SFP_XL710           0x1572
 #define I40E_DEV_ID_QSFP_A              0x1583

vnet/vnet/devices/dpdk/format.c

@@ -184,6 +184,10 @@ format_dpdk_device_name (u8 * s, va_list * args)
       device_name = "FortyGigabitEthernet";
       break;
 
+    case VNET_DPDK_PORT_TYPE_ETH_100G:
+      device_name = "HundredGigabitEthernet";
+      break;
+
     case VNET_DPDK_PORT_TYPE_ETH_BOND:
       return format (s, "BondEthernet%d", dm->devices[i].device_index);
 
@@ -268,6 +272,10 @@ format_dpdk_device_type (u8 * s, va_list * args)
       dev_type = "Chelsio T4/T5";
       break;
 
+    case VNET_DPDK_PMD_MLX5:
+      dev_type = "Mellanox ConnectX-4 Family";
+      break;
+
     case VNET_DPDK_PMD_VMXNET3:
       dev_type = "VMware VMXNET3";
       break;

vnet/vnet/devices/dpdk/init.c

@@ -293,11 +293,6 @@ dpdk_lib_init (dpdk_main_t * dm)
   vec_validate_aligned (dm->hqos_threads, tm->n_vlib_mains - 1,
 			CLIB_CACHE_LINE_BYTES);
 
-#ifdef NETMAP
-  if (rte_netmap_probe () < 0)
-    return clib_error_return (0, "rte netmap probe failed");
-#endif
-
   nports = rte_eth_dev_count ();
   if (nports < 1)
     {
@@ -448,6 +443,9 @@ dpdk_lib_init (dpdk_main_t * dm)
 	    else
 	    xd->pmd = VNET_DPDK_PMD_UNKNOWN;
 
+	  xd->port_type = VNET_DPDK_PORT_TYPE_UNKNOWN;
+	  xd->nb_rx_desc = DPDK_NB_RX_DESC_DEFAULT;
+	  xd->nb_tx_desc = DPDK_NB_TX_DESC_DEFAULT;
 
 	  switch (xd->pmd)
 	    {
@@ -463,8 +461,6 @@ dpdk_lib_init (dpdk_main_t * dm)
 	    case VNET_DPDK_PMD_IXGBEVF:
 	    case VNET_DPDK_PMD_THUNDERX:
 	      xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G;
-	      xd->nb_rx_desc = DPDK_NB_RX_DESC_10GE;
-	      xd->nb_tx_desc = DPDK_NB_TX_DESC_10GE;
 	      break;
 	    case VNET_DPDK_PMD_DPAA2:
 	      xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G;
@@ -474,26 +470,17 @@ dpdk_lib_init (dpdk_main_t * dm)
 	    case VNET_DPDK_PMD_ENIC:
 	      rte_eth_link_get_nowait (i, &l);
 	      xd->flags |= DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE;
-	      xd->nb_rx_desc = DPDK_NB_RX_DESC_ENIC;
 	      if (l.link_speed == 40000)
-		{
-		  xd->port_type = VNET_DPDK_PORT_TYPE_ETH_40G;
-		  xd->nb_tx_desc = DPDK_NB_TX_DESC_40GE;
-		}
+		xd->port_type = VNET_DPDK_PORT_TYPE_ETH_40G;
 	      else
-		{
-		  xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G;
-		  xd->nb_tx_desc = DPDK_NB_TX_DESC_10GE;
-		}
+		xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G;
 	      break;
 
 	      /* Intel Fortville */
 	    case VNET_DPDK_PMD_I40E:
 	    case VNET_DPDK_PMD_I40EVF:
 	      xd->flags |= DPDK_DEVICE_FLAG_PMD_SUPPORTS_PTYPE;
 	      xd->port_type = VNET_DPDK_PORT_TYPE_ETH_40G;
-	      xd->nb_rx_desc = DPDK_NB_RX_DESC_40GE;
-	      xd->nb_tx_desc = DPDK_NB_TX_DESC_40GE;
 
 	      switch (dev_info.pci_dev->id.device_id)
 		{
@@ -521,27 +508,74 @@ dpdk_lib_init (dpdk_main_t * dm)
 		{
 		case 0x540d:	/* T580-CR */
 		case 0x5410:	/* T580-LP-cr */
-		  xd->nb_rx_desc = DPDK_NB_RX_DESC_40GE;
-		  xd->nb_tx_desc = DPDK_NB_TX_DESC_40GE;
 		  xd->port_type = VNET_DPDK_PORT_TYPE_ETH_40G;
 		  break;
 		case 0x5403:	/* T540-CR */
-		  xd->nb_rx_desc = DPDK_NB_RX_DESC_10GE;
-		  xd->nb_tx_desc = DPDK_NB_TX_DESC_10GE;
 		  xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G;
 		  break;
 		default:
-		  xd->nb_rx_desc = DPDK_NB_RX_DESC_10GE;
-		  xd->nb_tx_desc = DPDK_NB_TX_DESC_10GE;
 		  xd->port_type = VNET_DPDK_PORT_TYPE_UNKNOWN;
 		}
 	      break;
 
+	    case VNET_DPDK_PMD_MLX5:
+	      {
+		char *pn_100g[] = { "MCX415A-CCAT", "MCX416A-CCAT", 0 };
+		char *pn_40g[] = { "MCX413A-BCAT", "MCX414A-BCAT",
+		  "MCX415A-BCAT", "MCX416A-BCAT", "MCX4131A-BCAT", 0
+		};
+		char *pn_10g[] = { "MCX4111A-XCAT", "MCX4121A-XCAT", 0 };
+
+		vlib_pci_device_t *pd = vlib_get_pci_device (&pci_addr);
+		u8 *pn = 0;
+		char **c;
+		int found = 0;
+		pn = format (0, "%U%c",
+			     format_vlib_pci_vpd, pd->vpd_r, "PN", 0);
+
+		if (!pn)
+		  break;
+
+		c = pn_100g;
+		while (!found && c[0])
+		  {
+		    if (strncmp ((char *) pn, c[0], strlen (c[0])) == 0)
+		      {
+			xd->port_type = VNET_DPDK_PORT_TYPE_ETH_100G;
+			break;
+		      }
+		    c++;
+		  }
+
+		c = pn_40g;
+		while (!found && c[0])
+		  {
+		    if (strncmp ((char *) pn, c[0], strlen (c[0])) == 0)
+		      {
+			xd->port_type = VNET_DPDK_PORT_TYPE_ETH_40G;
+			break;
+		      }
+		    c++;
+		  }
+
+		c = pn_10g;
+		while (!found && c[0])
+		  {
+		    if (strncmp ((char *) pn, c[0], strlen (c[0])) == 0)
+		      {
+			xd->port_type = VNET_DPDK_PORT_TYPE_ETH_10G;
+			break;
+		      }
+		    c++;
+		  }
+
+		vec_free (pn);
+	      }
+
+	      break;
 	      /* Intel Red Rock Canyon */
 	    case VNET_DPDK_PMD_FM10K:
 	      xd->port_type = VNET_DPDK_PORT_TYPE_ETH_SWITCH;
-	      xd->nb_rx_desc = DPDK_NB_RX_DESC_40GE;
-	      xd->nb_tx_desc = DPDK_NB_TX_DESC_40GE;
 	      break;
 
 	      /* virtio */

vpp/Makefile.am

@@ -121,6 +121,9 @@ vpp_LDFLAGS += -Wl,--no-whole-archive
 if WITH_DPDK_CRYPTO
 vpp_LDADD += -L$(AESNI_MULTI_BUFFER_LIB_PATH) -lIPSec_MB
 endif
+if WITH_DPDK_MLX5_PMD
+vpp_LDFLAGS += -libverbs -lmlx5 -lnuma
+endif
 endif
 
 vpp_LDADD += -lvppinfra -lm -lpthread -ldl

vpp/configure.ac

@@ -14,6 +14,11 @@ AC_ARG_WITH(dpdk_crypto,
             [with_dpdk_crypto=1],
             [with_dpdk_crypto=0])
 
+AC_ARG_WITH(dpdk_mlx5_pmd,
+            AC_HELP_STRING([--with-dpdk-mlx5-pmd],[Use Mellanox ConnetxX-4 DPDK PMD]),
+            [with_dpdk_mlx5_pmd=1],
+            [with_dpdk_mlx5_pmd=0])
+
 AC_ARG_ENABLE(dpdk-shared,
               AC_HELP_STRING([--enable-dpdk-shared],[Link with DPDK shared lib]),
               [enable_dpdk_shared=1],
@@ -44,7 +49,7 @@ AM_CONDITIONAL(ENABLE_DPDK_SHARED, test "$enable_dpdk_shared" = "1")
 AC_SUBST(DPDK,["-DDPDK=${with_dpdk} -DDPDK_SHARED_LIB=${enable_dpdk_shared}"])
 AM_CONDITIONAL(WITH_DPDK_CRYPTO, test "$with_dpdk_crypto" = "1")
 AC_SUBST(DPDK_CRYPTO,[-DDPDK_CRYPTO=${with_dpdk_crypto}])
-
+AM_CONDITIONAL(WITH_DPDK_MLX5_PMD, test "$with_dpdk_mlx5_pmd" = "1")
 
 AM_COND_IF(
 	[ENABLE_DPDK_SHARED],