The Tcpdump Group git mirrors - libpcap/blob - pcap-dpdk.c

   1 /*
   2  * Copyright (C) 2018 jingle YANG. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  *
   8  *   1. Redistributions of source code must retain the above copyright
   9  *      notice, this list of conditions and the following disclaimer.
  10  *   2. Redistributions in binary form must reproduce the above copyright
  11  *      notice, this list of conditions and the following disclaimer in the
  12  *      documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 /*
  28 Date: Dec 16, 2018
  29
  30 Description:
  31 1. Pcap-dpdk provides libpcap the ability to use DPDK with the device name as dpdk:{portid}, such as dpdk:0.
  32 2. DPDK is a set of libraries and drivers for fast packet processing. (https://www.dpdk.org/)
  33 3. The testprogs/capturetest provides 6.4Gbps/800,000 pps on Intel 10-Gigabit X540-AT2 with DPDK 18.11.
  34
  35 Limitations:
  36 1. DPDK support will be on if DPDK is available. Please set DIR for --with-dpdk[=DIR] with ./configure or -DDPDK_DIR[=DIR] with cmake if DPDK is installed manually.
  37 2. Only support link libdpdk.so dynamicly, because the libdpdk.a will not work correctly.
  38 3. Only support read operation, and packet injection has not been supported yet.
  39
  40 Usage:
  41 1. Compile DPDK as shared library and install.(https://github.com/DPDK/dpdk.git)
  42
  43 You shall modify the file $RTE_SDK/$RTE_TARGET/.config and set:
  44 CONFIG_RTE_BUILD_SHARED_LIB=y
  45 By the following command:
  46 sed -i 's/CONFIG_RTE_BUILD_SHARED_LIB=n/CONFIG_RTE_BUILD_SHARED_LIB=y/' $RTE_SDK/$RTE_TARGET/.config
  47
  48 2. Launch l2fwd that is one of DPDK examples correctly, and get device information.
  49
  50 You shall learn how to bind nic with DPDK-compatible driver by $RTE_SDK/usertools/dpdk-devbind.py, such as igb_uio.
  51 And enable hugepages by dpdk-setup.sh
  52
  53 Then launch the l2fwd with dynamic dirver support. For example:
  54 $RTE_SDK/examples/l2fwd/$RTE_TARGET/l2fwd -dlibrte_pmd_e1000.so -dlibrte_pmd_ixgbe.so -dlibrte_mempool_ring.so -- -p 0x1
  55
  56 3. Compile libpcap with dpdk options.
  57
  58 If DPDK has not been found automatically, you shall export DPDK envionment variable which are used for compiling DPDK. And then pass $RTE_SDK/$RTE_TARGET to --with-dpdk or -DDPDK_DIR
  59
  60 export RTE_SDK={your DPDK base directory}
  61 export RTE_TARGET={your target name}
  62
  63 3.1 With configure
  64
  65 ./configure --with-dpdk=$RTE_SDK/$RTE_TARGET && make -s all && make -s testprogs && make install
  66
  67 3.2 With cmake
  68
  69 mkdir -p build && cd build && cmake -DDPDK_DIR=$RTE_SDK/$RTE_TARGET ../ && make -s all && make -s testprogs && make install
  70
  71 4. Link your own program with libpcap, and use DPDK with the device name as dpdk:{portid}, such as dpdk:0.
  72 And you shall set DPDK configure options by environment variable DPDK_CFG
  73 For example, the testprogs/capturetest could be lanched by:
  74
  75 env DPDK_CFG="--log-level=debug -l0 -dlibrte_pmd_e1000.so -dlibrte_pmd_ixgbe.so -dlibrte_mempool_ring.so" ./capturetest -i dpdk:0
  76 */
  77
  78 #ifdef HAVE_CONFIG_H
  79 #include <config.h>
  80 #endif
  81
  82 #include <ctype.h>
  83 #include <errno.h>
  84 #include <netdb.h>
  85 #include <stdio.h>
  86 #include <stdlib.h>
  87 #include <string.h>
  88 #include <unistd.h>
  89 #include <time.h>
  90
  91 #include <sys/time.h>
  92
  93 //header for calling dpdk
  94 #include <rte_common.h>
  95 #include <rte_log.h>
  96 #include <rte_malloc.h>
  97 #include <rte_memory.h>
  98 #include <rte_eal.h>
  99 #include <rte_launch.h>
 100 #include <rte_atomic.h>
 101 #include <rte_cycles.h>
 102 #include <rte_lcore.h>
 103 #include <rte_per_lcore.h>
 104 #include <rte_branch_prediction.h>
 105 #include <rte_interrupts.h>
 106 #include <rte_random.h>
 107 #include <rte_debug.h>
 108 #include <rte_ether.h>
 109 #include <rte_ethdev.h>
 110 #include <rte_mempool.h>
 111 #include <rte_mbuf.h>
 112 #include <rte_bus.h>
 113
 114 #include "pcap-int.h"
 115 #include "pcap-dpdk.h"
 116
 117 #define DPDK_DEF_LOG_LEV RTE_LOG_ERR
 118 static int is_dpdk_pre_inited=0;
 119 #define DPDK_LIB_NAME "libpcap_dpdk"
 120 #define DPDK_DESC "Data Plane Development Kit (DPDK) Interface"
 121 #define DPDK_ERR_PERM_MSG "permission denied, DPDK needs root permission"
 122 #define DPDK_ARGC_MAX 64
 123 #define DPDK_CFG_MAX_LEN 1024
 124 #define DPDK_DEV_NAME_MAX 32
 125 #define DPDK_DEV_DESC_MAX 512
 126 #define DPDK_CFG_ENV_NAME "DPDK_CFG"
 127 #define DPDK_DEF_MIN_SLEEP_MS 1
 128 static char dpdk_cfg_buf[DPDK_CFG_MAX_LEN];
 129 #define DPDK_MAC_ADDR_SIZE 32
 130 #define DPDK_DEF_MAC_ADDR "00:00:00:00:00:00"
 131 #define DPDK_PCI_ADDR_SIZE 16
 132 #define DPDK_DEF_CFG "--log-level=error -l0 -dlibrte_pmd_e1000.so -dlibrte_pmd_ixgbe.so -dlibrte_mempool_ring.so"
 133 #define DPDK_PREFIX "dpdk:"
 134 #define DPDK_PORTID_MAX 65535U
 135 #define MBUF_POOL_NAME "mbuf_pool"
 136 #define DPDK_TX_BUF_NAME "tx_buffer"
 137 //The number of elements in the mbuf pool.
 138 #define DPDK_NB_MBUFS 8192U
 139 #define MEMPOOL_CACHE_SIZE 256
 140 #define MAX_PKT_BURST 32
 141 // Configurable number of RX/TX ring descriptors
 142 #define RTE_TEST_RX_DESC_DEFAULT 1024
 143 #define RTE_TEST_TX_DESC_DEFAULT 1024
 144
 145 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
 146 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
 147
 148 #define RTE_ETH_PCAP_SNAPLEN ETHER_MAX_JUMBO_FRAME_LEN
 149
 150 static struct rte_eth_dev_tx_buffer *tx_buffer;
 151
 152 struct dpdk_ts_helper{
 153         struct timeval start_time;
 154         uint64_t start_cycles;
 155         uint64_t hz;
 156 };
 157 struct pcap_dpdk{
 158         pcap_t * orig;
 159         uint16_t portid; // portid of DPDK
 160         int must_clear_promisc;
 161         uint64_t bpf_drop;
 162         int nonblock;
 163         struct timeval required_select_timeout;
 164         struct timeval prev_ts;
 165         struct rte_eth_stats prev_stats;
 166         struct timeval curr_ts;
 167         struct rte_eth_stats curr_stats;
 168         uint64_t pps;
 169         uint64_t bps;
 170         struct rte_mempool * pktmbuf_pool;
 171         struct dpdk_ts_helper ts_helper;
 172         struct ether_addr eth_addr;
 173         char mac_addr[DPDK_MAC_ADDR_SIZE];
 174         char pci_addr[DPDK_PCI_ADDR_SIZE];
 175         unsigned char pcap_tmp_buf[RTE_ETH_PCAP_SNAPLEN];
 176 };
 177
 178 static struct rte_eth_conf port_conf = {
 179         .rxmode = {
 180                 .split_hdr_size = 0,
 181         },
 182         .txmode = {
 183                 .mq_mode = ETH_MQ_TX_NONE,
 184         },
 185 };
 186
 187 static int dpdk_init_timer(struct pcap_dpdk *pd){
 188         gettimeofday(&(pd->ts_helper.start_time),NULL);
 189         pd->ts_helper.start_cycles = rte_get_timer_cycles();
 190         pd->ts_helper.hz = rte_get_timer_hz();
 191         if (pd->ts_helper.hz == 0){
 192                 return -1;
 193         }
 194         return 0;
 195 }
 196 static inline void calculate_timestamp(struct dpdk_ts_helper *helper,struct timeval *ts)
 197 {
 198         uint64_t cycles;
 199         // delta
 200         struct timeval cur_time;
 201         cycles = rte_get_timer_cycles() - helper->start_cycles;
 202         cur_time.tv_sec = (time_t)(cycles/helper->hz);
 203         cur_time.tv_usec = (suseconds_t)((cycles%helper->hz)*1e6/helper->hz);
 204         timeradd(&(helper->start_time), &cur_time, ts);
 205 }
 206
 207 static uint32_t dpdk_gather_data(unsigned char *data, int len, struct rte_mbuf *mbuf)
 208 {
 209         uint32_t total_len = 0;
 210         while (mbuf && (total_len+mbuf->data_len) < len ){
 211                 rte_memcpy(data+total_len, rte_pktmbuf_mtod(mbuf,void *),mbuf->data_len);
 212                 total_len+=mbuf->data_len;
 213                 mbuf=mbuf->next;
 214         }
 215         return total_len;
 216 }
 217
 218
 219 static int dpdk_read_with_timeout(pcap_t *p, uint16_t portid, uint16_t queueid,struct rte_mbuf **pkts_burst, const uint16_t burst_cnt){
 220         struct pcap_dpdk *pd = (struct pcap_dpdk*)(p->priv);
 221         int nb_rx = 0;
 222         int timeout_ms = p->opt.timeout;
 223         int sleep_ms = 0;
 224         if (pd->nonblock){
 225                 // In non-blocking mode, just read once, no mater how many packets are captured.
 226                 nb_rx = (int)rte_eth_rx_burst(pd->portid, 0, pkts_burst, burst_cnt);
 227         }else{
 228                 // In blocking mode, read many times until packets are captured or timeout or break_loop is setted.
 229                 // if timeout_ms == 0, it may be blocked forever.
 230                 while (timeout_ms == 0 || sleep_ms < timeout_ms){
 231                         nb_rx = (int)rte_eth_rx_burst(pd->portid, 0, pkts_burst, burst_cnt);
 232                         if (nb_rx){ // got packets within timeout_ms
 233                                 break;
 234                         }else{ // no packet arrives at this round.
 235                                 if (p->break_loop){
 236                                         break;
 237                                 }
 238                                 // sleep for a very short while.
 239                                 // block sleep is the only choice, since usleep() will impact performance dramatically.
 240                                 rte_delay_us_block(DPDK_DEF_MIN_SLEEP_MS*1000);
 241                                 sleep_ms += DPDK_DEF_MIN_SLEEP_MS;
 242                         }
 243                 }
 244         }
 245         return nb_rx;
 246 }
 247
 248 static int pcap_dpdk_dispatch(pcap_t *p, int max_cnt, pcap_handler cb, u_char *cb_arg)
 249 {
 250         struct pcap_dpdk *pd = (struct pcap_dpdk*)(p->priv);
 251         int burst_cnt = 0;
 252         int nb_rx = 0;
 253         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
 254         struct rte_mbuf *m;
 255         struct pcap_pkthdr pcap_header;
 256         uint16_t portid = pd->portid;
 257         // In DPDK, pkt_len is sum of lengths for all segments. And data_len is for one segment
 258         uint32_t pkt_len = 0;
 259         int caplen = 0;
 260         u_char *bp = NULL;
 261         int i=0;
 262         unsigned int gather_len =0;
 263         int pkt_cnt = 0;
 264         int is_accepted=0;
 265         u_char *large_buffer=NULL;
 266         int timeout_ms = p->opt.timeout;
 267
 268         if ( !PACKET_COUNT_IS_UNLIMITED(max_cnt) && max_cnt < MAX_PKT_BURST){
 269                 burst_cnt = max_cnt;
 270         }else{
 271                 burst_cnt = MAX_PKT_BURST;
 272         }
 273
 274         while( PACKET_COUNT_IS_UNLIMITED(max_cnt) || pkt_cnt < max_cnt){
 275                 if (p->break_loop){
 276                         p->break_loop = 0;
 277                         return PCAP_ERROR_BREAK;
 278                 }
 279                 // read once in non-blocking mode, or try many times waiting for timeout_ms.
 280                 // if timeout_ms == 0, it will be blocked until one packet arrives or break_loop is setted.
 281                 nb_rx = dpdk_read_with_timeout(p, portid, 0, pkts_burst, burst_cnt);
 282                 if (nb_rx == 0){
 283                         if (pd->nonblock){
 284                                 RTE_LOG(DEBUG, USER1, "dpdk: no packets available in non-blocking mode.\n");
 285                         }else{
 286                                 if (p->break_loop){
 287                                         RTE_LOG(DEBUG, USER1, "dpdk: no packets available and break_loop is setted in blocking mode.\n");
 288                                         p->break_loop = 0;
 289                                         return PCAP_ERROR_BREAK;
 290
 291                                 }
 292                                 RTE_LOG(DEBUG, USER1, "dpdk: no packets available for timeout %d ms in blocking mode.\n", timeout_ms);
 293                         }
 294                         // break if dpdk reads 0 packet, no matter in blocking(timeout) or non-blocking mode.
 295                         break;
 296                 }
 297                 pkt_cnt += nb_rx;
 298                 for ( i = 0; i < nb_rx; i++) {
 299                         m = pkts_burst[i];
 300                         calculate_timestamp(&(pd->ts_helper),&(pcap_header.ts));
 301                         pkt_len = rte_pktmbuf_pkt_len(m);
 302                         // caplen = min(pkt_len, p->snapshot);
 303                         // caplen will not be changed, no matter how long the rte_pktmbuf
 304                         caplen = pkt_len < p->snapshot ? pkt_len: p->snapshot;
 305                         pcap_header.caplen = caplen;
 306                         pcap_header.len = pkt_len;
 307                         // volatile prefetch
 308                         rte_prefetch0(rte_pktmbuf_mtod(m, void *));
 309                         bp = NULL;
 310                         if (m->nb_segs == 1)
 311                         {
 312                                 bp = rte_pktmbuf_mtod(m, u_char *);
 313                         }else{
 314                                 // use fast buffer pcap_tmp_buf if pkt_len is small, no need to call malloc and free
 315                                 if ( pkt_len <= ETHER_MAX_JUMBO_FRAME_LEN)
 316                                 {
 317                                         gather_len = dpdk_gather_data(pd->pcap_tmp_buf, RTE_ETH_PCAP_SNAPLEN, m);
 318                                         bp = pd->pcap_tmp_buf;
 319                                 }else{
 320                                         // need call free later
 321                                         large_buffer = (u_char *)malloc(caplen*sizeof(u_char));
 322                                         gather_len = dpdk_gather_data(large_buffer, caplen, m);
 323                                         bp = large_buffer;
 324                                 }
 325
 326                         }
 327                         if (bp){
 328                                 if (p->fcode.bf_insns==NULL || pcap_filter(p->fcode.bf_insns, bp, pcap_header.len, pcap_header.caplen)){
 329                                         cb(cb_arg, &pcap_header, bp);
 330                                 }else{
 331                                         pd->bpf_drop++;
 332                                 }
 333                         }
 334                         //free all pktmbuf
 335                         rte_pktmbuf_free(m);
 336                         if (large_buffer){
 337                                 free(large_buffer);
 338                                 large_buffer=NULL;
 339                         }
 340                 }
 341         }
 342         return pkt_cnt;
 343 }
 344
 345 static int pcap_dpdk_inject(pcap_t *p, const void *buf _U_, int size _U_)
 346 {
 347         //not implemented yet
 348         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 349                 errno, "dpdk error: Inject function has not been implemented yet");
 350         return PCAP_ERROR;
 351 }
 352
 353 static void pcap_dpdk_close(pcap_t *p)
 354 {
 355         struct pcap_dpdk *pd = p->priv;
 356         if (pd==NULL)
 357         {
 358                 return;
 359         }
 360         if (pd->must_clear_promisc)
 361         {
 362                 rte_eth_promiscuous_disable(pd->portid);
 363         }
 364         rte_eth_dev_stop(pd->portid);
 365         rte_eth_dev_close(pd->portid);
 366         pcap_cleanup_live_common(p);
 367 }
 368
 369 static void nic_stats_display(struct pcap_dpdk *pd)
 370 {
 371         uint16_t portid = pd->portid;
 372         struct rte_eth_stats stats;
 373         rte_eth_stats_get(portid, &stats);
 374         RTE_LOG(INFO,USER1, "portid:%d, RX-packets: %-10"PRIu64"  RX-errors:  %-10"PRIu64
 375                "  RX-bytes:  %-10"PRIu64"  RX-Imissed:  %-10"PRIu64"\n", portid, stats.ipackets, stats.ierrors,
 376                stats.ibytes,stats.imissed);
 377         RTE_LOG(INFO,USER1, "portid:%d, RX-PPS: %-10"PRIu64" RX-Mbps: %.2lf\n", portid, pd->pps, pd->bps/1e6f );
 378 }
 379
 380 static int pcap_dpdk_stats(pcap_t *p, struct pcap_stat *ps)
 381 {
 382         struct pcap_dpdk *pd = p->priv;
 383         calculate_timestamp(&(pd->ts_helper), &(pd->curr_ts));
 384         rte_eth_stats_get(pd->portid,&(pd->curr_stats));
 385         if (ps){
 386                 ps->ps_recv = pd->curr_stats.ipackets;
 387                 ps->ps_drop = pd->curr_stats.ierrors;
 388                 ps->ps_drop += pd->bpf_drop;
 389                 ps->ps_ifdrop = pd->curr_stats.imissed;
 390         }
 391         uint64_t delta_pkt = pd->curr_stats.ipackets - pd->prev_stats.ipackets;
 392         struct timeval delta_tm;
 393         timersub(&(pd->curr_ts),&(pd->prev_ts), &delta_tm);
 394         uint64_t delta_usec = delta_tm.tv_sec*1e6+delta_tm.tv_usec;
 395         uint64_t delta_bit = (pd->curr_stats.ibytes-pd->prev_stats.ibytes)*8;
 396         RTE_LOG(DEBUG, USER1, "delta_usec: %-10"PRIu64" delta_pkt: %-10"PRIu64" delta_bit: %-10"PRIu64"\n", delta_usec, delta_pkt, delta_bit);
 397         pd->pps = (uint64_t)(delta_pkt*1e6f/delta_usec);
 398         pd->bps = (uint64_t)(delta_bit*1e6f/delta_usec);
 399         nic_stats_display(pd);
 400         pd->prev_stats = pd->curr_stats;
 401         pd->prev_ts = pd->curr_ts;
 402         return 0;
 403 }
 404
 405 static int pcap_dpdk_setnonblock(pcap_t *p, int nonblock){
 406         struct pcap_dpdk *pd = (struct pcap_dpdk*)(p->priv);
 407         pd->nonblock = nonblock;
 408         return 0;
 409 }
 410
 411 static int pcap_dpdk_getnonblock(pcap_t *p){
 412         struct pcap_dpdk *pd = (struct pcap_dpdk*)(p->priv);
 413         return pd->nonblock;
 414 }
 415 static int check_link_status(uint16_t portid, struct rte_eth_link *plink)
 416 {
 417         // wait up to 9 seconds to get link status
 418         rte_eth_link_get(portid, plink);
 419         return plink->link_status == ETH_LINK_UP;
 420 }
 421 static void eth_addr_str(struct ether_addr *addrp, char* mac_str, int len)
 422 {
 423         int offset=0;
 424         if (addrp == NULL){
 425                 pcap_snprintf(mac_str, len-1, DPDK_DEF_MAC_ADDR);
 426                 return;
 427         }
 428         for (int i=0; i<6; i++)
 429         {
 430                 if (offset >= len)
 431                 { // buffer overflow
 432                         return;
 433                 }
 434                 if (i==0)
 435                 {
 436                         pcap_snprintf(mac_str+offset, len-1-offset, "%02X",addrp->addr_bytes[i]);
 437                         offset+=2; // FF
 438                 }else{
 439                         pcap_snprintf(mac_str+offset, len-1-offset, ":%02X", addrp->addr_bytes[i]);
 440                         offset+=3; // :FF
 441                 }
 442         }
 443         return;
 444 }
 445 // return portid by device name, otherwise return -1
 446 static uint16_t portid_by_device(char * device)
 447 {
 448         uint16_t ret = DPDK_PORTID_MAX;
 449         int len = strlen(device);
 450         int prefix_len = strlen(DPDK_PREFIX);
 451         unsigned long ret_ul = 0L;
 452         char *pEnd;
 453         if (len<=prefix_len || strncmp(device, DPDK_PREFIX, prefix_len)) // check prefix dpdk:
 454         {
 455                 return ret;
 456         }
 457         //check all chars are digital
 458         for (int i=prefix_len; device[i]; i++){
 459                 if (device[i]<'0' || device[i]>'9'){
 460                         return ret;
 461                 }
 462         }
 463         ret_ul = strtoul(&(device[prefix_len]), &pEnd, 10);
 464         if (pEnd == &(device[prefix_len]) || *pEnd != '\0'){
 465                 return ret;
 466         }
 467         // too large for portid
 468         if (ret_ul >= DPDK_PORTID_MAX){
 469                 return ret;
 470         }
 471         ret = (uint16_t)ret_ul;
 472         return ret;
 473 }
 474
 475 static int parse_dpdk_cfg(char* dpdk_cfg,char** dargv)
 476 {
 477         int cnt=0;
 478         memset(dargv,0,sizeof(dargv[0])*DPDK_ARGC_MAX);
 479         //current process name
 480         int skip_space = 1;
 481         int i=0;
 482         RTE_LOG(INFO, USER1,"dpdk cfg: %s\n",dpdk_cfg);
 483         // find first non space char
 484         // The last opt is NULL
 485         for (i=0;dpdk_cfg[i] && cnt<DPDK_ARGC_MAX-1;i++){
 486                 if (skip_space && dpdk_cfg[i]!=' '){ // not space
 487                         skip_space=!skip_space; // skip normal char
 488                         dargv[cnt++] = dpdk_cfg+i;
 489                 }
 490                 if (!skip_space && dpdk_cfg[i]==' '){ // fint a space
 491                         dpdk_cfg[i]=0x00; // end of this opt
 492                         skip_space=!skip_space; // skip space char
 493                 }
 494         }
 495         dargv[cnt]=NULL;
 496         return cnt;
 497 }
 498
 499 // only called once
 500 static int dpdk_pre_init(char * ebuf)
 501 {
 502         int dargv_cnt=0;
 503         char *dargv[DPDK_ARGC_MAX];
 504         char *ptr_dpdk_cfg = NULL;
 505         int ret = PCAP_ERROR;
 506         // globale var
 507         if (is_dpdk_pre_inited)
 508         {
 509                 // already inited
 510                 return 0;
 511         }
 512         // check for root permission
 513         if( geteuid() != 0)
 514         {
 515                 RTE_LOG(ERR, USER1, "%s\n", DPDK_ERR_PERM_MSG);
 516                 pcap_fmt_errmsg_for_errno(ebuf, PCAP_ERRBUF_SIZE,
 517                             errno, "dpdk error: %s",
 518                             DPDK_ERR_PERM_MSG);
 519                 ret = PCAP_ERROR_PERM_DENIED;
 520                 return ret;
 521         }
 522         // init EAL
 523         ptr_dpdk_cfg = getenv(DPDK_CFG_ENV_NAME);
 524         // set default log level to debug
 525         rte_log_set_global_level(DPDK_DEF_LOG_LEV);
 526         if (ptr_dpdk_cfg == NULL)
 527         {
 528                 RTE_LOG(INFO,USER1,"env $DPDK_CFG is unset, so using default: %s\n",DPDK_DEF_CFG);
 529                 ptr_dpdk_cfg = DPDK_DEF_CFG;
 530         }
 531         memset(dpdk_cfg_buf,0,sizeof(dpdk_cfg_buf));
 532         snprintf(dpdk_cfg_buf,DPDK_CFG_MAX_LEN-1,"%s %s",DPDK_LIB_NAME,ptr_dpdk_cfg);
 533         dargv_cnt = parse_dpdk_cfg(dpdk_cfg_buf,dargv);
 534         ret = rte_eal_init(dargv_cnt,dargv);
 535         // if init successed, we do not need to do it again later.
 536         if (ret == 0){
 537                 is_dpdk_pre_inited = 1;
 538         }
 539         return ret;
 540 }
 541
 542 static int pcap_dpdk_activate(pcap_t *p)
 543 {
 544         struct pcap_dpdk *pd = p->priv;
 545         pd->orig = p;
 546         int ret = PCAP_ERROR;
 547         uint16_t nb_ports=0;
 548         uint16_t portid= DPDK_PORTID_MAX;
 549         unsigned nb_mbufs = DPDK_NB_MBUFS;
 550         struct rte_eth_rxconf rxq_conf;
 551         struct rte_eth_txconf txq_conf;
 552         struct rte_eth_conf local_port_conf = port_conf;
 553         struct rte_eth_dev_info dev_info;
 554         int is_port_up = 0;
 555         struct rte_eth_link link;
 556         do{
 557                 //init EAL
 558                 ret = dpdk_pre_init(p->errbuf);
 559                 if (ret < 0)
 560                 {
 561                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 562                             errno, "dpdk error: Init failed with device %s",
 563                             p->opt.device);
 564                         ret = PCAP_ERROR;
 565                         break;
 566                 }
 567                 ret = dpdk_init_timer(pd);
 568                 if (ret<0)
 569                 {
 570                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 571                                 errno, "dpdk error: Init timer error with device %s",
 572                                 p->opt.device);
 573                         ret = PCAP_ERROR;
 574                         break;
 575                 }
 576
 577                 nb_ports = rte_eth_dev_count_avail();
 578                 if (nb_ports == 0)
 579                 {
 580                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 581                             errno, "dpdk error: No Ethernet ports");
 582                         ret = PCAP_ERROR;
 583                         break;
 584                 }
 585
 586                 portid = portid_by_device(p->opt.device);
 587                 if (portid == DPDK_PORTID_MAX){
 588                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 589                             errno, "dpdk error: portid is invalid. device %s",
 590                             p->opt.device);
 591                         ret = PCAP_ERROR_NO_SUCH_DEVICE;
 592                         break;
 593                 }
 594
 595                 pd->portid = portid;
 596
 597                 if (p->snapshot <= 0 || p->snapshot > MAXIMUM_SNAPLEN)
 598                 {
 599                         p->snapshot = MAXIMUM_SNAPLEN;
 600                 }
 601                 // create the mbuf pool
 602                 pd->pktmbuf_pool = rte_pktmbuf_pool_create(MBUF_POOL_NAME, nb_mbufs,
 603                         MEMPOOL_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE,
 604                         rte_socket_id());
 605                 if (pd->pktmbuf_pool == NULL)
 606                 {
 607                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 608                             errno, "dpdk error: Cannot init mbuf pool");
 609                         ret = PCAP_ERROR;
 610                         break;
 611                 }
 612                 // config dev
 613                 rte_eth_dev_info_get(portid, &dev_info);
 614                 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 615                 {
 616                         local_port_conf.txmode.offloads |=DEV_TX_OFFLOAD_MBUF_FAST_FREE;
 617                 }
 618                 // only support 1 queue
 619                 ret = rte_eth_dev_configure(portid, 1, 1, &local_port_conf);
 620                 if (ret < 0)
 621                 {
 622                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 623                             errno, "dpdk error: Cannot configure device: err=%d, port=%u",
 624                             ret, portid);
 625                         ret = PCAP_ERROR;
 626                         break;
 627                 }
 628                 // adjust rx tx
 629                 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, &nb_txd);
 630                 if (ret < 0)
 631                 {
 632                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 633                             errno, "dpdk error: Cannot adjust number of descriptors: err=%d, port=%u",
 634                             ret, portid);
 635                         ret = PCAP_ERROR;
 636                         break;
 637                 }
 638                 // get MAC addr
 639                 rte_eth_macaddr_get(portid, &(pd->eth_addr));
 640                 eth_addr_str(&(pd->eth_addr), pd->mac_addr, DPDK_MAC_ADDR_SIZE-1);
 641
 642                 // init one RX queue
 643                 rxq_conf = dev_info.default_rxconf;
 644                 rxq_conf.offloads = local_port_conf.rxmode.offloads;
 645                 ret = rte_eth_rx_queue_setup(portid, 0, nb_rxd,
 646                                              rte_eth_dev_socket_id(portid),
 647                                              &rxq_conf,
 648                                              pd->pktmbuf_pool);
 649                 if (ret < 0)
 650                 {
 651                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 652                             errno, "dpdk error: rte_eth_rx_queue_setup:err=%d, port=%u",
 653                             ret, portid);
 654                         ret = PCAP_ERROR;
 655                         break;
 656                 }
 657
 658                 // init one TX queue
 659                 txq_conf = dev_info.default_txconf;
 660                 txq_conf.offloads = local_port_conf.txmode.offloads;
 661                 ret = rte_eth_tx_queue_setup(portid, 0, nb_txd,
 662                                 rte_eth_dev_socket_id(portid),
 663                                 &txq_conf);
 664                 if (ret < 0)
 665                 {
 666                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 667                             errno, "dpdk error: rte_eth_tx_queue_setup:err=%d, port=%u",
 668                             ret, portid);
 669                         ret = PCAP_ERROR;
 670                         break;
 671                 }
 672                 // Initialize TX buffers
 673                 tx_buffer = rte_zmalloc_socket(DPDK_TX_BUF_NAME,
 674                                 RTE_ETH_TX_BUFFER_SIZE(MAX_PKT_BURST), 0,
 675                                 rte_eth_dev_socket_id(portid));
 676                 if (tx_buffer == NULL)
 677                 {
 678                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 679                             errno, "dpdk error: Cannot allocate buffer for tx on port %u", portid);
 680                         ret = PCAP_ERROR;
 681                         break;
 682                 }
 683                 rte_eth_tx_buffer_init(tx_buffer, MAX_PKT_BURST);
 684                 // Start device
 685                 ret = rte_eth_dev_start(portid);
 686                 if (ret < 0)
 687                 {
 688                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 689                             errno, "dpdk error: rte_eth_dev_start:err=%d, port=%u",
 690                             ret, portid);
 691                         ret = PCAP_ERROR;
 692                         break;
 693                 }
 694                 // set promiscuous mode
 695                 if (p->opt.promisc){
 696                         pd->must_clear_promisc=1;
 697                         rte_eth_promiscuous_enable(portid);
 698                 }
 699                 // check link status
 700                 is_port_up = check_link_status(portid, &link);
 701                 if (!is_port_up){
 702                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 703                             errno, "dpdk error: link is down, port=%u",portid);
 704                         ret = PCAP_ERROR_IFACE_NOT_UP;
 705                         break;
 706                 }
 707                 // reset statistics
 708                 rte_eth_stats_reset(pd->portid);
 709                 calculate_timestamp(&(pd->ts_helper), &(pd->prev_ts));
 710                 rte_eth_stats_get(pd->portid,&(pd->prev_stats));
 711                 // format pcap_t
 712                 pd->portid = portid;
 713                 p->fd = pd->portid;
 714                 if (p->snapshot <=0 || p->snapshot> MAXIMUM_SNAPLEN)
 715                 {
 716                         p->snapshot = MAXIMUM_SNAPLEN;
 717                 }
 718                 p->linktype = DLT_EN10MB; // Ethernet, the 10MB is historical.
 719                 p->selectable_fd = p->fd;
 720                 p->read_op = pcap_dpdk_dispatch;
 721                 p->inject_op = pcap_dpdk_inject;
 722                 // using pcap_filter currently, though DPDK provides their own BPF function. Because DPDK BPF needs load a ELF file as a filter.
 723                 p->setfilter_op = install_bpf_program;
 724                 p->setdirection_op = NULL;
 725                 p->set_datalink_op = NULL;
 726                 p->getnonblock_op = pcap_dpdk_getnonblock;
 727                 p->setnonblock_op = pcap_dpdk_setnonblock;
 728                 p->stats_op = pcap_dpdk_stats;
 729                 p->cleanup_op = pcap_dpdk_close;
 730                 p->breakloop_op = pcap_breakloop_common;
 731                 // set default timeout
 732                 pd->required_select_timeout.tv_sec = 0;
 733                 pd->required_select_timeout.tv_usec = DPDK_DEF_MIN_SLEEP_MS*1000;
 734                 p->required_select_timeout = &pd->required_select_timeout;
 735                 ret = 0; // OK
 736         }while(0);
 737
 738         if (ret <= PCAP_ERROR) // all kinds of error code
 739         {
 740                 pcap_cleanup_live_common(p);
 741         }else{
 742                 rte_eth_dev_get_name_by_port(portid,pd->pci_addr);
 743                 RTE_LOG(INFO, USER1,"Port %d device: %s, MAC:%s, PCI:%s\n", portid, p->opt.device, pd->mac_addr, pd->pci_addr);
 744                 RTE_LOG(INFO, USER1,"Port %d Link Up. Speed %u Mbps - %s\n",
 745                                                         portid, link.link_speed,
 746                                         (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
 747                                                 ("full-duplex") : ("half-duplex\n"));
 748         }
 749         return ret;
 750 }
 751
 752 // device name for dpdk shoud be in the form as dpdk:number, such as dpdk:0
 753 pcap_t * pcap_dpdk_create(const char *device, char *ebuf, int *is_ours)
 754 {
 755         pcap_t *p=NULL;
 756         *is_ours = 0;
 757
 758         *is_ours = !strncmp(device, "dpdk:", 5);
 759         if (! *is_ours)
 760                 return NULL;
 761         //memset will happen
 762         p = pcap_create_common(ebuf, sizeof(struct pcap_dpdk));
 763
 764         if (p == NULL)
 765                 return NULL;
 766         p->activate_op = pcap_dpdk_activate;
 767         return p;
 768 }
 769
 770 int pcap_dpdk_findalldevs(pcap_if_list_t *devlistp, char *ebuf)
 771 {
 772         int ret=0;
 773         int nb_ports = 0;
 774         char dpdk_name[DPDK_DEV_NAME_MAX];
 775         char dpdk_desc[DPDK_DEV_DESC_MAX];
 776         struct ether_addr eth_addr;
 777         char mac_addr[DPDK_MAC_ADDR_SIZE];
 778         char pci_addr[DPDK_PCI_ADDR_SIZE];
 779         do{
 780                 ret = dpdk_pre_init(ebuf);
 781                 if (ret < 0)
 782                 {
 783                         pcap_fmt_errmsg_for_errno(ebuf, PCAP_ERRBUF_SIZE,
 784                             errno, "error: Init failed with device");
 785                         ret = PCAP_ERROR;
 786                         break;
 787                 }
 788                 nb_ports = rte_eth_dev_count_avail();
 789                 if (nb_ports == 0)
 790                 {
 791                         pcap_fmt_errmsg_for_errno(ebuf, PCAP_ERRBUF_SIZE,
 792                             errno, "DPDK error: No Ethernet ports");
 793                         ret = PCAP_ERROR;
 794                         break;
 795                 }
 796                 for (int i=0; i<nb_ports; i++){
 797                         pcap_snprintf(dpdk_name,DPDK_DEV_NAME_MAX-1,"dpdk:%d",i);
 798                         // mac addr
 799                         rte_eth_macaddr_get(i, &eth_addr);
 800                         eth_addr_str(&eth_addr,mac_addr,DPDK_MAC_ADDR_SIZE);
 801                         // PCI addr
 802                         rte_eth_dev_get_name_by_port(i,pci_addr);
 803                         pcap_snprintf(dpdk_desc,DPDK_DEV_DESC_MAX-1,"%s %s, MAC:%s, PCI:%s", DPDK_DESC, dpdk_name, mac_addr, pci_addr);
 804                         if (add_dev(devlistp, dpdk_name, 0, dpdk_desc, ebuf)==NULL){
 805                                 ret = PCAP_ERROR;
 806                                 break;
 807                         }
 808                 }
 809         }while(0);
 810         return ret;
 811 }