The Tcpdump Group git mirrors - libpcap/blob - pcap-dpdk.c

   1 /*
   2  * Copyright (C) 2018 jingle YANG. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  *
   8  *   1. Redistributions of source code must retain the above copyright
   9  *      notice, this list of conditions and the following disclaimer.
  10  *   2. Redistributions in binary form must reproduce the above copyright
  11  *      notice, this list of conditions and the following disclaimer in the
  12  *      documentation and/or other materials provided with the distribution.
  13  *
  14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''AND
  15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  17  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  24  * SUCH DAMAGE.
  25  */
  26
  27 /*
  28 Date: Dec 16, 2018
  29
  30 Description:
  31 1. Pcap-dpdk provides libpcap the ability to use DPDK with the device name as dpdk:{portid}, such as dpdk:0.
  32 2. DPDK is a set of libraries and drivers for fast packet processing. (https://www.dpdk.org/)
  33 3. The testprogs/capturetest provides 6.4Gbps/800,000 pps on Intel 10-Gigabit X540-AT2 with DPDK 18.11.
  34
  35 Limitations:
  36 1. By default DPDK support is no, unless you explicitly set --enable-dpdk with ./configure or -DDISABLE_DPDK=OFF with cmake.
  37 2. Only support link libdpdk.so dynamicly, because the libdpdk.a will not work correctly.
  38 3. Only support read operation, and packet injection has not been supported yet.
  39
  40 Usage:
  41 1. compile DPDK as shared library and install.(https://github.com/DPDK/dpdk.git)
  42
  43 You shall modify the file $RTE_SDK/$RTE_TARGET/.config and set:
  44 CONFIG_RTE_BUILD_SHARED_LIB=y
  45 By the following command:
  46 sed -i 's/CONFIG_RTE_BUILD_SHARED_LIB=n/CONFIG_RTE_BUILD_SHARED_LIB=y/' $RTE_SDK/$RTE_TARGET/.config
  47
  48 2. launch l2fwd that is one of DPDK examples correctly, and get device information.
  49
  50 You shall learn how to bind nic with DPDK-compatible driver by $RTE_SDK/usertools/dpdk-devbind.py, such as igb_uio.
  51 And enable hugepages by dpdk-setup.sh
  52
  53 Then launch the l2fwd with dynamic dirver support. For example:
  54 $RTE_SDK/examples/l2fwd/$RTE_TARGET/l2fwd -dlibrte_pmd_e1000.so -dlibrte_pmd_ixgbe.so -dlibrte_mempool_ring.so -- -p 0x1
  55
  56 3. compile libpcap with dpdk options.
  57
  58 In order to find inlucde and lib automatically, you shall export DPDK envionment variable which are used for compiling DPDK.
  59
  60 export RTE_SDK={your DPDK base directory}
  61 export RTE_TARGET={your target name}
  62
  63 3.1 with configure
  64
  65 ./configure --enable-dpdk --with-dpdk-includes=$RTE_SDK/$RTE_TARGET/include --with-dpdk-libraries=$RTE_SDK/$RTE_TARGET/lib && make -s all && make -s testprogs && make install
  66
  67 3.2 with cmake
  68
  69 mkdir -p build && cd build && cmake -DDISABLE_DPDK=OFF -DDPDK_INC_DIR=$RTE_SDK/$RTE_TARGET/include -DDPDK_LIB_DIR=$RTE_SDK/$RTE_TARGET/lib" ../ && make -s all && make -s testprogs && make install
  70
  71 4. link your own program with libpcap, and use DPDK with the device name as dpdk:{portid}, such as dpdk:0.
  72 And you shall set DPDK configure options by environment variable DPDK_CFG
  73 For example, the testprogs/capturetest could be lanched by:
  74
  75 env DPDK_CFG="--log-level=debug -l0 -dlibrte_pmd_e1000.so -dlibrte_pmd_ixgbe.so -dlibrte_mempool_ring.so" ./capturetest -i dpdk:0
  76 */
  77
  78 #ifdef HAVE_CONFIG_H
  79 #include <config.h>
  80 #endif
  81
  82 #include <ctype.h>
  83 #include <errno.h>
  84 #include <netdb.h>
  85 #include <stdio.h>
  86 #include <stdlib.h>
  87 #include <string.h>
  88 #include <unistd.h>
  89 #include <time.h>
  90
  91 #include <sys/time.h>
  92
  93 //header for calling dpdk
  94 #include <rte_common.h>
  95 #include <rte_log.h>
  96 #include <rte_malloc.h>
  97 #include <rte_memory.h>
  98 #include <rte_eal.h>
  99 #include <rte_launch.h>
 100 #include <rte_atomic.h>
 101 #include <rte_cycles.h>
 102 #include <rte_lcore.h>
 103 #include <rte_per_lcore.h>
 104 #include <rte_branch_prediction.h>
 105 #include <rte_interrupts.h>
 106 #include <rte_random.h>
 107 #include <rte_debug.h>
 108 #include <rte_ether.h>
 109 #include <rte_ethdev.h>
 110 #include <rte_mempool.h>
 111 #include <rte_mbuf.h>
 112 #include <rte_bus.h>
 113
 114 #include "pcap-int.h"
 115 #include "pcap-dpdk.h"
 116
 117 #define DPDK_DEF_LOG_LEV RTE_LOG_ERR
 118 static int is_dpdk_pre_inited=0;
 119 #define DPDK_LIB_NAME "libpcap_dpdk"
 120 #define DPDK_DESC "Data Plane Development Kit (DPDK) Interface"
 121 #define DPDK_ERR_PERM_MSG "permission denied, DPDK needs root permission"
 122 #define DPDK_ARGC_MAX 64
 123 #define DPDK_CFG_MAX_LEN 1024
 124 #define DPDK_DEV_NAME_MAX 32
 125 #define DPDK_DEV_DESC_MAX 512
 126 #define DPDK_CFG_ENV_NAME "DPDK_CFG"
 127 #define DPDK_DEF_MIN_SLEEP_MS 1
 128 static char dpdk_cfg_buf[DPDK_CFG_MAX_LEN];
 129 #define DPDK_MAC_ADDR_SIZE 32
 130 #define DPDK_DEF_MAC_ADDR "00:00:00:00:00:00"
 131 #define DPDK_PCI_ADDR_SIZE 16
 132 #define DPDK_DEF_CFG "--log-level=error -l0 -dlibrte_pmd_e1000.so -dlibrte_pmd_ixgbe.so -dlibrte_mempool_ring.so"
 133 #define DPDK_PREFIX "dpdk:"
 134 #define DPDK_PORTID_MAX 65535U
 135 #define MBUF_POOL_NAME "mbuf_pool"
 136 #define DPDK_TX_BUF_NAME "tx_buffer"
 137 //The number of elements in the mbuf pool.
 138 #define DPDK_NB_MBUFS 8192U
 139 #define MEMPOOL_CACHE_SIZE 256
 140 #define MAX_PKT_BURST 32
 141 // Configurable number of RX/TX ring descriptors
 142 #define RTE_TEST_RX_DESC_DEFAULT 1024
 143 #define RTE_TEST_TX_DESC_DEFAULT 1024
 144
 145 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
 146 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
 147
 148 #define RTE_ETH_PCAP_SNAPLEN ETHER_MAX_JUMBO_FRAME_LEN
 149
 150 static struct rte_eth_dev_tx_buffer *tx_buffer;
 151
 152 struct dpdk_ts_helper{
 153         struct timeval start_time;
 154         uint64_t start_cycles;
 155         uint64_t hz;
 156 };
 157 struct pcap_dpdk{
 158         pcap_t * orig;
 159         uint16_t portid; // portid of DPDK
 160         int must_clear_promisc;
 161         uint64_t rx_pkts;
 162         uint64_t bpf_drop;
 163         int nonblock;
 164         struct timeval prev_ts;
 165         struct rte_eth_stats prev_stats;
 166         struct timeval curr_ts;
 167         struct rte_eth_stats curr_stats;
 168         uint64_t pps;
 169         uint64_t bps;
 170         struct rte_mempool * pktmbuf_pool;
 171         struct dpdk_ts_helper ts_helper;
 172         struct ether_addr eth_addr;
 173         char mac_addr[DPDK_MAC_ADDR_SIZE];
 174         char pci_addr[DPDK_PCI_ADDR_SIZE];
 175         unsigned char pcap_tmp_buf[RTE_ETH_PCAP_SNAPLEN];
 176 };
 177
 178 static struct rte_eth_conf port_conf = {
 179         .rxmode = {
 180                 .split_hdr_size = 0,
 181         },
 182         .txmode = {
 183                 .mq_mode = ETH_MQ_TX_NONE,
 184         },
 185 };
 186
 187 static int dpdk_init_timer(struct pcap_dpdk *pd){
 188         gettimeofday(&(pd->ts_helper.start_time),NULL);
 189         pd->ts_helper.start_cycles = rte_get_timer_cycles();
 190         pd->ts_helper.hz = rte_get_timer_hz();
 191         if (pd->ts_helper.hz == 0){
 192                 return -1;
 193         }
 194         return 0;
 195 }
 196 static inline void calculate_timestamp(struct dpdk_ts_helper *helper,struct timeval *ts)
 197 {
 198         uint64_t cycles;
 199         // delta
 200         struct timeval cur_time;
 201         cycles = rte_get_timer_cycles() - helper->start_cycles;
 202         cur_time.tv_sec = (time_t)(cycles/helper->hz);
 203         cur_time.tv_usec = (suseconds_t)((cycles%helper->hz)*1e6/helper->hz);
 204         timeradd(&(helper->start_time), &cur_time, ts);
 205 }
 206
 207 static uint32_t dpdk_gather_data(unsigned char *data, int len, struct rte_mbuf *mbuf)
 208 {
 209         uint32_t total_len = 0;
 210         while (mbuf && (total_len+mbuf->data_len) < len ){
 211                 rte_memcpy(data+total_len, rte_pktmbuf_mtod(mbuf,void *),mbuf->data_len);
 212                 total_len+=mbuf->data_len;
 213                 mbuf=mbuf->next;
 214         }
 215         return total_len;
 216 }
 217
 218
 219 static int dpdk_read_with_timeout(pcap_t *p, uint16_t portid, uint16_t queueid,struct rte_mbuf **pkts_burst, const uint16_t burst_cnt){
 220         struct pcap_dpdk *pd = (struct pcap_dpdk*)(p->priv);
 221         int nb_rx = 0;
 222         int timeout_ms = p->opt.timeout;
 223         int sleep_ms = 0;
 224         if (pd->nonblock){
 225                 // In non-blocking mode, just read once, no mater how many packets are captured.
 226                 nb_rx = (int)rte_eth_rx_burst(pd->portid, 0, pkts_burst, burst_cnt);
 227         }else{
 228                 // In blocking mode, read many times until packets are captured or timeout or break_loop is setted.
 229                 // if timeout_ms == 0, it may be blocked forever.
 230                 while (timeout_ms == 0 || sleep_ms < timeout_ms){
 231                         nb_rx = (int)rte_eth_rx_burst(pd->portid, 0, pkts_burst, burst_cnt);
 232                         if (nb_rx){ // got packets within timeout_ms
 233                                 break;
 234                         }else{ // no packet arrives at this round.
 235                                 if (p->break_loop){
 236                                         break;
 237                                 }
 238                                 // sleep for a very short while, but do not block CPU.
 239                                 rte_delay_us_block(DPDK_DEF_MIN_SLEEP_MS*1000);
 240                                 sleep_ms += DPDK_DEF_MIN_SLEEP_MS;
 241                         }
 242                 }
 243         }
 244         return nb_rx;
 245 }
 246
 247 static int pcap_dpdk_dispatch(pcap_t *p, int max_cnt, pcap_handler cb, u_char *cb_arg)
 248 {
 249         struct pcap_dpdk *pd = (struct pcap_dpdk*)(p->priv);
 250         int burst_cnt = 0;
 251         int nb_rx = 0;
 252         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
 253         struct rte_mbuf *m;
 254         struct pcap_pkthdr pcap_header;
 255         uint16_t portid = pd->portid;
 256         // In DPDK, pkt_len is sum of lengths for all segments. And data_len is for one segment
 257         uint32_t pkt_len = 0;
 258         int caplen = 0;
 259         u_char *bp = NULL;
 260         int i=0;
 261         unsigned int gather_len =0;
 262         int pkt_cnt = 0;
 263         int is_accepted=0;
 264         u_char *large_buffer=NULL;
 265         int timeout_ms = p->opt.timeout;
 266
 267         pd->rx_pkts = 0;
 268         if ( !PACKET_COUNT_IS_UNLIMITED(max_cnt) && max_cnt < MAX_PKT_BURST){
 269                 burst_cnt = max_cnt;
 270         }else{
 271                 burst_cnt = MAX_PKT_BURST;
 272         }
 273
 274         while( PACKET_COUNT_IS_UNLIMITED(max_cnt) || pkt_cnt < max_cnt){
 275                 if (p->break_loop){
 276                         p->break_loop = 0;
 277                         return PCAP_ERROR_BREAK;
 278                 }
 279                 // read once in non-blocking mode, or try many times waiting for timeout_ms.
 280                 // if timeout_ms == 0, it will be blocked until one packet arrives or break_loop is setted.
 281                 nb_rx = dpdk_read_with_timeout(p, portid, 0, pkts_burst, burst_cnt);
 282                 if (nb_rx == 0){
 283                         if (pd->nonblock){
 284                                 RTE_LOG(DEBUG, USER1, "dpdk: no packets available in non-blocking mode.\n");
 285                         }else{
 286                                 if (p->break_loop){
 287                                         RTE_LOG(DEBUG, USER1, "dpdk: no packets available and break_loop is setted in blocking mode.\n");
 288                                         p->break_loop = 0;
 289                                         return PCAP_ERROR_BREAK;
 290
 291                                 }
 292                                 RTE_LOG(DEBUG, USER1, "dpdk: no packets available for timeout %d ms in blocking mode.\n", timeout_ms);
 293                         }
 294                         // break if dpdk reads 0 packet, no matter in blocking(timeout) or non-blocking mode.
 295                         break;
 296                 }
 297                 pkt_cnt += nb_rx;
 298                 for ( i = 0; i < nb_rx; i++) {
 299                         m = pkts_burst[i];
 300                         calculate_timestamp(&(pd->ts_helper),&(pcap_header.ts));
 301                         pkt_len = rte_pktmbuf_pkt_len(m);
 302                         // caplen = min(pkt_len, p->snapshot);
 303                         // caplen will not be changed, no matter how long the rte_pktmbuf
 304                         caplen = pkt_len < p->snapshot ? pkt_len: p->snapshot;
 305                         pcap_header.caplen = caplen;
 306                         pcap_header.len = pkt_len;
 307                         // volatile prefetch
 308                         rte_prefetch0(rte_pktmbuf_mtod(m, void *));
 309                         bp = NULL;
 310                         if (m->nb_segs == 1)
 311                         {
 312                                 bp = rte_pktmbuf_mtod(m, u_char *);
 313                         }else{
 314                                 // use fast buffer pcap_tmp_buf if pkt_len is small, no need to call malloc and free
 315                                 if ( pkt_len <= ETHER_MAX_JUMBO_FRAME_LEN)
 316                                 {
 317                                         gather_len = dpdk_gather_data(pd->pcap_tmp_buf, RTE_ETH_PCAP_SNAPLEN, m);
 318                                         bp = pd->pcap_tmp_buf;
 319                                 }else{
 320                                         // need call free later
 321                                         large_buffer = (u_char *)malloc(caplen*sizeof(u_char));
 322                                         gather_len = dpdk_gather_data(large_buffer, caplen, m);
 323                                         bp = large_buffer;
 324                                 }
 325
 326                         }
 327                         if (bp){
 328                                 if (p->fcode.bf_insns==NULL || pcap_filter(p->fcode.bf_insns, bp, pcap_header.len, pcap_header.caplen)){
 329                                         cb(cb_arg, &pcap_header, bp);
 330                                 }else{
 331                                         pd->bpf_drop++;
 332                                 }
 333                         }
 334                         //free all pktmbuf
 335                         rte_pktmbuf_free(m);
 336                         if (large_buffer){
 337                                 free(large_buffer);
 338                                 large_buffer=NULL;
 339                         }
 340                 }
 341         }
 342         pd->rx_pkts = pkt_cnt;
 343         return pd->rx_pkts;
 344 }
 345
 346 static int pcap_dpdk_inject(pcap_t *p, const void *buf _U_, int size _U_)
 347 {
 348         //not implemented yet
 349         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 350                 errno, "dpdk error: Inject function has not been implemented yet");
 351         return PCAP_ERROR;
 352 }
 353
 354 static void pcap_dpdk_close(pcap_t *p)
 355 {
 356         struct pcap_dpdk *pd = p->priv;
 357         if (pd==NULL)
 358         {
 359                 return;
 360         }
 361         if (pd->must_clear_promisc)
 362         {
 363                 rte_eth_promiscuous_disable(pd->portid);
 364         }
 365         rte_eth_dev_stop(pd->portid);
 366         rte_eth_dev_close(pd->portid);
 367         pcap_cleanup_live_common(p);
 368 }
 369
 370 static void nic_stats_display(struct pcap_dpdk *pd)
 371 {
 372         uint16_t portid = pd->portid;
 373         struct rte_eth_stats stats;
 374         rte_eth_stats_get(portid, &stats);
 375         RTE_LOG(INFO,USER1, "portid:%d, RX-packets: %-10"PRIu64"  RX-errors:  %-10"PRIu64
 376                "  RX-bytes:  %-10"PRIu64"  RX-Imissed:  %-10"PRIu64"\n", portid, stats.ipackets, stats.ierrors,
 377                stats.ibytes,stats.imissed);
 378         RTE_LOG(INFO,USER1, "portid:%d, RX-PPS: %-10"PRIu64" RX-Mbps: %.2lf\n", portid, pd->pps, pd->bps/1e6f );
 379 }
 380
 381 static int pcap_dpdk_stats(pcap_t *p, struct pcap_stat *ps)
 382 {
 383         struct pcap_dpdk *pd = p->priv;
 384         calculate_timestamp(&(pd->ts_helper), &(pd->curr_ts));
 385         rte_eth_stats_get(pd->portid,&(pd->curr_stats));
 386         if (ps){
 387                 ps->ps_recv = pd->curr_stats.ipackets;
 388                 ps->ps_drop = pd->curr_stats.ierrors;
 389                 ps->ps_drop += pd->bpf_drop;
 390                 ps->ps_ifdrop = pd->curr_stats.imissed;
 391         }
 392         uint64_t delta_pkt = pd->curr_stats.ipackets - pd->prev_stats.ipackets;
 393         struct timeval delta_tm;
 394         timersub(&(pd->curr_ts),&(pd->prev_ts), &delta_tm);
 395         uint64_t delta_usec = delta_tm.tv_sec*1e6+delta_tm.tv_usec;
 396         uint64_t delta_bit = (pd->curr_stats.ibytes-pd->prev_stats.ibytes)*8;
 397         RTE_LOG(DEBUG, USER1, "delta_usec: %-10"PRIu64" delta_pkt: %-10"PRIu64" delta_bit: %-10"PRIu64"\n", delta_usec, delta_pkt, delta_bit);
 398         pd->pps = (uint64_t)(delta_pkt*1e6f/delta_usec);
 399         pd->bps = (uint64_t)(delta_bit*1e6f/delta_usec);
 400         nic_stats_display(pd);
 401         pd->prev_stats = pd->curr_stats;
 402         pd->prev_ts = pd->curr_ts;
 403         return 0;
 404 }
 405
 406 static int pcap_dpdk_setnonblock(pcap_t *p, int nonblock){
 407         struct pcap_dpdk *pd = (struct pcap_dpdk*)(p->priv);
 408         pd->nonblock = nonblock;
 409         return 0;
 410 }
 411
 412 static int pcap_dpdk_getnonblock(pcap_t *p){
 413         struct pcap_dpdk *pd = (struct pcap_dpdk*)(p->priv);
 414         return pd->nonblock;
 415 }
 416 static int check_link_status(uint16_t portid, struct rte_eth_link *plink)
 417 {
 418         // wait up to 9 seconds to get link status
 419         rte_eth_link_get(portid, plink);
 420         return plink->link_status == ETH_LINK_UP;
 421 }
 422 static void eth_addr_str(struct ether_addr *addrp, char* mac_str, int len)
 423 {
 424         int offset=0;
 425         if (addrp == NULL){
 426                 pcap_snprintf(mac_str, len-1, DPDK_DEF_MAC_ADDR);
 427                 return;
 428         }
 429         for (int i=0; i<6; i++)
 430         {
 431                 if (offset >= len)
 432                 { // buffer overflow
 433                         return;
 434                 }
 435                 if (i==0)
 436                 {
 437                         pcap_snprintf(mac_str+offset, len-1-offset, "%02X",addrp->addr_bytes[i]);
 438                         offset+=2; // FF
 439                 }else{
 440                         pcap_snprintf(mac_str+offset, len-1-offset, ":%02X", addrp->addr_bytes[i]);
 441                         offset+=3; // :FF
 442                 }
 443         }
 444         return;
 445 }
 446 // return portid by device name, otherwise return -1
 447 static uint16_t portid_by_device(char * device)
 448 {
 449         uint16_t ret = DPDK_PORTID_MAX;
 450         int len = strlen(device);
 451         int prefix_len = strlen(DPDK_PREFIX);
 452         unsigned long ret_ul = 0L;
 453         char *pEnd;
 454         if (len<=prefix_len || strncmp(device, DPDK_PREFIX, prefix_len)) // check prefix dpdk:
 455         {
 456                 return ret;
 457         }
 458         //check all chars are digital
 459         for (int i=prefix_len; device[i]; i++){
 460                 if (device[i]<'0' || device[i]>'9'){
 461                         return ret;
 462                 }
 463         }
 464         ret_ul = strtoul(&(device[prefix_len]), &pEnd, 10);
 465         if (pEnd == &(device[prefix_len]) || *pEnd != '\0'){
 466                 return ret;
 467         }
 468         // too large for portid
 469         if (ret_ul >= DPDK_PORTID_MAX){
 470                 return ret;
 471         }
 472         ret = (uint16_t)ret_ul;
 473         return ret;
 474 }
 475
 476 static int parse_dpdk_cfg(char* dpdk_cfg,char** dargv)
 477 {
 478         int cnt=0;
 479         memset(dargv,0,sizeof(dargv[0])*DPDK_ARGC_MAX);
 480         //current process name
 481         int skip_space = 1;
 482         int i=0;
 483         RTE_LOG(INFO, USER1,"dpdk cfg: %s\n",dpdk_cfg);
 484         // find first non space char
 485         // The last opt is NULL
 486         for (i=0;dpdk_cfg[i] && cnt<DPDK_ARGC_MAX-1;i++){
 487                 if (skip_space && dpdk_cfg[i]!=' '){ // not space
 488                         skip_space=!skip_space; // skip normal char
 489                         dargv[cnt++] = dpdk_cfg+i;
 490                 }
 491                 if (!skip_space && dpdk_cfg[i]==' '){ // fint a space
 492                         dpdk_cfg[i]=0x00; // end of this opt
 493                         skip_space=!skip_space; // skip space char
 494                 }
 495         }
 496         dargv[cnt]=NULL;
 497         return cnt;
 498 }
 499
 500 // only called once
 501 static int dpdk_pre_init(char * ebuf)
 502 {
 503         int dargv_cnt=0;
 504         char *dargv[DPDK_ARGC_MAX];
 505         char *ptr_dpdk_cfg = NULL;
 506         int ret = PCAP_ERROR;
 507         // globale var
 508         if (is_dpdk_pre_inited)
 509         {
 510                 // already inited
 511                 return 0;
 512         }
 513         // check for root permission
 514         if( geteuid() != 0)
 515         {
 516                 RTE_LOG(ERR, USER1, "%s\n", DPDK_ERR_PERM_MSG);
 517                 pcap_fmt_errmsg_for_errno(ebuf, PCAP_ERRBUF_SIZE,
 518                             errno, "dpdk error: %s",
 519                             DPDK_ERR_PERM_MSG);
 520                 ret = PCAP_ERROR_PERM_DENIED;
 521                 return ret;
 522         }
 523         // init EAL
 524         ptr_dpdk_cfg = getenv(DPDK_CFG_ENV_NAME);
 525         // set default log level to debug
 526         rte_log_set_global_level(DPDK_DEF_LOG_LEV);
 527         if (ptr_dpdk_cfg == NULL)
 528         {
 529                 RTE_LOG(INFO,USER1,"env $DPDK_CFG is unset, so using default: %s\n",DPDK_DEF_CFG);
 530                 ptr_dpdk_cfg = DPDK_DEF_CFG;
 531         }
 532         memset(dpdk_cfg_buf,0,sizeof(dpdk_cfg_buf));
 533         snprintf(dpdk_cfg_buf,DPDK_CFG_MAX_LEN-1,"%s %s",DPDK_LIB_NAME,ptr_dpdk_cfg);
 534         dargv_cnt = parse_dpdk_cfg(dpdk_cfg_buf,dargv);
 535         ret = rte_eal_init(dargv_cnt,dargv);
 536         // if init successed, we do not need to do it again later.
 537         if (ret == 0){
 538                 is_dpdk_pre_inited = 1;
 539         }
 540         return ret;
 541 }
 542
 543 static int pcap_dpdk_activate(pcap_t *p)
 544 {
 545         struct pcap_dpdk *pd = p->priv;
 546         pd->orig = p;
 547         int ret = PCAP_ERROR;
 548         uint16_t nb_ports=0;
 549         uint16_t portid= DPDK_PORTID_MAX;
 550         unsigned nb_mbufs = DPDK_NB_MBUFS;
 551         struct rte_eth_rxconf rxq_conf;
 552         struct rte_eth_txconf txq_conf;
 553         struct rte_eth_conf local_port_conf = port_conf;
 554         struct rte_eth_dev_info dev_info;
 555         int is_port_up = 0;
 556         struct rte_eth_link link;
 557         do{
 558                 //init EAL
 559                 ret = dpdk_pre_init(p->errbuf);
 560                 if (ret < 0)
 561                 {
 562                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 563                             errno, "dpdk error: Init failed with device %s",
 564                             p->opt.device);
 565                         ret = PCAP_ERROR;
 566                         break;
 567                 }
 568                 ret = dpdk_init_timer(pd);
 569                 if (ret<0)
 570                 {
 571                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 572                                 errno, "dpdk error: Init timer error with device %s",
 573                                 p->opt.device);
 574                         ret = PCAP_ERROR;
 575                         break;
 576                 }
 577
 578                 nb_ports = rte_eth_dev_count_avail();
 579                 if (nb_ports == 0)
 580                 {
 581                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 582                             errno, "dpdk error: No Ethernet ports");
 583                         ret = PCAP_ERROR;
 584                         break;
 585                 }
 586
 587                 portid = portid_by_device(p->opt.device);
 588                 if (portid == DPDK_PORTID_MAX){
 589                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 590                             errno, "dpdk error: portid is invalid. device %s",
 591                             p->opt.device);
 592                         ret = PCAP_ERROR_NO_SUCH_DEVICE;
 593                         break;
 594                 }
 595
 596                 pd->portid = portid;
 597
 598                 if (p->snapshot <= 0 || p->snapshot > MAXIMUM_SNAPLEN)
 599                 {
 600                         p->snapshot = MAXIMUM_SNAPLEN;
 601                 }
 602                 // create the mbuf pool
 603                 pd->pktmbuf_pool = rte_pktmbuf_pool_create(MBUF_POOL_NAME, nb_mbufs,
 604                         MEMPOOL_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE,
 605                         rte_socket_id());
 606                 if (pd->pktmbuf_pool == NULL)
 607                 {
 608                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 609                             errno, "dpdk error: Cannot init mbuf pool");
 610                         ret = PCAP_ERROR;
 611                         break;
 612                 }
 613                 // config dev
 614                 rte_eth_dev_info_get(portid, &dev_info);
 615                 if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 616                 {
 617                         local_port_conf.txmode.offloads |=DEV_TX_OFFLOAD_MBUF_FAST_FREE;
 618                 }
 619                 // only support 1 queue
 620                 ret = rte_eth_dev_configure(portid, 1, 1, &local_port_conf);
 621                 if (ret < 0)
 622                 {
 623                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 624                             errno, "dpdk error: Cannot configure device: err=%d, port=%u",
 625                             ret, portid);
 626                         ret = PCAP_ERROR;
 627                         break;
 628                 }
 629                 // adjust rx tx
 630                 ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, &nb_txd);
 631                 if (ret < 0)
 632                 {
 633                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 634                             errno, "dpdk error: Cannot adjust number of descriptors: err=%d, port=%u",
 635                             ret, portid);
 636                         ret = PCAP_ERROR;
 637                         break;
 638                 }
 639                 // get MAC addr
 640                 rte_eth_macaddr_get(portid, &(pd->eth_addr));
 641                 eth_addr_str(&(pd->eth_addr), pd->mac_addr, DPDK_MAC_ADDR_SIZE-1);
 642
 643                 // init one RX queue
 644                 rxq_conf = dev_info.default_rxconf;
 645                 rxq_conf.offloads = local_port_conf.rxmode.offloads;
 646                 ret = rte_eth_rx_queue_setup(portid, 0, nb_rxd,
 647                                              rte_eth_dev_socket_id(portid),
 648                                              &rxq_conf,
 649                                              pd->pktmbuf_pool);
 650                 if (ret < 0)
 651                 {
 652                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 653                             errno, "dpdk error: rte_eth_rx_queue_setup:err=%d, port=%u",
 654                             ret, portid);
 655                         ret = PCAP_ERROR;
 656                         break;
 657                 }
 658
 659                 // init one TX queue
 660                 txq_conf = dev_info.default_txconf;
 661                 txq_conf.offloads = local_port_conf.txmode.offloads;
 662                 ret = rte_eth_tx_queue_setup(portid, 0, nb_txd,
 663                                 rte_eth_dev_socket_id(portid),
 664                                 &txq_conf);
 665                 if (ret < 0)
 666                 {
 667                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 668                             errno, "dpdk error: rte_eth_tx_queue_setup:err=%d, port=%u",
 669                             ret, portid);
 670                         ret = PCAP_ERROR;
 671                         break;
 672                 }
 673                 // Initialize TX buffers
 674                 tx_buffer = rte_zmalloc_socket(DPDK_TX_BUF_NAME,
 675                                 RTE_ETH_TX_BUFFER_SIZE(MAX_PKT_BURST), 0,
 676                                 rte_eth_dev_socket_id(portid));
 677                 if (tx_buffer == NULL)
 678                 {
 679                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 680                             errno, "dpdk error: Cannot allocate buffer for tx on port %u", portid);
 681                         ret = PCAP_ERROR;
 682                         break;
 683                 }
 684                 rte_eth_tx_buffer_init(tx_buffer, MAX_PKT_BURST);
 685                 // Start device
 686                 ret = rte_eth_dev_start(portid);
 687                 if (ret < 0)
 688                 {
 689                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 690                             errno, "dpdk error: rte_eth_dev_start:err=%d, port=%u",
 691                             ret, portid);
 692                         ret = PCAP_ERROR;
 693                         break;
 694                 }
 695                 // set promiscuous mode
 696                 if (p->opt.promisc){
 697                         pd->must_clear_promisc=1;
 698                         rte_eth_promiscuous_enable(portid);
 699                 }
 700                 // check link status
 701                 is_port_up = check_link_status(portid, &link);
 702                 if (!is_port_up){
 703                         pcap_fmt_errmsg_for_errno(p->errbuf, PCAP_ERRBUF_SIZE,
 704                             errno, "dpdk error: link is down, port=%u",portid);
 705                         ret = PCAP_ERROR_IFACE_NOT_UP;
 706                         break;
 707                 }
 708                 // reset statistics
 709                 rte_eth_stats_reset(pd->portid);
 710                 calculate_timestamp(&(pd->ts_helper), &(pd->prev_ts));
 711                 rte_eth_stats_get(pd->portid,&(pd->prev_stats));
 712                 // format pcap_t
 713                 pd->portid = portid;
 714                 p->fd = pd->portid;
 715                 if (p->snapshot <=0 || p->snapshot> MAXIMUM_SNAPLEN)
 716                 {
 717                         p->snapshot = MAXIMUM_SNAPLEN;
 718                 }
 719                 p->linktype = DLT_EN10MB; // Ethernet, the 10MB is historical.
 720                 p->selectable_fd = p->fd;
 721                 p->read_op = pcap_dpdk_dispatch;
 722                 p->inject_op = pcap_dpdk_inject;
 723                 // using pcap_filter currently, though DPDK provides their own BPF function. Because DPDK BPF needs load a ELF file as a filter.
 724                 p->setfilter_op = install_bpf_program;
 725                 p->setdirection_op = NULL;
 726                 p->set_datalink_op = NULL;
 727                 p->getnonblock_op = pcap_dpdk_getnonblock;
 728                 p->setnonblock_op = pcap_dpdk_setnonblock;
 729                 p->stats_op = pcap_dpdk_stats;
 730                 p->cleanup_op = pcap_dpdk_close;
 731                 p->breakloop_op = pcap_breakloop_common;
 732                 ret = 0; // OK
 733         }while(0);
 734
 735         if (ret <= PCAP_ERROR) // all kinds of error code
 736         {
 737                 pcap_cleanup_live_common(p);
 738         }else{
 739                 rte_eth_dev_get_name_by_port(portid,pd->pci_addr);
 740                 RTE_LOG(INFO, USER1,"Port %d device: %s, MAC:%s, PCI:%s\n", portid, p->opt.device, pd->mac_addr, pd->pci_addr);
 741                 RTE_LOG(INFO, USER1,"Port %d Link Up. Speed %u Mbps - %s\n",
 742                                                         portid, link.link_speed,
 743                                         (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
 744                                                 ("full-duplex") : ("half-duplex\n"));
 745         }
 746         return ret;
 747 }
 748
 749 // device name for dpdk shoud be in the form as dpdk:number, such as dpdk:0
 750 pcap_t * pcap_dpdk_create(const char *device, char *ebuf, int *is_ours)
 751 {
 752         pcap_t *p=NULL;
 753         *is_ours = 0;
 754
 755         *is_ours = !strncmp(device, "dpdk:", 5);
 756         if (! *is_ours)
 757                 return NULL;
 758         //memset will happen
 759         p = pcap_create_common(ebuf, sizeof(struct pcap_dpdk));
 760
 761         if (p == NULL)
 762                 return NULL;
 763         p->activate_op = pcap_dpdk_activate;
 764         return p;
 765 }
 766
 767 int pcap_dpdk_findalldevs(pcap_if_list_t *devlistp, char *ebuf)
 768 {
 769         int ret=0;
 770         int nb_ports = 0;
 771         char dpdk_name[DPDK_DEV_NAME_MAX];
 772         char dpdk_desc[DPDK_DEV_DESC_MAX];
 773         struct ether_addr eth_addr;
 774         char mac_addr[DPDK_MAC_ADDR_SIZE];
 775         char pci_addr[DPDK_PCI_ADDR_SIZE];
 776         do{
 777                 ret = dpdk_pre_init(ebuf);
 778                 if (ret < 0)
 779                 {
 780                         pcap_fmt_errmsg_for_errno(ebuf, PCAP_ERRBUF_SIZE,
 781                             errno, "error: Init failed with device");
 782                         ret = PCAP_ERROR;
 783                         break;
 784                 }
 785                 nb_ports = rte_eth_dev_count_avail();
 786                 if (nb_ports == 0)
 787                 {
 788                         pcap_fmt_errmsg_for_errno(ebuf, PCAP_ERRBUF_SIZE,
 789                             errno, "DPDK error: No Ethernet ports");
 790                         ret = PCAP_ERROR;
 791                         break;
 792                 }
 793                 for (int i=0; i<nb_ports; i++){
 794                         pcap_snprintf(dpdk_name,DPDK_DEV_NAME_MAX-1,"dpdk:%d",i);
 795                         // mac addr
 796                         rte_eth_macaddr_get(i, &eth_addr);
 797                         eth_addr_str(&eth_addr,mac_addr,DPDK_MAC_ADDR_SIZE);
 798                         // PCI addr
 799                         rte_eth_dev_get_name_by_port(i,pci_addr);
 800                         pcap_snprintf(dpdk_desc,DPDK_DEV_DESC_MAX-1,"%s %s, MAC:%s, PCI:%s", DPDK_DESC, dpdk_name, mac_addr, pci_addr);
 801                         if (add_dev(devlistp, dpdk_name, 0, dpdk_desc, ebuf)==NULL){
 802                                 ret = PCAP_ERROR;
 803                                 break;
 804                         }
 805                 }
 806         }while(0);
 807         return ret;
 808 }