L2 Forwarding App

From: Venkatesh Vivekanandan <venkatesh.vivekanandan@linaro.org>

Initial L2 forwarding application based on example pktio that
resembles DPDK's in functionality.

It is based on raw sockets and not tested on any network HW accelerator.

Create pktio and queue for all threads before starting pthread.

It has two modes supported,
	1. burst mode(without ODP queues)
	2. queue mode (with ODP queues and scheduler)
   and three types,
	1. raw sockets
	2. multiple messages on sockets
	3. mmap on sockets

Signed-off-by: Venkatesh Vivekanandan <venkatesh.vivekanandan@linaro.org>
---
 test/l2fwd/Makefile |  44 ++++
 test/l2fwd/l2fwd.c  | 637 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 681 insertions(+)
 create mode 100644 test/l2fwd/Makefile
 create mode 100644 test/l2fwd/l2fwd.c

Taras


On 13 May 2014 10:18, <venkatesh.vivekanandan@linaro.org> wrote:

> From: Venkatesh Vivekanandan <venkatesh.vivekanandan@linaro.org>
>
> Initial L2 forwarding application based on example pktio that
> resembles DPDK's in functionality.
>
> It is based on raw sockets and not tested on any network HW accelerator.
>
> Create pktio and queue for all threads before starting pthread.
>
> It has two modes supported,
>         1. burst mode(without ODP queues)
>         2. queue mode (with ODP queues and scheduler)
>    and three types,
>         1. raw sockets
>         2. multiple messages on sockets
>         3. mmap on sockets
>
> Signed-off-by: Venkatesh Vivekanandan <venkatesh.vivekanandan@linaro.org>
> ---
>  test/l2fwd/Makefile |  44 ++++
>  test/l2fwd/l2fwd.c  | 637
> ++++++++++++++++++++++++++++++++++++++++++++++++++++
>  2 files changed, 681 insertions(+)
>  create mode 100644 test/l2fwd/Makefile
>  create mode 100644 test/l2fwd/l2fwd.c
>
> diff --git a/test/l2fwd/Makefile b/test/l2fwd/Makefile
> new file mode 100644
> index 0000000..3435caa
> --- /dev/null
> +++ b/test/l2fwd/Makefile
> @@ -0,0 +1,44 @@
> +# Copyright (c) 2014, Linaro Limited
> +# All rights reserved.
> +#
> +# SPDX-License-Identifier:     BSD-3-Clause
> +
> +ODP_ROOT = ../..
> +ODP_APP  = l2fwd
> +
> +include $(ODP_ROOT)/Makefile.inc
> +include ../Makefile.inc
> +
> +OBJS     =
> +OBJS    += $(OBJ_DIR)/l2fwd.o
> +
> +DEPS     = $(OBJS:.o=.d)
> +
> +.PHONY: default
> +default: $(OBJ_DIR) $(ODP_APP)
> +
> +-include $(DEPS)
> +
> +#
> +# Compile rules
> +#
> +$(OBJ_DIR)/%.o: %.c
> +       $(ECHO) Compiling $<
> +       $(CC) -c -MD $(EXTRA_CFLAGS) $(CFLAGS) -o $@ $<
> +
> +#
> +# Link rule
> +#
> +$(ODP_APP): $(ODP_LIB) $(OBJS)
> +       $(CC) $(LDFLAGS) $(OBJS) $(ODP_LIB) $(STD_LIBS) -o $@
> +
> +.PHONY: clean
> +clean:
> +       $(RMDIR) $(OBJ_DIR)
> +       $(RM) $(ODP_APP)
> +       $(MAKE) -C $(ODP_DIR) clean
> +
> +.PHONY: install
> +install:
> +       install -d $(DESTDIR)/share/odp
> +       install -m 0755 $(ODP_APP) $(DESTDIR)/share/odp/
> diff --git a/test/l2fwd/l2fwd.c b/test/l2fwd/l2fwd.c
> new file mode 100644
> index 0000000..fb1b949
> --- /dev/null
> +++ b/test/l2fwd/l2fwd.c
> @@ -0,0 +1,637 @@
> +/* Copyright (c) 2014, Linaro Limited
> + * All rights reserved.
> + *
> + * SPDX-License-Identifier:     BSD-3-Clause
> + */
> +
> +/**
> + * @file
> + *
> + * @example l2fwd.c  ODP basic forwarding application
> + */
> +
> +#include <stdlib.h>
> +#include <string.h>
> +#include <getopt.h>
> +#include <unistd.h>
> +
> +#include <odp.h>
> +#include <helper/odp_linux.h>
> +#include <helper/odp_packet_helper.h>
> +#include <helper/odp_eth.h>
> +#include <helper/odp_ip.h>
> +
> +#define MAX_WORKERS            32
> +#define SHM_PKT_POOL_SIZE      (512*2048)
> +#define SHM_PKT_POOL_BUF_SIZE  1856
> +#define MAX_PKT_BURST          16
> +
> +#define APPL_MODE_PKT_BURST    0
> +#define APPL_MODE_PKT_QUEUE    1
> +
> +#define PRINT_APPL_MODE(x) printf("%s(%i)\n", #x, (x))
> +
> +/** Get rid of path in filename - only for unix-type paths using '/' */
> +#define NO_PATH(file_name) (strrchr((file_name), '/') ? \
> +                           strrchr((file_name), '/') + 1 : (file_name))
> +/**
> + * Parsed command line application arguments
> + */
> +typedef struct {
> +       int core_count;
> +       int if_count;           /**< Number of interfaces to be used */
> +       char **if_names;        /**< Array of pointers to interface names
> */
> +       int mode;               /**< Packet IO mode */
> +       int type;               /**< Packet IO type */
> +       int fanout;             /**< Packet IO fanout */
> +       odp_buffer_pool_t pool; /**< Buffer pool for packet IO */
> +} appl_args_t;
> +
> +/**
> + * Thread specific arguments
> + */
> +typedef struct {
> +       char *srcif;            /**< Source Interface */
> +       char *dstif;            /**< Dest Interface */
> +       odp_buffer_pool_t pool; /**< Buffer pool for packet IO */
> +       odp_pktio_t srcpktio;   /**< Source pktio handle */
> +       odp_pktio_t dstpktio;   /**< Destination pktio handle */
> +       int mode;               /**< Thread mode */
> +       int type;               /**< Thread i/o type */
> +       int fanout;             /**< Thread i/o fanout */
> +} thread_args_t;
> +
> +/**
> + * Grouping of both parsed CL args and thread specific args - alloc
> together
> + */
> +typedef struct {
> +       /** Application (parsed) arguments */
> +       appl_args_t appl;
> +       /** Thread specific arguments */
> +       thread_args_t thread[MAX_WORKERS];
> +} args_t;
> +
> +/** Global pointer to args */
> +static args_t *args;
> +int num_workers;
> +
> +/* helper funcs */
> +static int drop_err_pkts(odp_packet_t pkt_tbl[], unsigned len);
> +static void parse_args(int argc, char *argv[], appl_args_t *appl_args);
> +static void print_info(char *progname, appl_args_t *appl_args);
> +static void usage(char *progname);
> +
> +/**
> + * Burst mode: pktio for each thread will be created with either same or
> + * different params
> + *
> + * @param arg  thread arguments of type 'thread_args_t *'
> + * @param pool is the packet pool from where buffers should be taken
> + */
> +static odp_pktio_t burst_mode_init_params(void *arg, int pool)
> +{
> +       thread_args_t *args;
> +       odp_pktio_params_t params;
> +       socket_params_t *sock_params = &params.sock_params;
> +       odp_pktio_t pktio;
> +
> +       args = arg;
> +       /* Open a packet IO instance for this thread */
> +       sock_params->type = args->type;
> +       sock_params->fanout = args->fanout;
> +       pktio = odp_pktio_open(args->srcif, pool, &params);
> +       if (pktio == ODP_PKTIO_INVALID)
> +               ODP_ERR("  Error: pktio create failed");
> +
> +       return pktio;
> +}
> +
> +/**
> + * Queue mode: pktio for each thread will be created with either same or
> + * different params. Queues are created and attached to the pktio.
> + *
> + * @param arg  thread arguments of type 'thread_args_t *'
> + * @param pool is the packet pool from where buffers should be taken
> + */
> +static odp_pktio_t queue_mode_init_params(void *arg, int pool)
> +{
> +       char inq_name[ODP_QUEUE_NAME_LEN];
> +       odp_queue_param_t qparam;
> +       odp_queue_t inq_def;
> +       int ret;
> +       odp_pktio_t pktio = ODP_PKTIO_INVALID;
> +
> +       pktio = burst_mode_init_params(arg, pool);
> +       if (pktio == ODP_PKTIO_INVALID)
> +               return pktio;
> +       /*
> +        * Create and set the default INPUT queue associated with the
> 'pktio'
> +        * resource
> +        */
> +       qparam.sched.prio  = ODP_SCHED_PRIO_DEFAULT;
> +       qparam.sched.sync  = ODP_SCHED_SYNC_ATOMIC;
> +       qparam.sched.group = ODP_SCHED_GROUP_DEFAULT;
> +       snprintf(inq_name, sizeof(inq_name), "%i-pktio_inq_def",
> (int)pktio);
> +       inq_name[ODP_QUEUE_NAME_LEN - 1] = '\0';
> +
> +       inq_def = odp_queue_create(inq_name, ODP_QUEUE_TYPE_PKTIN,
> &qparam);
> +       if (inq_def == ODP_QUEUE_INVALID) {
> +               ODP_ERR("  Error: pktio queue creation failed");
> +               return ODP_PKTIO_INVALID;
> +       }
> +
> +       ret = odp_pktio_inq_setdef(pktio, inq_def);
> +       if (ret != 0) {
> +               ODP_ERR("  Error: default input-Q setup");
> +               return ODP_PKTIO_INVALID;
> +       }
> +
> +       return pktio;
> +}
> +
> +/**
> + * Packet IO worker thread using ODP queues
> + *
> + * @param arg  thread arguments of type 'thread_args_t *'
> + */
> +static void *pktio_queue_thread(void *arg)
> +{
> +       int thr, i;
> +       thread_args_t *thr_args;
> +       char dstpktio[MAX_WORKERS+1];
> +       odp_queue_t outq_def;
> +       odp_packet_t pkt;
> +       odp_buffer_t buf;
> +       unsigned long pkt_cnt = 0;
> +       unsigned long err_cnt = 0;
> +
> +       thr = odp_thread_id();
> +       thr_args = arg;
> +
> +       if (thr_args->srcpktio == 0 || thr_args->dstpktio == 0) {
> +               ODP_ERR("Invalid srcpktio:%d dstpktio:%d\n",
> +                       thr_args->srcpktio, thr_args->dstpktio);
> +               return NULL;
> +       }
> +       printf("[%02i] srcif:%s dstif:%s spktio:%02i dpktio:%02i QUEUE
> mode\n",
> +              thr, thr_args->srcif, thr_args->dstif, thr_args->srcpktio,
> +              thr_args->dstpktio);
> +
> +       /* Populate an array of destination pktio's in all threads as the
> +        * scheduler can take packets from any input queue
> +        */
> +       for (i = 0; i < num_workers; i++)
> +               dstpktio[i+1] = args->thread[i].dstpktio;
> +
> +       /* Loop packets */
> +       for (;;) {
> +               odp_pktio_t pktio_tmp;
> +
> +               /* Use schedule to get buf from any input queue */
> +               buf = odp_schedule(NULL, ODP_SCHED_WAIT);
> +
> +               pkt = odp_packet_from_buffer(buf);
> +               /* Drop packets with errors */
> +               if (odp_unlikely(drop_err_pkts(&pkt, 1) == 0)) {
> +                       ODP_ERR("Drop frame - err_cnt:%lu\n", ++err_cnt);
> +                       continue;
> +               }
> +
> +               pktio_tmp = odp_pktio_get_input(pkt);
> +               outq_def = odp_pktio_outq_getdef(dstpktio[pktio_tmp]);
> +               if (outq_def == ODP_QUEUE_INVALID) {
> +                       ODP_ERR("  [%02i] Error: def output-Q query\n",
> thr);
> +                       return NULL;
> +               }
> +
> +               /* Enqueue the packet for output */
> +               odp_queue_enq(outq_def, buf);
> +
> +               /* Print packet counts every once in a while */
> +               if (odp_unlikely(pkt_cnt++ % 100000 == 0)) {
> +                       printf("  [%02i] pkt_cnt:%lu\n", thr, pkt_cnt);
> +                       fflush(NULL);
> +               }
> +       }
> +
> +/* unreachable */
> +}
> +
> +/**
> + * Packet IO worker thread using bursts from/to IO resources
> + *
> + * @param arg  thread arguments of type 'thread_args_t *'
> + */
> +static void *pktio_ifburst_thread(void *arg)
> +{
> +       int thr;
> +       thread_args_t *thr_args;
> +       int pkts, pkts_ok;
> +       odp_packet_t pkt_tbl[MAX_PKT_BURST];
> +       unsigned long pkt_cnt = 0;
> +       unsigned long err_cnt = 0;
> +       unsigned long tmp = 0;
> +
> +       thr = odp_thread_id();
> +       thr_args = arg;
> +
> +       if (thr_args->srcpktio == 0 || thr_args->dstpktio == 0) {
> +               ODP_ERR("Invalid srcpktio:%d dstpktio:%d\n",
> +                       thr_args->srcpktio, thr_args->dstpktio);
> +               return NULL;
> +       }
> +       printf("[%02i] srcif:%s dstif:%s spktio:%02i dpktio:%02i BURST
> mode\n",
> +              thr, thr_args->srcif, thr_args->dstif, thr_args->srcpktio,
> +              thr_args->dstpktio);
> +
> +       /* Loop packets */
> +       for (;;) {
> +               pkts = odp_pktio_recv(thr_args->srcpktio, pkt_tbl,
> +                                       MAX_PKT_BURST);
> +               if (pkts > 0) {
> +                       /* Drop packets with errors */
> +                       pkts_ok = drop_err_pkts(pkt_tbl, pkts);
> +                       if (pkts_ok > 0)
> +                               odp_pktio_send(thr_args->dstpktio, pkt_tbl,
> +                                              pkts_ok);
> +                       if (odp_unlikely(pkts_ok != pkts))
> +                               ODP_ERR("Dropped frames:%u -
> err_cnt:%lu\n",
> +                                       pkts-pkts_ok, ++err_cnt);
> +
> +                       /* Print packet counts every once in a while */
> +                       tmp += pkts_ok;
> +                       if (odp_unlikely((tmp >= 100000) || /* OR first
> print:*/
> +                           ((pkt_cnt == 0) && ((tmp-1) <
> MAX_PKT_BURST)))) {
> +                               pkt_cnt += tmp;
> +                               printf("  [%02i] pkt_cnt:%lu\n", thr,
> pkt_cnt);
> +                               fflush(NULL);
> +                               tmp = 0;
> +                       }
> +               }
> +       }
> +
> +/* unreachable */
> +}
> +
> +/**
> + * ODP L2 forwarding main function
> + */
> +int main(int argc, char *argv[])
> +{
> +       odp_linux_pthread_t thread_tbl[MAX_WORKERS];
> +       odp_buffer_pool_t pool;
> +       int thr_id;
> +       void *pool_base;
> +       int i;
> +       int first_core;
> +       int core_count;
> +       odp_pktio_t pktio;
> +
> +       /* Init ODP before calling anything else */
> +       if (odp_init_global()) {
> +               ODP_ERR("Error: ODP global init failed.\n");
> +               exit(EXIT_FAILURE);
> +       }
> +
> +       /* Reserve memory for args from shared mem */
> +       args = odp_shm_reserve("shm_args", sizeof(args_t),
> ODP_CACHE_LINE_SIZE);
> +       if (args == NULL) {
> +               ODP_ERR("Error: shared mem alloc failed.\n");
> +               exit(EXIT_FAILURE);
> +       }
> +       memset(args, 0, sizeof(*args));
> +
> +       /* Parse and store the application arguments */
> +       parse_args(argc, argv, &args->appl);
> +
> +       /* Print both system and application information */
> +       print_info(NO_PATH(argv[0]), &args->appl);
> +
> +       core_count  = odp_sys_core_count();
> +       num_workers = core_count;
> +
> +       if (args->appl.core_count)
> +               num_workers = args->appl.core_count;
> +
> +       if (num_workers > MAX_WORKERS)
> +               num_workers = MAX_WORKERS;
> +
> +       printf("Num worker threads: %i\n", num_workers);
> +
> +       if (num_workers < args->appl.if_count) {
> +               ODP_ERR("Error: core count %d is less than interface
> count\n",
> +                       num_workers);
> +               exit(EXIT_FAILURE);
> +       }
> +       if (args->appl.if_count % 2 != 0) {
> +               ODP_ERR("Error: interface count %d is odd in fwd appl.\n",
> +                       args->appl.if_count);
> +               exit(EXIT_FAILURE);
> +       }
> +       /*
> +        * By default core #0 runs Linux kernel background tasks.
> +        * Start mapping thread from core #1
> +        */
> +       first_core = 1;
> +
> +       if (core_count == 1)
> +               first_core = 0;
> +
> +       printf("First core:         %i\n\n", first_core);
> +
> +       /* Init this thread */
> +       thr_id = odp_thread_create(0);
> +       odp_init_local(thr_id);
> +
> +       /* Create packet pool */
> +       pool_base = odp_shm_reserve("shm_packet_pool",
> +                                   SHM_PKT_POOL_SIZE,
> ODP_CACHE_LINE_SIZE);
> +       if (pool_base == NULL) {
> +               ODP_ERR("Error: packet pool mem alloc failed.\n");
> +               exit(EXIT_FAILURE);
> +       }
> +
> +       pool = odp_buffer_pool_create("packet_pool", pool_base,
> +                                     SHM_PKT_POOL_SIZE,
> +                                     SHM_PKT_POOL_BUF_SIZE,
> +                                     ODP_CACHE_LINE_SIZE,
> +                                     ODP_BUFFER_TYPE_PACKET);
> +       if (pool == ODP_BUFFER_POOL_INVALID) {
> +               ODP_ERR("Error: packet pool create failed.\n");
> +               exit(EXIT_FAILURE);
> +       }
> +       odp_buffer_pool_print(pool);
> +
> +       memset(thread_tbl, 0, sizeof(thread_tbl));
> +       /* initialize threads params */
> +       for (i = 0; i < num_workers; ++i) {
> +               int if_idx;
> +
> +               if_idx = i % args->appl.if_count;
> +
> +               args->thread[i].srcif = args->appl.if_names[if_idx];
> +               if (if_idx % 2 == 0)
> +                       args->thread[i].dstif =
> args->appl.if_names[if_idx+1];
> +               else
> +                       args->thread[i].dstif =
> args->appl.if_names[if_idx-1];
> +               args->thread[i].pool = pool;
> +               args->thread[i].mode = args->appl.mode;
> +               args->thread[i].type = args->appl.type;
> +               args->thread[i].fanout = args->appl.fanout;
> +
> +               if (args->appl.mode == APPL_MODE_PKT_BURST) {
> +                       pktio = burst_mode_init_params(&args->thread[i],
> pool);
> +                       if (pktio == ODP_PKTIO_INVALID) {
> +                               ODP_ERR("  for thread:%02i\n", i);
> +                               exit(EXIT_FAILURE);
> +                       }
> +               } else { /* APPL_MODE_PKT_QUEUE */
> +                       pktio = queue_mode_init_params(&args->thread[i],
> pool);
> +                       if (pktio == ODP_PKTIO_INVALID) {
> +                               ODP_ERR("  for thread:%02i\n", i);
> +                               exit(EXIT_FAILURE);
> +                       }
> +               }
> +               args->thread[i].srcpktio = pktio;
> +       }
> +       for (i = 0; i < num_workers; ++i) {
> +               if (i % 2 == 0)
> +                       args->thread[i].dstpktio =
> args->thread[i+1].srcpktio;
> +               else
> +                       args->thread[i].dstpktio =
> args->thread[i-1].srcpktio;
> +       }
> +       /* Create worker threads */
> +       for (i = 0; i < num_workers; ++i) {
> +               void *(*thr_run_func) (void *);
> +               int core;
> +
> +               core = (first_core + i) % core_count;
> +
> +               if (args->appl.mode == APPL_MODE_PKT_BURST)
> +                       thr_run_func = pktio_ifburst_thread;
> +               else /* APPL_MODE_PKT_QUEUE */
> +                       thr_run_func = pktio_queue_thread;
> +               odp_linux_pthread_create(thread_tbl, 1, core, thr_run_func,
> +                                        &args->thread[i]);
> +       }
> +
> +       /* Master thread waits for other threads to exit */
> +       odp_linux_pthread_join(thread_tbl, num_workers);
> +
> +       printf("Exit\n\n");
> +
> +       return 0;
> +}
> +
> +/**
> + * Drop packets which input parsing marked as containing errors.
> + *
> + * Frees packets with error and modifies pkt_tbl[] to only contain
> packets with
> + * no detected errors.
> + *
> + * @param pkt_tbl  Array of packet
> + * @param len      Length of pkt_tbl[]
> + *
> + * @return Number of packets with no detected error
> + */
> +static int drop_err_pkts(odp_packet_t pkt_tbl[], unsigned len)
> +{
> +       odp_packet_t pkt;
> +       unsigned pkt_cnt = len;
> +       unsigned i, j;
> +
> +       for (i = 0, j = 0; i < len; ++i) {
> +               pkt = pkt_tbl[i];
> +
> +               if (odp_unlikely(odp_packet_error(pkt))) {
> +                       odp_packet_free(pkt); /* Drop */
> +                       pkt_cnt--;
> +               } else if (odp_unlikely(i != j++)) {
> +                       pkt_tbl[j] = pkt;
> +               }
> +       }
> +
> +       return pkt_cnt;
> +}
> +
> +/**
> + * Parse and store the command line arguments
> + *
> + * @param argc       argument count
> + * @param argv[]     argument vector
> + * @param appl_args  Store application arguments here
> + */
> +static void parse_args(int argc, char *argv[], appl_args_t *appl_args)
> +{
> +       int opt;
> +       int long_index;
> +       char *names, *str, *token, *save;
> +       size_t len;
> +       int i;
> +       static struct option longopts[] = {
> +               {"count", required_argument, NULL, 'c'},
> +               {"interface", required_argument, NULL, 'i'},    /* return
> 'i' */
> +               {"mode", required_argument, NULL, 'm'},         /* return
> 'm' */
> +               {"help", no_argument, NULL, 'h'},               /* return
> 'h' */
> +               {NULL, 0, NULL, 0}
> +       };
> +
> +       appl_args->mode = -1; /* Invalid, must be changed by parsing */
> +       appl_args->type = 3;  /* 3: ODP_PKTIO_TYPE_SOCKET_MMAP */
> +       appl_args->fanout = 1; /* turn off fanout by default for mmap */
> +
> +       while (1) {
> +               opt = getopt_long(argc, argv, "+c:i:m:t:f:h",
> +                                 longopts, &long_index);
> +
> +               if (opt == -1)
> +                       break;  /* No more options */
> +
> +               switch (opt) {
> +               case 'c':
> +                       appl_args->core_count = atoi(optarg);
> +                       break;
> +                       /* parse packet-io interface names */
> +               case 'i':
> +                       len = strlen(optarg);
> +                       if (len == 0) {
> +                               usage(argv[0]);
> +                               exit(EXIT_FAILURE);
> +                       }
> +                       len += 1;       /* add room for '\0' */
> +
> +                       names = malloc(len);
> +                       if (names == NULL) {
> +                               usage(argv[0]);
> +                               exit(EXIT_FAILURE);
> +                       }
> +
> +                       /* count the number of tokens separated by ',' */
> +                       strcpy(names, optarg);
> +                       for (str = names, i = 0;; str = NULL, i++) {
> +                               token = strtok_r(str, ",", &save);
> +                               if (token == NULL)
> +                                       break;
> +                       }
> +                       appl_args->if_count = i;
> +
> +                       if (appl_args->if_count == 0) {
> +                               usage(argv[0]);
> +                               exit(EXIT_FAILURE);
> +                       }
> +
> +                       /* allocate storage for the if names */
> +                       appl_args->if_names =
> +                           calloc(appl_args->if_count, sizeof(char *));
> +
> +                       /* store the if names (reset names string) */
> +                       strcpy(names, optarg);
> +                       for (str = names, i = 0;; str = NULL, i++) {
> +                               token = strtok_r(str, ",", &save);
> +                               if (token == NULL)
> +                                       break;
> +                               appl_args->if_names[i] = token;
> +                       }
> +                       break;
> +
> +               case 'm':
> +                       i = atoi(optarg);
> +                       if (i == 0)
> +                               appl_args->mode = APPL_MODE_PKT_BURST;
> +                       else
> +                               appl_args->mode = APPL_MODE_PKT_QUEUE;
> +                       break;
> +
> +               case 't':
> +                       appl_args->type = atoi(optarg);
> +                       break;
> +
> +               case 'f':
> +                       appl_args->fanout = atoi(optarg);
> +                       break;
> +
> +               case 'h':
> +                       usage(argv[0]);
> +                       exit(EXIT_SUCCESS);
> +                       break;
> +
> +               default:
> +                       break;
> +               }
> +       }
> +
> +       if (appl_args->if_count == 0 || appl_args->mode == -1) {
> +               usage(argv[0]);
> +               exit(EXIT_FAILURE);
> +       }
> +
> +       optind = 1;             /* reset 'extern optind' from the getopt
> lib */
> +}
> +
> +/**
> + * Print system and application info
> + */
> +static void print_info(char *progname, appl_args_t *appl_args)
> +{
> +       int i;
> +
> +       printf("\n"
> +              "ODP system info\n"
> +              "---------------\n"
> +              "ODP API version: %s\n"
> +              "CPU model:       %s\n"
> +              "CPU freq (hz):   %"PRIu64"\n"
> +              "Cache line size: %i\n"
> +              "Core count:      %i\n"
> +              "\n",
> +              odp_version_api_str(), odp_sys_cpu_model_str(),
> odp_sys_cpu_hz(),
> +              odp_sys_cache_line_size(), odp_sys_core_count());
> +
> +       printf("Running ODP appl: \"%s\"\n"
> +              "-----------------\n"
> +              "IF-count:        %i\n"
> +              "Using IFs:      ",
> +              progname, appl_args->if_count);
> +       for (i = 0; i < appl_args->if_count; ++i)
> +               printf(" %s", appl_args->if_names[i]);
> +       printf("\n"
> +              "Mode:            ");
> +       if (appl_args->mode == APPL_MODE_PKT_BURST)
> +               PRINT_APPL_MODE(APPL_MODE_PKT_BURST);
> +       else
> +               PRINT_APPL_MODE(APPL_MODE_PKT_QUEUE);
> +       printf("\n\n");
> +       fflush(NULL);
> +}
> +
> +/**
> + * Prinf usage information
> + */
> +static void usage(char *progname)
> +{
> +       printf("\n"
> +              "OpenDataPlane L2 forwarding application.\n"
> +              "\n"
> +              "Usage: %s OPTIONS\n"
> +              "  E.g. %s -i eth0,eth1,eth2,eth3 -m 0 -t 1\n"
> +              " In the above example,\n"
> +              " eth0 will send pkts to eth1 and vice versa\n"
> +              " eth2 will send pkts to eth3 and vice versa\n"
> +              "\n"
> +              "Mandatory OPTIONS:\n"
> +              "  -i, --interface Eth interfaces (comma-separated, no
> spaces)\n"
> +              "  -m, --mode      0: Burst send&receive packets (no
> queues)\n"
> +              "                  1: Send&receive packets through ODP
> queues.\n"
> +              " -t, --type   1: ODP_PKTIO_TYPE_SOCKET_BASIC\n"
> +              "              2: ODP_PKTIO_TYPE_SOCKET_MMSG\n"
> +              "              3: ODP_PKTIO_TYPE_SOCKET_MMAP\n"
> +              "              4: ODP_PKTIO_TYPE_NETMAP\n"
> +              "         Default: 3: ODP_PKTIO_TYPE_SOCKET_MMAP\n"
> +              " -f, --fanout 0: off 1: on (Default 1: on)\n"
> +              "\n"
> +              "Optional OPTIONS\n"
> +              "  -c, --count <number> Core count.\n"
> +              "  -h, --help           Display help and exit.\n\n"
> +              "\n", NO_PATH(progname), NO_PATH(progname)
> +           );
> +}
> --
> 1.9.1
>
>
> _______________________________________________
> lng-odp mailing list
> lng-odp@lists.linaro.org
> http://lists.linaro.org/mailman/listinfo/lng-odp
>

Taras,

Will you be able to test this on KS2 ?

Mike


On 13 May 2014 10:35, Mike Holmes <mike.holmes@linaro.org> wrote:

> Taras
>
>
> On 13 May 2014 10:18, <venkatesh.vivekanandan@linaro.org> wrote:
>
>> From: Venkatesh Vivekanandan <venkatesh.vivekanandan@linaro.org>
>>
>> Initial L2 forwarding application based on example pktio that
>> resembles DPDK's in functionality.
>>
>> It is based on raw sockets and not tested on any network HW accelerator.
>>
>> Create pktio and queue for all threads before starting pthread.
>>
>> It has two modes supported,
>>         1. burst mode(without ODP queues)
>>         2. queue mode (with ODP queues and scheduler)
>>    and three types,
>>         1. raw sockets
>>         2. multiple messages on sockets
>>         3. mmap on sockets
>>
>> Signed-off-by: Venkatesh Vivekanandan <venkatesh.vivekanandan@linaro.org>
>> ---
>>  test/l2fwd/Makefile |  44 ++++
>>  test/l2fwd/l2fwd.c  | 637
>> ++++++++++++++++++++++++++++++++++++++++++++++++++++
>>  2 files changed, 681 insertions(+)
>>  create mode 100644 test/l2fwd/Makefile
>>  create mode 100644 test/l2fwd/l2fwd.c
>>
>> diff --git a/test/l2fwd/Makefile b/test/l2fwd/Makefile
>> new file mode 100644
>> index 0000000..3435caa
>> --- /dev/null
>> +++ b/test/l2fwd/Makefile
>> @@ -0,0 +1,44 @@
>> +# Copyright (c) 2014, Linaro Limited
>> +# All rights reserved.
>> +#
>> +# SPDX-License-Identifier:     BSD-3-Clause
>> +
>> +ODP_ROOT = ../..
>> +ODP_APP  = l2fwd
>> +
>> +include $(ODP_ROOT)/Makefile.inc
>> +include ../Makefile.inc
>> +
>> +OBJS     =
>> +OBJS    += $(OBJ_DIR)/l2fwd.o
>> +
>> +DEPS     = $(OBJS:.o=.d)
>> +
>> +.PHONY: default
>> +default: $(OBJ_DIR) $(ODP_APP)
>> +
>> +-include $(DEPS)
>> +
>> +#
>> +# Compile rules
>> +#
>> +$(OBJ_DIR)/%.o: %.c
>> +       $(ECHO) Compiling $<
>> +       $(CC) -c -MD $(EXTRA_CFLAGS) $(CFLAGS) -o $@ $<
>> +
>> +#
>> +# Link rule
>> +#
>> +$(ODP_APP): $(ODP_LIB) $(OBJS)
>> +       $(CC) $(LDFLAGS) $(OBJS) $(ODP_LIB) $(STD_LIBS) -o $@
>> +
>> +.PHONY: clean
>> +clean:
>> +       $(RMDIR) $(OBJ_DIR)
>> +       $(RM) $(ODP_APP)
>> +       $(MAKE) -C $(ODP_DIR) clean
>> +
>> +.PHONY: install
>> +install:
>> +       install -d $(DESTDIR)/share/odp
>> +       install -m 0755 $(ODP_APP) $(DESTDIR)/share/odp/
>> diff --git a/test/l2fwd/l2fwd.c b/test/l2fwd/l2fwd.c
>> new file mode 100644
>> index 0000000..fb1b949
>> --- /dev/null
>> +++ b/test/l2fwd/l2fwd.c
>> @@ -0,0 +1,637 @@
>> +/* Copyright (c) 2014, Linaro Limited
>> + * All rights reserved.
>> + *
>> + * SPDX-License-Identifier:     BSD-3-Clause
>> + */
>> +
>> +/**
>> + * @file
>> + *
>> + * @example l2fwd.c  ODP basic forwarding application
>> + */
>> +
>> +#include <stdlib.h>
>> +#include <string.h>
>> +#include <getopt.h>
>> +#include <unistd.h>
>> +
>> +#include <odp.h>
>> +#include <helper/odp_linux.h>
>> +#include <helper/odp_packet_helper.h>
>> +#include <helper/odp_eth.h>
>> +#include <helper/odp_ip.h>
>> +
>> +#define MAX_WORKERS            32
>> +#define SHM_PKT_POOL_SIZE      (512*2048)
>> +#define SHM_PKT_POOL_BUF_SIZE  1856
>> +#define MAX_PKT_BURST          16
>> +
>> +#define APPL_MODE_PKT_BURST    0
>> +#define APPL_MODE_PKT_QUEUE    1
>> +
>> +#define PRINT_APPL_MODE(x) printf("%s(%i)\n", #x, (x))
>> +
>> +/** Get rid of path in filename - only for unix-type paths using '/' */
>> +#define NO_PATH(file_name) (strrchr((file_name), '/') ? \
>> +                           strrchr((file_name), '/') + 1 : (file_name))
>> +/**
>> + * Parsed command line application arguments
>> + */
>> +typedef struct {
>> +       int core_count;
>> +       int if_count;           /**< Number of interfaces to be used */
>> +       char **if_names;        /**< Array of pointers to interface names
>> */
>> +       int mode;               /**< Packet IO mode */
>> +       int type;               /**< Packet IO type */
>> +       int fanout;             /**< Packet IO fanout */
>> +       odp_buffer_pool_t pool; /**< Buffer pool for packet IO */
>> +} appl_args_t;
>> +
>> +/**
>> + * Thread specific arguments
>> + */
>> +typedef struct {
>> +       char *srcif;            /**< Source Interface */
>> +       char *dstif;            /**< Dest Interface */
>> +       odp_buffer_pool_t pool; /**< Buffer pool for packet IO */
>> +       odp_pktio_t srcpktio;   /**< Source pktio handle */
>> +       odp_pktio_t dstpktio;   /**< Destination pktio handle */
>> +       int mode;               /**< Thread mode */
>> +       int type;               /**< Thread i/o type */
>> +       int fanout;             /**< Thread i/o fanout */
>> +} thread_args_t;
>> +
>> +/**
>> + * Grouping of both parsed CL args and thread specific args - alloc
>> together
>> + */
>> +typedef struct {
>> +       /** Application (parsed) arguments */
>> +       appl_args_t appl;
>> +       /** Thread specific arguments */
>> +       thread_args_t thread[MAX_WORKERS];
>> +} args_t;
>> +
>> +/** Global pointer to args */
>> +static args_t *args;
>> +int num_workers;
>> +
>> +/* helper funcs */
>> +static int drop_err_pkts(odp_packet_t pkt_tbl[], unsigned len);
>> +static void parse_args(int argc, char *argv[], appl_args_t *appl_args);
>> +static void print_info(char *progname, appl_args_t *appl_args);
>> +static void usage(char *progname);
>> +
>> +/**
>> + * Burst mode: pktio for each thread will be created with either same or
>> + * different params
>> + *
>> + * @param arg  thread arguments of type 'thread_args_t *'
>> + * @param pool is the packet pool from where buffers should be taken
>> + */
>> +static odp_pktio_t burst_mode_init_params(void *arg, int pool)
>> +{
>> +       thread_args_t *args;
>> +       odp_pktio_params_t params;
>> +       socket_params_t *sock_params = &params.sock_params;
>> +       odp_pktio_t pktio;
>> +
>> +       args = arg;
>> +       /* Open a packet IO instance for this thread */
>> +       sock_params->type = args->type;
>> +       sock_params->fanout = args->fanout;
>> +       pktio = odp_pktio_open(args->srcif, pool, &params);
>> +       if (pktio == ODP_PKTIO_INVALID)
>> +               ODP_ERR("  Error: pktio create failed");
>> +
>> +       return pktio;
>> +}
>> +
>> +/**
>> + * Queue mode: pktio for each thread will be created with either same or
>> + * different params. Queues are created and attached to the pktio.
>> + *
>> + * @param arg  thread arguments of type 'thread_args_t *'
>> + * @param pool is the packet pool from where buffers should be taken
>> + */
>> +static odp_pktio_t queue_mode_init_params(void *arg, int pool)
>> +{
>> +       char inq_name[ODP_QUEUE_NAME_LEN];
>> +       odp_queue_param_t qparam;
>> +       odp_queue_t inq_def;
>> +       int ret;
>> +       odp_pktio_t pktio = ODP_PKTIO_INVALID;
>> +
>> +       pktio = burst_mode_init_params(arg, pool);
>> +       if (pktio == ODP_PKTIO_INVALID)
>> +               return pktio;
>> +       /*
>> +        * Create and set the default INPUT queue associated with the
>> 'pktio'
>> +        * resource
>> +        */
>> +       qparam.sched.prio  = ODP_SCHED_PRIO_DEFAULT;
>> +       qparam.sched.sync  = ODP_SCHED_SYNC_ATOMIC;
>> +       qparam.sched.group = ODP_SCHED_GROUP_DEFAULT;
>> +       snprintf(inq_name, sizeof(inq_name), "%i-pktio_inq_def",
>> (int)pktio);
>> +       inq_name[ODP_QUEUE_NAME_LEN - 1] = '\0';
>> +
>> +       inq_def = odp_queue_create(inq_name, ODP_QUEUE_TYPE_PKTIN,
>> &qparam);
>> +       if (inq_def == ODP_QUEUE_INVALID) {
>> +               ODP_ERR("  Error: pktio queue creation failed");
>> +               return ODP_PKTIO_INVALID;
>> +       }
>> +
>> +       ret = odp_pktio_inq_setdef(pktio, inq_def);
>> +       if (ret != 0) {
>> +               ODP_ERR("  Error: default input-Q setup");
>> +               return ODP_PKTIO_INVALID;
>> +       }
>> +
>> +       return pktio;
>> +}
>> +
>> +/**
>> + * Packet IO worker thread using ODP queues
>> + *
>> + * @param arg  thread arguments of type 'thread_args_t *'
>> + */
>> +static void *pktio_queue_thread(void *arg)
>> +{
>> +       int thr, i;
>> +       thread_args_t *thr_args;
>> +       char dstpktio[MAX_WORKERS+1];
>> +       odp_queue_t outq_def;
>> +       odp_packet_t pkt;
>> +       odp_buffer_t buf;
>> +       unsigned long pkt_cnt = 0;
>> +       unsigned long err_cnt = 0;
>> +
>> +       thr = odp_thread_id();
>> +       thr_args = arg;
>> +
>> +       if (thr_args->srcpktio == 0 || thr_args->dstpktio == 0) {
>> +               ODP_ERR("Invalid srcpktio:%d dstpktio:%d\n",
>> +                       thr_args->srcpktio, thr_args->dstpktio);
>> +               return NULL;
>> +       }
>> +       printf("[%02i] srcif:%s dstif:%s spktio:%02i dpktio:%02i QUEUE
>> mode\n",
>> +              thr, thr_args->srcif, thr_args->dstif, thr_args->srcpktio,
>> +              thr_args->dstpktio);
>> +
>> +       /* Populate an array of destination pktio's in all threads as the
>> +        * scheduler can take packets from any input queue
>> +        */
>> +       for (i = 0; i < num_workers; i++)
>> +               dstpktio[i+1] = args->thread[i].dstpktio;
>> +
>> +       /* Loop packets */
>> +       for (;;) {
>> +               odp_pktio_t pktio_tmp;
>> +
>> +               /* Use schedule to get buf from any input queue */
>> +               buf = odp_schedule(NULL, ODP_SCHED_WAIT);
>> +
>> +               pkt = odp_packet_from_buffer(buf);
>> +               /* Drop packets with errors */
>> +               if (odp_unlikely(drop_err_pkts(&pkt, 1) == 0)) {
>> +                       ODP_ERR("Drop frame - err_cnt:%lu\n", ++err_cnt);
>> +                       continue;
>> +               }
>> +
>> +               pktio_tmp = odp_pktio_get_input(pkt);
>> +               outq_def = odp_pktio_outq_getdef(dstpktio[pktio_tmp]);
>> +               if (outq_def == ODP_QUEUE_INVALID) {
>> +                       ODP_ERR("  [%02i] Error: def output-Q query\n",
>> thr);
>> +                       return NULL;
>> +               }
>> +
>> +               /* Enqueue the packet for output */
>> +               odp_queue_enq(outq_def, buf);
>> +
>> +               /* Print packet counts every once in a while */
>> +               if (odp_unlikely(pkt_cnt++ % 100000 == 0)) {
>> +                       printf("  [%02i] pkt_cnt:%lu\n", thr, pkt_cnt);
>> +                       fflush(NULL);
>> +               }
>> +       }
>> +
>> +/* unreachable */
>> +}
>> +
>> +/**
>> + * Packet IO worker thread using bursts from/to IO resources
>> + *
>> + * @param arg  thread arguments of type 'thread_args_t *'
>> + */
>> +static void *pktio_ifburst_thread(void *arg)
>> +{
>> +       int thr;
>> +       thread_args_t *thr_args;
>> +       int pkts, pkts_ok;
>> +       odp_packet_t pkt_tbl[MAX_PKT_BURST];
>> +       unsigned long pkt_cnt = 0;
>> +       unsigned long err_cnt = 0;
>> +       unsigned long tmp = 0;
>> +
>> +       thr = odp_thread_id();
>> +       thr_args = arg;
>> +
>> +       if (thr_args->srcpktio == 0 || thr_args->dstpktio == 0) {
>> +               ODP_ERR("Invalid srcpktio:%d dstpktio:%d\n",
>> +                       thr_args->srcpktio, thr_args->dstpktio);
>> +               return NULL;
>> +       }
>> +       printf("[%02i] srcif:%s dstif:%s spktio:%02i dpktio:%02i BURST
>> mode\n",
>> +              thr, thr_args->srcif, thr_args->dstif, thr_args->srcpktio,
>> +              thr_args->dstpktio);
>> +
>> +       /* Loop packets */
>> +       for (;;) {
>> +               pkts = odp_pktio_recv(thr_args->srcpktio, pkt_tbl,
>> +                                       MAX_PKT_BURST);
>> +               if (pkts > 0) {
>> +                       /* Drop packets with errors */
>> +                       pkts_ok = drop_err_pkts(pkt_tbl, pkts);
>> +                       if (pkts_ok > 0)
>> +                               odp_pktio_send(thr_args->dstpktio,
>> pkt_tbl,
>> +                                              pkts_ok);
>> +                       if (odp_unlikely(pkts_ok != pkts))
>> +                               ODP_ERR("Dropped frames:%u -
>> err_cnt:%lu\n",
>> +                                       pkts-pkts_ok, ++err_cnt);
>> +
>> +                       /* Print packet counts every once in a while */
>> +                       tmp += pkts_ok;
>> +                       if (odp_unlikely((tmp >= 100000) || /* OR first
>> print:*/
>> +                           ((pkt_cnt == 0) && ((tmp-1) <
>> MAX_PKT_BURST)))) {
>> +                               pkt_cnt += tmp;
>> +                               printf("  [%02i] pkt_cnt:%lu\n", thr,
>> pkt_cnt);
>> +                               fflush(NULL);
>> +                               tmp = 0;
>> +                       }
>> +               }
>> +       }
>> +
>> +/* unreachable */
>> +}
>> +
>> +/**
>> + * ODP L2 forwarding main function
>> + */
>> +int main(int argc, char *argv[])
>> +{
>> +       odp_linux_pthread_t thread_tbl[MAX_WORKERS];
>> +       odp_buffer_pool_t pool;
>> +       int thr_id;
>> +       void *pool_base;
>> +       int i;
>> +       int first_core;
>> +       int core_count;
>> +       odp_pktio_t pktio;
>> +
>> +       /* Init ODP before calling anything else */
>> +       if (odp_init_global()) {
>> +               ODP_ERR("Error: ODP global init failed.\n");
>> +               exit(EXIT_FAILURE);
>> +       }
>> +
>> +       /* Reserve memory for args from shared mem */
>> +       args = odp_shm_reserve("shm_args", sizeof(args_t),
>> ODP_CACHE_LINE_SIZE);
>> +       if (args == NULL) {
>> +               ODP_ERR("Error: shared mem alloc failed.\n");
>> +               exit(EXIT_FAILURE);
>> +       }
>> +       memset(args, 0, sizeof(*args));
>> +
>> +       /* Parse and store the application arguments */
>> +       parse_args(argc, argv, &args->appl);
>> +
>> +       /* Print both system and application information */
>> +       print_info(NO_PATH(argv[0]), &args->appl);
>> +
>> +       core_count  = odp_sys_core_count();
>> +       num_workers = core_count;
>> +
>> +       if (args->appl.core_count)
>> +               num_workers = args->appl.core_count;
>> +
>> +       if (num_workers > MAX_WORKERS)
>> +               num_workers = MAX_WORKERS;
>> +
>> +       printf("Num worker threads: %i\n", num_workers);
>> +
>> +       if (num_workers < args->appl.if_count) {
>> +               ODP_ERR("Error: core count %d is less than interface
>> count\n",
>> +                       num_workers);
>> +               exit(EXIT_FAILURE);
>> +       }
>> +       if (args->appl.if_count % 2 != 0) {
>> +               ODP_ERR("Error: interface count %d is odd in fwd appl.\n",
>> +                       args->appl.if_count);
>> +               exit(EXIT_FAILURE);
>> +       }
>> +       /*
>> +        * By default core #0 runs Linux kernel background tasks.
>> +        * Start mapping thread from core #1
>> +        */
>> +       first_core = 1;
>> +
>> +       if (core_count == 1)
>> +               first_core = 0;
>> +
>> +       printf("First core:         %i\n\n", first_core);
>> +
>> +       /* Init this thread */
>> +       thr_id = odp_thread_create(0);
>> +       odp_init_local(thr_id);
>> +
>> +       /* Create packet pool */
>> +       pool_base = odp_shm_reserve("shm_packet_pool",
>> +                                   SHM_PKT_POOL_SIZE,
>> ODP_CACHE_LINE_SIZE);
>> +       if (pool_base == NULL) {
>> +               ODP_ERR("Error: packet pool mem alloc failed.\n");
>> +               exit(EXIT_FAILURE);
>> +       }
>> +
>> +       pool = odp_buffer_pool_create("packet_pool", pool_base,
>> +                                     SHM_PKT_POOL_SIZE,
>> +                                     SHM_PKT_POOL_BUF_SIZE,
>> +                                     ODP_CACHE_LINE_SIZE,
>> +                                     ODP_BUFFER_TYPE_PACKET);
>> +       if (pool == ODP_BUFFER_POOL_INVALID) {
>> +               ODP_ERR("Error: packet pool create failed.\n");
>> +               exit(EXIT_FAILURE);
>> +       }
>> +       odp_buffer_pool_print(pool);
>> +
>> +       memset(thread_tbl, 0, sizeof(thread_tbl));
>> +       /* initialize threads params */
>> +       for (i = 0; i < num_workers; ++i) {
>> +               int if_idx;
>> +
>> +               if_idx = i % args->appl.if_count;
>> +
>> +               args->thread[i].srcif = args->appl.if_names[if_idx];
>> +               if (if_idx % 2 == 0)
>> +                       args->thread[i].dstif =
>> args->appl.if_names[if_idx+1];
>> +               else
>> +                       args->thread[i].dstif =
>> args->appl.if_names[if_idx-1];
>> +               args->thread[i].pool = pool;
>> +               args->thread[i].mode = args->appl.mode;
>> +               args->thread[i].type = args->appl.type;
>> +               args->thread[i].fanout = args->appl.fanout;
>> +
>> +               if (args->appl.mode == APPL_MODE_PKT_BURST) {
>> +                       pktio = burst_mode_init_params(&args->thread[i],
>> pool);
>> +                       if (pktio == ODP_PKTIO_INVALID) {
>> +                               ODP_ERR("  for thread:%02i\n", i);
>> +                               exit(EXIT_FAILURE);
>> +                       }
>> +               } else { /* APPL_MODE_PKT_QUEUE */
>> +                       pktio = queue_mode_init_params(&args->thread[i],
>> pool);
>> +                       if (pktio == ODP_PKTIO_INVALID) {
>> +                               ODP_ERR("  for thread:%02i\n", i);
>> +                               exit(EXIT_FAILURE);
>> +                       }
>> +               }
>> +               args->thread[i].srcpktio = pktio;
>> +       }
>> +       for (i = 0; i < num_workers; ++i) {
>> +               if (i % 2 == 0)
>> +                       args->thread[i].dstpktio =
>> args->thread[i+1].srcpktio;
>> +               else
>> +                       args->thread[i].dstpktio =
>> args->thread[i-1].srcpktio;
>> +       }
>> +       /* Create worker threads */
>> +       for (i = 0; i < num_workers; ++i) {
>> +               void *(*thr_run_func) (void *);
>> +               int core;
>> +
>> +               core = (first_core + i) % core_count;
>> +
>> +               if (args->appl.mode == APPL_MODE_PKT_BURST)
>> +                       thr_run_func = pktio_ifburst_thread;
>> +               else /* APPL_MODE_PKT_QUEUE */
>> +                       thr_run_func = pktio_queue_thread;
>> +               odp_linux_pthread_create(thread_tbl, 1, core,
>> thr_run_func,
>> +                                        &args->thread[i]);
>> +       }
>> +
>> +       /* Master thread waits for other threads to exit */
>> +       odp_linux_pthread_join(thread_tbl, num_workers);
>> +
>> +       printf("Exit\n\n");
>> +
>> +       return 0;
>> +}
>> +
>> +/**
>> + * Drop packets which input parsing marked as containing errors.
>> + *
>> + * Frees packets with error and modifies pkt_tbl[] to only contain
>> packets with
>> + * no detected errors.
>> + *
>> + * @param pkt_tbl  Array of packet
>> + * @param len      Length of pkt_tbl[]
>> + *
>> + * @return Number of packets with no detected error
>> + */
>> +static int drop_err_pkts(odp_packet_t pkt_tbl[], unsigned len)
>> +{
>> +       odp_packet_t pkt;
>> +       unsigned pkt_cnt = len;
>> +       unsigned i, j;
>> +
>> +       for (i = 0, j = 0; i < len; ++i) {
>> +               pkt = pkt_tbl[i];
>> +
>> +               if (odp_unlikely(odp_packet_error(pkt))) {
>> +                       odp_packet_free(pkt); /* Drop */
>> +                       pkt_cnt--;
>> +               } else if (odp_unlikely(i != j++)) {
>> +                       pkt_tbl[j] = pkt;
>> +               }
>> +       }
>> +
>> +       return pkt_cnt;
>> +}
>> +
>> +/**
>> + * Parse and store the command line arguments
>> + *
>> + * @param argc       argument count
>> + * @param argv[]     argument vector
>> + * @param appl_args  Store application arguments here
>> + */
>> +static void parse_args(int argc, char *argv[], appl_args_t *appl_args)
>> +{
>> +       int opt;
>> +       int long_index;
>> +       char *names, *str, *token, *save;
>> +       size_t len;
>> +       int i;
>> +       static struct option longopts[] = {
>> +               {"count", required_argument, NULL, 'c'},
>> +               {"interface", required_argument, NULL, 'i'},    /* return
>> 'i' */
>> +               {"mode", required_argument, NULL, 'm'},         /* return
>> 'm' */
>> +               {"help", no_argument, NULL, 'h'},               /* return
>> 'h' */
>> +               {NULL, 0, NULL, 0}
>> +       };
>> +
>> +       appl_args->mode = -1; /* Invalid, must be changed by parsing */
>> +       appl_args->type = 3;  /* 3: ODP_PKTIO_TYPE_SOCKET_MMAP */
>> +       appl_args->fanout = 1; /* turn off fanout by default for mmap */
>> +
>> +       while (1) {
>> +               opt = getopt_long(argc, argv, "+c:i:m:t:f:h",
>> +                                 longopts, &long_index);
>> +
>> +               if (opt == -1)
>> +                       break;  /* No more options */
>> +
>> +               switch (opt) {
>> +               case 'c':
>> +                       appl_args->core_count = atoi(optarg);
>> +                       break;
>> +                       /* parse packet-io interface names */
>> +               case 'i':
>> +                       len = strlen(optarg);
>> +                       if (len == 0) {
>> +                               usage(argv[0]);
>> +                               exit(EXIT_FAILURE);
>> +                       }
>> +                       len += 1;       /* add room for '\0' */
>> +
>> +                       names = malloc(len);
>> +                       if (names == NULL) {
>> +                               usage(argv[0]);
>> +                               exit(EXIT_FAILURE);
>> +                       }
>> +
>> +                       /* count the number of tokens separated by ',' */
>> +                       strcpy(names, optarg);
>> +                       for (str = names, i = 0;; str = NULL, i++) {
>> +                               token = strtok_r(str, ",", &save);
>> +                               if (token == NULL)
>> +                                       break;
>> +                       }
>> +                       appl_args->if_count = i;
>> +
>> +                       if (appl_args->if_count == 0) {
>> +                               usage(argv[0]);
>> +                               exit(EXIT_FAILURE);
>> +                       }
>> +
>> +                       /* allocate storage for the if names */
>> +                       appl_args->if_names =
>> +                           calloc(appl_args->if_count, sizeof(char *));
>> +
>> +                       /* store the if names (reset names string) */
>> +                       strcpy(names, optarg);
>> +                       for (str = names, i = 0;; str = NULL, i++) {
>> +                               token = strtok_r(str, ",", &save);
>> +                               if (token == NULL)
>> +                                       break;
>> +                               appl_args->if_names[i] = token;
>> +                       }
>> +                       break;
>> +
>> +               case 'm':
>> +                       i = atoi(optarg);
>> +                       if (i == 0)
>> +                               appl_args->mode = APPL_MODE_PKT_BURST;
>> +                       else
>> +                               appl_args->mode = APPL_MODE_PKT_QUEUE;
>> +                       break;
>> +
>> +               case 't':
>> +                       appl_args->type = atoi(optarg);
>> +                       break;
>> +
>> +               case 'f':
>> +                       appl_args->fanout = atoi(optarg);
>> +                       break;
>> +
>> +               case 'h':
>> +                       usage(argv[0]);
>> +                       exit(EXIT_SUCCESS);
>> +                       break;
>> +
>> +               default:
>> +                       break;
>> +               }
>> +       }
>> +
>> +       if (appl_args->if_count == 0 || appl_args->mode == -1) {
>> +               usage(argv[0]);
>> +               exit(EXIT_FAILURE);
>> +       }
>> +
>> +       optind = 1;             /* reset 'extern optind' from the getopt
>> lib */
>> +}
>> +
>> +/**
>> + * Print system and application info
>> + */
>> +static void print_info(char *progname, appl_args_t *appl_args)
>> +{
>> +       int i;
>> +
>> +       printf("\n"
>> +              "ODP system info\n"
>> +              "---------------\n"
>> +              "ODP API version: %s\n"
>> +              "CPU model:       %s\n"
>> +              "CPU freq (hz):   %"PRIu64"\n"
>> +              "Cache line size: %i\n"
>> +              "Core count:      %i\n"
>> +              "\n",
>> +              odp_version_api_str(), odp_sys_cpu_model_str(),
>> odp_sys_cpu_hz(),
>> +              odp_sys_cache_line_size(), odp_sys_core_count());
>> +
>> +       printf("Running ODP appl: \"%s\"\n"
>> +              "-----------------\n"
>> +              "IF-count:        %i\n"
>> +              "Using IFs:      ",
>> +              progname, appl_args->if_count);
>> +       for (i = 0; i < appl_args->if_count; ++i)
>> +               printf(" %s", appl_args->if_names[i]);
>> +       printf("\n"
>> +              "Mode:            ");
>> +       if (appl_args->mode == APPL_MODE_PKT_BURST)
>> +               PRINT_APPL_MODE(APPL_MODE_PKT_BURST);
>> +       else
>> +               PRINT_APPL_MODE(APPL_MODE_PKT_QUEUE);
>> +       printf("\n\n");
>> +       fflush(NULL);
>> +}
>> +
>> +/**
>> + * Prinf usage information
>> + */
>> +static void usage(char *progname)
>> +{
>> +       printf("\n"
>> +              "OpenDataPlane L2 forwarding application.\n"
>> +              "\n"
>> +              "Usage: %s OPTIONS\n"
>> +              "  E.g. %s -i eth0,eth1,eth2,eth3 -m 0 -t 1\n"
>> +              " In the above example,\n"
>> +              " eth0 will send pkts to eth1 and vice versa\n"
>> +              " eth2 will send pkts to eth3 and vice versa\n"
>> +              "\n"
>> +              "Mandatory OPTIONS:\n"
>> +              "  -i, --interface Eth interfaces (comma-separated, no
>> spaces)\n"
>> +              "  -m, --mode      0: Burst send&receive packets (no
>> queues)\n"
>> +              "                  1: Send&receive packets through ODP
>> queues.\n"
>> +              " -t, --type   1: ODP_PKTIO_TYPE_SOCKET_BASIC\n"
>> +              "              2: ODP_PKTIO_TYPE_SOCKET_MMSG\n"
>> +              "              3: ODP_PKTIO_TYPE_SOCKET_MMAP\n"
>> +              "              4: ODP_PKTIO_TYPE_NETMAP\n"
>> +              "         Default: 3: ODP_PKTIO_TYPE_SOCKET_MMAP\n"
>> +              " -f, --fanout 0: off 1: on (Default 1: on)\n"
>> +              "\n"
>> +              "Optional OPTIONS\n"
>> +              "  -c, --count <number> Core count.\n"
>> +              "  -h, --help           Display help and exit.\n\n"
>> +              "\n", NO_PATH(progname), NO_PATH(progname)
>> +           );
>> +}
>> --
>> 1.9.1
>>
>>
>> _______________________________________________
>> lng-odp mailing list
>> lng-odp@lists.linaro.org
>> http://lists.linaro.org/mailman/listinfo/lng-odp
>>
>
>
>
> --
> *Mike Holmes*
> Linaro Technical Manager / Lead
> LNG - ODP
>

On 05/13/2014 06:18 PM, venkatesh.vivekanandan@linaro.org wrote:
> +	static struct option longopts[] = {
> +		{"count", required_argument, NULL, 'c'},
> +		{"interface", required_argument, NULL, 'i'},	/* return 'i' */
> +		{"mode", required_argument, NULL, 'm'},		/* return 'm' */
> +		{"help", no_argument, NULL, 'h'},		/* return 'h' */
> +		{NULL, 0, NULL, 0}
> +	};
what about options -f -t and -i eth0,eth1,eth3? You have them in parsing 
and usage. Needed or extend longopts or
remove options completely.

Best regards,
Maxim.

On 05/13/2014 05:35 PM, Mike Holmes wrote:
> Taras,
>
> Will you be able to test this on KS2 ?

Yes I'll test it.

On 14 May 2014 17:45, Maxim Uvarov <maxim.uvarov@linaro.org> wrote:

> On 05/13/2014 06:18 PM, venkatesh.vivekanandan@linaro.org wrote:
>
>> +       static struct option longopts[] = {
>> +               {"count", required_argument, NULL, 'c'},
>> +               {"interface", required_argument, NULL, 'i'},    /* return
>> 'i' */
>> +               {"mode", required_argument, NULL, 'm'},         /* return
>> 'm' */
>> +               {"help", no_argument, NULL, 'h'},               /* return
>> 'h' */
>> +               {NULL, 0, NULL, 0}
>> +       };
>>
> what about options -f -t and -i eth0,eth1,eth3? You have them in parsing
> and usage. Needed or extend longopts or
>
"-i" is a mandatory argument and "-f" and "-t" are optional arguments. I
will add them and send out patch v2.

> remove options completely.
>
> Best regards,
> Maxim.
>
>
> _______________________________________________
> lng-odp mailing list
> lng-odp@lists.linaro.org
> http://lists.linaro.org/mailman/listinfo/lng-odp
>

Applied, thanks!

Maxim.

On 05/13/2014 06:18 PM, venkatesh.vivekanandan@linaro.org wrote:
> From: Venkatesh Vivekanandan <venkatesh.vivekanandan@linaro.org>
>
> Initial L2 forwarding application based on example pktio that
> resembles DPDK's in functionality.
>
> It is based on raw sockets and not tested on any network HW accelerator.
>
> Create pktio and queue for all threads before starting pthread.
>
> It has two modes supported,
> 	1. burst mode(without ODP queues)
> 	2. queue mode (with ODP queues and scheduler)
>     and three types,
> 	1. raw sockets
> 	2. multiple messages on sockets
> 	3. mmap on sockets
>
> Signed-off-by: Venkatesh Vivekanandan <venkatesh.vivekanandan@linaro.org>
> ---
>   test/l2fwd/Makefile |  44 ++++
>   test/l2fwd/l2fwd.c  | 637 ++++++++++++++++++++++++++++++++++++++++++++++++++++
>   2 files changed, 681 insertions(+)
>   create mode 100644 test/l2fwd/Makefile
>   create mode 100644 test/l2fwd/l2fwd.c
>
> diff --git a/test/l2fwd/Makefile b/test/l2fwd/Makefile
> new file mode 100644
> index 0000000..3435caa
> --- /dev/null
> +++ b/test/l2fwd/Makefile
> @@ -0,0 +1,44 @@
> +# Copyright (c) 2014, Linaro Limited
> +# All rights reserved.
> +#
> +# SPDX-License-Identifier:     BSD-3-Clause
> +
> +ODP_ROOT = ../..
> +ODP_APP  = l2fwd
> +
> +include $(ODP_ROOT)/Makefile.inc
> +include ../Makefile.inc
> +
> +OBJS     =
> +OBJS    += $(OBJ_DIR)/l2fwd.o
> +
> +DEPS     = $(OBJS:.o=.d)
> +
> +.PHONY: default
> +default: $(OBJ_DIR) $(ODP_APP)
> +
> +-include $(DEPS)
> +
> +#
> +# Compile rules
> +#
> +$(OBJ_DIR)/%.o: %.c
> +	$(ECHO) Compiling $<
> +	$(CC) -c -MD $(EXTRA_CFLAGS) $(CFLAGS) -o $@ $<
> +
> +#
> +# Link rule
> +#
> +$(ODP_APP): $(ODP_LIB) $(OBJS)
> +	$(CC) $(LDFLAGS) $(OBJS) $(ODP_LIB) $(STD_LIBS) -o $@
> +
> +.PHONY: clean
> +clean:
> +	$(RMDIR) $(OBJ_DIR)
> +	$(RM) $(ODP_APP)
> +	$(MAKE) -C $(ODP_DIR) clean
> +
> +.PHONY: install
> +install:
> +	install -d $(DESTDIR)/share/odp
> +	install -m 0755 $(ODP_APP) $(DESTDIR)/share/odp/
> diff --git a/test/l2fwd/l2fwd.c b/test/l2fwd/l2fwd.c
> new file mode 100644
> index 0000000..fb1b949
> --- /dev/null
> +++ b/test/l2fwd/l2fwd.c
> @@ -0,0 +1,637 @@
> +/* Copyright (c) 2014, Linaro Limited
> + * All rights reserved.
> + *
> + * SPDX-License-Identifier:     BSD-3-Clause
> + */
> +
> +/**
> + * @file
> + *
> + * @example l2fwd.c  ODP basic forwarding application
> + */
> +
> +#include <stdlib.h>
> +#include <string.h>
> +#include <getopt.h>
> +#include <unistd.h>
> +
> +#include <odp.h>
> +#include <helper/odp_linux.h>
> +#include <helper/odp_packet_helper.h>
> +#include <helper/odp_eth.h>
> +#include <helper/odp_ip.h>
> +
> +#define MAX_WORKERS            32
> +#define SHM_PKT_POOL_SIZE      (512*2048)
> +#define SHM_PKT_POOL_BUF_SIZE  1856
> +#define MAX_PKT_BURST          16
> +
> +#define APPL_MODE_PKT_BURST    0
> +#define APPL_MODE_PKT_QUEUE    1
> +
> +#define PRINT_APPL_MODE(x) printf("%s(%i)\n", #x, (x))
> +
> +/** Get rid of path in filename - only for unix-type paths using '/' */
> +#define NO_PATH(file_name) (strrchr((file_name), '/') ? \
> +			    strrchr((file_name), '/') + 1 : (file_name))
> +/**
> + * Parsed command line application arguments
> + */
> +typedef struct {
> +	int core_count;
> +	int if_count;		/**< Number of interfaces to be used */
> +	char **if_names;	/**< Array of pointers to interface names */
> +	int mode;		/**< Packet IO mode */
> +	int type;		/**< Packet IO type */
> +	int fanout;		/**< Packet IO fanout */
> +	odp_buffer_pool_t pool;	/**< Buffer pool for packet IO */
> +} appl_args_t;
> +
> +/**
> + * Thread specific arguments
> + */
> +typedef struct {
> +	char *srcif;		/**< Source Interface */
> +	char *dstif;		/**< Dest Interface */
> +	odp_buffer_pool_t pool;	/**< Buffer pool for packet IO */
> +	odp_pktio_t srcpktio;	/**< Source pktio handle */
> +	odp_pktio_t dstpktio;	/**< Destination pktio handle */
> +	int mode;		/**< Thread mode */
> +	int type;		/**< Thread i/o type */
> +	int fanout;		/**< Thread i/o fanout */
> +} thread_args_t;
> +
> +/**
> + * Grouping of both parsed CL args and thread specific args - alloc together
> + */
> +typedef struct {
> +	/** Application (parsed) arguments */
> +	appl_args_t appl;
> +	/** Thread specific arguments */
> +	thread_args_t thread[MAX_WORKERS];
> +} args_t;
> +
> +/** Global pointer to args */
> +static args_t *args;
> +int num_workers;
> +
> +/* helper funcs */
> +static int drop_err_pkts(odp_packet_t pkt_tbl[], unsigned len);
> +static void parse_args(int argc, char *argv[], appl_args_t *appl_args);
> +static void print_info(char *progname, appl_args_t *appl_args);
> +static void usage(char *progname);
> +
> +/**
> + * Burst mode: pktio for each thread will be created with either same or
> + * different params
> + *
> + * @param arg  thread arguments of type 'thread_args_t *'
> + * @param pool is the packet pool from where buffers should be taken
> + */
> +static odp_pktio_t burst_mode_init_params(void *arg, int pool)
> +{
> +	thread_args_t *args;
> +	odp_pktio_params_t params;
> +	socket_params_t *sock_params = &params.sock_params;
> +	odp_pktio_t pktio;
> +
> +	args = arg;
> +	/* Open a packet IO instance for this thread */
> +	sock_params->type = args->type;
> +	sock_params->fanout = args->fanout;
> +	pktio = odp_pktio_open(args->srcif, pool, &params);
> +	if (pktio == ODP_PKTIO_INVALID)
> +		ODP_ERR("  Error: pktio create failed");
> +
> +	return pktio;
> +}
> +
> +/**
> + * Queue mode: pktio for each thread will be created with either same or
> + * different params. Queues are created and attached to the pktio.
> + *
> + * @param arg  thread arguments of type 'thread_args_t *'
> + * @param pool is the packet pool from where buffers should be taken
> + */
> +static odp_pktio_t queue_mode_init_params(void *arg, int pool)
> +{
> +	char inq_name[ODP_QUEUE_NAME_LEN];
> +	odp_queue_param_t qparam;
> +	odp_queue_t inq_def;
> +	int ret;
> +	odp_pktio_t pktio = ODP_PKTIO_INVALID;
> +
> +	pktio = burst_mode_init_params(arg, pool);
> +	if (pktio == ODP_PKTIO_INVALID)
> +		return pktio;
> +	/*
> +	 * Create and set the default INPUT queue associated with the 'pktio'
> +	 * resource
> +	 */
> +	qparam.sched.prio  = ODP_SCHED_PRIO_DEFAULT;
> +	qparam.sched.sync  = ODP_SCHED_SYNC_ATOMIC;
> +	qparam.sched.group = ODP_SCHED_GROUP_DEFAULT;
> +	snprintf(inq_name, sizeof(inq_name), "%i-pktio_inq_def", (int)pktio);
> +	inq_name[ODP_QUEUE_NAME_LEN - 1] = '\0';
> +
> +	inq_def = odp_queue_create(inq_name, ODP_QUEUE_TYPE_PKTIN, &qparam);
> +	if (inq_def == ODP_QUEUE_INVALID) {
> +		ODP_ERR("  Error: pktio queue creation failed");
> +		return ODP_PKTIO_INVALID;
> +	}
> +
> +	ret = odp_pktio_inq_setdef(pktio, inq_def);
> +	if (ret != 0) {
> +		ODP_ERR("  Error: default input-Q setup");
> +		return ODP_PKTIO_INVALID;
> +	}
> +
> +	return pktio;
> +}
> +
> +/**
> + * Packet IO worker thread using ODP queues
> + *
> + * @param arg  thread arguments of type 'thread_args_t *'
> + */
> +static void *pktio_queue_thread(void *arg)
> +{
> +	int thr, i;
> +	thread_args_t *thr_args;
> +	char dstpktio[MAX_WORKERS+1];
> +	odp_queue_t outq_def;
> +	odp_packet_t pkt;
> +	odp_buffer_t buf;
> +	unsigned long pkt_cnt = 0;
> +	unsigned long err_cnt = 0;
> +
> +	thr = odp_thread_id();
> +	thr_args = arg;
> +
> +	if (thr_args->srcpktio == 0 || thr_args->dstpktio == 0) {
> +		ODP_ERR("Invalid srcpktio:%d dstpktio:%d\n",
> +			thr_args->srcpktio, thr_args->dstpktio);
> +		return NULL;
> +	}
> +	printf("[%02i] srcif:%s dstif:%s spktio:%02i dpktio:%02i QUEUE mode\n",
> +	       thr, thr_args->srcif, thr_args->dstif, thr_args->srcpktio,
> +	       thr_args->dstpktio);
> +
> +	/* Populate an array of destination pktio's in all threads as the
> +	 * scheduler can take packets from any input queue
> +	 */
> +	for (i = 0; i < num_workers; i++)
> +		dstpktio[i+1] = args->thread[i].dstpktio;
> +
> +	/* Loop packets */
> +	for (;;) {
> +		odp_pktio_t pktio_tmp;
> +
> +		/* Use schedule to get buf from any input queue */
> +		buf = odp_schedule(NULL, ODP_SCHED_WAIT);
> +
> +		pkt = odp_packet_from_buffer(buf);
> +		/* Drop packets with errors */
> +		if (odp_unlikely(drop_err_pkts(&pkt, 1) == 0)) {
> +			ODP_ERR("Drop frame - err_cnt:%lu\n", ++err_cnt);
> +			continue;
> +		}
> +
> +		pktio_tmp = odp_pktio_get_input(pkt);
> +		outq_def = odp_pktio_outq_getdef(dstpktio[pktio_tmp]);
> +		if (outq_def == ODP_QUEUE_INVALID) {
> +			ODP_ERR("  [%02i] Error: def output-Q query\n", thr);
> +			return NULL;
> +		}
> +
> +		/* Enqueue the packet for output */
> +		odp_queue_enq(outq_def, buf);
> +
> +		/* Print packet counts every once in a while */
> +		if (odp_unlikely(pkt_cnt++ % 100000 == 0)) {
> +			printf("  [%02i] pkt_cnt:%lu\n", thr, pkt_cnt);
> +			fflush(NULL);
> +		}
> +	}
> +
> +/* unreachable */
> +}
> +
> +/**
> + * Packet IO worker thread using bursts from/to IO resources
> + *
> + * @param arg  thread arguments of type 'thread_args_t *'
> + */
> +static void *pktio_ifburst_thread(void *arg)
> +{
> +	int thr;
> +	thread_args_t *thr_args;
> +	int pkts, pkts_ok;
> +	odp_packet_t pkt_tbl[MAX_PKT_BURST];
> +	unsigned long pkt_cnt = 0;
> +	unsigned long err_cnt = 0;
> +	unsigned long tmp = 0;
> +
> +	thr = odp_thread_id();
> +	thr_args = arg;
> +
> +	if (thr_args->srcpktio == 0 || thr_args->dstpktio == 0) {
> +		ODP_ERR("Invalid srcpktio:%d dstpktio:%d\n",
> +			thr_args->srcpktio, thr_args->dstpktio);
> +		return NULL;
> +	}
> +	printf("[%02i] srcif:%s dstif:%s spktio:%02i dpktio:%02i BURST mode\n",
> +	       thr, thr_args->srcif, thr_args->dstif, thr_args->srcpktio,
> +	       thr_args->dstpktio);
> +
> +	/* Loop packets */
> +	for (;;) {
> +		pkts = odp_pktio_recv(thr_args->srcpktio, pkt_tbl,
> +					MAX_PKT_BURST);
> +		if (pkts > 0) {
> +			/* Drop packets with errors */
> +			pkts_ok = drop_err_pkts(pkt_tbl, pkts);
> +			if (pkts_ok > 0)
> +				odp_pktio_send(thr_args->dstpktio, pkt_tbl,
> +					       pkts_ok);
> +			if (odp_unlikely(pkts_ok != pkts))
> +				ODP_ERR("Dropped frames:%u - err_cnt:%lu\n",
> +					pkts-pkts_ok, ++err_cnt);
> +
> +			/* Print packet counts every once in a while */
> +			tmp += pkts_ok;
> +			if (odp_unlikely((tmp >= 100000) || /* OR first print:*/
> +			    ((pkt_cnt == 0) && ((tmp-1) < MAX_PKT_BURST)))) {
> +				pkt_cnt += tmp;
> +				printf("  [%02i] pkt_cnt:%lu\n", thr, pkt_cnt);
> +				fflush(NULL);
> +				tmp = 0;
> +			}
> +		}
> +	}
> +
> +/* unreachable */
> +}
> +
> +/**
> + * ODP L2 forwarding main function
> + */
> +int main(int argc, char *argv[])
> +{
> +	odp_linux_pthread_t thread_tbl[MAX_WORKERS];
> +	odp_buffer_pool_t pool;
> +	int thr_id;
> +	void *pool_base;
> +	int i;
> +	int first_core;
> +	int core_count;
> +	odp_pktio_t pktio;
> +
> +	/* Init ODP before calling anything else */
> +	if (odp_init_global()) {
> +		ODP_ERR("Error: ODP global init failed.\n");
> +		exit(EXIT_FAILURE);
> +	}
> +
> +	/* Reserve memory for args from shared mem */
> +	args = odp_shm_reserve("shm_args", sizeof(args_t), ODP_CACHE_LINE_SIZE);
> +	if (args == NULL) {
> +		ODP_ERR("Error: shared mem alloc failed.\n");
> +		exit(EXIT_FAILURE);
> +	}
> +	memset(args, 0, sizeof(*args));
> +
> +	/* Parse and store the application arguments */
> +	parse_args(argc, argv, &args->appl);
> +
> +	/* Print both system and application information */
> +	print_info(NO_PATH(argv[0]), &args->appl);
> +
> +	core_count  = odp_sys_core_count();
> +	num_workers = core_count;
> +
> +	if (args->appl.core_count)
> +		num_workers = args->appl.core_count;
> +
> +	if (num_workers > MAX_WORKERS)
> +		num_workers = MAX_WORKERS;
> +
> +	printf("Num worker threads: %i\n", num_workers);
> +
> +	if (num_workers < args->appl.if_count) {
> +		ODP_ERR("Error: core count %d is less than interface count\n",
> +			num_workers);
> +		exit(EXIT_FAILURE);
> +	}
> +	if (args->appl.if_count % 2 != 0) {
> +		ODP_ERR("Error: interface count %d is odd in fwd appl.\n",
> +			args->appl.if_count);
> +		exit(EXIT_FAILURE);
> +	}
> +	/*
> +	 * By default core #0 runs Linux kernel background tasks.
> +	 * Start mapping thread from core #1
> +	 */
> +	first_core = 1;
> +
> +	if (core_count == 1)
> +		first_core = 0;
> +
> +	printf("First core:         %i\n\n", first_core);
> +
> +	/* Init this thread */
> +	thr_id = odp_thread_create(0);
> +	odp_init_local(thr_id);
> +
> +	/* Create packet pool */
> +	pool_base = odp_shm_reserve("shm_packet_pool",
> +				    SHM_PKT_POOL_SIZE, ODP_CACHE_LINE_SIZE);
> +	if (pool_base == NULL) {
> +		ODP_ERR("Error: packet pool mem alloc failed.\n");
> +		exit(EXIT_FAILURE);
> +	}
> +
> +	pool = odp_buffer_pool_create("packet_pool", pool_base,
> +				      SHM_PKT_POOL_SIZE,
> +				      SHM_PKT_POOL_BUF_SIZE,
> +				      ODP_CACHE_LINE_SIZE,
> +				      ODP_BUFFER_TYPE_PACKET);
> +	if (pool == ODP_BUFFER_POOL_INVALID) {
> +		ODP_ERR("Error: packet pool create failed.\n");
> +		exit(EXIT_FAILURE);
> +	}
> +	odp_buffer_pool_print(pool);
> +
> +	memset(thread_tbl, 0, sizeof(thread_tbl));
> +	/* initialize threads params */
> +	for (i = 0; i < num_workers; ++i) {
> +		int if_idx;
> +
> +		if_idx = i % args->appl.if_count;
> +
> +		args->thread[i].srcif = args->appl.if_names[if_idx];
> +		if (if_idx % 2 == 0)
> +			args->thread[i].dstif = args->appl.if_names[if_idx+1];
> +		else
> +			args->thread[i].dstif = args->appl.if_names[if_idx-1];
> +		args->thread[i].pool = pool;
> +		args->thread[i].mode = args->appl.mode;
> +		args->thread[i].type = args->appl.type;
> +		args->thread[i].fanout = args->appl.fanout;
> +
> +		if (args->appl.mode == APPL_MODE_PKT_BURST) {
> +			pktio = burst_mode_init_params(&args->thread[i], pool);
> +			if (pktio == ODP_PKTIO_INVALID) {
> +				ODP_ERR("  for thread:%02i\n", i);
> +				exit(EXIT_FAILURE);
> +			}
> +		} else { /* APPL_MODE_PKT_QUEUE */
> +			pktio = queue_mode_init_params(&args->thread[i], pool);
> +			if (pktio == ODP_PKTIO_INVALID) {
> +				ODP_ERR("  for thread:%02i\n", i);
> +				exit(EXIT_FAILURE);
> +			}
> +		}
> +		args->thread[i].srcpktio = pktio;
> +	}
> +	for (i = 0; i < num_workers; ++i) {
> +		if (i % 2 == 0)
> +			args->thread[i].dstpktio = args->thread[i+1].srcpktio;
> +		else
> +			args->thread[i].dstpktio = args->thread[i-1].srcpktio;
> +	}
> +	/* Create worker threads */
> +	for (i = 0; i < num_workers; ++i) {
> +		void *(*thr_run_func) (void *);
> +		int core;
> +
> +		core = (first_core + i) % core_count;
> +
> +		if (args->appl.mode == APPL_MODE_PKT_BURST)
> +			thr_run_func = pktio_ifburst_thread;
> +		else /* APPL_MODE_PKT_QUEUE */
> +			thr_run_func = pktio_queue_thread;
> +		odp_linux_pthread_create(thread_tbl, 1, core, thr_run_func,
> +					 &args->thread[i]);
> +	}
> +
> +	/* Master thread waits for other threads to exit */
> +	odp_linux_pthread_join(thread_tbl, num_workers);
> +
> +	printf("Exit\n\n");
> +
> +	return 0;
> +}
> +
> +/**
> + * Drop packets which input parsing marked as containing errors.
> + *
> + * Frees packets with error and modifies pkt_tbl[] to only contain packets with
> + * no detected errors.
> + *
> + * @param pkt_tbl  Array of packet
> + * @param len      Length of pkt_tbl[]
> + *
> + * @return Number of packets with no detected error
> + */
> +static int drop_err_pkts(odp_packet_t pkt_tbl[], unsigned len)
> +{
> +	odp_packet_t pkt;
> +	unsigned pkt_cnt = len;
> +	unsigned i, j;
> +
> +	for (i = 0, j = 0; i < len; ++i) {
> +		pkt = pkt_tbl[i];
> +
> +		if (odp_unlikely(odp_packet_error(pkt))) {
> +			odp_packet_free(pkt); /* Drop */
> +			pkt_cnt--;
> +		} else if (odp_unlikely(i != j++)) {
> +			pkt_tbl[j] = pkt;
> +		}
> +	}
> +
> +	return pkt_cnt;
> +}
> +
> +/**
> + * Parse and store the command line arguments
> + *
> + * @param argc       argument count
> + * @param argv[]     argument vector
> + * @param appl_args  Store application arguments here
> + */
> +static void parse_args(int argc, char *argv[], appl_args_t *appl_args)
> +{
> +	int opt;
> +	int long_index;
> +	char *names, *str, *token, *save;
> +	size_t len;
> +	int i;
> +	static struct option longopts[] = {
> +		{"count", required_argument, NULL, 'c'},
> +		{"interface", required_argument, NULL, 'i'},	/* return 'i' */
> +		{"mode", required_argument, NULL, 'm'},		/* return 'm' */
> +		{"help", no_argument, NULL, 'h'},		/* return 'h' */
> +		{NULL, 0, NULL, 0}
> +	};
> +
> +	appl_args->mode = -1; /* Invalid, must be changed by parsing */
> +	appl_args->type = 3;  /* 3: ODP_PKTIO_TYPE_SOCKET_MMAP */
> +	appl_args->fanout = 1; /* turn off fanout by default for mmap */
> +
> +	while (1) {
> +		opt = getopt_long(argc, argv, "+c:i:m:t:f:h",
> +				  longopts, &long_index);
> +
> +		if (opt == -1)
> +			break;	/* No more options */
> +
> +		switch (opt) {
> +		case 'c':
> +			appl_args->core_count = atoi(optarg);
> +			break;
> +			/* parse packet-io interface names */
> +		case 'i':
> +			len = strlen(optarg);
> +			if (len == 0) {
> +				usage(argv[0]);
> +				exit(EXIT_FAILURE);
> +			}
> +			len += 1;	/* add room for '\0' */
> +
> +			names = malloc(len);
> +			if (names == NULL) {
> +				usage(argv[0]);
> +				exit(EXIT_FAILURE);
> +			}
> +
> +			/* count the number of tokens separated by ',' */
> +			strcpy(names, optarg);
> +			for (str = names, i = 0;; str = NULL, i++) {
> +				token = strtok_r(str, ",", &save);
> +				if (token == NULL)
> +					break;
> +			}
> +			appl_args->if_count = i;
> +
> +			if (appl_args->if_count == 0) {
> +				usage(argv[0]);
> +				exit(EXIT_FAILURE);
> +			}
> +
> +			/* allocate storage for the if names */
> +			appl_args->if_names =
> +			    calloc(appl_args->if_count, sizeof(char *));
> +
> +			/* store the if names (reset names string) */
> +			strcpy(names, optarg);
> +			for (str = names, i = 0;; str = NULL, i++) {
> +				token = strtok_r(str, ",", &save);
> +				if (token == NULL)
> +					break;
> +				appl_args->if_names[i] = token;
> +			}
> +			break;
> +
> +		case 'm':
> +			i = atoi(optarg);
> +			if (i == 0)
> +				appl_args->mode = APPL_MODE_PKT_BURST;
> +			else
> +				appl_args->mode = APPL_MODE_PKT_QUEUE;
> +			break;
> +
> +		case 't':
> +			appl_args->type = atoi(optarg);
> +			break;
> +
> +		case 'f':
> +			appl_args->fanout = atoi(optarg);
> +			break;
> +
> +		case 'h':
> +			usage(argv[0]);
> +			exit(EXIT_SUCCESS);
> +			break;
> +
> +		default:
> +			break;
> +		}
> +	}
> +
> +	if (appl_args->if_count == 0 || appl_args->mode == -1) {
> +		usage(argv[0]);
> +		exit(EXIT_FAILURE);
> +	}
> +
> +	optind = 1;		/* reset 'extern optind' from the getopt lib */
> +}
> +
> +/**
> + * Print system and application info
> + */
> +static void print_info(char *progname, appl_args_t *appl_args)
> +{
> +	int i;
> +
> +	printf("\n"
> +	       "ODP system info\n"
> +	       "---------------\n"
> +	       "ODP API version: %s\n"
> +	       "CPU model:       %s\n"
> +	       "CPU freq (hz):   %"PRIu64"\n"
> +	       "Cache line size: %i\n"
> +	       "Core count:      %i\n"
> +	       "\n",
> +	       odp_version_api_str(), odp_sys_cpu_model_str(), odp_sys_cpu_hz(),
> +	       odp_sys_cache_line_size(), odp_sys_core_count());
> +
> +	printf("Running ODP appl: \"%s\"\n"
> +	       "-----------------\n"
> +	       "IF-count:        %i\n"
> +	       "Using IFs:      ",
> +	       progname, appl_args->if_count);
> +	for (i = 0; i < appl_args->if_count; ++i)
> +		printf(" %s", appl_args->if_names[i]);
> +	printf("\n"
> +	       "Mode:            ");
> +	if (appl_args->mode == APPL_MODE_PKT_BURST)
> +		PRINT_APPL_MODE(APPL_MODE_PKT_BURST);
> +	else
> +		PRINT_APPL_MODE(APPL_MODE_PKT_QUEUE);
> +	printf("\n\n");
> +	fflush(NULL);
> +}
> +
> +/**
> + * Prinf usage information
> + */
> +static void usage(char *progname)
> +{
> +	printf("\n"
> +	       "OpenDataPlane L2 forwarding application.\n"
> +	       "\n"
> +	       "Usage: %s OPTIONS\n"
> +	       "  E.g. %s -i eth0,eth1,eth2,eth3 -m 0 -t 1\n"
> +	       " In the above example,\n"
> +	       " eth0 will send pkts to eth1 and vice versa\n"
> +	       " eth2 will send pkts to eth3 and vice versa\n"
> +	       "\n"
> +	       "Mandatory OPTIONS:\n"
> +	       "  -i, --interface Eth interfaces (comma-separated, no spaces)\n"
> +	       "  -m, --mode      0: Burst send&receive packets (no queues)\n"
> +	       "                  1: Send&receive packets through ODP queues.\n"
> +	       " -t, --type   1: ODP_PKTIO_TYPE_SOCKET_BASIC\n"
> +	       "	      2: ODP_PKTIO_TYPE_SOCKET_MMSG\n"
> +	       "	      3: ODP_PKTIO_TYPE_SOCKET_MMAP\n"
> +	       "	      4: ODP_PKTIO_TYPE_NETMAP\n"
> +	       "	 Default: 3: ODP_PKTIO_TYPE_SOCKET_MMAP\n"
> +	       " -f, --fanout 0: off 1: on (Default 1: on)\n"
> +	       "\n"
> +	       "Optional OPTIONS\n"
> +	       "  -c, --count <number> Core count.\n"
> +	       "  -h, --help           Display help and exit.\n\n"
> +	       "\n", NO_PATH(progname), NO_PATH(progname)
> +	    );
> +}