diff mbox

[v2,6/6] linux-generic: odp-ipfw: New file example/ipfw/extra/odp_ipfw.c

Message ID 1457702830-25087-1-git-send-email-venkatesh.vivekanandan@linaro.org
State Superseded
Headers show

Commit Message

Venkatesh Vivekanandan March 11, 2016, 1:27 p.m. UTC
From: Venkatesh Vivekanandan <venkatesh.vivekanandan@linaro.org>

This is a derived work from l2fwd with some hooks to odp_uipfw and
odp_ipfw.

Signed-off-by: Venkatesh Vivekanandan <venkatesh.vivekanandan@linaro.org>
---
 example/ipfw/Makefile.am      |    1 +
 example/ipfw/extra/missing.h  |    2 +
 example/ipfw/extra/odp_ipfw.c | 1506 +++++++++++++++++++++++++++++++++++++++++
 example/ipfw/extra/session.c  |    4 +
 4 files changed, 1513 insertions(+)
 create mode 100644 example/ipfw/extra/odp_ipfw.c
diff mbox

Patch

diff --git a/example/ipfw/Makefile.am b/example/ipfw/Makefile.am
index 032f992..0e1732d 100644
--- a/example/ipfw/Makefile.am
+++ b/example/ipfw/Makefile.am
@@ -85,6 +85,7 @@  IPFW_SRCS       = $(SRCS_IPFW) $(SRCS_NET) $(SRCS_NETINET)
 IPFW_SRCS += extra/ipfw2_mod.c # bsd_compat.c
 
 IPFW_SRCS += extra/missing.c extra/session.c
+IPFW_SRCS += extra/odp_ipfw.c
 
 AM_CFLAGS  = $(E_CFLAGS)
 AM_CFLAGS += -I$(top_srcdir)/platform/$(with_platform)/arch/$(ARCH)
diff --git a/example/ipfw/extra/missing.h b/example/ipfw/extra/missing.h
index b5b65b2..d849542 100644
--- a/example/ipfw/extra/missing.h
+++ b/example/ipfw/extra/missing.h
@@ -797,5 +797,7 @@  struct sess *
 new_session(int fd, handler_t *func, void *arg, enum flags_t flags);
 
 
+void odp_ipfw_exit(void);
+int odp_ipfw_main(int argc, char *argv[]);
 void netmap_add_port(const char *dev);
 #endif /* !_MISSING_H_ */
diff --git a/example/ipfw/extra/odp_ipfw.c b/example/ipfw/extra/odp_ipfw.c
new file mode 100644
index 0000000..fc10649
--- /dev/null
+++ b/example/ipfw/extra/odp_ipfw.c
@@ -0,0 +1,1506 @@ 
+/* Copyright (c) 2014, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * @example odp_ipfw.c  ODP ip firewall application
+ */
+
+/** enable strtok */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <stdlib.h>
+#include <getopt.h>
+#include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/malloc.h> /* M_NOWAIT */
+#include <sys/mbuf.h>   /* mbuf */
+#include <sys/cpuset.h> /* FreeBSD */
+#include <net/pfil.h>   /* PFIL_IN */
+
+/* args for ipfw */
+#include <netinet/ip_fw.h>
+#include <netpfil/ipfw/ip_fw_private.h>
+
+#undef LOG_ERR
+#include <test_debug.h>
+
+#include <odp_api.h>
+#include <odp/helper/linux.h>
+#include <odp/helper/eth.h>
+#include <odp/helper/ip.h>
+
+#include <odp_packet_internal.h>
+
+/** @def MAX_WORKERS
+ * @brief Maximum number of worker threads
+ */
+#define MAX_WORKERS            32
+
+/** @def SHM_PKT_POOL_SIZE
+ * @brief Size of the shared memory block
+ */
+#define SHM_PKT_POOL_SIZE      8192
+
+/** @def SHM_PKT_POOL_BUF_SIZE
+ * @brief Buffer size of the packet pool buffer
+ */
+#define SHM_PKT_POOL_BUF_SIZE  1856
+
+/** @def MAX_PKT_BURST
+ * @brief Maximum number of packet in a burst
+ */
+#define MAX_PKT_BURST          32
+
+/** Maximum number of pktio queues per interface */
+#define MAX_QUEUES             32
+
+/** Maximum number of pktio interfaces */
+#define MAX_PKTIOS             8
+
+/**
+ * Packet input mode
+ */
+typedef enum pktin_mode_t {
+	DIRECT_RECV,
+	PLAIN_QUEUE,
+	SCHED_PARALLEL,
+	SCHED_ATOMIC,
+	SCHED_ORDERED,
+} pktin_mode_t;
+
+/**
+ * Packet output modes
+ */
+typedef enum pktout_mode_t {
+	PKTOUT_DIRECT,
+	PKTOUT_QUEUE
+} pktout_mode_t;
+
+static inline int sched_mode(pktin_mode_t in_mode)
+{
+	return (in_mode == SCHED_PARALLEL) ||
+	       (in_mode == SCHED_ATOMIC)   ||
+	       (in_mode == SCHED_ORDERED);
+}
+
+/** Get rid of path in filename - only for unix-type paths using '/' */
+#define NO_PATH(file_name) (strrchr((file_name), '/') ? \
+			    strrchr((file_name), '/') + 1 : (file_name))
+/**
+ * Parsed command line application arguments
+ */
+typedef struct {
+	int cpu_count;
+	int if_count;		/**< Number of interfaces to be used */
+	int addr_count;		/**< Number of dst addresses to be used */
+	int num_workers;	/**< Number of worker threads */
+	char **if_names;	/**< Array of pointers to interface names */
+	odph_ethaddr_t addrs[MAX_PKTIOS]; /**< Array of dst addresses */
+	pktin_mode_t in_mode;	/**< Packet input mode */
+	pktout_mode_t out_mode; /**< Packet output mode */
+	int time;		/**< Time in seconds to run. */
+	int accuracy;		/**< Number of seconds to get and print statistics */
+	char *if_str;		/**< Storage for interface names */
+	int dst_change;		/**< Change destination eth addresses */
+	int src_change;		/**< Change source eth addresses */
+	int error_check;        /**< Check packet errors */
+} appl_args_t;
+
+static int exit_threads;	/**< Break workers loop if set to 1 */
+
+/**
+ * Statistics
+ */
+typedef union {
+	struct {
+		/** Number of forwarded packets */
+		uint64_t packets;
+		/** Packets dropped due to receive error */
+		uint64_t rx_drops;
+		/** Packets dropped due to transmit error */
+		uint64_t tx_drops;
+	} s;
+
+	uint8_t padding[ODP_CACHE_LINE_SIZE];
+} stats_t ODP_ALIGNED_CACHE;
+
+/**
+ * Thread specific arguments
+ */
+typedef struct thread_args_t {
+	int thr_idx;
+	int num_pktio;
+
+	struct {
+		odp_pktio_t rx_pktio;
+		odp_pktio_t tx_pktio;
+		odp_pktin_queue_t pktin;
+		odp_pktout_queue_t pktout;
+		odp_queue_t rx_queue;
+		odp_queue_t tx_queue;
+		int rx_idx;
+		int tx_idx;
+		int rx_queue_idx;
+		int tx_queue_idx;
+	} pktio[MAX_PKTIOS];
+
+	stats_t *stats;	/**< Pointer to per thread stats */
+} thread_args_t;
+
+/**
+ * Grouping of all global data
+ */
+typedef struct {
+	/** Per thread packet stats */
+	stats_t stats[MAX_WORKERS];
+	/** Application (parsed) arguments */
+	appl_args_t appl;
+	/** Thread specific arguments */
+	thread_args_t thread[MAX_WORKERS];
+	/** Table of port ethernet addresses */
+	odph_ethaddr_t port_eth_addr[MAX_PKTIOS];
+	/** Table of dst ethernet addresses */
+	odph_ethaddr_t dst_eth_addr[MAX_PKTIOS];
+	/** Table of dst ports */
+	int dst_port[MAX_PKTIOS];
+	/** Table of pktio handles */
+	struct {
+		odp_pktio_t pktio;
+		odp_pktin_queue_t pktin[MAX_QUEUES];
+		odp_pktout_queue_t pktout[MAX_QUEUES];
+		odp_queue_t rx_q[MAX_QUEUES];
+		odp_queue_t tx_q[MAX_QUEUES];
+		int num_rx_thr;
+		int num_tx_thr;
+		int num_rx_queue;
+		int num_tx_queue;
+		int next_rx_queue;
+		int next_tx_queue;
+	} pktios[MAX_PKTIOS];
+} args_t;
+
+/** Global pointer to args */
+static args_t *gbl_args;
+/** Global barrier to synchronize main and workers */
+static odp_barrier_t barrier;
+
+/**
+ * Lookup the destination port for a given packet
+ *
+ * @param pkt  ODP packet handle
+ */
+static inline int lookup_dest_port(odp_packet_t pkt)
+{
+	int i, src_idx;
+	odp_pktio_t pktio_src;
+
+	pktio_src = odp_packet_input(pkt);
+
+	for (src_idx = -1, i = 0; gbl_args->pktios[i].pktio
+				  != ODP_PKTIO_INVALID; ++i)
+		if (gbl_args->pktios[i].pktio == pktio_src)
+			src_idx = i;
+
+	if (src_idx == -1)
+		LOG_ABORT("Failed to determine pktio input\n");
+
+	return gbl_args->dst_port[src_idx];
+}
+
+/**
+ * Drop packets which input parsing marked as containing errors.
+ *
+ * Frees packets with error and modifies pkt_tbl[] to only contain packets with
+ * no detected errors.
+ *
+ * @param pkt_tbl  Array of packets
+ * @param num      Number of packets in pkt_tbl[]
+ *
+ * @return Number of packets dropped
+ */
+static inline int drop_err_pkts(odp_packet_t pkt_tbl[], unsigned num)
+{
+	odp_packet_t pkt;
+	unsigned dropped = 0;
+	unsigned i, j;
+
+	for (i = 0, j = 0; i < num; ++i) {
+		pkt = pkt_tbl[i];
+
+		if (odp_unlikely(odp_packet_has_error(pkt))) {
+			odp_packet_free(pkt); /* Drop */
+			dropped++;
+		} else if (odp_unlikely(i != j++)) {
+			pkt_tbl[j - 1] = pkt;
+		}
+	}
+
+	return dropped;
+}
+
+/**
+ * Fill packets' eth addresses according to the destination port
+ *
+ * @param pkt_tbl  Array of packets
+ * @param num      Number of packets in the array
+ * @param dst_port Destination port
+ */
+static inline void fill_eth_addrs(odp_packet_t pkt_tbl[],
+				  unsigned num, int dst_port)
+{
+	odp_packet_t pkt;
+	odph_ethhdr_t *eth;
+	unsigned i;
+
+	if (!gbl_args->appl.dst_change && !gbl_args->appl.src_change)
+		return;
+
+	for (i = 0; i < num; ++i) {
+		pkt = pkt_tbl[i];
+		if (odp_packet_has_eth(pkt)) {
+			eth = (odph_ethhdr_t *)odp_packet_l2_ptr(pkt, NULL);
+
+			if (gbl_args->appl.src_change)
+				eth->src = gbl_args->port_eth_addr[dst_port];
+
+			if (gbl_args->appl.dst_change)
+				eth->dst = gbl_args->dst_eth_addr[dst_port];
+		}
+	}
+}
+
+static inline int event_queue_send(odp_queue_t queue, odp_packet_t *pkt_tbl,
+				   unsigned pkts)
+{
+	int ret;
+	unsigned i;
+	unsigned sent = 0;
+	odp_event_t ev_tbl[pkts];
+
+	for (i = 0; i < pkts; i++)
+		ev_tbl[i] = odp_packet_to_event(pkt_tbl[i]);
+
+	while (sent < pkts) {
+		ret = odp_queue_enq_multi(queue, &ev_tbl[sent], pkts - sent);
+
+		if (ret < 0) {
+			LOG_ERR("Failed to send packet as events\n");
+			break;
+		}
+
+		sent += ret;
+	}
+
+	return sent;
+}
+
+/**
+ * check whether the packet conforms to a set of rules defined in ipfw
+ *
+ * @param pkt    packet that needs to be checked for conformance against
+ *               ipfw rule
+ */
+struct mbuf *odp_ipfw_rule_check(odp_packet_t pkt)
+{
+	/* This code is taken from netmap_read() function of
+	 * extra/netmap_io.c. netmap-ipfw code can be found at
+	 * https://github.com/luigirizzo/netmap-ipfw.git
+	 */
+	struct ip_fw_args args;
+	struct mbuf dm, dm0;
+	odp_buffer_t buf;
+	int len, hdrlen = 0;
+
+	bzero(&dm0, sizeof(dm0));
+	dm = dm0;
+	args.m = &dm;
+	len = odp_packet_len(pkt);
+	dm.m_flags = M_STACK;
+	buf = _odp_packet_to_buffer(pkt);
+	dm.__m_extbuf = buf;
+	dm.__m_extlen = len;
+	/* skip mac + vlan hdr if any */
+	dm.m_data = odp_packet_data(pkt) + hdrlen;
+	dm.m_pkthdr.len = len - hdrlen;
+	dm.m_len = dm.m_pkthdr.len;
+	dm.__max_m_len = dm.m_len;
+	ipfw_check_frame(NULL, &args.m, NULL, PFIL_IN, NULL);
+
+	return args.m;
+}
+
+/**
+ * Packet IO worker thread using scheduled queues
+ *
+ * @param sess a pointer to session structure which is not used, but
+ *             it is required as per session semantics
+ * @param arg  thread arguments of type 'thread_args_t *'
+ */
+int run_worker_sched_mode(struct sess *sess, void *arg)
+{
+	odp_event_t  ev_tbl[MAX_PKT_BURST];
+	odp_packet_t pkt_tbl[MAX_PKT_BURST];
+	int pkts;
+	int thr;
+	uint64_t wait;
+	int dst_idx;
+	int thr_idx;
+	int i;
+	odp_pktout_queue_t pktout[MAX_PKTIOS];
+	odp_queue_t tx_queue[MAX_PKTIOS];
+	thread_args_t *thr_args = arg;
+	stats_t *stats = thr_args->stats;
+	int use_event_queue = gbl_args->appl.out_mode;
+	pktin_mode_t in_mode = gbl_args->appl.in_mode;
+
+	(void)sess;
+	thr = odp_thread_id();
+	thr_idx = thr_args->thr_idx;
+
+	memset(pktout, 0, sizeof(pktout));
+
+	for (i = 0; i < MAX_PKTIOS; i++)
+		tx_queue[i] = ODP_QUEUE_INVALID;
+
+	for (i = 0; i < gbl_args->appl.if_count; i++) {
+		if (gbl_args->pktios[i].num_tx_queue ==
+		    gbl_args->appl.num_workers) {
+			pktout[i]   = gbl_args->pktios[i].pktout[thr_idx];
+			tx_queue[i] = gbl_args->pktios[i].tx_q[thr_idx];
+		} else if (gbl_args->pktios[i].num_tx_queue == 1) {
+			pktout[i]   = gbl_args->pktios[i].pktout[0];
+			tx_queue[i] = gbl_args->pktios[i].tx_q[0];
+		} else {
+			LOG_ABORT("Bad number of output queues %i\n", i);
+		}
+	}
+
+	wait = odp_schedule_wait_time(ODP_TIME_MSEC_IN_NS * 100);
+
+	/* Loop packets */
+	{
+		int sent;
+
+		pkts = odp_schedule_multi(NULL, wait, ev_tbl, MAX_PKT_BURST);
+
+		if (pkts <= 0)
+			return 0;
+
+		for (i = 0; i < pkts; i++)
+			pkt_tbl[i] = odp_packet_from_event(ev_tbl[i]);
+
+		if (gbl_args->appl.error_check) {
+			int rx_drops;
+
+			/* Drop packets with errors */
+			rx_drops = drop_err_pkts(pkt_tbl, pkts);
+
+			if (odp_unlikely(rx_drops)) {
+				stats->s.rx_drops += rx_drops;
+				if (pkts == rx_drops)
+					return 0;
+
+				pkts -= rx_drops;
+			}
+		}
+
+		/* packets from the same queue are from the same interface */
+		dst_idx = lookup_dest_port(pkt_tbl[0]);
+		fill_eth_addrs(pkt_tbl, pkts, dst_idx);
+
+		/* IPFW rules gets applied here */
+		for (i = 0; i < pkts; i++) {
+			struct mbuf *args_m;
+
+			args_m = odp_ipfw_rule_check(pkt_tbl[i]);
+			/* If the packet is ACCEPT */
+			if (args_m != NULL) {
+				if (odp_unlikely(use_event_queue))
+					event_queue_send(tx_queue[dst_idx],
+							 &pkt_tbl[i], 1);
+				else
+					odp_pktout_send(pktout[dst_idx],
+							&pkt_tbl[i], 1);
+
+			} else  { /* Drop */
+				stats->s.tx_drops += 1;
+				odp_packet_free(pkt_tbl[i]);
+			}
+		}
+		stats->s.packets += pkts;
+	}
+
+	/* Make sure that latest stat writes are visible to other threads */
+	odp_mb_full();
+
+	return 0;
+}
+
+/**
+ * Packet IO worker thread using plain queues
+ *
+ * @param sess a pointer to session structure which is not used, but
+ *             it is required as per session semantics
+ * @param arg  thread arguments of type 'thread_args_t *'
+ */
+int run_worker_plain_queue_mode(struct sess *sess, void *arg)
+{
+	int thr;
+	int pkts;
+	odp_packet_t pkt_tbl[MAX_PKT_BURST];
+	int dst_idx, num_pktio;
+	odp_queue_t queue;
+	odp_pktout_queue_t pktout;
+	odp_queue_t tx_queue;
+	int pktio = 0;
+	thread_args_t *thr_args = arg;
+	stats_t *stats = thr_args->stats;
+	int use_event_queue = gbl_args->appl.out_mode;
+
+	(void)sess;
+	thr = odp_thread_id();
+
+	num_pktio = thr_args->num_pktio;
+	dst_idx   = thr_args->pktio[pktio].tx_idx;
+	queue     = thr_args->pktio[pktio].rx_queue;
+	pktout    = thr_args->pktio[pktio].pktout;
+	tx_queue  = thr_args->pktio[pktio].tx_queue;
+
+	/* Loop packets */
+	{
+		int sent;
+		odp_event_t event[MAX_PKT_BURST];
+		int i;
+
+		if (num_pktio > 1) {
+			dst_idx   = thr_args->pktio[pktio].tx_idx;
+			queue     = thr_args->pktio[pktio].rx_queue;
+			pktout    = thr_args->pktio[pktio].pktout;
+			pktio++;
+			if (pktio == num_pktio)
+				pktio = 0;
+		}
+
+		pkts = odp_queue_deq_multi(queue, event, MAX_PKT_BURST);
+		if (odp_unlikely(pkts <= 0))
+			return 0;
+
+		for (i = 0; i < pkts; i++)
+			pkt_tbl[i] = odp_packet_from_event(event[i]);
+
+		if (gbl_args->appl.error_check) {
+			int rx_drops;
+
+			/* Drop packets with errors */
+			rx_drops = drop_err_pkts(pkt_tbl, pkts);
+
+			if (odp_unlikely(rx_drops)) {
+				stats->s.rx_drops += rx_drops;
+				if (pkts == rx_drops)
+					return 0;
+
+				pkts -= rx_drops;
+			}
+		}
+
+		fill_eth_addrs(pkt_tbl, pkts, dst_idx);
+
+		/* IPFW rules gets applied here */
+		for (i = 0; i < pkts; i++) {
+			struct mbuf *args_m;
+
+			args_m = odp_ipfw_rule_check(pkt_tbl[i]);
+			/* If the packet is ACCEPT */
+			if (args_m != NULL) {
+				if (odp_unlikely(use_event_queue))
+					event_queue_send(tx_queue,
+							 &pkt_tbl[i], 1);
+				else
+					odp_pktout_send(pktout,
+							&pkt_tbl[i], 1);
+
+			} else  { /* Drop */
+				stats->s.tx_drops += 1;
+				odp_packet_free(pkt_tbl[i]);
+			}
+		}
+		stats->s.packets += pkts;
+	}
+
+	/* Make sure that latest stat writes are visible to other threads */
+	odp_mb_full();
+
+	return 0;
+}
+
+/**
+ * Packet IO worker thread accessing IO resources directly
+ *
+ * @param sess a pointer to session structure which is not used, but
+ *             it is required as per session semantics
+ * @param arg  thread arguments of type 'thread_args_t *'
+ */
+int run_worker_direct_mode(struct sess *sess, void *arg)
+{
+	int thr;
+	int pkts;
+	odp_packet_t pkt_tbl[MAX_PKT_BURST];
+	int dst_idx, num_pktio;
+	odp_pktin_queue_t pktin;
+	odp_pktout_queue_t pktout;
+	odp_queue_t tx_queue;
+	int pktio = 0;
+	thread_args_t *thr_args = arg;
+	stats_t *stats = thr_args->stats;
+	int use_event_queue = gbl_args->appl.out_mode;
+
+	(void)sess;
+	thr = odp_thread_id();
+
+	num_pktio = thr_args->num_pktio;
+	dst_idx   = thr_args->pktio[pktio].tx_idx;
+	pktin     = thr_args->pktio[pktio].pktin;
+	pktout    = thr_args->pktio[pktio].pktout;
+	tx_queue  = thr_args->pktio[pktio].tx_queue;
+
+	/* Loop packets */
+	{
+		int sent, i;
+
+		if (num_pktio > 1) {
+			dst_idx   = thr_args->pktio[pktio].tx_idx;
+			pktin     = thr_args->pktio[pktio].pktin;
+			pktout    = thr_args->pktio[pktio].pktout;
+			pktio++;
+			if (pktio == num_pktio)
+				pktio = 0;
+		}
+
+		pkts = odp_pktin_recv(pktin, pkt_tbl, MAX_PKT_BURST);
+		if (odp_unlikely(pkts <= 0))
+			return 0;
+
+		if (gbl_args->appl.error_check) {
+			int rx_drops;
+
+			/* Drop packets with errors */
+			rx_drops = drop_err_pkts(pkt_tbl, pkts);
+
+			if (odp_unlikely(rx_drops)) {
+				stats->s.rx_drops += rx_drops;
+				if (pkts == rx_drops)
+					return 0;
+
+				pkts -= rx_drops;
+			}
+		}
+
+		fill_eth_addrs(pkt_tbl, pkts, dst_idx);
+
+		/* IPFW rules gets applied here */
+		for (i = 0; i < pkts; i++) {
+			struct mbuf *args_m;
+
+			args_m = odp_ipfw_rule_check(pkt_tbl[i]);
+			/* If the packet is ACCEPT */
+			if (args_m != NULL) {
+				if (odp_unlikely(use_event_queue))
+					event_queue_send(tx_queue,
+							 &pkt_tbl[i], 1);
+				else
+					odp_pktout_send(pktout,
+							&pkt_tbl[i], 1);
+
+			} else  { /* Drop */
+				stats->s.tx_drops += 1;
+				odp_packet_free(pkt_tbl[i]);
+			}
+		}
+		stats->s.packets += pkts;
+	}
+
+	/* Make sure that latest stat writes are visible to other threads */
+	odp_mb_full();
+
+	return 0;
+}
+
+/**
+ * Create a pktio handle, optionally associating a default input queue.
+ *
+ * @param dev   Name of device to open
+ * @param index Pktio index
+ * @param pool  Pool to associate with device for packet RX/TX
+ *
+ * @retval 0 on success
+ * @retval -1 on failure
+ */
+static int create_pktio(const char *dev, int idx, int num_rx, int num_tx,
+			odp_pool_t pool)
+{
+	odp_pktio_t pktio;
+	odp_pktio_param_t pktio_param;
+	odp_schedule_sync_t  sync_mode;
+	odp_pktio_capability_t capa;
+	odp_pktin_queue_param_t pktin_param;
+	odp_pktout_queue_param_t pktout_param;
+	odp_pktio_op_mode_t mode_rx;
+	odp_pktio_op_mode_t mode_tx;
+	pktin_mode_t in_mode = gbl_args->appl.in_mode;
+	int num_tx_shared;
+
+	odp_pktio_param_init(&pktio_param);
+
+	if (gbl_args->appl.in_mode == PLAIN_QUEUE)
+		pktio_param.in_mode = ODP_PKTIN_MODE_QUEUE;
+	else if (gbl_args->appl.in_mode != DIRECT_RECV) /* pktin_mode SCHED_* */
+		pktio_param.in_mode = ODP_PKTIN_MODE_SCHED;
+
+	if (gbl_args->appl.out_mode != PKTOUT_DIRECT)
+		pktio_param.out_mode = ODP_PKTOUT_MODE_QUEUE;
+
+	pktio = odp_pktio_open(dev, pool, &pktio_param);
+	if (pktio == ODP_PKTIO_INVALID) {
+		LOG_ERR("Error: failed to open %s\n", dev);
+		return -1;
+	}
+
+	printf("created pktio %" PRIu64 " (%s)\n",
+	       odp_pktio_to_u64(pktio), dev);
+
+	if (odp_pktio_capability(pktio, &capa)) {
+		LOG_ERR("Error: capability query failed %s\n", dev);
+		return -1;
+	}
+
+	odp_pktin_queue_param_init(&pktin_param);
+	odp_pktout_queue_param_init(&pktout_param);
+
+	if (sched_mode(in_mode)) {
+		num_tx_shared = 1;
+		mode_tx = ODP_PKTIO_OP_MT;
+		mode_rx = ODP_PKTIO_OP_MT;
+
+		if (gbl_args->appl.in_mode == SCHED_ATOMIC)
+			sync_mode = ODP_SCHED_SYNC_ATOMIC;
+		else if (gbl_args->appl.in_mode == SCHED_ORDERED)
+			sync_mode = ODP_SCHED_SYNC_ORDERED;
+		else
+			sync_mode = ODP_SCHED_SYNC_PARALLEL;
+
+		pktin_param.queue_param.sched.prio  = ODP_SCHED_PRIO_DEFAULT;
+		pktin_param.queue_param.sched.sync  = sync_mode;
+		pktin_param.queue_param.sched.group = ODP_SCHED_GROUP_ALL;
+	} else {
+		num_tx_shared = capa.max_output_queues;
+		mode_tx = ODP_PKTIO_OP_MT_UNSAFE;
+		mode_rx = ODP_PKTIO_OP_MT_UNSAFE;
+	}
+
+	if (num_rx > (int)capa.max_input_queues) {
+		printf("Sharing %i input queues between %i workers\n",
+		       capa.max_input_queues, num_rx);
+		num_rx  = capa.max_input_queues;
+		mode_rx = ODP_PKTIO_OP_MT;
+	}
+
+	if (num_tx > (int)capa.max_output_queues) {
+		printf("Sharing %i output queues between %i workers\n",
+		       num_tx_shared, num_tx);
+		num_tx  = num_tx_shared;
+		mode_tx = ODP_PKTIO_OP_MT;
+	}
+
+	pktin_param.hash_enable = 1;
+	pktin_param.hash_proto.proto.ipv4_udp = 1;
+	pktin_param.num_queues  = num_rx;
+	pktin_param.op_mode     = mode_rx;
+
+	pktout_param.op_mode    = mode_tx;
+	pktout_param.num_queues = num_tx;
+
+	if (odp_pktin_queue_config(pktio, &pktin_param)) {
+		LOG_ERR("Error: input queue config failed %s\n", dev);
+		return -1;
+	}
+
+	if (odp_pktout_queue_config(pktio, &pktout_param)) {
+		LOG_ERR("Error: output queue config failed %s\n", dev);
+		return -1;
+	}
+
+	if (gbl_args->appl.in_mode == DIRECT_RECV) {
+		if (odp_pktin_queue(pktio, gbl_args->pktios[idx].pktin,
+				    num_rx) != num_rx) {
+			LOG_ERR("Error: pktin queue query failed %s\n",
+				dev);
+			return -1;
+		}
+	} else {
+		if (odp_pktin_event_queue(pktio,
+					  gbl_args->pktios[idx].rx_q,
+					  num_rx) != num_rx) {
+			LOG_ERR("Error: pktin event queue query failed %s\n",
+				dev);
+			return -1;
+		}
+	}
+
+	if (gbl_args->appl.out_mode == PKTOUT_DIRECT) {
+		if (odp_pktout_queue(pktio,
+				     gbl_args->pktios[idx].pktout,
+				     num_tx) != num_tx) {
+			LOG_ERR("Error: pktout queue query failed %s\n", dev);
+			return -1;
+		}
+	} else {
+		if (odp_pktout_event_queue(pktio,
+					   gbl_args->pktios[idx].tx_q,
+					   num_tx) != num_tx) {
+			LOG_ERR("Error: event queue query failed %s\n", dev);
+			return -1;
+		}
+	}
+
+	printf("created %i input and %i output queues on (%s)\n",
+	       num_rx, num_tx, dev);
+
+	gbl_args->pktios[idx].num_rx_queue = num_rx;
+	gbl_args->pktios[idx].num_tx_queue = num_tx;
+	gbl_args->pktios[idx].pktio        = pktio;
+
+	return 0;
+}
+
+/**
+ *  Print statistics
+ *
+ * @param num_workers Number of worker threads
+ * @param thr_stats Pointer to stats storage
+ * @param duration Number of seconds to loop in
+ * @param timeout Number of seconds for stats calculation
+ *
+ */
+static int print_speed_stats(int num_workers, stats_t *thr_stats,
+			     int duration, int timeout)
+{
+	uint64_t pkts = 0;
+	uint64_t pkts_prev = 0;
+	uint64_t pps;
+	uint64_t rx_drops, tx_drops;
+	uint64_t maximum_pps = 0;
+	int i;
+	int elapsed = 0;
+	int stats_enabled = 1;
+	int loop_forever = (duration == 0);
+
+	if (timeout <= 0) {
+		stats_enabled = 0;
+		timeout = 1;
+	}
+	/* Wait for all threads to be ready*/
+	odp_barrier_wait(&barrier);
+
+	do {
+		pkts = 0;
+		rx_drops = 0;
+		tx_drops = 0;
+
+		sleep(timeout);
+
+		for (i = 0; i < num_workers; i++) {
+			pkts += thr_stats[i].s.packets;
+			rx_drops += thr_stats[i].s.rx_drops;
+			tx_drops += thr_stats[i].s.tx_drops;
+		}
+		if (stats_enabled) {
+			pps = (pkts - pkts_prev) / timeout;
+			if (pps > maximum_pps)
+				maximum_pps = pps;
+			printf("%" PRIu64 " pps, %" PRIu64 " max pps, ",  pps,
+			       maximum_pps);
+
+			printf(" %" PRIu64 " rx drops, %" PRIu64 " tx drops\n",
+			       rx_drops, tx_drops);
+
+			pkts_prev = pkts;
+		}
+		elapsed += timeout;
+	} while (loop_forever || (elapsed < duration));
+
+	if (stats_enabled)
+		printf("TEST RESULT: %" PRIu64 " maximum packets per second.\n",
+		       maximum_pps);
+
+	return pkts > 100 ? 0 : -1;
+}
+
+static void print_port_mapping(void)
+{
+	int if_count, num_workers;
+	int thr, pktio;
+
+	if_count    = gbl_args->appl.if_count;
+	num_workers = gbl_args->appl.num_workers;
+
+	printf("\nWorker mapping table (port[queue])\n--------------------\n");
+
+	for (thr = 0; thr < num_workers; thr++) {
+		int rx_idx, tx_idx;
+		int rx_queue_idx, tx_queue_idx;
+		thread_args_t *thr_args = &gbl_args->thread[thr];
+		int num = thr_args->num_pktio;
+
+		printf("Worker %i\n", thr);
+
+		for (pktio = 0; pktio < num; pktio++) {
+			rx_idx = thr_args->pktio[pktio].rx_idx;
+			tx_idx = thr_args->pktio[pktio].tx_idx;
+			rx_queue_idx = thr_args->pktio[pktio].rx_queue_idx;
+			tx_queue_idx = thr_args->pktio[pktio].tx_queue_idx;
+			printf("  %i[%i] ->  %i[%i]\n",
+			       rx_idx, rx_queue_idx, tx_idx, tx_queue_idx);
+		}
+	}
+
+	printf("\nPort config\n--------------------\n");
+
+	for (pktio = 0; pktio < if_count; pktio++) {
+		const char *dev = gbl_args->appl.if_names[pktio];
+
+		printf("Port %i (%s)\n", pktio, dev);
+		printf("  rx workers %i\n",
+		       gbl_args->pktios[pktio].num_rx_thr);
+		printf("  tx workers %i\n",
+		       gbl_args->pktios[pktio].num_tx_thr);
+		printf("  rx queues %i\n",
+		       gbl_args->pktios[pktio].num_rx_queue);
+		printf("  tx queues %i\n",
+		       gbl_args->pktios[pktio].num_tx_queue);
+	}
+
+	printf("\n");
+}
+
+/**
+ * Find the destination port for a given input port
+ *
+ * @param port  Input port index
+ */
+static int find_dest_port(int port)
+{
+	/* Even number of ports */
+	if (gbl_args->appl.if_count % 2 == 0)
+		return (port % 2 == 0) ? port + 1 : port - 1;
+
+	/* Odd number of ports */
+	if (port == gbl_args->appl.if_count - 1)
+		return 0;
+	else
+		return port + 1;
+}
+
+/*
+ * Bind worker threads to interfaces and calculate number of queues needed
+ *
+ * less workers (N) than interfaces (M)
+ *  - assign each worker to process every Nth interface
+ *  - workers process inequal number of interfaces, when M is not divisible by N
+ *  - needs only single queue per interface
+ * otherwise
+ *  - assign an interface to every Mth worker
+ *  - interfaces are processed by inequal number of workers, when N is not
+ *    divisible by M
+ *  - tries to configure a queue per worker per interface
+ *  - shares queues, if interface capability does not allows a queue per worker
+ */
+static void bind_workers(void)
+{
+	int if_count, num_workers;
+	int rx_idx, tx_idx, thr, pktio;
+	thread_args_t *thr_args;
+
+	if_count    = gbl_args->appl.if_count;
+	num_workers = gbl_args->appl.num_workers;
+
+	/* initialize port forwarding table */
+	for (rx_idx = 0; rx_idx < if_count; rx_idx++)
+		gbl_args->dst_port[rx_idx] = find_dest_port(rx_idx);
+
+	if (if_count > num_workers) {
+		thr = 0;
+
+		for (rx_idx = 0; rx_idx < if_count; rx_idx++) {
+			thr_args = &gbl_args->thread[thr];
+			pktio    = thr_args->num_pktio;
+			tx_idx   = gbl_args->dst_port[rx_idx];
+			thr_args->pktio[pktio].rx_idx = rx_idx;
+			thr_args->pktio[pktio].tx_idx = tx_idx;
+			thr_args->num_pktio++;
+
+			gbl_args->pktios[rx_idx].num_rx_thr++;
+			gbl_args->pktios[tx_idx].num_tx_thr++;
+
+			thr++;
+			if (thr >= num_workers)
+				thr = 0;
+		}
+	} else {
+		rx_idx = 0;
+
+		for (thr = 0; thr < num_workers; thr++) {
+			thr_args = &gbl_args->thread[thr];
+			pktio    = thr_args->num_pktio;
+			tx_idx   = gbl_args->dst_port[rx_idx];
+			thr_args->pktio[pktio].rx_idx = rx_idx;
+			thr_args->pktio[pktio].tx_idx = tx_idx;
+			thr_args->num_pktio++;
+
+			gbl_args->pktios[rx_idx].num_rx_thr++;
+			gbl_args->pktios[tx_idx].num_tx_thr++;
+
+			rx_idx++;
+			if (rx_idx >= if_count)
+				rx_idx = 0;
+		}
+	}
+}
+
+/*
+ * Bind queues to threads and fill in missing thread arguments (handles)
+ */
+static void bind_queues(void)
+{
+	int num_workers;
+	int thr, pktio;
+
+	num_workers = gbl_args->appl.num_workers;
+
+	for (thr = 0; thr < num_workers; thr++) {
+		int rx_idx, tx_idx;
+		thread_args_t *thr_args = &gbl_args->thread[thr];
+		int num = thr_args->num_pktio;
+
+		for (pktio = 0; pktio < num; pktio++) {
+			int rx_queue, tx_queue;
+
+			rx_idx   = thr_args->pktio[pktio].rx_idx;
+			tx_idx   = thr_args->pktio[pktio].tx_idx;
+			rx_queue = gbl_args->pktios[rx_idx].next_rx_queue;
+			tx_queue = gbl_args->pktios[tx_idx].next_tx_queue;
+
+			thr_args->pktio[pktio].rx_queue_idx = rx_queue;
+			thr_args->pktio[pktio].tx_queue_idx = tx_queue;
+			thr_args->pktio[pktio].pktin =
+				gbl_args->pktios[rx_idx].pktin[rx_queue];
+			thr_args->pktio[pktio].pktout =
+				gbl_args->pktios[tx_idx].pktout[tx_queue];
+			thr_args->pktio[pktio].rx_queue =
+				gbl_args->pktios[rx_idx].rx_q[rx_queue];
+			thr_args->pktio[pktio].tx_queue =
+				gbl_args->pktios[tx_idx].tx_q[tx_queue];
+			thr_args->pktio[pktio].rx_pktio =
+				gbl_args->pktios[rx_idx].pktio;
+			thr_args->pktio[pktio].tx_pktio =
+				gbl_args->pktios[tx_idx].pktio;
+
+			rx_queue++;
+			tx_queue++;
+
+			if (rx_queue >= gbl_args->pktios[rx_idx].num_rx_queue)
+				rx_queue = 0;
+			if (tx_queue >= gbl_args->pktios[tx_idx].num_tx_queue)
+				tx_queue = 0;
+
+			gbl_args->pktios[rx_idx].next_rx_queue = rx_queue;
+			gbl_args->pktios[tx_idx].next_tx_queue = tx_queue;
+		}
+	}
+}
+
+/**
+ * Prinf usage information
+ */
+static void usage(char *progname)
+{
+	printf("\n"
+	       "OpenDataPlane ip firewall application.\n"
+	       "\n"
+	       "Usage: %s OPTIONS\n"
+	       "  E.g. %s -i eth0,eth1 -m 0 -t 1\n"
+	       " In the above example,\n"
+	       " eth0 will send pkts to eth1 and vice versa\n"
+	       "\n"
+	       "Mandatory OPTIONS:\n"
+	       "  -i, --interface Eth interfaces (comma-separated, no spaces)\n"
+	       "                  Interface count min 1, max 2\n"
+	       "\n"
+	       "Optional OPTIONS:\n"
+	       "  -m, --mode      Packet input mode\n"
+	       "                  0: Direct mode: PKTIN_MODE_DIRECT (default)\n"
+	       "                  1: Scheduler mode with parallel queues: PKTIN_MODE_SCHED + SCHED_SYNC_PARALLEL\n"
+	       "                  2: Scheduler mode with atomic queues:   PKTIN_MODE_SCHED + SCHED_SYNC_ATOMIC\n"
+	       "                  3: Scheduler mode with ordered queues:  PKTIN_MODE_SCHED + SCHED_SYNC_ORDERED\n"
+	       "                  4: Plain queue mode: ODP_PKTIN_MODE_QUEUE\n"
+	       "  -o, --out_mode  Packet output mode\n"
+	       "                  0: Direct mode: PKTOUT_MODE_DIRECT (default)\n"
+	       "                  1: Queue mode:  PKTOUT_MODE_QUEUE\n"
+	       "  -c, --count <number> CPU count.\n"
+	       "  -t, --time  <number> Time in seconds to run.\n"
+	       "  -d, --dst_change  0: Don't change packets' dst eth addresses (default)\n"
+	       "                    1: Change packets' dst eth addresses\n"
+	       "  -s, --src_change  0: Don't change packets' src eth addresses\n"
+	       "                    1: Change packets' src eth addresses (default)\n"
+	       "  -r, --dst_addr    Destination addresses (comma-separated, no spaces)\n"
+	       "                    Requires also the -d flag to be set\n"
+	       "  -e, --error_check 0: Don't check packet errors (default)\n"
+	       "                    1: Check packet errors\n"
+	       "  -h, --help           Display help and exit.\n\n"
+	       " environment variables: ODP_PKTIO_DISABLE_NETMAP\n"
+	       "                        ODP_PKTIO_DISABLE_SOCKET_MMAP\n"
+	       "                        ODP_PKTIO_DISABLE_SOCKET_MMSG\n"
+	       " can be used to advanced pkt I/O selection for linux-generic\n"
+	       "\n", NO_PATH(progname), NO_PATH(progname)
+	    );
+}
+
+/**
+ * Parse and store the command line arguments
+ *
+ * @param argc       argument count
+ * @param argv[]     argument vector
+ * @param appl_args  Store application arguments here
+ */
+static void parse_args(int argc, char *argv[], appl_args_t *appl_args)
+{
+	int opt;
+	int long_index;
+	char *token;
+	char *addr_str;
+	size_t len;
+	int i;
+	static struct option longopts[] = {
+		{"count", required_argument, NULL, 'c'},
+		{"time", required_argument, NULL, 't'},
+		{"interface", required_argument, NULL, 'i'},
+		{"mode", required_argument, NULL, 'm'},
+		{"out_mode", required_argument, NULL, 'o'},
+		{"dst_addr", required_argument, NULL, 'r'},
+		{"dst_change", required_argument, NULL, 'd'},
+		{"src_change", required_argument, NULL, 's'},
+		{"error_check", required_argument, NULL, 'e'},
+		{"help", no_argument, NULL, 'h'},
+		{NULL, 0, NULL, 0}
+	};
+
+	appl_args->time = 0; /* loop forever if time to run is 0 */
+	appl_args->src_change = 1; /* change eth src address by default */
+	appl_args->error_check = 0; /* don't check packet errors by default */
+
+	while (1) {
+		opt = getopt_long(argc, argv, "+c:+t:+a:i:m:o:r:d:s:e:h",
+				  longopts, &long_index);
+
+		if (opt == -1)
+			break;	/* No more options */
+
+		switch (opt) {
+		case 'c':
+			appl_args->cpu_count = atoi(optarg);
+			break;
+		case 't':
+			appl_args->time = atoi(optarg);
+			break;
+		case 'a':
+			appl_args->accuracy = atoi(optarg);
+			break;
+			/* parse packet-io interface names */
+		case 'r':
+			len = strlen(optarg);
+			if (len == 0) {
+				usage(argv[0]);
+				exit(EXIT_FAILURE);
+			}
+			len += 1;	/* add room for '\0' */
+
+			addr_str = calloc(1, len);
+			if (addr_str == NULL) {
+				usage(argv[0]);
+				exit(EXIT_FAILURE);
+			}
+
+			/* store the mac addresses names */
+			strcpy(addr_str, optarg);
+			for (token = strtok(addr_str, ","), i = 0;
+			     token != NULL; token = strtok(NULL, ","), i++) {
+				if (i >= MAX_PKTIOS) {
+					printf("too many MAC addresses\n");
+					usage(argv[0]);
+					exit(EXIT_FAILURE);
+				}
+				if (odph_eth_addr_parse(&appl_args->addrs[i],
+							token) != 0) {
+					printf("invalid MAC address\n");
+					usage(argv[0]);
+					exit(EXIT_FAILURE);
+				}
+			}
+			appl_args->addr_count = i;
+			if (appl_args->addr_count < 1) {
+				usage(argv[0]);
+				exit(EXIT_FAILURE);
+			}
+			kern_free(addr_str);
+			break;
+		case 'i':
+			len = strlen(optarg);
+			if (len == 0) {
+				usage(argv[0]);
+				exit(EXIT_FAILURE);
+			}
+			len += 1;	/* add room for '\0' */
+
+			appl_args->if_str = calloc(1, len);
+			if (appl_args->if_str == NULL) {
+				usage(argv[0]);
+				exit(EXIT_FAILURE);
+			}
+
+			/* count the number of tokens separated by ',' */
+			strcpy(appl_args->if_str, optarg);
+			for (token = strtok(appl_args->if_str, ","), i = 0;
+			     token != NULL;
+			     token = strtok(NULL, ","), i++)
+				;
+
+			appl_args->if_count = i;
+
+			if (appl_args->if_count < 1 ||
+			    appl_args->if_count > MAX_PKTIOS) {
+				usage(argv[0]);
+				exit(EXIT_FAILURE);
+			}
+
+			/* allocate storage for the if names */
+			appl_args->if_names =
+			    calloc(appl_args->if_count, sizeof(char *));
+
+			/* store the if names (reset names string) */
+			strcpy(appl_args->if_str, optarg);
+			for (token = strtok(appl_args->if_str, ","), i = 0;
+			     token != NULL; token = strtok(NULL, ","), i++) {
+				appl_args->if_names[i] = token;
+			}
+			break;
+		case 'm':
+			i = atoi(optarg);
+			if (i == 1)
+				appl_args->in_mode = SCHED_PARALLEL;
+			else if (i == 2)
+				appl_args->in_mode = SCHED_ATOMIC;
+			else if (i == 3)
+				appl_args->in_mode = SCHED_ORDERED;
+			else if (i == 4)
+				appl_args->in_mode = PLAIN_QUEUE;
+			else
+				appl_args->in_mode = DIRECT_RECV;
+			break;
+		case 'o':
+			i = atoi(optarg);
+			if (i != 0)
+				appl_args->out_mode = PKTOUT_QUEUE;
+			break;
+		case 'd':
+			appl_args->dst_change = atoi(optarg);
+			break;
+		case 's':
+			appl_args->src_change = atoi(optarg);
+			break;
+		case 'e':
+			appl_args->error_check = atoi(optarg);
+			break;
+		case 'h':
+			usage(argv[0]);
+			exit(EXIT_SUCCESS);
+			break;
+		default:
+			break;
+		}
+	}
+
+	if (appl_args->if_count == 0) {
+		usage(argv[0]);
+		exit(EXIT_FAILURE);
+	}
+	if (appl_args->addr_count != 0 &&
+	    appl_args->addr_count != appl_args->if_count) {
+		printf("Number of destination addresses differs from number"
+		       " of interfaces\n");
+		usage(argv[0]);
+		exit(EXIT_FAILURE);
+	}
+
+	optind = 1;		/* reset 'extern optind' from the getopt lib */
+}
+
+/**
+ * Print system and application info
+ */
+static void print_info(char *progname, appl_args_t *appl_args)
+{
+	int i;
+
+	printf("\n"
+	       "ODP system info\n"
+	       "---------------\n"
+	       "ODP API version: %s\n"
+	       "ODP impl name:   %s\n"
+	       "CPU model:       %s\n"
+	       "CPU freq (hz):   %" PRIu64 "\n"
+	       "Cache line size: %i\n"
+	       "CPU count:       %i\n"
+	       "\n",
+	       odp_version_api_str(), odp_version_impl_name(),
+	       odp_cpu_model_str(), odp_cpu_hz_max(),
+	       odp_sys_cache_line_size(), odp_cpu_count());
+
+	printf("Running ODP appl: \"%s\"\n"
+	       "-----------------\n"
+	       "IF-count:        %i\n"
+	       "Using IFs:      ",
+	       progname, appl_args->if_count);
+	for (i = 0; i < appl_args->if_count; ++i)
+		printf(" %s", appl_args->if_names[i]);
+	printf("\n"
+	       "Mode:            ");
+	if (appl_args->in_mode == DIRECT_RECV)
+		printf("PKTIN_DIRECT, ");
+	else if (appl_args->in_mode == PLAIN_QUEUE)
+		printf("PKTIN_QUEUE, ");
+	else if (appl_args->in_mode == SCHED_PARALLEL)
+		printf("PKTIN_SCHED_PARALLEL, ");
+	else if (appl_args->in_mode == SCHED_ATOMIC)
+		printf("PKTIN_SCHED_ATOMIC, ");
+	else if (appl_args->in_mode == SCHED_ORDERED)
+		printf("PKTIN_SCHED_ORDERED, ");
+
+	if (appl_args->out_mode)
+		printf("PKTOUT_QUEUE");
+	else
+		printf("PKTOUT_DIRECT");
+
+	printf("\n\n");
+	fflush(NULL);
+}
+
+static void gbl_args_init(args_t *args)
+{
+	int pktio, queue;
+
+	memset(args, 0, sizeof(args_t));
+
+	for (pktio = 0; pktio < MAX_PKTIOS; pktio++) {
+		args->pktios[pktio].pktio = ODP_PKTIO_INVALID;
+
+		for (queue = 0; queue < MAX_QUEUES; queue++)
+			args->pktios[pktio].rx_q[queue] = ODP_QUEUE_INVALID;
+	}
+}
+
+void odp_ipfw_exit(void)
+{
+	kern_free(gbl_args->appl.if_names);
+	kern_free(gbl_args->appl.if_str);
+	LOG_ERR("IPfw Exit\n\n");
+}
+
+/**
+ * ODP IP firewall main function
+ */
+int odp_ipfw_main(int argc, char *argv[])
+{
+	odph_linux_pthread_t thread_tbl[MAX_WORKERS];
+	odp_pool_t pool;
+	int i;
+	int cpu;
+	int num_workers;
+	odp_shm_t shm;
+	odp_cpumask_t cpumask;
+	char cpumaskstr[ODP_CPUMASK_STR_SIZE];
+	odph_ethaddr_t new_addr;
+	odp_pool_param_t params;
+	int ret;
+	stats_t *stats;
+	int if_count;
+	int (*thr_run_func)(struct sess*, void *);
+
+	/* Init ODP before calling anything else */
+	if (odp_init_global(NULL, NULL)) {
+		LOG_ERR("Error: ODP global init failed.\n");
+		exit(EXIT_FAILURE);
+	}
+
+	/* Init this thread */
+	if (odp_init_local(ODP_THREAD_CONTROL)) {
+		LOG_ERR("Error: ODP local init failed.\n");
+		exit(EXIT_FAILURE);
+	}
+
+	/* Reserve memory for args from shared mem */
+	shm = odp_shm_reserve("shm_args", sizeof(args_t),
+			      ODP_CACHE_LINE_SIZE, 0);
+	gbl_args = odp_shm_addr(shm);
+
+	if (gbl_args == NULL) {
+		LOG_ERR("Error: shared mem alloc failed.\n");
+		exit(EXIT_FAILURE);
+	}
+	gbl_args_init(gbl_args);
+
+	/* Parse and store the application arguments */
+	parse_args(argc, argv, &gbl_args->appl);
+
+	/* Print both system and application information */
+	print_info(NO_PATH(argv[0]), &gbl_args->appl);
+
+	/* Default to 2, one thread per interface */
+	num_workers = 2;
+	if (gbl_args->appl.cpu_count)
+		num_workers = gbl_args->appl.cpu_count;
+	if (num_workers > 2) {
+		LOG_ERR("Error: currently no multi thread support"
+			" available. please use cpu_count as 2\n");
+		exit(EXIT_FAILURE);
+	}
+
+	/* Get default worker cpumask */
+	num_workers = odp_cpumask_default_worker(&cpumask, num_workers);
+	(void)odp_cpumask_to_str(&cpumask, cpumaskstr, sizeof(cpumaskstr));
+
+	gbl_args->appl.num_workers = num_workers;
+
+	for (i = 0; i < num_workers; i++)
+		gbl_args->thread[i].thr_idx    = i;
+
+	if_count = gbl_args->appl.if_count;
+
+	printf("num worker threads: %i\n", num_workers);
+	printf("first CPU:          %i\n", odp_cpumask_first(&cpumask));
+	printf("cpu mask:           %s\n", cpumaskstr);
+
+	/* Create packet pool */
+	odp_pool_param_init(&params);
+	params.pkt.seg_len = SHM_PKT_POOL_BUF_SIZE;
+	params.pkt.len     = SHM_PKT_POOL_BUF_SIZE;
+	params.pkt.num     = SHM_PKT_POOL_SIZE;
+	params.type        = ODP_POOL_PACKET;
+
+	pool = odp_pool_create("packet pool", &params);
+
+	if (pool == ODP_POOL_INVALID) {
+		LOG_ERR("Error: packet pool create failed.\n");
+		exit(EXIT_FAILURE);
+	}
+	odp_pool_print(pool);
+
+	bind_workers();
+
+	for (i = 0; i < if_count; ++i) {
+		const char *dev = gbl_args->appl.if_names[i];
+		int num_rx, num_tx;
+
+		/* A queue per worker in scheduled mode */
+		num_rx = num_workers;
+		num_tx = num_workers;
+
+		if (gbl_args->appl.in_mode == DIRECT_RECV ||
+		    gbl_args->appl.in_mode == PLAIN_QUEUE) {
+			/* A queue per assigned worker */
+			num_rx = gbl_args->pktios[i].num_rx_thr;
+			num_tx = gbl_args->pktios[i].num_tx_thr;
+		}
+
+		if (create_pktio(dev, i, num_rx, num_tx, pool))
+			exit(EXIT_FAILURE);
+
+		/* Save interface ethernet address */
+		if (odp_pktio_mac_addr(gbl_args->pktios[i].pktio,
+				       gbl_args->port_eth_addr[i].addr,
+				       ODPH_ETHADDR_LEN) != ODPH_ETHADDR_LEN) {
+			LOG_ERR("Error: interface ethernet address unknown\n");
+			exit(EXIT_FAILURE);
+		}
+
+		/* Save destination eth address */
+		if (gbl_args->appl.dst_change) {
+			/* 02:00:00:00:00:XX */
+			memset(&new_addr, 0, sizeof(odph_ethaddr_t));
+			if (gbl_args->appl.addr_count) {
+				memcpy(&new_addr, &gbl_args->appl.addrs[i],
+				       sizeof(odph_ethaddr_t));
+			} else {
+				new_addr.addr[0] = 0x02;
+				new_addr.addr[5] = i;
+			}
+			gbl_args->dst_eth_addr[i] = new_addr;
+		}
+	}
+
+	gbl_args->pktios[i].pktio = ODP_PKTIO_INVALID;
+
+	bind_queues();
+
+	if (gbl_args->appl.in_mode == DIRECT_RECV ||
+	    gbl_args->appl.in_mode == PLAIN_QUEUE)
+		print_port_mapping();
+
+	memset(thread_tbl, 0, sizeof(thread_tbl));
+
+	stats = gbl_args->stats;
+
+	odp_barrier_init(&barrier, num_workers + 1);
+
+	if (gbl_args->appl.in_mode == DIRECT_RECV)
+		thr_run_func = run_worker_direct_mode;
+	else if (gbl_args->appl.in_mode == PLAIN_QUEUE)
+		thr_run_func = run_worker_plain_queue_mode;
+	else /* SCHED_PARALLEL / SCHED_ATOMIC / SCHED_ORDERED */
+		thr_run_func = run_worker_sched_mode;
+
+	for (i = 0; i < num_workers; ++i)
+		gbl_args->thread[i].stats = &stats[i];
+
+	/* Create worker processes that will be called from a tight loop
+	 * at example/ipfw/extra/session.c
+	 */
+	for (i = 0; i < gbl_args->appl.if_count; ++i) {
+		int fd;
+
+		fd = open("/dev/zero", O_RDWR);
+		/* create sessions for odp-ipfw for each port */
+		new_session(fd, thr_run_func, (void *)&gbl_args->thread[i],
+			    WANT_READ);
+	}
+
+	/* Start packet receive and transmit */
+	for (i = 0; i < if_count; ++i) {
+		odp_pktio_t pktio;
+
+		pktio = gbl_args->pktios[i].pktio;
+		ret   = odp_pktio_start(pktio);
+		if (ret) {
+			LOG_ERR("Error: unable to start %s\n",
+				gbl_args->appl.if_names[i]);
+			exit(EXIT_FAILURE);
+		}
+	}
+	return ret;
+}
diff --git a/example/ipfw/extra/session.c b/example/ipfw/extra/session.c
index 333edd3..cf47f71 100644
--- a/example/ipfw/extra/session.c
+++ b/example/ipfw/extra/session.c
@@ -610,6 +610,9 @@  mainloop(int argc, char *argv[])
 	}
 #endif /* WITH_NETMAP */
 
+	/* ODP initialization of ports */
+	odp_ipfw_main(argc, argv);
+
 #if 0 // test code: a telnet on 5556 becomes an infinite source
 	{
 		int net_fd = do_server(addr, port+1);
@@ -639,6 +642,7 @@  mainloop(int argc, char *argv[])
 		}
 		RD(1, "callouts %lu skipped %lu", (u_long)callouts, (u_long)skipped);
 	}
+	odp_ipfw_exit();
 	ipfw_destroy();
 	return 0;
 }