@@ -31,8 +31,7 @@ extern "C" {
*/
typedef uint32_t odp_buffer_t;
-#define ODP_BUFFER_INVALID (0xffffffff) /**< Invalid buffer */
-
+#define ODP_BUFFER_INVALID (0) /**< Invalid buffer */
/**
* Buffer start address
@@ -40,7 +40,6 @@
#error "platform not defined or unsupported!"
#endif
-#define TI_ODP_PUBLIC_DESC_SIZE (64u)
#define TI_ODP_PUBLIC_DESC_NUM (4096u)
#define TI_ODP_REGION_NUM (2) /* local regions are not used on Linux */
@@ -22,21 +22,23 @@ extern "C" {
#include <odp_atomic.h>
#include <odp_buffer_pool.h>
#include <odp_buffer.h>
+#include <odp_queue.h>
#include <odp_debug.h>
#include <odp_align.h>
-/* TODO: move these to correct files */
-
-typedef uint64_t odp_phys_addr_t;
-
-#define ODP_BUFFER_MAX_INDEX (ODP_BUFFER_MAX_BUFFERS - 2)
-#define ODP_BUFFER_INVALID_INDEX (ODP_BUFFER_MAX_BUFFERS - 1)
+#include <event_machine_macros.h>
+#include <event_machine_types.h>
+#include <event_machine_group.h>
+#include <event_machine_hw_macros.h>
+#include <event_machine_hw_types.h>
+#include <event_machine_hw_ti_macros.h>
+#include <event_machine_hw_ti_types.h>
+#include <ti_em_osal_cppi.h>
+#include <src/event_machine_hwpform.h>
-#define ODP_BUFS_PER_CHUNK 16
-#define ODP_BUFS_PER_SCATTER 4
-
-#define ODP_BUFFER_TYPE_CHUNK 0xffff
+/* TODO: move these to correct files */
+typedef uintptr_t odp_phys_addr_t;
#define ODP_BUFFER_POOL_BITS 4
#define ODP_BUFFER_INDEX_BITS (32 - ODP_BUFFER_POOL_BITS)
@@ -53,56 +55,33 @@ typedef union odp_buffer_bits_t {
};
} odp_buffer_bits_t;
-
-/* forward declaration */
-struct odp_buffer_hdr_t;
-
-
-/*
- * Scatter/gather list of buffers
- */
-typedef struct odp_buffer_scatter_t {
- /* buffer pointers */
- struct odp_buffer_hdr_t *buf[ODP_BUFS_PER_SCATTER];
- int num_bufs; /* num buffers */
- int pos; /* position on the list */
- size_t total_len; /* Total length */
-} odp_buffer_scatter_t;
+typedef struct odp_buffer_hdr_t {
+ Cppi_HostDesc desc;
+ void *buf_vaddr;
+ odp_queue_t free_queue;
+ int type;
+ struct odp_buffer_hdr_t *next; /* next buf in a list */
+ odp_buffer_bits_t handle; /* handle */
+} odp_buffer_hdr_t;
/*
* Chunk of buffers (in single pool)
*/
-typedef struct odp_buffer_chunk_t {
- uint32_t num_bufs; /* num buffers */
- uint32_t buf_index[ODP_BUFS_PER_CHUNK]; /* buffers */
-} odp_buffer_chunk_t;
-
-typedef struct odp_buffer_hdr_t {
- struct odp_buffer_hdr_t *next; /* next buf in a list */
- odp_buffer_bits_t handle; /* handle */
- odp_phys_addr_t phys_addr; /* physical data start address */
- void *addr; /* virtual data start address */
- uint32_t index; /* buf index in the pool */
- size_t size; /* max data size */
- size_t cur_offset; /* current offset */
- odp_atomic_int_t ref_count; /* reference count */
- odp_buffer_scatter_t scatter; /* Scatter/gather list */
- int type; /* type of next header */
- odp_buffer_pool_t pool; /* buffer pool */
-
- uint8_t payload[]; /* next header or data */
-} odp_buffer_hdr_t;
-
-ODP_ASSERT(sizeof(odp_buffer_hdr_t) == ODP_OFFSETOF(odp_buffer_hdr_t, payload),
+ODP_ASSERT(sizeof(odp_buffer_hdr_t) <= ODP_CACHE_LINE_SIZE,
ODP_BUFFER_HDR_T__SIZE_ERROR);
+static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf)
+{
+ return (odp_buffer_hdr_t *)buf;
+}
+static inline odp_buffer_t hdr_to_odp_buf(odp_buffer_hdr_t *hdr)
+{
+ return (odp_buffer_t)hdr;
+}
-typedef struct odp_buffer_chunk_hdr_t {
- odp_buffer_hdr_t buf_hdr;
- odp_buffer_chunk_t chunk;
-} odp_buffer_chunk_hdr_t;
+extern odp_buffer_pool_t odp_buf_to_pool(odp_buffer_t buf);
int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf);
@@ -39,7 +39,6 @@ extern "C" {
#include <odp_spinlock.h>
#endif
-
struct pool_entry_s {
#ifdef POOL_USE_TICKETLOCK
odp_ticketlock_t lock ODP_ALIGNED_CACHE;
@@ -47,24 +46,25 @@ struct pool_entry_s {
odp_spinlock_t lock ODP_ALIGNED_CACHE;
#endif
- odp_buffer_chunk_hdr_t *head;
uint64_t free_bufs;
char name[ODP_BUFFER_POOL_NAME_LEN];
odp_buffer_pool_t pool ODP_ALIGNED_CACHE;
- uintptr_t buf_base;
- size_t buf_size;
- size_t buf_offset;
uint64_t num_bufs;
void *pool_base_addr;
+ uintptr_t pool_base_paddr;
uint64_t pool_size;
size_t payload_size;
size_t payload_align;
int buf_type;
+ odp_queue_t free_queue;
+
+ uintptr_t buf_base;
+ size_t buf_size;
+ size_t buf_offset;
size_t hdr_size;
};
-
extern void *pool_entry_ptr[];
@@ -73,41 +73,6 @@ static inline void *get_pool_entry(odp_buffer_pool_t pool_id)
return pool_entry_ptr[pool_id];
}
-
-static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf)
-{
- odp_buffer_bits_t handle;
- uint32_t pool_id;
- uint32_t index;
- struct pool_entry_s *pool;
- odp_buffer_hdr_t *hdr;
-
- handle.u32 = buf;
- pool_id = handle.pool;
- index = handle.index;
-
-#ifdef POOL_ERROR_CHECK
- if (odp_unlikely(pool_id > ODP_CONFIG_BUFFER_POOLS)) {
- ODP_ERR("odp_buf_to_hdr: Bad pool id\n");
- return NULL;
- }
-#endif
-
- pool = get_pool_entry(pool_id);
-
-#ifdef POOL_ERROR_CHECK
- if (odp_unlikely(index > pool->num_bufs - 1)) {
- ODP_ERR("odp_buf_to_hdr: Bad buffer index\n");
- return NULL;
- }
-#endif
-
- hdr = (odp_buffer_hdr_t *)(pool->buf_base + index * pool->buf_size);
-
- return hdr;
-}
-
-
#ifdef __cplusplus
}
#endif
@@ -7,59 +7,38 @@
#include <odp_buffer.h>
#include <odp_buffer_internal.h>
#include <odp_buffer_pool_internal.h>
-
-#include <string.h>
-#include <stdio.h>
-
+#include <ti_em_rh.h>
void *odp_buffer_addr(odp_buffer_t buf)
{
- odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf);
-
- return hdr->addr;
+ return odp_buf_to_hdr(buf)->buf_vaddr;
}
-
size_t odp_buffer_size(odp_buffer_t buf)
{
- odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf);
-
- return hdr->size;
+ return (size_t)odp_buf_to_hdr(buf)->desc.origBufferLen;
}
-
int odp_buffer_type(odp_buffer_t buf)
{
- odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf);
-
- return hdr->type;
+ return odp_buf_to_hdr(buf)->type;
}
-
int odp_buffer_is_scatter(odp_buffer_t buf)
{
- odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf);
-
- if (hdr->scatter.num_bufs == 0)
- return 0;
- else
- return 1;
+ return (odp_buf_to_hdr(buf)->desc.nextBDPtr) ? 1 : 0;
}
int odp_buffer_is_valid(odp_buffer_t buf)
{
- odp_buffer_bits_t handle;
-
- handle.u32 = buf;
-
- return (handle.index != ODP_BUFFER_INVALID_INDEX);
+ return (buf != ODP_BUFFER_INVALID);
}
int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf)
{
- odp_buffer_hdr_t *hdr;
+ odp_buffer_hdr_t *desc;
int len = 0;
if (!odp_buffer_is_valid(buf)) {
@@ -67,34 +46,27 @@ int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf)
return len;
}
- hdr = odp_buf_to_hdr(buf);
+ desc = odp_buf_to_hdr(buf);
len += snprintf(&str[len], n-len,
"Buffer\n");
len += snprintf(&str[len], n-len,
- " pool %i\n", hdr->pool);
- len += snprintf(&str[len], n-len,
- " index %"PRIu32"\n", hdr->index);
- len += snprintf(&str[len], n-len,
- " phy_addr %"PRIu64"\n", hdr->phys_addr);
- len += snprintf(&str[len], n-len,
- " addr %p\n", hdr->addr);
- len += snprintf(&str[len], n-len,
- " size %zu\n", hdr->size);
+ " desc_vaddr %p\n", desc);
len += snprintf(&str[len], n-len,
- " cur_offset %zu\n", hdr->cur_offset);
+ " buf_vaddr %p\n", desc->buf_vaddr);
len += snprintf(&str[len], n-len,
- " ref_count %i\n", hdr->ref_count);
+ " buf_paddr_o 0x%x\n", desc->desc.origBuffPtr);
len += snprintf(&str[len], n-len,
- " type %i\n", hdr->type);
+ " buf_paddr 0x%x\n", desc->desc.buffPtr);
len += snprintf(&str[len], n-len,
- " Scatter list\n");
+ " pool %i\n", odp_buf_to_pool(buf));
len += snprintf(&str[len], n-len,
- " num_bufs %i\n", hdr->scatter.num_bufs);
- len += snprintf(&str[len], n-len,
- " pos %i\n", hdr->scatter.pos);
- len += snprintf(&str[len], n-len,
- " total_len %zu\n", hdr->scatter.total_len);
+ " free_queue %u\n", desc->free_queue);
+
+ len += snprintf(&str[len], n-len, "\n");
+
+ ti_em_rh_dump_mem(desc, sizeof(*desc), "Descriptor dump");
+ ti_em_rh_dump_mem(desc->buf_vaddr, 64, "Buffer start");
return len;
}
@@ -10,15 +10,18 @@
#include <odp_buffer_internal.h>
#include <odp_packet_internal.h>
#include <odp_shared_memory.h>
+#include <odp_shared_memory_internal.h>
#include <odp_align.h>
#include <odp_internal.h>
#include <odp_config.h>
+#include <configs/odp_config_platform.h>
#include <odp_hints.h>
#include <odp_debug.h>
+#include <odp_sync.h>
#include <string.h>
#include <stdlib.h>
-
+#include <ti_em_rh.h>
#ifdef POOL_USE_TICKETLOCK
#include <odp_ticketlock.h>
@@ -33,13 +36,11 @@
#endif
-#if ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS
-#error ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS
-#endif
-
#define NULL_INDEX ((uint32_t)-1)
+
+
typedef union pool_entry_u {
struct pool_entry_s s;
@@ -53,6 +54,10 @@ typedef struct pool_table_t {
} pool_table_t;
+typedef struct {
+ uintptr_t p;
+ uintptr_t v;
+} pvaddr_t;
/* The pool table */
static pool_table_t *pool_tbl;
@@ -60,25 +65,29 @@ static pool_table_t *pool_tbl;
/* Pool entry pointers (for inlining) */
void *pool_entry_ptr[ODP_CONFIG_BUFFER_POOLS];
-
-static __thread odp_buffer_chunk_hdr_t *local_chunk[ODP_CONFIG_BUFFER_POOLS];
-
-
-static inline void set_handle(odp_buffer_hdr_t *hdr,
- pool_entry_t *pool, uint32_t index)
+static uint32_t ti_odp_alloc_public_desc(uint32_t num)
{
- uint32_t pool_id = (uint32_t) pool->s.pool;
+ static uint32_t free_desc_id;
+ uint32_t tmp;
- if (pool_id > ODP_CONFIG_BUFFER_POOLS)
- ODP_ERR("set_handle: Bad pool id\n");
+ if (free_desc_id + num > TI_ODP_PUBLIC_DESC_NUM)
+ return -1;
- if (index > ODP_BUFFER_MAX_INDEX)
- ODP_ERR("set_handle: Bad buffer index\n");
+ tmp = __sync_fetch_and_add(&free_desc_id, num);
- hdr->handle.pool = pool_id;
- hdr->handle.index = index;
+ if (tmp + num > TI_ODP_PUBLIC_DESC_NUM) {
+ __sync_fetch_and_sub(&free_desc_id, num);
+ return -1;
+ }
+ return tmp;
}
+odp_buffer_pool_t odp_buf_to_pool(odp_buffer_t buf)
+{
+ odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf);
+ pool_entry_t *pool = get_pool_entry(0);
+ return hdr->free_queue - pool->s.free_queue;
+}
int odp_buffer_pool_init_global(void)
{
@@ -98,231 +107,133 @@ int odp_buffer_pool_init_global(void)
pool_entry_t *pool = &pool_tbl->pool[i];
LOCK_INIT(&pool->s.lock);
pool->s.pool = i;
-
pool_entry_ptr[i] = pool;
+ pool->s.free_queue = TI_ODP_FREE_QUEUE_BASE_IDX + i;
}
ODP_DBG("\nBuffer pool init global\n");
ODP_DBG(" pool_entry_s size %zu\n", sizeof(struct pool_entry_s));
ODP_DBG(" pool_entry_t size %zu\n", sizeof(pool_entry_t));
- ODP_DBG(" odp_buffer_hdr_t size %zu\n", sizeof(odp_buffer_hdr_t));
ODP_DBG("\n");
return 0;
}
+#define MAX_BUFS_PER_POOL 2000
-static odp_buffer_hdr_t *index_to_hdr(pool_entry_t *pool, uint32_t index)
-{
- odp_buffer_hdr_t *hdr;
-
- hdr = (odp_buffer_hdr_t *)(pool->s.buf_base + index * pool->s.buf_size);
- return hdr;
-}
-
-
-static void add_buf_index(odp_buffer_chunk_hdr_t *chunk_hdr, uint32_t index)
-{
- uint32_t i = chunk_hdr->chunk.num_bufs;
- chunk_hdr->chunk.buf_index[i] = index;
- chunk_hdr->chunk.num_bufs++;
-}
-
-
-static uint32_t rem_buf_index(odp_buffer_chunk_hdr_t *chunk_hdr)
-{
- uint32_t index;
- uint32_t i;
-
- i = chunk_hdr->chunk.num_bufs - 1;
- index = chunk_hdr->chunk.buf_index[i];
- chunk_hdr->chunk.num_bufs--;
- return index;
-}
-
-
-static odp_buffer_chunk_hdr_t *next_chunk(pool_entry_t *pool,
- odp_buffer_chunk_hdr_t *chunk_hdr)
-{
- uint32_t index;
-
- index = chunk_hdr->chunk.buf_index[ODP_BUFS_PER_CHUNK-1];
- if (index == NULL_INDEX)
- return NULL;
- else
- return (odp_buffer_chunk_hdr_t *)index_to_hdr(pool, index);
-}
-
-
-static odp_buffer_chunk_hdr_t *rem_chunk(pool_entry_t *pool)
+static int link_bufs(pool_entry_t *pool)
{
- odp_buffer_chunk_hdr_t *chunk_hdr;
-
- chunk_hdr = pool->s.head;
- if (chunk_hdr == NULL) {
- /* Pool is empty */
- return NULL;
+ size_t buf_size, buf_align;
+ uint64_t pool_size;
+ uintptr_t pool_base;
+ pvaddr_t buf_addr, desc_addr;
+ uint32_t desc_index;
+ uint32_t num_bufs, i;
+
+ buf_align = pool->s.payload_align;
+ buf_size = ODP_ALIGN_ROUNDUP(pool->s.payload_size, buf_align);
+ pool_size = pool->s.pool_size;
+ pool_base = (uintptr_t) pool->s.pool_base_addr;
+ /* First buffer */
+ buf_addr.v = ODP_ALIGN_ROUNDUP(pool_base, buf_align);
+ buf_addr.p = _odp_shm_get_paddr((void *)buf_addr.v);
+ pool->s.buf_base = buf_addr.v;
+
+ num_bufs = (pool_size - (buf_addr.v - pool_base)) / buf_size;
+ /*
+ * FIXME: Currently a number of HW descriptors is limited,
+ * so temporary limit max number of buffers per pool
+ * to be albe to run ODP example apps.
+ * Descriptor management have to be made more intelligent
+ * To remove this limitation.
+ */
+ if (num_bufs > MAX_BUFS_PER_POOL) {
+ ODP_DBG("Limiting number of buffer in %s from %d to %d\n",
+ pool->s.name, num_bufs, MAX_BUFS_PER_POOL);
+ num_bufs = MAX_BUFS_PER_POOL;
}
- pool->s.head = next_chunk(pool, chunk_hdr);
- pool->s.free_bufs -= ODP_BUFS_PER_CHUNK;
-
- /* unlink */
- rem_buf_index(chunk_hdr);
- return chunk_hdr;
-}
-
-
-static void add_chunk(pool_entry_t *pool, odp_buffer_chunk_hdr_t *chunk_hdr)
-{
- if (pool->s.head) {
- /* link pool head to the chunk */
- add_buf_index(chunk_hdr, pool->s.head->buf_hdr.index);
- } else
- add_buf_index(chunk_hdr, NULL_INDEX);
-
- pool->s.head = chunk_hdr;
- pool->s.free_bufs += ODP_BUFS_PER_CHUNK;
-}
-
-
-static void check_align(pool_entry_t *pool, odp_buffer_hdr_t *hdr)
-{
- if (!ODP_ALIGNED_CHECK_POWER_2(hdr->addr, pool->s.payload_align)) {
- ODP_ERR("check_align: payload align error %p, align %zu\n",
- hdr->addr, pool->s.payload_align);
- exit(0);
+ desc_index = ti_odp_alloc_public_desc(num_bufs);
+
+ ODP_DBG("%s: buf_size: %zu, buf_align: %zu\n", __func__,
+ buf_size, buf_align);
+ ODP_DBG("%s: pool_size: %llu, pool_base: 0x%p\n", __func__,
+ pool_size, (void *)pool_base);
+ ODP_DBG("%s: buf_addr.v: 0x%p, buf_addr.p: 0x%p\n", __func__,
+ (void *)buf_addr.v, (void *)buf_addr.p);
+ ODP_DBG("%s: num_bufs: %u, desc_index: %u\n", __func__,
+ num_bufs, desc_index);
+
+ /* FIXME: Need to define error codes somewhere */
+ if (desc_index == (uint32_t)-1) {
+ ODP_ERR("Failed to allocate %u descriptors for pool %s\n",
+ num_bufs, pool->s.name);
+ return -1;
}
- if (!ODP_ALIGNED_CHECK_POWER_2(hdr, ODP_CACHE_LINE_SIZE)) {
- ODP_ERR("check_align: hdr align error %p, align %i\n",
- hdr, ODP_CACHE_LINE_SIZE);
- exit(0);
+ if (ti_em_osal_hw_queue_open(pool->s.free_queue) != EM_OK) {
+ ODP_ERR("Failed to open HW queue %u\n", pool->s.free_queue);
+ return -1;
}
-}
-
-
-static void fill_hdr(void *ptr, pool_entry_t *pool, uint32_t index,
- int buf_type)
-{
- odp_buffer_hdr_t *hdr = (odp_buffer_hdr_t *)ptr;
- size_t size = pool->s.hdr_size;
- uint8_t *payload = hdr->payload;
- if (buf_type == ODP_BUFFER_TYPE_CHUNK)
- size = sizeof(odp_buffer_chunk_hdr_t);
+ for (i = 0; i < num_bufs; i++) {
+ Cppi_DescTag tag;
+ odp_buffer_hdr_t *hdr;
- if (pool->s.buf_type == ODP_BUFFER_TYPE_PACKET) {
- odp_packet_hdr_t *packet_hdr = ptr;
- payload = packet_hdr->payload;
+ /*
+ * TODO: Need to get descriptor size here and shift
+ * descriptor address, but not query it on every iteration.
+ */
+ desc_addr.v = (uintptr_t)ti_em_rh_public_desc_addr(desc_index,
+ &desc_addr.p);
+ hdr = (odp_buffer_hdr_t *)desc_addr.v;
+ memset((void *)hdr, 0, sizeof(*hdr));
+
+ hdr->free_queue = pool->s.free_queue;
+ hdr->buf_vaddr = (void *)buf_addr.v;
+
+ /* Set defaults in descriptor */
+ hdr->desc.descInfo = (Cppi_DescType_HOST << 30) |
+ (Cppi_PSLoc_PS_IN_DESC << 22) |
+ (buf_size & 0xFFFF);
+ hdr->desc.packetInfo =
+ (((uint32_t) Cppi_EPIB_EPIB_PRESENT) << 31) |
+ (0x2 << 16) |
+ (((uint32_t) Cppi_ReturnPolicy_RETURN_BUFFER) << 15) |
+ (pool->s.free_queue & 0x3FFF);
+ hdr->desc.origBuffPtr = buf_addr.p;
+ hdr->desc.buffPtr = buf_addr.p;
+ hdr->desc.origBufferLen = buf_size;
+ hdr->desc.buffLen = buf_size;
+
+ /* TODO: pslen is set to 0, but should be configurable */
+ ti_em_cppi_set_pslen(Cppi_DescType_HOST,
+ (Cppi_Desc *)(hdr), 0);
+
+ tag.srcTagHi = 0x00;
+ tag.srcTagLo = 0xFF;
+ tag.destTagHi = 0x00;
+ tag.destTagLo = 0x00;
+ ti_em_cppi_set_tag(Cppi_DescType_HOST,
+ (Cppi_Desc *)(hdr),
+ &tag);
+
+ odp_sync_stores();
+ ti_em_osal_hw_queue_push_size(pool->s.free_queue,
+ (void *)hdr,
+ sizeof(Cppi_HostDesc),
+ TI_EM_MEM_PUBLIC_DESC);
+ buf_addr.v += buf_size;
+ buf_addr.p += buf_size;
+ desc_index++;
}
- memset(hdr, 0, size);
-
- set_handle(hdr, pool, index);
-
- hdr->addr = &payload[pool->s.buf_offset - pool->s.hdr_size];
- hdr->index = index;
- hdr->size = pool->s.payload_size;
- hdr->pool = pool->s.pool;
- hdr->type = buf_type;
-
- check_align(pool, hdr);
-}
-
-
-static void link_bufs(pool_entry_t *pool)
-{
- odp_buffer_chunk_hdr_t *chunk_hdr;
- size_t hdr_size;
- size_t payload_size;
- size_t payload_align;
- size_t size;
- size_t offset;
- size_t min_size;
- uint64_t pool_size;
- uintptr_t buf_base;
- uint32_t index;
- uintptr_t pool_base;
- int buf_type;
-
- buf_type = pool->s.buf_type;
- payload_size = pool->s.payload_size;
- payload_align = pool->s.payload_align;
- pool_size = pool->s.pool_size;
- pool_base = (uintptr_t) pool->s.pool_base_addr;
-
- if (buf_type == ODP_BUFFER_TYPE_RAW)
- hdr_size = sizeof(odp_buffer_hdr_t);
- else if (buf_type == ODP_BUFFER_TYPE_PACKET)
- hdr_size = sizeof(odp_packet_hdr_t);
- else {
- ODP_ERR("odp_buffer_pool_create: Bad type %i\n",
- buf_type);
- exit(0);
- }
-
- /* Chunk must fit into buffer payload.*/
- min_size = sizeof(odp_buffer_chunk_hdr_t) - hdr_size;
- if (payload_size < min_size)
- payload_size = min_size;
-
- /* Roundup payload size to full cachelines */
- payload_size = ODP_CACHE_LINE_SIZE_ROUNDUP(payload_size);
-
- /* Min cacheline alignment for buffer header and payload */
- payload_align = ODP_CACHE_LINE_SIZE_ROUNDUP(payload_align);
- offset = ODP_CACHE_LINE_SIZE_ROUNDUP(hdr_size);
-
- /* Multiples of cacheline size */
- if (payload_size > payload_align)
- size = payload_size + offset;
- else
- size = payload_align + offset;
-
- /* First buffer */
- buf_base = ODP_ALIGN_ROUNDUP(pool_base + offset, payload_align)
- - offset;
-
- pool->s.hdr_size = hdr_size;
- pool->s.buf_base = buf_base;
- pool->s.buf_size = size;
- pool->s.buf_offset = offset;
- index = 0;
-
- chunk_hdr = (odp_buffer_chunk_hdr_t *)index_to_hdr(pool, index);
- pool->s.head = NULL;
- pool_size -= buf_base - pool_base;
-
- while (pool_size > ODP_BUFS_PER_CHUNK * size) {
- int i;
-
- fill_hdr(chunk_hdr, pool, index, ODP_BUFFER_TYPE_CHUNK);
-
- index++;
-
- for (i = 0; i < ODP_BUFS_PER_CHUNK - 1; i++) {
- odp_buffer_hdr_t *hdr = index_to_hdr(pool, index);
-
- fill_hdr(hdr, pool, index, buf_type);
-
- add_buf_index(chunk_hdr, index);
- index++;
- }
-
- add_chunk(pool, chunk_hdr);
-
- chunk_hdr = (odp_buffer_chunk_hdr_t *)index_to_hdr(pool,
- index);
- pool->s.num_bufs += ODP_BUFS_PER_CHUNK;
- pool_size -= ODP_BUFS_PER_CHUNK * size;
- }
+ return 0;
}
-
odp_buffer_pool_t odp_buffer_pool_create(const char *name,
- void *base_addr, uint64_t size,
- size_t buf_size, size_t buf_align,
- int buf_type)
+ void *base_addr, uint64_t size,
+ size_t buf_size, size_t buf_align,
+ int buf_type)
{
odp_buffer_pool_t i;
pool_entry_t *pool;
@@ -335,7 +246,8 @@ odp_buffer_pool_t odp_buffer_pool_create(const char *name,
if (pool->s.buf_base == 0) {
/* found free pool */
-
+ ODP_DBG("%s: found free pool id: %u for %s\n", __func__,
+ i, name);
strncpy(pool->s.name, name,
ODP_BUFFER_POOL_NAME_LEN - 1);
pool->s.name[ODP_BUFFER_POOL_NAME_LEN - 1] = 0;
@@ -344,12 +256,12 @@ odp_buffer_pool_t odp_buffer_pool_create(const char *name,
pool->s.payload_size = buf_size;
pool->s.payload_align = buf_align;
pool->s.buf_type = buf_type;
+ pool->s.buf_base = (uintptr_t)ODP_ALIGN_ROUNDUP_PTR(
+ base_addr, buf_align);
- link_bufs(pool);
-
+ if (link_bufs(pool) != -1)
+ pool_id = i;
UNLOCK(&pool->s.lock);
-
- pool_id = i;
break;
}
@@ -359,7 +271,6 @@ odp_buffer_pool_t odp_buffer_pool_create(const char *name,
return pool_id;
}
-
odp_buffer_pool_t odp_buffer_pool_lookup(const char *name)
{
odp_buffer_pool_t i;
@@ -383,129 +294,22 @@ odp_buffer_pool_t odp_buffer_pool_lookup(const char *name)
odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool_id)
{
- pool_entry_t *pool;
- odp_buffer_chunk_hdr_t *chunk;
- odp_buffer_bits_t handle;
-
- pool = get_pool_entry(pool_id);
- chunk = local_chunk[pool_id];
-
- if (chunk == NULL) {
- LOCK(&pool->s.lock);
- chunk = rem_chunk(pool);
- UNLOCK(&pool->s.lock);
-
- if (chunk == NULL)
- return ODP_BUFFER_INVALID;
-
- local_chunk[pool_id] = chunk;
- }
-
- if (chunk->chunk.num_bufs == 0) {
- /* give the chunk buffer */
- local_chunk[pool_id] = NULL;
- chunk->buf_hdr.type = pool->s.buf_type;
-
- handle = chunk->buf_hdr.handle;
- } else {
- odp_buffer_hdr_t *hdr;
- uint32_t index;
- index = rem_buf_index(chunk);
- hdr = index_to_hdr(pool, index);
-
- handle = hdr->handle;
- }
-
- return handle.u32;
+ pool_entry_t *pool = get_pool_entry(pool_id);
+ return (odp_buffer_t)ti_em_osal_hw_queue_pop(pool->s.free_queue,
+ TI_EM_MEM_PUBLIC_DESC);
}
void odp_buffer_free(odp_buffer_t buf)
{
- odp_buffer_hdr_t *hdr;
- odp_buffer_pool_t pool_id;
- pool_entry_t *pool;
- odp_buffer_chunk_hdr_t *chunk_hdr;
-
- hdr = odp_buf_to_hdr(buf);
- pool_id = hdr->pool;
- pool = get_pool_entry(pool_id);
- chunk_hdr = local_chunk[pool_id];
-
- if (chunk_hdr && chunk_hdr->chunk.num_bufs == ODP_BUFS_PER_CHUNK - 1) {
- /* Current chunk is full. Push back to the pool */
- LOCK(&pool->s.lock);
- add_chunk(pool, chunk_hdr);
- UNLOCK(&pool->s.lock);
- chunk_hdr = NULL;
- }
-
- if (chunk_hdr == NULL) {
- /* Use this buffer */
- chunk_hdr = (odp_buffer_chunk_hdr_t *)hdr;
- local_chunk[pool_id] = chunk_hdr;
- chunk_hdr->chunk.num_bufs = 0;
- } else {
- /* Add to current chunk */
- add_buf_index(chunk_hdr, hdr->index);
- }
+ odp_buffer_hdr_t *hdr = odp_buf_to_hdr(buf);
+ ti_em_osal_hw_queue_push_size(hdr->free_queue,
+ (void *)hdr,
+ sizeof(Cppi_HostDesc),
+ TI_EM_MEM_PUBLIC_DESC);
}
-
void odp_buffer_pool_print(odp_buffer_pool_t pool_id)
{
- pool_entry_t *pool;
- odp_buffer_chunk_hdr_t *chunk_hdr;
- uint32_t i;
-
- pool = get_pool_entry(pool_id);
-
- printf("Pool info\n");
- printf("---------\n");
- printf(" pool %i\n", pool->s.pool);
- printf(" name %s\n", pool->s.name);
- printf(" pool base %p\n", pool->s.pool_base_addr);
- printf(" buf base 0x%"PRIxPTR"\n", pool->s.buf_base);
- printf(" pool size 0x%"PRIx64"\n", pool->s.pool_size);
- printf(" buf size %zu\n", pool->s.payload_size);
- printf(" buf align %zu\n", pool->s.payload_align);
- printf(" hdr size %zu\n", pool->s.hdr_size);
- printf(" alloc size %zu\n", pool->s.buf_size);
- printf(" offset to hdr %zu\n", pool->s.buf_offset);
- printf(" num bufs %"PRIu64"\n", pool->s.num_bufs);
- printf(" free bufs %"PRIu64"\n", pool->s.free_bufs);
-
- /* first chunk */
- chunk_hdr = pool->s.head;
-
- if (chunk_hdr == NULL) {
- ODP_ERR(" POOL EMPTY\n");
- return;
- }
-
- printf("\n First chunk\n");
-
- for (i = 0; i < chunk_hdr->chunk.num_bufs - 1; i++) {
- uint32_t index;
- odp_buffer_hdr_t *hdr;
-
- index = chunk_hdr->chunk.buf_index[i];
- hdr = index_to_hdr(pool, index);
-
- printf(" [%i] addr %p, id %"PRIu32"\n", i, hdr->addr, index);
- }
-
- printf(" [%i] addr %p, id %"PRIu32"\n", i, chunk_hdr->buf_hdr.addr,
- chunk_hdr->buf_hdr.index);
-
- /* next chunk */
- chunk_hdr = next_chunk(pool, chunk_hdr);
-
- if (chunk_hdr) {
- printf(" Next chunk\n");
- printf(" addr %p, id %"PRIu32"\n", chunk_hdr->buf_hdr.addr,
- chunk_hdr->buf_hdr.index);
- }
-
- printf("\n");
+ (void)pool_id;
}
@@ -12,6 +12,7 @@
#include <ti_em_osal_queue.h>
#include <ti_em_rh.h>
#include <odp_config.h>
+#include <odp_buffer_internal.h>
/*
* Make region_configs[] global, because hw_config is saved in
@@ -47,7 +48,8 @@ static int ti_init_hw_config(void)
/* Define descriptor regions */
reg_config = ®ion_configs[TI_EM_RH_PUBLIC];
reg_config->region_idx = TI_ODP_PUBLIC_REGION_IDX;
- reg_config->desc_size = TI_ODP_PUBLIC_DESC_SIZE;
+ reg_config->desc_size =
+ ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_buffer_hdr_t));
reg_config->desc_num = TI_ODP_PUBLIC_DESC_NUM;
reg_config->desc_base = TI_ODP_PUBLIC_DESC_BASE;
reg_config->desc_vbase = TI_ODP_PUBLIC_DESC_VBASE;
@@ -331,38 +331,7 @@ void odp_packet_print(odp_packet_t pkt)
int odp_packet_copy(odp_packet_t pkt_dst, odp_packet_t pkt_src)
{
- odp_packet_hdr_t *const pkt_hdr_dst = odp_packet_hdr(pkt_dst);
- odp_packet_hdr_t *const pkt_hdr_src = odp_packet_hdr(pkt_src);
- const size_t start_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, buf_hdr);
- uint8_t *start_src;
- uint8_t *start_dst;
- size_t len;
-
- if (pkt_dst == ODP_PACKET_INVALID || pkt_src == ODP_PACKET_INVALID)
- return -1;
-
- if (pkt_hdr_dst->buf_hdr.size <
- pkt_hdr_src->frame_len + pkt_hdr_src->frame_offset)
- return -1;
-
- /* Copy packet header */
- start_dst = (uint8_t *)pkt_hdr_dst + start_offset;
- start_src = (uint8_t *)pkt_hdr_src + start_offset;
- len = ODP_OFFSETOF(odp_packet_hdr_t, payload) - start_offset;
- memcpy(start_dst, start_src, len);
-
- /* Copy frame payload */
- start_dst = (uint8_t *)odp_packet_start(pkt_dst);
- start_src = (uint8_t *)odp_packet_start(pkt_src);
- len = pkt_hdr_src->frame_len;
- memcpy(start_dst, start_src, len);
-
- /* Copy useful things from the buffer header */
- pkt_hdr_dst->buf_hdr.cur_offset = pkt_hdr_src->buf_hdr.cur_offset;
-
- /* Create a copy of the scatter list */
- odp_buffer_copy_scatter(odp_buffer_from_packet(pkt_dst),
- odp_buffer_from_packet(pkt_src));
-
- return 0;
+ (void) pkt_dst;
+ (void) pkt_src;
+ return -1;
}
@@ -393,7 +393,7 @@ int odp_queue_deq_multi(odp_queue_t handle, odp_buffer_t buf[], int num)
ret = queue->s.dequeue_multi(queue, buf_hdr, num);
for (i = 0; i < ret; i++)
- buf[i] = buf_hdr[i]->handle.handle;
+ buf[i] = hdr_to_odp_buf(buf_hdr[i]);
return ret;
}
@@ -408,7 +408,7 @@ odp_buffer_t odp_queue_deq(odp_queue_t handle)
buf_hdr = queue->s.dequeue(queue);
if (buf_hdr)
- return buf_hdr->handle.handle;
+ return hdr_to_odp_buf(buf_hdr);
return ODP_BUFFER_INVALID;
}
Buffer pools are managed as a hw queue filled with free buffers. Allocating buffer -> dequeue from pool's free queue. Freeing buffer -> enqueue to pool's free queue. This approach has important limitation - all free buffers are linked to cppi descriptor, so max number of free buffers is limited to max number of descriptors. If fast internal linking ram is used, then max number of desctiptors is 16k. Applications may create pools with much more buffers. As a temporary solution max pool size is limited to 2000 buffers, but buffer management should be made more flexible. Signed-off-by: Taras Kondratiuk <taras.kondratiuk@linaro.org> --- platform/linux-keystone2/include/api/odp_buffer.h | 3 +- .../include/configs/odp_config_platform.h | 1 - .../linux-keystone2/include/odp_buffer_internal.h | 81 ++-- .../include/odp_buffer_pool_internal.h | 47 +- platform/linux-keystone2/source/odp_buffer.c | 66 +-- platform/linux-keystone2/source/odp_buffer_pool.c | 486 ++++++-------------- platform/linux-keystone2/source/odp_init.c | 4 +- platform/linux-keystone2/source/odp_packet.c | 37 +- platform/linux-keystone2/source/odp_queue.c | 4 +- 9 files changed, 209 insertions(+), 520 deletions(-)