| Message ID | 20170713125828.28230-1-petri.savolainen@linaro.org |
|---|---|
| State | Accepted |
| Commit | dcea50f7ad1f5473d74797e58eac5ba4abb939d1 |
| Headers | show |
| Series | [v2,1/3] linux-gen: buffer: change buffer handle to pointer | expand |
Merged with review from v1. Maxim. On 07/13/17 15:58, Petri Savolainen wrote: > Changed buffer, event and timeout handles to be pointers. > Packet handles are already pointers. This enabled code > optimization as some conversions may be removed and remaining > once are just type casts. > > Signed-off-by: Petri Savolainen <petri.savolainen@linaro.org> > --- > include/odp/arch/default/api/abi/buffer.h | 7 +- > include/odp/arch/default/api/abi/event.h | 2 +- > .../include/odp/api/plat/buffer_types.h | 6 +- > .../include/odp/api/plat/event_types.h | 2 +- > .../include/odp/api/plat/timer_types.h | 2 +- > .../linux-generic/include/odp_buffer_inlines.h | 9 +- > .../linux-generic/include/odp_buffer_internal.h | 23 ++-- > .../linux-generic/include/odp_packet_internal.h | 5 +- > platform/linux-generic/include/odp_pool_internal.h | 30 +----- > platform/linux-generic/odp_buffer.c | 4 +- > platform/linux-generic/odp_packet.c | 56 +++++----- > platform/linux-generic/odp_pool.c | 120 +++++++++++---------- > platform/linux-generic/odp_queue.c | 6 +- > platform/linux-generic/odp_traffic_mngr.c | 2 +- > platform/linux-generic/pktio/ipc.c | 11 +- > 15 files changed, 126 insertions(+), 159 deletions(-) > > diff --git a/include/odp/arch/default/api/abi/buffer.h b/include/odp/arch/default/api/abi/buffer.h > index eec6f01f..d8bfc913 100644 > --- a/include/odp/arch/default/api/abi/buffer.h > +++ b/include/odp/arch/default/api/abi/buffer.h > @@ -14,18 +14,13 @@ extern "C" { > /** @internal Dummy type for strong typing */ > typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_buffer_t; > > -/** @internal Dummy type for strong typing */ > -typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_buffer_seg_t; > - > /** @ingroup odp_buffer > * @{ > */ > > typedef _odp_abi_buffer_t *odp_buffer_t; > -typedef _odp_abi_buffer_seg_t *odp_buffer_seg_t; > > -#define ODP_BUFFER_INVALID ((odp_buffer_t)0xffffffff) > -#define ODP_SEGMENT_INVALID ((odp_buffer_seg_t)0xffffffff) > +#define ODP_BUFFER_INVALID ((odp_buffer_t)NULL) > > /** > * @} > diff --git a/include/odp/arch/default/api/abi/event.h b/include/odp/arch/default/api/abi/event.h > index 4f6596b1..fd86f25c 100644 > --- a/include/odp/arch/default/api/abi/event.h > +++ b/include/odp/arch/default/api/abi/event.h > @@ -22,7 +22,7 @@ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_event_t; > > typedef _odp_abi_event_t *odp_event_t; > > -#define ODP_EVENT_INVALID ((odp_event_t)0xffffffff) > +#define ODP_EVENT_INVALID ((odp_event_t)NULL) > > typedef enum odp_event_type_t { > ODP_EVENT_BUFFER = 1, > diff --git a/platform/linux-generic/include/odp/api/plat/buffer_types.h b/platform/linux-generic/include/odp/api/plat/buffer_types.h > index 809768f3..8b79bb52 100644 > --- a/platform/linux-generic/include/odp/api/plat/buffer_types.h > +++ b/platform/linux-generic/include/odp/api/plat/buffer_types.h > @@ -31,11 +31,7 @@ extern "C" { > > typedef ODP_HANDLE_T(odp_buffer_t); > > -#define ODP_BUFFER_INVALID _odp_cast_scalar(odp_buffer_t, 0xffffffff) > - > -typedef ODP_HANDLE_T(odp_buffer_seg_t); > - > -#define ODP_SEGMENT_INVALID ((odp_buffer_seg_t)ODP_BUFFER_INVALID) > +#define ODP_BUFFER_INVALID _odp_cast_scalar(odp_buffer_t, NULL) > > /** > * @} > diff --git a/platform/linux-generic/include/odp/api/plat/event_types.h b/platform/linux-generic/include/odp/api/plat/event_types.h > index a1aa0e45..559a2fa0 100644 > --- a/platform/linux-generic/include/odp/api/plat/event_types.h > +++ b/platform/linux-generic/include/odp/api/plat/event_types.h > @@ -32,7 +32,7 @@ extern "C" { > > typedef ODP_HANDLE_T(odp_event_t); > > -#define ODP_EVENT_INVALID _odp_cast_scalar(odp_event_t, 0xffffffff) > +#define ODP_EVENT_INVALID _odp_cast_scalar(odp_event_t, NULL) > > typedef enum odp_event_type_t { > ODP_EVENT_BUFFER = 1, > diff --git a/platform/linux-generic/include/odp/api/plat/timer_types.h b/platform/linux-generic/include/odp/api/plat/timer_types.h > index 8821bed6..a8891f11 100644 > --- a/platform/linux-generic/include/odp/api/plat/timer_types.h > +++ b/platform/linux-generic/include/odp/api/plat/timer_types.h > @@ -38,7 +38,7 @@ typedef ODP_HANDLE_T(odp_timer_t); > > typedef ODP_HANDLE_T(odp_timeout_t); > > -#define ODP_TIMEOUT_INVALID _odp_cast_scalar(odp_timeout_t, 0xffffffff) > +#define ODP_TIMEOUT_INVALID _odp_cast_scalar(odp_timeout_t, NULL) > > /** > * @} > diff --git a/platform/linux-generic/include/odp_buffer_inlines.h b/platform/linux-generic/include/odp_buffer_inlines.h > index cf817d90..1dbc725b 100644 > --- a/platform/linux-generic/include/odp_buffer_inlines.h > +++ b/platform/linux-generic/include/odp_buffer_inlines.h > @@ -23,9 +23,14 @@ odp_event_type_t _odp_buffer_event_type(odp_buffer_t buf); > void _odp_buffer_event_type_set(odp_buffer_t buf, int ev); > int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf); > > -static inline odp_buffer_t odp_hdr_to_buf(odp_buffer_hdr_t *hdr) > +static inline odp_buffer_t buf_from_buf_hdr(odp_buffer_hdr_t *hdr) > { > - return hdr->handle.handle; > + return (odp_buffer_t)hdr; > +} > + > +static inline odp_event_t event_from_buf_hdr(odp_buffer_hdr_t *hdr) > +{ > + return (odp_event_t)hdr; > } > > #ifdef __cplusplus > diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h > index 076abe96..1e8a6486 100644 > --- a/platform/linux-generic/include/odp_buffer_internal.h > +++ b/platform/linux-generic/include/odp_buffer_internal.h > @@ -33,25 +33,12 @@ extern "C" { > #include <odp_schedule_if.h> > #include <stddef.h> > > -typedef union odp_buffer_bits_t { > - odp_buffer_t handle; > - > - union { > - uint32_t u32; > - > - struct { > - uint32_t pool_id: 8; > - uint32_t index: 24; > - }; > - }; > -} odp_buffer_bits_t; > - > #define BUFFER_BURST_SIZE CONFIG_BURST_SIZE > > /* Common buffer header */ > struct odp_buffer_hdr_t { > - /* Handle union */ > - odp_buffer_bits_t handle; > + /* Buffer index in the pool */ > + uint32_t index; > > /* Initial buffer data pointer and length */ > uint8_t *base_data; > @@ -104,9 +91,13 @@ struct odp_buffer_hdr_t { > * offset has to be used */ > uint64_t ipc_data_offset; > > - /* Pool handle */ > + /* Pool handle: will be removed, used only for odp_packet_pool() > + * inlining */ > odp_pool_t pool_hdl; > > + /* Pool pointer */ > + void *pool_ptr; > + > /* Data or next header */ > uint8_t data[0]; > }; > diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h > index d513f05e..d38f831f 100644 > --- a/platform/linux-generic/include/odp_packet_internal.h > +++ b/platform/linux-generic/include/odp_packet_internal.h > @@ -208,11 +208,8 @@ int packet_parse_layer(odp_packet_hdr_t *pkt_hdr, > /* Reset parser metadata for a new parse */ > void packet_parse_reset(odp_packet_hdr_t *pkt_hdr); > > -/* Convert a packet handle to a buffer handle */ > -odp_buffer_t _odp_packet_to_buffer(odp_packet_t pkt); > - > /* Convert a buffer handle to a packet handle */ > -odp_packet_t _odp_packet_from_buffer(odp_buffer_t buf); > +odp_packet_t _odp_packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr); > > static inline int packet_hdr_has_l2(odp_packet_hdr_t *pkt_hdr) > { > diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h > index ebb779da..1fb55d7d 100644 > --- a/platform/linux-generic/include/odp_pool_internal.h > +++ b/platform/linux-generic/include/odp_pool_internal.h > @@ -28,8 +28,7 @@ extern "C" { > > typedef struct pool_cache_t { > uint32_t num; > - > - odp_buffer_t buf[CONFIG_POOL_CACHE_SIZE]; > + uint32_t buf_index[CONFIG_POOL_CACHE_SIZE]; > > } pool_cache_t ODP_ALIGNED_CACHE; > > @@ -92,34 +91,9 @@ static inline pool_t *pool_entry_from_hdl(odp_pool_t pool_hdl) > return &pool_tbl->pool[_odp_typeval(pool_hdl)]; > } > > -static inline odp_buffer_hdr_t *pool_buf_hdl_to_hdr(pool_t *pool, > - odp_buffer_t buf) > -{ > - odp_buffer_bits_t handle; > - uint32_t index, block_offset; > - odp_buffer_hdr_t *buf_hdr; > - > - handle.handle = buf; > - index = handle.index; > - block_offset = index * pool->block_size; > - > - /* clang requires cast to uintptr_t */ > - buf_hdr = (odp_buffer_hdr_t *)(uintptr_t)&pool->base_addr[block_offset]; > - > - return buf_hdr; > -} > - > static inline odp_buffer_hdr_t *buf_hdl_to_hdr(odp_buffer_t buf) > { > - odp_buffer_bits_t handle; > - uint32_t pool_id; > - pool_t *pool; > - > - handle.handle = buf; > - pool_id = handle.pool_id; > - pool = pool_entry(pool_id); > - > - return pool_buf_hdl_to_hdr(pool, buf); > + return (odp_buffer_hdr_t *)(uintptr_t)buf; > } > > int buffer_alloc_multi(pool_t *pool, odp_buffer_t buf[], > diff --git a/platform/linux-generic/odp_buffer.c b/platform/linux-generic/odp_buffer.c > index 88c8140b..9c7dc1f5 100644 > --- a/platform/linux-generic/odp_buffer.c > +++ b/platform/linux-generic/odp_buffer.c > @@ -41,6 +41,7 @@ uint32_t odp_buffer_size(odp_buffer_t buf) > int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf) > { > odp_buffer_hdr_t *hdr; > + pool_t *pool; > int len = 0; > > if (!odp_buffer_is_valid(buf)) { > @@ -49,12 +50,13 @@ int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf) > } > > hdr = buf_hdl_to_hdr(buf); > + pool = hdr->pool_ptr; > > len += snprintf(&str[len], n-len, > "Buffer\n"); > len += snprintf(&str[len], n-len, > " pool %" PRIu64 "\n", > - odp_pool_to_u64(hdr->pool_hdl)); > + odp_pool_to_u64(pool->pool_hdl)); > len += snprintf(&str[len], n-len, > " addr %p\n", hdr->seg[0].data); > len += snprintf(&str[len], n-len, > diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c > index eb66af2d..e6de2558 100644 > --- a/platform/linux-generic/odp_packet.c > +++ b/platform/linux-generic/odp_packet.c > @@ -54,7 +54,7 @@ static inline odp_packet_hdr_t *packet_hdr(odp_packet_t pkt) > > static inline odp_buffer_t buffer_handle(odp_packet_hdr_t *pkt_hdr) > { > - return pkt_hdr->buf_hdr.handle.handle; > + return (odp_buffer_t)pkt_hdr; > } > > static inline odp_packet_hdr_t *buf_to_packet_hdr(odp_buffer_t buf) > @@ -62,6 +62,16 @@ static inline odp_packet_hdr_t *buf_to_packet_hdr(odp_buffer_t buf) > return (odp_packet_hdr_t *)buf_hdl_to_hdr(buf); > } > > +odp_packet_t _odp_packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr) > +{ > + return (odp_packet_t)buf_hdr; > +} > + > +static inline odp_buffer_t packet_to_buffer(odp_packet_t pkt) > +{ > + return (odp_buffer_t)pkt; > +} > + > static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, > uint32_t seg_idx) > { > @@ -617,7 +627,7 @@ void odp_packet_free_multi(const odp_packet_t pkt[], int num) > int odp_packet_reset(odp_packet_t pkt, uint32_t len) > { > odp_packet_hdr_t *const pkt_hdr = packet_hdr(pkt); > - pool_t *pool = pool_entry_from_hdl(pkt_hdr->buf_hdr.pool_hdl); > + pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; > > if (len > pool->headroom + pool->data_size + pool->tailroom) > return -1; > @@ -627,22 +637,6 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len) > return 0; > } > > -odp_packet_t _odp_packet_from_buffer(odp_buffer_t buf) > -{ > - if (odp_unlikely(buf == ODP_BUFFER_INVALID)) > - return ODP_PACKET_INVALID; > - > - return (odp_packet_t)buf_to_packet_hdr(buf); > -} > - > -odp_buffer_t _odp_packet_to_buffer(odp_packet_t pkt) > -{ > - if (odp_unlikely(pkt == ODP_PACKET_INVALID)) > - return ODP_BUFFER_INVALID; > - > - return buffer_handle(packet_hdr(pkt)); > -} > - > odp_packet_t odp_packet_from_event(odp_event_t ev) > { > if (odp_unlikely(ev == ODP_EVENT_INVALID)) > @@ -874,7 +868,7 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, > int ret = 0; > > if (len > headroom) { > - pool_t *pool = pool_entry_from_hdl(pkt_hdr->buf_hdr.pool_hdl); > + pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; > int num; > int segs; > > @@ -1040,7 +1034,7 @@ int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, > int ret = 0; > > if (len > tailroom) { > - pool_t *pool = pool_entry_from_hdl(pkt_hdr->buf_hdr.pool_hdl); > + pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; > int num; > int segs; > > @@ -1337,12 +1331,13 @@ int odp_packet_add_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) > odp_packet_t pkt = *pkt_ptr; > odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); > uint32_t pktlen = pkt_hdr->frame_len; > + pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; > odp_packet_t newpkt; > > if (offset > pktlen) > return -1; > > - newpkt = odp_packet_alloc(pkt_hdr->buf_hdr.pool_hdl, pktlen + len); > + newpkt = odp_packet_alloc(pool->pool_hdl, pktlen + len); > > if (newpkt == ODP_PACKET_INVALID) > return -1; > @@ -1366,12 +1361,13 @@ int odp_packet_rem_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) > odp_packet_t pkt = *pkt_ptr; > odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); > uint32_t pktlen = pkt_hdr->frame_len; > + pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; > odp_packet_t newpkt; > > if (offset > pktlen || offset + len > pktlen) > return -1; > > - newpkt = odp_packet_alloc(pkt_hdr->buf_hdr.pool_hdl, pktlen - len); > + newpkt = odp_packet_alloc(pool->pool_hdl, pktlen - len); > > if (newpkt == ODP_PACKET_INVALID) > return -1; > @@ -1436,12 +1432,12 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) > { > odp_packet_hdr_t *dst_hdr = packet_hdr(*dst); > odp_packet_hdr_t *src_hdr = packet_hdr(src); > - int dst_segs = dst_hdr->buf_hdr.segcount; > - int src_segs = src_hdr->buf_hdr.segcount; > - odp_pool_t dst_pool = dst_hdr->buf_hdr.pool_hdl; > - odp_pool_t src_pool = src_hdr->buf_hdr.pool_hdl; > - uint32_t dst_len = dst_hdr->frame_len; > - uint32_t src_len = src_hdr->frame_len; > + int dst_segs = dst_hdr->buf_hdr.segcount; > + int src_segs = src_hdr->buf_hdr.segcount; > + pool_t *dst_pool = dst_hdr->buf_hdr.pool_ptr; > + pool_t *src_pool = src_hdr->buf_hdr.pool_ptr; > + uint32_t dst_len = dst_hdr->frame_len; > + uint32_t src_len = src_hdr->frame_len; > > /* Do a copy if resulting packet would be out of segments or packets > * are from different pools. */ > @@ -1703,7 +1699,7 @@ void odp_packet_print(odp_packet_t pkt) > int len = 0; > int n = max_len - 1; > odp_packet_hdr_t *hdr = packet_hdr(pkt); > - odp_buffer_t buf = _odp_packet_to_buffer(pkt); > + odp_buffer_t buf = packet_to_buffer(pkt); > > len += snprintf(&str[len], n - len, "Packet "); > len += odp_buffer_snprint(&str[len], n - len, buf); > @@ -1751,7 +1747,7 @@ void odp_packet_print(odp_packet_t pkt) > > int odp_packet_is_valid(odp_packet_t pkt) > { > - if (odp_buffer_is_valid(_odp_packet_to_buffer(pkt)) == 0) > + if (odp_buffer_is_valid(packet_to_buffer(pkt)) == 0) > return 0; > > if (odp_event_type(odp_packet_to_event(pkt)) != ODP_EVENT_PACKET) > diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c > index 9dba7341..7e0c7606 100644 > --- a/platform/linux-generic/odp_pool.c > +++ b/platform/linux-generic/odp_pool.c > @@ -52,12 +52,32 @@ static inline odp_pool_t pool_index_to_handle(uint32_t pool_idx) > return _odp_cast_scalar(odp_pool_t, pool_idx); > } > > -static inline uint32_t pool_id_from_buf(odp_buffer_t buf) > +static inline pool_t *pool_from_buf(odp_buffer_t buf) > { > - odp_buffer_bits_t handle; > + odp_buffer_hdr_t *buf_hdr = buf_hdl_to_hdr(buf); > > - handle.handle = buf; > - return handle.pool_id; > + return buf_hdr->pool_ptr; > +} > + > +static inline odp_buffer_hdr_t *buf_hdr_from_index(pool_t *pool, > + uint32_t buffer_idx) > +{ > + uint32_t block_offset; > + odp_buffer_hdr_t *buf_hdr; > + > + block_offset = buffer_idx * pool->block_size; > + > + /* clang requires cast to uintptr_t */ > + buf_hdr = (odp_buffer_hdr_t *)(uintptr_t)&pool->base_addr[block_offset]; > + > + return buf_hdr; > +} > + > +static inline uint32_t buf_index_from_hdl(odp_buffer_t buf) > +{ > + odp_buffer_hdr_t *buf_hdr = buf_hdl_to_hdr(buf); > + > + return buf_hdr->index; > } > > int odp_pool_init_global(void) > @@ -141,16 +161,14 @@ static void flush_cache(pool_cache_t *cache, pool_t *pool) > { > ring_t *ring; > uint32_t mask; > - uint32_t cache_num, i, data; > + uint32_t cache_num, i; > > ring = &pool->ring->hdr; > mask = pool->ring_mask; > cache_num = cache->num; > > - for (i = 0; i < cache_num; i++) { > - data = (uint32_t)(uintptr_t)cache->buf[i]; > - ring_enq(ring, mask, data); > - } > + for (i = 0; i < cache_num; i++) > + ring_enq(ring, mask, cache->buf_index[i]); > > cache->num = 0; > } > @@ -202,23 +220,11 @@ static pool_t *reserve_pool(void) > return NULL; > } > > -static odp_buffer_t form_buffer_handle(uint32_t pool_idx, uint32_t buffer_idx) > -{ > - odp_buffer_bits_t bits; > - > - bits.handle = 0; > - bits.pool_id = pool_idx; > - bits.index = buffer_idx; > - > - return bits.handle; > -} > - > static void init_buffers(pool_t *pool) > { > uint32_t i; > odp_buffer_hdr_t *buf_hdr; > odp_packet_hdr_t *pkt_hdr; > - odp_buffer_t buf_hdl; > void *addr; > void *uarea = NULL; > uint8_t *data; > @@ -256,10 +262,12 @@ static void init_buffers(pool_t *pool) > seg_size = pool->headroom + pool->data_size + pool->tailroom; > > /* Initialize buffer metadata */ > + buf_hdr->index = i; > buf_hdr->size = seg_size; > buf_hdr->type = type; > buf_hdr->event_type = type; > buf_hdr->pool_hdl = pool->pool_hdl; > + buf_hdr->pool_ptr = pool; > buf_hdr->uarea_addr = uarea; > /* Show user requested size through API */ > buf_hdr->uarea_size = pool->params.pkt.uarea_size; > @@ -275,11 +283,8 @@ static void init_buffers(pool_t *pool) > buf_hdr->buf_end = &data[offset + pool->data_size + > pool->tailroom]; > > - buf_hdl = form_buffer_handle(pool->pool_idx, i); > - buf_hdr->handle.handle = buf_hdl; > - > - /* Store buffer into the global pool */ > - ring_enq(ring, mask, (uint32_t)(uintptr_t)buf_hdl); > + /* Store buffer index into the global pool */ > + ring_enq(ring, mask, i); > } > } > > @@ -600,7 +605,7 @@ int odp_pool_info(odp_pool_t pool_hdl, odp_pool_info_t *info) > } > > int buffer_alloc_multi(pool_t *pool, odp_buffer_t buf[], > - odp_buffer_hdr_t *buf_hdr[], int max_num) > + odp_buffer_hdr_t *buf_hdr_out[], int max_num) > { > ring_t *ring; > uint32_t mask, i; > @@ -626,10 +631,13 @@ int buffer_alloc_multi(pool_t *pool, odp_buffer_t buf[], > > /* Get buffers from the cache */ > for (i = 0; i < num_ch; i++) { > - buf[i] = cache->buf[cache_num - num_ch + i]; > + uint32_t j = cache_num - num_ch + i; > + > + hdr = buf_hdr_from_index(pool, cache->buf_index[j]); > + buf[i] = buf_from_buf_hdr(hdr); > > - if (odp_likely(buf_hdr != NULL)) > - buf_hdr[i] = pool_buf_hdl_to_hdr(pool, buf[i]); > + if (odp_likely(buf_hdr_out != NULL)) > + buf_hdr_out[i] = hdr; > } > > /* If needed, get more from the global pool */ > @@ -651,18 +659,18 @@ int buffer_alloc_multi(pool_t *pool, odp_buffer_t buf[], > for (i = 0; i < num_deq; i++) { > uint32_t idx = num_ch + i; > > - buf[idx] = (odp_buffer_t)(uintptr_t)data[i]; > - hdr = pool_buf_hdl_to_hdr(pool, buf[idx]); > + hdr = buf_hdr_from_index(pool, data[i]); > odp_prefetch(hdr); > > - if (odp_likely(buf_hdr != NULL)) > - buf_hdr[idx] = hdr; > + buf[idx] = buf_from_buf_hdr(hdr); > + > + if (odp_likely(buf_hdr_out != NULL)) > + buf_hdr_out[idx] = hdr; > } > > /* Cache extra buffers. Cache is currently empty. */ > for (i = 0; i < cache_num; i++) > - cache->buf[i] = (odp_buffer_t) > - (uintptr_t)data[num_deq + i]; > + cache->buf_index[i] = data[num_deq + i]; > > cache->num = cache_num; > } else { > @@ -672,26 +680,28 @@ int buffer_alloc_multi(pool_t *pool, odp_buffer_t buf[], > return num_ch + num_deq; > } > > -static inline void buffer_free_to_pool(uint32_t pool_id, > +static inline void buffer_free_to_pool(pool_t *pool, > const odp_buffer_t buf[], int num) > { > - pool_t *pool; > int i; > ring_t *ring; > uint32_t mask; > pool_cache_t *cache; > uint32_t cache_num; > > - cache = local.cache[pool_id]; > - pool = pool_entry(pool_id); > + cache = local.cache[pool->pool_idx]; > > /* Special case of a very large free. Move directly to > * the global pool. */ > if (odp_unlikely(num > CONFIG_POOL_CACHE_SIZE)) { > + uint32_t buf_index[num]; > + > ring = &pool->ring->hdr; > mask = pool->ring_mask; > for (i = 0; i < num; i++) > - ring_enq(ring, mask, (uint32_t)(uintptr_t)buf[i]); > + buf_index[i] = buf_index_from_hdl(buf[i]); > + > + ring_enq_multi(ring, mask, buf_index, num); > > return; > } > @@ -718,8 +728,7 @@ static inline void buffer_free_to_pool(uint32_t pool_id, > index = cache_num - burst; > > for (i = 0; i < burst; i++) > - data[i] = (uint32_t) > - (uintptr_t)cache->buf[index + i]; > + data[i] = cache->buf_index[index + i]; > > ring_enq_multi(ring, mask, data, burst); > } > @@ -728,14 +737,14 @@ static inline void buffer_free_to_pool(uint32_t pool_id, > } > > for (i = 0; i < num; i++) > - cache->buf[cache_num + i] = buf[i]; > + cache->buf_index[cache_num + i] = buf_index_from_hdl(buf[i]); > > cache->num = cache_num + num; > } > > void buffer_free_multi(const odp_buffer_t buf[], int num_total) > { > - uint32_t pool_id; > + pool_t *pool; > int num; > int i; > int first = 0; > @@ -743,18 +752,19 @@ void buffer_free_multi(const odp_buffer_t buf[], int num_total) > while (1) { > num = 1; > i = 1; > - pool_id = pool_id_from_buf(buf[first]); > + > + pool = pool_from_buf(buf[first]); > > /* 'num' buffers are from the same pool */ > if (num_total > 1) { > for (i = first; i < num_total; i++) > - if (pool_id != pool_id_from_buf(buf[i])) > + if (pool != pool_from_buf(buf[i])) > break; > > num = i - first; > } > > - buffer_free_to_pool(pool_id, &buf[first], num); > + buffer_free_to_pool(pool, &buf[first], num); > > if (i == num_total) > return; > @@ -871,9 +881,9 @@ void odp_pool_print(odp_pool_t pool_hdl) > > odp_pool_t odp_buffer_pool(odp_buffer_t buf) > { > - uint32_t pool_id = pool_id_from_buf(buf); > + pool_t *pool = pool_from_buf(buf); > > - return pool_index_to_handle(pool_id); > + return pool->pool_hdl; > } > > void odp_pool_param_init(odp_pool_param_t *params) > @@ -901,15 +911,15 @@ void seg_free_tail(odp_buffer_hdr_t *buf_hdr, int segcount) > > int odp_buffer_is_valid(odp_buffer_t buf) > { > - odp_buffer_bits_t handle; > pool_t *pool; > > - handle.handle = buf; > - > - if (handle.pool_id >= ODP_CONFIG_POOLS) > + if (buf == ODP_BUFFER_INVALID) > return 0; > > - pool = pool_entry(handle.pool_id); > + pool = pool_from_buf(buf); > + > + if (pool->pool_idx >= ODP_CONFIG_POOLS) > + return 0; > > if (pool->reserved == 0) > return 0; > diff --git a/platform/linux-generic/odp_queue.c b/platform/linux-generic/odp_queue.c > index d52814b2..348d609a 100644 > --- a/platform/linux-generic/odp_queue.c > +++ b/platform/linux-generic/odp_queue.c > @@ -559,7 +559,7 @@ static int queue_deq_multi(odp_queue_t handle, odp_event_t events[], int num) > ret = queue->s.dequeue_multi(qentry_to_int(queue), buf_hdr, num); > > for (i = 0; i < ret; i++) > - events[i] = odp_buffer_to_event(buf_hdr[i]->handle.handle); > + events[i] = event_from_buf_hdr(buf_hdr[i]); > > return ret; > } > @@ -574,7 +574,7 @@ static odp_event_t queue_deq(odp_queue_t handle) > buf_hdr = queue->s.dequeue(qentry_to_int(queue)); > > if (buf_hdr) > - return odp_buffer_to_event(buf_hdr->handle.handle); > + return event_from_buf_hdr(buf_hdr); > > return ODP_EVENT_INVALID; > } > @@ -686,7 +686,7 @@ int sched_cb_queue_deq_multi(uint32_t queue_index, odp_event_t ev[], int num) > > if (ret > 0) > for (i = 0; i < ret; i++) > - ev[i] = odp_buffer_to_event(buf_hdr[i]->handle.handle); > + ev[i] = event_from_buf_hdr(buf_hdr[i]); > > return ret; > } > diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c > index ec2d331f..53e10471 100644 > --- a/platform/linux-generic/odp_traffic_mngr.c > +++ b/platform/linux-generic/odp_traffic_mngr.c > @@ -104,7 +104,7 @@ static int queue_tm_reenq(queue_t queue, odp_buffer_hdr_t *buf_hdr) > odp_tm_queue_t tm_queue = MAKE_ODP_TM_QUEUE((uint8_t *)queue - > offsetof(tm_queue_obj_t, > tm_qentry)); > - odp_packet_t pkt = _odp_packet_from_buffer(buf_hdr->handle.handle); > + odp_packet_t pkt = _odp_packet_from_buf_hdr(buf_hdr); > > return odp_tm_enq(tm_queue, pkt); > } > diff --git a/platform/linux-generic/pktio/ipc.c b/platform/linux-generic/pktio/ipc.c > index f0facd8c..acdfc4ff 100644 > --- a/platform/linux-generic/pktio/ipc.c > +++ b/platform/linux-generic/pktio/ipc.c > @@ -580,12 +580,13 @@ static int ipc_pktio_send_lockless(pktio_entry_t *pktio_entry, > for (i = 0; i < len; i++) { > odp_packet_t pkt = pkt_table[i]; > pool_t *ipc_pool = pool_entry_from_hdl(pktio_entry->s.ipc.pool); > - odp_buffer_bits_t handle; > - uint32_t pkt_pool_id; > + odp_packet_hdr_t *pkt_hdr; > + pool_t *pool; > > - handle.handle = _odp_packet_to_buffer(pkt); > - pkt_pool_id = handle.pool_id; > - if (pkt_pool_id != ipc_pool->pool_idx) { > + pkt_hdr = odp_packet_hdr(pkt); > + pool = pkt_hdr->buf_hdr.pool_ptr; > + > + if (pool->pool_idx != ipc_pool->pool_idx) { > odp_packet_t newpkt; > > newpkt = odp_packet_copy(pkt, pktio_entry->s.ipc.pool); >
diff --git a/include/odp/arch/default/api/abi/buffer.h b/include/odp/arch/default/api/abi/buffer.h index eec6f01f..d8bfc913 100644 --- a/include/odp/arch/default/api/abi/buffer.h +++ b/include/odp/arch/default/api/abi/buffer.h @@ -14,18 +14,13 @@ extern "C" { /** @internal Dummy type for strong typing */ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_buffer_t; -/** @internal Dummy type for strong typing */ -typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_buffer_seg_t; - /** @ingroup odp_buffer * @{ */ typedef _odp_abi_buffer_t *odp_buffer_t; -typedef _odp_abi_buffer_seg_t *odp_buffer_seg_t; -#define ODP_BUFFER_INVALID ((odp_buffer_t)0xffffffff) -#define ODP_SEGMENT_INVALID ((odp_buffer_seg_t)0xffffffff) +#define ODP_BUFFER_INVALID ((odp_buffer_t)NULL) /** * @} diff --git a/include/odp/arch/default/api/abi/event.h b/include/odp/arch/default/api/abi/event.h index 4f6596b1..fd86f25c 100644 --- a/include/odp/arch/default/api/abi/event.h +++ b/include/odp/arch/default/api/abi/event.h @@ -22,7 +22,7 @@ typedef struct { char dummy; /**< @internal Dummy */ } _odp_abi_event_t; typedef _odp_abi_event_t *odp_event_t; -#define ODP_EVENT_INVALID ((odp_event_t)0xffffffff) +#define ODP_EVENT_INVALID ((odp_event_t)NULL) typedef enum odp_event_type_t { ODP_EVENT_BUFFER = 1, diff --git a/platform/linux-generic/include/odp/api/plat/buffer_types.h b/platform/linux-generic/include/odp/api/plat/buffer_types.h index 809768f3..8b79bb52 100644 --- a/platform/linux-generic/include/odp/api/plat/buffer_types.h +++ b/platform/linux-generic/include/odp/api/plat/buffer_types.h @@ -31,11 +31,7 @@ extern "C" { typedef ODP_HANDLE_T(odp_buffer_t); -#define ODP_BUFFER_INVALID _odp_cast_scalar(odp_buffer_t, 0xffffffff) - -typedef ODP_HANDLE_T(odp_buffer_seg_t); - -#define ODP_SEGMENT_INVALID ((odp_buffer_seg_t)ODP_BUFFER_INVALID) +#define ODP_BUFFER_INVALID _odp_cast_scalar(odp_buffer_t, NULL) /** * @} diff --git a/platform/linux-generic/include/odp/api/plat/event_types.h b/platform/linux-generic/include/odp/api/plat/event_types.h index a1aa0e45..559a2fa0 100644 --- a/platform/linux-generic/include/odp/api/plat/event_types.h +++ b/platform/linux-generic/include/odp/api/plat/event_types.h @@ -32,7 +32,7 @@ extern "C" { typedef ODP_HANDLE_T(odp_event_t); -#define ODP_EVENT_INVALID _odp_cast_scalar(odp_event_t, 0xffffffff) +#define ODP_EVENT_INVALID _odp_cast_scalar(odp_event_t, NULL) typedef enum odp_event_type_t { ODP_EVENT_BUFFER = 1, diff --git a/platform/linux-generic/include/odp/api/plat/timer_types.h b/platform/linux-generic/include/odp/api/plat/timer_types.h index 8821bed6..a8891f11 100644 --- a/platform/linux-generic/include/odp/api/plat/timer_types.h +++ b/platform/linux-generic/include/odp/api/plat/timer_types.h @@ -38,7 +38,7 @@ typedef ODP_HANDLE_T(odp_timer_t); typedef ODP_HANDLE_T(odp_timeout_t); -#define ODP_TIMEOUT_INVALID _odp_cast_scalar(odp_timeout_t, 0xffffffff) +#define ODP_TIMEOUT_INVALID _odp_cast_scalar(odp_timeout_t, NULL) /** * @} diff --git a/platform/linux-generic/include/odp_buffer_inlines.h b/platform/linux-generic/include/odp_buffer_inlines.h index cf817d90..1dbc725b 100644 --- a/platform/linux-generic/include/odp_buffer_inlines.h +++ b/platform/linux-generic/include/odp_buffer_inlines.h @@ -23,9 +23,14 @@ odp_event_type_t _odp_buffer_event_type(odp_buffer_t buf); void _odp_buffer_event_type_set(odp_buffer_t buf, int ev); int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf); -static inline odp_buffer_t odp_hdr_to_buf(odp_buffer_hdr_t *hdr) +static inline odp_buffer_t buf_from_buf_hdr(odp_buffer_hdr_t *hdr) { - return hdr->handle.handle; + return (odp_buffer_t)hdr; +} + +static inline odp_event_t event_from_buf_hdr(odp_buffer_hdr_t *hdr) +{ + return (odp_event_t)hdr; } #ifdef __cplusplus diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index 076abe96..1e8a6486 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -33,25 +33,12 @@ extern "C" { #include <odp_schedule_if.h> #include <stddef.h> -typedef union odp_buffer_bits_t { - odp_buffer_t handle; - - union { - uint32_t u32; - - struct { - uint32_t pool_id: 8; - uint32_t index: 24; - }; - }; -} odp_buffer_bits_t; - #define BUFFER_BURST_SIZE CONFIG_BURST_SIZE /* Common buffer header */ struct odp_buffer_hdr_t { - /* Handle union */ - odp_buffer_bits_t handle; + /* Buffer index in the pool */ + uint32_t index; /* Initial buffer data pointer and length */ uint8_t *base_data; @@ -104,9 +91,13 @@ struct odp_buffer_hdr_t { * offset has to be used */ uint64_t ipc_data_offset; - /* Pool handle */ + /* Pool handle: will be removed, used only for odp_packet_pool() + * inlining */ odp_pool_t pool_hdl; + /* Pool pointer */ + void *pool_ptr; + /* Data or next header */ uint8_t data[0]; }; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index d513f05e..d38f831f 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -208,11 +208,8 @@ int packet_parse_layer(odp_packet_hdr_t *pkt_hdr, /* Reset parser metadata for a new parse */ void packet_parse_reset(odp_packet_hdr_t *pkt_hdr); -/* Convert a packet handle to a buffer handle */ -odp_buffer_t _odp_packet_to_buffer(odp_packet_t pkt); - /* Convert a buffer handle to a packet handle */ -odp_packet_t _odp_packet_from_buffer(odp_buffer_t buf); +odp_packet_t _odp_packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr); static inline int packet_hdr_has_l2(odp_packet_hdr_t *pkt_hdr) { diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index ebb779da..1fb55d7d 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -28,8 +28,7 @@ extern "C" { typedef struct pool_cache_t { uint32_t num; - - odp_buffer_t buf[CONFIG_POOL_CACHE_SIZE]; + uint32_t buf_index[CONFIG_POOL_CACHE_SIZE]; } pool_cache_t ODP_ALIGNED_CACHE; @@ -92,34 +91,9 @@ static inline pool_t *pool_entry_from_hdl(odp_pool_t pool_hdl) return &pool_tbl->pool[_odp_typeval(pool_hdl)]; } -static inline odp_buffer_hdr_t *pool_buf_hdl_to_hdr(pool_t *pool, - odp_buffer_t buf) -{ - odp_buffer_bits_t handle; - uint32_t index, block_offset; - odp_buffer_hdr_t *buf_hdr; - - handle.handle = buf; - index = handle.index; - block_offset = index * pool->block_size; - - /* clang requires cast to uintptr_t */ - buf_hdr = (odp_buffer_hdr_t *)(uintptr_t)&pool->base_addr[block_offset]; - - return buf_hdr; -} - static inline odp_buffer_hdr_t *buf_hdl_to_hdr(odp_buffer_t buf) { - odp_buffer_bits_t handle; - uint32_t pool_id; - pool_t *pool; - - handle.handle = buf; - pool_id = handle.pool_id; - pool = pool_entry(pool_id); - - return pool_buf_hdl_to_hdr(pool, buf); + return (odp_buffer_hdr_t *)(uintptr_t)buf; } int buffer_alloc_multi(pool_t *pool, odp_buffer_t buf[], diff --git a/platform/linux-generic/odp_buffer.c b/platform/linux-generic/odp_buffer.c index 88c8140b..9c7dc1f5 100644 --- a/platform/linux-generic/odp_buffer.c +++ b/platform/linux-generic/odp_buffer.c @@ -41,6 +41,7 @@ uint32_t odp_buffer_size(odp_buffer_t buf) int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf) { odp_buffer_hdr_t *hdr; + pool_t *pool; int len = 0; if (!odp_buffer_is_valid(buf)) { @@ -49,12 +50,13 @@ int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf) } hdr = buf_hdl_to_hdr(buf); + pool = hdr->pool_ptr; len += snprintf(&str[len], n-len, "Buffer\n"); len += snprintf(&str[len], n-len, " pool %" PRIu64 "\n", - odp_pool_to_u64(hdr->pool_hdl)); + odp_pool_to_u64(pool->pool_hdl)); len += snprintf(&str[len], n-len, " addr %p\n", hdr->seg[0].data); len += snprintf(&str[len], n-len, diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index eb66af2d..e6de2558 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -54,7 +54,7 @@ static inline odp_packet_hdr_t *packet_hdr(odp_packet_t pkt) static inline odp_buffer_t buffer_handle(odp_packet_hdr_t *pkt_hdr) { - return pkt_hdr->buf_hdr.handle.handle; + return (odp_buffer_t)pkt_hdr; } static inline odp_packet_hdr_t *buf_to_packet_hdr(odp_buffer_t buf) @@ -62,6 +62,16 @@ static inline odp_packet_hdr_t *buf_to_packet_hdr(odp_buffer_t buf) return (odp_packet_hdr_t *)buf_hdl_to_hdr(buf); } +odp_packet_t _odp_packet_from_buf_hdr(odp_buffer_hdr_t *buf_hdr) +{ + return (odp_packet_t)buf_hdr; +} + +static inline odp_buffer_t packet_to_buffer(odp_packet_t pkt) +{ + return (odp_buffer_t)pkt; +} + static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) { @@ -617,7 +627,7 @@ void odp_packet_free_multi(const odp_packet_t pkt[], int num) int odp_packet_reset(odp_packet_t pkt, uint32_t len) { odp_packet_hdr_t *const pkt_hdr = packet_hdr(pkt); - pool_t *pool = pool_entry_from_hdl(pkt_hdr->buf_hdr.pool_hdl); + pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; if (len > pool->headroom + pool->data_size + pool->tailroom) return -1; @@ -627,22 +637,6 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len) return 0; } -odp_packet_t _odp_packet_from_buffer(odp_buffer_t buf) -{ - if (odp_unlikely(buf == ODP_BUFFER_INVALID)) - return ODP_PACKET_INVALID; - - return (odp_packet_t)buf_to_packet_hdr(buf); -} - -odp_buffer_t _odp_packet_to_buffer(odp_packet_t pkt) -{ - if (odp_unlikely(pkt == ODP_PACKET_INVALID)) - return ODP_BUFFER_INVALID; - - return buffer_handle(packet_hdr(pkt)); -} - odp_packet_t odp_packet_from_event(odp_event_t ev) { if (odp_unlikely(ev == ODP_EVENT_INVALID)) @@ -874,7 +868,7 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, int ret = 0; if (len > headroom) { - pool_t *pool = pool_entry_from_hdl(pkt_hdr->buf_hdr.pool_hdl); + pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; int num; int segs; @@ -1040,7 +1034,7 @@ int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, int ret = 0; if (len > tailroom) { - pool_t *pool = pool_entry_from_hdl(pkt_hdr->buf_hdr.pool_hdl); + pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; int num; int segs; @@ -1337,12 +1331,13 @@ int odp_packet_add_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) odp_packet_t pkt = *pkt_ptr; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); uint32_t pktlen = pkt_hdr->frame_len; + pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; odp_packet_t newpkt; if (offset > pktlen) return -1; - newpkt = odp_packet_alloc(pkt_hdr->buf_hdr.pool_hdl, pktlen + len); + newpkt = odp_packet_alloc(pool->pool_hdl, pktlen + len); if (newpkt == ODP_PACKET_INVALID) return -1; @@ -1366,12 +1361,13 @@ int odp_packet_rem_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) odp_packet_t pkt = *pkt_ptr; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); uint32_t pktlen = pkt_hdr->frame_len; + pool_t *pool = pkt_hdr->buf_hdr.pool_ptr; odp_packet_t newpkt; if (offset > pktlen || offset + len > pktlen) return -1; - newpkt = odp_packet_alloc(pkt_hdr->buf_hdr.pool_hdl, pktlen - len); + newpkt = odp_packet_alloc(pool->pool_hdl, pktlen - len); if (newpkt == ODP_PACKET_INVALID) return -1; @@ -1436,12 +1432,12 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) { odp_packet_hdr_t *dst_hdr = packet_hdr(*dst); odp_packet_hdr_t *src_hdr = packet_hdr(src); - int dst_segs = dst_hdr->buf_hdr.segcount; - int src_segs = src_hdr->buf_hdr.segcount; - odp_pool_t dst_pool = dst_hdr->buf_hdr.pool_hdl; - odp_pool_t src_pool = src_hdr->buf_hdr.pool_hdl; - uint32_t dst_len = dst_hdr->frame_len; - uint32_t src_len = src_hdr->frame_len; + int dst_segs = dst_hdr->buf_hdr.segcount; + int src_segs = src_hdr->buf_hdr.segcount; + pool_t *dst_pool = dst_hdr->buf_hdr.pool_ptr; + pool_t *src_pool = src_hdr->buf_hdr.pool_ptr; + uint32_t dst_len = dst_hdr->frame_len; + uint32_t src_len = src_hdr->frame_len; /* Do a copy if resulting packet would be out of segments or packets * are from different pools. */ @@ -1703,7 +1699,7 @@ void odp_packet_print(odp_packet_t pkt) int len = 0; int n = max_len - 1; odp_packet_hdr_t *hdr = packet_hdr(pkt); - odp_buffer_t buf = _odp_packet_to_buffer(pkt); + odp_buffer_t buf = packet_to_buffer(pkt); len += snprintf(&str[len], n - len, "Packet "); len += odp_buffer_snprint(&str[len], n - len, buf); @@ -1751,7 +1747,7 @@ void odp_packet_print(odp_packet_t pkt) int odp_packet_is_valid(odp_packet_t pkt) { - if (odp_buffer_is_valid(_odp_packet_to_buffer(pkt)) == 0) + if (odp_buffer_is_valid(packet_to_buffer(pkt)) == 0) return 0; if (odp_event_type(odp_packet_to_event(pkt)) != ODP_EVENT_PACKET) diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 9dba7341..7e0c7606 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -52,12 +52,32 @@ static inline odp_pool_t pool_index_to_handle(uint32_t pool_idx) return _odp_cast_scalar(odp_pool_t, pool_idx); } -static inline uint32_t pool_id_from_buf(odp_buffer_t buf) +static inline pool_t *pool_from_buf(odp_buffer_t buf) { - odp_buffer_bits_t handle; + odp_buffer_hdr_t *buf_hdr = buf_hdl_to_hdr(buf); - handle.handle = buf; - return handle.pool_id; + return buf_hdr->pool_ptr; +} + +static inline odp_buffer_hdr_t *buf_hdr_from_index(pool_t *pool, + uint32_t buffer_idx) +{ + uint32_t block_offset; + odp_buffer_hdr_t *buf_hdr; + + block_offset = buffer_idx * pool->block_size; + + /* clang requires cast to uintptr_t */ + buf_hdr = (odp_buffer_hdr_t *)(uintptr_t)&pool->base_addr[block_offset]; + + return buf_hdr; +} + +static inline uint32_t buf_index_from_hdl(odp_buffer_t buf) +{ + odp_buffer_hdr_t *buf_hdr = buf_hdl_to_hdr(buf); + + return buf_hdr->index; } int odp_pool_init_global(void) @@ -141,16 +161,14 @@ static void flush_cache(pool_cache_t *cache, pool_t *pool) { ring_t *ring; uint32_t mask; - uint32_t cache_num, i, data; + uint32_t cache_num, i; ring = &pool->ring->hdr; mask = pool->ring_mask; cache_num = cache->num; - for (i = 0; i < cache_num; i++) { - data = (uint32_t)(uintptr_t)cache->buf[i]; - ring_enq(ring, mask, data); - } + for (i = 0; i < cache_num; i++) + ring_enq(ring, mask, cache->buf_index[i]); cache->num = 0; } @@ -202,23 +220,11 @@ static pool_t *reserve_pool(void) return NULL; } -static odp_buffer_t form_buffer_handle(uint32_t pool_idx, uint32_t buffer_idx) -{ - odp_buffer_bits_t bits; - - bits.handle = 0; - bits.pool_id = pool_idx; - bits.index = buffer_idx; - - return bits.handle; -} - static void init_buffers(pool_t *pool) { uint32_t i; odp_buffer_hdr_t *buf_hdr; odp_packet_hdr_t *pkt_hdr; - odp_buffer_t buf_hdl; void *addr; void *uarea = NULL; uint8_t *data; @@ -256,10 +262,12 @@ static void init_buffers(pool_t *pool) seg_size = pool->headroom + pool->data_size + pool->tailroom; /* Initialize buffer metadata */ + buf_hdr->index = i; buf_hdr->size = seg_size; buf_hdr->type = type; buf_hdr->event_type = type; buf_hdr->pool_hdl = pool->pool_hdl; + buf_hdr->pool_ptr = pool; buf_hdr->uarea_addr = uarea; /* Show user requested size through API */ buf_hdr->uarea_size = pool->params.pkt.uarea_size; @@ -275,11 +283,8 @@ static void init_buffers(pool_t *pool) buf_hdr->buf_end = &data[offset + pool->data_size + pool->tailroom]; - buf_hdl = form_buffer_handle(pool->pool_idx, i); - buf_hdr->handle.handle = buf_hdl; - - /* Store buffer into the global pool */ - ring_enq(ring, mask, (uint32_t)(uintptr_t)buf_hdl); + /* Store buffer index into the global pool */ + ring_enq(ring, mask, i); } } @@ -600,7 +605,7 @@ int odp_pool_info(odp_pool_t pool_hdl, odp_pool_info_t *info) } int buffer_alloc_multi(pool_t *pool, odp_buffer_t buf[], - odp_buffer_hdr_t *buf_hdr[], int max_num) + odp_buffer_hdr_t *buf_hdr_out[], int max_num) { ring_t *ring; uint32_t mask, i; @@ -626,10 +631,13 @@ int buffer_alloc_multi(pool_t *pool, odp_buffer_t buf[], /* Get buffers from the cache */ for (i = 0; i < num_ch; i++) { - buf[i] = cache->buf[cache_num - num_ch + i]; + uint32_t j = cache_num - num_ch + i; + + hdr = buf_hdr_from_index(pool, cache->buf_index[j]); + buf[i] = buf_from_buf_hdr(hdr); - if (odp_likely(buf_hdr != NULL)) - buf_hdr[i] = pool_buf_hdl_to_hdr(pool, buf[i]); + if (odp_likely(buf_hdr_out != NULL)) + buf_hdr_out[i] = hdr; } /* If needed, get more from the global pool */ @@ -651,18 +659,18 @@ int buffer_alloc_multi(pool_t *pool, odp_buffer_t buf[], for (i = 0; i < num_deq; i++) { uint32_t idx = num_ch + i; - buf[idx] = (odp_buffer_t)(uintptr_t)data[i]; - hdr = pool_buf_hdl_to_hdr(pool, buf[idx]); + hdr = buf_hdr_from_index(pool, data[i]); odp_prefetch(hdr); - if (odp_likely(buf_hdr != NULL)) - buf_hdr[idx] = hdr; + buf[idx] = buf_from_buf_hdr(hdr); + + if (odp_likely(buf_hdr_out != NULL)) + buf_hdr_out[idx] = hdr; } /* Cache extra buffers. Cache is currently empty. */ for (i = 0; i < cache_num; i++) - cache->buf[i] = (odp_buffer_t) - (uintptr_t)data[num_deq + i]; + cache->buf_index[i] = data[num_deq + i]; cache->num = cache_num; } else { @@ -672,26 +680,28 @@ int buffer_alloc_multi(pool_t *pool, odp_buffer_t buf[], return num_ch + num_deq; } -static inline void buffer_free_to_pool(uint32_t pool_id, +static inline void buffer_free_to_pool(pool_t *pool, const odp_buffer_t buf[], int num) { - pool_t *pool; int i; ring_t *ring; uint32_t mask; pool_cache_t *cache; uint32_t cache_num; - cache = local.cache[pool_id]; - pool = pool_entry(pool_id); + cache = local.cache[pool->pool_idx]; /* Special case of a very large free. Move directly to * the global pool. */ if (odp_unlikely(num > CONFIG_POOL_CACHE_SIZE)) { + uint32_t buf_index[num]; + ring = &pool->ring->hdr; mask = pool->ring_mask; for (i = 0; i < num; i++) - ring_enq(ring, mask, (uint32_t)(uintptr_t)buf[i]); + buf_index[i] = buf_index_from_hdl(buf[i]); + + ring_enq_multi(ring, mask, buf_index, num); return; } @@ -718,8 +728,7 @@ static inline void buffer_free_to_pool(uint32_t pool_id, index = cache_num - burst; for (i = 0; i < burst; i++) - data[i] = (uint32_t) - (uintptr_t)cache->buf[index + i]; + data[i] = cache->buf_index[index + i]; ring_enq_multi(ring, mask, data, burst); } @@ -728,14 +737,14 @@ static inline void buffer_free_to_pool(uint32_t pool_id, } for (i = 0; i < num; i++) - cache->buf[cache_num + i] = buf[i]; + cache->buf_index[cache_num + i] = buf_index_from_hdl(buf[i]); cache->num = cache_num + num; } void buffer_free_multi(const odp_buffer_t buf[], int num_total) { - uint32_t pool_id; + pool_t *pool; int num; int i; int first = 0; @@ -743,18 +752,19 @@ void buffer_free_multi(const odp_buffer_t buf[], int num_total) while (1) { num = 1; i = 1; - pool_id = pool_id_from_buf(buf[first]); + + pool = pool_from_buf(buf[first]); /* 'num' buffers are from the same pool */ if (num_total > 1) { for (i = first; i < num_total; i++) - if (pool_id != pool_id_from_buf(buf[i])) + if (pool != pool_from_buf(buf[i])) break; num = i - first; } - buffer_free_to_pool(pool_id, &buf[first], num); + buffer_free_to_pool(pool, &buf[first], num); if (i == num_total) return; @@ -871,9 +881,9 @@ void odp_pool_print(odp_pool_t pool_hdl) odp_pool_t odp_buffer_pool(odp_buffer_t buf) { - uint32_t pool_id = pool_id_from_buf(buf); + pool_t *pool = pool_from_buf(buf); - return pool_index_to_handle(pool_id); + return pool->pool_hdl; } void odp_pool_param_init(odp_pool_param_t *params) @@ -901,15 +911,15 @@ void seg_free_tail(odp_buffer_hdr_t *buf_hdr, int segcount) int odp_buffer_is_valid(odp_buffer_t buf) { - odp_buffer_bits_t handle; pool_t *pool; - handle.handle = buf; - - if (handle.pool_id >= ODP_CONFIG_POOLS) + if (buf == ODP_BUFFER_INVALID) return 0; - pool = pool_entry(handle.pool_id); + pool = pool_from_buf(buf); + + if (pool->pool_idx >= ODP_CONFIG_POOLS) + return 0; if (pool->reserved == 0) return 0; diff --git a/platform/linux-generic/odp_queue.c b/platform/linux-generic/odp_queue.c index d52814b2..348d609a 100644 --- a/platform/linux-generic/odp_queue.c +++ b/platform/linux-generic/odp_queue.c @@ -559,7 +559,7 @@ static int queue_deq_multi(odp_queue_t handle, odp_event_t events[], int num) ret = queue->s.dequeue_multi(qentry_to_int(queue), buf_hdr, num); for (i = 0; i < ret; i++) - events[i] = odp_buffer_to_event(buf_hdr[i]->handle.handle); + events[i] = event_from_buf_hdr(buf_hdr[i]); return ret; } @@ -574,7 +574,7 @@ static odp_event_t queue_deq(odp_queue_t handle) buf_hdr = queue->s.dequeue(qentry_to_int(queue)); if (buf_hdr) - return odp_buffer_to_event(buf_hdr->handle.handle); + return event_from_buf_hdr(buf_hdr); return ODP_EVENT_INVALID; } @@ -686,7 +686,7 @@ int sched_cb_queue_deq_multi(uint32_t queue_index, odp_event_t ev[], int num) if (ret > 0) for (i = 0; i < ret; i++) - ev[i] = odp_buffer_to_event(buf_hdr[i]->handle.handle); + ev[i] = event_from_buf_hdr(buf_hdr[i]); return ret; } diff --git a/platform/linux-generic/odp_traffic_mngr.c b/platform/linux-generic/odp_traffic_mngr.c index ec2d331f..53e10471 100644 --- a/platform/linux-generic/odp_traffic_mngr.c +++ b/platform/linux-generic/odp_traffic_mngr.c @@ -104,7 +104,7 @@ static int queue_tm_reenq(queue_t queue, odp_buffer_hdr_t *buf_hdr) odp_tm_queue_t tm_queue = MAKE_ODP_TM_QUEUE((uint8_t *)queue - offsetof(tm_queue_obj_t, tm_qentry)); - odp_packet_t pkt = _odp_packet_from_buffer(buf_hdr->handle.handle); + odp_packet_t pkt = _odp_packet_from_buf_hdr(buf_hdr); return odp_tm_enq(tm_queue, pkt); } diff --git a/platform/linux-generic/pktio/ipc.c b/platform/linux-generic/pktio/ipc.c index f0facd8c..acdfc4ff 100644 --- a/platform/linux-generic/pktio/ipc.c +++ b/platform/linux-generic/pktio/ipc.c @@ -580,12 +580,13 @@ static int ipc_pktio_send_lockless(pktio_entry_t *pktio_entry, for (i = 0; i < len; i++) { odp_packet_t pkt = pkt_table[i]; pool_t *ipc_pool = pool_entry_from_hdl(pktio_entry->s.ipc.pool); - odp_buffer_bits_t handle; - uint32_t pkt_pool_id; + odp_packet_hdr_t *pkt_hdr; + pool_t *pool; - handle.handle = _odp_packet_to_buffer(pkt); - pkt_pool_id = handle.pool_id; - if (pkt_pool_id != ipc_pool->pool_idx) { + pkt_hdr = odp_packet_hdr(pkt); + pool = pkt_hdr->buf_hdr.pool_ptr; + + if (pool->pool_idx != ipc_pool->pool_idx) { odp_packet_t newpkt; newpkt = odp_packet_copy(pkt, pktio_entry->s.ipc.pool);
Changed buffer, event and timeout handles to be pointers. Packet handles are already pointers. This enabled code optimization as some conversions may be removed and remaining once are just type casts. Signed-off-by: Petri Savolainen <petri.savolainen@linaro.org> --- include/odp/arch/default/api/abi/buffer.h | 7 +- include/odp/arch/default/api/abi/event.h | 2 +- .../include/odp/api/plat/buffer_types.h | 6 +- .../include/odp/api/plat/event_types.h | 2 +- .../include/odp/api/plat/timer_types.h | 2 +- .../linux-generic/include/odp_buffer_inlines.h | 9 +- .../linux-generic/include/odp_buffer_internal.h | 23 ++-- .../linux-generic/include/odp_packet_internal.h | 5 +- platform/linux-generic/include/odp_pool_internal.h | 30 +----- platform/linux-generic/odp_buffer.c | 4 +- platform/linux-generic/odp_packet.c | 56 +++++----- platform/linux-generic/odp_pool.c | 120 +++++++++++---------- platform/linux-generic/odp_queue.c | 6 +- platform/linux-generic/odp_traffic_mngr.c | 2 +- platform/linux-generic/pktio/ipc.c | 11 +- 15 files changed, 126 insertions(+), 159 deletions(-) -- 2.13.0