| Message ID | 1458600381-7288-1-git-send-email-bill.fischofer@linaro.org |
|---|---|
| State | Superseded |
| Headers | show |
The problem with adding multi-segment functionality to push/pull is that the resulting semantics are incomplete since they assume that the operations don't cross segment boundaries. v3 adds the needed seglen output parameter to make the APIs complete under the new definition, however this means that everything has do be done as a single update since the API signatures change. On Mon, Mar 21, 2016 at 5:46 PM, Bill Fischofer <bill.fischofer@linaro.org> wrote: > Add support for multi-segment push/pull operations for Monarch compliance. > If a push for more than the available headroom/tailroom is requested, then > allocate additional head/tail segments if possible to complete the > operation. Similarly, when pulling more than a single segment, allow > empty segments to exist that consist entirely of headroom/tailroom. > > Signed-off-by: Bill Fischofer <bill.fischofer@linaro.org> > --- > example/ipsec/odp_ipsec.c | 4 +- > example/ipsec/odp_ipsec_stream.c | 2 +- > include/odp/api/spec/packet.h | 78 > ++++++++++++---------- > .../linux-generic/include/odp_buffer_internal.h | 4 ++ > .../linux-generic/include/odp_packet_internal.h | 49 ++++++++++++++ > platform/linux-generic/odp_packet.c | 22 +++--- > platform/linux-generic/odp_pool.c | 75 > +++++++++++++++++++++ > test/validation/packet/packet.c | 70 > +++++++++++++------ > 8 files changed, 240 insertions(+), 64 deletions(-) > > diff --git a/example/ipsec/odp_ipsec.c b/example/ipsec/odp_ipsec.c > index 2e93fcd..052c387 100644 > --- a/example/ipsec/odp_ipsec.c > +++ b/example/ipsec/odp_ipsec.c > @@ -763,7 +763,7 @@ pkt_disposition_e do_ipsec_in_finish(odp_packet_t pkt, > > /* We have a tunneled IPv4 packet */ > if (ip->proto == ODPH_IPV4) { > - odp_packet_pull_head(pkt, sizeof(*ip) + hdr_len); > + odp_packet_pull_head(pkt, sizeof(*ip) + hdr_len, NULL); > odp_packet_pull_tail(pkt, trl_len); > odph_ethhdr_t *eth; > > @@ -925,7 +925,7 @@ pkt_disposition_e do_ipsec_out_classify(odp_packet_t > pkt, > > /* Set IPv4 length before authentication */ > ipv4_adjust_len(ip, hdr_len + trl_len); > - if (!odp_packet_push_tail(pkt, hdr_len + trl_len)) > + if (!odp_packet_push_tail(pkt, hdr_len + trl_len, NULL)) > return PKT_DROP; > > /* Save remaining context */ > diff --git a/example/ipsec/odp_ipsec_stream.c > b/example/ipsec/odp_ipsec_stream.c > index 4dc9acf..22ddc32 100644 > --- a/example/ipsec/odp_ipsec_stream.c > +++ b/example/ipsec/odp_ipsec_stream.c > @@ -358,7 +358,7 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t > *stream, > } > > /* Correct set packet length offsets */ > - odp_packet_push_tail(pkt, data - base); > + odp_packet_push_tail(pkt, data - base, NULL); > odp_packet_l2_offset_set(pkt, (uint8_t *)eth - base); > odp_packet_l3_offset_set(pkt, (uint8_t *)ip - base); > odp_packet_l4_offset_set(pkt, ((uint8_t *)ip - base) + > sizeof(*ip)); > diff --git a/include/odp/api/spec/packet.h b/include/odp/api/spec/packet.h > index 9c63b5f..66ef220 100644 > --- a/include/odp/api/spec/packet.h > +++ b/include/odp/api/spec/packet.h > @@ -294,9 +294,13 @@ void *odp_packet_tail(odp_packet_t pkt); > * Push out packet head > * > * Increase packet data length by moving packet head into packet headroom. > - * Packet headroom is decreased with the same amount. The packet head may > be > - * pushed out up to 'headroom' bytes. Packet is not modified if there's > not > - * enough headroom space. > + * Packet headroom is decreased by the same amount. If there is > insufficient > + * headroom available in the current segment the packet MAY be extended > with > + * additional segment(s) to accommodate the push request. Note that such > + * extension may change the segmentation of the packet but does not affect > + * the packet handle. As a result, the entire requested length may not be > + * contiguously addressable from the returned data pointer. Use > + * odp_packet_offset() to obtain this information if needed. > * > * odp_packet_xxx: > * seg_len += len > @@ -304,27 +308,27 @@ void *odp_packet_tail(odp_packet_t pkt); > * headroom -= len > * data -= len > * > - * Operation does not modify packet segmentation or move data. Handles and > - * pointers remain valid. User is responsible to update packet metadata > - * offsets when needed. > + * Following this operation packet handles and pointers remain valid. > User is > + * responsible to update packet metadata offsets when needed. > * > - * @param pkt Packet handle > - * @param len Number of bytes to push the head (0 ... headroom) > + * @param pkt Packet handle > + * @param len Number of bytes to push the head (0 or more) > + * @param[out] seglen Number of bytes contiguously addressable from the > returned > + * data pointer. Ignored when NULL. > * > * @return The new data pointer > * @retval NULL Requested offset exceeds available headroom > * > * @see odp_packet_headroom(), odp_packet_pull_head() > */ > -void *odp_packet_push_head(odp_packet_t pkt, uint32_t len); > +void *odp_packet_push_head(odp_packet_t pkt, uint32_t len, uint32_t > *seglen); > > /** > * Pull in packet head > * > * Decrease packet data length by removing data from the head of the > packet. > - * Packet headroom is increased with the same amount. Packet head may be > pulled > - * in up to seg_len - 1 bytes (i.e. packet data pointer must stay in the > - * first segment). Packet is not modified if there's not enough data. > + * Packet headroom is increased with the same amount. Packet is not > modified > + * if there's not enough data. > * > * odp_packet_xxx: > * seg_len -= len > @@ -332,27 +336,34 @@ void *odp_packet_push_head(odp_packet_t pkt, > uint32_t len); > * headroom += len > * data += len > * > - * Operation does not modify packet segmentation or move data. Handles and > - * pointers remain valid. User is responsible to update packet metadata > - * offsets when needed. > + * Following this operation packet handles and pointers remain valid. > User is > + * responsible to update packet metadata offsets when needed. > * > * @param pkt Packet handle > - * @param len Number of bytes to pull the head (0 ... seg_len - 1) > + * @param len Number of bytes to pull the head (0 ... packet length) > + * @param[out] seglen Number of contiguously addressable bytes available > at > + * the returned data pointer. Ignored when NULL. > * > * @return The new data pointer > - * @retval NULL Requested offset exceeds packet segment length > + * @retval NULL Requested offset exceeds packet length > * > * @see odp_packet_seg_len(), odp_packet_push_head() > */ > -void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len); > +void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len, uint32_t > *seglen); > > /** > * Push out packet tail > * > * Increase packet data length by moving packet tail into packet tailroom. > - * Packet tailroom is decreased with the same amount. The packet tail may > be > - * pushed out up to 'tailroom' bytes. Packet is not modified if there's > not > - * enough tailroom. > + * Packet tailroom is decreased with the same amount. If the requested len > + * exceeds the current tailroom, the packet MAY be extended with > additional > + * segment(s) to accommodate the push request. Note that such extension > may > + * change the segmentation of the packet but does not affect the packet > + * handle. As a result, the entire requested length may not be > contiguously > + * addressable from the returned data pointer. Use odp_packet_offset() to > + * obtain this information if needed. > + * > + * Packet is not modified if there's not enough tailroom. > * > * last_seg: > * data_len += len > @@ -362,26 +373,26 @@ void *odp_packet_pull_head(odp_packet_t pkt, > uint32_t len); > * tail += len > * tailroom -= len > * > - * Operation does not modify packet segmentation or move data. Handles, > - * pointers and offsets remain valid. > + * Following this operation packet handles, pointers and offsets remain > valid. > * > - * @param pkt Packet handle > - * @param len Number of bytes to push the tail (0 ... tailroom) > + * @param pkt Packet handle > + * @param len Number of bytes to push the tail (0 or more) > + * @param[out] seglen Number of contiguously addressable bytes available > at > + * returned data pointer. Ignored when NULL. > * > * @return The old tail pointer > * @retval NULL Requested offset exceeds available tailroom > * > * @see odp_packet_tailroom(), odp_packet_pull_tail() > */ > -void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len); > +void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len, uint32_t > *seglen); > > /** > * Pull in packet tail > * > * Decrease packet data length by removing data from the tail of the > packet. > - * Packet tailroom is increased with the same amount. Packet tail may be > pulled > - * in up to last segment data_len - 1 bytes. (i.e. packet tail must stay > in the > - * last segment). Packet is not modified if there's not enough data. > + * Packet tailroom is increased with the same amount. Packet is not > modified > + * if there's not enough data. > * > * last_seg: > * data_len -= len > @@ -391,15 +402,14 @@ void *odp_packet_push_tail(odp_packet_t pkt, > uint32_t len); > * tail -= len > * tailroom += len > * > - * Operation does not modify packet segmentation or move data. Handles and > - * pointers remain valid. User is responsible to update packet metadata > - * offsets when needed. > + * Following this operation packet handles and pointers remain valid. > User is > + * responsible to update packet metadata offsets when needed. > * > * @param pkt Packet handle > - * @param len Number of bytes to pull the tail (0 ... last_seg:data_len > - 1) > + * @param len Number of bytes to pull the tail (0 ... packet length) > * > * @return The new tail pointer > - * @retval NULL The specified offset exceeds allowable data length > + * @retval NULL The specified offset exceeds packet length > */ > void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len); > > diff --git a/platform/linux-generic/include/odp_buffer_internal.h > b/platform/linux-generic/include/odp_buffer_internal.h > index ea092ca..0a4c290 100644 > --- a/platform/linux-generic/include/odp_buffer_internal.h > +++ b/platform/linux-generic/include/odp_buffer_internal.h > @@ -170,6 +170,10 @@ typedef struct { > odp_buffer_t buffer_alloc(odp_pool_t pool, size_t size); > int buffer_alloc_multi(odp_pool_t pool_hdl, size_t size, > odp_buffer_t buf[], int num); > +int seg_alloc_head(odp_buffer_hdr_t *buf_hdr, int segcount); > +void seg_free_head(odp_buffer_hdr_t *buf_hdr, int segcount); > +int seg_alloc_tail(odp_buffer_hdr_t *buf_hdr, int segcount); > +void seg_free_tail(odp_buffer_hdr_t *buf_hdr, int segcount); > > #ifdef __cplusplus > } > diff --git a/platform/linux-generic/include/odp_packet_internal.h > b/platform/linux-generic/include/odp_packet_internal.h > index b632ece..77e32fe 100644 > --- a/platform/linux-generic/include/odp_packet_internal.h > +++ b/platform/linux-generic/include/odp_packet_internal.h > @@ -210,6 +210,30 @@ static inline void pull_head(odp_packet_hdr_t > *pkt_hdr, size_t len) > pkt_hdr->frame_len -= len; > } > > +static inline int push_head_seg(odp_packet_hdr_t *pkt_hdr, size_t len) > +{ > + uint32_t newsegcount = > + (len - pkt_hdr->headroom + pkt_hdr->buf_hdr.size - 1) / > + pkt_hdr->buf_hdr.size; > + > + if (pkt_hdr->buf_hdr.segcount + newsegcount > ODP_BUFFER_MAX_SEG) > + return -1; > + > + if (seg_alloc_head(&pkt_hdr->buf_hdr, newsegcount)) > + return -1; > + > + pkt_hdr->headroom += newsegcount * pkt_hdr->buf_hdr.size; > + return 0; > +} > + > +static inline void pull_head_seg(odp_packet_hdr_t *pkt_hdr) > +{ > + uint32_t extrasegs = pkt_hdr->headroom / pkt_hdr->buf_hdr.size; > + > + seg_free_head(&pkt_hdr->buf_hdr, extrasegs); > + pkt_hdr->headroom -= extrasegs * pkt_hdr->buf_hdr.size; > +} > + > static inline void push_tail(odp_packet_hdr_t *pkt_hdr, size_t len) > { > pkt_hdr->tailroom -= len; > @@ -223,6 +247,31 @@ static inline void pull_tail(odp_packet_hdr_t > *pkt_hdr, size_t len) > pkt_hdr->frame_len -= len; > } > > +static inline int push_tail_seg(odp_packet_hdr_t *pkt_hdr, size_t len) > +{ > + uint32_t newsegcount = > + (len - pkt_hdr->tailroom + pkt_hdr->buf_hdr.size - 1) / > + pkt_hdr->buf_hdr.size; > + > + if (pkt_hdr->buf_hdr.segcount + newsegcount > ODP_BUFFER_MAX_SEG) > + return -1; > + > + if (seg_alloc_tail(&pkt_hdr->buf_hdr, newsegcount)) > + return -1; > + > + pkt_hdr->tailroom += newsegcount * pkt_hdr->buf_hdr.size; > + return 0; > +} > + > +static inline void pull_tail_seg(odp_packet_hdr_t *pkt_hdr) > +{ > + uint32_t extrasegs = pkt_hdr->tailroom / pkt_hdr->buf_hdr.size; > + > + seg_free_tail(&pkt_hdr->buf_hdr, extrasegs); > + > + pkt_hdr->tailroom -= extrasegs * pkt_hdr->buf_hdr.size; > +} > + > static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr) > { > return pkt_hdr->frame_len; > diff --git a/platform/linux-generic/odp_packet.c > b/platform/linux-generic/odp_packet.c > index aac42b6..64b618a 100644 > --- a/platform/linux-generic/odp_packet.c > +++ b/platform/linux-generic/odp_packet.c > @@ -251,18 +251,18 @@ void *odp_packet_tail(odp_packet_t pkt) > return packet_map(pkt_hdr, pkt_hdr->frame_len, NULL); > } > > -void *odp_packet_push_head(odp_packet_t pkt, uint32_t len) > +void *odp_packet_push_head(odp_packet_t pkt, uint32_t len, uint32_t > *seglen) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > > - if (len > pkt_hdr->headroom) > + if (len > pkt_hdr->headroom && push_head_seg(pkt_hdr, len)) > return NULL; > > push_head(pkt_hdr, len); > - return packet_map(pkt_hdr, 0, NULL); > + return packet_map(pkt_hdr, 0, seglen); > } > > -void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) > +void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len, uint32_t > *seglen) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > > @@ -270,19 +270,22 @@ void *odp_packet_pull_head(odp_packet_t pkt, > uint32_t len) > return NULL; > > pull_head(pkt_hdr, len); > - return packet_map(pkt_hdr, 0, NULL); > + if (pkt_hdr->headroom > pkt_hdr->buf_hdr.size) > + pull_head_seg(pkt_hdr); > + > + return packet_map(pkt_hdr, 0, seglen); > } > > -void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) > +void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len, uint32_t > *seglen) > { > odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); > uint32_t origin = pkt_hdr->frame_len; > > - if (len > pkt_hdr->tailroom) > + if (len > pkt_hdr->tailroom && push_tail_seg(pkt_hdr, len)) > return NULL; > > push_tail(pkt_hdr, len); > - return packet_map(pkt_hdr, origin, NULL); > + return packet_map(pkt_hdr, origin, seglen); > } > > void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) > @@ -293,6 +296,9 @@ void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t > len) > return NULL; > > pull_tail(pkt_hdr, len); > + if (pkt_hdr->tailroom > pkt_hdr->buf_hdr.size) > + pull_tail_seg(pkt_hdr); > + > return packet_map(pkt_hdr, pkt_hdr->frame_len, NULL); > } > > diff --git a/platform/linux-generic/odp_pool.c > b/platform/linux-generic/odp_pool.c > index f6fa8f5..e494d5a 100644 > --- a/platform/linux-generic/odp_pool.c > +++ b/platform/linux-generic/odp_pool.c > @@ -500,6 +500,81 @@ int odp_pool_destroy(odp_pool_t pool_hdl) > return 0; > } > > +int seg_alloc_head(odp_buffer_hdr_t *buf_hdr, int segcount) > +{ > + uint32_t pool_id = pool_handle_to_index(buf_hdr->pool_hdl); > + pool_entry_t *pool = get_pool_entry(pool_id); > + void *newsegs[segcount]; > + int i; > + > + for (i = 0; i < segcount; i++) { > + newsegs[i] = get_blk(&pool->s); > + if (newsegs[i] == NULL) { > + while (--i >= 0) > + ret_blk(&pool->s, newsegs[i]); > + return -1; > + } > + } > + > + for (i = buf_hdr->segcount - 1; i >= 0; i--) > + buf_hdr->addr[i + segcount] = buf_hdr->addr[i]; > + > + for (i = 0; i < segcount; i++) > + buf_hdr->addr[i] = newsegs[i]; > + > + buf_hdr->segcount += segcount; > + return 0; > +} > + > +void seg_free_head(odp_buffer_hdr_t *buf_hdr, int segcount) > +{ > + uint32_t pool_id = pool_handle_to_index(buf_hdr->pool_hdl); > + pool_entry_t *pool = get_pool_entry(pool_id); > + int s_cnt = buf_hdr->segcount; > + int i; > + > + for (i = 0; i < s_cnt; i++) > + ret_blk(&pool->s, buf_hdr->addr[i]); > + > + for (i = 0; i < s_cnt - segcount; i++) > + buf_hdr->addr[i] = buf_hdr->addr[i + segcount]; > + > + buf_hdr->segcount -= segcount; > +} > + > +int seg_alloc_tail(odp_buffer_hdr_t *buf_hdr, int segcount) > +{ > + uint32_t pool_id = pool_handle_to_index(buf_hdr->pool_hdl); > + pool_entry_t *pool = get_pool_entry(pool_id); > + uint32_t s_cnt = buf_hdr->segcount; > + int i; > + > + for (i = 0; i < segcount; i++) { > + buf_hdr->addr[s_cnt + i] = get_blk(&pool->s); > + if (buf_hdr->addr[s_cnt + i] == NULL) { > + while (--i >= 0) > + ret_blk(&pool->s, buf_hdr->addr[s_cnt + > i]); > + return -1; > + } > + } > + > + buf_hdr->segcount += segcount; > + return 0; > +} > + > +void seg_free_tail(odp_buffer_hdr_t *buf_hdr, int segcount) > +{ > + uint32_t pool_id = pool_handle_to_index(buf_hdr->pool_hdl); > + pool_entry_t *pool = get_pool_entry(pool_id); > + int s_cnt = buf_hdr->segcount; > + int i; > + > + for (i = s_cnt - 1; i > s_cnt - segcount; i--) > + ret_blk(&pool->s, buf_hdr->addr[i]); > + > + buf_hdr->segcount -= segcount; > +} > + > odp_buffer_t buffer_alloc(odp_pool_t pool_hdl, size_t size) > { > uint32_t pool_id = pool_handle_to_index(pool_hdl); > diff --git a/test/validation/packet/packet.c > b/test/validation/packet/packet.c > index a764ed9..0258dde 100644 > --- a/test/validation/packet/packet.c > +++ b/test/validation/packet/packet.c > @@ -363,22 +363,30 @@ static void _verify_headroom_shift(odp_packet_t > packet, > uint32_t room = odp_packet_headroom(packet); > uint32_t seg_data_len = odp_packet_seg_len(packet); > uint32_t pkt_data_len = odp_packet_len(packet); > + uint32_t *seglen = NULL; > void *data; > char *data_orig = odp_packet_data(packet); > char *head_orig = odp_packet_head(packet); > + uint32_t seg_count_before = odp_packet_num_segs(packet); > + uint32_t seg_count_after; > > if (shift >= 0) > - data = odp_packet_push_head(packet, shift); > + data = odp_packet_push_head(packet, shift, seglen); > else > - data = odp_packet_pull_head(packet, -shift); > + data = odp_packet_pull_head(packet, -shift, seglen); > + seg_count_after = odp_packet_num_segs(packet); > > CU_ASSERT_PTR_NOT_NULL(data); > - CU_ASSERT(odp_packet_headroom(packet) == room - shift); > - CU_ASSERT(odp_packet_seg_len(packet) == seg_data_len + shift); > - CU_ASSERT(odp_packet_len(packet) == pkt_data_len + shift); > - CU_ASSERT(odp_packet_data(packet) == data); > - CU_ASSERT(odp_packet_head(packet) == head_orig); > - CU_ASSERT(data == data_orig - shift); > + if (seg_count_before == seg_count_after) { > + CU_ASSERT(odp_packet_headroom(packet) == room - shift); > + CU_ASSERT(odp_packet_seg_len(packet) == seg_data_len + > shift); > + CU_ASSERT(odp_packet_len(packet) == pkt_data_len + shift); > + CU_ASSERT(odp_packet_data(packet) == data); > + CU_ASSERT(odp_packet_head(packet) == head_orig); > + CU_ASSERT(data == data_orig - shift); > + } else { > + CU_ASSERT(data != NULL); > + } > } > > void packet_test_headroom(void) > @@ -403,6 +411,13 @@ void packet_test_headroom(void) > _verify_headroom_shift(pkt, push_val + pull_val); > _verify_headroom_shift(pkt, -push_val); > _verify_headroom_shift(pkt, 0); > + > + if (segmentation_supported) { > + push_val = room * 2; > + _verify_headroom_shift(pkt, push_val); > + _verify_headroom_shift(pkt, 0); > + _verify_headroom_shift(pkt, -push_val); > + } > } > > static void _verify_tailroom_shift(odp_packet_t pkt, > @@ -411,8 +426,11 @@ static void _verify_tailroom_shift(odp_packet_t pkt, > odp_packet_seg_t seg; > uint32_t room; > uint32_t seg_data_len, pkt_data_len; > + uint32_t *seglen = NULL; > void *tail; > char *tail_orig; > + uint32_t seg_count_before = odp_packet_num_segs(pkt); > + uint32_t seg_count_after; > > room = odp_packet_tailroom(pkt); > pkt_data_len = odp_packet_len(pkt); > @@ -429,7 +447,7 @@ static void _verify_tailroom_shift(odp_packet_t pkt, > l3_off = odp_packet_l3_offset(pkt); > l4_off = odp_packet_l4_offset(pkt); > > - tail = odp_packet_push_tail(pkt, shift); > + tail = odp_packet_push_tail(pkt, shift, seglen); > > CU_ASSERT(l2_off == odp_packet_l2_offset(pkt)); > CU_ASSERT(l3_off == odp_packet_l3_offset(pkt)); > @@ -437,19 +455,26 @@ static void _verify_tailroom_shift(odp_packet_t pkt, > } else { > tail = odp_packet_pull_tail(pkt, -shift); > } > + seg_count_after = odp_packet_num_segs(pkt); > > CU_ASSERT_PTR_NOT_NULL(tail); > - CU_ASSERT(odp_packet_seg_data_len(pkt, seg) == seg_data_len + > shift); > - CU_ASSERT(odp_packet_len(pkt) == pkt_data_len + shift); > - CU_ASSERT(odp_packet_tailroom(pkt) == room - shift); > - if (room == 0 || (room - shift) == 0) > - return; > - if (shift >= 0) { > - CU_ASSERT(odp_packet_tail(pkt) == tail_orig + shift); > - CU_ASSERT(tail == tail_orig); > + > + if (seg_count_before == seg_count_after) { > + CU_ASSERT(odp_packet_seg_data_len(pkt, seg) == > + seg_data_len + shift); > + CU_ASSERT(odp_packet_len(pkt) == pkt_data_len + shift); > + CU_ASSERT(odp_packet_tailroom(pkt) == room - shift); > + if (room == 0 || (room - shift) == 0) > + return; > + if (shift >= 0) { > + CU_ASSERT(odp_packet_tail(pkt) == tail_orig + > shift); > + CU_ASSERT(tail == tail_orig); > + } else { > + CU_ASSERT(odp_packet_tail(pkt) == tail); > + CU_ASSERT(tail == tail_orig + shift); > + } > } else { > - CU_ASSERT(odp_packet_tail(pkt) == tail); > - CU_ASSERT(tail == tail_orig + shift); > + CU_ASSERT(odp_packet_data(pkt) != NULL); > } > } > > @@ -478,6 +503,13 @@ void packet_test_tailroom(void) > _verify_tailroom_shift(pkt, push_val + pull_val); > _verify_tailroom_shift(pkt, -push_val); > _verify_tailroom_shift(pkt, 0); > + > + if (segmentation_supported) { > + push_val = room * 2; > + _verify_tailroom_shift(pkt, push_val); > + _verify_tailroom_shift(pkt, 0); > + _verify_tailroom_shift(pkt, -push_val); > + } > } > > void packet_test_segments(void) > -- > 2.5.0 > >
diff --git a/example/ipsec/odp_ipsec.c b/example/ipsec/odp_ipsec.c index 2e93fcd..052c387 100644 --- a/example/ipsec/odp_ipsec.c +++ b/example/ipsec/odp_ipsec.c @@ -763,7 +763,7 @@ pkt_disposition_e do_ipsec_in_finish(odp_packet_t pkt, /* We have a tunneled IPv4 packet */ if (ip->proto == ODPH_IPV4) { - odp_packet_pull_head(pkt, sizeof(*ip) + hdr_len); + odp_packet_pull_head(pkt, sizeof(*ip) + hdr_len, NULL); odp_packet_pull_tail(pkt, trl_len); odph_ethhdr_t *eth; @@ -925,7 +925,7 @@ pkt_disposition_e do_ipsec_out_classify(odp_packet_t pkt, /* Set IPv4 length before authentication */ ipv4_adjust_len(ip, hdr_len + trl_len); - if (!odp_packet_push_tail(pkt, hdr_len + trl_len)) + if (!odp_packet_push_tail(pkt, hdr_len + trl_len, NULL)) return PKT_DROP; /* Save remaining context */ diff --git a/example/ipsec/odp_ipsec_stream.c b/example/ipsec/odp_ipsec_stream.c index 4dc9acf..22ddc32 100644 --- a/example/ipsec/odp_ipsec_stream.c +++ b/example/ipsec/odp_ipsec_stream.c @@ -358,7 +358,7 @@ odp_packet_t create_ipv4_packet(stream_db_entry_t *stream, } /* Correct set packet length offsets */ - odp_packet_push_tail(pkt, data - base); + odp_packet_push_tail(pkt, data - base, NULL); odp_packet_l2_offset_set(pkt, (uint8_t *)eth - base); odp_packet_l3_offset_set(pkt, (uint8_t *)ip - base); odp_packet_l4_offset_set(pkt, ((uint8_t *)ip - base) + sizeof(*ip)); diff --git a/include/odp/api/spec/packet.h b/include/odp/api/spec/packet.h index 9c63b5f..66ef220 100644 --- a/include/odp/api/spec/packet.h +++ b/include/odp/api/spec/packet.h @@ -294,9 +294,13 @@ void *odp_packet_tail(odp_packet_t pkt); * Push out packet head * * Increase packet data length by moving packet head into packet headroom. - * Packet headroom is decreased with the same amount. The packet head may be - * pushed out up to 'headroom' bytes. Packet is not modified if there's not - * enough headroom space. + * Packet headroom is decreased by the same amount. If there is insufficient + * headroom available in the current segment the packet MAY be extended with + * additional segment(s) to accommodate the push request. Note that such + * extension may change the segmentation of the packet but does not affect + * the packet handle. As a result, the entire requested length may not be + * contiguously addressable from the returned data pointer. Use + * odp_packet_offset() to obtain this information if needed. * * odp_packet_xxx: * seg_len += len @@ -304,27 +308,27 @@ void *odp_packet_tail(odp_packet_t pkt); * headroom -= len * data -= len * - * Operation does not modify packet segmentation or move data. Handles and - * pointers remain valid. User is responsible to update packet metadata - * offsets when needed. + * Following this operation packet handles and pointers remain valid. User is + * responsible to update packet metadata offsets when needed. * - * @param pkt Packet handle - * @param len Number of bytes to push the head (0 ... headroom) + * @param pkt Packet handle + * @param len Number of bytes to push the head (0 or more) + * @param[out] seglen Number of bytes contiguously addressable from the returned + * data pointer. Ignored when NULL. * * @return The new data pointer * @retval NULL Requested offset exceeds available headroom * * @see odp_packet_headroom(), odp_packet_pull_head() */ -void *odp_packet_push_head(odp_packet_t pkt, uint32_t len); +void *odp_packet_push_head(odp_packet_t pkt, uint32_t len, uint32_t *seglen); /** * Pull in packet head * * Decrease packet data length by removing data from the head of the packet. - * Packet headroom is increased with the same amount. Packet head may be pulled - * in up to seg_len - 1 bytes (i.e. packet data pointer must stay in the - * first segment). Packet is not modified if there's not enough data. + * Packet headroom is increased with the same amount. Packet is not modified + * if there's not enough data. * * odp_packet_xxx: * seg_len -= len @@ -332,27 +336,34 @@ void *odp_packet_push_head(odp_packet_t pkt, uint32_t len); * headroom += len * data += len * - * Operation does not modify packet segmentation or move data. Handles and - * pointers remain valid. User is responsible to update packet metadata - * offsets when needed. + * Following this operation packet handles and pointers remain valid. User is + * responsible to update packet metadata offsets when needed. * * @param pkt Packet handle - * @param len Number of bytes to pull the head (0 ... seg_len - 1) + * @param len Number of bytes to pull the head (0 ... packet length) + * @param[out] seglen Number of contiguously addressable bytes available at + * the returned data pointer. Ignored when NULL. * * @return The new data pointer - * @retval NULL Requested offset exceeds packet segment length + * @retval NULL Requested offset exceeds packet length * * @see odp_packet_seg_len(), odp_packet_push_head() */ -void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len); +void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len, uint32_t *seglen); /** * Push out packet tail * * Increase packet data length by moving packet tail into packet tailroom. - * Packet tailroom is decreased with the same amount. The packet tail may be - * pushed out up to 'tailroom' bytes. Packet is not modified if there's not - * enough tailroom. + * Packet tailroom is decreased with the same amount. If the requested len + * exceeds the current tailroom, the packet MAY be extended with additional + * segment(s) to accommodate the push request. Note that such extension may + * change the segmentation of the packet but does not affect the packet + * handle. As a result, the entire requested length may not be contiguously + * addressable from the returned data pointer. Use odp_packet_offset() to + * obtain this information if needed. + * + * Packet is not modified if there's not enough tailroom. * * last_seg: * data_len += len @@ -362,26 +373,26 @@ void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len); * tail += len * tailroom -= len * - * Operation does not modify packet segmentation or move data. Handles, - * pointers and offsets remain valid. + * Following this operation packet handles, pointers and offsets remain valid. * - * @param pkt Packet handle - * @param len Number of bytes to push the tail (0 ... tailroom) + * @param pkt Packet handle + * @param len Number of bytes to push the tail (0 or more) + * @param[out] seglen Number of contiguously addressable bytes available at + * returned data pointer. Ignored when NULL. * * @return The old tail pointer * @retval NULL Requested offset exceeds available tailroom * * @see odp_packet_tailroom(), odp_packet_pull_tail() */ -void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len); +void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len, uint32_t *seglen); /** * Pull in packet tail * * Decrease packet data length by removing data from the tail of the packet. - * Packet tailroom is increased with the same amount. Packet tail may be pulled - * in up to last segment data_len - 1 bytes. (i.e. packet tail must stay in the - * last segment). Packet is not modified if there's not enough data. + * Packet tailroom is increased with the same amount. Packet is not modified + * if there's not enough data. * * last_seg: * data_len -= len @@ -391,15 +402,14 @@ void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len); * tail -= len * tailroom += len * - * Operation does not modify packet segmentation or move data. Handles and - * pointers remain valid. User is responsible to update packet metadata - * offsets when needed. + * Following this operation packet handles and pointers remain valid. User is + * responsible to update packet metadata offsets when needed. * * @param pkt Packet handle - * @param len Number of bytes to pull the tail (0 ... last_seg:data_len - 1) + * @param len Number of bytes to pull the tail (0 ... packet length) * * @return The new tail pointer - * @retval NULL The specified offset exceeds allowable data length + * @retval NULL The specified offset exceeds packet length */ void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len); diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h index ea092ca..0a4c290 100644 --- a/platform/linux-generic/include/odp_buffer_internal.h +++ b/platform/linux-generic/include/odp_buffer_internal.h @@ -170,6 +170,10 @@ typedef struct { odp_buffer_t buffer_alloc(odp_pool_t pool, size_t size); int buffer_alloc_multi(odp_pool_t pool_hdl, size_t size, odp_buffer_t buf[], int num); +int seg_alloc_head(odp_buffer_hdr_t *buf_hdr, int segcount); +void seg_free_head(odp_buffer_hdr_t *buf_hdr, int segcount); +int seg_alloc_tail(odp_buffer_hdr_t *buf_hdr, int segcount); +void seg_free_tail(odp_buffer_hdr_t *buf_hdr, int segcount); #ifdef __cplusplus } diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index b632ece..77e32fe 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -210,6 +210,30 @@ static inline void pull_head(odp_packet_hdr_t *pkt_hdr, size_t len) pkt_hdr->frame_len -= len; } +static inline int push_head_seg(odp_packet_hdr_t *pkt_hdr, size_t len) +{ + uint32_t newsegcount = + (len - pkt_hdr->headroom + pkt_hdr->buf_hdr.size - 1) / + pkt_hdr->buf_hdr.size; + + if (pkt_hdr->buf_hdr.segcount + newsegcount > ODP_BUFFER_MAX_SEG) + return -1; + + if (seg_alloc_head(&pkt_hdr->buf_hdr, newsegcount)) + return -1; + + pkt_hdr->headroom += newsegcount * pkt_hdr->buf_hdr.size; + return 0; +} + +static inline void pull_head_seg(odp_packet_hdr_t *pkt_hdr) +{ + uint32_t extrasegs = pkt_hdr->headroom / pkt_hdr->buf_hdr.size; + + seg_free_head(&pkt_hdr->buf_hdr, extrasegs); + pkt_hdr->headroom -= extrasegs * pkt_hdr->buf_hdr.size; +} + static inline void push_tail(odp_packet_hdr_t *pkt_hdr, size_t len) { pkt_hdr->tailroom -= len; @@ -223,6 +247,31 @@ static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, size_t len) pkt_hdr->frame_len -= len; } +static inline int push_tail_seg(odp_packet_hdr_t *pkt_hdr, size_t len) +{ + uint32_t newsegcount = + (len - pkt_hdr->tailroom + pkt_hdr->buf_hdr.size - 1) / + pkt_hdr->buf_hdr.size; + + if (pkt_hdr->buf_hdr.segcount + newsegcount > ODP_BUFFER_MAX_SEG) + return -1; + + if (seg_alloc_tail(&pkt_hdr->buf_hdr, newsegcount)) + return -1; + + pkt_hdr->tailroom += newsegcount * pkt_hdr->buf_hdr.size; + return 0; +} + +static inline void pull_tail_seg(odp_packet_hdr_t *pkt_hdr) +{ + uint32_t extrasegs = pkt_hdr->tailroom / pkt_hdr->buf_hdr.size; + + seg_free_tail(&pkt_hdr->buf_hdr, extrasegs); + + pkt_hdr->tailroom -= extrasegs * pkt_hdr->buf_hdr.size; +} + static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr) { return pkt_hdr->frame_len; diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index aac42b6..64b618a 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -251,18 +251,18 @@ void *odp_packet_tail(odp_packet_t pkt) return packet_map(pkt_hdr, pkt_hdr->frame_len, NULL); } -void *odp_packet_push_head(odp_packet_t pkt, uint32_t len) +void *odp_packet_push_head(odp_packet_t pkt, uint32_t len, uint32_t *seglen) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); - if (len > pkt_hdr->headroom) + if (len > pkt_hdr->headroom && push_head_seg(pkt_hdr, len)) return NULL; push_head(pkt_hdr, len); - return packet_map(pkt_hdr, 0, NULL); + return packet_map(pkt_hdr, 0, seglen); } -void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) +void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len, uint32_t *seglen) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); @@ -270,19 +270,22 @@ void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) return NULL; pull_head(pkt_hdr, len); - return packet_map(pkt_hdr, 0, NULL); + if (pkt_hdr->headroom > pkt_hdr->buf_hdr.size) + pull_head_seg(pkt_hdr); + + return packet_map(pkt_hdr, 0, seglen); } -void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) +void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len, uint32_t *seglen) { odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); uint32_t origin = pkt_hdr->frame_len; - if (len > pkt_hdr->tailroom) + if (len > pkt_hdr->tailroom && push_tail_seg(pkt_hdr, len)) return NULL; push_tail(pkt_hdr, len); - return packet_map(pkt_hdr, origin, NULL); + return packet_map(pkt_hdr, origin, seglen); } void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) @@ -293,6 +296,9 @@ void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) return NULL; pull_tail(pkt_hdr, len); + if (pkt_hdr->tailroom > pkt_hdr->buf_hdr.size) + pull_tail_seg(pkt_hdr); + return packet_map(pkt_hdr, pkt_hdr->frame_len, NULL); } diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index f6fa8f5..e494d5a 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -500,6 +500,81 @@ int odp_pool_destroy(odp_pool_t pool_hdl) return 0; } +int seg_alloc_head(odp_buffer_hdr_t *buf_hdr, int segcount) +{ + uint32_t pool_id = pool_handle_to_index(buf_hdr->pool_hdl); + pool_entry_t *pool = get_pool_entry(pool_id); + void *newsegs[segcount]; + int i; + + for (i = 0; i < segcount; i++) { + newsegs[i] = get_blk(&pool->s); + if (newsegs[i] == NULL) { + while (--i >= 0) + ret_blk(&pool->s, newsegs[i]); + return -1; + } + } + + for (i = buf_hdr->segcount - 1; i >= 0; i--) + buf_hdr->addr[i + segcount] = buf_hdr->addr[i]; + + for (i = 0; i < segcount; i++) + buf_hdr->addr[i] = newsegs[i]; + + buf_hdr->segcount += segcount; + return 0; +} + +void seg_free_head(odp_buffer_hdr_t *buf_hdr, int segcount) +{ + uint32_t pool_id = pool_handle_to_index(buf_hdr->pool_hdl); + pool_entry_t *pool = get_pool_entry(pool_id); + int s_cnt = buf_hdr->segcount; + int i; + + for (i = 0; i < s_cnt; i++) + ret_blk(&pool->s, buf_hdr->addr[i]); + + for (i = 0; i < s_cnt - segcount; i++) + buf_hdr->addr[i] = buf_hdr->addr[i + segcount]; + + buf_hdr->segcount -= segcount; +} + +int seg_alloc_tail(odp_buffer_hdr_t *buf_hdr, int segcount) +{ + uint32_t pool_id = pool_handle_to_index(buf_hdr->pool_hdl); + pool_entry_t *pool = get_pool_entry(pool_id); + uint32_t s_cnt = buf_hdr->segcount; + int i; + + for (i = 0; i < segcount; i++) { + buf_hdr->addr[s_cnt + i] = get_blk(&pool->s); + if (buf_hdr->addr[s_cnt + i] == NULL) { + while (--i >= 0) + ret_blk(&pool->s, buf_hdr->addr[s_cnt + i]); + return -1; + } + } + + buf_hdr->segcount += segcount; + return 0; +} + +void seg_free_tail(odp_buffer_hdr_t *buf_hdr, int segcount) +{ + uint32_t pool_id = pool_handle_to_index(buf_hdr->pool_hdl); + pool_entry_t *pool = get_pool_entry(pool_id); + int s_cnt = buf_hdr->segcount; + int i; + + for (i = s_cnt - 1; i > s_cnt - segcount; i--) + ret_blk(&pool->s, buf_hdr->addr[i]); + + buf_hdr->segcount -= segcount; +} + odp_buffer_t buffer_alloc(odp_pool_t pool_hdl, size_t size) { uint32_t pool_id = pool_handle_to_index(pool_hdl); diff --git a/test/validation/packet/packet.c b/test/validation/packet/packet.c index a764ed9..0258dde 100644 --- a/test/validation/packet/packet.c +++ b/test/validation/packet/packet.c @@ -363,22 +363,30 @@ static void _verify_headroom_shift(odp_packet_t packet, uint32_t room = odp_packet_headroom(packet); uint32_t seg_data_len = odp_packet_seg_len(packet); uint32_t pkt_data_len = odp_packet_len(packet); + uint32_t *seglen = NULL; void *data; char *data_orig = odp_packet_data(packet); char *head_orig = odp_packet_head(packet); + uint32_t seg_count_before = odp_packet_num_segs(packet); + uint32_t seg_count_after; if (shift >= 0) - data = odp_packet_push_head(packet, shift); + data = odp_packet_push_head(packet, shift, seglen); else - data = odp_packet_pull_head(packet, -shift); + data = odp_packet_pull_head(packet, -shift, seglen); + seg_count_after = odp_packet_num_segs(packet); CU_ASSERT_PTR_NOT_NULL(data); - CU_ASSERT(odp_packet_headroom(packet) == room - shift); - CU_ASSERT(odp_packet_seg_len(packet) == seg_data_len + shift); - CU_ASSERT(odp_packet_len(packet) == pkt_data_len + shift); - CU_ASSERT(odp_packet_data(packet) == data); - CU_ASSERT(odp_packet_head(packet) == head_orig); - CU_ASSERT(data == data_orig - shift); + if (seg_count_before == seg_count_after) { + CU_ASSERT(odp_packet_headroom(packet) == room - shift); + CU_ASSERT(odp_packet_seg_len(packet) == seg_data_len + shift); + CU_ASSERT(odp_packet_len(packet) == pkt_data_len + shift); + CU_ASSERT(odp_packet_data(packet) == data); + CU_ASSERT(odp_packet_head(packet) == head_orig); + CU_ASSERT(data == data_orig - shift); + } else { + CU_ASSERT(data != NULL); + } } void packet_test_headroom(void) @@ -403,6 +411,13 @@ void packet_test_headroom(void) _verify_headroom_shift(pkt, push_val + pull_val); _verify_headroom_shift(pkt, -push_val); _verify_headroom_shift(pkt, 0); + + if (segmentation_supported) { + push_val = room * 2; + _verify_headroom_shift(pkt, push_val); + _verify_headroom_shift(pkt, 0); + _verify_headroom_shift(pkt, -push_val); + } } static void _verify_tailroom_shift(odp_packet_t pkt, @@ -411,8 +426,11 @@ static void _verify_tailroom_shift(odp_packet_t pkt, odp_packet_seg_t seg; uint32_t room; uint32_t seg_data_len, pkt_data_len; + uint32_t *seglen = NULL; void *tail; char *tail_orig; + uint32_t seg_count_before = odp_packet_num_segs(pkt); + uint32_t seg_count_after; room = odp_packet_tailroom(pkt); pkt_data_len = odp_packet_len(pkt); @@ -429,7 +447,7 @@ static void _verify_tailroom_shift(odp_packet_t pkt, l3_off = odp_packet_l3_offset(pkt); l4_off = odp_packet_l4_offset(pkt); - tail = odp_packet_push_tail(pkt, shift); + tail = odp_packet_push_tail(pkt, shift, seglen); CU_ASSERT(l2_off == odp_packet_l2_offset(pkt)); CU_ASSERT(l3_off == odp_packet_l3_offset(pkt)); @@ -437,19 +455,26 @@ static void _verify_tailroom_shift(odp_packet_t pkt, } else { tail = odp_packet_pull_tail(pkt, -shift); } + seg_count_after = odp_packet_num_segs(pkt); CU_ASSERT_PTR_NOT_NULL(tail); - CU_ASSERT(odp_packet_seg_data_len(pkt, seg) == seg_data_len + shift); - CU_ASSERT(odp_packet_len(pkt) == pkt_data_len + shift); - CU_ASSERT(odp_packet_tailroom(pkt) == room - shift); - if (room == 0 || (room - shift) == 0) - return; - if (shift >= 0) { - CU_ASSERT(odp_packet_tail(pkt) == tail_orig + shift); - CU_ASSERT(tail == tail_orig); + + if (seg_count_before == seg_count_after) { + CU_ASSERT(odp_packet_seg_data_len(pkt, seg) == + seg_data_len + shift); + CU_ASSERT(odp_packet_len(pkt) == pkt_data_len + shift); + CU_ASSERT(odp_packet_tailroom(pkt) == room - shift); + if (room == 0 || (room - shift) == 0) + return; + if (shift >= 0) { + CU_ASSERT(odp_packet_tail(pkt) == tail_orig + shift); + CU_ASSERT(tail == tail_orig); + } else { + CU_ASSERT(odp_packet_tail(pkt) == tail); + CU_ASSERT(tail == tail_orig + shift); + } } else { - CU_ASSERT(odp_packet_tail(pkt) == tail); - CU_ASSERT(tail == tail_orig + shift); + CU_ASSERT(odp_packet_data(pkt) != NULL); } } @@ -478,6 +503,13 @@ void packet_test_tailroom(void) _verify_tailroom_shift(pkt, push_val + pull_val); _verify_tailroom_shift(pkt, -push_val); _verify_tailroom_shift(pkt, 0); + + if (segmentation_supported) { + push_val = room * 2; + _verify_tailroom_shift(pkt, push_val); + _verify_tailroom_shift(pkt, 0); + _verify_tailroom_shift(pkt, -push_val); + } } void packet_test_segments(void)
Add support for multi-segment push/pull operations for Monarch compliance. If a push for more than the available headroom/tailroom is requested, then allocate additional head/tail segments if possible to complete the operation. Similarly, when pulling more than a single segment, allow empty segments to exist that consist entirely of headroom/tailroom. Signed-off-by: Bill Fischofer <bill.fischofer@linaro.org> --- example/ipsec/odp_ipsec.c | 4 +- example/ipsec/odp_ipsec_stream.c | 2 +- include/odp/api/spec/packet.h | 78 ++++++++++++---------- .../linux-generic/include/odp_buffer_internal.h | 4 ++ .../linux-generic/include/odp_packet_internal.h | 49 ++++++++++++++ platform/linux-generic/odp_packet.c | 22 +++--- platform/linux-generic/odp_pool.c | 75 +++++++++++++++++++++ test/validation/packet/packet.c | 70 +++++++++++++------ 8 files changed, 240 insertions(+), 64 deletions(-)