From patchwork Sun Mar 12 12:30:35 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Bill Fischofer X-Patchwork-Id: 95167 Delivered-To: patch@linaro.org Received: by 10.140.82.71 with SMTP id g65csp783048qgd; Sun, 12 Mar 2017 05:31:02 -0700 (PDT) X-Received: by 10.200.43.185 with SMTP id m54mr27818881qtm.122.1489321862709; Sun, 12 Mar 2017 05:31:02 -0700 (PDT) Return-Path: Received: from lists.linaro.org (lists.linaro.org. [54.225.227.206]) by mx.google.com with ESMTP id p53si12311459qtp.306.2017.03.12.05.31.01; Sun, 12 Mar 2017 05:31:02 -0700 (PDT) Received-SPF: pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) client-ip=54.225.227.206; Authentication-Results: mx.google.com; spf=pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) smtp.mailfrom=lng-odp-bounces@lists.linaro.org; dmarc=pass (p=NONE sp=NONE dis=NONE) header.from=linaro.org Received: by lists.linaro.org (Postfix, from userid 109) id 5F81563C96; Sun, 12 Mar 2017 12:31:01 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on ip-10-142-244-252 X-Spam-Level: X-Spam-Status: No, score=-1.9 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_NONE, RCVD_IN_MSPIKE_H3, RCVD_IN_MSPIKE_WL, URIBL_BLOCKED autolearn=disabled version=3.4.0 Received: from [127.0.0.1] (localhost [127.0.0.1]) by lists.linaro.org (Postfix) with ESMTP id C142963C87; Sun, 12 Mar 2017 12:30:54 +0000 (UTC) X-Original-To: lng-odp@lists.linaro.org Delivered-To: lng-odp@lists.linaro.org Received: by lists.linaro.org (Postfix, from userid 109) id E21A263C87; Sun, 12 Mar 2017 12:30:52 +0000 (UTC) Received: from mail-ot0-f175.google.com (mail-ot0-f175.google.com [74.125.82.175]) by lists.linaro.org (Postfix) with ESMTPS id 2281662D0B for ; Sun, 12 Mar 2017 12:30:52 +0000 (UTC) Received: by mail-ot0-f175.google.com with SMTP id x37so99512846ota.2 for ; Sun, 12 Mar 2017 05:30:52 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references; bh=qvNAvCkh66aRCbTK/+40hHURcGOU0Ipi7hDzps+uuh4=; b=qGGSjYywbbul1QbH1Q7Cy1NAXdq1cy5X30jSZT7z1gakykoBag85wqftTyHRBZZPLE u/lHAroYjNzMxxyBPSfrbd9FjtUeLVa8i0ck/+YqghMHK456710LxyXqh6X6LioKJ6yZ Vimd9NHs7sIMVt7R6I6XiOeBNgHQOdkBnyiGtuFbYqcSc0h6AuNas9gBkan+pCZKsl8o cUoRLQrzOeYSV4pNQP3pl4Dlo161EsMQL/gVB08Ui0qQqheZLlj2kYIDEMjsM5rS8GvZ UCMC27WVvK1MaGU7WZIDCK5R1HJzoJSDUtYiiRiNdD50wxvUivWhUZAt5uwDuymUKceP RQlg== X-Gm-Message-State: AFeK/H3a1EdRYh5OQb/hOBOJLWsz6+TPYocFrnvA5OzUu892GWjTVTCJ7CqMm/AMjQr1vABs7+eq1g== X-Received: by 10.157.82.91 with SMTP id q27mr13902815otg.50.1489321851452; Sun, 12 Mar 2017 05:30:51 -0700 (PDT) Received: from localhost.localdomain (cpe-70-121-83-241.austin.res.rr.com. [70.121.83.241]) by smtp.gmail.com with ESMTPSA id 62sm6663155otq.34.2017.03.12.05.30.50 (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Sun, 12 Mar 2017 05:30:50 -0700 (PDT) From: Bill Fischofer To: lng-odp@lists.linaro.org Date: Sun, 12 Mar 2017 07:30:35 -0500 Message-Id: <20170312123045.3852-2-bill.fischofer@linaro.org> X-Mailer: git-send-email 2.12.0.rc1 In-Reply-To: <20170312123045.3852-1-bill.fischofer@linaro.org> References: <20170312123045.3852-1-bill.fischofer@linaro.org> Subject: [lng-odp] [PATCHv2 01/11] linux-generic: pool: restructure pool routines X-BeenThere: lng-odp@lists.linaro.org X-Mailman-Version: 2.1.16 Precedence: list List-Id: "The OpenDataPlane \(ODP\) List" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: lng-odp-bounces@lists.linaro.org Sender: "lng-odp" Restructure pool routines in preparation for introducing packet references. Signed-off-by: Bill Fischofer --- platform/linux-generic/include/odp_pool_internal.h | 33 ++++++++++++++- platform/linux-generic/odp_pool.c | 47 +--------------------- 2 files changed, 34 insertions(+), 46 deletions(-) -- 2.12.0.rc1 diff --git a/platform/linux-generic/include/odp_pool_internal.h b/platform/linux-generic/include/odp_pool_internal.h index ebb779da..e767251a 100644 --- a/platform/linux-generic/include/odp_pool_internal.h +++ b/platform/linux-generic/include/odp_pool_internal.h @@ -122,9 +122,40 @@ static inline odp_buffer_hdr_t *buf_hdl_to_hdr(odp_buffer_t buf) return pool_buf_hdl_to_hdr(pool, buf); } +static inline odp_pool_t pool_index_to_handle(uint32_t pool_idx) +{ + return _odp_cast_scalar(odp_pool_t, pool_idx); +} + +static inline uint32_t pool_id_from_buf(odp_buffer_t buf) +{ + odp_buffer_bits_t handle; + + handle.handle = buf; + return handle.pool_id; +} + int buffer_alloc_multi(pool_t *pool, odp_buffer_t buf[], odp_buffer_hdr_t *buf_hdr[], int num); -void buffer_free_multi(const odp_buffer_t buf[], int num_free); +void buffer_free_to_pool(uint32_t pool_id, const odp_buffer_t buf[], int num); + +static inline void buffer_free_multi(const odp_buffer_t buf[], int num) +{ + uint32_t next_pool, pool_id = pool_id_from_buf(buf[0]); + int i, first = 0; + + for (i = 1; i < num; i++) { + next_pool = pool_id_from_buf(buf[i]); + + if (odp_unlikely(next_pool != pool_id)) { + buffer_free_to_pool(pool_id, &buf[first], i - first); + first = i; + pool_id = next_pool; + } + } + + buffer_free_to_pool(pool_id, &buf[first], num - first); +} #ifdef __cplusplus } diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c index 145002d9..9df9ba21 100644 --- a/platform/linux-generic/odp_pool.c +++ b/platform/linux-generic/odp_pool.c @@ -47,19 +47,6 @@ typedef struct pool_local_t { pool_table_t *pool_tbl; static __thread pool_local_t local; -static inline odp_pool_t pool_index_to_handle(uint32_t pool_idx) -{ - return _odp_cast_scalar(odp_pool_t, pool_idx); -} - -static inline uint32_t pool_id_from_buf(odp_buffer_t buf) -{ - odp_buffer_bits_t handle; - - handle.handle = buf; - return handle.pool_id; -} - int odp_pool_init_global(void) { uint32_t i; @@ -672,8 +659,8 @@ int buffer_alloc_multi(pool_t *pool, odp_buffer_t buf[], return num_ch + num_deq; } -static inline void buffer_free_to_pool(uint32_t pool_id, - const odp_buffer_t buf[], int num) +void buffer_free_to_pool(uint32_t pool_id, + const odp_buffer_t buf[], int num) { pool_t *pool; int i; @@ -733,36 +720,6 @@ static inline void buffer_free_to_pool(uint32_t pool_id, cache->num = cache_num + num; } -void buffer_free_multi(const odp_buffer_t buf[], int num_total) -{ - uint32_t pool_id; - int num; - int i; - int first = 0; - - while (1) { - num = 1; - i = 1; - pool_id = pool_id_from_buf(buf[first]); - - /* 'num' buffers are from the same pool */ - if (num_total > 1) { - for (i = first; i < num_total; i++) - if (pool_id != pool_id_from_buf(buf[i])) - break; - - num = i - first; - } - - buffer_free_to_pool(pool_id, &buf[first], num); - - if (i == num_total) - return; - - first = i; - } -} - odp_buffer_t odp_buffer_alloc(odp_pool_t pool_hdl) { odp_buffer_t buf; From patchwork Sun Mar 12 12:30:36 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Bill Fischofer X-Patchwork-Id: 95169 Delivered-To: patch@linaro.org Received: by 10.140.82.71 with SMTP id g65csp783357qgd; Sun, 12 Mar 2017 05:32:09 -0700 (PDT) X-Received: by 10.237.34.125 with SMTP id o58mr27114044qtc.108.1489321929342; Sun, 12 Mar 2017 05:32:09 -0700 (PDT) Return-Path: Received: from lists.linaro.org (lists.linaro.org. [54.225.227.206]) by mx.google.com with ESMTP id y2si12328712qta.108.2017.03.12.05.32.09; Sun, 12 Mar 2017 05:32:09 -0700 (PDT) Received-SPF: pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) client-ip=54.225.227.206; Authentication-Results: mx.google.com; spf=pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) smtp.mailfrom=lng-odp-bounces@lists.linaro.org; dmarc=pass (p=NONE sp=NONE dis=NONE) header.from=linaro.org Received: by lists.linaro.org (Postfix, from userid 109) id 082BD63C94; Sun, 12 Mar 2017 12:32:09 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on ip-10-142-244-252 X-Spam-Level: X-Spam-Status: No, score=-1.9 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_NONE, RCVD_IN_MSPIKE_H3, RCVD_IN_MSPIKE_WL, URIBL_BLOCKED autolearn=disabled version=3.4.0 Received: from [127.0.0.1] (localhost [127.0.0.1]) by lists.linaro.org (Postfix) with ESMTP id 0C21E63C95; Sun, 12 Mar 2017 12:31:02 +0000 (UTC) X-Original-To: lng-odp@lists.linaro.org Delivered-To: lng-odp@lists.linaro.org Received: by lists.linaro.org (Postfix, from userid 109) id 06EFB62D0B; Sun, 12 Mar 2017 12:30:54 +0000 (UTC) Received: from mail-ot0-f180.google.com (mail-ot0-f180.google.com [74.125.82.180]) by lists.linaro.org (Postfix) with ESMTPS id 102DA62D0B for ; Sun, 12 Mar 2017 12:30:53 +0000 (UTC) Received: by mail-ot0-f180.google.com with SMTP id 19so99643156oti.0 for ; Sun, 12 Mar 2017 05:30:53 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references; bh=cT4Pj146x43Qg6YKe3Z/UNUWeIXm0rKuzF7pcGWO5r8=; b=cImGmn4MdR2WGUWNoMYIPfNN+VmI6dJxhD/f/ejD6XoSupBnZ8903L8VQt8MS/ZjGY nlOs9wjFurXID0Si+0UDSFuUgwg3MEm+XJCeCGPZYVYe0KUdkAliCQAIFAomzVJDE20U Np5bCxLJ2xlY9rzkrziiVWRmdb/b7lBtSAiRv4QxHOEwMOkjFgIlRDAkcyx8pI4kNyD2 FJ+N448IcqhWm2w3PFYX/77b8uCIB+mj2ctl3sSCEYiZswohtbrHmcHL8MVueBhwK6im wFMMdhCcao3742IUGbNRAw/nnebV3DeRO0d7/RwcHfwh03UPuMMHQIWwW/iPsd/xpXuY 55EQ== X-Gm-Message-State: AFeK/H3Wq3639YaPbfVsAFZnY2StjgNlRewU+32gYWc48lR3V4EMGh88fmYH5PXb+SN14uQ/HgO6yw== X-Received: by 10.157.41.65 with SMTP id d59mr13696513otb.154.1489321852360; Sun, 12 Mar 2017 05:30:52 -0700 (PDT) Received: from localhost.localdomain (cpe-70-121-83-241.austin.res.rr.com. [70.121.83.241]) by smtp.gmail.com with ESMTPSA id 62sm6663155otq.34.2017.03.12.05.30.51 (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Sun, 12 Mar 2017 05:30:51 -0700 (PDT) From: Bill Fischofer To: lng-odp@lists.linaro.org Date: Sun, 12 Mar 2017 07:30:36 -0500 Message-Id: <20170312123045.3852-3-bill.fischofer@linaro.org> X-Mailer: git-send-email 2.12.0.rc1 In-Reply-To: <20170312123045.3852-1-bill.fischofer@linaro.org> References: <20170312123045.3852-1-bill.fischofer@linaro.org> Subject: [lng-odp] [PATCHv2 02/11] linux-generic: packet: move pull_tail to packet mainline for consistency X-BeenThere: lng-odp@lists.linaro.org X-Mailman-Version: 2.1.16 Precedence: list List-Id: "The OpenDataPlane \(ODP\) List" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: lng-odp-bounces@lists.linaro.org Sender: "lng-odp" Move the pull_tail() internal routine to odp_packet.c for consistency with other internal head/tail manipulation routines. Signed-off-by: Bill Fischofer --- platform/linux-generic/include/odp_packet_internal.h | 9 --------- platform/linux-generic/odp_packet.c | 9 +++++++++ 2 files changed, 9 insertions(+), 9 deletions(-) -- 2.12.0.rc1 diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 0a9f1779..589e204a 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -184,15 +184,6 @@ static inline void copy_packet_cls_metadata(odp_packet_hdr_t *src_hdr, dst_hdr->op_result = src_hdr->op_result; } -static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) -{ - int last = pkt_hdr->buf_hdr.segcount - 1; - - pkt_hdr->tailroom += len; - pkt_hdr->frame_len -= len; - pkt_hdr->buf_hdr.seg[last].len -= len; -} - static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr) { return pkt_hdr->frame_len; diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 60eef3a6..d0f76332 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -145,6 +145,15 @@ static inline void push_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) pkt_hdr->buf_hdr.seg[last].len += len; } +static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) +{ + int last = pkt_hdr->buf_hdr.segcount - 1; + + pkt_hdr->tailroom += len; + pkt_hdr->frame_len -= len; + pkt_hdr->buf_hdr.seg[last].len -= len; +} + /* Copy all metadata for segmentation modification. Segment data and lengths * are not copied. */ static inline void packet_seg_copy_md(odp_packet_hdr_t *dst, From patchwork Sun Mar 12 12:30:37 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Bill Fischofer X-Patchwork-Id: 95170 Delivered-To: patch@linaro.org Received: by 10.140.82.71 with SMTP id g65csp783514qgd; Sun, 12 Mar 2017 05:32:43 -0700 (PDT) X-Received: by 10.55.190.129 with SMTP id o123mr29303894qkf.110.1489321963648; Sun, 12 Mar 2017 05:32:43 -0700 (PDT) Return-Path: Received: from lists.linaro.org (lists.linaro.org. [54.225.227.206]) by mx.google.com with ESMTP id m33si12344931qtd.72.2017.03.12.05.32.43; Sun, 12 Mar 2017 05:32:43 -0700 (PDT) Received-SPF: pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) client-ip=54.225.227.206; Authentication-Results: mx.google.com; spf=pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) smtp.mailfrom=lng-odp-bounces@lists.linaro.org; dmarc=pass (p=NONE sp=NONE dis=NONE) header.from=linaro.org Received: by lists.linaro.org (Postfix, from userid 109) id 3CC8F606B4; Sun, 12 Mar 2017 12:32:43 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on ip-10-142-244-252 X-Spam-Level: X-Spam-Status: No, score=-1.9 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_NONE, RCVD_IN_MSPIKE_H3, RCVD_IN_MSPIKE_WL, URIBL_BLOCKED autolearn=disabled version=3.4.0 Received: from [127.0.0.1] (localhost [127.0.0.1]) by lists.linaro.org (Postfix) with ESMTP id 4DA3163C9C; Sun, 12 Mar 2017 12:31:05 +0000 (UTC) X-Original-To: lng-odp@lists.linaro.org Delivered-To: lng-odp@lists.linaro.org Received: by lists.linaro.org (Postfix, from userid 109) id D164163C8F; Sun, 12 Mar 2017 12:30:57 +0000 (UTC) Received: from mail-ot0-f171.google.com (mail-ot0-f171.google.com [74.125.82.171]) by lists.linaro.org (Postfix) with ESMTPS id 31AD360F4C for ; Sun, 12 Mar 2017 12:30:54 +0000 (UTC) Received: by mail-ot0-f171.google.com with SMTP id x37so99513050ota.2 for ; Sun, 12 Mar 2017 05:30:54 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references; bh=IocZ58Qkj+vbFvxiMLGrMDp4S2oRZLXhFEQOA89WIeY=; b=YN8qD/+jJj6t9cbmOlFHgRtN/C8gDy2ecot/8vKYAHq9wpg3cn9oO0eTHessQM931Y JCT5cUFjK3yRdOj6tDPtIK/q35xhNA+jgNVohFbfDrzOlfz/mFpR5Bv4DYfDcQcEPotQ EQEYDd/jjN0a1007WNRHrZkVRpSPjyNiO0Dibz7OzpLr9JUpby4l+fpqHo7TdQWvLkWx KIA3pv5tHUtnH3N69E3SYbJUlVQb76ljTxqKxN9x4T8LUU0nUmnrF9g/jyDYhP7fsptE cFZJaoo0wF2FKl5WGxP6hdepV2d407UO2qrO7PePD2RIwhK7wOu9d+vMz2tDF5QD2tkK vvbQ== X-Gm-Message-State: AFeK/H2MxK0gmVgEAu7AC+A5DcRGj557qdLIDt4ZDmMYgakqVcgzyIOSEZ2UsexaojFX1UIPvajPwA== X-Received: by 10.157.13.72 with SMTP id 66mr16577126oti.6.1489321853441; Sun, 12 Mar 2017 05:30:53 -0700 (PDT) Received: from localhost.localdomain (cpe-70-121-83-241.austin.res.rr.com. [70.121.83.241]) by smtp.gmail.com with ESMTPSA id 62sm6663155otq.34.2017.03.12.05.30.52 (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Sun, 12 Mar 2017 05:30:52 -0700 (PDT) From: Bill Fischofer To: lng-odp@lists.linaro.org Date: Sun, 12 Mar 2017 07:30:37 -0500 Message-Id: <20170312123045.3852-4-bill.fischofer@linaro.org> X-Mailer: git-send-email 2.12.0.rc1 In-Reply-To: <20170312123045.3852-1-bill.fischofer@linaro.org> References: <20170312123045.3852-1-bill.fischofer@linaro.org> Subject: [lng-odp] [PATCHv2 03/11] linux-generic: packet: restructure inline routines to use macros X-BeenThere: lng-odp@lists.linaro.org X-Mailman-Version: 2.1.16 Precedence: list List-Id: "The OpenDataPlane \(ODP\) List" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: lng-odp-bounces@lists.linaro.org Sender: "lng-odp" Use an internal macro for accessing inline fields in preparation for introducing packet reference support. Signed-off-by: Bill Fischofer --- .../include/odp/api/plat/packet_flag_inlines.h | 7 +++- .../include/odp/api/plat/packet_inlines.h | 44 +++++++++------------- 2 files changed, 23 insertions(+), 28 deletions(-) -- 2.12.0.rc1 diff --git a/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h index 2de04c81..738fced3 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_flag_inlines.h @@ -16,14 +16,17 @@ #include #include +/** @internal Packet field accessor */ +#define _odp_pkt_get(pkt, cast, field) \ + (*(cast *)(uintptr_t)((uint8_t *)pkt + _odp_packet_inline.field)) + /** @internal Inline function offsets */ extern const _odp_packet_inline_offset_t _odp_packet_inline; /** @internal Inline function @param pkt @return */ static inline uint64_t _odp_packet_input_flags(odp_packet_t pkt) { - return *(uint64_t *)(uintptr_t)((uint8_t *)pkt + - _odp_packet_inline.input_flags); + return _odp_pkt_get(pkt, uint64_t, input_flags); } /** @internal Inline function @param pkt @return */ diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index eb36aa96..02295b08 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -18,97 +18,89 @@ #include #include +/** @internal Packet field accessor */ +#define _odp_pkt_get(pkt, cast, field) \ + (*(cast *)(uintptr_t)((uint8_t *)pkt + _odp_packet_inline.field)) + /** @internal Inline function offsets */ extern const _odp_packet_inline_offset_t _odp_packet_inline; /** @internal Inline function @param pkt @return */ static inline void *_odp_packet_data(odp_packet_t pkt) { - return *(void **)(uintptr_t)((uint8_t *)pkt + _odp_packet_inline.data); + return _odp_pkt_get(pkt, void *, data); } /** @internal Inline function @param pkt @return */ static inline uint32_t _odp_packet_seg_len(odp_packet_t pkt) { - return *(uint32_t *)(uintptr_t)((uint8_t *)pkt + - _odp_packet_inline.seg_len); + return _odp_pkt_get(pkt, uint32_t, seg_len); } /** @internal Inline function @param pkt @return */ static inline uint32_t _odp_packet_len(odp_packet_t pkt) { - return *(uint32_t *)(uintptr_t)((uint8_t *)pkt + - _odp_packet_inline.frame_len); + return _odp_pkt_get(pkt, uint32_t, frame_len); } /** @internal Inline function @param pkt @return */ static inline uint32_t _odp_packet_headroom(odp_packet_t pkt) { - return *(uint32_t *)(uintptr_t)((uint8_t *)pkt + - _odp_packet_inline.headroom); + return _odp_pkt_get(pkt, uint32_t, headroom); } /** @internal Inline function @param pkt @return */ static inline uint32_t _odp_packet_tailroom(odp_packet_t pkt) { - return *(uint32_t *)(uintptr_t)((uint8_t *)pkt + - _odp_packet_inline.tailroom); + return _odp_pkt_get(pkt, uint32_t, tailroom); } /** @internal Inline function @param pkt @return */ static inline odp_pool_t _odp_packet_pool(odp_packet_t pkt) { - return *(odp_pool_t *)(uintptr_t)((uint8_t *)pkt + - _odp_packet_inline.pool); + return _odp_pkt_get(pkt, odp_pool_t, pool); } /** @internal Inline function @param pkt @return */ static inline odp_pktio_t _odp_packet_input(odp_packet_t pkt) { - return *(odp_pktio_t *)(uintptr_t)((uint8_t *)pkt + - _odp_packet_inline.input); + return _odp_pkt_get(pkt, odp_pktio_t, input); } /** @internal Inline function @param pkt @return */ static inline int _odp_packet_num_segs(odp_packet_t pkt) { - return *(uint8_t *)(uintptr_t)((uint8_t *)pkt + - _odp_packet_inline.segcount); + return _odp_pkt_get(pkt, uint8_t, segcount); } /** @internal Inline function @param pkt @return */ static inline void *_odp_packet_user_ptr(odp_packet_t pkt) { - return *(void **)(uintptr_t)((uint8_t *)pkt + - _odp_packet_inline.user_ptr); + return _odp_pkt_get(pkt, void *, user_ptr); } /** @internal Inline function @param pkt @return */ static inline void *_odp_packet_user_area(odp_packet_t pkt) { - return *(void **)(uintptr_t)((uint8_t *)pkt + - _odp_packet_inline.user_area); + return _odp_pkt_get(pkt, void *, user_area); } /** @internal Inline function @param pkt @return */ static inline uint32_t _odp_packet_user_area_size(odp_packet_t pkt) { - return *(uint32_t *)(uintptr_t)((uint8_t *)pkt + - _odp_packet_inline.user_area_size); + return _odp_pkt_get(pkt, uint32_t, user_area_size); } /** @internal Inline function @param pkt @return */ static inline uint32_t _odp_packet_flow_hash(odp_packet_t pkt) { - return *(uint32_t *)(uintptr_t)((uint8_t *)pkt + - _odp_packet_inline.flow_hash); + return _odp_pkt_get(pkt, uint32_t, flow_hash); } /** @internal Inline function @param pkt @return */ static inline odp_time_t _odp_packet_ts(odp_packet_t pkt) { - return *(odp_time_t *)(uintptr_t)((uint8_t *)pkt + - _odp_packet_inline.timestamp); + return _odp_pkt_get(pkt, odp_time_t, timestamp); } /** @internal Inline function @param pkt @return */ @@ -120,7 +112,7 @@ static inline void *_odp_packet_head(odp_packet_t pkt) /** @internal Inline function @param pkt @return */ static inline int _odp_packet_is_segmented(odp_packet_t pkt) { - return _odp_packet_num_segs(pkt) > 1; + return _odp_pkt_get(pkt, uint8_t, segcount) > 1; } /** @internal Inline function @param pkt @return */ From patchwork Sun Mar 12 12:30:38 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Bill Fischofer X-Patchwork-Id: 95171 Delivered-To: patch@linaro.org Received: by 10.140.82.71 with SMTP id g65csp783723qgd; Sun, 12 Mar 2017 05:33:27 -0700 (PDT) X-Received: by 10.55.47.69 with SMTP id v66mr26726333qkh.222.1489322007502; Sun, 12 Mar 2017 05:33:27 -0700 (PDT) Return-Path: Received: from lists.linaro.org (lists.linaro.org. [54.225.227.206]) by mx.google.com with ESMTP id e49si12324938qta.160.2017.03.12.05.33.27; Sun, 12 Mar 2017 05:33:27 -0700 (PDT) Received-SPF: pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) client-ip=54.225.227.206; Authentication-Results: mx.google.com; spf=pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) smtp.mailfrom=lng-odp-bounces@lists.linaro.org; dmarc=pass (p=NONE sp=NONE dis=NONE) header.from=linaro.org Received: by lists.linaro.org (Postfix, from userid 109) id 24BE6607FD; Sun, 12 Mar 2017 12:33:27 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on ip-10-142-244-252 X-Spam-Level: X-Spam-Status: No, score=-1.9 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_NONE, RCVD_IN_MSPIKE_H2, URIBL_BLOCKED autolearn=disabled version=3.4.0 Received: from [127.0.0.1] (localhost [127.0.0.1]) by lists.linaro.org (Postfix) with ESMTP id 96F3563CA1; Sun, 12 Mar 2017 12:31:09 +0000 (UTC) X-Original-To: lng-odp@lists.linaro.org Delivered-To: lng-odp@lists.linaro.org Received: by lists.linaro.org (Postfix, from userid 109) id C604363C97; Sun, 12 Mar 2017 12:31:03 +0000 (UTC) Received: from mail-oi0-f47.google.com (mail-oi0-f47.google.com [209.85.218.47]) by lists.linaro.org (Postfix) with ESMTPS id 231DF63C8B for ; Sun, 12 Mar 2017 12:30:55 +0000 (UTC) Received: by mail-oi0-f47.google.com with SMTP id 126so68150290oig.3 for ; Sun, 12 Mar 2017 05:30:55 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references; bh=M3ZVf3LUEvWMt6ZHLAebsC4/+VyV3y4YE4Fzs5aSLCY=; b=NuL6EVxXJKnXCSCLv9Ylz76+JVfIM6A335puRBV2lJqsn/zdnylXA0KZGgI5pbHgOP KYgw2P7iVFppOAlJpaO3KhCb356YBulVbS3UhMnjdceO3D7ra2wfJRkI/fJ9o8jF6k+5 kvknFddk8yVNsvTYTckiB9pFTcxzYoM47M9xZFk+CCO35DU0O2pQyaK/hJAct7Po71wg o2/bbtsd7UU3XeLa4MDYp1S9jNPfICAVRmCrWEtJmDTbzD3xoaO8LFA5zir0uToFFeL5 02jbrc8z1A9n8uHMhD/iN1SWNhacscnakIPpHYIfm+9HcV6PATp/7eOqQWCt51zmc6A4 mT/Q== X-Gm-Message-State: AMke39mNWTzSq6IRs7dhfn2AxZzPLlI5hcDPWtYQSrB0+S04vveXjV1kTINJ6nb51uAOgeXSUVI= X-Received: by 10.202.182.198 with SMTP id g189mr15398285oif.180.1489321854481; Sun, 12 Mar 2017 05:30:54 -0700 (PDT) Received: from localhost.localdomain (cpe-70-121-83-241.austin.res.rr.com. [70.121.83.241]) by smtp.gmail.com with ESMTPSA id 62sm6663155otq.34.2017.03.12.05.30.53 (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Sun, 12 Mar 2017 05:30:53 -0700 (PDT) From: Bill Fischofer To: lng-odp@lists.linaro.org Date: Sun, 12 Mar 2017 07:30:38 -0500 Message-Id: <20170312123045.3852-5-bill.fischofer@linaro.org> X-Mailer: git-send-email 2.12.0.rc1 In-Reply-To: <20170312123045.3852-1-bill.fischofer@linaro.org> References: <20170312123045.3852-1-bill.fischofer@linaro.org> Subject: [lng-odp] [PATCHv2 04/11] linux-generic: packet: add packet fields needed for references X-BeenThere: lng-odp@lists.linaro.org X-Mailman-Version: 2.1.16 Precedence: list List-Id: "The OpenDataPlane \(ODP\) List" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: lng-odp-bounces@lists.linaro.org Sender: "lng-odp" Add new fields to the odp_packet_hdr_t used for support of zero-copy references. Also added are the offsets needed to access them from inlined functions. Signed-off-by: Bill Fischofer --- platform/linux-generic/include/odp/api/plat/packet_types.h | 8 ++++++++ platform/linux-generic/include/odp_packet_internal.h | 9 ++++++++- platform/linux-generic/odp_packet.c | 4 ++++ 3 files changed, 20 insertions(+), 1 deletion(-) -- 2.12.0.rc1 diff --git a/platform/linux-generic/include/odp/api/plat/packet_types.h b/platform/linux-generic/include/odp/api/plat/packet_types.h index b8f665d2..16e4b467 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_types.h +++ b/platform/linux-generic/include/odp/api/plat/packet_types.h @@ -70,6 +70,14 @@ typedef struct _odp_packet_inline_offset_t { /** @internal field offset */ size_t tailroom; /** @internal field offset */ + size_t unshared_len; + /** @internal field offset */ + size_t ref_hdr; + /** @internal field offset */ + size_t ref_offset; + /** *internal field offset */ + size_t ref_len; + /** @internal field offset */ size_t pool; /** @internal field offset */ size_t input; diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 589e204a..31d46df4 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -119,7 +119,7 @@ typedef struct { * packet_init(). Because of this any new fields added must be reviewed for * initialization requirements. */ -typedef struct { +typedef struct odp_packet_hdr_t { /* common buffer header */ odp_buffer_hdr_t buf_hdr; @@ -135,6 +135,13 @@ typedef struct { uint32_t headroom; uint32_t tailroom; + /* Fields used to support packet references */ + uint32_t unshared_len; + struct odp_packet_hdr_t *ref_hdr; + uint32_t ref_offset; + uint32_t ref_len; + odp_atomic_u32_t ref_count; + /* * Members below are not initialized by packet_init() */ diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index d0f76332..067d0fcb 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -31,6 +31,10 @@ const _odp_packet_inline_offset_t _odp_packet_inline ODP_ALIGNED_CACHE = { .frame_len = offsetof(odp_packet_hdr_t, frame_len), .headroom = offsetof(odp_packet_hdr_t, headroom), .tailroom = offsetof(odp_packet_hdr_t, tailroom), + .unshared_len = offsetof(odp_packet_hdr_t, unshared_len), + .ref_hdr = offsetof(odp_packet_hdr_t, ref_hdr), + .ref_offset = offsetof(odp_packet_hdr_t, ref_offset), + .ref_len = offsetof(odp_packet_hdr_t, ref_len), .pool = offsetof(odp_packet_hdr_t, buf_hdr.pool_hdl), .input = offsetof(odp_packet_hdr_t, input), .segcount = offsetof(odp_packet_hdr_t, buf_hdr.segcount), From patchwork Sun Mar 12 12:30:39 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Bill Fischofer X-Patchwork-Id: 95174 Delivered-To: patch@linaro.org Received: by 10.140.82.71 with SMTP id g65csp784310qgd; Sun, 12 Mar 2017 05:35:37 -0700 (PDT) X-Received: by 10.55.221.1 with SMTP id n1mr26138657qki.221.1489322137108; Sun, 12 Mar 2017 05:35:37 -0700 (PDT) Return-Path: Received: from lists.linaro.org (lists.linaro.org. [54.225.227.206]) by mx.google.com with ESMTP id o36si12316424qtb.332.2017.03.12.05.35.36; Sun, 12 Mar 2017 05:35:37 -0700 (PDT) Received-SPF: pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) client-ip=54.225.227.206; Authentication-Results: mx.google.com; spf=pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) smtp.mailfrom=lng-odp-bounces@lists.linaro.org; dmarc=pass (p=NONE sp=NONE dis=NONE) header.from=linaro.org Received: by lists.linaro.org (Postfix, from userid 109) id B4C1A60945; Sun, 12 Mar 2017 12:35:36 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on ip-10-142-244-252 X-Spam-Level: X-Spam-Status: No, score=-1.9 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_NONE, RCVD_IN_MSPIKE_H2, URIBL_BLOCKED autolearn=disabled version=3.4.0 Received: from [127.0.0.1] (localhost [127.0.0.1]) by lists.linaro.org (Postfix) with ESMTP id 5547963C8C; Sun, 12 Mar 2017 12:31:43 +0000 (UTC) X-Original-To: lng-odp@lists.linaro.org Delivered-To: lng-odp@lists.linaro.org Received: by lists.linaro.org (Postfix, from userid 109) id 44BB26091D; Sun, 12 Mar 2017 12:31:33 +0000 (UTC) Received: from mail-oi0-f44.google.com (mail-oi0-f44.google.com [209.85.218.44]) by lists.linaro.org (Postfix) with ESMTPS id 0EA0162D0B for ; Sun, 12 Mar 2017 12:30:56 +0000 (UTC) Received: by mail-oi0-f44.google.com with SMTP id 126so68150358oig.3 for ; Sun, 12 Mar 2017 05:30:56 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references; bh=fXTKoNdMf2kjZp23/ZCUVwKNOFuRpYUnZV+Yuz9C6qc=; b=PE/NEYnpdri/UxdkZ8Lf4puu7VCP52Cf+Utv+bP9xKyAhozS/U2AGENoxcC4wDYhTy 77YAj/H4UzM+BBAf2vFoPMJ5x/tGsKBnWaIoxgUir6ZXeLeKa5lNv/pX7tJXGI4yroDf 8U1mJlX7U0StgL13D9ToNsRlKyOc80DpbJ1FZX3g0DXcmpXnsY32v/GIbl439yU53pmm WsZDgJqvylQWd7XFnHBjtnnl/NqJyLQaOwfcbcdAbFOMuxSWbj3Cv8ITQQtW19OaHKtx f/RghJTE40qZL+ygpWexcfC8W0Eb5RukYdHcEdrf+NIfrUMZMzCYjPHaWFzF1bClw8kA RZAQ== X-Gm-Message-State: AMke39lC8EOSdxaK0IVVSwIT/Lco51pTfi/dVJy2E+eZJ7pbnVde5Pa3ndiQtsNjQZTSfdZ6lNncGA== X-Received: by 10.202.104.84 with SMTP id d81mr15325655oic.73.1489321855310; Sun, 12 Mar 2017 05:30:55 -0700 (PDT) Received: from localhost.localdomain (cpe-70-121-83-241.austin.res.rr.com. [70.121.83.241]) by smtp.gmail.com with ESMTPSA id 62sm6663155otq.34.2017.03.12.05.30.54 (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Sun, 12 Mar 2017 05:30:54 -0700 (PDT) From: Bill Fischofer To: lng-odp@lists.linaro.org Date: Sun, 12 Mar 2017 07:30:39 -0500 Message-Id: <20170312123045.3852-6-bill.fischofer@linaro.org> X-Mailer: git-send-email 2.12.0.rc1 In-Reply-To: <20170312123045.3852-1-bill.fischofer@linaro.org> References: <20170312123045.3852-1-bill.fischofer@linaro.org> Subject: [lng-odp] [PATCHv2 05/11] linux-generic: packet: add routines for manipulating reference counts X-BeenThere: lng-odp@lists.linaro.org X-Mailman-Version: 2.1.16 Precedence: list List-Id: "The OpenDataPlane \(ODP\) List" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: lng-odp-bounces@lists.linaro.org Sender: "lng-odp" Signed-off-by: Bill Fischofer --- platform/linux-generic/include/odp_packet_internal.h | 19 +++++++++++++++++++ platform/linux-generic/odp_packet.c | 10 ++++++++++ 2 files changed, 29 insertions(+) -- 2.12.0.rc1 diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 31d46df4..39deae20 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -196,6 +196,25 @@ static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr) return pkt_hdr->frame_len; } +static inline uint32_t packet_ref_count(odp_packet_hdr_t *pkt_hdr) +{ + /* Breach the atomic type to do a peek at the ref count. This + * is used to bypass atomic operations if ref_count == 1 for + * performance reasons. + */ + return pkt_hdr->ref_count.v; +} + +static inline void packet_ref_count_set(odp_packet_hdr_t *pkt_hdr, uint32_t n) +{ + /* Only used during init when there are no other possible + * references to this pkt, so avoid the "atomic" overhead by + * a controlled breach of the atomic type here. This saves + * over 10% of the pathlength in routines like packet_alloc(). + */ + pkt_hdr->ref_count.v = n; +} + static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len) { pkt_hdr->frame_len = len; diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 067d0fcb..4bbb6199 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -57,6 +57,16 @@ static inline odp_buffer_t buffer_handle(odp_packet_hdr_t *pkt_hdr) return pkt_hdr->buf_hdr.handle.handle; } +static inline uint32_t packet_ref_inc(odp_packet_hdr_t *pkt_hdr) +{ + return odp_atomic_fetch_inc_u32(&pkt_hdr->ref_count); +} + +static inline uint32_t packet_ref_dec(odp_packet_hdr_t *pkt_hdr) +{ + return odp_atomic_fetch_dec_u32(&pkt_hdr->ref_count); +} + static inline odp_packet_hdr_t *buf_to_packet_hdr(odp_buffer_t buf) { return (odp_packet_hdr_t *)buf_hdl_to_hdr(buf); From patchwork Sun Mar 12 12:30:40 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Bill Fischofer X-Patchwork-Id: 95172 Delivered-To: patch@linaro.org Received: by 10.140.82.71 with SMTP id g65csp783983qgd; Sun, 12 Mar 2017 05:34:26 -0700 (PDT) X-Received: by 10.200.44.235 with SMTP id 40mr28070667qtx.178.1489322066925; Sun, 12 Mar 2017 05:34:26 -0700 (PDT) Return-Path: Received: from lists.linaro.org (lists.linaro.org. [54.225.227.206]) by mx.google.com with ESMTP id m72si12330591qki.161.2017.03.12.05.34.26; Sun, 12 Mar 2017 05:34:26 -0700 (PDT) Received-SPF: pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) client-ip=54.225.227.206; Authentication-Results: mx.google.com; spf=pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) smtp.mailfrom=lng-odp-bounces@lists.linaro.org; dmarc=pass (p=NONE sp=NONE dis=NONE) header.from=linaro.org Received: by lists.linaro.org (Postfix, from userid 109) id 98AA8609A4; Sun, 12 Mar 2017 12:34:26 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on ip-10-142-244-252 X-Spam-Level: X-Spam-Status: No, score=-1.9 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_NONE, RCVD_IN_MSPIKE_H3, RCVD_IN_MSPIKE_WL, URIBL_BLOCKED autolearn=disabled version=3.4.0 Received: from [127.0.0.1] (localhost [127.0.0.1]) by lists.linaro.org (Postfix) with ESMTP id A523560945; Sun, 12 Mar 2017 12:31:36 +0000 (UTC) X-Original-To: lng-odp@lists.linaro.org Delivered-To: lng-odp@lists.linaro.org Received: by lists.linaro.org (Postfix, from userid 109) id E19D160945; Sun, 12 Mar 2017 12:31:31 +0000 (UTC) Received: from mail-oi0-f49.google.com (mail-oi0-f49.google.com [209.85.218.49]) by lists.linaro.org (Postfix) with ESMTPS id B066663C8D for ; Sun, 12 Mar 2017 12:30:56 +0000 (UTC) Received: by mail-oi0-f49.google.com with SMTP id 126so68150404oig.3 for ; Sun, 12 Mar 2017 05:30:56 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references; bh=SeaVa3wAev1oE0//8+ahpG52SBajO9EDZ+8Yx3Uinio=; b=hshIfm4MRTuivJC5vZKosnwIMp0Ms6frce5bSPpA41y0SdbuqdJE0tdS6vfrxrJAyP eytUSbzl5O5sTcgApIBm7wv3lXRn0pKtW0M9QJ41ckTCGPFGiO3mE3QJRe82+ipqQbSm qsHG0GyVzhWnrDvnfOcWekuC3e9wIrRyYc4KlgEpwXvKXhaDJC80ICYRORp5QeZDZ/V3 DSHTsUmWLTAEPNQG5n7NPemHIHLRGEAnAeIOrO+qgz8p8zTc3+nvLyZQ8GqilPWT1pvW /BuAeJkKIPox+GDI+m5ErNfZ3+WOu5wU+oB60LCw8p2xbD7DabRgEzFy46MudeC/wmeZ x5LQ== X-Gm-Message-State: AMke39l2QE7UcB0HOMUk3UIW2aHIJARJXwDODzCpLPnrfwqwuLcvXWi3Po+U7guTsVRQ+8LB57o= X-Received: by 10.202.89.139 with SMTP id n133mr13606585oib.70.1489321856210; Sun, 12 Mar 2017 05:30:56 -0700 (PDT) Received: from localhost.localdomain (cpe-70-121-83-241.austin.res.rr.com. [70.121.83.241]) by smtp.gmail.com with ESMTPSA id 62sm6663155otq.34.2017.03.12.05.30.55 (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Sun, 12 Mar 2017 05:30:55 -0700 (PDT) From: Bill Fischofer To: lng-odp@lists.linaro.org Date: Sun, 12 Mar 2017 07:30:40 -0500 Message-Id: <20170312123045.3852-7-bill.fischofer@linaro.org> X-Mailer: git-send-email 2.12.0.rc1 In-Reply-To: <20170312123045.3852-1-bill.fischofer@linaro.org> References: <20170312123045.3852-1-bill.fischofer@linaro.org> Subject: [lng-odp] [PATCHv2 06/11] linux-generic: packet: add initializations for new hdr fields X-BeenThere: lng-odp@lists.linaro.org X-Mailman-Version: 2.1.16 Precedence: list List-Id: "The OpenDataPlane \(ODP\) List" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: lng-odp-bounces@lists.linaro.org Sender: "lng-odp" Add initializations for new packet hdr fields needed to support packet references. Signed-off-by: Bill Fischofer --- platform/linux-generic/odp_packet.c | 9 +++++++++ 1 file changed, 9 insertions(+) -- 2.12.0.rc1 diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 4bbb6199..16b482d3 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -255,6 +255,9 @@ void packet_parse_reset(odp_packet_hdr_t *pkt_hdr) pkt_hdr->p.l2_offset = 0; pkt_hdr->p.l3_offset = ODP_PACKET_OFFSET_INVALID; pkt_hdr->p.l4_offset = ODP_PACKET_OFFSET_INVALID; + + /* Ensure dummy pkt_hdrs used in I/O recv classification are valid */ + pkt_hdr->ref_hdr = NULL; } /** @@ -300,6 +303,10 @@ static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len, CONFIG_PACKET_TAILROOM; pkt_hdr->input = ODP_PKTIO_INVALID; + + /* By default packet has no references */ + pkt_hdr->unshared_len = len; + pkt_hdr->ref_hdr = NULL; } static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) @@ -312,6 +319,7 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) hdr->buf_hdr.seg[0].data = hdr->buf_hdr.base_data; hdr->buf_hdr.seg[0].len = BASE_LEN; + packet_ref_count_set(hdr, 1); /* Link segments */ if (CONFIG_PACKET_MAX_SEGS != 1) { @@ -321,6 +329,7 @@ static inline void init_segments(odp_packet_hdr_t *pkt_hdr[], int num) for (i = 1; i < num; i++) { odp_buffer_hdr_t *buf_hdr; + packet_ref_count_set(pkt_hdr[i], 1); buf_hdr = &pkt_hdr[i]->buf_hdr; hdr->buf_hdr.seg[i].hdr = buf_hdr; hdr->buf_hdr.seg[i].data = buf_hdr->base_data; From patchwork Sun Mar 12 12:30:41 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Bill Fischofer X-Patchwork-Id: 95173 Delivered-To: patch@linaro.org Received: by 10.140.82.71 with SMTP id g65csp784123qgd; Sun, 12 Mar 2017 05:34:58 -0700 (PDT) X-Received: by 10.237.45.194 with SMTP id i60mr31563229qtd.135.1489322098093; Sun, 12 Mar 2017 05:34:58 -0700 (PDT) Return-Path: Received: from lists.linaro.org (lists.linaro.org. [54.225.227.206]) by mx.google.com with ESMTP id w76si12339807qka.93.2017.03.12.05.34.57; Sun, 12 Mar 2017 05:34:58 -0700 (PDT) Received-SPF: pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) client-ip=54.225.227.206; Authentication-Results: mx.google.com; spf=pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) smtp.mailfrom=lng-odp-bounces@lists.linaro.org; dmarc=pass (p=NONE sp=NONE dis=NONE) header.from=linaro.org Received: by lists.linaro.org (Postfix, from userid 109) id C0D7760945; Sun, 12 Mar 2017 12:34:57 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on ip-10-142-244-252 X-Spam-Level: X-Spam-Status: No, score=-1.9 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_NONE, RCVD_IN_MSPIKE_H3, RCVD_IN_MSPIKE_WL, URIBL_BLOCKED autolearn=disabled version=3.4.0 Received: from [127.0.0.1] (localhost [127.0.0.1]) by lists.linaro.org (Postfix) with ESMTP id 01D30609B8; Sun, 12 Mar 2017 12:31:40 +0000 (UTC) X-Original-To: lng-odp@lists.linaro.org Delivered-To: lng-odp@lists.linaro.org Received: by lists.linaro.org (Postfix, from userid 109) id 777DB60945; Sun, 12 Mar 2017 12:31:32 +0000 (UTC) Received: from mail-oi0-f50.google.com (mail-oi0-f50.google.com [209.85.218.50]) by lists.linaro.org (Postfix) with ESMTPS id A72C963C8E for ; Sun, 12 Mar 2017 12:30:57 +0000 (UTC) Received: by mail-oi0-f50.google.com with SMTP id 2so67861986oif.0 for ; Sun, 12 Mar 2017 05:30:57 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references; bh=O1mOwTK2Ps4Ozns/Kofe38Ow+TvdMltyFPERh3RsU/I=; b=jDLfhsTdkPc6laqjgmRWPugfYHgCE/ngkLrzEhYj9XoFAnKpGY/9fFDaCrI4C+biSi OkOc6yM0+/RsJv97rBW/LUZ9cjVxFJuBAnSNSCOQ4Ols5+S2Vxp4OwSO0ztu7nBYchVy J5T6brZPr8YHDJqriQCrMT4Df/NVaxXeIMSukoqIHhW9kWvHklIQafLyQkUJeVGtIVW+ c9cRehhXhhu8qaMl72z/aXAAziFWrlRuAozu3xvJo4fIwDsVCuNP4u+Si9qYS+wmIpQl O1o2EjLuwOZZoPe/9YZSGiSfFU19zmkr89uhmvfVvJSRjUbRxw2b1qYJgL8/xUCiKYfu V44A== X-Gm-Message-State: AMke39nszX5qA12WDChzj0cju2q+lFCehRQFU/8dUFQHgKcbbaJ/rhDJNLBbS/Wfmu+z9V+x4fU= X-Received: by 10.202.85.193 with SMTP id j184mr13739246oib.199.1489321857069; Sun, 12 Mar 2017 05:30:57 -0700 (PDT) Received: from localhost.localdomain (cpe-70-121-83-241.austin.res.rr.com. [70.121.83.241]) by smtp.gmail.com with ESMTPSA id 62sm6663155otq.34.2017.03.12.05.30.56 (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Sun, 12 Mar 2017 05:30:56 -0700 (PDT) From: Bill Fischofer To: lng-odp@lists.linaro.org Date: Sun, 12 Mar 2017 07:30:41 -0500 Message-Id: <20170312123045.3852-8-bill.fischofer@linaro.org> X-Mailer: git-send-email 2.12.0.rc1 In-Reply-To: <20170312123045.3852-1-bill.fischofer@linaro.org> References: <20170312123045.3852-1-bill.fischofer@linaro.org> Subject: [lng-odp] [PATCHv2 07/11] linux-generic: packet: unshared_len maintenance X-BeenThere: lng-odp@lists.linaro.org X-Mailman-Version: 2.1.16 Precedence: list List-Id: "The OpenDataPlane \(ODP\) List" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: lng-odp-bounces@lists.linaro.org Sender: "lng-odp" Add code to initialize and maintain unshared_len packet metadata Signed-off-by: Bill Fischofer --- .../linux-generic/include/odp_packet_internal.h | 4 ++ platform/linux-generic/odp_packet.c | 43 +++++++++++++++++----- 2 files changed, 37 insertions(+), 10 deletions(-) -- 2.12.0.rc1 diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index 39deae20..b6afd581 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -19,6 +19,7 @@ extern "C" { #include #include +#include #include #include #include @@ -217,7 +218,10 @@ static inline void packet_ref_count_set(odp_packet_hdr_t *pkt_hdr, uint32_t n) static inline void packet_set_len(odp_packet_hdr_t *pkt_hdr, uint32_t len) { + ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); + pkt_hdr->frame_len = len; + pkt_hdr->unshared_len = len; } static inline int packet_parse_l2_not_done(packet_parser_t *prs) diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 16b482d3..d4910214 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -138,6 +138,7 @@ static inline void push_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) { pkt_hdr->headroom -= len; pkt_hdr->frame_len += len; + pkt_hdr->unshared_len += len; pkt_hdr->buf_hdr.seg[0].data -= len; pkt_hdr->buf_hdr.seg[0].len += len; } @@ -146,6 +147,7 @@ static inline void pull_head(odp_packet_hdr_t *pkt_hdr, uint32_t len) { pkt_hdr->headroom += len; pkt_hdr->frame_len -= len; + pkt_hdr->unshared_len -= len; pkt_hdr->buf_hdr.seg[0].data += len; pkt_hdr->buf_hdr.seg[0].len -= len; } @@ -156,6 +158,7 @@ static inline void push_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) pkt_hdr->tailroom -= len; pkt_hdr->frame_len += len; + pkt_hdr->unshared_len += len; pkt_hdr->buf_hdr.seg[last].len += len; } @@ -163,8 +166,11 @@ static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len) { int last = pkt_hdr->buf_hdr.segcount - 1; + ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); + pkt_hdr->tailroom += len; pkt_hdr->frame_len -= len; + pkt_hdr->unshared_len -= len; pkt_hdr->buf_hdr.seg[last].len -= len; } @@ -192,6 +198,10 @@ static inline void packet_seg_copy_md(odp_packet_hdr_t *dst, dst->buf_hdr.uarea_addr = src->buf_hdr.uarea_addr; dst->buf_hdr.uarea_size = src->buf_hdr.uarea_size; + /* reference related metadata */ + dst->ref_len = src->ref_len; + dst->unshared_len = src->unshared_len; + /* segmentation data is not copied: * buf_hdr.seg[] * buf_hdr.segcount @@ -433,9 +443,10 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, new_hdr->buf_hdr.seg[0].len = seg_len; packet_seg_copy_md(new_hdr, pkt_hdr); - new_hdr->frame_len = pkt_hdr->frame_len + len; - new_hdr->headroom = pool->headroom + offset; - new_hdr->tailroom = pkt_hdr->tailroom; + new_hdr->frame_len = pkt_hdr->frame_len + len; + new_hdr->unshared_len = pkt_hdr->unshared_len + len; + new_hdr->headroom = pool->headroom + offset; + new_hdr->tailroom = pkt_hdr->tailroom; pkt_hdr = new_hdr; } else { @@ -448,8 +459,9 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, last = packet_last_seg(pkt_hdr); pkt_hdr->buf_hdr.seg[last].len = seg_len; - pkt_hdr->frame_len += len; - pkt_hdr->tailroom = pool->tailroom + offset; + pkt_hdr->frame_len += len; + pkt_hdr->unshared_len += len; + pkt_hdr->tailroom = pool->tailroom + offset; } return pkt_hdr; @@ -487,9 +499,10 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, packet_seg_copy_md(new_hdr, pkt_hdr); /* Tailroom not changed */ - new_hdr->tailroom = pkt_hdr->tailroom; - new_hdr->headroom = seg_headroom(new_hdr, 0); - new_hdr->frame_len = pkt_hdr->frame_len - free_len; + new_hdr->tailroom = pkt_hdr->tailroom; + new_hdr->headroom = seg_headroom(new_hdr, 0); + new_hdr->frame_len = pkt_hdr->frame_len - free_len; + new_hdr->unshared_len = pkt_hdr->unshared_len - free_len; pull_head(new_hdr, pull_len); @@ -504,6 +517,7 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, * of the metadata. */ pkt_hdr->buf_hdr.segcount = num_remain; pkt_hdr->frame_len -= free_len; + pkt_hdr->unshared_len -= free_len; pkt_hdr->tailroom = seg_tailroom(pkt_hdr, num_remain - 1); pull_tail(pkt_hdr, pull_len); @@ -971,6 +985,7 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, pkt_hdr->buf_hdr.segcount = segs; pkt_hdr->frame_len = frame_len; + pkt_hdr->unshared_len = frame_len; pkt_hdr->headroom = offset + pool->headroom; pkt_hdr->tailroom = pool->tailroom; @@ -1012,6 +1027,8 @@ void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) if (len > pkt_hdr->frame_len) return NULL; + ODP_ASSERT(len <= pkt_hdr->unshared_len); + pull_head(pkt_hdr, len); return packet_data(pkt_hdr); } @@ -1387,6 +1404,8 @@ int odp_packet_add_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) if (offset > pktlen) return -1; + ODP_ASSERT(odp_packet_unshared_len(*pkt_ptr) >= offset); + newpkt = odp_packet_alloc(pkt_hdr->buf_hdr.pool_hdl, pktlen + len); if (newpkt == ODP_PACKET_INVALID) @@ -1506,8 +1525,9 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) add_all_segs(dst_hdr, src_hdr); - dst_hdr->frame_len = dst_len + src_len; - dst_hdr->tailroom = src_hdr->tailroom; + dst_hdr->frame_len = dst_len + src_len; + dst_hdr->unshared_len = dst_len + src_len; + dst_hdr->tailroom = src_hdr->tailroom; /* Data was not moved in memory */ return 0; @@ -1520,6 +1540,7 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) if (len >= pktlen || tail == NULL) return -1; + ODP_ASSERT(odp_packet_unshared_len(*pkt) >= len); *tail = odp_packet_copy_part(*pkt, len, pktlen - len, odp_packet_pool(*pkt)); @@ -1605,6 +1626,8 @@ int odp_packet_copy_from_mem(odp_packet_t pkt, uint32_t offset, if (offset + len > pkt_hdr->frame_len) return -1; + ODP_ASSERT(odp_packet_unshared_len(pkt) >= offset + len); + while (len > 0) { mapaddr = packet_map(pkt_hdr, offset, &seglen, NULL); cpylen = len > seglen ? seglen : len; From patchwork Sun Mar 12 12:30:42 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Bill Fischofer X-Patchwork-Id: 95175 Delivered-To: patch@linaro.org Received: by 10.140.82.71 with SMTP id g65csp784467qgd; Sun, 12 Mar 2017 05:36:08 -0700 (PDT) X-Received: by 10.200.38.39 with SMTP id u36mr30766976qtu.58.1489322168722; Sun, 12 Mar 2017 05:36:08 -0700 (PDT) Return-Path: Received: from lists.linaro.org (lists.linaro.org. [54.225.227.206]) by mx.google.com with ESMTP id b65si12317781qkf.245.2017.03.12.05.36.08; Sun, 12 Mar 2017 05:36:08 -0700 (PDT) Received-SPF: pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) client-ip=54.225.227.206; Authentication-Results: mx.google.com; spf=pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) smtp.mailfrom=lng-odp-bounces@lists.linaro.org; dmarc=pass (p=NONE sp=NONE dis=NONE) header.from=linaro.org Received: by lists.linaro.org (Postfix, from userid 109) id 5BF3B60945; Sun, 12 Mar 2017 12:36:08 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on ip-10-142-244-252 X-Spam-Level: X-Spam-Status: No, score=-1.9 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_NONE, RCVD_IN_MSPIKE_H3, RCVD_IN_MSPIKE_WL, URIBL_BLOCKED autolearn=disabled version=3.4.0 Received: from [127.0.0.1] (localhost [127.0.0.1]) by lists.linaro.org (Postfix) with ESMTP id A3A8063CB2; Sun, 12 Mar 2017 12:31:48 +0000 (UTC) X-Original-To: lng-odp@lists.linaro.org Delivered-To: lng-odp@lists.linaro.org Received: by lists.linaro.org (Postfix, from userid 109) id F09EE609AB; Sun, 12 Mar 2017 12:31:38 +0000 (UTC) Received: from mail-oi0-f54.google.com (mail-oi0-f54.google.com [209.85.218.54]) by lists.linaro.org (Postfix) with ESMTPS id 8CCD263C91 for ; Sun, 12 Mar 2017 12:30:58 +0000 (UTC) Received: by mail-oi0-f54.google.com with SMTP id 2so67862056oif.0 for ; Sun, 12 Mar 2017 05:30:58 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references; bh=a+H3eLAtVIH/f8jdo7iGFon2uNomQFLRb7w93kvLPKk=; b=cFMYh6yVBY1TmZ+LI5VSiMa5tQSjETB1rkewjkrer+TypAyYWFi12Nnvh3kFKQlbzq FWnMO1qjmU0u5bLIThIbNux35gvqkU+wwshsYnkvi96zdFuBbcyuGr9GtTuNNgb6asNe 2tT1aVDExMHyzqFAv8yR337Tg07cRjJ8lcyTiwxdBDyBnKrShchWPu4qB2PlGb1V8kEo oj4YWk7sB5eH9NkPJ5ja225k1P0T9QCRMdEcz4bmEU0vdlyHPpq36Bo331IFjpPYzBoS 1MOYHU0q+R/saQpOnA4XP1gNDQHEdrT27g8VwUaGGQE7kbc0HhqHAwXiZB70BsyUUVMU GiCQ== X-Gm-Message-State: AMke39mpG9FxAB0O8EBZZJtIJ1Ja+2HeVkPqGO22wUiMYT9tQR2VURWBjcBpoulKdezksIfnGfo= X-Received: by 10.202.183.139 with SMTP id h133mr13211130oif.183.1489321857836; Sun, 12 Mar 2017 05:30:57 -0700 (PDT) Received: from localhost.localdomain (cpe-70-121-83-241.austin.res.rr.com. [70.121.83.241]) by smtp.gmail.com with ESMTPSA id 62sm6663155otq.34.2017.03.12.05.30.57 (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Sun, 12 Mar 2017 05:30:57 -0700 (PDT) From: Bill Fischofer To: lng-odp@lists.linaro.org Date: Sun, 12 Mar 2017 07:30:42 -0500 Message-Id: <20170312123045.3852-9-bill.fischofer@linaro.org> X-Mailer: git-send-email 2.12.0.rc1 In-Reply-To: <20170312123045.3852-1-bill.fischofer@linaro.org> References: <20170312123045.3852-1-bill.fischofer@linaro.org> Subject: [lng-odp] [PATCHv2 08/11] linux-generic: packet: segment manipulation with references X-BeenThere: lng-odp@lists.linaro.org X-Mailman-Version: 2.1.16 Precedence: list List-Id: "The OpenDataPlane \(ODP\) List" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: lng-odp-bounces@lists.linaro.org Sender: "lng-odp" Add support for segment operation with references Signed-off-by: Bill Fischofer --- .../include/odp/api/plat/packet_inlines.h | 39 ++--- .../include/odp/api/plat/packet_inlines_api.h | 16 -- .../linux-generic/include/odp_packet_internal.h | 55 ++++++- platform/linux-generic/odp_packet.c | 161 ++++++++++++++++++--- 4 files changed, 206 insertions(+), 65 deletions(-) -- 2.12.0.rc1 diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 02295b08..8bfd85fe 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -40,7 +40,19 @@ static inline uint32_t _odp_packet_seg_len(odp_packet_t pkt) /** @internal Inline function @param pkt @return */ static inline uint32_t _odp_packet_len(odp_packet_t pkt) { - return _odp_pkt_get(pkt, uint32_t, frame_len); + uint32_t pkt_len = _odp_pkt_get(pkt, uint32_t, frame_len); + void *ref_nxt = _odp_pkt_get(pkt, void *, ref_hdr); + void *ref_pkt = (void *)pkt; + + while (ref_nxt) { + pkt_len += _odp_pkt_get(ref_pkt, uint32_t, ref_len) - + _odp_pkt_get(ref_pkt, uint32_t, ref_offset); + + ref_pkt = ref_nxt; + ref_nxt = _odp_pkt_get(ref_nxt, void *, ref_hdr); + } + + return pkt_len; } /** @internal Inline function @param pkt @return */ @@ -68,12 +80,6 @@ static inline odp_pktio_t _odp_packet_input(odp_packet_t pkt) } /** @internal Inline function @param pkt @return */ -static inline int _odp_packet_num_segs(odp_packet_t pkt) -{ - return _odp_pkt_get(pkt, uint8_t, segcount); -} - -/** @internal Inline function @param pkt @return */ static inline void *_odp_packet_user_ptr(odp_packet_t pkt) { return _odp_pkt_get(pkt, void *, user_ptr); @@ -112,7 +118,8 @@ static inline void *_odp_packet_head(odp_packet_t pkt) /** @internal Inline function @param pkt @return */ static inline int _odp_packet_is_segmented(odp_packet_t pkt) { - return _odp_pkt_get(pkt, uint8_t, segcount) > 1; + return _odp_pkt_get(pkt, uint8_t, segcount) > 1 || + _odp_pkt_get(pkt, void *, ref_hdr) != NULL; } /** @internal Inline function @param pkt @return */ @@ -123,22 +130,6 @@ static inline odp_packet_seg_t _odp_packet_first_seg(odp_packet_t pkt) return 0; } -/** @internal Inline function @param pkt @return */ -static inline odp_packet_seg_t _odp_packet_last_seg(odp_packet_t pkt) -{ - return _odp_packet_num_segs(pkt) - 1; -} - -/** @internal Inline function @param pkt @param seg @return */ -static inline odp_packet_seg_t _odp_packet_next_seg(odp_packet_t pkt, - odp_packet_seg_t seg) -{ - if (odp_unlikely(seg >= _odp_packet_last_seg(pkt))) - return ODP_PACKET_SEG_INVALID; - - return seg + 1; -} - /** @internal Inline function @param pkt @param offset @param len */ static inline void _odp_packet_prefetch(odp_packet_t pkt, uint32_t offset, uint32_t len) diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h b/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h index 233bc876..f818f820 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h @@ -48,11 +48,6 @@ _ODP_INLINE odp_pktio_t odp_packet_input(odp_packet_t pkt) return _odp_packet_input(pkt); } -_ODP_INLINE int odp_packet_num_segs(odp_packet_t pkt) -{ - return _odp_packet_num_segs(pkt); -} - _ODP_INLINE void *odp_packet_user_ptr(odp_packet_t pkt) { return _odp_packet_user_ptr(pkt); @@ -93,17 +88,6 @@ _ODP_INLINE odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt) return _odp_packet_first_seg(pkt); } -_ODP_INLINE odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) -{ - return _odp_packet_last_seg(pkt); -} - -_ODP_INLINE odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, - odp_packet_seg_t seg) -{ - return _odp_packet_next_seg(pkt, seg); -} - _ODP_INLINE void odp_packet_prefetch(odp_packet_t pkt, uint32_t offset, uint32_t len) { diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index b6afd581..c99e9843 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -171,6 +171,50 @@ static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt) return (odp_packet_hdr_t *)(uintptr_t)pkt; } +static inline odp_packet_hdr_t *packet_last_hdr(odp_packet_t pkt, + uint32_t *offset) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + odp_packet_hdr_t *prev_hdr = pkt_hdr; + uint32_t ref_offset = 0; + + while (pkt_hdr->ref_hdr) { + ref_offset = pkt_hdr->ref_offset; + prev_hdr = pkt_hdr; + pkt_hdr = pkt_hdr->ref_hdr; + } + + if (offset) { + if (prev_hdr != pkt_hdr) + ref_offset += pkt_hdr->frame_len - prev_hdr->ref_len; + *offset = ref_offset; + } + + return pkt_hdr; +} + +static inline odp_packet_hdr_t *packet_prev_hdr(odp_packet_hdr_t *pkt_hdr, + odp_packet_hdr_t *cur_hdr, + uint32_t *offset) +{ + uint32_t ref_offset = 0; + odp_packet_hdr_t *prev_hdr = pkt_hdr; + + while (pkt_hdr->ref_hdr != cur_hdr) { + ref_offset = pkt_hdr->ref_offset; + prev_hdr = pkt_hdr; + pkt_hdr = pkt_hdr->ref_hdr; + } + + if (offset) { + if (prev_hdr != pkt_hdr) + ref_offset += pkt_hdr->frame_len - prev_hdr->ref_len; + *offset = ref_offset; + } + + return pkt_hdr; +} + static inline odp_packet_t packet_handle(odp_packet_hdr_t *pkt_hdr) { return (odp_packet_t)pkt_hdr; @@ -194,7 +238,16 @@ static inline void copy_packet_cls_metadata(odp_packet_hdr_t *src_hdr, static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr) { - return pkt_hdr->frame_len; + uint32_t pkt_len = pkt_hdr->frame_len; + odp_packet_hdr_t *ref_hdr = pkt_hdr->ref_hdr; + + while (ref_hdr) { + pkt_len += (pkt_hdr->ref_len - pkt_hdr->ref_offset); + pkt_hdr = ref_hdr; + ref_hdr = ref_hdr->ref_hdr; + } + + return pkt_len; } static inline uint32_t packet_ref_count(odp_packet_hdr_t *pkt_hdr) diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index d4910214..c9cb3ef2 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -54,7 +54,7 @@ static inline odp_packet_hdr_t *packet_hdr(odp_packet_t pkt) static inline odp_buffer_t buffer_handle(odp_packet_hdr_t *pkt_hdr) { - return pkt_hdr->buf_hdr.handle.handle; + return odp_hdr_to_buf(&pkt_hdr->buf_hdr); } static inline uint32_t packet_ref_inc(odp_packet_hdr_t *pkt_hdr) @@ -78,7 +78,8 @@ static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, return pkt_hdr->buf_hdr.seg[seg_idx].len; } -static inline void *packet_seg_data(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) +static inline uint8_t *packet_seg_data(odp_packet_hdr_t *pkt_hdr, + uint32_t seg_idx) { return pkt_hdr->buf_hdr.seg[seg_idx].data; } @@ -216,7 +217,15 @@ static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, int seg = 0; int seg_count = pkt_hdr->buf_hdr.segcount; - if (odp_unlikely(offset >= pkt_hdr->frame_len)) + /* Special processing for references */ + while (offset >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { + offset -= (pkt_hdr->frame_len - pkt_hdr->ref_offset); + offset += (pkt_hdr->ref_hdr->frame_len - pkt_hdr->ref_len); + pkt_hdr = pkt_hdr->ref_hdr; + seg_count = pkt_hdr->buf_hdr.segcount; + } + + if (odp_unlikely(offset > pkt_hdr->frame_len)) return NULL; if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || seg_count == 1)) { @@ -719,13 +728,47 @@ odp_event_t odp_packet_to_event(odp_packet_t pkt) uint32_t odp_packet_buf_len(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t buf_len = 0; + + do { + buf_len += pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount; + pkt_hdr = pkt_hdr->ref_hdr; + } while (pkt_hdr); - return pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount; + return buf_len; } -void *odp_packet_tail(odp_packet_t pkt) +uint32_t odp_packet_unshared_len(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t pkt_len = 0, offset = 0; + + if (packet_ref_count(pkt_hdr) == 1) + pkt_hdr->unshared_len = pkt_hdr->frame_len; + + do { + if (packet_ref_count(pkt_hdr) > 1) { + if (offset == 0) + pkt_len += pkt_hdr->unshared_len; + break; + } + + pkt_len += pkt_hdr->frame_len - offset; + offset = pkt_hdr->ref_offset; + + if (pkt_hdr->ref_hdr) + offset += (pkt_hdr->ref_hdr->frame_len - + pkt_hdr->ref_len); + + pkt_hdr = pkt_hdr->ref_hdr; + } while (pkt_hdr); + + return pkt_len; +} + +void *odp_packet_tail(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = packet_last_hdr(pkt, NULL); return packet_tail(pkt_hdr); } @@ -1360,6 +1403,45 @@ void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp) pkt_hdr->p.input_flags.timestamp = 1; } +int odp_packet_num_segs(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t segcount = 0, i; + uint32_t seg_offset = 0, offset; + + do { + segcount += pkt_hdr->buf_hdr.segcount - seg_offset; + offset = pkt_hdr->ref_offset; + pkt_hdr = pkt_hdr->ref_hdr; + if (pkt_hdr) { + for (i = 0, seg_offset = 0; + i < pkt_hdr->buf_hdr.segcount; + i++, seg_offset++) { + if (offset < pkt_hdr->buf_hdr.seg[i].len) + break; + offset -= pkt_hdr->buf_hdr.seg[i].len; + } + } + } while (pkt_hdr); + + return segcount; +} + +odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) +{ + return (odp_packet_seg_t)(odp_packet_num_segs(pkt) - 1); +} + +odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + if (odp_unlikely(seg >= packet_last_seg(pkt_hdr))) + return ODP_PACKET_SEG_INVALID; + + return seg + 1; +} + /* * * Segment level @@ -1370,21 +1452,51 @@ void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp) void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t seg_offset = 0, offset = 0, i; + + while (seg >= pkt_hdr->buf_hdr.segcount - seg_offset && + pkt_hdr->ref_hdr) { + seg -= (pkt_hdr->buf_hdr.segcount - seg_offset); + offset = pkt_hdr->ref_offset; + pkt_hdr = pkt_hdr->ref_hdr; + for (i = 0, seg_offset = 0; + i < pkt_hdr->buf_hdr.segcount; + i++, seg_offset++) { + if (offset < pkt_hdr->buf_hdr.seg[i].len) + break; + offset -= pkt_hdr->buf_hdr.seg[i].len; + } + } - if (odp_unlikely(seg >= pkt_hdr->buf_hdr.segcount)) + if (odp_unlikely(seg + seg_offset >= pkt_hdr->buf_hdr.segcount)) return NULL; - return packet_seg_data(pkt_hdr, seg); + return packet_seg_data(pkt_hdr, seg + seg_offset) + offset; } uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t seg_offset = 0, offset = 0, i; + + while (seg >= pkt_hdr->buf_hdr.segcount - seg_offset && + pkt_hdr->ref_hdr) { + seg -= (pkt_hdr->buf_hdr.segcount - seg_offset); + offset = pkt_hdr->ref_offset; + pkt_hdr = pkt_hdr->ref_hdr; + for (i = 0, seg_offset = 0; + i < pkt_hdr->buf_hdr.segcount; + i++, seg_offset++) { + if (offset < pkt_hdr->buf_hdr.seg[i].len) + break; + offset -= pkt_hdr->buf_hdr.seg[i].len; + } + } - if (odp_unlikely(seg >= pkt_hdr->buf_hdr.segcount)) + if (odp_unlikely(seg + seg_offset >= pkt_hdr->buf_hdr.segcount)) return 0; - return packet_seg_len(pkt_hdr, seg); + return packet_seg_len(pkt_hdr, seg + seg_offset) - offset; } /* @@ -1398,7 +1510,7 @@ int odp_packet_add_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) { odp_packet_t pkt = *pkt_ptr; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t pktlen = pkt_hdr->frame_len; + uint32_t pktlen = packet_len(pkt_hdr); odp_packet_t newpkt; if (offset > pktlen) @@ -1468,6 +1580,8 @@ int odp_packet_align(odp_packet_t *pkt, uint32_t offset, uint32_t len, if (align > ODP_CACHE_LINE_SIZE) return -1; + ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); + if (seglen >= len) { misalign = align <= 1 ? 0 : ROUNDUP_ALIGN(uaddr, align) - uaddr; @@ -1507,10 +1621,13 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) uint32_t dst_len = dst_hdr->frame_len; uint32_t src_len = src_hdr->frame_len; + ODP_ASSERT(packet_ref_count(dst_hdr) == 1); + /* Do a copy if resulting packet would be out of segments or packets - * are from different pools. */ + * are from different pools or src is a reference. */ if (odp_unlikely((dst_segs + src_segs) > CONFIG_PACKET_MAX_SEGS) || - odp_unlikely(dst_pool != src_pool)) { + odp_unlikely(dst_pool != src_pool) || + odp_unlikely(packet_ref_count(src_hdr)) > 1) { if (odp_packet_extend_tail(dst, src_len, NULL, NULL) >= 0) { (void)odp_packet_copy_from_pkt(*dst, dst_len, src, 0, src_len); @@ -1559,8 +1676,7 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_pool_t pool) { - odp_packet_hdr_t *srchdr = packet_hdr(pkt); - uint32_t pktlen = srchdr->frame_len; + uint32_t pktlen = odp_packet_len(pkt); odp_packet_t newpkt = odp_packet_alloc(pool, pktlen); if (newpkt != ODP_PACKET_INVALID) { @@ -1599,7 +1715,7 @@ int odp_packet_copy_to_mem(odp_packet_t pkt, uint32_t offset, uint8_t *dstaddr = (uint8_t *)dst; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - if (offset + len > pkt_hdr->frame_len) + if (offset + len > packet_len(pkt_hdr)) return -1; while (len > 0) { @@ -1623,7 +1739,7 @@ int odp_packet_copy_from_mem(odp_packet_t pkt, uint32_t offset, const uint8_t *srcaddr = (const uint8_t *)src; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - if (offset + len > pkt_hdr->frame_len) + if (offset + len > packet_len(pkt_hdr)) return -1; ODP_ASSERT(odp_packet_unshared_len(pkt) >= offset + len); @@ -1653,10 +1769,12 @@ int odp_packet_copy_from_pkt(odp_packet_t dst, uint32_t dst_offset, uint32_t src_seglen = 0; /* GCC */ int overlap; - if (dst_offset + len > dst_hdr->frame_len || - src_offset + len > src_hdr->frame_len) + if (dst_offset + len > packet_len(dst_hdr) || + src_offset + len > packet_len(src_hdr)) return -1; + ODP_ASSERT(odp_packet_unshared_len(dst) >= dst_offset + len); + overlap = (dst_hdr == src_hdr && ((dst_offset <= src_offset && dst_offset + len >= src_offset) || @@ -1741,7 +1859,7 @@ void odp_packet_print(odp_packet_t pkt) len += snprintf(&str[len], n - len, " l4_offset %" PRIu32 "\n", hdr->p.l4_offset); len += snprintf(&str[len], n - len, - " frame_len %" PRIu32 "\n", hdr->frame_len); + " frame_len %" PRIu32 "\n", packet_len(hdr)); len += snprintf(&str[len], n - len, " input %" PRIu64 "\n", odp_pktio_to_u64(hdr->input)); @@ -2268,11 +2386,6 @@ int odp_packet_has_ref(odp_packet_t pkt) return 0; } -uint32_t odp_packet_unshared_len(odp_packet_t pkt) -{ - return odp_packet_len(pkt); -} - /* Include non-inlined versions of API functions */ #if ODP_ABI_COMPAT == 1 #include From patchwork Sun Mar 12 12:30:43 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Bill Fischofer X-Patchwork-Id: 95176 Delivered-To: patch@linaro.org Received: by 10.140.82.71 with SMTP id g65csp784916qgd; Sun, 12 Mar 2017 05:37:45 -0700 (PDT) X-Received: by 10.98.159.149 with SMTP id v21mr32238902pfk.57.1489322265394; Sun, 12 Mar 2017 05:37:45 -0700 (PDT) Return-Path: Received: from lists.linaro.org (lists.linaro.org. [54.225.227.206]) by mx.google.com with ESMTP id z5si8644454pgn.374.2017.03.12.05.37.44; Sun, 12 Mar 2017 05:37:45 -0700 (PDT) Received-SPF: pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) client-ip=54.225.227.206; Authentication-Results: mx.google.com; spf=pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) smtp.mailfrom=lng-odp-bounces@lists.linaro.org; dmarc=pass (p=NONE sp=NONE dis=NONE) header.from=linaro.org Received: by lists.linaro.org (Postfix, from userid 109) id 9F844607FD; Sun, 12 Mar 2017 12:37:44 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on ip-10-142-244-252 X-Spam-Level: X-Spam-Status: No, score=-1.9 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_NONE, RCVD_IN_MSPIKE_H2, URIBL_BLOCKED autolearn=disabled version=3.4.0 Received: from [127.0.0.1] (localhost [127.0.0.1]) by lists.linaro.org (Postfix) with ESMTP id 40A9E60A2E; Sun, 12 Mar 2017 12:32:03 +0000 (UTC) X-Original-To: lng-odp@lists.linaro.org Delivered-To: lng-odp@lists.linaro.org Received: by lists.linaro.org (Postfix, from userid 109) id 3B2C363CB3; Sun, 12 Mar 2017 12:31:58 +0000 (UTC) Received: from mail-oi0-f48.google.com (mail-oi0-f48.google.com [209.85.218.48]) by lists.linaro.org (Postfix) with ESMTPS id 6930963C94 for ; Sun, 12 Mar 2017 12:30:59 +0000 (UTC) Received: by mail-oi0-f48.google.com with SMTP id 2so67862123oif.0 for ; Sun, 12 Mar 2017 05:30:59 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references; bh=Eqy28sdBs4VzcJkTkeEgAv4AFOQjJa3f9n9DtwaJke4=; b=W2JrImksQABBaMFmk/MPSoPG/7RWDcFK440K+NtbPlHInK36SqyGskmkPDlP4n8+ZT Fzlyp8bQMKEBk+1NVHugBfoaubUc45u/ljTbL/BCtpbLOYY/hit/TsHc2JOsx2PBApxH 3+gx8x9yLQG5Gioku9h2YiMdN+rUJ9tS1GbooQKHjlTDsvHS/xMuJh3rLX91DoMmHQbl 4hLQSxjjYXEsoBYF5JHD+s9gywd+8QoTQhHTUsd6mPYJ5HNzvQGcz1TMh8X4TCbqD3M6 1R9UpaFA596ucUmH5I+NdSoamIwTUE763KXwI5U39GvOmCUEiw1NexPJL0WckO8nkHSl fgFg== X-Gm-Message-State: AMke39kg1QN/hrfAtjN9d5WIkCQHBYIfanksOCH2Y8WZtprdz4Iwp6fBGTqw9J146JEU/5x4ZThUdw== X-Received: by 10.202.76.215 with SMTP id z206mr14946664oia.68.1489321858575; Sun, 12 Mar 2017 05:30:58 -0700 (PDT) Received: from localhost.localdomain (cpe-70-121-83-241.austin.res.rr.com. [70.121.83.241]) by smtp.gmail.com with ESMTPSA id 62sm6663155otq.34.2017.03.12.05.30.57 (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Sun, 12 Mar 2017 05:30:58 -0700 (PDT) From: Bill Fischofer To: lng-odp@lists.linaro.org Date: Sun, 12 Mar 2017 07:30:43 -0500 Message-Id: <20170312123045.3852-10-bill.fischofer@linaro.org> X-Mailer: git-send-email 2.12.0.rc1 In-Reply-To: <20170312123045.3852-1-bill.fischofer@linaro.org> References: <20170312123045.3852-1-bill.fischofer@linaro.org> Subject: [lng-odp] [PATCHv2 09/11] linux-generic: packet: restructure free routines for references X-BeenThere: lng-odp@lists.linaro.org X-Mailman-Version: 2.1.16 Precedence: list List-Id: "The OpenDataPlane \(ODP\) List" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: lng-odp-bounces@lists.linaro.org Sender: "lng-odp" Restructure the various packet/segment free routines to support references. Signed-off-by: Bill Fischofer --- platform/linux-generic/odp_packet.c | 146 +++++++++++++++++++++++++++--------- 1 file changed, 111 insertions(+), 35 deletions(-) -- 2.12.0.rc1 diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index c9cb3ef2..9ddc8522 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -478,13 +478,18 @@ static inline odp_packet_hdr_t *add_segments(odp_packet_hdr_t *pkt_hdr, static inline void free_bufs(odp_packet_hdr_t *pkt_hdr, int first, int num) { - int i; + int i, nfree; odp_buffer_t buf[num]; - for (i = 0; i < num; i++) - buf[i] = buffer_handle(pkt_hdr->buf_hdr.seg[first + i].hdr); + for (i = 0, nfree = 0; i < num; i++) { + odp_packet_hdr_t *hdr = pkt_hdr->buf_hdr.seg[first + i].hdr; + + if (packet_ref_count(hdr) == 1 || packet_ref_dec(hdr) == 1) + buf[nfree++] = buffer_handle(hdr); + } - buffer_free_multi(buf, num); + if (nfree > 0) + buffer_free_multi(buf, nfree); } static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, @@ -495,11 +500,16 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, if (head) { odp_packet_hdr_t *new_hdr; - int i; + int i, nfree; odp_buffer_t buf[num]; - for (i = 0; i < num; i++) - buf[i] = buffer_handle(pkt_hdr->buf_hdr.seg[i].hdr); + for (i = 0, nfree = 0; i < num; i++) { + new_hdr = pkt_hdr->buf_hdr.seg[i].hdr; + + if (packet_ref_count(new_hdr) == 1 || + packet_ref_dec(new_hdr) == 1) + buf[nfree++] = buffer_handle(new_hdr); + } /* First remaining segment is the new packet descriptor */ new_hdr = pkt_hdr->buf_hdr.seg[num].hdr; @@ -517,7 +527,8 @@ static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr, pkt_hdr = new_hdr; - buffer_free_multi(buf, num); + if (nfree > 0) + buffer_free_multi(buf, nfree); } else { /* Free last 'num' bufs */ free_bufs(pkt_hdr, num_remain, num); @@ -630,47 +641,109 @@ int odp_packet_alloc_multi(odp_pool_t pool_hdl, uint32_t len, return num; } -void odp_packet_free(odp_packet_t pkt) +static inline odp_packet_hdr_t *packet_free_to_list(odp_packet_hdr_t *pkt_hdr, + odp_buffer_t buf[], + int nbufs, + int *nfree) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - odp_buffer_t hdl = buffer_handle(pkt_hdr); + odp_packet_hdr_t *ref_hdr; + uint32_t ref_count; + int num_seg, i; - int num_seg = pkt_hdr->buf_hdr.segcount; + do { + ref_count = packet_ref_count(pkt_hdr); + num_seg = pkt_hdr->buf_hdr.segcount; + ref_hdr = pkt_hdr->ref_hdr; - if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1)) - buffer_free_multi(&hdl, 1); - else - free_bufs(pkt_hdr, 0, num_seg); + if (odp_likely((CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1) && + ref_count == 1)) { + if (*nfree >= nbufs) + break; + + buf[(*nfree)++] = buffer_handle(pkt_hdr); + } else { + if (*nfree + num_seg >= nbufs) + break; + + for (i = 0; i < num_seg; i++) { + odp_packet_hdr_t *hdr = + pkt_hdr->buf_hdr.seg[i].hdr; + + if (packet_ref_count(hdr) == 1 || + packet_ref_dec(hdr) == 1) + buf[(*nfree)++] = buffer_handle(hdr); + } + } + + pkt_hdr = ref_hdr; + } while (pkt_hdr); + + return pkt_hdr; +} + +static inline void packet_free(odp_packet_hdr_t *pkt_hdr) +{ + odp_packet_hdr_t *ref_hdr; + uint32_t ref_count; + int num_seg; + + do { + ref_count = packet_ref_count(pkt_hdr); + num_seg = pkt_hdr->buf_hdr.segcount; + ref_hdr = pkt_hdr->ref_hdr; + + if (odp_likely((CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1) && + ref_count == 1)) + buffer_free_multi((odp_buffer_t *) + &pkt_hdr->buf_hdr.handle.handle, 1); + else + free_bufs(pkt_hdr, 0, num_seg); + + pkt_hdr = ref_hdr; + } while (pkt_hdr); +} + +void odp_packet_free(odp_packet_t pkt) +{ + packet_free(packet_hdr(pkt)); } void odp_packet_free_multi(const odp_packet_t pkt[], int num) { - if (CONFIG_PACKET_MAX_SEGS == 1) { - buffer_free_multi((const odp_buffer_t * const)pkt, num); - } else { - odp_buffer_t buf[num * CONFIG_PACKET_MAX_SEGS]; - int i, j; - int bufs = 0; + int nbufs = num * CONFIG_PACKET_MAX_SEGS * 2; + odp_buffer_t buf[nbufs]; + int nfree = 0; + int i = 0; + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt[0]); + uint32_t pool_id = pool_id_from_buf(pkt_hdr->buf_hdr.handle.handle); + uint32_t next_pool; + + while (1) { + pkt_hdr = packet_free_to_list(pkt_hdr, buf, nbufs, &nfree); - for (i = 0; i < num; i++) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt[i]); - int num_seg = pkt_hdr->buf_hdr.segcount; - odp_buffer_hdr_t *buf_hdr = &pkt_hdr->buf_hdr; + if (pkt_hdr) { + buffer_free_to_pool(pool_id, buf, nfree); + nfree = 0; + continue; + } - buf[bufs] = buffer_handle(pkt_hdr); - bufs++; + if (++i == num) + break; - if (odp_likely(num_seg == 1)) - continue; + pkt_hdr = packet_hdr(pkt[i]); + next_pool = pool_id_from_buf(pkt_hdr->buf_hdr.handle.handle); - for (j = 1; j < num_seg; j++) { - buf[bufs] = buffer_handle(buf_hdr->seg[j].hdr); - bufs++; + if (odp_unlikely(next_pool != pool_id)) { + if (nfree > 0) { + buffer_free_to_pool(pool_id, buf, nfree); + nfree = 0; } + pool_id = next_pool; } - - buffer_free_multi(buf, bufs); } + + if (nfree > 0) + buffer_free_to_pool(pool_id, buf, nfree); } int odp_packet_reset(odp_packet_t pkt, uint32_t len) @@ -681,6 +754,9 @@ int odp_packet_reset(odp_packet_t pkt, uint32_t len) if (len > pool->headroom + pool->data_size + pool->tailroom) return -1; + if (pkt_hdr->ref_hdr) + packet_free(pkt_hdr->ref_hdr); + packet_init(pkt_hdr, len, 0); return 0; From patchwork Sun Mar 12 12:30:44 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Bill Fischofer X-Patchwork-Id: 95177 Delivered-To: patch@linaro.org Received: by 10.140.82.71 with SMTP id g65csp785120qgd; Sun, 12 Mar 2017 05:38:32 -0700 (PDT) X-Received: by 10.237.53.231 with SMTP id d36mr27073519qte.240.1489322312796; Sun, 12 Mar 2017 05:38:32 -0700 (PDT) Return-Path: Received: from lists.linaro.org (lists.linaro.org. [54.225.227.206]) by mx.google.com with ESMTP id h184si12323939qkf.257.2017.03.12.05.38.32; Sun, 12 Mar 2017 05:38:32 -0700 (PDT) Received-SPF: pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) client-ip=54.225.227.206; Authentication-Results: mx.google.com; spf=pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) smtp.mailfrom=lng-odp-bounces@lists.linaro.org; dmarc=pass (p=NONE sp=NONE dis=NONE) header.from=linaro.org Received: by lists.linaro.org (Postfix, from userid 109) id 621FC607B6; Sun, 12 Mar 2017 12:38:32 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on ip-10-142-244-252 X-Spam-Level: X-Spam-Status: No, score=-1.9 required=5.0 tests=BAYES_00, RCVD_IN_MSPIKE_H3, RCVD_IN_MSPIKE_WL, URIBL_BLOCKED autolearn=disabled version=3.4.0 Received: from [127.0.0.1] (localhost [127.0.0.1]) by lists.linaro.org (Postfix) with ESMTP id 8916763CAF; Sun, 12 Mar 2017 12:32:06 +0000 (UTC) X-Original-To: lng-odp@lists.linaro.org Delivered-To: lng-odp@lists.linaro.org Received: by lists.linaro.org (Postfix, from userid 109) id 68DA963C94; Sun, 12 Mar 2017 12:31:58 +0000 (UTC) Received: from mail-oi0-f52.google.com (mail-oi0-f52.google.com [209.85.218.52]) by lists.linaro.org (Postfix) with ESMTPS id C3EC063703 for ; Sun, 12 Mar 2017 12:31:00 +0000 (UTC) Received: by mail-oi0-f52.google.com with SMTP id 62so68270757oih.2 for ; Sun, 12 Mar 2017 05:31:00 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references; bh=M+9vYBx8KbRT2VnWbBPXLCF1jufuZw52ixDacvZGamE=; b=PDlX+IylQMU0ty1POABBYiSSIT32Wjq9NF3q+n8kETs/i3j1Du4+N1pkxhQc1cp17I pAFXBM1E3gYciHkxXGZJnw8p/2BRcmiUIku9s8BrcB77HSkq1JCPlRd3xY1lddigZ4Cn hdZuLhp2mUNHHzOn5Edm+N6C1aV3+gqBTVEKHS5BLq9ac41qrFq8jlnnkZypx1aHhTlO H1QartEIDd/gQ0NHKi4p65kS0/Lr7tqFWAz9FmoK802ZQNuCUb14VDoTwsgnlo8Yv6aN s7MDl/G55Q+8xsQykSvLsNmWlAOY3XYFWeEvMop9lvNfEK4YjE8xGDmRYKQQ0WmsJHdd CwXQ== X-Gm-Message-State: AMke39npQhvera7QLBARGJYINLJM9cdjs1IaxV3xXP/hdFYhVidWs7hy4qo5fEBLACa9V99xjwqJbA== X-Received: by 10.202.81.203 with SMTP id f194mr12810683oib.120.1489321860060; Sun, 12 Mar 2017 05:31:00 -0700 (PDT) Received: from localhost.localdomain (cpe-70-121-83-241.austin.res.rr.com. [70.121.83.241]) by smtp.gmail.com with ESMTPSA id 62sm6663155otq.34.2017.03.12.05.30.58 (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Sun, 12 Mar 2017 05:30:58 -0700 (PDT) From: Bill Fischofer To: lng-odp@lists.linaro.org Date: Sun, 12 Mar 2017 07:30:44 -0500 Message-Id: <20170312123045.3852-11-bill.fischofer@linaro.org> X-Mailer: git-send-email 2.12.0.rc1 In-Reply-To: <20170312123045.3852-1-bill.fischofer@linaro.org> References: <20170312123045.3852-1-bill.fischofer@linaro.org> Subject: [lng-odp] [PATCHv2 10/11] linux-generic: packet: add reference support for head/tail manipulation X-BeenThere: lng-odp@lists.linaro.org X-Mailman-Version: 2.1.16 Precedence: list List-Id: "The OpenDataPlane \(ODP\) List" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: lng-odp-bounces@lists.linaro.org Sender: "lng-odp" Signed-off-by: Bill Fischofer --- platform/linux-generic/odp_packet.c | 96 +++++++++++++++++++++++++++++-------- 1 file changed, 77 insertions(+), 19 deletions(-) -- 2.12.0.rc1 diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 9ddc8522..403b21f1 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -1053,6 +1053,13 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, segs = pkt_hdr->buf_hdr.segcount; if (odp_unlikely((segs + num) > CONFIG_PACKET_MAX_SEGS)) { + /* Corner case: fail request if packet has + * references since we cannot shuffle segments + * since another thread may be accessing them + * concurrently */ + if (packet_ref_count(pkt_hdr) > 1) + return -1; + /* Cannot directly add new segments */ odp_packet_hdr_t *new_hdr; int new_segs = 0; @@ -1130,11 +1137,16 @@ int odp_packet_extend_head(odp_packet_t *pkt, uint32_t len, push_head(pkt_hdr, len); } - if (data_ptr) - *data_ptr = packet_data(pkt_hdr); + if (data_ptr || seg_len) { + uint32_t seg_ln = 0; + void *data = packet_map(pkt_hdr, 0, &seg_ln, NULL); - if (seg_len) - *seg_len = packet_first_seg_len(pkt_hdr); + if (data_ptr) + *data_ptr = data; + + if (seg_len) + *seg_len = seg_ln; + } return ret; } @@ -1155,15 +1167,35 @@ void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len) int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len_out) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); + odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt), *nxt_hdr; uint32_t seg_len = packet_first_seg_len(pkt_hdr); + int ret = 0; - if (len > pkt_hdr->frame_len) + if (len > packet_len(pkt_hdr)) return -1; - if (len < seg_len) { + ODP_ASSERT(len <= odp_packet_unshared_len(*pkt)); + + /* Special processing for references */ + while (len >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { + ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); + nxt_hdr = pkt_hdr->ref_hdr; + len -= pkt_hdr->frame_len; + len += pkt_hdr->ref_offset + + (nxt_hdr->frame_len - pkt_hdr->ref_len); + pkt_hdr->ref_hdr = NULL; + packet_free(pkt_hdr); + pkt_hdr = nxt_hdr; + seg_len = packet_first_seg_len(pkt_hdr); + *pkt = packet_handle(pkt_hdr); + ret = 1; + } + + if (CONFIG_PACKET_MAX_SEGS == 1 || + len < seg_len || + pkt_hdr->buf_hdr.segcount == 1) { pull_head(pkt_hdr, len); - } else if (CONFIG_PACKET_MAX_SEGS != 1) { + } else { int num = 0; uint32_t pull_len = 0; @@ -1178,23 +1210,29 @@ int odp_packet_trunc_head(odp_packet_t *pkt, uint32_t len, *pkt = packet_handle(pkt_hdr); } - if (data_ptr) - *data_ptr = packet_data(pkt_hdr); + if (data_ptr || seg_len_out) { + void *data_head = packet_map(pkt_hdr, 0, &seg_len, NULL); - if (seg_len_out) - *seg_len_out = packet_first_seg_len(pkt_hdr); + if (data_ptr) + *data_ptr = data_head; - return 0; + if (seg_len_out) + *seg_len_out = seg_len; + } + + return ret; } void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + odp_packet_hdr_t *pkt_hdr = packet_last_hdr(pkt, NULL); void *old_tail; if (len > pkt_hdr->tailroom) return NULL; + ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); + old_tail = packet_tail(pkt_hdr); push_tail(pkt_hdr, len); @@ -1204,12 +1242,14 @@ void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len) int odp_packet_extend_tail(odp_packet_t *pkt, uint32_t len, void **data_ptr, uint32_t *seg_len_out) { - odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); + odp_packet_hdr_t *pkt_hdr = packet_last_hdr(*pkt, NULL); uint32_t frame_len = pkt_hdr->frame_len; uint32_t tailroom = pkt_hdr->tailroom; uint32_t tail_off = frame_len; int ret = 0; + ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); + if (len > tailroom) { pool_t *pool = pool_entry_from_hdl(pkt_hdr->buf_hdr.pool_hdl); int num; @@ -1300,6 +1340,7 @@ void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len) if (len > packet_last_seg_len(pkt_hdr)) return NULL; + ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); pull_tail(pkt_hdr, len); return packet_tail(pkt_hdr); @@ -1310,17 +1351,34 @@ int odp_packet_trunc_tail(odp_packet_t *pkt, uint32_t len, { int last; uint32_t seg_len; - odp_packet_hdr_t *pkt_hdr = packet_hdr(*pkt); + uint32_t offset; + odp_packet_hdr_t *first_hdr = packet_hdr(*pkt); + odp_packet_hdr_t *pkt_hdr, *prev_hdr; - if (len > pkt_hdr->frame_len) + if (len > packet_len(first_hdr)) return -1; + pkt_hdr = packet_last_hdr(*pkt, &offset); + + /* Special processing for references */ + while (len >= pkt_hdr->frame_len - offset && first_hdr->ref_hdr) { + len -= (pkt_hdr->frame_len - offset); + prev_hdr = packet_prev_hdr(first_hdr, pkt_hdr, &offset); + ODP_ASSERT(packet_ref_count(prev_hdr) == 1); + prev_hdr->ref_hdr = NULL; + packet_free(pkt_hdr); + pkt_hdr = prev_hdr; + } + + ODP_ASSERT(packet_ref_count(pkt_hdr) == 1); last = packet_last_seg(pkt_hdr); seg_len = packet_seg_len(pkt_hdr, last); - if (len < seg_len) { + if (CONFIG_PACKET_MAX_SEGS == 1 || + len < seg_len || + pkt_hdr->buf_hdr.segcount == 1) { pull_tail(pkt_hdr, len); - } else if (CONFIG_PACKET_MAX_SEGS != 1) { + } else { int num = 0; uint32_t pull_len = 0; From patchwork Sun Mar 12 12:30:45 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Bill Fischofer X-Patchwork-Id: 95178 Delivered-To: patch@linaro.org Received: by 10.140.82.71 with SMTP id g65csp785149qgd; Sun, 12 Mar 2017 05:38:39 -0700 (PDT) X-Received: by 10.200.35.6 with SMTP id a6mr27685163qta.210.1489322319112; Sun, 12 Mar 2017 05:38:39 -0700 (PDT) Return-Path: Received: from lists.linaro.org (lists.linaro.org. [54.225.227.206]) by mx.google.com with ESMTP id o6si12365819qtd.44.2017.03.12.05.38.38; Sun, 12 Mar 2017 05:38:39 -0700 (PDT) Received-SPF: pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) client-ip=54.225.227.206; Authentication-Results: mx.google.com; spf=pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) smtp.mailfrom=lng-odp-bounces@lists.linaro.org; dmarc=pass (p=NONE sp=NONE dis=NONE) header.from=linaro.org Received: by lists.linaro.org (Postfix, from userid 109) id C9918607FD; Sun, 12 Mar 2017 12:38:38 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on ip-10-142-244-252 X-Spam-Level: X-Spam-Status: No, score=-1.9 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_NONE, RCVD_IN_MSPIKE_H3, RCVD_IN_MSPIKE_WL, URIBL_BLOCKED autolearn=disabled version=3.4.0 Received: from [127.0.0.1] (localhost [127.0.0.1]) by lists.linaro.org (Postfix) with ESMTP id D3B2863C94; Sun, 12 Mar 2017 12:32:09 +0000 (UTC) X-Original-To: lng-odp@lists.linaro.org Delivered-To: lng-odp@lists.linaro.org Received: by lists.linaro.org (Postfix, from userid 109) id BA41663C8F; Sun, 12 Mar 2017 12:32:00 +0000 (UTC) Received: from mail-oi0-f54.google.com (mail-oi0-f54.google.com [209.85.218.54]) by lists.linaro.org (Postfix) with ESMTPS id CD70863C87 for ; Sun, 12 Mar 2017 12:31:01 +0000 (UTC) Received: by mail-oi0-f54.google.com with SMTP id 62so68270837oih.2 for ; Sun, 12 Mar 2017 05:31:01 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references; bh=Py47rnxas9QR/SlV0t4m3ARO84DC1P/dqucnkNbtthA=; b=UiuCqF+bXBVPBMsUrU30311c3VDRc08PtZ3b006j91K7VRneMoS78KXByyGDJSljvm o8bcQZonMdyfeS9tyKc8kEydCjv6meFx7T9PXix+F/zswtnms9G8v0m5y2Ee65BGMq/Q 4KyRTDcvNXt9oX6dfeJk/Pu+aufJ0BGiUFy3efuf3eZL5FLgH5s0Jo31chH2347lbLie yCqEZ37gw84PNptKo+0PrVafoRxnwub/lbo8bQjY0938k+rX97ehVxgNtBND/423loq0 rnzhrCaHaMogcDW5vQwn0qK9s47K+MUDwc7ECYbtYGQrnxMXP6Ho0szclZqLDJBdCpY+ vxQg== X-Gm-Message-State: AMke39l9gojzHH5l+wlXs+Cd+P3gaB/V51iZpJCgXi5L8Jgbo5yknEYxYTLQZ9JRlbXBfZVV7R+dzQ== X-Received: by 10.202.231.148 with SMTP id e142mr12616491oih.3.1489321861112; Sun, 12 Mar 2017 05:31:01 -0700 (PDT) Received: from localhost.localdomain (cpe-70-121-83-241.austin.res.rr.com. [70.121.83.241]) by smtp.gmail.com with ESMTPSA id 62sm6663155otq.34.2017.03.12.05.31.00 (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 bits=128/128); Sun, 12 Mar 2017 05:31:00 -0700 (PDT) From: Bill Fischofer To: lng-odp@lists.linaro.org Date: Sun, 12 Mar 2017 07:30:45 -0500 Message-Id: <20170312123045.3852-12-bill.fischofer@linaro.org> X-Mailer: git-send-email 2.12.0.rc1 In-Reply-To: <20170312123045.3852-1-bill.fischofer@linaro.org> References: <20170312123045.3852-1-bill.fischofer@linaro.org> Subject: [lng-odp] [PATCHv2 11/11] linux-generic: packet: enable zero-copy reference creation X-BeenThere: lng-odp@lists.linaro.org X-Mailman-Version: 2.1.16 Precedence: list List-Id: "The OpenDataPlane \(ODP\) List" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: lng-odp-bounces@lists.linaro.org Sender: "lng-odp" Signed-off-by: Bill Fischofer --- platform/linux-generic/odp_packet.c | 172 +++++++++++++++++++++--------------- 1 file changed, 103 insertions(+), 69 deletions(-) -- 2.12.0.rc1 diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index 403b21f1..b220eb5e 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -1802,6 +1802,109 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) } /* + * References + */ + +static inline void packet_ref(odp_packet_hdr_t *pkt_hdr) +{ + uint32_t i; + odp_packet_hdr_t *hdr; + + do { + for (i = 0; i < pkt_hdr->buf_hdr.segcount; i++) { + hdr = pkt_hdr->buf_hdr.seg[i].hdr; + packet_ref_inc(hdr); + } + + pkt_hdr = pkt_hdr->ref_hdr; + } while (pkt_hdr); +} + +static inline odp_packet_t packet_splice(odp_packet_hdr_t *pkt_hdr, + uint32_t offset, + odp_packet_hdr_t *ref_hdr) +{ + /* Catch attempted references to stale handles in debug builds */ + ODP_ASSERT(packet_ref_count(pkt_hdr) > 0); + + /* Splicing is from the last section of src pkt */ + while (ref_hdr->ref_hdr) + ref_hdr = ref_hdr->ref_hdr; + + /* Find section where splice begins */ + while (offset >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { + offset -= (pkt_hdr->frame_len - pkt_hdr->ref_offset); + offset += (pkt_hdr->ref_hdr->frame_len - pkt_hdr->ref_len); + pkt_hdr = pkt_hdr->ref_hdr; + } + + ref_hdr->ref_hdr = pkt_hdr; + ref_hdr->ref_offset = offset; + ref_hdr->ref_len = pkt_hdr->frame_len; + + if (packet_ref_count(pkt_hdr) == 1 || offset < pkt_hdr->unshared_len) + pkt_hdr->unshared_len = offset; + + packet_ref(pkt_hdr); + return packet_handle(ref_hdr); +} + +odp_packet_t odp_packet_ref_static(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + packet_ref(pkt_hdr); + pkt_hdr->unshared_len = 0; + return pkt; +} + +odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) +{ + odp_packet_t hdr; + odp_packet_hdr_t *pkt_hdr; + + if (pkt == ODP_PACKET_INVALID) + return ODP_PACKET_INVALID; + + pkt_hdr = packet_hdr(pkt); + if (offset >= packet_len(pkt_hdr)) + return ODP_PACKET_INVALID; + + hdr = odp_packet_alloc(odp_packet_pool(pkt), 0); + + if (hdr == ODP_PACKET_INVALID) + return ODP_PACKET_INVALID; + + return packet_splice(pkt_hdr, offset, packet_hdr(hdr)); +} + +odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, + odp_packet_t hdr) +{ + odp_packet_hdr_t *pkt_hdr; + + if (pkt == ODP_PACKET_INVALID || + hdr == ODP_PACKET_INVALID || + pkt == hdr) + return ODP_PACKET_INVALID; + + ODP_ASSERT(odp_packet_has_ref(hdr) == 0); + + pkt_hdr = packet_hdr(pkt); + if (offset >= packet_len(pkt_hdr)) + return ODP_PACKET_INVALID; + + return packet_splice(pkt_hdr, offset, packet_hdr(hdr)); +} + +int odp_packet_has_ref(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + return pkt_hdr->ref_hdr != NULL || packet_ref_count(pkt_hdr) > 1; +} + +/* * * Copy * ******************************************************** @@ -2451,75 +2554,6 @@ uint64_t odp_packet_seg_to_u64(odp_packet_seg_t hdl) return _odp_pri(hdl); } -odp_packet_t odp_packet_ref_static(odp_packet_t pkt) -{ - return odp_packet_copy(pkt, odp_packet_pool(pkt)); -} - -odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset) -{ - odp_packet_t new; - int ret; - - new = odp_packet_copy(pkt, odp_packet_pool(pkt)); - - if (new == ODP_PACKET_INVALID) { - ODP_ERR("copy failed\n"); - return ODP_PACKET_INVALID; - } - - ret = odp_packet_trunc_head(&new, offset, NULL, NULL); - - if (ret < 0) { - ODP_ERR("trunk_head failed\n"); - odp_packet_free(new); - return ODP_PACKET_INVALID; - } - - return new; -} - -odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset, - odp_packet_t hdr) -{ - odp_packet_t new; - int ret; - - new = odp_packet_copy(pkt, odp_packet_pool(pkt)); - - if (new == ODP_PACKET_INVALID) { - ODP_ERR("copy failed\n"); - return ODP_PACKET_INVALID; - } - - if (offset) { - ret = odp_packet_trunc_head(&new, offset, NULL, NULL); - - if (ret < 0) { - ODP_ERR("trunk_head failed\n"); - odp_packet_free(new); - return ODP_PACKET_INVALID; - } - } - - ret = odp_packet_concat(&hdr, new); - - if (ret < 0) { - ODP_ERR("concat failed\n"); - odp_packet_free(new); - return ODP_PACKET_INVALID; - } - - return hdr; -} - -int odp_packet_has_ref(odp_packet_t pkt) -{ - (void)pkt; - - return 0; -} - /* Include non-inlined versions of API functions */ #if ODP_ABI_COMPAT == 1 #include