| Message ID | 1494950422-11135-11-git-send-email-odpbot@yandex.ru |
|---|---|
| State | Superseded |
| Headers | show |
| Series | [API-NEXT,v5,1/11] api: ipsec: add soft limit expiration event | expand |
On 05/16/17 19:00, Github ODP bot wrote: > From: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov@linaro.org> > > For now it's only a preview with the following limitation: > - No inbound inline processing support > - Only IPv4 support > - No zeroing of mutable IPv4 options for AH ICV calculation > - No replay protection > - No ESN support > - No SA options support: DF, DSCP, UDP, ESN > > Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov@linaro.org> > --- > /** Email created from pull request 28 (lumag:ipsec) > ** https://github.com/Linaro/odp/pull/28 > ** Patch: https://github.com/Linaro/odp/pull/28.patch > ** Base sha: f4c213cccc49b4b28557506ad19fba49b872d24f > ** Merge commit sha: e6ef5a70e5261ee32e5edb33fc7710d103010d95 > **/ > platform/linux-generic/include/odp_internal.h | 4 + > platform/linux-generic/odp_init.c | 13 + > platform/linux-generic/odp_ipsec.c | 1069 ++++++++++++++++++++++++- > 3 files changed, 1054 insertions(+), 32 deletions(-) > > diff --git a/platform/linux-generic/include/odp_internal.h b/platform/linux-generic/include/odp_internal.h > index 4662651..85ebbca 100644 > --- a/platform/linux-generic/include/odp_internal.h > +++ b/platform/linux-generic/include/odp_internal.h > @@ -72,6 +72,7 @@ enum init_stage { > NAME_TABLE_INIT, > IPSEC_EVENTS_INIT, > IPSEC_SAD_INIT, > + IPSEC_INIT, > MODULES_INIT, > ALL_INIT /* All init stages completed */ > }; > @@ -131,6 +132,9 @@ int _odp_ishm_init_local(void); > int _odp_ishm_term_global(void); > int _odp_ishm_term_local(void); > > +int odp_ipsec_init_global(void); > +int odp_ipsec_term_global(void); > + > int odp_ipsec_events_init_global(void); > int odp_ipsec_events_term_global(void); > > diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c > index 30dc54d..483ee74 100644 > --- a/platform/linux-generic/odp_init.c > +++ b/platform/linux-generic/odp_init.c > @@ -278,6 +278,12 @@ int odp_init_global(odp_instance_t *instance, > } > stage = IPSEC_SAD_INIT; > > + if (odp_ipsec_init_global()) { like all other function it has to be _odp_ipsec_init_global(). Or init has to be called from application. > + ODP_ERR("ODP IPsec init failed.\n"); > + goto init_failed; > + } > + stage = IPSEC_INIT; > + > if (_odp_modules_init_global()) { > ODP_ERR("ODP modules init failed\n"); > goto init_failed; > @@ -308,6 +314,13 @@ int _odp_term_global(enum init_stage stage) > switch (stage) { > case ALL_INIT: > case MODULES_INIT: > + case IPSEC_INIT: > + if (odp_ipsec_term_global()) { > + ODP_ERR("ODP IPsec term failed.\n"); > + rc = -1; > + } > + /* Fall through */ > + > case IPSEC_SAD_INIT: > if (odp_ipsec_sad_term_global()) { > ODP_ERR("ODP IPsec SAD term failed.\n"); > diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c > index 6620daf..b3e7107 100644 > --- a/platform/linux-generic/odp_ipsec.c > +++ b/platform/linux-generic/odp_ipsec.c > @@ -4,101 +4,1106 @@ > * SPDX-License-Identifier: BSD-3-Clause > */ > > +#include <odp/api/atomic.h> > #include <odp/api/ipsec.h> > +#include <odp/api/packet.h> > +#include <odp/api/shared_memory.h> > > +#include <odp_buffer_internal.h> > +#include <odp_buffer_inlines.h> > +#include <odp_debug_internal.h> > #include <odp_ipsec_internal.h> > +#include <odp_packet_internal.h> > +#include <odp_pool_internal.h> > > -#include <string.h> > +#include <protocols/ip.h> > +#include <protocols/ipsec.h> > + > +typedef void (*ipsec_postprocess_t)(ipsec_ctx_t *ctx); > + > +/** > + * Per packet IPsec processing context > + */ > +struct ipsec_ctx_s { > + odp_buffer_t buffer; /**< Buffer for context */ > + ipsec_ctx_t *next; /**< Next context in event */ > + > + ipsec_postprocess_t postprocess; > + ipsec_sa_t *ipsec_sa; > + odp_crypto_op_result_t crypto; > + odp_ipsec_op_status_t status; > + > + uint8_t ip_tos; /**< Saved IP TOS value */ > + uint8_t ip_ttl; /**< Saved IP TTL value */ > + uint16_t ip_frag_offset; /**< Saved IP flags value */ > + unsigned hdr_len; /**< Length of IPsec headers */ > + unsigned trl_len; /**< Length of IPsec trailers */ > + > + uint32_t src_ip; /**< SA source IP address */ > + uint32_t dst_ip; /**< SA dest IP address */ > + uint16_t ipsec_offset; /**< Offset of IPsec header from > + buffer start */ > + uint8_t iv[MAX_IV_LEN]; /**< ESP IV storage */ > + > + unsigned pkt_out : 1; /**< Packet was output to application */ > +}; > + > +static odp_pool_t ipsec_ctx_pool = ODP_POOL_INVALID; > + > +#define IPSEC_CTX_POOL_BUF_COUNT 1024 > + > +int odp_ipsec_init_global(void) > +{ > + odp_pool_param_t param; > + > + /* Create context buffer pool */ > + param.buf.size = sizeof(ipsec_ctx_t); > + param.buf.align = 0; > + param.buf.num = IPSEC_CTX_POOL_BUF_COUNT; > + param.type = ODP_POOL_BUFFER; > + > + ipsec_ctx_pool = odp_pool_create("ipsec_ctx_pool", ¶m); > + if (ODP_POOL_INVALID == ipsec_ctx_pool) { > + ODP_ERR("Error: context pool create failed.\n"); > + return -1; > + } > + > + return 0; > +} > + > +int odp_ipsec_term_global(void) > +{ > + int ret = 0; set to 0 is not needed. > + int rc = 0; > + > + ret = odp_pool_destroy(ipsec_ctx_pool); > + if (ret < 0) { > + ODP_ERR("ctx pool destroy failed"); > + rc = -1; > + } > + > + return rc; > +} > > int odp_ipsec_capability(odp_ipsec_capability_t *capa) > { > + int rc; > + odp_crypto_capability_t crypto_capa; > + > memset(capa, 0, sizeof(odp_ipsec_capability_t)); > > + capa->op_mode_sync = ODP_SUPPORT_PREFERRED; > + capa->op_mode_async = ODP_SUPPORT_PREFERRED; > + capa->op_mode_inline_out = ODP_SUPPORT_YES; > + > + capa->proto_ah = ODP_SUPPORT_YES; > + > + capa->max_num_sa = ODP_CONFIG_IPSEC_SAS; > + > + rc = odp_crypto_capability(&crypto_capa); > + if (rc < 0) > + return rc; > + > + capa->ciphers = crypto_capa.ciphers; > + capa->auths = crypto_capa.auths; > + > return 0; > } > > int odp_ipsec_cipher_capability(odp_cipher_alg_t cipher, > odp_crypto_cipher_capability_t capa[], int num) > { > - (void)cipher; > - (void)capa; > - (void)num; > - > - return -1; > + return odp_crypto_cipher_capability(cipher, capa, num); > } > > int odp_ipsec_auth_capability(odp_auth_alg_t auth, > odp_crypto_auth_capability_t capa[], int num) > { > - (void)auth; > - (void)capa; > - (void)num; > - > - return -1; > + return odp_crypto_auth_capability(auth, capa, num); > } > > void odp_ipsec_config_init(odp_ipsec_config_t *config) > { > memset(config, 0, sizeof(odp_ipsec_config_t)); > + config->inbound_mode = ODP_IPSEC_OP_MODE_SYNC; > + config->outbound_mode = ODP_IPSEC_OP_MODE_SYNC; > + config->max_num_sa = ODP_CONFIG_IPSEC_SAS; > + config->inbound.default_queue = ODP_QUEUE_INVALID; > + config->inbound.lookup.min_spi = 0; > + config->inbound.lookup.max_spi = UINT32_MAX; > } > > +static odp_ipsec_config_t ipsec_config; > + > int odp_ipsec_config(const odp_ipsec_config_t *config) > { > - (void)config; > + /* FIXME: unsupported for now */ > + if (ODP_IPSEC_OP_MODE_INLINE == config->inbound_mode) > + return -1; > + > + if (ODP_CONFIG_IPSEC_SAS > config->max_num_sa) > + return -1; > > - return -1; > + ipsec_config = *config; > + > + return 0; > +} > + > +static > +void ipsec_ctx_init(ipsec_ctx_t *ctx, odp_buffer_t buf) > +{ > + memset(ctx, 0, sizeof(*ctx)); > + ctx->buffer = buf; > + > + ctx->crypto.pkt = ODP_PACKET_INVALID; > + ctx->crypto.ok = true; > +} > + > +/** > + * Allocate per packet processing context. > + * > + * @return pointer to context area > + */ > +static > +ipsec_ctx_t *ipsec_ctx_alloc(void) > +{ > + odp_buffer_t ctx_buf = odp_buffer_alloc(ipsec_ctx_pool); > + ipsec_ctx_t *ctx; > + > + if (odp_unlikely(ODP_BUFFER_INVALID == ctx_buf)) > + return NULL; > + > + ctx = odp_buffer_addr(ctx_buf); > + ipsec_ctx_init(ctx, ctx_buf); > + > + return ctx; > } > > void _odp_ipsec_ctx_free(ipsec_ctx_t *ctx) > { > - (void)ctx; > + while (NULL != ctx) { > + ipsec_ctx_t *next = ctx->next; > + > + if (!ctx->pkt_out && ODP_PACKET_INVALID != ctx->crypto.pkt) > + odp_packet_free(ctx->crypto.pkt); > + > + odp_buffer_free(ctx->buffer); > + > + ctx = next; > + } > +} > + > +/** > + * Checksum > + * > + * @param buffer calculate chksum for buffer > + * @param len buffer length > + * > + * @return checksum value in host cpu order > + */ > +static inline > +odp_u16sum_t _odp_chksum(void *buffer, int len) > +{ > + uint16_t *buf = (uint16_t *)buffer; > + uint32_t sum = 0; > + uint16_t result; > + > + for (sum = 0; len > 1; len -= 2) > + sum += *buf++; > + > + if (len == 1) > + sum += *(unsigned char *)buf; > + > + sum = (sum >> 16) + (sum & 0xFFFF); > + sum += (sum >> 16); > + result = ~sum; > + > + return (__odp_force odp_u16sum_t) result; > } > > +/** > + * Calculate and fill in IPv4 checksum > + * > + * @note when using this api to populate data destined for the wire > + * odp_cpu_to_be_16() can be used to remove sparse warnings > + * > + * @param pkt ODP packet > + * > + * @return IPv4 checksum in host cpu order, or 0 on failure > + */ > +static inline odp_u16sum_t > +_odp_ipv4_csum_update(odp_packet_t pkt) > +{ > + uint16_t *w; > + _odp_ipv4hdr_t *ip; > + int nleft = sizeof(_odp_ipv4hdr_t); > + > + ip = (_odp_ipv4hdr_t *)odp_packet_l3_ptr(pkt, NULL); > + if (ip == NULL) > + return 0; > + > + ip->chksum = 0; > + w = (uint16_t *)(void *)ip; > + ip->chksum = _odp_chksum(w, nleft); > + return ip->chksum; > +} Ok that function came from helper. It's good for now but I think we need to have code in some common place. I think we can update in one palace and forgot to update in other place. > + > +#define ipv4_hdr_len(ip) (_ODP_IPV4HDR_IHL(ip->ver_ihl) * 4) > +static inline > +void ipv4_adjust_len(_odp_ipv4hdr_t *ip, int adj) > +{ > + ip->tot_len = odp_cpu_to_be_16(odp_be_to_cpu_16(ip->tot_len) + adj); > +} > + > +static > +void ipsec_finish(ipsec_ctx_t *ctx, > + odp_ipsec_packet_result_t *res, > + odp_packet_t *pkt) > +{ > + res->status = ctx->status; > + > + /* Check crypto result */ > + if (!ctx->crypto.ok) { > + if (ctx->crypto.cipher_status.alg_err != ODP_CRYPTO_ALG_ERR_NONE || > + ctx->crypto.cipher_status.hw_err != ODP_CRYPTO_HW_ERR_NONE) > + res->status.error.alg = 1; > + > + if (ctx->crypto.auth_status.alg_err != ODP_CRYPTO_ALG_ERR_NONE || > + ctx->crypto.auth_status.hw_err != ODP_CRYPTO_HW_ERR_NONE) > + res->status.error.auth = 1; > + } else { > + if (ctx->postprocess) > + ctx->postprocess(ctx); > + } > + > + *pkt = ctx->crypto.pkt; > + ctx->pkt_out = 1; > + > + if (NULL != ctx->ipsec_sa) { > + res->sa = ctx->ipsec_sa->ipsec_sa_hdl; > + _odp_ipsec_sa_unuse(ctx->ipsec_sa); > + } else { > + res->sa = ODP_IPSEC_SA_INVALID; > + } > +} > + > +static > +int ipsec_in_check_sa(ipsec_ctx_t *ctx, odp_ipsec_protocol_t proto, > + uint32_t spi, void *dst_addr) > +{ > + if (NULL == ctx->ipsec_sa) { > + ipsec_sa_lookup_t lookup; > + > + lookup.proto = proto; > + lookup.spi = spi; > + lookup.dst_addr = dst_addr; > + > + ctx->ipsec_sa = _odp_ipsec_sa_lookup(&lookup); > + if (NULL == ctx->ipsec_sa) { > + ctx->status.error.sa_lookup = 1; > + return -1; > + } > + } else if (ctx->ipsec_sa->spi != spi || > + ctx->ipsec_sa->proto != proto) { > + ctx->status.error.proto = 1; > + return -1; > + } > + > + return 0; > + > +} > + > +static const uint8_t ipsec_padding[255] = { > + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, > + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, > + 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, > + 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, > + 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, > + 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, > + 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, > + 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, > + 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, > + 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, > + 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, > + 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, > + 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, > + 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, > + 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, > + 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, > + 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, > + 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, > + 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, > + 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, > + 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, > + 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, > + 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, > + 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, > + 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, > + 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, > + 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, > + 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, > + 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, > + 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, > + 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, > + 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, > +}; where this magic numbers came from? It might be reasonable to add some comment about that. Was it taken from some other project? > + > +static void ipsec_in_postprocess(ipsec_ctx_t *ctx); > + > +static > +void ipsec_in_single(ipsec_ctx_t *ctx) > +{ > + odp_packet_t pkt = ctx->crypto.pkt; > + uint32_t ip_offset = odp_packet_l3_offset(pkt); > + _odp_ipv4hdr_t *ip = odp_packet_l3_ptr(pkt, NULL); > + uint16_t ip_hdr_len = ipv4_hdr_len(ip); > + odp_crypto_op_param_t param; > + odp_bool_t posted = 0; > + int rc = -1; > + > + ODP_ASSERT(ODP_PACKET_OFFSET_INVALID != ip_offset); > + ODP_ASSERT(NULL != ip); > + > + /* Initialize parameters block */ > + memset(¶m, 0, sizeof(param)); > + param.ctx = ctx; > + > + ctx->postprocess = ipsec_in_postprocess; > + ctx->ipsec_offset = ip_offset + ip_hdr_len; > + > + /* Check IP header for IPSec protocols and look it up */ > + if (_ODP_IPPROTO_ESP == ip->proto) { you have too long lines here and checkpatch should complane about it. switch instead if if will be more compact. > + _odp_esphdr_t esp; > + > + if (odp_packet_copy_to_mem(pkt, ctx->ipsec_offset, sizeof(esp), &esp) < 0) { > + ctx->status.error.alg = 1; > + return; > + } > + > + if (ipsec_in_check_sa(ctx, ODP_IPSEC_ESP, odp_be_to_cpu_32(esp.spi), &ip->dst_addr) < 0) > + return; > + > + if (odp_packet_copy_to_mem(pkt, ctx->ipsec_offset + _ODP_ESPHDR_LEN, ctx->ipsec_sa->esp_iv_len, ctx->iv) < 0) { > + ctx->status.error.alg = 1; > + return; > + } > + > + ctx->hdr_len = _ODP_ESPHDR_LEN + ctx->ipsec_sa->esp_iv_len; > + ctx->trl_len = _ODP_ESPTRL_LEN + ctx->ipsec_sa->icv_len; > + > + param.cipher_range.offset = ctx->ipsec_offset + ctx->hdr_len; > + param.cipher_range.length = odp_be_to_cpu_16(ip->tot_len) - ip_hdr_len - ctx->hdr_len - ctx->ipsec_sa->icv_len; > + param.override_iv_ptr = ctx->iv; > + > + param.auth_range.offset = ctx->ipsec_offset; > + param.auth_range.length = odp_be_to_cpu_16(ip->tot_len) - ip_hdr_len - ctx->ipsec_sa->icv_len; > + param.hash_result_offset = ip_offset + odp_be_to_cpu_16(ip->tot_len) - ctx->ipsec_sa->icv_len; > + } else if (_ODP_IPPROTO_AH == ip->proto) { > + _odp_ahhdr_t ah; > + > + if (odp_packet_copy_to_mem(pkt, ctx->ipsec_offset, sizeof(ah), &ah) < 0) { > + ctx->status.error.alg = 1; > + return; > + } > + > + if (ipsec_in_check_sa(ctx, ODP_IPSEC_AH, odp_be_to_cpu_32(ah.spi), &ip->dst_addr) < 0) > + return; > + > + ctx->hdr_len = (ah.ah_len + 2) * 4; > + ctx->trl_len = 0; > + > + /* Save everything to context */ > + ctx->ip_tos = ip->tos; > + ctx->ip_frag_offset = odp_be_to_cpu_16(ip->frag_offset); > + ctx->ip_ttl = ip->ttl; > + > + /* If authenticating, zero the mutable fields build the request */ > + ip->chksum = 0; > + ip->tos = 0; > + ip->frag_offset = 0; > + ip->ttl = 0; > + > + param.auth_range.offset = ip_offset; > + param.auth_range.length = odp_be_to_cpu_16(ip->tot_len); > + param.hash_result_offset = ctx->ipsec_offset + _ODP_AHHDR_LEN; > + } else { > + ctx->status.error.proto = 1; > + return; > + } > + > + if (_odp_ipsec_sa_update_stats(ctx->ipsec_sa, odp_packet_len(pkt), &ctx->status) < 0) > + return; > + > + param.session = ctx->ipsec_sa->session; > + param.pkt = pkt; > + /* Create new packet after all length extensions */ > + if (ctx->ipsec_sa->in_place) { > + param.out_pkt = pkt; > + } else { > + param.out_pkt = odp_packet_alloc(odp_packet_pool(pkt), > + odp_packet_len(pkt)); missing check for allocation. > + /* uarea will be copied by odp_crypto_operation */ > + odp_packet_user_ptr_set(param.out_pkt, > + odp_packet_user_ptr(param.pkt)); > + } > + > + rc = odp_crypto_operation(¶m, &posted, &ctx->crypto); > + if (rc < 0) { odp_packet_free(param.out_pkt) for above comment case ? > + ODP_DBG("Crypto failed\n"); > + ctx->status.error.alg = 1; > + return; > + } > + > + ODP_ASSERT(!posted); > +} > + > +static > +void ipsec_in_postprocess(ipsec_ctx_t *ctx) > +{ > + odp_packet_t pkt = ctx->crypto.pkt; > + uint32_t ip_offset = odp_packet_l3_offset(pkt); > + _odp_ipv4hdr_t *ip = odp_packet_l3_ptr(pkt, NULL); > + uint16_t ip_hdr_len = ipv4_hdr_len(ip); > + > + if (_ODP_IPPROTO_ESP == ip->proto) { > + /* > + * Finish cipher by finding ESP trailer and processing > + */ > + _odp_esptrl_t esptrl; > + uint32_t esptrl_offset = ip_offset + odp_be_to_cpu_16(ip->tot_len) - ctx->trl_len; > + > + if (odp_packet_copy_to_mem(pkt, esptrl_offset, sizeof(esptrl), &esptrl) < 0) { > + ctx->status.error.proto = 1; > + return; > + } > + > + if (ip_offset + esptrl.pad_len > esptrl_offset) { > + ctx->status.error.proto = 1; > + return; > + } > + > + if (_odp_packet_cmp_data(pkt, esptrl_offset - esptrl.pad_len, ipsec_padding, esptrl.pad_len) != 0) { > + ctx->status.error.proto = 1; > + return; > + } > + > + ip->proto = esptrl.next_header; > + ctx->trl_len += esptrl.pad_len; > + } else if (_ODP_IPPROTO_AH == ip->proto) { > + /* > + * Finish auth > + */ > + _odp_ahhdr_t ah; > + > + if (odp_packet_copy_to_mem(pkt, ctx->ipsec_offset, sizeof(ah), &ah) < 0) { > + ctx->status.error.alg = 1; > + return; > + } > + > + ip->proto = ah.next_header; > + > + /* Restore mutable fields */ > + ip->ttl = ctx->ip_ttl; > + ip->tos = ctx->ip_tos; > + ip->frag_offset = odp_cpu_to_be_16(ctx->ip_frag_offset); > + } else { > + ctx->status.error.proto = 1; > + return; > + } > + > + if (ctx->ipsec_sa->mode == ODP_IPSEC_MODE_TUNNEL) { > + /* FIXME: is this correct place/header ??? */ > + ip->ttl -= ctx->ipsec_sa->dec_ttl; > + _odp_ipv4_csum_update(pkt); > + > + /* We have a tunneled IPv4 packet, strip outer and IPsec headers */ > + odp_packet_move_data(pkt, ip_hdr_len + ctx->hdr_len, 0, ip_offset); > + if (odp_packet_trunc_head(&pkt, ip_hdr_len + ctx->hdr_len, NULL, NULL) < 0) { > + ctx->status.error.alg = 1; > + return; > + } > + } else { > + /* Finalize the IPv4 header */ > + ipv4_adjust_len(ip, -(ctx->hdr_len + ctx->trl_len)); > + > + _odp_ipv4_csum_update(pkt); > + > + odp_packet_move_data(pkt, ctx->hdr_len, 0, ip_offset + ip_hdr_len); > + if (odp_packet_trunc_head(&pkt, ctx->hdr_len, NULL, NULL) < 0) { > + ctx->status.error.alg = 1; > + return; > + } > + } > + > + if (odp_packet_trunc_tail(&pkt, ctx->trl_len, NULL, NULL) < 0) > + ctx->status.error.alg = 1; > + > + ctx->crypto.pkt = pkt; > +} > + > +/** Helper for calculating encode length using data length and block size */ > +#define ESP_ENCODE_LEN(x, b) ((((x) + ((b) - 1)) / (b)) * (b)) > + > +static > +int ipsec_out_extend_packet(ipsec_ctx_t *ctx, odp_packet_t *pkt) > +{ > + int rc = -1; > + > + if (odp_packet_extend_tail(pkt, ctx->trl_len, NULL, NULL) < 0) { > + ctx->status.error.alg = 1; > + goto out; > + } > + > + if (odp_packet_extend_head(pkt, ctx->hdr_len, NULL, NULL) < 0) { > + ctx->status.error.alg = 1; > + goto out; > + } > + > + odp_packet_move_data(*pkt, 0, ctx->hdr_len, ctx->ipsec_offset); > + > + return 0; > +out: > + ctx->crypto.pkt = *pkt; > + > + return rc; > +} if some funtion you return in the middle and here use goto. Without goto code will be 3 lines shorter and rc is not needed. > + > +static void ipsec_out_postprocess(ipsec_ctx_t *ctx); > + > +static > +void ipsec_out_single(ipsec_ctx_t *ctx) > +{ > + odp_packet_t pkt = ctx->crypto.pkt; > + uint32_t ip_offset = odp_packet_l3_offset(pkt); > + _odp_ipv4hdr_t *ip = odp_packet_l3_ptr(pkt, NULL); > + uint16_t ip_hdr_len = ipv4_hdr_len(ip); > + odp_crypto_op_param_t param; > + odp_bool_t posted = 0; > + int rc = -1; > + > + ODP_ASSERT(ODP_PACKET_OFFSET_INVALID != ip_offset); > + ODP_ASSERT(NULL != ip); > + ODP_ASSERT(NULL != ctx->ipsec_sa); > + > + /* Initialize parameters block */ > + memset(¶m, 0, sizeof(param)); > + param.ctx = ctx; > + > + if (ctx->ipsec_sa->mode == ODP_IPSEC_MODE_TUNNEL) { > + _odp_ipv4hdr_t out_ip; > + > + ip->ttl -= ctx->ipsec_sa->dec_ttl; > + > + out_ip.ver_ihl = 0x45; > + out_ip.tos = ip->tos; /* FIXME */ > + out_ip.tot_len = odp_cpu_to_be_16(odp_be_to_cpu_16(ip->tot_len) + _ODP_IPV4HDR_LEN); > + /* No need to convert to BE: ID just should not be duplicated */ > + out_ip.id = odp_atomic_fetch_add_u32(&ctx->ipsec_sa->tun_hdr_id, 1) & 0xffff; > + out_ip.frag_offset = 0; > + out_ip.ttl = ctx->ipsec_sa->tun_ttl; > + out_ip.proto = _ODP_IPV4; > + out_ip.src_addr = ctx->ipsec_sa->tun_src_ip; > + out_ip.dst_addr = ctx->ipsec_sa->tun_dst_ip; > + > + if (odp_packet_extend_head(&pkt, _ODP_IPV4HDR_LEN, NULL, NULL) < 0) { > + ctx->status.error.alg = 1; > + return; > + } > + ctx->crypto.pkt = pkt; > + > + odp_packet_move_data(pkt, 0, _ODP_IPV4HDR_LEN, ip_offset); > + > + odp_packet_copy_from_mem(pkt, ip_offset, _ODP_IPV4HDR_LEN, &out_ip); > + > + odp_packet_l4_offset_set(pkt, ip_offset + _ODP_IPV4HDR_LEN); > + > + ip = odp_packet_l3_ptr(pkt, NULL); > + ip_hdr_len = _ODP_IPV4HDR_LEN; > + } > + > + ctx->postprocess = ipsec_out_postprocess; > + > + ctx->ipsec_offset = ip_offset + ip_hdr_len; > + > + if (ctx->ipsec_sa->proto == ODP_IPSEC_ESP) { > + _odp_esphdr_t esp; > + _odp_esptrl_t esptrl; > + uint32_t encrypt_len; > + uint16_t ip_data_len = odp_be_to_cpu_16(ip->tot_len) - ip_hdr_len; > + uint32_t pad_block = ctx->ipsec_sa->esp_block_len; > + > + /* ESP trailer should be 32-bit right aligned */ > + if (pad_block < 4) > + pad_block = 4; > + > + encrypt_len = ESP_ENCODE_LEN(ip_data_len + _ODP_ESPTRL_LEN, > + pad_block); > + > + ctx->hdr_len += _ODP_ESPHDR_LEN + ctx->ipsec_sa->esp_iv_len; > + ctx->trl_len = encrypt_len - ip_data_len + ctx->ipsec_sa->icv_len; > + > + if (ctx->ipsec_sa->esp_iv_len) { > + /* FIXME: this is correct only for CBC ciphers ! */ > + uint32_t len = odp_random_data(ctx->iv, ctx->ipsec_sa->esp_iv_len, ODP_RANDOM_CRYPTO); > + if (len != ctx->ipsec_sa->esp_iv_len) { > + ctx->status.error.alg = 1; > + return; > + } > + > + param.override_iv_ptr = ctx->iv; > + } > + > + if (ipsec_out_extend_packet(ctx, &pkt) < 0) > + return; > + > + ip = odp_packet_l3_ptr(pkt, NULL); > + > + /* Set IPv4 length before authentication */ > + ipv4_adjust_len(ip, ctx->hdr_len + ctx->trl_len); > + > + uint32_t esptrl_offset = ip_offset + ip_hdr_len + ctx->hdr_len + encrypt_len - _ODP_ESPTRL_LEN; > + > + memset(&esp, 0, sizeof(esp)); > + esp.spi = odp_cpu_to_be_32(ctx->ipsec_sa->spi); > + esp.seq_no = odp_cpu_to_be_32(odp_atomic_fetch_add_u32(&ctx->ipsec_sa->seq, 1) + 1); > + > + memset(&esptrl, 0, sizeof(esptrl)); > + esptrl.pad_len = encrypt_len - ip_data_len - _ODP_ESPTRL_LEN; > + esptrl.next_header = ip->proto; > + ip->proto = _ODP_IPPROTO_ESP; > + > + odp_packet_copy_from_mem(pkt, ctx->ipsec_offset, _ODP_ESPHDR_LEN, &esp); > + /* FIXME: this is correct only for CBC ciphers ! */ > + odp_packet_copy_from_mem(pkt, ctx->ipsec_offset + _ODP_ESPHDR_LEN, ctx->ipsec_sa->esp_iv_len, ctx->iv); > + odp_packet_copy_from_mem(pkt, esptrl_offset - esptrl.pad_len, esptrl.pad_len, ipsec_padding); > + odp_packet_copy_from_mem(pkt, esptrl_offset, _ODP_ESPTRL_LEN, &esptrl); > + > + param.cipher_range.offset = ctx->ipsec_offset + ctx->hdr_len; > + param.cipher_range.length = odp_be_to_cpu_16(ip->tot_len) - ip_hdr_len - ctx->hdr_len - ctx->ipsec_sa->icv_len; > + > + param.auth_range.offset = ctx->ipsec_offset; > + param.auth_range.length = odp_be_to_cpu_16(ip->tot_len) - ip_hdr_len - ctx->ipsec_sa->icv_len; > + param.hash_result_offset = ip_offset + odp_be_to_cpu_16(ip->tot_len) - ctx->ipsec_sa->icv_len; > + } else if (ctx->ipsec_sa->proto == ODP_IPSEC_AH) { > + _odp_ahhdr_t ah; > + > + ctx->hdr_len = _ODP_AHHDR_LEN + ctx->ipsec_sa->icv_len; > + ctx->trl_len = 0; > + > + /* Save IPv4 stuff */ > + ctx->ip_tos = ip->tos; > + ctx->ip_frag_offset = odp_be_to_cpu_16(ip->frag_offset); > + ctx->ip_ttl = ip->ttl; > + > + if (ipsec_out_extend_packet(ctx, &pkt) < 0) > + return; > + > + ip = odp_packet_l3_ptr(pkt, NULL); > + > + /* Set IPv4 length before authentication */ > + ipv4_adjust_len(ip, ctx->hdr_len + ctx->trl_len); > + > + memset(&ah, 0, sizeof(ah)); > + ah.spi = odp_cpu_to_be_32(ctx->ipsec_sa->spi); > + ah.ah_len = 1 + (ctx->ipsec_sa->icv_len / 4); > + ah.seq_no = odp_cpu_to_be_32(odp_atomic_fetch_add_u32(&ctx->ipsec_sa->seq, 1) + 1); > + ah.next_header = ip->proto; > + ip->proto = _ODP_IPPROTO_AH; > + > + odp_packet_copy_from_mem(pkt, ctx->ipsec_offset, _ODP_AHHDR_LEN, &ah); > + _odp_packet_set_data(pkt, ctx->ipsec_offset + _ODP_AHHDR_LEN, 0, ctx->ipsec_sa->icv_len); > + > + ip->chksum = 0; > + ip->tos = 0; > + ip->frag_offset = 0; > + ip->ttl = 0; > + > + param.auth_range.offset = ip_offset; > + param.auth_range.length = odp_be_to_cpu_16(ip->tot_len); > + param.hash_result_offset = ctx->ipsec_offset + _ODP_AHHDR_LEN; > + } else { > + ctx->status.error.alg = 1; > + return; > + } > + > + if (_odp_ipsec_sa_update_stats(ctx->ipsec_sa, odp_packet_len(pkt), &ctx->status) < 0) > + return; > + > + param.session = ctx->ipsec_sa->session; > + param.pkt = pkt; > + /* Create new packet after all length extensions */ > + if (ctx->ipsec_sa->in_place) { > + param.out_pkt = pkt; > + } else { > + param.out_pkt = odp_packet_alloc(odp_packet_pool(pkt), > + odp_packet_len(pkt)); > + odp_packet_user_ptr_set(param.out_pkt, > + odp_packet_user_ptr(param.pkt)); > + } > + > + rc = odp_crypto_operation(¶m, &posted, &ctx->crypto); > + if (rc < 0) { > + ODP_DBG("Crypto failed\n"); > + ctx->status.error.alg = 1; > + return; > + } > + > + ODP_ASSERT(!posted); > +} > + > +static > +void ipsec_out_postprocess(ipsec_ctx_t *ctx) > +{ > + odp_packet_t pkt = ctx->crypto.pkt; > + _odp_ipv4hdr_t *ip = odp_packet_l3_ptr(pkt, NULL); > + > + /* Finalize the IPv4 header */ > + if (ip->proto == _ODP_IPPROTO_AH) { > + ip->ttl = ctx->ip_ttl; > + ip->tos = ctx->ip_tos; > + ip->frag_offset = odp_cpu_to_be_16(ctx->ip_frag_offset); > + } > + > + _odp_ipv4_csum_update(pkt); > +} > + > +#if 0 > +static odp_ipsec_op_opt_t default_opt = { > + .mode = ODP_IPSEC_FRAG_DISABLED, > +}; > +#endif you have a lot if if 0s in the code. What is the plane for that? > + > int odp_ipsec_in(const odp_ipsec_op_param_t *input, > odp_ipsec_op_result_t *output) > { > - (void)input; > - (void)output; > + int in_pkt = 0; > + int out_pkt = 0; > + unsigned sa_idx = 0; > + unsigned opt_idx = 0; > + unsigned sa_inc = (input->num_sa > 1) ? 1 : 0; > + unsigned opt_inc = (input->num_opt > 1) ? 1 : 0; > + > + while (in_pkt < input->num_pkt && out_pkt < output->num_pkt) { > + ipsec_ctx_t ctx; > + > + ipsec_ctx_init(&ctx, ODP_BUFFER_INVALID); > + > +#if 0 > + odp_ipsec_op_opt_t *opt; > + > + if (0 == input->num_opt) > + opt = &default_opt; > + else > + opt = &input->opt[opt_idx]; > +#endif > + > + ctx.crypto.pkt = input->pkt[in_pkt]; > + > + if (0 == input->num_sa) { > + ctx.ipsec_sa = NULL; > + } else { > + ctx.ipsec_sa = _odp_ipsec_sa_use(input->sa[sa_idx]); > + ODP_ASSERT(NULL != ctx.ipsec_sa); > + } > > - return -1; > + ipsec_in_single(&ctx); > + > + ipsec_finish(&ctx, &output->res[out_pkt], &output->pkt[out_pkt]); > + > + in_pkt++; > + out_pkt++; > + sa_idx += sa_inc; > + opt_idx += opt_inc; > + } > + > + return in_pkt; > } > > int odp_ipsec_out(const odp_ipsec_op_param_t *input, > - odp_ipsec_op_result_t *output) > + odp_ipsec_op_result_t *output) > { > - (void)input; > - (void)output; > + int in_pkt = 0; > + int out_pkt = 0; > + unsigned sa_idx = 0; > + unsigned opt_idx = 0; > + unsigned sa_inc = (input->num_sa > 1) ? 1 : 0; > + unsigned opt_inc = (input->num_opt > 1) ? 1 : 0; > + > + ODP_ASSERT(input->num_sa != 0); > + > + while (in_pkt < input->num_pkt && out_pkt < output->num_pkt) { > + odp_ipsec_sa_t sa; > + ipsec_ctx_t ctx; > + > + ipsec_ctx_init(&ctx, ODP_BUFFER_INVALID); > + > + sa = input->sa[sa_idx]; > > - return -1; > + ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); > + > +#if 0 > + odp_ipsec_op_opt_t *opt; > + > + if (0 == input->num_opt) > + opt = &default_opt; > + else > + opt = &input->opt[opt_idx]; > +#endif > + > + ctx.crypto.pkt = input->pkt[in_pkt]; > + ctx.ipsec_sa = _odp_ipsec_sa_use(sa); > + > + ipsec_out_single(&ctx); > + > + ipsec_finish(&ctx, &output->res[out_pkt], &output->pkt[out_pkt]); > + > + in_pkt++; > + out_pkt++; > + sa_idx += sa_inc; > + opt_idx += opt_inc; > + } > + > + return in_pkt; > } > > int odp_ipsec_in_enq(const odp_ipsec_op_param_t *input) > { > - (void)input; > + int in_pkt = 0; > + unsigned sa_idx = 0; > + unsigned opt_idx = 0; > + unsigned sa_inc = (input->num_sa > 1) ? 1 : 0; > + unsigned opt_inc = (input->num_opt > 1) ? 1 : 0; > + > + while (in_pkt < input->num_pkt) { > + ipsec_ctx_t *ctx; > + odp_queue_t queue; > > - return -1; > + ctx = ipsec_ctx_alloc(); > + if (NULL == ctx) > + break; > + > +#if 0 > + odp_ipsec_op_opt_t *opt; > + > + if (0 == input->num_opt) > + opt = &default_opt; > + else > + opt = &input->opt[opt_idx]; > +#endif > + > + ctx->crypto.pkt = input->pkt[in_pkt]; > + > + if (0 == input->num_sa) { > + ctx->ipsec_sa = NULL; > + } else { > + ctx->ipsec_sa = _odp_ipsec_sa_use(input->sa[sa_idx]); > + ODP_ASSERT(NULL != ctx->ipsec_sa); > + } > + > + ipsec_in_single(ctx); > + > + in_pkt++; > + sa_idx += sa_inc; > + opt_idx += opt_inc; > + > + /* IN might have looked up SA for the packet */ > + if (NULL == ctx->ipsec_sa) > + queue = ipsec_config.inbound.default_queue; > + else > + queue = ctx->ipsec_sa->queue; > + if (odp_unlikely(_odp_ipsec_result_send(queue, ctx) < 0)) { > + _odp_ipsec_ctx_free(ctx); > + break; > + } > + } > + > + return in_pkt; > } > > int odp_ipsec_out_enq(const odp_ipsec_op_param_t *input) > { > - (void)input; > + int in_pkt = 0; > + unsigned sa_idx = 0; > + unsigned opt_idx = 0; > + unsigned sa_inc = (input->num_sa > 1) ? 1 : 0; > + unsigned opt_inc = (input->num_opt > 1) ? 1 : 0; > + > + ODP_ASSERT(input->num_sa != 0); > + > + while (in_pkt < input->num_pkt) { > + odp_ipsec_sa_t sa; > + ipsec_ctx_t *ctx; > + > + ctx = ipsec_ctx_alloc(); > + if (NULL == ctx) > + break; > + > + sa = input->sa[sa_idx]; > + > + ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); > + > +#if 0 > + odp_ipsec_op_opt_t *opt; > + > + if (0 == input->num_opt) > + opt = &default_opt; > + else > + opt = &input->opt[opt_idx]; > +#endif > + > + ctx->crypto.pkt = input->pkt[in_pkt]; > + ctx->ipsec_sa = _odp_ipsec_sa_use(sa); > + > + ipsec_out_single(ctx); > + > + in_pkt++; > + sa_idx += sa_inc; > + opt_idx += opt_inc; > + > + if (odp_unlikely(_odp_ipsec_result_send(ctx->ipsec_sa->queue, ctx) < 0)) { > + _odp_ipsec_ctx_free(ctx); > + break; > + } > + } > + > + return in_pkt; > +} > + > +static > +odp_bool_t _odp_ipsec_out_inline_send(ipsec_ctx_t *ctx, > + const odp_ipsec_inline_op_param_t *inline_param) > +{ > + if (ctx->status.all_error || !ctx->crypto.ok) > + return false; > + > + while (ctx) { > + ipsec_ctx_t *next = ctx->next; > + odp_ipsec_packet_result_t dummy; > + odp_packet_t pkt; > + uint32_t offset; > + odp_pktout_queue_t queue; > + uint32_t hdr_len = inline_param->outer_hdr.len; > + > + ctx->next = NULL; > + > + ipsec_finish(ctx, &dummy, &pkt); > + offset = odp_packet_l3_offset(pkt); > + > + if (offset >= hdr_len) { > + offset = offset - hdr_len; > + } else { > + if (odp_packet_extend_head(&pkt, hdr_len - offset, NULL, NULL) < 0) { > + ctx->status.error.alg = 1; > + goto out; > + } > + > + odp_packet_l3_offset_set(pkt, hdr_len); > + > + offset = 0; > + } > + > + if (odp_packet_copy_from_mem(pkt, offset, hdr_len, inline_param->outer_hdr.ptr) < 0) { > + ctx->status.error.alg = 1; > + goto out; > + } > + > + if (odp_pktout_queue(inline_param->pktio, &queue, 1) < 0) { > + ctx->status.error.alg = 1; > + goto out; > + } > + > + if (odp_pktout_send(queue, &pkt, 1) < 0) { > + ctx->status.error.alg = 1; > + goto out; > + } > > - return -1; > +out: > + if (ctx->status.all_error) { > + if (odp_unlikely(_odp_ipsec_result_send(ctx->ipsec_sa->queue, ctx) < 0)) > + _odp_ipsec_ctx_free(ctx); > + } else { > + _odp_ipsec_ctx_free(ctx); > + } > + ctx = next; > + } > + > + return true; > } > > -int odp_ipsec_out_inline(const odp_ipsec_op_param_t *op_param, > +int odp_ipsec_out_inline(const odp_ipsec_op_param_t *input, > const odp_ipsec_inline_op_param_t *inline_param) > { > - (void)op_param; > - (void)inline_param; > + int in_pkt = 0; > + unsigned sa_idx = 0; > + unsigned opt_idx = 0; > + unsigned sa_inc = (input->num_sa > 1) ? 1 : 0; > + unsigned opt_inc = (input->num_opt > 1) ? 1 : 0; > + > + ODP_ASSERT(input->num_sa != 0); > + > + while (in_pkt < input->num_pkt) { > + odp_ipsec_sa_t sa; > + ipsec_ctx_t *ctx; > + > + ctx = ipsec_ctx_alloc(); > + if (NULL == ctx) > + break; > + > + sa = input->sa[sa_idx]; > + > + ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); > + > +#if 0 > + odp_ipsec_op_opt_t *opt; > + > + if (0 == input->num_opt) > + opt = &default_opt; > + else > + opt = &input->opt[opt_idx]; > +#endif > > - return -1; > + ctx->crypto.pkt = input->pkt[in_pkt]; > + ctx->ipsec_sa = _odp_ipsec_sa_use(sa); > + > + ipsec_out_single(ctx); > + > + in_pkt++; > + sa_idx += sa_inc; > + opt_idx += opt_inc; > + > + /* FIXME: inline_param should have been put into context */ > + if (!_odp_ipsec_out_inline_send(ctx, &inline_param[in_pkt - 1])) { > + /* In case of an error, submit result event */ > + if (odp_unlikely(_odp_ipsec_result_send(ctx->ipsec_sa->queue, ctx) < 0)) { > + _odp_ipsec_ctx_free(ctx); > + break; > + } > + } > + } > + > + return in_pkt; > } > > int _odp_ipsec_ctx_result(ipsec_ctx_t *ctx, odp_ipsec_op_result_t *result) > { > - (void)ctx; > - (void)result; > + int out_pkt = 0; > + > + if (NULL == result) > + goto count; > + > + while (NULL != ctx && out_pkt < result->num_pkt) { > + ipsec_finish(ctx, &result->res[out_pkt], &result->pkt[out_pkt]); > + out_pkt++; > + ctx = ctx->next; > + } > + > + result->num_pkt = out_pkt; > + > +count: > + while (NULL != ctx) { > + out_pkt++; > + ctx = ctx->next; > + } > > - return -1; > + return out_pkt; > } >
diff --git a/platform/linux-generic/include/odp_internal.h b/platform/linux-generic/include/odp_internal.h index 4662651..85ebbca 100644 --- a/platform/linux-generic/include/odp_internal.h +++ b/platform/linux-generic/include/odp_internal.h @@ -72,6 +72,7 @@ enum init_stage { NAME_TABLE_INIT, IPSEC_EVENTS_INIT, IPSEC_SAD_INIT, + IPSEC_INIT, MODULES_INIT, ALL_INIT /* All init stages completed */ }; @@ -131,6 +132,9 @@ int _odp_ishm_init_local(void); int _odp_ishm_term_global(void); int _odp_ishm_term_local(void); +int odp_ipsec_init_global(void); +int odp_ipsec_term_global(void); + int odp_ipsec_events_init_global(void); int odp_ipsec_events_term_global(void); diff --git a/platform/linux-generic/odp_init.c b/platform/linux-generic/odp_init.c index 30dc54d..483ee74 100644 --- a/platform/linux-generic/odp_init.c +++ b/platform/linux-generic/odp_init.c @@ -278,6 +278,12 @@ int odp_init_global(odp_instance_t *instance, } stage = IPSEC_SAD_INIT; + if (odp_ipsec_init_global()) { + ODP_ERR("ODP IPsec init failed.\n"); + goto init_failed; + } + stage = IPSEC_INIT; + if (_odp_modules_init_global()) { ODP_ERR("ODP modules init failed\n"); goto init_failed; @@ -308,6 +314,13 @@ int _odp_term_global(enum init_stage stage) switch (stage) { case ALL_INIT: case MODULES_INIT: + case IPSEC_INIT: + if (odp_ipsec_term_global()) { + ODP_ERR("ODP IPsec term failed.\n"); + rc = -1; + } + /* Fall through */ + case IPSEC_SAD_INIT: if (odp_ipsec_sad_term_global()) { ODP_ERR("ODP IPsec SAD term failed.\n"); diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c index 6620daf..b3e7107 100644 --- a/platform/linux-generic/odp_ipsec.c +++ b/platform/linux-generic/odp_ipsec.c @@ -4,101 +4,1106 @@ * SPDX-License-Identifier: BSD-3-Clause */ +#include <odp/api/atomic.h> #include <odp/api/ipsec.h> +#include <odp/api/packet.h> +#include <odp/api/shared_memory.h> +#include <odp_buffer_internal.h> +#include <odp_buffer_inlines.h> +#include <odp_debug_internal.h> #include <odp_ipsec_internal.h> +#include <odp_packet_internal.h> +#include <odp_pool_internal.h> -#include <string.h> +#include <protocols/ip.h> +#include <protocols/ipsec.h> + +typedef void (*ipsec_postprocess_t)(ipsec_ctx_t *ctx); + +/** + * Per packet IPsec processing context + */ +struct ipsec_ctx_s { + odp_buffer_t buffer; /**< Buffer for context */ + ipsec_ctx_t *next; /**< Next context in event */ + + ipsec_postprocess_t postprocess; + ipsec_sa_t *ipsec_sa; + odp_crypto_op_result_t crypto; + odp_ipsec_op_status_t status; + + uint8_t ip_tos; /**< Saved IP TOS value */ + uint8_t ip_ttl; /**< Saved IP TTL value */ + uint16_t ip_frag_offset; /**< Saved IP flags value */ + unsigned hdr_len; /**< Length of IPsec headers */ + unsigned trl_len; /**< Length of IPsec trailers */ + + uint32_t src_ip; /**< SA source IP address */ + uint32_t dst_ip; /**< SA dest IP address */ + uint16_t ipsec_offset; /**< Offset of IPsec header from + buffer start */ + uint8_t iv[MAX_IV_LEN]; /**< ESP IV storage */ + + unsigned pkt_out : 1; /**< Packet was output to application */ +}; + +static odp_pool_t ipsec_ctx_pool = ODP_POOL_INVALID; + +#define IPSEC_CTX_POOL_BUF_COUNT 1024 + +int odp_ipsec_init_global(void) +{ + odp_pool_param_t param; + + /* Create context buffer pool */ + param.buf.size = sizeof(ipsec_ctx_t); + param.buf.align = 0; + param.buf.num = IPSEC_CTX_POOL_BUF_COUNT; + param.type = ODP_POOL_BUFFER; + + ipsec_ctx_pool = odp_pool_create("ipsec_ctx_pool", ¶m); + if (ODP_POOL_INVALID == ipsec_ctx_pool) { + ODP_ERR("Error: context pool create failed.\n"); + return -1; + } + + return 0; +} + +int odp_ipsec_term_global(void) +{ + int ret = 0; + int rc = 0; + + ret = odp_pool_destroy(ipsec_ctx_pool); + if (ret < 0) { + ODP_ERR("ctx pool destroy failed"); + rc = -1; + } + + return rc; +} int odp_ipsec_capability(odp_ipsec_capability_t *capa) { + int rc; + odp_crypto_capability_t crypto_capa; + memset(capa, 0, sizeof(odp_ipsec_capability_t)); + capa->op_mode_sync = ODP_SUPPORT_PREFERRED; + capa->op_mode_async = ODP_SUPPORT_PREFERRED; + capa->op_mode_inline_out = ODP_SUPPORT_YES; + + capa->proto_ah = ODP_SUPPORT_YES; + + capa->max_num_sa = ODP_CONFIG_IPSEC_SAS; + + rc = odp_crypto_capability(&crypto_capa); + if (rc < 0) + return rc; + + capa->ciphers = crypto_capa.ciphers; + capa->auths = crypto_capa.auths; + return 0; } int odp_ipsec_cipher_capability(odp_cipher_alg_t cipher, odp_crypto_cipher_capability_t capa[], int num) { - (void)cipher; - (void)capa; - (void)num; - - return -1; + return odp_crypto_cipher_capability(cipher, capa, num); } int odp_ipsec_auth_capability(odp_auth_alg_t auth, odp_crypto_auth_capability_t capa[], int num) { - (void)auth; - (void)capa; - (void)num; - - return -1; + return odp_crypto_auth_capability(auth, capa, num); } void odp_ipsec_config_init(odp_ipsec_config_t *config) { memset(config, 0, sizeof(odp_ipsec_config_t)); + config->inbound_mode = ODP_IPSEC_OP_MODE_SYNC; + config->outbound_mode = ODP_IPSEC_OP_MODE_SYNC; + config->max_num_sa = ODP_CONFIG_IPSEC_SAS; + config->inbound.default_queue = ODP_QUEUE_INVALID; + config->inbound.lookup.min_spi = 0; + config->inbound.lookup.max_spi = UINT32_MAX; } +static odp_ipsec_config_t ipsec_config; + int odp_ipsec_config(const odp_ipsec_config_t *config) { - (void)config; + /* FIXME: unsupported for now */ + if (ODP_IPSEC_OP_MODE_INLINE == config->inbound_mode) + return -1; + + if (ODP_CONFIG_IPSEC_SAS > config->max_num_sa) + return -1; - return -1; + ipsec_config = *config; + + return 0; +} + +static +void ipsec_ctx_init(ipsec_ctx_t *ctx, odp_buffer_t buf) +{ + memset(ctx, 0, sizeof(*ctx)); + ctx->buffer = buf; + + ctx->crypto.pkt = ODP_PACKET_INVALID; + ctx->crypto.ok = true; +} + +/** + * Allocate per packet processing context. + * + * @return pointer to context area + */ +static +ipsec_ctx_t *ipsec_ctx_alloc(void) +{ + odp_buffer_t ctx_buf = odp_buffer_alloc(ipsec_ctx_pool); + ipsec_ctx_t *ctx; + + if (odp_unlikely(ODP_BUFFER_INVALID == ctx_buf)) + return NULL; + + ctx = odp_buffer_addr(ctx_buf); + ipsec_ctx_init(ctx, ctx_buf); + + return ctx; } void _odp_ipsec_ctx_free(ipsec_ctx_t *ctx) { - (void)ctx; + while (NULL != ctx) { + ipsec_ctx_t *next = ctx->next; + + if (!ctx->pkt_out && ODP_PACKET_INVALID != ctx->crypto.pkt) + odp_packet_free(ctx->crypto.pkt); + + odp_buffer_free(ctx->buffer); + + ctx = next; + } +} + +/** + * Checksum + * + * @param buffer calculate chksum for buffer + * @param len buffer length + * + * @return checksum value in host cpu order + */ +static inline +odp_u16sum_t _odp_chksum(void *buffer, int len) +{ + uint16_t *buf = (uint16_t *)buffer; + uint32_t sum = 0; + uint16_t result; + + for (sum = 0; len > 1; len -= 2) + sum += *buf++; + + if (len == 1) + sum += *(unsigned char *)buf; + + sum = (sum >> 16) + (sum & 0xFFFF); + sum += (sum >> 16); + result = ~sum; + + return (__odp_force odp_u16sum_t) result; } +/** + * Calculate and fill in IPv4 checksum + * + * @note when using this api to populate data destined for the wire + * odp_cpu_to_be_16() can be used to remove sparse warnings + * + * @param pkt ODP packet + * + * @return IPv4 checksum in host cpu order, or 0 on failure + */ +static inline odp_u16sum_t +_odp_ipv4_csum_update(odp_packet_t pkt) +{ + uint16_t *w; + _odp_ipv4hdr_t *ip; + int nleft = sizeof(_odp_ipv4hdr_t); + + ip = (_odp_ipv4hdr_t *)odp_packet_l3_ptr(pkt, NULL); + if (ip == NULL) + return 0; + + ip->chksum = 0; + w = (uint16_t *)(void *)ip; + ip->chksum = _odp_chksum(w, nleft); + return ip->chksum; +} + +#define ipv4_hdr_len(ip) (_ODP_IPV4HDR_IHL(ip->ver_ihl) * 4) +static inline +void ipv4_adjust_len(_odp_ipv4hdr_t *ip, int adj) +{ + ip->tot_len = odp_cpu_to_be_16(odp_be_to_cpu_16(ip->tot_len) + adj); +} + +static +void ipsec_finish(ipsec_ctx_t *ctx, + odp_ipsec_packet_result_t *res, + odp_packet_t *pkt) +{ + res->status = ctx->status; + + /* Check crypto result */ + if (!ctx->crypto.ok) { + if (ctx->crypto.cipher_status.alg_err != ODP_CRYPTO_ALG_ERR_NONE || + ctx->crypto.cipher_status.hw_err != ODP_CRYPTO_HW_ERR_NONE) + res->status.error.alg = 1; + + if (ctx->crypto.auth_status.alg_err != ODP_CRYPTO_ALG_ERR_NONE || + ctx->crypto.auth_status.hw_err != ODP_CRYPTO_HW_ERR_NONE) + res->status.error.auth = 1; + } else { + if (ctx->postprocess) + ctx->postprocess(ctx); + } + + *pkt = ctx->crypto.pkt; + ctx->pkt_out = 1; + + if (NULL != ctx->ipsec_sa) { + res->sa = ctx->ipsec_sa->ipsec_sa_hdl; + _odp_ipsec_sa_unuse(ctx->ipsec_sa); + } else { + res->sa = ODP_IPSEC_SA_INVALID; + } +} + +static +int ipsec_in_check_sa(ipsec_ctx_t *ctx, odp_ipsec_protocol_t proto, + uint32_t spi, void *dst_addr) +{ + if (NULL == ctx->ipsec_sa) { + ipsec_sa_lookup_t lookup; + + lookup.proto = proto; + lookup.spi = spi; + lookup.dst_addr = dst_addr; + + ctx->ipsec_sa = _odp_ipsec_sa_lookup(&lookup); + if (NULL == ctx->ipsec_sa) { + ctx->status.error.sa_lookup = 1; + return -1; + } + } else if (ctx->ipsec_sa->spi != spi || + ctx->ipsec_sa->proto != proto) { + ctx->status.error.proto = 1; + return -1; + } + + return 0; + +} + +static const uint8_t ipsec_padding[255] = { + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, + 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20, + 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, + 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, + 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, + 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, + 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, + 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, + 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, + 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60, + 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, + 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, + 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, + 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80, + 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, + 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, + 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, + 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, + 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, + 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, + 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, + 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, + 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, + 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, + 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, + 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, + 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, + 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, + 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, + 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, +}; + +static void ipsec_in_postprocess(ipsec_ctx_t *ctx); + +static +void ipsec_in_single(ipsec_ctx_t *ctx) +{ + odp_packet_t pkt = ctx->crypto.pkt; + uint32_t ip_offset = odp_packet_l3_offset(pkt); + _odp_ipv4hdr_t *ip = odp_packet_l3_ptr(pkt, NULL); + uint16_t ip_hdr_len = ipv4_hdr_len(ip); + odp_crypto_op_param_t param; + odp_bool_t posted = 0; + int rc = -1; + + ODP_ASSERT(ODP_PACKET_OFFSET_INVALID != ip_offset); + ODP_ASSERT(NULL != ip); + + /* Initialize parameters block */ + memset(¶m, 0, sizeof(param)); + param.ctx = ctx; + + ctx->postprocess = ipsec_in_postprocess; + ctx->ipsec_offset = ip_offset + ip_hdr_len; + + /* Check IP header for IPSec protocols and look it up */ + if (_ODP_IPPROTO_ESP == ip->proto) { + _odp_esphdr_t esp; + + if (odp_packet_copy_to_mem(pkt, ctx->ipsec_offset, sizeof(esp), &esp) < 0) { + ctx->status.error.alg = 1; + return; + } + + if (ipsec_in_check_sa(ctx, ODP_IPSEC_ESP, odp_be_to_cpu_32(esp.spi), &ip->dst_addr) < 0) + return; + + if (odp_packet_copy_to_mem(pkt, ctx->ipsec_offset + _ODP_ESPHDR_LEN, ctx->ipsec_sa->esp_iv_len, ctx->iv) < 0) { + ctx->status.error.alg = 1; + return; + } + + ctx->hdr_len = _ODP_ESPHDR_LEN + ctx->ipsec_sa->esp_iv_len; + ctx->trl_len = _ODP_ESPTRL_LEN + ctx->ipsec_sa->icv_len; + + param.cipher_range.offset = ctx->ipsec_offset + ctx->hdr_len; + param.cipher_range.length = odp_be_to_cpu_16(ip->tot_len) - ip_hdr_len - ctx->hdr_len - ctx->ipsec_sa->icv_len; + param.override_iv_ptr = ctx->iv; + + param.auth_range.offset = ctx->ipsec_offset; + param.auth_range.length = odp_be_to_cpu_16(ip->tot_len) - ip_hdr_len - ctx->ipsec_sa->icv_len; + param.hash_result_offset = ip_offset + odp_be_to_cpu_16(ip->tot_len) - ctx->ipsec_sa->icv_len; + } else if (_ODP_IPPROTO_AH == ip->proto) { + _odp_ahhdr_t ah; + + if (odp_packet_copy_to_mem(pkt, ctx->ipsec_offset, sizeof(ah), &ah) < 0) { + ctx->status.error.alg = 1; + return; + } + + if (ipsec_in_check_sa(ctx, ODP_IPSEC_AH, odp_be_to_cpu_32(ah.spi), &ip->dst_addr) < 0) + return; + + ctx->hdr_len = (ah.ah_len + 2) * 4; + ctx->trl_len = 0; + + /* Save everything to context */ + ctx->ip_tos = ip->tos; + ctx->ip_frag_offset = odp_be_to_cpu_16(ip->frag_offset); + ctx->ip_ttl = ip->ttl; + + /* If authenticating, zero the mutable fields build the request */ + ip->chksum = 0; + ip->tos = 0; + ip->frag_offset = 0; + ip->ttl = 0; + + param.auth_range.offset = ip_offset; + param.auth_range.length = odp_be_to_cpu_16(ip->tot_len); + param.hash_result_offset = ctx->ipsec_offset + _ODP_AHHDR_LEN; + } else { + ctx->status.error.proto = 1; + return; + } + + if (_odp_ipsec_sa_update_stats(ctx->ipsec_sa, odp_packet_len(pkt), &ctx->status) < 0) + return; + + param.session = ctx->ipsec_sa->session; + param.pkt = pkt; + /* Create new packet after all length extensions */ + if (ctx->ipsec_sa->in_place) { + param.out_pkt = pkt; + } else { + param.out_pkt = odp_packet_alloc(odp_packet_pool(pkt), + odp_packet_len(pkt)); + /* uarea will be copied by odp_crypto_operation */ + odp_packet_user_ptr_set(param.out_pkt, + odp_packet_user_ptr(param.pkt)); + } + + rc = odp_crypto_operation(¶m, &posted, &ctx->crypto); + if (rc < 0) { + ODP_DBG("Crypto failed\n"); + ctx->status.error.alg = 1; + return; + } + + ODP_ASSERT(!posted); +} + +static +void ipsec_in_postprocess(ipsec_ctx_t *ctx) +{ + odp_packet_t pkt = ctx->crypto.pkt; + uint32_t ip_offset = odp_packet_l3_offset(pkt); + _odp_ipv4hdr_t *ip = odp_packet_l3_ptr(pkt, NULL); + uint16_t ip_hdr_len = ipv4_hdr_len(ip); + + if (_ODP_IPPROTO_ESP == ip->proto) { + /* + * Finish cipher by finding ESP trailer and processing + */ + _odp_esptrl_t esptrl; + uint32_t esptrl_offset = ip_offset + odp_be_to_cpu_16(ip->tot_len) - ctx->trl_len; + + if (odp_packet_copy_to_mem(pkt, esptrl_offset, sizeof(esptrl), &esptrl) < 0) { + ctx->status.error.proto = 1; + return; + } + + if (ip_offset + esptrl.pad_len > esptrl_offset) { + ctx->status.error.proto = 1; + return; + } + + if (_odp_packet_cmp_data(pkt, esptrl_offset - esptrl.pad_len, ipsec_padding, esptrl.pad_len) != 0) { + ctx->status.error.proto = 1; + return; + } + + ip->proto = esptrl.next_header; + ctx->trl_len += esptrl.pad_len; + } else if (_ODP_IPPROTO_AH == ip->proto) { + /* + * Finish auth + */ + _odp_ahhdr_t ah; + + if (odp_packet_copy_to_mem(pkt, ctx->ipsec_offset, sizeof(ah), &ah) < 0) { + ctx->status.error.alg = 1; + return; + } + + ip->proto = ah.next_header; + + /* Restore mutable fields */ + ip->ttl = ctx->ip_ttl; + ip->tos = ctx->ip_tos; + ip->frag_offset = odp_cpu_to_be_16(ctx->ip_frag_offset); + } else { + ctx->status.error.proto = 1; + return; + } + + if (ctx->ipsec_sa->mode == ODP_IPSEC_MODE_TUNNEL) { + /* FIXME: is this correct place/header ??? */ + ip->ttl -= ctx->ipsec_sa->dec_ttl; + _odp_ipv4_csum_update(pkt); + + /* We have a tunneled IPv4 packet, strip outer and IPsec headers */ + odp_packet_move_data(pkt, ip_hdr_len + ctx->hdr_len, 0, ip_offset); + if (odp_packet_trunc_head(&pkt, ip_hdr_len + ctx->hdr_len, NULL, NULL) < 0) { + ctx->status.error.alg = 1; + return; + } + } else { + /* Finalize the IPv4 header */ + ipv4_adjust_len(ip, -(ctx->hdr_len + ctx->trl_len)); + + _odp_ipv4_csum_update(pkt); + + odp_packet_move_data(pkt, ctx->hdr_len, 0, ip_offset + ip_hdr_len); + if (odp_packet_trunc_head(&pkt, ctx->hdr_len, NULL, NULL) < 0) { + ctx->status.error.alg = 1; + return; + } + } + + if (odp_packet_trunc_tail(&pkt, ctx->trl_len, NULL, NULL) < 0) + ctx->status.error.alg = 1; + + ctx->crypto.pkt = pkt; +} + +/** Helper for calculating encode length using data length and block size */ +#define ESP_ENCODE_LEN(x, b) ((((x) + ((b) - 1)) / (b)) * (b)) + +static +int ipsec_out_extend_packet(ipsec_ctx_t *ctx, odp_packet_t *pkt) +{ + int rc = -1; + + if (odp_packet_extend_tail(pkt, ctx->trl_len, NULL, NULL) < 0) { + ctx->status.error.alg = 1; + goto out; + } + + if (odp_packet_extend_head(pkt, ctx->hdr_len, NULL, NULL) < 0) { + ctx->status.error.alg = 1; + goto out; + } + + odp_packet_move_data(*pkt, 0, ctx->hdr_len, ctx->ipsec_offset); + + return 0; +out: + ctx->crypto.pkt = *pkt; + + return rc; +} + +static void ipsec_out_postprocess(ipsec_ctx_t *ctx); + +static +void ipsec_out_single(ipsec_ctx_t *ctx) +{ + odp_packet_t pkt = ctx->crypto.pkt; + uint32_t ip_offset = odp_packet_l3_offset(pkt); + _odp_ipv4hdr_t *ip = odp_packet_l3_ptr(pkt, NULL); + uint16_t ip_hdr_len = ipv4_hdr_len(ip); + odp_crypto_op_param_t param; + odp_bool_t posted = 0; + int rc = -1; + + ODP_ASSERT(ODP_PACKET_OFFSET_INVALID != ip_offset); + ODP_ASSERT(NULL != ip); + ODP_ASSERT(NULL != ctx->ipsec_sa); + + /* Initialize parameters block */ + memset(¶m, 0, sizeof(param)); + param.ctx = ctx; + + if (ctx->ipsec_sa->mode == ODP_IPSEC_MODE_TUNNEL) { + _odp_ipv4hdr_t out_ip; + + ip->ttl -= ctx->ipsec_sa->dec_ttl; + + out_ip.ver_ihl = 0x45; + out_ip.tos = ip->tos; /* FIXME */ + out_ip.tot_len = odp_cpu_to_be_16(odp_be_to_cpu_16(ip->tot_len) + _ODP_IPV4HDR_LEN); + /* No need to convert to BE: ID just should not be duplicated */ + out_ip.id = odp_atomic_fetch_add_u32(&ctx->ipsec_sa->tun_hdr_id, 1) & 0xffff; + out_ip.frag_offset = 0; + out_ip.ttl = ctx->ipsec_sa->tun_ttl; + out_ip.proto = _ODP_IPV4; + out_ip.src_addr = ctx->ipsec_sa->tun_src_ip; + out_ip.dst_addr = ctx->ipsec_sa->tun_dst_ip; + + if (odp_packet_extend_head(&pkt, _ODP_IPV4HDR_LEN, NULL, NULL) < 0) { + ctx->status.error.alg = 1; + return; + } + ctx->crypto.pkt = pkt; + + odp_packet_move_data(pkt, 0, _ODP_IPV4HDR_LEN, ip_offset); + + odp_packet_copy_from_mem(pkt, ip_offset, _ODP_IPV4HDR_LEN, &out_ip); + + odp_packet_l4_offset_set(pkt, ip_offset + _ODP_IPV4HDR_LEN); + + ip = odp_packet_l3_ptr(pkt, NULL); + ip_hdr_len = _ODP_IPV4HDR_LEN; + } + + ctx->postprocess = ipsec_out_postprocess; + + ctx->ipsec_offset = ip_offset + ip_hdr_len; + + if (ctx->ipsec_sa->proto == ODP_IPSEC_ESP) { + _odp_esphdr_t esp; + _odp_esptrl_t esptrl; + uint32_t encrypt_len; + uint16_t ip_data_len = odp_be_to_cpu_16(ip->tot_len) - ip_hdr_len; + uint32_t pad_block = ctx->ipsec_sa->esp_block_len; + + /* ESP trailer should be 32-bit right aligned */ + if (pad_block < 4) + pad_block = 4; + + encrypt_len = ESP_ENCODE_LEN(ip_data_len + _ODP_ESPTRL_LEN, + pad_block); + + ctx->hdr_len += _ODP_ESPHDR_LEN + ctx->ipsec_sa->esp_iv_len; + ctx->trl_len = encrypt_len - ip_data_len + ctx->ipsec_sa->icv_len; + + if (ctx->ipsec_sa->esp_iv_len) { + /* FIXME: this is correct only for CBC ciphers ! */ + uint32_t len = odp_random_data(ctx->iv, ctx->ipsec_sa->esp_iv_len, ODP_RANDOM_CRYPTO); + if (len != ctx->ipsec_sa->esp_iv_len) { + ctx->status.error.alg = 1; + return; + } + + param.override_iv_ptr = ctx->iv; + } + + if (ipsec_out_extend_packet(ctx, &pkt) < 0) + return; + + ip = odp_packet_l3_ptr(pkt, NULL); + + /* Set IPv4 length before authentication */ + ipv4_adjust_len(ip, ctx->hdr_len + ctx->trl_len); + + uint32_t esptrl_offset = ip_offset + ip_hdr_len + ctx->hdr_len + encrypt_len - _ODP_ESPTRL_LEN; + + memset(&esp, 0, sizeof(esp)); + esp.spi = odp_cpu_to_be_32(ctx->ipsec_sa->spi); + esp.seq_no = odp_cpu_to_be_32(odp_atomic_fetch_add_u32(&ctx->ipsec_sa->seq, 1) + 1); + + memset(&esptrl, 0, sizeof(esptrl)); + esptrl.pad_len = encrypt_len - ip_data_len - _ODP_ESPTRL_LEN; + esptrl.next_header = ip->proto; + ip->proto = _ODP_IPPROTO_ESP; + + odp_packet_copy_from_mem(pkt, ctx->ipsec_offset, _ODP_ESPHDR_LEN, &esp); + /* FIXME: this is correct only for CBC ciphers ! */ + odp_packet_copy_from_mem(pkt, ctx->ipsec_offset + _ODP_ESPHDR_LEN, ctx->ipsec_sa->esp_iv_len, ctx->iv); + odp_packet_copy_from_mem(pkt, esptrl_offset - esptrl.pad_len, esptrl.pad_len, ipsec_padding); + odp_packet_copy_from_mem(pkt, esptrl_offset, _ODP_ESPTRL_LEN, &esptrl); + + param.cipher_range.offset = ctx->ipsec_offset + ctx->hdr_len; + param.cipher_range.length = odp_be_to_cpu_16(ip->tot_len) - ip_hdr_len - ctx->hdr_len - ctx->ipsec_sa->icv_len; + + param.auth_range.offset = ctx->ipsec_offset; + param.auth_range.length = odp_be_to_cpu_16(ip->tot_len) - ip_hdr_len - ctx->ipsec_sa->icv_len; + param.hash_result_offset = ip_offset + odp_be_to_cpu_16(ip->tot_len) - ctx->ipsec_sa->icv_len; + } else if (ctx->ipsec_sa->proto == ODP_IPSEC_AH) { + _odp_ahhdr_t ah; + + ctx->hdr_len = _ODP_AHHDR_LEN + ctx->ipsec_sa->icv_len; + ctx->trl_len = 0; + + /* Save IPv4 stuff */ + ctx->ip_tos = ip->tos; + ctx->ip_frag_offset = odp_be_to_cpu_16(ip->frag_offset); + ctx->ip_ttl = ip->ttl; + + if (ipsec_out_extend_packet(ctx, &pkt) < 0) + return; + + ip = odp_packet_l3_ptr(pkt, NULL); + + /* Set IPv4 length before authentication */ + ipv4_adjust_len(ip, ctx->hdr_len + ctx->trl_len); + + memset(&ah, 0, sizeof(ah)); + ah.spi = odp_cpu_to_be_32(ctx->ipsec_sa->spi); + ah.ah_len = 1 + (ctx->ipsec_sa->icv_len / 4); + ah.seq_no = odp_cpu_to_be_32(odp_atomic_fetch_add_u32(&ctx->ipsec_sa->seq, 1) + 1); + ah.next_header = ip->proto; + ip->proto = _ODP_IPPROTO_AH; + + odp_packet_copy_from_mem(pkt, ctx->ipsec_offset, _ODP_AHHDR_LEN, &ah); + _odp_packet_set_data(pkt, ctx->ipsec_offset + _ODP_AHHDR_LEN, 0, ctx->ipsec_sa->icv_len); + + ip->chksum = 0; + ip->tos = 0; + ip->frag_offset = 0; + ip->ttl = 0; + + param.auth_range.offset = ip_offset; + param.auth_range.length = odp_be_to_cpu_16(ip->tot_len); + param.hash_result_offset = ctx->ipsec_offset + _ODP_AHHDR_LEN; + } else { + ctx->status.error.alg = 1; + return; + } + + if (_odp_ipsec_sa_update_stats(ctx->ipsec_sa, odp_packet_len(pkt), &ctx->status) < 0) + return; + + param.session = ctx->ipsec_sa->session; + param.pkt = pkt; + /* Create new packet after all length extensions */ + if (ctx->ipsec_sa->in_place) { + param.out_pkt = pkt; + } else { + param.out_pkt = odp_packet_alloc(odp_packet_pool(pkt), + odp_packet_len(pkt)); + odp_packet_user_ptr_set(param.out_pkt, + odp_packet_user_ptr(param.pkt)); + } + + rc = odp_crypto_operation(¶m, &posted, &ctx->crypto); + if (rc < 0) { + ODP_DBG("Crypto failed\n"); + ctx->status.error.alg = 1; + return; + } + + ODP_ASSERT(!posted); +} + +static +void ipsec_out_postprocess(ipsec_ctx_t *ctx) +{ + odp_packet_t pkt = ctx->crypto.pkt; + _odp_ipv4hdr_t *ip = odp_packet_l3_ptr(pkt, NULL); + + /* Finalize the IPv4 header */ + if (ip->proto == _ODP_IPPROTO_AH) { + ip->ttl = ctx->ip_ttl; + ip->tos = ctx->ip_tos; + ip->frag_offset = odp_cpu_to_be_16(ctx->ip_frag_offset); + } + + _odp_ipv4_csum_update(pkt); +} + +#if 0 +static odp_ipsec_op_opt_t default_opt = { + .mode = ODP_IPSEC_FRAG_DISABLED, +}; +#endif + int odp_ipsec_in(const odp_ipsec_op_param_t *input, odp_ipsec_op_result_t *output) { - (void)input; - (void)output; + int in_pkt = 0; + int out_pkt = 0; + unsigned sa_idx = 0; + unsigned opt_idx = 0; + unsigned sa_inc = (input->num_sa > 1) ? 1 : 0; + unsigned opt_inc = (input->num_opt > 1) ? 1 : 0; + + while (in_pkt < input->num_pkt && out_pkt < output->num_pkt) { + ipsec_ctx_t ctx; + + ipsec_ctx_init(&ctx, ODP_BUFFER_INVALID); + +#if 0 + odp_ipsec_op_opt_t *opt; + + if (0 == input->num_opt) + opt = &default_opt; + else + opt = &input->opt[opt_idx]; +#endif + + ctx.crypto.pkt = input->pkt[in_pkt]; + + if (0 == input->num_sa) { + ctx.ipsec_sa = NULL; + } else { + ctx.ipsec_sa = _odp_ipsec_sa_use(input->sa[sa_idx]); + ODP_ASSERT(NULL != ctx.ipsec_sa); + } - return -1; + ipsec_in_single(&ctx); + + ipsec_finish(&ctx, &output->res[out_pkt], &output->pkt[out_pkt]); + + in_pkt++; + out_pkt++; + sa_idx += sa_inc; + opt_idx += opt_inc; + } + + return in_pkt; } int odp_ipsec_out(const odp_ipsec_op_param_t *input, - odp_ipsec_op_result_t *output) + odp_ipsec_op_result_t *output) { - (void)input; - (void)output; + int in_pkt = 0; + int out_pkt = 0; + unsigned sa_idx = 0; + unsigned opt_idx = 0; + unsigned sa_inc = (input->num_sa > 1) ? 1 : 0; + unsigned opt_inc = (input->num_opt > 1) ? 1 : 0; + + ODP_ASSERT(input->num_sa != 0); + + while (in_pkt < input->num_pkt && out_pkt < output->num_pkt) { + odp_ipsec_sa_t sa; + ipsec_ctx_t ctx; + + ipsec_ctx_init(&ctx, ODP_BUFFER_INVALID); + + sa = input->sa[sa_idx]; - return -1; + ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); + +#if 0 + odp_ipsec_op_opt_t *opt; + + if (0 == input->num_opt) + opt = &default_opt; + else + opt = &input->opt[opt_idx]; +#endif + + ctx.crypto.pkt = input->pkt[in_pkt]; + ctx.ipsec_sa = _odp_ipsec_sa_use(sa); + + ipsec_out_single(&ctx); + + ipsec_finish(&ctx, &output->res[out_pkt], &output->pkt[out_pkt]); + + in_pkt++; + out_pkt++; + sa_idx += sa_inc; + opt_idx += opt_inc; + } + + return in_pkt; } int odp_ipsec_in_enq(const odp_ipsec_op_param_t *input) { - (void)input; + int in_pkt = 0; + unsigned sa_idx = 0; + unsigned opt_idx = 0; + unsigned sa_inc = (input->num_sa > 1) ? 1 : 0; + unsigned opt_inc = (input->num_opt > 1) ? 1 : 0; + + while (in_pkt < input->num_pkt) { + ipsec_ctx_t *ctx; + odp_queue_t queue; - return -1; + ctx = ipsec_ctx_alloc(); + if (NULL == ctx) + break; + +#if 0 + odp_ipsec_op_opt_t *opt; + + if (0 == input->num_opt) + opt = &default_opt; + else + opt = &input->opt[opt_idx]; +#endif + + ctx->crypto.pkt = input->pkt[in_pkt]; + + if (0 == input->num_sa) { + ctx->ipsec_sa = NULL; + } else { + ctx->ipsec_sa = _odp_ipsec_sa_use(input->sa[sa_idx]); + ODP_ASSERT(NULL != ctx->ipsec_sa); + } + + ipsec_in_single(ctx); + + in_pkt++; + sa_idx += sa_inc; + opt_idx += opt_inc; + + /* IN might have looked up SA for the packet */ + if (NULL == ctx->ipsec_sa) + queue = ipsec_config.inbound.default_queue; + else + queue = ctx->ipsec_sa->queue; + if (odp_unlikely(_odp_ipsec_result_send(queue, ctx) < 0)) { + _odp_ipsec_ctx_free(ctx); + break; + } + } + + return in_pkt; } int odp_ipsec_out_enq(const odp_ipsec_op_param_t *input) { - (void)input; + int in_pkt = 0; + unsigned sa_idx = 0; + unsigned opt_idx = 0; + unsigned sa_inc = (input->num_sa > 1) ? 1 : 0; + unsigned opt_inc = (input->num_opt > 1) ? 1 : 0; + + ODP_ASSERT(input->num_sa != 0); + + while (in_pkt < input->num_pkt) { + odp_ipsec_sa_t sa; + ipsec_ctx_t *ctx; + + ctx = ipsec_ctx_alloc(); + if (NULL == ctx) + break; + + sa = input->sa[sa_idx]; + + ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); + +#if 0 + odp_ipsec_op_opt_t *opt; + + if (0 == input->num_opt) + opt = &default_opt; + else + opt = &input->opt[opt_idx]; +#endif + + ctx->crypto.pkt = input->pkt[in_pkt]; + ctx->ipsec_sa = _odp_ipsec_sa_use(sa); + + ipsec_out_single(ctx); + + in_pkt++; + sa_idx += sa_inc; + opt_idx += opt_inc; + + if (odp_unlikely(_odp_ipsec_result_send(ctx->ipsec_sa->queue, ctx) < 0)) { + _odp_ipsec_ctx_free(ctx); + break; + } + } + + return in_pkt; +} + +static +odp_bool_t _odp_ipsec_out_inline_send(ipsec_ctx_t *ctx, + const odp_ipsec_inline_op_param_t *inline_param) +{ + if (ctx->status.all_error || !ctx->crypto.ok) + return false; + + while (ctx) { + ipsec_ctx_t *next = ctx->next; + odp_ipsec_packet_result_t dummy; + odp_packet_t pkt; + uint32_t offset; + odp_pktout_queue_t queue; + uint32_t hdr_len = inline_param->outer_hdr.len; + + ctx->next = NULL; + + ipsec_finish(ctx, &dummy, &pkt); + offset = odp_packet_l3_offset(pkt); + + if (offset >= hdr_len) { + offset = offset - hdr_len; + } else { + if (odp_packet_extend_head(&pkt, hdr_len - offset, NULL, NULL) < 0) { + ctx->status.error.alg = 1; + goto out; + } + + odp_packet_l3_offset_set(pkt, hdr_len); + + offset = 0; + } + + if (odp_packet_copy_from_mem(pkt, offset, hdr_len, inline_param->outer_hdr.ptr) < 0) { + ctx->status.error.alg = 1; + goto out; + } + + if (odp_pktout_queue(inline_param->pktio, &queue, 1) < 0) { + ctx->status.error.alg = 1; + goto out; + } + + if (odp_pktout_send(queue, &pkt, 1) < 0) { + ctx->status.error.alg = 1; + goto out; + } - return -1; +out: + if (ctx->status.all_error) { + if (odp_unlikely(_odp_ipsec_result_send(ctx->ipsec_sa->queue, ctx) < 0)) + _odp_ipsec_ctx_free(ctx); + } else { + _odp_ipsec_ctx_free(ctx); + } + ctx = next; + } + + return true; } -int odp_ipsec_out_inline(const odp_ipsec_op_param_t *op_param, +int odp_ipsec_out_inline(const odp_ipsec_op_param_t *input, const odp_ipsec_inline_op_param_t *inline_param) { - (void)op_param; - (void)inline_param; + int in_pkt = 0; + unsigned sa_idx = 0; + unsigned opt_idx = 0; + unsigned sa_inc = (input->num_sa > 1) ? 1 : 0; + unsigned opt_inc = (input->num_opt > 1) ? 1 : 0; + + ODP_ASSERT(input->num_sa != 0); + + while (in_pkt < input->num_pkt) { + odp_ipsec_sa_t sa; + ipsec_ctx_t *ctx; + + ctx = ipsec_ctx_alloc(); + if (NULL == ctx) + break; + + sa = input->sa[sa_idx]; + + ODP_ASSERT(ODP_IPSEC_SA_INVALID != sa); + +#if 0 + odp_ipsec_op_opt_t *opt; + + if (0 == input->num_opt) + opt = &default_opt; + else + opt = &input->opt[opt_idx]; +#endif - return -1; + ctx->crypto.pkt = input->pkt[in_pkt]; + ctx->ipsec_sa = _odp_ipsec_sa_use(sa); + + ipsec_out_single(ctx); + + in_pkt++; + sa_idx += sa_inc; + opt_idx += opt_inc; + + /* FIXME: inline_param should have been put into context */ + if (!_odp_ipsec_out_inline_send(ctx, &inline_param[in_pkt - 1])) { + /* In case of an error, submit result event */ + if (odp_unlikely(_odp_ipsec_result_send(ctx->ipsec_sa->queue, ctx) < 0)) { + _odp_ipsec_ctx_free(ctx); + break; + } + } + } + + return in_pkt; } int _odp_ipsec_ctx_result(ipsec_ctx_t *ctx, odp_ipsec_op_result_t *result) { - (void)ctx; - (void)result; + int out_pkt = 0; + + if (NULL == result) + goto count; + + while (NULL != ctx && out_pkt < result->num_pkt) { + ipsec_finish(ctx, &result->res[out_pkt], &result->pkt[out_pkt]); + out_pkt++; + ctx = ctx->next; + } + + result->num_pkt = out_pkt; + +count: + while (NULL != ctx) { + out_pkt++; + ctx = ctx->next; + } - return -1; + return out_pkt; }