diff mbox series

[API-NEXT,v13,2/16] linux-gen: modularize IPsec implementation

Message ID 1513098047-19804-3-git-send-email-odpbot@yandex.ru
State Superseded
Headers show
Series [API-NEXT,v13,1/16] validation: ipsec: add ipv4 name parts | expand

Commit Message

Github ODP bot Dec. 12, 2017, 5 p.m. UTC 
From: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov@linaro.org>


To ease adding IPv6/IPcomp/etc modularize IPsec implementation,
refactoring out functions handling ESP/AH and header parsing/tunneling.

Signed-off-by: Dmitry Eremin-Solenikov <dmitry.ereminsolenikov@linaro.org>

---
/** Email created from pull request 304 (lumag:ipsec-ipv6-2)
 ** https://github.com/Linaro/odp/pull/304
 ** Patch: https://github.com/Linaro/odp/pull/304.patch
 ** Base sha: 65d690fbcf03c6a4d5eb3f01bde36807833feaaa
 ** Merge commit sha: a17bf2b46352969fad4fbaf801dc8b5da0de7a78
 **/
 platform/linux-generic/odp_ipsec.c | 1089 ++++++++++++++++++++----------------
 1 file changed, 597 insertions(+), 492 deletions(-)
diff mbox series

Patch

diff --git a/platform/linux-generic/odp_ipsec.c b/platform/linux-generic/odp_ipsec.c
index 8735e6052..2b9de5771 100644
--- a/platform/linux-generic/odp_ipsec.c
+++ b/platform/linux-generic/odp_ipsec.c
@@ -123,8 +123,8 @@  static inline int _odp_ipv4_csum(odp_packet_t pkt,
 	return 0;
 }
 
-/** @internal Checksum offset in IPv4 header */
-#define _ODP_IPV4HDR_CSUM_OFFSET	10
+#define _ODP_IPV4HDR_CSUM_OFFSET ODP_OFFSETOF(_odp_ipv4hdr_t, chksum)
+#define _ODP_IPV4HDR_PROTO_OFFSET ODP_OFFSETOF(_odp_ipv4hdr_t, proto)
 
 /**
  * Calculate and fill in IPv4 checksum
@@ -158,7 +158,7 @@  static inline int _odp_ipv4_csum_update(odp_packet_t pkt)
 					2, &chksum);
 }
 
-#define ipv4_hdr_len(ip) (_ODP_IPV4HDR_IHL(ip->ver_ihl) * 4)
+#define ipv4_hdr_len(ip) (_ODP_IPV4HDR_IHL((ip)->ver_ihl) * 4)
 static inline
 void ipv4_adjust_len(_odp_ipv4hdr_t *ip, int adj)
 {
@@ -218,200 +218,310 @@  static inline odp_pktio_parser_layer_t parse_layer(odp_ipsec_proto_layer_t l)
 	return ODP_PKTIO_PARSER_LAYER_NONE;
 }
 
-static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt,
-				   odp_ipsec_sa_t sa,
-				   odp_packet_t *pkt_out,
-				   odp_ipsec_op_status_t *status)
-{
-	ipsec_sa_t *ipsec_sa = NULL;
-	uint32_t ip_offset = odp_packet_l3_offset(pkt);
-	_odp_ipv4hdr_t *ip = odp_packet_l3_ptr(pkt, NULL);
-	uint16_t ip_hdr_len = ipv4_hdr_len(ip);
-	odp_crypto_packet_op_param_t param;
-	int rc;
+typedef struct {
+	_odp_ipv4hdr_t *ip;
 	unsigned stats_length;
-	uint16_t ipsec_offset;   /**< Offset of IPsec header from
-				      buffer start */
-	uint8_t	iv[IPSEC_MAX_IV_LEN];  /**< ESP IV storage */
-	ipsec_aad_t aad;         /**< AAD, note ESN is not fully supported */
-	unsigned hdr_len;        /**< Length of IPsec headers */
-	unsigned trl_len;        /**< Length of IPsec trailers */
-	uint8_t  ip_tos;         /**< Saved IP TOS value */
-	uint8_t  ip_ttl;         /**< Saved IP TTL value */
-	uint16_t ip_frag_offset; /**< Saved IP flags value */
-	odp_crypto_packet_result_t crypto; /**< Crypto operation result */
-	odp_packet_hdr_t *pkt_hdr;
+	uint16_t ip_offset;
+	uint16_t ip_hdr_len;
+	uint16_t ip_tot_len;
+	union {
+		struct {
+			uint16_t ip_df;
+			uint8_t  ip_tos;
+		} out_tunnel;
+		struct {
+			uint16_t hdr_len;
+			uint16_t trl_len;
+		} in;
+	};
+	union {
+		struct {
+			uint8_t  tos;
+			uint8_t  ttl;
+			uint16_t frag_offset;
+		} ah_ipv4;
+	};
+	ipsec_aad_t aad;
+	uint8_t	iv[IPSEC_MAX_IV_LEN];
+} ipsec_state_t;
+
+static int ipsec_parse_ipv4(ipsec_state_t *state)
+{
+	if (_ODP_IPV4HDR_IS_FRAGMENT(odp_be_to_cpu_16(state->ip->frag_offset)))
+		return -1;
 
-	ODP_ASSERT(ODP_PACKET_OFFSET_INVALID != ip_offset);
-	ODP_ASSERT(NULL != ip);
+	state->ip_hdr_len = ipv4_hdr_len(state->ip);
+	state->ip_tot_len = odp_be_to_cpu_16(state->ip->tot_len);
 
-	ip_tos = 0;
-	ip_ttl = 0;
-	ip_frag_offset = 0;
+	return 0;
+}
 
-	/* Initialize parameters block */
-	memset(&param, 0, sizeof(param));
+static inline ipsec_sa_t *ipsec_get_sa(odp_ipsec_sa_t sa,
+				       odp_ipsec_protocol_t proto,
+				       uint32_t spi,
+				       void *dst_addr,
+				       odp_ipsec_op_status_t *status)
+{
+	ipsec_sa_t *ipsec_sa;
+
+	if (ODP_IPSEC_SA_INVALID == sa) {
+		ipsec_sa_lookup_t lookup;
+
+		lookup.proto = proto;
+		lookup.spi = spi;
+		lookup.dst_addr = dst_addr;
+
+		ipsec_sa = _odp_ipsec_sa_lookup(&lookup);
+		if (NULL == ipsec_sa) {
+			status->error.sa_lookup = 1;
+			return NULL;
+		}
+	} else {
+		ipsec_sa = _odp_ipsec_sa_use(sa);
+		ODP_ASSERT(NULL != ipsec_sa);
+		if (ipsec_sa->proto != proto ||
+		    ipsec_sa->spi != spi) {
+			status->error.proto = 1;
+			return ipsec_sa;
+		}
+	}
+
+	return ipsec_sa;
+}
 
-	ipsec_offset = ip_offset + ip_hdr_len;
+static int ipsec_in_iv(odp_packet_t pkt,
+		       ipsec_state_t *state,
+		       ipsec_sa_t *ipsec_sa,
+		       uint16_t iv_offset)
+{
+	memcpy(state->iv, ipsec_sa->salt, ipsec_sa->salt_length);
+	if (odp_packet_copy_to_mem(pkt,
+				   iv_offset,
+				   ipsec_sa->esp_iv_len,
+				   state->iv + ipsec_sa->salt_length) < 0)
+		return -1;
 
-	if (odp_be_to_cpu_16(ip->tot_len) + ip_offset > odp_packet_len(pkt)) {
+	if (ipsec_sa->aes_ctr_iv) {
+		state->iv[12] = 0;
+		state->iv[13] = 0;
+		state->iv[14] = 0;
+		state->iv[15] = 1;
+	}
+
+	return 0;
+}
+
+static int ipsec_in_esp(odp_packet_t *pkt,
+			ipsec_state_t *state,
+			ipsec_sa_t **_ipsec_sa,
+			odp_ipsec_sa_t sa,
+			odp_crypto_packet_op_param_t *param,
+			odp_ipsec_op_status_t *status)
+{
+	_odp_esphdr_t esp;
+	uint16_t ipsec_offset;
+	ipsec_sa_t *ipsec_sa;
+
+	ipsec_offset = state->ip_offset + state->ip_hdr_len;
+
+	if (odp_packet_copy_to_mem(*pkt, ipsec_offset,
+				   sizeof(esp), &esp) < 0) {
 		status->error.alg = 1;
-		goto err;
+		return -1;
 	}
 
-	if (_ODP_IPV4HDR_IS_FRAGMENT(odp_be_to_cpu_16(ip->frag_offset))) {
-		status->error.proto = 1;
-		goto err;
+	ipsec_sa = ipsec_get_sa(sa, ODP_IPSEC_ESP,
+				odp_be_to_cpu_32(esp.spi),
+				&state->ip->dst_addr, status);
+	*_ipsec_sa = ipsec_sa;
+	if (status->error.all)
+		return -1;
+
+	if (ipsec_in_iv(*pkt, state, ipsec_sa,
+			ipsec_offset + _ODP_ESPHDR_LEN) < 0) {
+		status->error.alg = 1;
+		return -1;
 	}
 
-	/* Check IP header for IPSec protocols and look it up */
-	if (_ODP_IPPROTO_ESP == ip->proto) {
-		_odp_esphdr_t esp;
+	state->in.hdr_len = _ODP_ESPHDR_LEN + ipsec_sa->esp_iv_len;
+	state->in.trl_len = _ODP_ESPTRL_LEN + ipsec_sa->icv_len;
 
-		if (odp_packet_copy_to_mem(pkt, ipsec_offset,
-					   sizeof(esp), &esp) < 0) {
-			status->error.alg = 1;
-			goto err;
-		}
+	param->cipher_range.offset = ipsec_offset + state->in.hdr_len;
+	param->cipher_range.length = state->ip_tot_len -
+				    state->ip_hdr_len -
+				    state->in.hdr_len -
+				    ipsec_sa->icv_len;
+	param->override_iv_ptr = state->iv;
 
-		if (ODP_IPSEC_SA_INVALID == sa) {
-			ipsec_sa_lookup_t lookup;
+	state->aad.spi = esp.spi;
+	state->aad.seq_no = esp.seq_no;
 
-			lookup.proto = ODP_IPSEC_ESP;
-			lookup.spi = odp_be_to_cpu_32(esp.spi);
-			lookup.dst_addr = &ip->dst_addr;
+	param->aad.ptr = (uint8_t *)&state->aad;
 
-			ipsec_sa = _odp_ipsec_sa_lookup(&lookup);
-			if (NULL == ipsec_sa) {
-				status->error.sa_lookup = 1;
-				goto err;
-			}
-		} else {
-			ipsec_sa = _odp_ipsec_sa_use(sa);
-			ODP_ASSERT(NULL != ipsec_sa);
-			if (ipsec_sa->proto != ODP_IPSEC_ESP ||
-			    ipsec_sa->spi != odp_be_to_cpu_32(esp.spi)) {
-				status->error.proto = 1;
-				goto err;
-			}
-		}
+	param->auth_range.offset = ipsec_offset;
+	param->auth_range.length = state->ip_tot_len -
+				  state->ip_hdr_len -
+				  ipsec_sa->icv_len;
+	param->hash_result_offset = state->ip_offset +
+				   state->ip_tot_len -
+				   ipsec_sa->icv_len;
 
-		memcpy(iv, ipsec_sa->salt, ipsec_sa->salt_length);
-		if (odp_packet_copy_to_mem(pkt,
-					   ipsec_offset + _ODP_ESPHDR_LEN,
-					   ipsec_sa->esp_iv_len,
-					   iv + ipsec_sa->salt_length) < 0) {
-			status->error.alg = 1;
-			goto err;
-		}
+	state->stats_length = param->cipher_range.length;
 
-		if (ipsec_sa->aes_ctr_iv) {
-			iv[12] = 0;
-			iv[13] = 0;
-			iv[14] = 0;
-			iv[15] = 1;
-		}
+	return 0;
+}
 
-		hdr_len = _ODP_ESPHDR_LEN + ipsec_sa->esp_iv_len;
-		trl_len = _ODP_ESPTRL_LEN + ipsec_sa->icv_len;
+static int ipsec_in_esp_post(odp_packet_t pkt,
+			     ipsec_state_t *state)
+{
+	_odp_esptrl_t esptrl;
+	uint32_t esptrl_offset = state->ip_offset +
+				 state->ip_tot_len -
+				 state->in.trl_len;
+
+	if (odp_packet_copy_to_mem(pkt, esptrl_offset,
+				   sizeof(esptrl), &esptrl) < 0 ||
+	    state->ip_offset + esptrl.pad_len > esptrl_offset ||
+	    _odp_packet_cmp_data(pkt, esptrl_offset - esptrl.pad_len,
+				 ipsec_padding, esptrl.pad_len) != 0)
+		return -1;
 
-		param.cipher_range.offset = ipsec_offset + hdr_len;
-		param.cipher_range.length = odp_be_to_cpu_16(ip->tot_len) -
-					    ip_hdr_len -
-					    hdr_len -
-					    ipsec_sa->icv_len;
-		param.override_iv_ptr = iv;
+	state->ip->proto = esptrl.next_header;
+	state->in.trl_len += esptrl.pad_len;
 
-		aad.spi = esp.spi;
-		aad.seq_no = esp.seq_no;
+	return 0;
+}
 
-		param.aad.ptr = (uint8_t *)&aad;
+static int ipsec_in_ah(odp_packet_t *pkt,
+		       ipsec_state_t *state,
+		       ipsec_sa_t **_ipsec_sa,
+		       odp_ipsec_sa_t sa,
+		       odp_crypto_packet_op_param_t *param,
+		       odp_ipsec_op_status_t *status)
+{
+	_odp_ahhdr_t ah;
+	uint16_t ipsec_offset;
+	ipsec_sa_t *ipsec_sa;
 
-		param.auth_range.offset = ipsec_offset;
-		param.auth_range.length = odp_be_to_cpu_16(ip->tot_len) -
-					  ip_hdr_len -
-					  ipsec_sa->icv_len;
-		param.hash_result_offset = ip_offset +
-					   odp_be_to_cpu_16(ip->tot_len) -
-					   ipsec_sa->icv_len;
+	ipsec_offset = state->ip_offset + state->ip_hdr_len;
 
-		stats_length = param.cipher_range.length;
-	} else if (_ODP_IPPROTO_AH == ip->proto) {
-		_odp_ahhdr_t ah;
+	if (odp_packet_copy_to_mem(*pkt, ipsec_offset,
+				   sizeof(ah), &ah) < 0) {
+		status->error.alg = 1;
+		return -1;
+	}
 
-		if (odp_packet_copy_to_mem(pkt, ipsec_offset,
-					   sizeof(ah), &ah) < 0) {
-			status->error.alg = 1;
-			goto err;
-		}
+	ipsec_sa = ipsec_get_sa(sa, ODP_IPSEC_AH,
+				odp_be_to_cpu_32(ah.spi),
+				&state->ip->dst_addr, status);
+	*_ipsec_sa = ipsec_sa;
+	if (status->error.all)
+		return -1;
 
-		if (ODP_IPSEC_SA_INVALID == sa) {
-			ipsec_sa_lookup_t lookup;
+	if (ipsec_in_iv(*pkt, state, ipsec_sa,
+			ipsec_offset + _ODP_AHHDR_LEN) < 0) {
+		status->error.alg = 1;
+		return -1;
+	}
 
-			lookup.proto = ODP_IPSEC_AH;
-			lookup.spi = odp_be_to_cpu_32(ah.spi);
-			lookup.dst_addr = &ip->dst_addr;
+	param->override_iv_ptr = state->iv;
 
-			ipsec_sa = _odp_ipsec_sa_lookup(&lookup);
-			if (NULL == ipsec_sa) {
-				status->error.sa_lookup = 1;
-				goto err;
-			}
-		} else {
-			ipsec_sa = _odp_ipsec_sa_use(sa);
-			ODP_ASSERT(NULL != ipsec_sa);
-			if (ipsec_sa->proto != ODP_IPSEC_AH ||
-			    ipsec_sa->spi != odp_be_to_cpu_32(ah.spi)) {
-				status->error.proto = 1;
-				goto err;
-			}
-		}
+	state->in.hdr_len = (ah.ah_len + 2) * 4;
+	state->in.trl_len = 0;
 
-		memcpy(iv, ipsec_sa->salt, ipsec_sa->salt_length);
-		if (odp_packet_copy_to_mem(pkt,
-					   ipsec_offset + _ODP_AHHDR_LEN,
-					   ipsec_sa->esp_iv_len,
-					   iv + ipsec_sa->salt_length) < 0) {
-			status->error.alg = 1;
-			goto err;
-		}
-		param.override_iv_ptr = iv;
+	/* Save everything to context */
+	state->ah_ipv4.tos = state->ip->tos;
+	state->ah_ipv4.frag_offset = state->ip->frag_offset;
+	state->ah_ipv4.ttl = state->ip->ttl;
+
+	/* FIXME: zero copy of header, passing it to crypto! */
+	/*
+	 * If authenticating, zero the mutable fields build the request
+	 */
+	state->ip->chksum = 0;
+	state->ip->tos = 0;
+	state->ip->frag_offset = 0;
+	state->ip->ttl = 0;
 
-		hdr_len = (ah.ah_len + 2) * 4;
-		trl_len = 0;
+	state->aad.spi = ah.spi;
+	state->aad.seq_no = ah.seq_no;
 
-		/* Save everything to context */
-		ip_tos = ip->tos;
-		ip_frag_offset = odp_be_to_cpu_16(ip->frag_offset);
-		ip_ttl = ip->ttl;
+	param->aad.ptr = (uint8_t *)&state->aad;
 
-		/* FIXME: zero copy of header, passing it to crypto! */
-		/*
-		 * If authenticating, zero the mutable fields build the request
-		 */
-		ip->chksum = 0;
-		ip->tos = 0;
-		ip->frag_offset = 0;
-		ip->ttl = 0;
+	param->auth_range.offset = state->ip_offset;
+	param->auth_range.length = state->ip_tot_len;
+	param->hash_result_offset = ipsec_offset + _ODP_AHHDR_LEN +
+				ipsec_sa->esp_iv_len;
 
-		aad.spi = ah.spi;
-		aad.seq_no = ah.seq_no;
+	state->stats_length = param->auth_range.length;
 
-		param.aad.ptr = (uint8_t *)&aad;
+	return 0;
+}
+
+static int ipsec_in_ah_post(odp_packet_t pkt,
+			    ipsec_state_t *state)
+{
+	_odp_ahhdr_t ah;
+	uint16_t ipsec_offset;
 
-		param.auth_range.offset = ip_offset;
-		param.auth_range.length = odp_be_to_cpu_16(ip->tot_len);
-		param.hash_result_offset = ipsec_offset + _ODP_AHHDR_LEN +
-					ipsec_sa->esp_iv_len;
+	ipsec_offset = state->ip_offset + state->ip_hdr_len;
 
-		stats_length = param.auth_range.length;
+	if (odp_packet_copy_to_mem(pkt, ipsec_offset,
+				   sizeof(ah), &ah) < 0)
+		return -1;
+
+	state->ip->proto = ah.next_header;
+
+	/* Restore mutable fields */
+	state->ip->ttl = state->ah_ipv4.ttl;
+	state->ip->tos = state->ah_ipv4.tos;
+	state->ip->frag_offset = state->ah_ipv4.frag_offset;
+
+	return 0;
+}
+
+static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt,
+				   odp_ipsec_sa_t sa,
+				   odp_packet_t *pkt_out,
+				   odp_ipsec_op_status_t *status)
+{
+	ipsec_state_t state;
+	ipsec_sa_t *ipsec_sa = NULL;
+	odp_crypto_packet_op_param_t param;
+	int rc;
+	odp_crypto_packet_result_t crypto; /**< Crypto operation result */
+	odp_packet_hdr_t *pkt_hdr;
+
+	state.ip_offset = odp_packet_l3_offset(pkt);
+	ODP_ASSERT(ODP_PACKET_OFFSET_INVALID != state.ip_offset);
+
+	state.ip = odp_packet_l3_ptr(pkt, NULL);
+	ODP_ASSERT(NULL != state.ip);
+
+	/* Initialize parameters block */
+	memset(&param, 0, sizeof(param));
+
+	rc = ipsec_parse_ipv4(&state);
+	if (rc < 0 ||
+	    state.ip_tot_len + state.ip_offset > odp_packet_len(pkt)) {
+		status->error.alg = 1;
+		goto err;
+	}
+
+	/* Check IP header for IPSec protocols and look it up */
+	if (_ODP_IPPROTO_ESP == state.ip->proto) {
+		rc = ipsec_in_esp(&pkt, &state, &ipsec_sa, sa, &param, status);
+	} else if (_ODP_IPPROTO_AH == state.ip->proto) {
+		rc = ipsec_in_ah(&pkt, &state, &ipsec_sa, sa, &param, status);
 	} else {
 		status->error.proto = 1;
 		goto err;
 	}
+	if (rc < 0)
+		goto err;
 
 	if (_odp_ipsec_sa_replay_precheck(ipsec_sa,
-					  odp_be_to_cpu_32(aad.seq_no),
+					  odp_be_to_cpu_32(state.aad.seq_no),
 					  status) < 0)
 		goto err;
 
@@ -450,70 +560,30 @@  static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt,
 		goto err;
 	}
 
-	if (_odp_ipsec_sa_stats_update(ipsec_sa, stats_length, status) < 0)
+	if (_odp_ipsec_sa_stats_update(ipsec_sa,
+				       state.stats_length,
+				       status) < 0)
 		goto err;
 
 	if (_odp_ipsec_sa_replay_update(ipsec_sa,
-					odp_be_to_cpu_32(aad.seq_no),
+					odp_be_to_cpu_32(state.aad.seq_no),
 					status) < 0)
 		goto err;
 
-	ip_offset = odp_packet_l3_offset(pkt);
-	ip = odp_packet_l3_ptr(pkt, NULL);
-	ip_hdr_len = ipv4_hdr_len(ip);
-
-	if (_ODP_IPPROTO_ESP == ip->proto) {
-		/*
-		 * Finish cipher by finding ESP trailer and processing
-		 */
-		_odp_esptrl_t esptrl;
-		uint32_t esptrl_offset = ip_offset +
-					 odp_be_to_cpu_16(ip->tot_len) -
-					 trl_len;
-
-		if (odp_packet_copy_to_mem(pkt, esptrl_offset,
-					   sizeof(esptrl), &esptrl) < 0) {
-			status->error.proto = 1;
-			goto err;
-		}
-
-		if (ip_offset + esptrl.pad_len > esptrl_offset) {
-			status->error.proto = 1;
-			goto err;
-		}
-
-		if (_odp_packet_cmp_data(pkt, esptrl_offset - esptrl.pad_len,
-					 ipsec_padding, esptrl.pad_len) != 0) {
-			status->error.proto = 1;
-			goto err;
-		}
+	state.ip = odp_packet_l3_ptr(pkt, NULL);
 
-		ip->proto = esptrl.next_header;
-		trl_len += esptrl.pad_len;
-	} else if (_ODP_IPPROTO_AH == ip->proto) {
-		/*
-		 * Finish auth
-		 */
-		_odp_ahhdr_t ah;
-
-		if (odp_packet_copy_to_mem(pkt, ipsec_offset,
-					   sizeof(ah), &ah) < 0) {
-			status->error.alg = 1;
-			goto err;
-		}
-
-		ip->proto = ah.next_header;
-
-		/* Restore mutable fields */
-		ip->ttl = ip_ttl;
-		ip->tos = ip_tos;
-		ip->frag_offset = odp_cpu_to_be_16(ip_frag_offset);
-	} else {
+	if (ODP_IPSEC_ESP == ipsec_sa->proto)
+		rc = ipsec_in_esp_post(pkt, &state);
+	else if (ODP_IPSEC_AH == ipsec_sa->proto)
+		rc = ipsec_in_ah_post(pkt, &state);
+	else
+		rc = -1;
+	if (rc < 0) {
 		status->error.proto = 1;
 		goto err;
 	}
 
-	if (odp_packet_trunc_tail(&pkt, trl_len, NULL, NULL) < 0) {
+	if (odp_packet_trunc_tail(&pkt, state.in.trl_len, NULL, NULL) < 0) {
 		status->error.alg = 1;
 		goto err;
 	}
@@ -521,32 +591,36 @@  static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt,
 	if (ODP_IPSEC_MODE_TUNNEL == ipsec_sa->mode) {
 		/* We have a tunneled IPv4 packet, strip outer and IPsec
 		 * headers */
-		odp_packet_move_data(pkt, ip_hdr_len + hdr_len, 0,
-				     ip_offset);
-		if (odp_packet_trunc_head(&pkt, ip_hdr_len + hdr_len,
+		odp_packet_move_data(pkt, state.ip_hdr_len + state.in.hdr_len,
+				     0,
+				     state.ip_offset);
+		if (odp_packet_trunc_head(&pkt, state.ip_hdr_len +
+					  state.in.hdr_len,
 					  NULL, NULL) < 0) {
 			status->error.alg = 1;
 			goto err;
 		}
+
+		if (odp_packet_len(pkt) > sizeof(*state.ip)) {
+			state.ip = odp_packet_l3_ptr(pkt, NULL);
+			state.ip->ttl -= ipsec_sa->dec_ttl;
+			_odp_ipv4_csum_update(pkt);
+		}
 	} else {
-		odp_packet_move_data(pkt, hdr_len, 0,
-				     ip_offset + ip_hdr_len);
-		if (odp_packet_trunc_head(&pkt, hdr_len,
+		odp_packet_move_data(pkt, state.in.hdr_len, 0,
+				     state.ip_offset + state.ip_hdr_len);
+		if (odp_packet_trunc_head(&pkt, state.in.hdr_len,
 					  NULL, NULL) < 0) {
 			status->error.alg = 1;
 			goto err;
 		}
-	}
-
-	/* Finalize the IPv4 header */
-	if (odp_packet_len(pkt) > sizeof(*ip)) {
-		ip = odp_packet_l3_ptr(pkt, NULL);
-
-		if (ODP_IPSEC_MODE_TRANSPORT == ipsec_sa->mode)
-			ipv4_adjust_len(ip, -(hdr_len + trl_len));
 
-		ip->ttl -= ipsec_sa->dec_ttl;
-		_odp_ipv4_csum_update(pkt);
+		if (odp_packet_len(pkt) > sizeof(*state.ip)) {
+			state.ip = odp_packet_l3_ptr(pkt, NULL);
+			ipv4_adjust_len(state.ip,
+					-(state.in.hdr_len + state.in.trl_len));
+			_odp_ipv4_csum_update(pkt);
+		}
 	}
 
 	pkt_hdr = odp_packet_hdr(pkt);
@@ -554,7 +628,7 @@  static ipsec_sa_t *ipsec_in_single(odp_packet_t pkt,
 	packet_parse_reset(pkt_hdr);
 
 	packet_parse_l3_l4(pkt_hdr, parse_layer(ipsec_config.inbound.parse),
-			   ip_offset, _ODP_ETHTYPE_IPV4);
+			   state.ip_offset, _ODP_ETHTYPE_IPV4);
 
 	*pkt_out = pkt;
 
@@ -577,317 +651,353 @@  uint32_t ipsec_seq_no(ipsec_sa_t *ipsec_sa)
 }
 
 /* Helper for calculating encode length using data length and block size */
-#define ESP_ENCODE_LEN(x, b) ((((x) + ((b) - 1)) / (b)) * (b))
+#define IPSEC_PAD_LEN(x, b) ((((x) + ((b) - 1)) / (b)) * (b))
 
-static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt,
-				    odp_ipsec_sa_t sa,
-				    odp_packet_t *pkt_out,
-				    const odp_ipsec_out_opt_t *opt ODP_UNUSED,
-				    odp_ipsec_op_status_t *status)
+static int ipsec_out_tunnel_parse_ipv4(ipsec_state_t *state,
+				       ipsec_sa_t *ipsec_sa)
 {
-	ipsec_sa_t *ipsec_sa = NULL;
-	uint32_t ip_offset = odp_packet_l3_offset(pkt);
-	_odp_ipv4hdr_t *ip = odp_packet_l3_ptr(pkt, NULL);
-	uint16_t ip_hdr_len = ipv4_hdr_len(ip);
-	odp_crypto_packet_op_param_t param;
-	unsigned stats_length;
-	int rc;
-	uint16_t ipsec_offset;   /**< Offset of IPsec header from
-				      buffer start */
-	uint8_t	iv[IPSEC_MAX_IV_LEN];  /**< ESP IV storage */
-	ipsec_aad_t aad;         /**< AAD, note ESN is not fully supported */
-	unsigned hdr_len;        /**< Length of IPsec headers */
-	unsigned trl_len;        /**< Length of IPsec trailers */
-	uint8_t  ip_tos;         /**< Saved IP TOS value */
-	uint8_t  ip_ttl;         /**< Saved IP TTL value */
-	uint16_t ip_frag_offset; /**< Saved IP flags value */
-	odp_crypto_packet_result_t crypto; /**< Crypto operation result */
-	odp_packet_hdr_t *pkt_hdr;
+	_odp_ipv4hdr_t *ipv4hdr = state->ip;
+	uint16_t flags = odp_be_to_cpu_16(ipv4hdr->frag_offset);
 
-	ODP_ASSERT(ODP_PACKET_OFFSET_INVALID != ip_offset);
-	ODP_ASSERT(NULL != ip);
+	ipv4hdr->ttl -= ipsec_sa->dec_ttl;
+	state->out_tunnel.ip_tos = ipv4hdr->tos;
+	state->out_tunnel.ip_df = _ODP_IPV4HDR_FLAGS_DONT_FRAG(flags);
 
-	ip_tos = 0;
-	ip_ttl = 0;
-	ip_frag_offset = 0;
+	return 0;
+}
 
-	ipsec_sa = _odp_ipsec_sa_use(sa);
-	ODP_ASSERT(NULL != ipsec_sa);
+static int ipsec_out_tunnel_ipv4(odp_packet_t *pkt,
+				 ipsec_state_t *state,
+				 ipsec_sa_t *ipsec_sa)
+{
+	_odp_ipv4hdr_t out_ip;
+	uint16_t flags;
+
+	out_ip.ver_ihl = 0x45;
+	if (ipsec_sa->copy_dscp)
+		out_ip.tos = state->out_tunnel.ip_tos;
+	else
+		out_ip.tos = (state->out_tunnel.ip_tos &
+			      ~_ODP_IP_TOS_DSCP_MASK) |
+			     (ipsec_sa->out.tun_dscp <<
+			      _ODP_IP_TOS_DSCP_SHIFT);
+	state->ip_tot_len = odp_packet_len(*pkt) - state->ip_offset;
+	state->ip_tot_len += _ODP_IPV4HDR_LEN;
+
+	out_ip.tot_len = odp_cpu_to_be_16(state->ip_tot_len);
+	/* No need to convert to BE: ID just should not be duplicated */
+	out_ip.id = odp_atomic_fetch_add_u32(&ipsec_sa->out.tun_hdr_id,
+					     1);
+	if (ipsec_sa->copy_df)
+		flags = state->out_tunnel.ip_df;
+	else
+		flags = ((uint16_t)ipsec_sa->out.tun_df) << 14;
+	out_ip.frag_offset = odp_cpu_to_be_16(flags);
+	out_ip.ttl = ipsec_sa->out.tun_ttl;
+	out_ip.proto = _ODP_IPPROTO_IPIP;
+	/* Will be filled later by packet checksum update */
+	out_ip.chksum = 0;
+	out_ip.src_addr = ipsec_sa->out.tun_src_ip;
+	out_ip.dst_addr = ipsec_sa->out.tun_dst_ip;
+
+	if (odp_packet_extend_head(pkt, _ODP_IPV4HDR_LEN,
+				   NULL, NULL) < 0)
+		return -1;
 
-	/* Initialize parameters block */
-	memset(&param, 0, sizeof(param));
+	odp_packet_move_data(*pkt, 0, _ODP_IPV4HDR_LEN, state->ip_offset);
 
-	if (ODP_IPSEC_MODE_TRANSPORT == ipsec_sa->mode &&
-	    _ODP_IPV4HDR_IS_FRAGMENT(odp_be_to_cpu_16(ip->frag_offset))) {
-		status->error.alg = 1;
-		goto err;
-	}
+	odp_packet_copy_from_mem(*pkt, state->ip_offset,
+				 _ODP_IPV4HDR_LEN, &out_ip);
 
-	if (odp_be_to_cpu_16(ip->tot_len) + ip_offset > odp_packet_len(pkt)) {
-		status->error.alg = 1;
-		goto err;
-	}
+	odp_packet_l4_offset_set(*pkt, state->ip_offset + _ODP_IPV4HDR_LEN);
 
-	if (ODP_IPSEC_MODE_TUNNEL == ipsec_sa->mode) {
-		_odp_ipv4hdr_t out_ip;
-		uint16_t tot_len;
+	state->ip = odp_packet_l3_ptr(*pkt, NULL);
+	state->ip_hdr_len = _ODP_IPV4HDR_LEN;
 
-		ip->ttl -= ipsec_sa->dec_ttl;
+	return 0;
+}
 
-		out_ip.ver_ihl = 0x45;
-		if (ipsec_sa->copy_dscp)
-			out_ip.tos = ip->tos;
-		else
-			out_ip.tos = (ip->tos & ~_ODP_IP_TOS_DSCP_MASK) |
-				     (ipsec_sa->out.tun_dscp <<
-				      _ODP_IP_TOS_DSCP_SHIFT);
-		tot_len = odp_be_to_cpu_16(ip->tot_len) + _ODP_IPV4HDR_LEN;
-		out_ip.tot_len = odp_cpu_to_be_16(tot_len);
-		/* No need to convert to BE: ID just should not be duplicated */
-		out_ip.id = odp_atomic_fetch_add_u32(&ipsec_sa->out.tun_hdr_id,
-						     1);
-		if (ipsec_sa->copy_df)
-			out_ip.frag_offset = ip->frag_offset & 0x4000;
-		else
-			out_ip.frag_offset =
-				((uint16_t)ipsec_sa->out.tun_df) << 14;
-		out_ip.ttl = ipsec_sa->out.tun_ttl;
-		out_ip.proto = _ODP_IPV4;
-		/* Will be filled later by packet checksum update */
-		out_ip.chksum = 0;
-		out_ip.src_addr = ipsec_sa->out.tun_src_ip;
-		out_ip.dst_addr = ipsec_sa->out.tun_dst_ip;
-
-		if (odp_packet_extend_head(&pkt, _ODP_IPV4HDR_LEN,
-					   NULL, NULL) < 0) {
-			status->error.alg = 1;
-			goto err;
+static int ipsec_out_iv(ipsec_state_t *state,
+			ipsec_sa_t *ipsec_sa)
+{
+	if (ipsec_sa->use_counter_iv) {
+		uint64_t ctr;
+
+		/* Both GCM and CTR use 8-bit counters */
+		ODP_ASSERT(sizeof(ctr) == ipsec_sa->esp_iv_len);
+
+		ctr = odp_atomic_fetch_add_u64(&ipsec_sa->out.counter,
+					       1);
+		/* Check for overrun */
+		if (ctr == 0)
+			return -1;
+
+		memcpy(state->iv, ipsec_sa->salt, ipsec_sa->salt_length);
+		memcpy(state->iv + ipsec_sa->salt_length, &ctr,
+		       ipsec_sa->esp_iv_len);
+
+		if (ipsec_sa->aes_ctr_iv) {
+			state->iv[12] = 0;
+			state->iv[13] = 0;
+			state->iv[14] = 0;
+			state->iv[15] = 1;
 		}
+	} else if (ipsec_sa->esp_iv_len) {
+		uint32_t len;
 
-		odp_packet_move_data(pkt, 0, _ODP_IPV4HDR_LEN, ip_offset);
+		len = odp_random_data(state->iv, ipsec_sa->esp_iv_len,
+				      ODP_RANDOM_CRYPTO);
 
-		odp_packet_copy_from_mem(pkt, ip_offset,
-					 _ODP_IPV4HDR_LEN, &out_ip);
+		if (len != ipsec_sa->esp_iv_len)
+			return -1;
+	}
 
-		odp_packet_l4_offset_set(pkt, ip_offset + _ODP_IPV4HDR_LEN);
+	return 0;
+}
 
-		ip = odp_packet_l3_ptr(pkt, NULL);
-		ip_hdr_len = _ODP_IPV4HDR_LEN;
+static int ipsec_out_esp(odp_packet_t *pkt,
+			 ipsec_state_t *state,
+			 ipsec_sa_t *ipsec_sa,
+			 odp_crypto_packet_op_param_t *param,
+			 odp_ipsec_op_status_t *status)
+{
+	_odp_esphdr_t esp;
+	_odp_esptrl_t esptrl;
+	uint32_t encrypt_len;
+	uint16_t ip_data_len = state->ip_tot_len -
+			       state->ip_hdr_len;
+	uint32_t pad_block = ipsec_sa->esp_block_len;
+	uint16_t ipsec_offset = state->ip_offset + state->ip_hdr_len;
+	unsigned hdr_len;
+	unsigned trl_len;
+
+	/* ESP trailer should be 32-bit right aligned */
+	if (pad_block < 4)
+		pad_block = 4;
+
+	encrypt_len = IPSEC_PAD_LEN(ip_data_len + _ODP_ESPTRL_LEN,
+				    pad_block);
+
+	hdr_len = _ODP_ESPHDR_LEN + ipsec_sa->esp_iv_len;
+	trl_len = encrypt_len -
+		       ip_data_len +
+		       ipsec_sa->icv_len;
+
+	if (ipsec_out_iv(state, ipsec_sa) < 0) {
+		status->error.alg = 1;
+		return -1;
 	}
 
-	ipsec_offset = ip_offset + ip_hdr_len;
+	param->override_iv_ptr = state->iv;
 
-	if (ipsec_sa->proto == ODP_IPSEC_ESP) {
-		_odp_esphdr_t esp;
-		_odp_esptrl_t esptrl;
-		uint32_t encrypt_len;
-		uint16_t ip_data_len = odp_be_to_cpu_16(ip->tot_len) -
-				       ip_hdr_len;
-		uint32_t pad_block = ipsec_sa->esp_block_len;
+	if (odp_packet_extend_tail(pkt, trl_len, NULL, NULL) < 0 ||
+	    odp_packet_extend_head(pkt, hdr_len, NULL, NULL) < 0) {
+		status->error.alg = 1;
+		return -1;
+	}
 
-		/* ESP trailer should be 32-bit right aligned */
-		if (pad_block < 4)
-			pad_block = 4;
+	odp_packet_move_data(*pkt, 0, hdr_len, ipsec_offset);
+
+	state->ip = odp_packet_l3_ptr(*pkt, NULL);
+
+	/* Set IPv4 length before authentication */
+	ipv4_adjust_len(state->ip, hdr_len + trl_len);
+	state->ip_tot_len += hdr_len + trl_len;
+
+	uint32_t esptrl_offset = state->ip_offset +
+				 state->ip_hdr_len +
+				 hdr_len +
+				 encrypt_len -
+				 _ODP_ESPTRL_LEN;
+
+	memset(&esp, 0, sizeof(esp));
+	esp.spi = odp_cpu_to_be_32(ipsec_sa->spi);
+	esp.seq_no = odp_cpu_to_be_32(ipsec_seq_no(ipsec_sa));
+
+	state->aad.spi = esp.spi;
+	state->aad.seq_no = esp.seq_no;
+
+	param->aad.ptr = (uint8_t *)&state->aad;
+
+	memset(&esptrl, 0, sizeof(esptrl));
+	esptrl.pad_len = encrypt_len - ip_data_len - _ODP_ESPTRL_LEN;
+	esptrl.next_header = state->ip->proto;
+	state->ip->proto = _ODP_IPPROTO_ESP;
+
+	odp_packet_copy_from_mem(*pkt,
+				 ipsec_offset, _ODP_ESPHDR_LEN,
+				 &esp);
+	odp_packet_copy_from_mem(*pkt,
+				 ipsec_offset + _ODP_ESPHDR_LEN,
+				 ipsec_sa->esp_iv_len,
+				 state->iv + ipsec_sa->salt_length);
+	odp_packet_copy_from_mem(*pkt,
+				 esptrl_offset - esptrl.pad_len,
+				 esptrl.pad_len, ipsec_padding);
+	odp_packet_copy_from_mem(*pkt,
+				 esptrl_offset, _ODP_ESPTRL_LEN,
+				 &esptrl);
+
+	param->cipher_range.offset = ipsec_offset + hdr_len;
+	param->cipher_range.length = state->ip_tot_len -
+				    state->ip_hdr_len -
+				    hdr_len -
+				    ipsec_sa->icv_len;
+
+	param->auth_range.offset = ipsec_offset;
+	param->auth_range.length = state->ip_tot_len -
+				  state->ip_hdr_len -
+				  ipsec_sa->icv_len;
+	param->hash_result_offset = state->ip_offset +
+				   state->ip_tot_len -
+				   ipsec_sa->icv_len;
+
+	state->stats_length = param->cipher_range.length;
 
-		encrypt_len = ESP_ENCODE_LEN(ip_data_len + _ODP_ESPTRL_LEN,
-					     pad_block);
+	return 0;
+}
 
-		hdr_len = _ODP_ESPHDR_LEN + ipsec_sa->esp_iv_len;
-		trl_len = encrypt_len -
-			       ip_data_len +
-			       ipsec_sa->icv_len;
+static int ipsec_out_ah(odp_packet_t *pkt,
+			ipsec_state_t *state,
+			ipsec_sa_t *ipsec_sa,
+			odp_crypto_packet_op_param_t *param,
+			odp_ipsec_op_status_t *status)
+{
+	_odp_ahhdr_t ah;
+	unsigned hdr_len = _ODP_AHHDR_LEN + ipsec_sa->esp_iv_len +
+		ipsec_sa->icv_len;
+	uint16_t ipsec_offset = state->ip_offset + state->ip_hdr_len;
 
-		if (ipsec_sa->use_counter_iv) {
-			uint64_t ctr;
+	/* Save IPv4 stuff */
+	state->ah_ipv4.tos = state->ip->tos;
+	state->ah_ipv4.frag_offset = state->ip->frag_offset;
+	state->ah_ipv4.ttl = state->ip->ttl;
 
-			/* Both GCM and CTR use 8-bit counters */
-			ODP_ASSERT(sizeof(ctr) == ipsec_sa->esp_iv_len);
+	if (odp_packet_extend_head(pkt, hdr_len, NULL, NULL) < 0) {
+		status->error.alg = 1;
+		return -1;
+	}
 
-			ctr = odp_atomic_fetch_add_u64(&ipsec_sa->out.counter,
-						       1);
-			/* Check for overrun */
-			if (ctr == 0)
-				goto err;
+	odp_packet_move_data(*pkt, 0, hdr_len, ipsec_offset);
 
-			memcpy(iv, ipsec_sa->salt, ipsec_sa->salt_length);
-			memcpy(iv + ipsec_sa->salt_length, &ctr,
-			       ipsec_sa->esp_iv_len);
+	state->ip = odp_packet_l3_ptr(*pkt, NULL);
 
-			if (ipsec_sa->aes_ctr_iv) {
-				iv[12] = 0;
-				iv[13] = 0;
-				iv[14] = 0;
-				iv[15] = 1;
-			}
-		} else if (ipsec_sa->esp_iv_len) {
-			uint32_t len;
+	/* Set IPv4 length before authentication */
+	ipv4_adjust_len(state->ip, hdr_len);
+	state->ip_tot_len += hdr_len;
 
-			len = odp_random_data(iv, ipsec_sa->esp_iv_len,
-					      ODP_RANDOM_CRYPTO);
+	memset(&ah, 0, sizeof(ah));
+	ah.spi = odp_cpu_to_be_32(ipsec_sa->spi);
+	ah.ah_len = 1 + (ipsec_sa->esp_iv_len + ipsec_sa->icv_len) / 4;
+	ah.seq_no = odp_cpu_to_be_32(ipsec_seq_no(ipsec_sa));
+	ah.next_header = state->ip->proto;
+	state->ip->proto = _ODP_IPPROTO_AH;
 
-			if (len != ipsec_sa->esp_iv_len) {
-				status->error.alg = 1;
-				goto err;
-			}
-		}
+	state->aad.spi = ah.spi;
+	state->aad.seq_no = ah.seq_no;
 
-		param.override_iv_ptr = iv;
+	param->aad.ptr = (uint8_t *)&state->aad;
 
-		if (odp_packet_extend_tail(&pkt, trl_len, NULL, NULL) < 0) {
-			status->error.alg = 1;
-			goto err;
-		}
+	/* For GMAC */
+	if (ipsec_out_iv(state, ipsec_sa) < 0) {
+		status->error.alg = 1;
+		return -1;
+	}
 
-		if (odp_packet_extend_head(&pkt, hdr_len, NULL, NULL) < 0) {
-			status->error.alg = 1;
-			goto err;
-		}
+	param->override_iv_ptr = state->iv;
 
-		odp_packet_move_data(pkt, 0, hdr_len, ipsec_offset);
-
-		ip = odp_packet_l3_ptr(pkt, NULL);
-
-		/* Set IPv4 length before authentication */
-		ipv4_adjust_len(ip, hdr_len + trl_len);
-
-		uint32_t esptrl_offset = ip_offset +
-					 ip_hdr_len +
-					 hdr_len +
-					 encrypt_len -
-					 _ODP_ESPTRL_LEN;
-
-		memset(&esp, 0, sizeof(esp));
-		esp.spi = odp_cpu_to_be_32(ipsec_sa->spi);
-		esp.seq_no = odp_cpu_to_be_32(ipsec_seq_no(ipsec_sa));
-
-		aad.spi = esp.spi;
-		aad.seq_no = esp.seq_no;
-
-		param.aad.ptr = (uint8_t *)&aad;
-
-		memset(&esptrl, 0, sizeof(esptrl));
-		esptrl.pad_len = encrypt_len - ip_data_len - _ODP_ESPTRL_LEN;
-		esptrl.next_header = ip->proto;
-		ip->proto = _ODP_IPPROTO_ESP;
-
-		odp_packet_copy_from_mem(pkt,
-					 ipsec_offset, _ODP_ESPHDR_LEN,
-					 &esp);
-		odp_packet_copy_from_mem(pkt,
-					 ipsec_offset + _ODP_ESPHDR_LEN,
-					 ipsec_sa->esp_iv_len,
-					 iv + ipsec_sa->salt_length);
-		odp_packet_copy_from_mem(pkt,
-					 esptrl_offset - esptrl.pad_len,
-					 esptrl.pad_len, ipsec_padding);
-		odp_packet_copy_from_mem(pkt,
-					 esptrl_offset, _ODP_ESPTRL_LEN,
-					 &esptrl);
-
-		param.cipher_range.offset = ipsec_offset + hdr_len;
-		param.cipher_range.length = odp_be_to_cpu_16(ip->tot_len) -
-					    ip_hdr_len -
-					    hdr_len -
-					    ipsec_sa->icv_len;
-
-		param.auth_range.offset = ipsec_offset;
-		param.auth_range.length = odp_be_to_cpu_16(ip->tot_len) -
-					  ip_hdr_len -
-					  ipsec_sa->icv_len;
-		param.hash_result_offset = ip_offset +
-					   odp_be_to_cpu_16(ip->tot_len) -
-					   ipsec_sa->icv_len;
-
-		stats_length = param.cipher_range.length;
-	} else if (ipsec_sa->proto == ODP_IPSEC_AH) {
-		_odp_ahhdr_t ah;
-
-		hdr_len = _ODP_AHHDR_LEN + ipsec_sa->esp_iv_len +
-			ipsec_sa->icv_len;
-		trl_len = 0;
-
-		/* Save IPv4 stuff */
-		ip_tos = ip->tos;
-		ip_frag_offset = odp_be_to_cpu_16(ip->frag_offset);
-		ip_ttl = ip->ttl;
-
-		if (odp_packet_extend_tail(&pkt, trl_len, NULL, NULL) < 0) {
-			status->error.alg = 1;
-			goto err;
-		}
+	odp_packet_copy_from_mem(*pkt,
+				 ipsec_offset, _ODP_AHHDR_LEN,
+				 &ah);
+	odp_packet_copy_from_mem(*pkt,
+				 ipsec_offset + _ODP_AHHDR_LEN,
+				 ipsec_sa->esp_iv_len,
+				 state->iv + ipsec_sa->salt_length);
+	_odp_packet_set_data(*pkt,
+			     ipsec_offset + _ODP_AHHDR_LEN +
+			       ipsec_sa->esp_iv_len,
+			     0, ipsec_sa->icv_len);
 
-		if (odp_packet_extend_head(&pkt, hdr_len, NULL, NULL) < 0) {
-			status->error.alg = 1;
-			goto err;
-		}
+	state->ip->chksum = 0;
+	state->ip->tos = 0;
+	state->ip->frag_offset = 0;
+	state->ip->ttl = 0;
 
-		odp_packet_move_data(pkt, 0, hdr_len, ipsec_offset);
+	param->auth_range.offset = state->ip_offset;
+	param->auth_range.length = state->ip_tot_len;
+	param->hash_result_offset = ipsec_offset + _ODP_AHHDR_LEN +
+				ipsec_sa->esp_iv_len;
 
-		ip = odp_packet_l3_ptr(pkt, NULL);
+	state->stats_length = param->auth_range.length;
 
-		/* Set IPv4 length before authentication */
-		ipv4_adjust_len(ip, hdr_len + trl_len);
+	return 0;
+}
 
-		memset(&ah, 0, sizeof(ah));
-		ah.spi = odp_cpu_to_be_32(ipsec_sa->spi);
-		ah.ah_len = 1 + (ipsec_sa->esp_iv_len + ipsec_sa->icv_len) / 4;
-		ah.seq_no = odp_cpu_to_be_32(ipsec_seq_no(ipsec_sa));
-		ah.next_header = ip->proto;
-		ip->proto = _ODP_IPPROTO_AH;
+static void ipsec_out_ah_post(ipsec_state_t *state)
+{
+	state->ip->ttl = state->ah_ipv4.ttl;
+	state->ip->tos = state->ah_ipv4.tos;
+	state->ip->frag_offset = state->ah_ipv4.frag_offset;
+}
 
-		aad.spi = ah.spi;
-		aad.seq_no = ah.seq_no;
+static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt,
+				    odp_ipsec_sa_t sa,
+				    odp_packet_t *pkt_out,
+				    const odp_ipsec_out_opt_t *opt ODP_UNUSED,
+				    odp_ipsec_op_status_t *status)
+{
+	ipsec_state_t state;
+	ipsec_sa_t *ipsec_sa;
+	odp_crypto_packet_op_param_t param;
+	int rc;
+	odp_crypto_packet_result_t crypto; /**< Crypto operation result */
+	odp_packet_hdr_t *pkt_hdr;
 
-		param.aad.ptr = (uint8_t *)&aad;
+	state.ip_offset = odp_packet_l3_offset(pkt);
+	ODP_ASSERT(ODP_PACKET_OFFSET_INVALID != state.ip_offset);
 
-		/* For GMAC */
-		if (ipsec_sa->use_counter_iv) {
-			uint64_t ctr;
+	state.ip = odp_packet_l3_ptr(pkt, NULL);
+	ODP_ASSERT(NULL != state.ip);
 
-			ODP_ASSERT(sizeof(ctr) == ipsec_sa->esp_iv_len);
+	ipsec_sa = _odp_ipsec_sa_use(sa);
+	ODP_ASSERT(NULL != ipsec_sa);
 
-			ctr = odp_atomic_fetch_add_u64(&ipsec_sa->out.counter,
-						       1);
-			/* Check for overrun */
-			if (ctr == 0)
-				goto err;
+	/* Initialize parameters block */
+	memset(&param, 0, sizeof(param));
 
-			memcpy(iv, ipsec_sa->salt, ipsec_sa->salt_length);
-			memcpy(iv + ipsec_sa->salt_length, &ctr,
-			       ipsec_sa->esp_iv_len);
-			param.override_iv_ptr = iv;
+	if (ODP_IPSEC_MODE_TRANSPORT == ipsec_sa->mode) {
+		rc = ipsec_parse_ipv4(&state);
+		if (state.ip_tot_len + state.ip_offset != odp_packet_len(pkt))
+			rc = -1;
+	} else {
+		rc = ipsec_out_tunnel_parse_ipv4(&state, ipsec_sa);
+		if (rc < 0) {
+			status->error.alg = 1;
+			goto err;
 		}
 
-		odp_packet_copy_from_mem(pkt,
-					 ipsec_offset, _ODP_AHHDR_LEN,
-					 &ah);
-		odp_packet_copy_from_mem(pkt,
-					 ipsec_offset + _ODP_AHHDR_LEN,
-					 ipsec_sa->esp_iv_len,
-					 iv + ipsec_sa->salt_length);
-		_odp_packet_set_data(pkt,
-				     ipsec_offset + _ODP_AHHDR_LEN +
-				       ipsec_sa->esp_iv_len,
-				     0, ipsec_sa->icv_len);
-
-		ip->chksum = 0;
-		ip->tos = 0;
-		ip->frag_offset = 0;
-		ip->ttl = 0;
-
-		param.auth_range.offset = ip_offset;
-		param.auth_range.length = odp_be_to_cpu_16(ip->tot_len);
-		param.hash_result_offset = ipsec_offset + _ODP_AHHDR_LEN +
-					ipsec_sa->esp_iv_len;
-
-		stats_length = param.auth_range.length;
+		rc = ipsec_out_tunnel_ipv4(&pkt, &state, ipsec_sa);
+	}
+	if (rc < 0) {
+		status->error.alg = 1;
+		goto err;
+	}
+
+	if (ODP_IPSEC_ESP == ipsec_sa->proto) {
+		rc = ipsec_out_esp(&pkt, &state, ipsec_sa, &param, status);
+	} else if (ODP_IPSEC_AH == ipsec_sa->proto) {
+		rc = ipsec_out_ah(&pkt, &state, ipsec_sa, &param, status);
 	} else {
 		status->error.alg = 1;
 		goto err;
 	}
+	if (rc < 0)
+		goto err;
 
 	/* No need to run precheck here, we know that packet is authentic */
-	if (_odp_ipsec_sa_stats_update(ipsec_sa, stats_length, status) < 0)
+	if (_odp_ipsec_sa_stats_update(ipsec_sa,
+				       state.stats_length,
+				       status) < 0)
 		goto err;
 
 	param.session = ipsec_sa->session;
@@ -922,14 +1032,9 @@  static ipsec_sa_t *ipsec_out_single(odp_packet_t pkt,
 		goto err;
 	}
 
-	ip = odp_packet_l3_ptr(pkt, NULL);
-
 	/* Finalize the IPv4 header */
-	if (ip->proto == _ODP_IPPROTO_AH) {
-		ip->ttl = ip_ttl;
-		ip->tos = ip_tos;
-		ip->frag_offset = odp_cpu_to_be_16(ip_frag_offset);
-	}
+	if (ODP_IPSEC_AH == ipsec_sa->proto)
+		ipsec_out_ah_post(&state);
 
 	_odp_ipv4_csum_update(pkt);