@@ -740,6 +740,109 @@ static int caam_ctrl_rng_init(struct device *dev)
return 0;
}
+/* Indicate if the internal state of the CAAM is lost during PM */
+static int caam_off_during_pm(void)
+{
+ bool not_off_during_pm = of_machine_is_compatible("fsl,imx6q") ||
+ of_machine_is_compatible("fsl,imx6qp") ||
+ of_machine_is_compatible("fsl,imx6dl");
+
+ return not_off_during_pm ? 0 : 1;
+}
+
+static void caam_state_save(struct device *dev)
+{
+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev);
+ struct caam_ctl_state *state = &ctrlpriv->state;
+ struct caam_ctrl __iomem *ctrl = ctrlpriv->ctrl;
+ u32 deco_inst, jr_inst;
+ int i;
+
+ state->mcr = rd_reg32(&ctrl->mcr);
+ state->scfgr = rd_reg32(&ctrl->scfgr);
+
+ deco_inst = (rd_reg32(&ctrl->perfmon.cha_num_ms) &
+ CHA_ID_MS_DECO_MASK) >> CHA_ID_MS_DECO_SHIFT;
+ for (i = 0; i < deco_inst; i++) {
+ state->deco_mid[i].liodn_ms =
+ rd_reg32(&ctrl->deco_mid[i].liodn_ms);
+ state->deco_mid[i].liodn_ls =
+ rd_reg32(&ctrl->deco_mid[i].liodn_ls);
+ }
+
+ jr_inst = (rd_reg32(&ctrl->perfmon.cha_num_ms) &
+ CHA_ID_MS_JR_MASK) >> CHA_ID_MS_JR_SHIFT;
+ for (i = 0; i < jr_inst; i++) {
+ state->jr_mid[i].liodn_ms =
+ rd_reg32(&ctrl->jr_mid[i].liodn_ms);
+ state->jr_mid[i].liodn_ls =
+ rd_reg32(&ctrl->jr_mid[i].liodn_ls);
+ }
+}
+
+static void caam_state_restore(const struct device *dev)
+{
+ const struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev);
+ const struct caam_ctl_state *state = &ctrlpriv->state;
+ struct caam_ctrl __iomem *ctrl = ctrlpriv->ctrl;
+ u32 deco_inst, jr_inst;
+ int i;
+
+ wr_reg32(&ctrl->mcr, state->mcr);
+ wr_reg32(&ctrl->scfgr, state->scfgr);
+
+ deco_inst = (rd_reg32(&ctrl->perfmon.cha_num_ms) &
+ CHA_ID_MS_DECO_MASK) >> CHA_ID_MS_DECO_SHIFT;
+ for (i = 0; i < deco_inst; i++) {
+ wr_reg32(&ctrl->deco_mid[i].liodn_ms,
+ state->deco_mid[i].liodn_ms);
+ wr_reg32(&ctrl->deco_mid[i].liodn_ls,
+ state->deco_mid[i].liodn_ls);
+ }
+
+ jr_inst = (rd_reg32(&ctrl->perfmon.cha_num_ms) &
+ CHA_ID_MS_JR_MASK) >> CHA_ID_MS_JR_SHIFT;
+ for (i = 0; i < jr_inst; i++) {
+ wr_reg32(&ctrl->jr_mid[i].liodn_ms,
+ state->jr_mid[i].liodn_ms);
+ wr_reg32(&ctrl->jr_mid[i].liodn_ls,
+ state->jr_mid[i].liodn_ls);
+ }
+
+ if (ctrlpriv->virt_en == 1)
+ clrsetbits_32(&ctrl->jrstart, 0, JRSTART_JR0_START |
+ JRSTART_JR1_START | JRSTART_JR2_START |
+ JRSTART_JR3_START);
+}
+
+static int caam_ctrl_suspend(struct device *dev)
+{
+ const struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev);
+
+ if (ctrlpriv->caam_off_during_pm && !ctrlpriv->optee_en)
+ caam_state_save(dev);
+
+ return 0;
+}
+
+static int caam_ctrl_resume(struct device *dev)
+{
+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev);
+ int ret = 0;
+
+ if (ctrlpriv->caam_off_during_pm && !ctrlpriv->optee_en) {
+ caam_state_restore(dev);
+
+ /* HW and rng will be reset so deinstantiation can be removed */
+ devm_remove_action(dev, devm_deinstantiate_rng, dev);
+ ret = caam_ctrl_rng_init(dev);
+ }
+
+ return ret;
+}
+
+static SIMPLE_DEV_PM_OPS(caam_ctrl_pm_ops, caam_ctrl_suspend, caam_ctrl_resume);
+
/* Probe routine for CAAM top (controller) level */
static int caam_probe(struct platform_device *pdev)
{
@@ -771,6 +874,8 @@ static int caam_probe(struct platform_device *pdev)
caam_imx = (bool)imx_soc_match;
+ ctrlpriv->caam_off_during_pm = caam_imx && caam_off_during_pm();
+
if (imx_soc_match) {
/*
* Until Layerscape and i.MX OP-TEE get in sync,
@@ -1033,6 +1138,7 @@ static struct platform_driver caam_driver = {
.driver = {
.name = "caam",
.of_match_table = caam_match,
+ .pm = &caam_ctrl_pm_ops,
},
.probe = caam_probe,
};
@@ -4,7 +4,7 @@
* Private/internal definitions between modules
*
* Copyright 2008-2011 Freescale Semiconductor, Inc.
- * Copyright 2019 NXP
+ * Copyright 2019, 2023 NXP
*/
#ifndef INTERN_H
@@ -47,6 +47,16 @@ struct caam_jrentry_info {
u32 desc_size; /* Stored size for postprocessing, header derived */
};
+struct caam_jr_state {
+ dma_addr_t inpbusaddr;
+ dma_addr_t outbusaddr;
+};
+
+struct caam_jr_dequeue_params {
+ struct device *dev;
+ int enable_itr;
+};
+
/* Private sub-storage for a single JobR */
struct caam_drv_private_jr {
struct list_head list_node; /* Job Ring device list */
@@ -54,6 +64,7 @@ struct caam_drv_private_jr {
int ridx;
struct caam_job_ring __iomem *rregs; /* JobR's register space */
struct tasklet_struct irqtask;
+ struct caam_jr_dequeue_params tasklet_params;
int irq; /* One per queue */
bool hwrng;
@@ -71,6 +82,15 @@ struct caam_drv_private_jr {
int tail; /* entinfo (s/w ring) tail index */
void *outring; /* Base of output ring, DMA-safe */
struct crypto_engine *engine;
+
+ struct caam_jr_state state; /* State of the JR during PM */
+};
+
+struct caam_ctl_state {
+ struct masterid deco_mid[16];
+ struct masterid jr_mid[4];
+ u32 mcr;
+ u32 scfgr;
};
/*
@@ -116,6 +136,9 @@ struct caam_drv_private {
struct dentry *ctl; /* controller dir */
struct debugfs_blob_wrapper ctl_kek_wrap, ctl_tkek_wrap, ctl_tdsk_wrap;
#endif
+
+ int caam_off_during_pm; /* If the CAAM is reset after suspend */
+ struct caam_ctl_state state; /* State of the CTL during PM */
};
#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API
@@ -117,6 +117,23 @@ static int caam_jr_flush(struct device *dev)
return caam_jr_stop_processing(dev, JRCR_RESET);
}
+/* The resume can be used after a park or a flush if CAAM has not been reset */
+static int caam_jr_restart_processing(struct device *dev)
+{
+ struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
+ u32 halt_status = rd_reg32(&jrp->rregs->jrintstatus) &
+ JRINT_ERR_HALT_MASK;
+
+ /* Check that the flush/park is completed */
+ if (halt_status != JRINT_ERR_HALT_COMPLETE)
+ return -1;
+
+ /* Resume processing of jobs */
+ clrsetbits_32(&jrp->rregs->jrintstatus, 0, JRINT_ERR_HALT_COMPLETE);
+
+ return 0;
+}
+
static int caam_reset_hw_jr(struct device *dev)
{
struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
@@ -245,7 +262,8 @@ static irqreturn_t caam_jr_interrupt(int irq, void *st_dev)
static void caam_jr_dequeue(unsigned long devarg)
{
int hw_idx, sw_idx, i, head, tail;
- struct device *dev = (struct device *)devarg;
+ struct caam_jr_dequeue_params *params = (void *)devarg;
+ struct device *dev = params->dev;
struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg);
u32 *userdesc, userstatus;
@@ -319,8 +337,9 @@ static void caam_jr_dequeue(unsigned long devarg)
outring_used--;
}
- /* reenable / unmask IRQs */
- clrsetbits_32(&jrp->rregs->rconfig_lo, JRCFG_IMSK, 0);
+ if (params->enable_itr)
+ /* reenable / unmask IRQs */
+ clrsetbits_32(&jrp->rregs->rconfig_lo, JRCFG_IMSK, 0);
}
/**
@@ -470,6 +489,29 @@ int caam_jr_enqueue(struct device *dev, u32 *desc,
}
EXPORT_SYMBOL(caam_jr_enqueue);
+static void caam_jr_init_hw(struct device *dev, dma_addr_t inpbusaddr,
+ dma_addr_t outbusaddr)
+{
+ struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
+
+ wr_reg64(&jrp->rregs->inpring_base, inpbusaddr);
+ wr_reg64(&jrp->rregs->outring_base, outbusaddr);
+ wr_reg32(&jrp->rregs->inpring_size, JOBR_DEPTH);
+ wr_reg32(&jrp->rregs->outring_size, JOBR_DEPTH);
+
+ /* Select interrupt coalescing parameters */
+ clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JOBR_INTC |
+ (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) |
+ (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT));
+}
+
+static void caam_jr_reset_index(struct caam_drv_private_jr *jrp)
+{
+ jrp->out_ring_read_index = 0;
+ jrp->head = 0;
+ jrp->tail = 0;
+}
+
/*
* Init JobR independent of platform property detection
*/
@@ -506,25 +548,16 @@ static int caam_jr_init(struct device *dev)
jrp->entinfo[i].desc_addr_dma = !0;
/* Setup rings */
- jrp->out_ring_read_index = 0;
- jrp->head = 0;
- jrp->tail = 0;
-
- wr_reg64(&jrp->rregs->inpring_base, inpbusaddr);
- wr_reg64(&jrp->rregs->outring_base, outbusaddr);
- wr_reg32(&jrp->rregs->inpring_size, JOBR_DEPTH);
- wr_reg32(&jrp->rregs->outring_size, JOBR_DEPTH);
-
+ caam_jr_reset_index(jrp);
jrp->inpring_avail = JOBR_DEPTH;
+ caam_jr_init_hw(dev, inpbusaddr, outbusaddr);
spin_lock_init(&jrp->inplock);
- /* Select interrupt coalescing parameters */
- clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JOBR_INTC |
- (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) |
- (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT));
-
- tasklet_init(&jrp->irqtask, caam_jr_dequeue, (unsigned long)dev);
+ jrp->tasklet_params.dev = dev;
+ jrp->tasklet_params.enable_itr = 1;
+ tasklet_init(&jrp->irqtask, caam_jr_dequeue,
+ (unsigned long)&jrp->tasklet_params);
/* Connect job ring interrupt handler. */
error = devm_request_irq(dev, jrp->irq, caam_jr_interrupt, IRQF_SHARED,
@@ -635,11 +668,134 @@ static int caam_jr_probe(struct platform_device *pdev)
atomic_set(&jrpriv->tfm_count, 0);
+ device_init_wakeup(&pdev->dev, 1);
+ device_set_wakeup_enable(&pdev->dev, false);
+
register_algs(jrpriv, jrdev->parent);
return 0;
}
+static void caam_jr_get_hw_state(struct device *dev)
+{
+ struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
+
+ jrp->state.inpbusaddr = rd_reg64(&jrp->rregs->inpring_base);
+ jrp->state.outbusaddr = rd_reg64(&jrp->rregs->outring_base);
+}
+
+static int caam_jr_suspend(struct device *dev)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct caam_drv_private_jr *jrpriv = platform_get_drvdata(pdev);
+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev->parent);
+ struct caam_jr_dequeue_params suspend_params = {
+ .dev = dev,
+ .enable_itr = 0,
+ };
+
+ /* Remove the node from Physical JobR list maintained by driver */
+ spin_lock(&driver_data.jr_alloc_lock);
+ list_del(&jrpriv->list_node);
+ spin_unlock(&driver_data.jr_alloc_lock);
+
+ if (jrpriv->hwrng)
+ caam_rng_exit(dev->parent);
+
+ if (ctrlpriv->caam_off_during_pm) {
+ int err;
+
+ tasklet_disable(&jrpriv->irqtask);
+
+ /* mask itr to call flush */
+ clrsetbits_32(&jrpriv->rregs->rconfig_lo, 0, JRCFG_IMSK);
+
+ /* Invalid job in process */
+ err = caam_jr_flush(dev);
+ if (err) {
+ dev_err(dev, "Failed to flush\n");
+ return err;
+ }
+
+ /* Dequeing jobs flushed */
+ caam_jr_dequeue((unsigned long)&suspend_params);
+
+ /* Save state */
+ caam_jr_get_hw_state(dev);
+ } else if (device_may_wakeup(&pdev->dev)) {
+ enable_irq_wake(jrpriv->irq);
+ }
+
+ return 0;
+}
+
+static int caam_jr_resume(struct device *dev)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct caam_drv_private_jr *jrpriv = platform_get_drvdata(pdev);
+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev->parent);
+
+ if (ctrlpriv->caam_off_during_pm) {
+ u64 inp_addr;
+ int err;
+
+ /*
+ * Check if the CAAM has been resetted checking the address of
+ * the input ring
+ */
+ inp_addr = rd_reg64(&jrpriv->rregs->inpring_base);
+ if (inp_addr != 0) {
+ /* JR still has some configuration */
+ if (inp_addr == jrpriv->state.inpbusaddr) {
+ /* JR has not been resetted */
+ err = caam_jr_restart_processing(dev);
+ if (err) {
+ dev_err(dev,
+ "Restart processing failed\n");
+ return err;
+ }
+
+ tasklet_enable(&jrpriv->irqtask);
+
+ clrsetbits_32(&jrpriv->rregs->rconfig_lo,
+ JRCFG_IMSK, 0);
+
+ goto add_jr;
+ } else if (ctrlpriv->optee_en) {
+ /* JR has been used by OPTEE, reset it */
+ err = caam_reset_hw_jr(dev);
+ if (err) {
+ dev_err(dev, "Failed to reset JR\n");
+ return err;
+ }
+ } else {
+ /* No explanation, return error */
+ return -EIO;
+ }
+ }
+
+ caam_jr_reset_index(jrpriv);
+ caam_jr_init_hw(dev, jrpriv->state.inpbusaddr,
+ jrpriv->state.outbusaddr);
+
+ tasklet_enable(&jrpriv->irqtask);
+ } else if (device_may_wakeup(&pdev->dev)) {
+ disable_irq_wake(jrpriv->irq);
+ }
+
+add_jr:
+ spin_lock(&driver_data.jr_alloc_lock);
+ list_add_tail(&jrpriv->list_node, &driver_data.jr_list);
+ spin_unlock(&driver_data.jr_alloc_lock);
+
+ if (jrpriv->hwrng)
+ jrpriv->hwrng = !caam_rng_init(dev->parent);
+
+ return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(caam_jr_pm_ops, caam_jr_suspend, caam_jr_resume);
+
static const struct of_device_id caam_jr_match[] = {
{
.compatible = "fsl,sec-v4.0-job-ring",
@@ -655,6 +811,7 @@ static struct platform_driver caam_jr_driver = {
.driver = {
.name = "caam_jr",
.of_match_table = caam_jr_match,
+ .pm = &caam_jr_pm_ops,
},
.probe = caam_jr_probe,
.remove = caam_jr_remove,
@@ -584,8 +584,7 @@ struct caam_ctrl {
u32 deco_rsr; /* DECORSR - Deco Request Source */
u32 rsvd11;
u32 deco_rq; /* DECORR - DECO Request */
- struct masterid deco_mid[5]; /* DECOxLIODNR - 1 per DECO */
- u32 rsvd5[22];
+ struct masterid deco_mid[16]; /* DECOxLIODNR - 1 per DECO */
/* DECO Availability/Reset Section 120-3ff */
u32 deco_avail; /* DAR - DECO availability */