| Message ID | 20240224-pci-dbi-rework-v8-0-64c7fd0cfe64@linaro.org |
|---|---|
| Headers | show |
| Series | PCI: dwc: ep: Fix DBI access failure for drivers requiring refclk from host | expand |
On Sat, Feb 24, 2024 at 12:24:07PM +0530, Manivannan Sadhasivam wrote: > deinit() callback was solely introduced for the pcie-rcar-gen4 driver where > it is used to do platform specific resource deallocation. And this callback > is called right at the end of the dw_pcie_ep_exit() API. So it doesn't > matter whether it is called within or outside of dw_pcie_ep_exit() API. > > So let's remove this callback and directly call rcar_gen4_pcie_ep_deinit() > in pcie-rcar-gen4 driver to do resource deallocation after the completion > of dw_pcie_ep_exit() API in rcar_gen4_remove_dw_pcie_ep(). > > This simplifies the DWC layer. > > Signed-off-by: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org> Reviewed-by: Frank Li <Frank.Li@nxp.com> > --- > drivers/pci/controller/dwc/pcie-designware-ep.c | 9 +-------- > drivers/pci/controller/dwc/pcie-designware.h | 1 - > drivers/pci/controller/dwc/pcie-rcar-gen4.c | 14 ++++++++------ > 3 files changed, 9 insertions(+), 15 deletions(-) > > diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c > index 5befed2dc02b..d305f9b4cdfe 100644 > --- a/drivers/pci/controller/dwc/pcie-designware-ep.c > +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c > @@ -575,9 +575,6 @@ void dw_pcie_ep_exit(struct dw_pcie_ep *ep) > epc->mem->window.page_size); > > pci_epc_mem_exit(epc); > - > - if (ep->ops->deinit) > - ep->ops->deinit(ep); > } > EXPORT_SYMBOL_GPL(dw_pcie_ep_exit); > > @@ -738,7 +735,7 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep) > ep->page_size); > if (ret < 0) { > dev_err(dev, "Failed to initialize address space\n"); > - goto err_ep_deinit; > + return ret; > } > > ep->msi_mem = pci_epc_mem_alloc_addr(epc, &ep->msi_mem_phys, > @@ -775,10 +772,6 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep) > err_exit_epc_mem: > pci_epc_mem_exit(epc); > > -err_ep_deinit: > - if (ep->ops->deinit) > - ep->ops->deinit(ep); > - > return ret; > } > EXPORT_SYMBOL_GPL(dw_pcie_ep_init); > diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h > index 26dae4837462..ab7431a37209 100644 > --- a/drivers/pci/controller/dwc/pcie-designware.h > +++ b/drivers/pci/controller/dwc/pcie-designware.h > @@ -333,7 +333,6 @@ struct dw_pcie_rp { > struct dw_pcie_ep_ops { > void (*pre_init)(struct dw_pcie_ep *ep); > void (*init)(struct dw_pcie_ep *ep); > - void (*deinit)(struct dw_pcie_ep *ep); > int (*raise_irq)(struct dw_pcie_ep *ep, u8 func_no, > unsigned int type, u16 interrupt_num); > const struct pci_epc_features* (*get_features)(struct dw_pcie_ep *ep); > diff --git a/drivers/pci/controller/dwc/pcie-rcar-gen4.c b/drivers/pci/controller/dwc/pcie-rcar-gen4.c > index e9166619b1f9..ac97d594ea47 100644 > --- a/drivers/pci/controller/dwc/pcie-rcar-gen4.c > +++ b/drivers/pci/controller/dwc/pcie-rcar-gen4.c > @@ -352,11 +352,8 @@ static void rcar_gen4_pcie_ep_init(struct dw_pcie_ep *ep) > dw_pcie_ep_reset_bar(pci, bar); > } > > -static void rcar_gen4_pcie_ep_deinit(struct dw_pcie_ep *ep) > +static void rcar_gen4_pcie_ep_deinit(struct rcar_gen4_pcie *rcar) > { > - struct dw_pcie *dw = to_dw_pcie_from_ep(ep); > - struct rcar_gen4_pcie *rcar = to_rcar_gen4_pcie(dw); > - > writel(0, rcar->base + PCIEDMAINTSTSEN); > rcar_gen4_pcie_common_deinit(rcar); > } > @@ -408,7 +405,6 @@ static unsigned int rcar_gen4_pcie_ep_get_dbi2_offset(struct dw_pcie_ep *ep, > static const struct dw_pcie_ep_ops pcie_ep_ops = { > .pre_init = rcar_gen4_pcie_ep_pre_init, > .init = rcar_gen4_pcie_ep_init, > - .deinit = rcar_gen4_pcie_ep_deinit, > .raise_irq = rcar_gen4_pcie_ep_raise_irq, > .get_features = rcar_gen4_pcie_ep_get_features, > .get_dbi_offset = rcar_gen4_pcie_ep_get_dbi_offset, > @@ -418,18 +414,24 @@ static const struct dw_pcie_ep_ops pcie_ep_ops = { > static int rcar_gen4_add_dw_pcie_ep(struct rcar_gen4_pcie *rcar) > { > struct dw_pcie_ep *ep = &rcar->dw.ep; > + int ret; > > if (!IS_ENABLED(CONFIG_PCIE_RCAR_GEN4_EP)) > return -ENODEV; > > ep->ops = &pcie_ep_ops; > > - return dw_pcie_ep_init(ep); > + ret = dw_pcie_ep_init(ep); > + if (ret) > + rcar_gen4_pcie_ep_deinit(rcar); > + > + return ret; > } > > static void rcar_gen4_remove_dw_pcie_ep(struct rcar_gen4_pcie *rcar) > { > dw_pcie_ep_exit(&rcar->dw.ep); > + rcar_gen4_pcie_ep_deinit(rcar); > } > > /* Common */ > > -- > 2.25.1 >
On Sat, Feb 24, 2024 at 12:24:09PM +0530, Manivannan Sadhasivam wrote: > For DWC glue drivers supporting PERST# (currently Qcom and Tegra194), some > of the DWC resources like eDMA should be cleaned up during the PERST# > assert time. > > So let's introduce a dw_pcie_ep_cleanup() API that could be called by these > drivers to cleanup the DWC specific resources. Currently, it just removes > eDMA. > > Reported-by: Niklas Cassel <cassel@kernel.org> > Closes: https://lore.kernel.org/linux-pci/ZWYmX8Y%2F7Q9WMxES@x1-carbon > Signed-off-by: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org> Reviewed-by: Frank Li <Frank.Li@nxp.com> > --- > drivers/pci/controller/dwc/pcie-designware-ep.c | 11 +++++++++-- > drivers/pci/controller/dwc/pcie-designware.h | 5 +++++ > drivers/pci/controller/dwc/pcie-qcom-ep.c | 1 + > drivers/pci/controller/dwc/pcie-tegra194.c | 2 ++ > 4 files changed, 17 insertions(+), 2 deletions(-) > > diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c > index 2b11290aab4c..1205bfba8310 100644 > --- a/drivers/pci/controller/dwc/pcie-designware-ep.c > +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c > @@ -564,12 +564,19 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, > return 0; > } > > -void dw_pcie_ep_deinit(struct dw_pcie_ep *ep) > +void dw_pcie_ep_cleanup(struct dw_pcie_ep *ep) > { > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > - struct pci_epc *epc = ep->epc; > > dw_pcie_edma_remove(pci); > +} > +EXPORT_SYMBOL_GPL(dw_pcie_ep_cleanup); > + > +void dw_pcie_ep_deinit(struct dw_pcie_ep *ep) > +{ > + struct pci_epc *epc = ep->epc; > + > + dw_pcie_ep_cleanup(ep); > > pci_epc_mem_free_addr(epc, ep->msi_mem_phys, ep->msi_mem, > epc->mem->window.page_size); > diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h > index 61465203bb60..351d2fe3ea4d 100644 > --- a/drivers/pci/controller/dwc/pcie-designware.h > +++ b/drivers/pci/controller/dwc/pcie-designware.h > @@ -672,6 +672,7 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep); > int dw_pcie_ep_init_complete(struct dw_pcie_ep *ep); > void dw_pcie_ep_init_notify(struct dw_pcie_ep *ep); > void dw_pcie_ep_deinit(struct dw_pcie_ep *ep); > +void dw_pcie_ep_cleanup(struct dw_pcie_ep *ep); > int dw_pcie_ep_raise_intx_irq(struct dw_pcie_ep *ep, u8 func_no); > int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, > u8 interrupt_num); > @@ -705,6 +706,10 @@ static inline void dw_pcie_ep_deinit(struct dw_pcie_ep *ep) > { > } > > +static inline void dw_pcie_ep_cleanup(struct dw_pcie_ep *ep) > +{ > +} > + > static inline int dw_pcie_ep_raise_intx_irq(struct dw_pcie_ep *ep, u8 func_no) > { > return 0; > diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c > index 36e5e80cd22f..59b1c0110288 100644 > --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c > +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c > @@ -507,6 +507,7 @@ static void qcom_pcie_perst_assert(struct dw_pcie *pci) > return; > } > > + dw_pcie_ep_cleanup(&pci->ep); > qcom_pcie_disable_resources(pcie_ep); > pcie_ep->link_status = QCOM_PCIE_EP_LINK_DISABLED; > } > diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c > index 7afa9e9aabe2..68bfeed3429b 100644 > --- a/drivers/pci/controller/dwc/pcie-tegra194.c > +++ b/drivers/pci/controller/dwc/pcie-tegra194.c > @@ -1715,6 +1715,8 @@ static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie) > if (ret) > dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret); > > + dw_pcie_ep_cleanup(&pcie->pci.ep); > + > reset_control_assert(pcie->core_rst); > > tegra_pcie_disable_phy(pcie); > > -- > 2.25.1 >
On Sat, Feb 24, 2024 at 12:24:11PM +0530, Manivannan Sadhasivam wrote: > The goal of the dw_pcie_ep_init_complete() API is to initialize the DWC > specific registers post registering the controller with the EP framework. > > But the naming doesn't reflect its functionality and causes confusion. So, > let's rename it to dw_pcie_ep_init_registers() to make it clear that it > initializes the DWC specific registers. > > Signed-off-by: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org> Reviewed-by: Frank Li <Frank.Li@nxp.com> > --- > drivers/pci/controller/dwc/pcie-designware-ep.c | 6 +++--- > drivers/pci/controller/dwc/pcie-designware.h | 4 ++-- > drivers/pci/controller/dwc/pcie-qcom-ep.c | 2 +- > drivers/pci/controller/dwc/pcie-tegra194.c | 2 +- > 4 files changed, 7 insertions(+), 7 deletions(-) > > diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c > index 99d66b0fa59b..ed1f2afd830a 100644 > --- a/drivers/pci/controller/dwc/pcie-designware-ep.c > +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c > @@ -603,7 +603,7 @@ static unsigned int dw_pcie_ep_find_ext_capability(struct dw_pcie *pci, int cap) > return 0; > } > > -int dw_pcie_ep_init_complete(struct dw_pcie_ep *ep) > +int dw_pcie_ep_init_registers(struct dw_pcie_ep *ep) > { > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > struct dw_pcie_ep_func *ep_func; > @@ -718,7 +718,7 @@ int dw_pcie_ep_init_complete(struct dw_pcie_ep *ep) > > return ret; > } > -EXPORT_SYMBOL_GPL(dw_pcie_ep_init_complete); > +EXPORT_SYMBOL_GPL(dw_pcie_ep_init_registers); > > int dw_pcie_ep_init(struct dw_pcie_ep *ep) > { > @@ -788,7 +788,7 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep) > * (Ex: tegra194). Any hardware access on such platforms result > * in system hang. > */ > - ret = dw_pcie_ep_init_complete(ep); > + ret = dw_pcie_ep_init_registers(ep); > if (ret) > goto err_free_epc_mem; > > diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h > index 351d2fe3ea4d..f8e5431a207b 100644 > --- a/drivers/pci/controller/dwc/pcie-designware.h > +++ b/drivers/pci/controller/dwc/pcie-designware.h > @@ -669,7 +669,7 @@ static inline void __iomem *dw_pcie_own_conf_map_bus(struct pci_bus *bus, > #ifdef CONFIG_PCIE_DW_EP > void dw_pcie_ep_linkup(struct dw_pcie_ep *ep); > int dw_pcie_ep_init(struct dw_pcie_ep *ep); > -int dw_pcie_ep_init_complete(struct dw_pcie_ep *ep); > +int dw_pcie_ep_init_registers(struct dw_pcie_ep *ep); > void dw_pcie_ep_init_notify(struct dw_pcie_ep *ep); > void dw_pcie_ep_deinit(struct dw_pcie_ep *ep); > void dw_pcie_ep_cleanup(struct dw_pcie_ep *ep); > @@ -693,7 +693,7 @@ static inline int dw_pcie_ep_init(struct dw_pcie_ep *ep) > return 0; > } > > -static inline int dw_pcie_ep_init_complete(struct dw_pcie_ep *ep) > +static inline int dw_pcie_ep_init_registers(struct dw_pcie_ep *ep) > { > return 0; > } > diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c > index 59b1c0110288..3697b4a944cc 100644 > --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c > +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c > @@ -463,7 +463,7 @@ static int qcom_pcie_perst_deassert(struct dw_pcie *pci) > PARF_INT_ALL_LINK_UP | PARF_INT_ALL_EDMA; > writel_relaxed(val, pcie_ep->parf + PARF_INT_ALL_MASK); > > - ret = dw_pcie_ep_init_complete(&pcie_ep->pci.ep); > + ret = dw_pcie_ep_init_registers(&pcie_ep->pci.ep); > if (ret) { > dev_err(dev, "Failed to complete initialization: %d\n", ret); > goto err_disable_resources; > diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c > index 68bfeed3429b..264ee76bf008 100644 > --- a/drivers/pci/controller/dwc/pcie-tegra194.c > +++ b/drivers/pci/controller/dwc/pcie-tegra194.c > @@ -1897,7 +1897,7 @@ static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie) > val = (upper_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK); > dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val); > > - ret = dw_pcie_ep_init_complete(ep); > + ret = dw_pcie_ep_init_registers(ep); > if (ret) { > dev_err(dev, "Failed to complete initialization: %d\n", ret); > goto fail_init_complete; > > -- > 2.25.1 >
On Mon, Feb 26, 2024 at 12:20:41PM -0500, Frank Li wrote: > On Sat, Feb 24, 2024 at 12:24:15PM +0530, Manivannan Sadhasivam wrote: > > Now that the API is available, let's make use of it. It also handles the > > reinitialization of DWC non-sticky registers in addition to sending the > > notification to EPF drivers. > > > > Signed-off-by: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org> > > --- > > drivers/pci/controller/dwc/pcie-qcom-ep.c | 2 +- > > 1 file changed, 1 insertion(+), 1 deletion(-) > > > > diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c > > index 2fb8c15e7a91..4e45bc4bca45 100644 > > --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c > > +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c > > @@ -640,7 +640,7 @@ static irqreturn_t qcom_pcie_ep_global_irq_thread(int irq, void *data) > > if (FIELD_GET(PARF_INT_ALL_LINK_DOWN, status)) { > > dev_dbg(dev, "Received Linkdown event\n"); > > pcie_ep->link_status = QCOM_PCIE_EP_LINK_DOWN; > > - pci_epc_linkdown(pci->ep.epc); > > + dw_pcie_ep_linkdown(&pci->ep); > > Suppose pci_epc_linkdown() will call dw_pcie_ep_linkdown() ? > why need direct call dw_pcie_ep_linkdown() here? > I've already justified this in the commit message. Here is the excerpt: "It also handles the reinitialization of DWC non-sticky registers in addition to sending the notification to EPF drivers." - Mani
On Tue, Feb 27, 2024 at 06:02:30PM +0530, Manivannan Sadhasivam wrote: > On Mon, Feb 26, 2024 at 12:20:41PM -0500, Frank Li wrote: > > On Sat, Feb 24, 2024 at 12:24:15PM +0530, Manivannan Sadhasivam wrote: > > > Now that the API is available, let's make use of it. It also handles the > > > reinitialization of DWC non-sticky registers in addition to sending the > > > notification to EPF drivers. > > > > > > Signed-off-by: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org> > > > --- > > > drivers/pci/controller/dwc/pcie-qcom-ep.c | 2 +- > > > 1 file changed, 1 insertion(+), 1 deletion(-) > > > > > > diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c > > > index 2fb8c15e7a91..4e45bc4bca45 100644 > > > --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c > > > +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c > > > @@ -640,7 +640,7 @@ static irqreturn_t qcom_pcie_ep_global_irq_thread(int irq, void *data) > > > if (FIELD_GET(PARF_INT_ALL_LINK_DOWN, status)) { > > > dev_dbg(dev, "Received Linkdown event\n"); > > > pcie_ep->link_status = QCOM_PCIE_EP_LINK_DOWN; > > > - pci_epc_linkdown(pci->ep.epc); > > > + dw_pcie_ep_linkdown(&pci->ep); > > > > Suppose pci_epc_linkdown() will call dw_pcie_ep_linkdown() ? > > why need direct call dw_pcie_ep_linkdown() here? > > > > I've already justified this in the commit message. Here is the excerpt: > > "It also handles the reinitialization of DWC non-sticky registers in addition > to sending the notification to EPF drivers." API function name is too similar. It is hard to know difference from API naming. It'd better to know what function do from function name. Frank > > - Mani > > -- > மணிவண்ணன் சதாசிவம்
On Tue, Feb 27, 2024 at 12:34:15PM -0500, Frank Li wrote: > On Tue, Feb 27, 2024 at 06:02:30PM +0530, Manivannan Sadhasivam wrote: > > On Mon, Feb 26, 2024 at 12:20:41PM -0500, Frank Li wrote: > > > On Sat, Feb 24, 2024 at 12:24:15PM +0530, Manivannan Sadhasivam wrote: > > > > Now that the API is available, let's make use of it. It also handles the > > > > reinitialization of DWC non-sticky registers in addition to sending the > > > > notification to EPF drivers. > > > > > > > > Signed-off-by: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org> > > > > --- > > > > drivers/pci/controller/dwc/pcie-qcom-ep.c | 2 +- > > > > 1 file changed, 1 insertion(+), 1 deletion(-) > > > > > > > > diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c > > > > index 2fb8c15e7a91..4e45bc4bca45 100644 > > > > --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c > > > > +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c > > > > @@ -640,7 +640,7 @@ static irqreturn_t qcom_pcie_ep_global_irq_thread(int irq, void *data) > > > > if (FIELD_GET(PARF_INT_ALL_LINK_DOWN, status)) { > > > > dev_dbg(dev, "Received Linkdown event\n"); > > > > pcie_ep->link_status = QCOM_PCIE_EP_LINK_DOWN; > > > > - pci_epc_linkdown(pci->ep.epc); > > > > + dw_pcie_ep_linkdown(&pci->ep); > > > > > > Suppose pci_epc_linkdown() will call dw_pcie_ep_linkdown() ? > > > why need direct call dw_pcie_ep_linkdown() here? > > > > > > > I've already justified this in the commit message. Here is the excerpt: > > > > "It also handles the reinitialization of DWC non-sticky registers in addition > > to sending the notification to EPF drivers." > > API function name is too similar. It is hard to know difference from API > naming. It'd better to know what function do from function name. > In reality we cannot name a function based on everything it does. The naming is mostly based on what is the primary motive of the API and here it is handling Link down event. Maybe dw_pcie_ep_handle_linkdown() would be an apt one, but that's out of scope of this series (since changing that would also require changes to other similar APIs). - Mani > Frank > > > > - Mani > > > > -- > > மணிவண்ணன் சதாசிவம்
On Sat, Feb 24, 2024 at 12:24:09PM +0530, Manivannan Sadhasivam wrote: > For DWC glue drivers supporting PERST# (currently Qcom and Tegra194), some > of the DWC resources like eDMA should be cleaned up during the PERST# > assert time. > > So let's introduce a dw_pcie_ep_cleanup() API that could be called by these > drivers to cleanup the DWC specific resources. Currently, it just removes > eDMA. > > Reported-by: Niklas Cassel <cassel@kernel.org> > Closes: https://lore.kernel.org/linux-pci/ZWYmX8Y%2F7Q9WMxES@x1-carbon > Signed-off-by: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org> > --- > drivers/pci/controller/dwc/pcie-designware-ep.c | 11 +++++++++-- > drivers/pci/controller/dwc/pcie-designware.h | 5 +++++ > drivers/pci/controller/dwc/pcie-qcom-ep.c | 1 + > drivers/pci/controller/dwc/pcie-tegra194.c | 2 ++ > 4 files changed, 17 insertions(+), 2 deletions(-) > > diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c > index 2b11290aab4c..1205bfba8310 100644 > --- a/drivers/pci/controller/dwc/pcie-designware-ep.c > +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c > @@ -564,12 +564,19 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, > return 0; > } > > -void dw_pcie_ep_deinit(struct dw_pcie_ep *ep) > +void dw_pcie_ep_cleanup(struct dw_pcie_ep *ep) > { > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > - struct pci_epc *epc = ep->epc; > > dw_pcie_edma_remove(pci); Hello Mani, In this message: https://lore.kernel.org/linux-pci/20240130062938.GB32821@thinkpad/ You mentioned that you were going to clean up the BARs. (Like I wrote in that thread, I really think that we should merge a fix for the broken "do we have a saved value from find_first_zero_bit() in the array", by using a "if (!saved_value[bar])", when find_first_zero_bit() returns zero.) However, regardless of that, I do not see that this series (neither dw_pcie_ep_cleanup(), nor dw_pcie_ep_linkdown()), calls any function which will clean up the BARs. Since e.g. qcom-ep.c does a reset_control_assert() during perst assert/deassert, which should clear sticky registers, I think that you should let dw_pcie_ep_cleanup() clean up the BARs using dw_pcie_ep_clear_bar(). Kind regards, Niklas > +} > +EXPORT_SYMBOL_GPL(dw_pcie_ep_cleanup); > + > +void dw_pcie_ep_deinit(struct dw_pcie_ep *ep) > +{ > + struct pci_epc *epc = ep->epc; > + > + dw_pcie_ep_cleanup(ep); > > pci_epc_mem_free_addr(epc, ep->msi_mem_phys, ep->msi_mem, > epc->mem->window.page_size); > diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h > index 61465203bb60..351d2fe3ea4d 100644 > --- a/drivers/pci/controller/dwc/pcie-designware.h > +++ b/drivers/pci/controller/dwc/pcie-designware.h > @@ -672,6 +672,7 @@ int dw_pcie_ep_init(struct dw_pcie_ep *ep); > int dw_pcie_ep_init_complete(struct dw_pcie_ep *ep); > void dw_pcie_ep_init_notify(struct dw_pcie_ep *ep); > void dw_pcie_ep_deinit(struct dw_pcie_ep *ep); > +void dw_pcie_ep_cleanup(struct dw_pcie_ep *ep); > int dw_pcie_ep_raise_intx_irq(struct dw_pcie_ep *ep, u8 func_no); > int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, > u8 interrupt_num); > @@ -705,6 +706,10 @@ static inline void dw_pcie_ep_deinit(struct dw_pcie_ep *ep) > { > } > > +static inline void dw_pcie_ep_cleanup(struct dw_pcie_ep *ep) > +{ > +} > + > static inline int dw_pcie_ep_raise_intx_irq(struct dw_pcie_ep *ep, u8 func_no) > { > return 0; > diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c > index 36e5e80cd22f..59b1c0110288 100644 > --- a/drivers/pci/controller/dwc/pcie-qcom-ep.c > +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c > @@ -507,6 +507,7 @@ static void qcom_pcie_perst_assert(struct dw_pcie *pci) > return; > } > > + dw_pcie_ep_cleanup(&pci->ep); > qcom_pcie_disable_resources(pcie_ep); > pcie_ep->link_status = QCOM_PCIE_EP_LINK_DISABLED; > } > diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c > index 7afa9e9aabe2..68bfeed3429b 100644 > --- a/drivers/pci/controller/dwc/pcie-tegra194.c > +++ b/drivers/pci/controller/dwc/pcie-tegra194.c > @@ -1715,6 +1715,8 @@ static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie) > if (ret) > dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret); > > + dw_pcie_ep_cleanup(&pcie->pci.ep); > + > reset_control_assert(pcie->core_rst); > > tegra_pcie_disable_phy(pcie); > > -- > 2.25.1 >
On Thu, Feb 29, 2024 at 12:23:16PM +0100, Niklas Cassel wrote: > Hello Mani, > > On Sat, Feb 24, 2024 at 12:24:13PM +0530, Manivannan Sadhasivam wrote: > > "core_init_notifier" flag is set by the glue drivers requiring refclk from > > the host to complete the DWC core initialization. Also, those drivers will > > send a notification to the EPF drivers once the initialization is fully > > completed using the pci_epc_init_notify() API. Only then, the EPF drivers > > will start functioning. > > > > For the rest of the drivers generating refclk locally, EPF drivers will > > start functioning post binding with them. EPF drivers rely on the > > 'core_init_notifier' flag to differentiate between the drivers. > > Unfortunately, this creates two different flows for the EPF drivers. > > > > So to avoid that, let's get rid of the "core_init_notifier" flag and follow > > a single initialization flow for the EPF drivers. This is done by calling > > the dw_pcie_ep_init_notify() from all glue drivers after the completion of > > dw_pcie_ep_init_registers() API. This will allow all the glue drivers to > > send the notification to the EPF drivers once the initialization is fully > > completed. > > > > Only difference here is that, the drivers requiring refclk from host will > > send the notification once refclk is received, while others will send it > > during probe time itself. > > > > Signed-off-by: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org> > > --- > > drivers/pci/controller/dwc/pci-dra7xx.c | 2 ++ > > drivers/pci/controller/dwc/pci-imx6.c | 2 ++ > > drivers/pci/controller/dwc/pci-keystone.c | 2 ++ > > drivers/pci/controller/dwc/pci-layerscape-ep.c | 2 ++ > > drivers/pci/controller/dwc/pcie-designware-plat.c | 2 ++ > > drivers/pci/controller/dwc/pcie-qcom-ep.c | 1 - > > drivers/pci/controller/dwc/pcie-rcar-gen4.c | 2 ++ > > drivers/pci/controller/dwc/pcie-tegra194.c | 1 - > > drivers/pci/controller/dwc/pcie-uniphier-ep.c | 2 ++ > > drivers/pci/endpoint/functions/pci-epf-test.c | 18 +++++------------- > > include/linux/pci-epc.h | 3 --- > > pcie-artpec6.c:static const struct dw_pcie_ep_ops pcie_ep_ops = { > pcie-keembay.c:static const struct dw_pcie_ep_ops keembay_pcie_ep_ops = { > > Where is the love for these drivers? ;) > Ah, my grep skills got exposed :( Will fix them. - Mani
On Thu, Feb 29, 2024 at 01:40:29PM +0100, Niklas Cassel wrote: > On Sat, Feb 24, 2024 at 12:24:09PM +0530, Manivannan Sadhasivam wrote: > > For DWC glue drivers supporting PERST# (currently Qcom and Tegra194), some > > of the DWC resources like eDMA should be cleaned up during the PERST# > > assert time. > > > > So let's introduce a dw_pcie_ep_cleanup() API that could be called by these > > drivers to cleanup the DWC specific resources. Currently, it just removes > > eDMA. > > > > Reported-by: Niklas Cassel <cassel@kernel.org> > > Closes: https://lore.kernel.org/linux-pci/ZWYmX8Y%2F7Q9WMxES@x1-carbon > > Signed-off-by: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org> > > --- > > drivers/pci/controller/dwc/pcie-designware-ep.c | 11 +++++++++-- > > drivers/pci/controller/dwc/pcie-designware.h | 5 +++++ > > drivers/pci/controller/dwc/pcie-qcom-ep.c | 1 + > > drivers/pci/controller/dwc/pcie-tegra194.c | 2 ++ > > 4 files changed, 17 insertions(+), 2 deletions(-) > > > > diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c > > index 2b11290aab4c..1205bfba8310 100644 > > --- a/drivers/pci/controller/dwc/pcie-designware-ep.c > > +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c > > @@ -564,12 +564,19 @@ int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, > > return 0; > > } > > > > -void dw_pcie_ep_deinit(struct dw_pcie_ep *ep) > > +void dw_pcie_ep_cleanup(struct dw_pcie_ep *ep) > > { > > struct dw_pcie *pci = to_dw_pcie_from_ep(ep); > > - struct pci_epc *epc = ep->epc; > > > > dw_pcie_edma_remove(pci); > > Hello Mani, > > In this message: > https://lore.kernel.org/linux-pci/20240130062938.GB32821@thinkpad/ > > You mentioned that you were going to clean up the BARs. Yes, I did and it is still in my queue. > (Like I wrote in that thread, I really think that we should merge a fix for > the broken "do we have a saved value from find_first_zero_bit() in the array", > by using a "if (!saved_value[bar])", when find_first_zero_bit() returns zero.) > Hmm, yeah that logic is flawed. Let me take another look. > However, regardless of that, I do not see that this series (neither > dw_pcie_ep_cleanup(), nor dw_pcie_ep_linkdown()), calls any function which > will clean up the BARs. > > Since e.g. qcom-ep.c does a reset_control_assert() during perst > assert/deassert, which should clear sticky registers, I think that > you should let dw_pcie_ep_cleanup() clean up the BARs using > dw_pcie_ep_clear_bar(). > As I mentioned earlier, it is the job of the EPF drivers to clear the BARs since they allocate them. I'm trying to reduce the implicit resetting wherever we could. The proper fix is to add the LINK_DOWN callback to EPF drivers and do cleanup. I'm planning to submit a series for that after this one. - Mani
On Mon, Mar 04, 2024 at 01:47:13PM +0530, Manivannan Sadhasivam wrote: > On Thu, Feb 29, 2024 at 01:40:29PM +0100, Niklas Cassel wrote: > > On Sat, Feb 24, 2024 at 12:24:09PM +0530, Manivannan Sadhasivam wrote: > > > > Since e.g. qcom-ep.c does a reset_control_assert() during perst > > assert/deassert, which should clear sticky registers, I think that > > you should let dw_pcie_ep_cleanup() clean up the BARs using > > dw_pcie_ep_clear_bar(). > > > > As I mentioned earlier, it is the job of the EPF drivers to clear the BARs since > they allocate them. I'm trying to reduce the implicit resetting wherever we > could. > > The proper fix is to add the LINK_DOWN callback to EPF drivers and do cleanup. > I'm planning to submit a series for that after this one. Currently, pci-epf-test allocates memory for the BARs in .bind(). Likewise it frees the memory for the BARs in .unbind(). AFAICT, most iATU registers, and most BAR registers are sticky registers, so they will not get reset on link down. (The currently selected BAR size, in case of Resizable BAR is an exception.) That means that even on link down, we do not need to free the memory, or change the iATU settings. (This applies to all drivers.) However, on PERST (for the drivers call dw_pcie_ep_cleanup()), they call reset_control_assert(), so they will clear sticky registers, which means that they need to at least re-write the iATU and BAR registers. (I guess they could free + allocate the memory for the BARs again, but I don't think that is strictly necessary.) That is why I suggested that you call dw_pcie_ep_clear_bar() from dw_pcie_ep_cleanup(). If you free the memory for the BARs in link_down() (this callback exists for many drivers, even drivers without a PERST handler), where are you supposted to alloc the memory for the BARs again? Allocating them at link_up() is too late (because as soon as the link is up, the host is allowed to enumerate the EP BARs.) The proper place is to allocate them when receiving PERST, but not all drivers have a PERST handler. (My understanding is that 1) PERST assert 2) PERST deassert 3) link is up.) unbind() undos what was done in bind(), so shouldn't link_down() undo what was done in link_up()? With that logic, if you move the alloc to .core_init(), should we perhaps have a .core_deinit() callback for EPF drivers? (I guess only drivers which perform a reset during PERST would call this.) But considering that free+alloc is not strictly needed, why not just keep the allocation + free in .bind()/.unbind() ? (To avoid the need to create a .core_deinit()), and let dw_pcie_ep_cleanup() call dw_pcie_ep_clear_bar() ? I guess my point is that it seems a bit pointless for drivers that do not clear sticky registers to free+alloc memory on link down, for no good reason. (Memory might get fragmented over time, so it might not be possible to perform a big allocation after the device has been running for a really long time.) So I'm thinking that we either 1) Keep the alloc/free in bind/unbind, and let dw_pcie_ep_cleanup() call dw_pcie_ep_clear_bar(), or 2) Introduce a .deinit_core() callback which will free the BARs. (Because I don't see how you will (re-)allocate memory for all drivers if you free the memory in link_down().) Kind regards, Niklas
On Mon, Mar 04, 2024 at 11:51:04AM +0100, Niklas Cassel wrote: > On Mon, Mar 04, 2024 at 01:47:13PM +0530, Manivannan Sadhasivam wrote: > > On Thu, Feb 29, 2024 at 01:40:29PM +0100, Niklas Cassel wrote: > > > On Sat, Feb 24, 2024 at 12:24:09PM +0530, Manivannan Sadhasivam wrote: > > > > > > Since e.g. qcom-ep.c does a reset_control_assert() during perst > > > assert/deassert, which should clear sticky registers, I think that > > > you should let dw_pcie_ep_cleanup() clean up the BARs using > > > dw_pcie_ep_clear_bar(). > > > > > > > As I mentioned earlier, it is the job of the EPF drivers to clear the BARs since > > they allocate them. I'm trying to reduce the implicit resetting wherever we > > could. > > > > The proper fix is to add the LINK_DOWN callback to EPF drivers and do cleanup. > > I'm planning to submit a series for that after this one. > > Currently, pci-epf-test allocates memory for the BARs in .bind(). > Likewise it frees the memory for the BARs in .unbind(). > > AFAICT, most iATU registers, and most BAR registers are sticky registers, > so they will not get reset on link down. > (The currently selected BAR size, in case of Resizable BAR is an exception.) > > That means that even on link down, we do not need to free the memory, > or change the iATU settings. (This applies to all drivers.) > > > > However, on PERST (for the drivers call dw_pcie_ep_cleanup()), they call > reset_control_assert(), so they will clear sticky registers, which means > that they need to at least re-write the iATU and BAR registers. > (I guess they could free + allocate the memory for the BARs again, > but I don't think that is strictly necessary.) > That is why I suggested that you call dw_pcie_ep_clear_bar() from > dw_pcie_ep_cleanup(). > Sorry, I keep assuming the flow w.r.t PERST# supported platforms :/ My bad! > > > If you free the memory for the BARs in link_down() (this callback exists > for many drivers, even drivers without a PERST handler), where are you > supposted to alloc the memory for the BARs again? > > Allocating them at link_up() is too late (because as soon as the link is > up, the host is allowed to enumerate the EP BARs.) The proper place is to > allocate them when receiving PERST, but not all drivers have a PERST handler. > > (My understanding is that 1) PERST assert 2) PERST deassert 3) link is up.) > > > > unbind() undos what was done in bind(), so shouldn't link_down() undo what was > done in link_up()? With that logic, if you move the alloc to .core_init(), > should we perhaps have a .core_deinit() callback for EPF drivers? > (I guess only drivers which perform a reset during PERST would call this.) > > But considering that free+alloc is not strictly needed, why not just keep > the allocation + free in .bind()/.unbind() ? > (To avoid the need to create a .core_deinit()), and let dw_pcie_ep_cleanup() > call dw_pcie_ep_clear_bar() ? > > I guess my point is that it seems a bit pointless for drivers that do not > clear sticky registers to free+alloc memory on link down, for no good > reason. (Memory might get fragmented over time, so it might not be possible > to perform a big allocation after the device has been running for a really > long time.) > > > > So I'm thinking that we either > 1) Keep the alloc/free in bind/unbind, and let dw_pcie_ep_cleanup() call > dw_pcie_ep_clear_bar(), > or > 2) Introduce a .deinit_core() callback which will free the BARs. > (Because I don't see how you will (re-)allocate memory for all drivers > if you free the memory in link_down().) > I think option 2 is the better solution. In my view, calling dw_pcie_ep_clear_bar() from EPC drivers is a layering violation since the allocation happens from EPF drivers. So clearing the BARs during the deinit() callback that gets called when PERST# assert happens is the way to go. - Mani
Hello, This series is the continuation of previous work by Vidya Sagar [1] to fix the issues related to accessing DBI register space before completing the core initialization in some EP platforms like Tegra194/234 and Qcom EP. Since Vidya is busy, I took over the series based on his consent (off-list discussion). NOTE ==== Based on the comments received in v7 [2], I've heavily modified the series to fix several other issues reported by Bjorn and Niklas. One noticeable change is getting rid of the 'core_init_notifer' flag added to differentiate between glue drivers requiring refclk from host and drivers getting refclk locally. By getting rid of this flag, now both the DWC EP driver and the EPF drivers can use a single flow and need not distinguish between the glue drivers. We can also get rid of the 'link_up_notifier' flag in the future by following the same convention. Testing ======= I've tested the series on Qcom SM8450 based dev board that depends on refclk from host with EPF_MHI driver. It'd be good to test this series on platforms that generate refclk locally and also with EPF_TEST driver. - Mani [1] https://lore.kernel.org/linux-pci/20221013175712.7539-1-vidyas@nvidia.com/ [2] https://lore.kernel.org/linux-pci/20231120084014.108274-1-manivannan.sadhasivam@linaro.org/ Changes in v8: - Rebased on top of v6.8-rc1 - Removed the deinit callback from struct dw_pcie_ep_ops - Renamed dw_pcie_ep_exit() to dw_pcie_ep_deinit() - Introduced dw_pcie_ep_cleanup() API for drivers supporting PERST# - Renamed dw_pcie_ep_init_complete() to dw_pcie_ep_init_registers() - Called dw_pcie_ep_init_registers() API directly from all glue drivers - Removed "core_init_notifier" flag - Added a generic dw_pcie_ep_linkdown() API to handle LINK_DOWN event and used it in qcom driver - Added Kernel-doc comments for DWC EP APIs Changes in v7: - Rebased on top of v6.7-rc1 - Kept the current dw_pcie_ep_init_complete() API instead of renaming it to dw_pcie_ep_init_late(), since changing the name causes a slight ambiguity. - Splitted the change that moves pci_epc_init_notify() inside dw_pcie_ep_init_notify() to help bisecting and also to avoid build issue. - Added a new patch that moves pci_epc_init_notify() inside dw_pcie_ep_init_notify(). - Took over the authorship and dropped the previous Ack as the patches are heavily modified. Changes in v6: - Rebased on top of pci/next (6e2fca71e187) - removed ep_init_late() callback as it is no longer necessary For previous changelog, please refer [1]. Signed-off-by: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org> --- Manivannan Sadhasivam (10): PCI: dwc: ep: Remove deinit() callback from struct dw_pcie_ep_ops PCI: dwc: ep: Rename dw_pcie_ep_exit() to dw_pcie_ep_deinit() PCI: dwc: ep: Introduce dw_pcie_ep_cleanup() API for drivers supporting PERST# PCI: dwc: ep: Fix DBI access failure for drivers requiring refclk from host PCI: dwc: ep: Rename dw_pcie_ep_init_complete() to dw_pcie_ep_init_registers() PCI: dwc: ep: Call dw_pcie_ep_init_registers() API directly from all glue drivers PCI: dwc: ep: Remove "core_init_notifier" flag PCI: dwc: ep: Add a generic dw_pcie_ep_linkdown() API to handle LINK_DOWN event PCI: qcom-ep: Use the generic dw_pcie_ep_linkdown() API to handle LINK_DOWN event PCI: dwc: ep: Add Kernel-doc comments for APIs drivers/pci/controller/dwc/pci-dra7xx.c | 9 + drivers/pci/controller/dwc/pci-imx6.c | 10 + drivers/pci/controller/dwc/pci-keystone.c | 11 + drivers/pci/controller/dwc/pci-layerscape-ep.c | 9 + drivers/pci/controller/dwc/pcie-designware-ep.c | 307 +++++++++++++++------- drivers/pci/controller/dwc/pcie-designware-plat.c | 11 + drivers/pci/controller/dwc/pcie-designware.h | 19 +- drivers/pci/controller/dwc/pcie-qcom-ep.c | 6 +- drivers/pci/controller/dwc/pcie-rcar-gen4.c | 28 +- drivers/pci/controller/dwc/pcie-tegra194.c | 5 +- drivers/pci/controller/dwc/pcie-uniphier-ep.c | 15 +- drivers/pci/endpoint/functions/pci-epf-test.c | 18 +- include/linux/pci-epc.h | 3 - 13 files changed, 321 insertions(+), 130 deletions(-) --- base-commit: 6613476e225e090cc9aad49be7fa504e290dd33d change-id: 20240224-pci-dbi-rework-b2e99a62930c Best regards,