@@ -537,8 +537,8 @@ static int ufshcd_mcq_sq_start(struct ufs_hba *hba, struct ufs_hw_queue *hwq)
*/
int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag)
{
- struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
- struct scsi_cmnd *cmd = lrbp->cmd;
+ struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
struct ufs_hw_queue *hwq;
void __iomem *reg, *opr_sqd_base;
u32 nexus, id, val;
@@ -623,7 +623,8 @@ static void ufshcd_mcq_nullify_sqe(struct utp_transfer_req_desc *utrd)
static bool ufshcd_mcq_sqe_search(struct ufs_hba *hba,
struct ufs_hw_queue *hwq, int task_tag)
{
- struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
+ struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
struct utp_transfer_req_desc *utrd;
__le64 cmd_desc_base_addr;
bool ret = false;
@@ -674,12 +675,12 @@ int ufshcd_mcq_abort(struct scsi_cmnd *cmd)
struct Scsi_Host *host = cmd->device->host;
struct ufs_hba *hba = shost_priv(host);
int tag = scsi_cmd_to_rq(cmd)->tag;
- struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
struct ufs_hw_queue *hwq;
unsigned long flags;
int err;
- if (!ufshcd_cmd_inflight(lrbp->cmd)) {
+ if (!ufshcd_cmd_inflight(cmd)) {
dev_err(hba->dev,
"%s: skip abort. cmd at tag %d already completed.\n",
__func__, tag);
@@ -718,8 +719,8 @@ int ufshcd_mcq_abort(struct scsi_cmnd *cmd)
}
spin_lock_irqsave(&hwq->cq_lock, flags);
- if (ufshcd_cmd_inflight(lrbp->cmd))
- ufshcd_release_scsi_cmd(hba, lrbp);
+ if (ufshcd_cmd_inflight(cmd))
+ ufshcd_release_scsi_cmd(hba, cmd);
spin_unlock_irqrestore(&hwq->cq_lock, flags);
return SUCCESS;
@@ -38,10 +38,10 @@ ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp,
}
static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba,
- struct ufshcd_lrb *lrbp)
+ struct scsi_cmnd *cmd)
{
- struct scsi_cmnd *cmd = lrbp->cmd;
const struct bio_crypt_ctx *crypt_ctx = scsi_cmd_to_rq(cmd)->crypt_ctx;
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
if (crypt_ctx && hba->vops && hba->vops->fill_crypto_prdt)
return hba->vops->fill_crypto_prdt(hba, crypt_ctx,
@@ -51,17 +51,19 @@ static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba,
}
static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba,
- struct ufshcd_lrb *lrbp)
+ struct scsi_cmnd *cmd)
{
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+
if (!(hba->quirks & UFSHCD_QUIRK_KEYS_IN_PRDT))
return;
- if (!(scsi_cmd_to_rq(lrbp->cmd)->crypt_ctx))
+ if (!(scsi_cmd_to_rq(cmd)->crypt_ctx))
return;
/* Zeroize the PRDT because it can contain cryptographic keys. */
memzero_explicit(lrbp->ucd_prdt_ptr,
- ufshcd_sg_entry_size(hba) * scsi_sg_count(lrbp->cmd));
+ ufshcd_sg_entry_size(hba) * scsi_sg_count(cmd));
}
bool ufshcd_crypto_enable(struct ufs_hba *hba);
@@ -82,13 +84,15 @@ ufshcd_prepare_req_desc_hdr_crypto(struct ufshcd_lrb *lrbp,
struct request_desc_header *h) { }
static inline int ufshcd_crypto_fill_prdt(struct ufs_hba *hba,
- struct ufshcd_lrb *lrbp)
+ struct scsi_cmnd *cmd)
{
return 0;
}
static inline void ufshcd_crypto_clear_prdt(struct ufs_hba *hba,
- struct ufshcd_lrb *lrbp) { }
+ struct scsi_cmnd *cmd)
+{
+}
static inline bool ufshcd_crypto_enable(struct ufs_hba *hba)
{
@@ -75,8 +75,7 @@ bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd);
int ufshcd_mcq_sq_cleanup(struct ufs_hba *hba, int task_tag);
int ufshcd_mcq_abort(struct scsi_cmnd *cmd);
int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag);
-void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
- struct ufshcd_lrb *lrbp);
+void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd);
#define SD_ASCII_STD true
#define SD_RAW false
@@ -361,6 +360,28 @@ static inline bool ufs_is_valid_unit_desc_lun(struct ufs_dev_info *dev_info, u8
return lun == UFS_UPIU_RPMB_WLUN || (lun < dev_info->max_lu_supported);
}
+/*
+ * Convert a block layer tag into a SCSI command pointer. This function is
+ * called once per I/O completion path and is also called from error paths.
+ */
+static inline struct scsi_cmnd *ufshcd_tag_to_cmd(struct ufs_hba *hba, u32 tag)
+{
+ struct blk_mq_tags *tags = hba->host->tag_set.tags[0];
+ struct request *rq;
+
+ /*
+ * Use .static_rqs[] for reserved commands because blk_mq_get_tag()
+ * is not called for reserved commands by the UFS driver.
+ */
+ rq = tag < UFSHCD_NUM_RESERVED ? tags->static_rqs[tag] :
+ blk_mq_tag_to_rq(tags, tag);
+
+ if (WARN_ON_ONCE(!rq))
+ return NULL;
+
+ return blk_mq_rq_to_pdu(rq);
+}
+
static inline void ufshcd_inc_sq_tail(struct ufs_hw_queue *q)
__must_hold(&q->sq_lock)
{
@@ -28,6 +28,7 @@
#include <scsi/scsi_dbg.h>
#include <scsi/scsi_driver.h>
#include <scsi/scsi_eh.h>
+#include <scsi/scsi_tcq.h>
#include "ufshcd-priv.h"
#include <ufs/ufs_quirks.h>
#include <ufs/unipro.h>
@@ -435,7 +436,7 @@ static void ufshcd_add_command_trace(struct ufs_hba *hba, struct scsi_cmnd *cmd,
int hwq_id = -1;
struct request *rq = scsi_cmd_to_rq(cmd);
unsigned int tag = rq->tag;
- struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int transfer_len = -1;
/* trace UPIU also */
@@ -546,14 +547,13 @@ static void ufshcd_print_evt_hist(struct ufs_hba *hba)
ufshcd_vops_dbg_register_dump(hba);
}
-static
-void ufshcd_print_tr(struct ufs_hba *hba, int tag, bool pr_prdt)
+static void ufshcd_print_tr(struct ufs_hba *hba, struct scsi_cmnd *cmd,
+ bool pr_prdt)
{
- const struct ufshcd_lrb *lrbp;
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+ const int tag = lrbp->task_tag;
int prdt_length;
- lrbp = &hba->lrb[tag];
-
dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
tag, div_u64(lrbp->issue_time_stamp_local_clock, 1000));
dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
@@ -594,7 +594,7 @@ static bool ufshcd_print_tr_iter(struct request *req, void *priv)
struct Scsi_Host *shost = sdev->host;
struct ufs_hba *hba = shost_priv(shost);
- ufshcd_print_tr(hba, req->tag, *(bool *)priv);
+ ufshcd_print_tr(hba, blk_mq_rq_to_pdu(req), *(bool *)priv);
return true;
}
@@ -2240,8 +2240,7 @@ static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba,
struct scsi_cmnd *cmd)
{
const struct ufs_hba_monitor *m = &hba->monitor;
- struct request *rq = scsi_cmd_to_rq(cmd);
- struct ufshcd_lrb *lrbp = &hba->lrb[rq->tag];
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
return m->enabled &&
(!m->chunk_size || m->chunk_size == cmd->sdb.length) &&
@@ -2263,7 +2262,7 @@ static void ufshcd_update_monitor(struct ufs_hba *hba,
const struct scsi_cmnd *cmd)
{
const struct request *req = scsi_cmd_to_rq(cmd);
- struct ufshcd_lrb *lrbp = &hba->lrb[req->tag];
+ const struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int dir = ufshcd_monitor_opcode2dir(cmd->cmnd[0]);
unsigned long flags;
@@ -2293,27 +2292,37 @@ static void ufshcd_update_monitor(struct ufs_hba *hba,
spin_unlock_irqrestore(hba->host->host_lock, flags);
}
+/*
+ * Returns %true if @cmd represents a SCSI command that is in flight and %false
+ * if it represents a device management command.
+ */
+static bool ufshcd_is_scsi_cmd(struct scsi_cmnd *cmd)
+{
+ return blk_mq_request_started(scsi_cmd_to_rq(cmd));
+}
+
/**
* ufshcd_send_command - Send SCSI or device management commands
* @hba: per adapter instance
- * @lrbp: Local reference block of SCSI command
+ * @cmd: SCSI command or device management command pointer
* @hwq: pointer to hardware queue instance
*/
static inline void ufshcd_send_command(struct ufs_hba *hba,
- struct ufshcd_lrb *lrbp,
+ struct scsi_cmnd *cmd,
struct ufs_hw_queue *hwq)
{
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
unsigned long flags;
lrbp->issue_time_stamp = ktime_get();
lrbp->issue_time_stamp_local_clock = local_clock();
lrbp->compl_time_stamp = ktime_set(0, 0);
lrbp->compl_time_stamp_local_clock = 0;
- if (lrbp->cmd) {
- ufshcd_add_command_trace(hba, lrbp->cmd, UFS_CMD_SEND);
+ if (ufshcd_is_scsi_cmd(cmd)) {
+ ufshcd_add_command_trace(hba, cmd, UFS_CMD_SEND);
ufshcd_clk_scaling_start_busy(hba);
- if (unlikely(ufshcd_should_inform_monitor(hba, lrbp->cmd)))
- ufshcd_start_monitor(hba, lrbp->cmd);
+ if (unlikely(ufshcd_should_inform_monitor(hba, cmd)))
+ ufshcd_start_monitor(hba, cmd);
}
if (hba->mcq_enabled) {
@@ -2330,7 +2339,7 @@ static inline void ufshcd_send_command(struct ufs_hba *hba,
spin_lock_irqsave(&hba->outstanding_lock, flags);
if (hba->vops && hba->vops->setup_xfer_req)
hba->vops->setup_xfer_req(hba, lrbp->task_tag,
- !!lrbp->cmd);
+ ufshcd_is_scsi_cmd(cmd));
__set_bit(lrbp->task_tag, &hba->outstanding_reqs);
ufshcd_writel(hba, 1 << lrbp->task_tag,
REG_UTP_TRANSFER_REQ_DOOR_BELL);
@@ -2340,11 +2349,12 @@ static inline void ufshcd_send_command(struct ufs_hba *hba,
/**
* ufshcd_copy_sense_data - Copy sense data in case of check condition
- * @lrbp: pointer to local reference block
+ * @cmd: SCSI command
*/
-static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
+static inline void ufshcd_copy_sense_data(struct scsi_cmnd *cmd)
{
- u8 *const sense_buffer = lrbp->cmd->sense_buffer;
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+ u8 *const sense_buffer = cmd->sense_buffer;
u16 resp_len;
int len;
@@ -2644,13 +2654,13 @@ static void ufshcd_sgl_to_prdt(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int
/**
* ufshcd_map_sg - Map scatter-gather list to prdt
* @hba: per adapter instance
- * @lrbp: pointer to local reference block
+ * @cmd: SCSI command
*
* Return: 0 in case of success, non-zero value in case of failure.
*/
-static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
+static int ufshcd_map_sg(struct ufs_hba *hba, struct scsi_cmnd *cmd)
{
- struct scsi_cmnd *cmd = lrbp->cmd;
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int sg_segments = scsi_dma_map(cmd);
if (sg_segments < 0)
@@ -2658,7 +2668,7 @@ static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
ufshcd_sgl_to_prdt(hba, lrbp, sg_segments, scsi_sglist(cmd));
- return ufshcd_crypto_fill_prdt(hba, lrbp);
+ return ufshcd_crypto_fill_prdt(hba, cmd);
}
/**
@@ -2743,13 +2753,13 @@ ufshcd_prepare_req_desc_hdr(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
/**
* ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
* for scsi commands
- * @lrbp: local reference block pointer
+ * @cmd: SCSI command
* @upiu_flags: flags
*/
-static
-void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags)
+static void ufshcd_prepare_utp_scsi_cmd_upiu(struct scsi_cmnd *cmd,
+ u8 upiu_flags)
{
- struct scsi_cmnd *cmd = lrbp->cmd;
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
unsigned short cdb_len;
@@ -2856,20 +2866,23 @@ static int ufshcd_compose_devman_upiu(struct ufs_hba *hba,
* @hba: per adapter instance
* @lrbp: pointer to local reference block
*/
-static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
+static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct scsi_cmnd *cmd)
{
- struct request *rq = scsi_cmd_to_rq(lrbp->cmd);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+ struct request *rq = scsi_cmd_to_rq(cmd);
unsigned int ioprio_class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq));
u8 upiu_flags;
- ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, lrbp->cmd->sc_data_direction, 0);
+ ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags,
+ cmd->sc_data_direction, 0);
if (ioprio_class == IOPRIO_CLASS_RT)
upiu_flags |= UPIU_CMD_FLAGS_CP;
- ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
+ ufshcd_prepare_utp_scsi_cmd_upiu(cmd, upiu_flags);
}
-static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
+static void ufshcd_init_lrb(struct ufs_hba *hba, struct scsi_cmnd *cmd)
{
+ const int i = scsi_cmd_to_rq(cmd)->tag;
struct utp_transfer_cmd_desc *cmd_descp =
(void *)hba->ucdl_base_addr + i * ufshcd_get_ucd_size(hba);
struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
@@ -2877,6 +2890,7 @@ static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
hba->ucdl_dma_addr + i * ufshcd_get_ucd_size(hba);
u16 response_offset = le16_to_cpu(utrdlp[i].response_upiu_offset);
u16 prdt_offset = le16_to_cpu(utrdlp[i].prd_table_offset);
+ struct ufshcd_lrb *lrb = scsi_cmd_priv(cmd);
lrb->utr_descriptor_ptr = utrdlp + i;
lrb->utrd_dma_addr =
@@ -2889,27 +2903,30 @@ static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
}
-static void __ufshcd_setup_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
- struct scsi_cmnd *cmd, u8 lun, int tag)
+static void __ufshcd_setup_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
+ u8 lun, int tag)
{
- ufshcd_init_lrb(hba, lrbp, tag);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+
+ ufshcd_init_lrb(hba, cmd);
memset(lrbp->ucd_req_ptr, 0, sizeof(*lrbp->ucd_req_ptr));
- lrbp->cmd = cmd;
lrbp->task_tag = tag;
lrbp->lun = lun;
ufshcd_prepare_lrbp_crypto(cmd ? scsi_cmd_to_rq(cmd) : NULL, lrbp);
}
-static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
- struct scsi_cmnd *cmd, u8 lun, int tag)
+static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
+ u8 lun, int tag)
{
- __ufshcd_setup_cmd(hba, lrbp, cmd, lun, tag);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+
+ __ufshcd_setup_cmd(hba, cmd, lun, tag);
lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba);
lrbp->req_abort_skip = false;
- ufshcd_comp_scsi_upiu(hba, lrbp);
+ ufshcd_comp_scsi_upiu(hba, cmd);
}
/**
@@ -2980,7 +2997,6 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
{
struct ufs_hba *hba = shost_priv(host);
int tag = scsi_cmd_to_rq(cmd)->tag;
- struct ufshcd_lrb *lrbp;
int err = 0;
struct ufs_hw_queue *hwq = NULL;
@@ -3031,11 +3047,10 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
ufshcd_hold(hba);
- lrbp = &hba->lrb[tag];
+ ufshcd_setup_scsi_cmd(hba, cmd,
+ ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag);
- ufshcd_setup_scsi_cmd(hba, lrbp, cmd, ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag);
-
- err = ufshcd_map_sg(hba, lrbp);
+ err = ufshcd_map_sg(hba, cmd);
if (err) {
ufshcd_release(hba);
goto out;
@@ -3044,7 +3059,7 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
if (hba->mcq_enabled)
hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
- ufshcd_send_command(hba, lrbp, hwq);
+ ufshcd_send_command(hba, cmd, hwq);
out:
if (ufs_trigger_eh(hba)) {
@@ -3058,18 +3073,22 @@ static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
return err;
}
-static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
- enum dev_cmd_type cmd_type, u8 lun, int tag)
+static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
+ enum dev_cmd_type cmd_type, u8 lun, int tag)
{
- __ufshcd_setup_cmd(hba, lrbp, NULL, lun, tag);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+
+ __ufshcd_setup_cmd(hba, cmd, lun, tag);
lrbp->intr_cmd = true; /* No interrupt aggregation */
hba->dev_cmd.type = cmd_type;
}
-static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
- struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
+static int ufshcd_compose_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
+ enum dev_cmd_type cmd_type, int tag)
{
- ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+
+ ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag);
return ufshcd_compose_devman_upiu(hba, lrbp);
}
@@ -3272,13 +3291,14 @@ static void ufshcd_dev_man_unlock(struct ufs_hba *hba)
ufshcd_release(hba);
}
-static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
- const u32 tag, int timeout)
+static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd,
+ const u32 tag, int timeout)
{
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int err;
ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
- ufshcd_send_command(hba, lrbp, hba->dev_cmd_queue);
+ ufshcd_send_command(hba, cmd, hba->dev_cmd_queue);
err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
@@ -3302,17 +3322,17 @@ static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
enum dev_cmd_type cmd_type, int timeout)
{
const u32 tag = hba->reserved_slot;
- struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+ struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag);
int err;
/* Protects use of hba->reserved_slot. */
lockdep_assert_held(&hba->dev_cmd.lock);
- err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
+ err = ufshcd_compose_dev_cmd(hba, cmd, cmd_type, tag);
if (unlikely(err))
return err;
- return ufshcd_issue_dev_cmd(hba, lrbp, tag, timeout);
+ return ufshcd_issue_dev_cmd(hba, cmd, tag, timeout);
}
/**
@@ -3921,14 +3941,6 @@ static int ufshcd_memory_alloc(struct ufs_hba *hba)
}
skip_utmrdl:
- /* Allocate memory for local reference block */
- hba->lrb = devm_kcalloc(hba->dev,
- hba->nutrs, sizeof(struct ufshcd_lrb),
- GFP_KERNEL);
- if (!hba->lrb) {
- dev_err(hba->dev, "LRB Memory allocation failed\n");
- goto out;
- }
return 0;
out:
return -ENOMEM;
@@ -5329,19 +5341,18 @@ static void ufshcd_sdev_destroy(struct scsi_device *sdev)
/**
* ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
- * @lrbp: pointer to local reference block of completed command
+ * @cmd: SCSI command
* @scsi_status: SCSI command status
*
* Return: value base on SCSI command status.
*/
-static inline int
-ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
+static inline int ufshcd_scsi_cmd_status(struct scsi_cmnd *cmd, int scsi_status)
{
int result = 0;
switch (scsi_status) {
case SAM_STAT_CHECK_CONDITION:
- ufshcd_copy_sense_data(lrbp);
+ ufshcd_copy_sense_data(cmd);
fallthrough;
case SAM_STAT_GOOD:
result |= DID_OK << 16 | scsi_status;
@@ -5349,7 +5360,7 @@ ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
case SAM_STAT_TASK_SET_FULL:
case SAM_STAT_BUSY:
case SAM_STAT_TASK_ABORTED:
- ufshcd_copy_sense_data(lrbp);
+ ufshcd_copy_sense_data(cmd);
result |= scsi_status;
break;
default:
@@ -5363,15 +5374,16 @@ ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
/**
* ufshcd_transfer_rsp_status - Get overall status of the response
* @hba: per adapter instance
- * @lrbp: pointer to local reference block of completed command
+ * @cmd: SCSI command
* @cqe: pointer to the completion queue entry
*
* Return: result of the command to notify SCSI midlayer.
*/
-static inline int
-ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
- struct cq_entry *cqe)
+static inline int ufshcd_transfer_rsp_status(struct ufs_hba *hba,
+ struct scsi_cmnd *cmd,
+ struct cq_entry *cqe)
{
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int result = 0;
int scsi_status;
enum utp_ocs ocs;
@@ -5385,7 +5397,7 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
* not set either flag.
*/
if (resid && !(upiu_flags & UPIU_RSP_FLAG_OVERFLOW))
- scsi_set_resid(lrbp->cmd, resid);
+ scsi_set_resid(cmd, resid);
/* overall command status of utrd */
ocs = ufshcd_get_tr_ocs(lrbp, cqe);
@@ -5406,7 +5418,7 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
* to notify the SCSI midlayer of the command status
*/
scsi_status = lrbp->ucd_rsp_ptr->header.status;
- result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
+ result = ufshcd_scsi_cmd_status(cmd, scsi_status);
/*
* Currently we are only supporting BKOPs exception
@@ -5469,7 +5481,7 @@ ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
if ((host_byte(result) != DID_OK) &&
(host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs)
- ufshcd_print_tr(hba, lrbp->task_tag, true);
+ ufshcd_print_tr(hba, cmd, true);
return result;
}
@@ -5538,13 +5550,10 @@ static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
}
/* Release the resources allocated for processing a SCSI command. */
-void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
- struct ufshcd_lrb *lrbp)
+void ufshcd_release_scsi_cmd(struct ufs_hba *hba, struct scsi_cmnd *cmd)
{
- struct scsi_cmnd *cmd = lrbp->cmd;
-
scsi_dma_unmap(cmd);
- ufshcd_crypto_clear_prdt(hba, lrbp);
+ ufshcd_crypto_clear_prdt(hba, cmd);
ufshcd_release(hba);
ufshcd_clk_scaling_update_busy(hba);
}
@@ -5558,20 +5567,18 @@ void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag,
struct cq_entry *cqe)
{
- struct ufshcd_lrb *lrbp;
- struct scsi_cmnd *cmd;
+ struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, task_tag);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
enum utp_ocs ocs;
- lrbp = &hba->lrb[task_tag];
lrbp->compl_time_stamp = ktime_get();
lrbp->compl_time_stamp_local_clock = local_clock();
- cmd = lrbp->cmd;
- if (cmd) {
- if (unlikely(ufshcd_should_inform_monitor(hba, lrbp->cmd)))
- ufshcd_update_monitor(hba, lrbp->cmd);
+ if (ufshcd_is_scsi_cmd(cmd)) {
+ if (unlikely(ufshcd_should_inform_monitor(hba, cmd)))
+ ufshcd_update_monitor(hba, cmd);
ufshcd_add_command_trace(hba, cmd, UFS_CMD_COMP);
- cmd->result = ufshcd_transfer_rsp_status(hba, lrbp, cqe);
- ufshcd_release_scsi_cmd(hba, lrbp);
+ cmd->result = ufshcd_transfer_rsp_status(hba, cmd, cqe);
+ ufshcd_release_scsi_cmd(hba, cmd);
/* Do not touch lrbp after scsi done */
scsi_done(cmd);
} else {
@@ -5608,7 +5615,7 @@ static void ufshcd_clear_polled(struct ufs_hba *hba,
int tag;
for_each_set_bit(tag, completed_reqs, hba->nutrs) {
- struct scsi_cmnd *cmd = hba->lrb[tag].cmd;
+ struct scsi_cmnd *cmd = scsi_host_find_tag(hba->host, tag);
if (!cmd)
continue;
@@ -5659,7 +5666,6 @@ static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv)
struct scsi_device *sdev = rq->q->queuedata;
struct Scsi_Host *shost = sdev->host;
struct ufs_hba *hba = shost_priv(shost);
- struct ufshcd_lrb *lrbp = &hba->lrb[rq->tag];
struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq);
unsigned long flags;
@@ -5671,7 +5677,7 @@ static bool ufshcd_mcq_force_compl_one(struct request *rq, void *priv)
spin_lock_irqsave(&hwq->cq_lock, flags);
if (cmd && !test_bit(SCMD_STATE_COMPLETE, &cmd->state)) {
set_host_byte(cmd, DID_REQUEUE);
- ufshcd_release_scsi_cmd(hba, lrbp);
+ ufshcd_release_scsi_cmd(hba, cmd);
scsi_done(cmd);
}
spin_unlock_irqrestore(&hwq->cq_lock, flags);
@@ -6533,7 +6539,7 @@ static bool ufshcd_abort_one(struct request *rq, void *priv)
*ret = ufshcd_try_to_abort_task(hba, tag);
dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag,
- hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1,
+ ufshcd_is_scsi_cmd(cmd) ? cmd->cmnd[0] : -1,
*ret ? "failed" : "succeeded");
return *ret == 0;
@@ -7224,14 +7230,15 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
enum query_opcode desc_op)
{
const u32 tag = hba->reserved_slot;
- struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+ struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int err = 0;
u8 upiu_flags;
/* Protects use of hba->reserved_slot. */
lockdep_assert_held(&hba->dev_cmd.lock);
- ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag);
+ ufshcd_setup_dev_cmd(hba, cmd, cmd_type, 0, tag);
ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0);
@@ -7256,7 +7263,7 @@ static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
* bound to fail since dev_cmd.query and dev_cmd.type were left empty.
* read the response directly ignoring all errors.
*/
- ufshcd_issue_dev_cmd(hba, lrbp, tag, QUERY_REQ_TIMEOUT);
+ ufshcd_issue_dev_cmd(hba, cmd, tag, QUERY_REQ_TIMEOUT);
/* just copy the upiu response as it is */
memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
@@ -7372,7 +7379,8 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r
enum dma_data_direction dir)
{
const u32 tag = hba->reserved_slot;
- struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+ struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int err = 0;
int result;
u8 upiu_flags;
@@ -7383,7 +7391,8 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r
/* Protects use of hba->reserved_slot. */
ufshcd_dev_man_lock(hba);
- ufshcd_setup_dev_cmd(hba, lrbp, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN, tag);
+ ufshcd_setup_dev_cmd(hba, cmd, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN,
+ tag);
ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, ehs);
@@ -7400,7 +7409,7 @@ int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *r
memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
- err = ufshcd_issue_dev_cmd(hba, lrbp, tag, ADVANCED_RPMB_REQ_TIMEOUT);
+ err = ufshcd_issue_dev_cmd(hba, cmd, tag, ADVANCED_RPMB_REQ_TIMEOUT);
if (!err) {
/* Just copy the upiu response as it is */
@@ -7499,11 +7508,12 @@ static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
{
- struct ufshcd_lrb *lrbp;
int tag;
for_each_set_bit(tag, &bitmap, hba->nutrs) {
- lrbp = &hba->lrb[tag];
+ struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
+
lrbp->req_abort_skip = true;
}
}
@@ -7511,7 +7521,7 @@ static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
/**
* ufshcd_try_to_abort_task - abort a specific task
* @hba: Pointer to adapter instance
- * @tag: Task tag/index to be aborted
+ * @tag: Tag of the task to be aborted
*
* Abort the pending command in device by sending UFS_ABORT_TASK task management
* command, and in host controller by clearing the door-bell register. There can
@@ -7523,7 +7533,8 @@ static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
*/
int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
{
- struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+ struct scsi_cmnd *cmd = ufshcd_tag_to_cmd(hba, tag);
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
int err;
int poll_cnt;
u8 resp = 0xF;
@@ -7545,7 +7556,7 @@ int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
hba->dev,
"%s: cmd with tag %d not pending in the device.\n",
__func__, tag);
- if (!ufshcd_cmd_inflight(lrbp->cmd)) {
+ if (!ufshcd_cmd_inflight(cmd)) {
dev_info(hba->dev,
"%s: cmd with tag=%d completed.\n",
__func__, tag);
@@ -7593,7 +7604,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
struct Scsi_Host *host = cmd->device->host;
struct ufs_hba *hba = shost_priv(host);
int tag = scsi_cmd_to_rq(cmd)->tag;
- struct ufshcd_lrb *lrbp = &hba->lrb[tag];
+ struct ufshcd_lrb *lrbp = scsi_cmd_priv(cmd);
unsigned long flags;
int err = FAILED;
bool outstanding;
@@ -7628,9 +7639,9 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
ufshcd_print_evt_hist(hba);
ufshcd_print_host_state(hba);
ufshcd_print_pwr_info(hba);
- ufshcd_print_tr(hba, tag, true);
+ ufshcd_print_tr(hba, cmd, true);
} else {
- ufshcd_print_tr(hba, tag, false);
+ ufshcd_print_tr(hba, cmd, false);
}
hba->req_abort_count++;
@@ -7674,7 +7685,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
goto release;
}
- err = ufshcd_try_to_abort_task(hba, tag);
+ err = ufshcd_try_to_abort_task(hba, lrbp->task_tag);
if (err) {
dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
@@ -7691,7 +7702,7 @@ static int ufshcd_abort(struct scsi_cmnd *cmd)
spin_unlock_irqrestore(&hba->outstanding_lock, flags);
if (outstanding)
- ufshcd_release_scsi_cmd(hba, lrbp);
+ ufshcd_release_scsi_cmd(hba, cmd);
err = SUCCESS;
@@ -8726,11 +8737,6 @@ static void ufshcd_memory_free(struct ufs_hba *hba)
hba->utrdl_base_addr = NULL;
hba->utrdl_dma_addr = 0;
}
-
- if (hba->lrb) {
- devm_kfree(hba->dev, hba->lrb);
- hba->lrb = NULL;
- }
}
static int ufshcd_alloc_mcq(struct ufs_hba *hba, u32 ufs_dev_qd)
@@ -9040,6 +9046,7 @@ static const struct scsi_host_template ufshcd_driver_template = {
.name = UFSHCD,
.proc_name = UFSHCD,
.map_queues = ufshcd_map_queues,
+ .cmd_size = sizeof(struct ufshcd_lrb),
.init_cmd_priv = ufshcd_init_cmd_priv,
.queuecommand = ufshcd_queuecommand,
.mq_poll = ufshcd_poll,
@@ -161,7 +161,6 @@ struct ufs_pm_lvl_states {
* @ucd_prdt_dma_addr: PRDT dma address for debug
* @ucd_rsp_dma_addr: UPIU response dma address for debug
* @ucd_req_dma_addr: UPIU request dma address for debug
- * @cmd: pointer to SCSI command
* @scsi_status: SCSI status of the command
* @command_type: SCSI, UFS, Query.
* @task_tag: Task tag of the command
@@ -186,7 +185,6 @@ struct ufshcd_lrb {
dma_addr_t ucd_rsp_dma_addr;
dma_addr_t ucd_prdt_dma_addr;
- struct scsi_cmnd *cmd;
int scsi_status;
int command_type;
@@ -866,7 +864,6 @@ enum ufshcd_mcq_opr {
* @spm_lvl: desired UFS power management level during system PM.
* @pm_op_in_progress: whether or not a PM operation is in progress.
* @ahit: value of Auto-Hibernate Idle Timer register.
- * @lrb: local reference block
* @outstanding_tasks: Bits representing outstanding task requests
* @outstanding_lock: Protects @outstanding_reqs.
* @outstanding_reqs: Bits representing outstanding transfer requests
@@ -1009,8 +1006,6 @@ struct ufs_hba {
/* Auto-Hibernate Idle Timer register value */
u32 ahit;
- struct ufshcd_lrb *lrb;
-
unsigned long outstanding_tasks;
spinlock_t outstanding_lock;
unsigned long outstanding_reqs;
Convert the cmd->lrbp and lrbp->cmd memory loads into pointer offset calculations. Signed-off-by: Bart Van Assche <bvanassche@acm.org> --- drivers/ufs/core/ufs-mcq.c | 15 +- drivers/ufs/core/ufshcd-crypto.h | 18 ++- drivers/ufs/core/ufshcd-priv.h | 25 +++- drivers/ufs/core/ufshcd.c | 243 ++++++++++++++++--------------- include/ufs/ufshcd.h | 5 - 5 files changed, 167 insertions(+), 139 deletions(-)