[SRU][R:linux-gcp][PATCH v2 01/10] RDMA/irdma: Provide scratch buffers to firmware for internal use
Ian Whitfield
ian.whitfield at canonical.com
Wed Sep 2 19:32:55 UTC 2026
From: Jay Bhat <jay.bhat at intel.com>
BugLink: https://bugs.launchpad.net/bugs/2165062
For GEN3 and higher, FW requires scratch buffers for bookkeeping
during cleanup, specifically during QP and MR destroy ops.
Signed-off-by: Jay Bhat <jay.bhat at intel.com>
Signed-off-by: Tatyana Nikolova <tatyana.e.nikolova at intel.com>
Signed-off-by: Leon Romanovsky <leon at kernel.org>
(cherry picked from commit 9d6ba4ced73433df3d3c5909d73a49945bedd5c6)
Signed-off-by: Ian Whitfield <ian.whitfield at canonical.com>
---
drivers/infiniband/hw/irdma/ctrl.c | 43 +++++++++++++++++++++++++++++-
drivers/infiniband/hw/irdma/defs.h | 4 +++
drivers/infiniband/hw/irdma/hw.c | 11 ++++++++
drivers/infiniband/hw/irdma/type.h | 2 ++
drivers/infiniband/hw/irdma/user.h | 4 +--
5 files changed, 61 insertions(+), 3 deletions(-)
diff --git a/drivers/infiniband/hw/irdma/ctrl.c b/drivers/infiniband/hw/irdma/ctrl.c
index 45c7433c96f36..13820f1a48a40 100644
--- a/drivers/infiniband/hw/irdma/ctrl.c
+++ b/drivers/infiniband/hw/irdma/ctrl.c
@@ -3570,6 +3570,41 @@ static int irdma_sc_parse_fpm_query_buf(struct irdma_sc_dev *dev, __le64 *buf,
hmc_fpm_misc->loc_mem_pages = (u32)FIELD_GET(IRDMA_QUERY_FPM_LOC_MEM_PAGES, temp);
if (!hmc_fpm_misc->loc_mem_pages)
return -EINVAL;
+
+ get_64bit_val(buf, 184, &temp);
+ if (temp) {
+ hmc_fpm_misc->fw_scratch_buf0.size = temp;
+ hmc_fpm_misc->fw_scratch_buf0.va =
+ dma_alloc_coherent(dev->hw->device,
+ hmc_fpm_misc->fw_scratch_buf0.size,
+ &hmc_fpm_misc->fw_scratch_buf0.pa,
+ GFP_KERNEL);
+
+ if (!hmc_fpm_misc->fw_scratch_buf0.va) {
+ hmc_fpm_misc->fw_scratch_buf0.size = 0;
+ return -ENOMEM;
+ }
+ }
+ get_64bit_val(buf, 192, &temp);
+ if (temp) {
+ hmc_fpm_misc->fw_scratch_buf1.size = temp;
+ hmc_fpm_misc->fw_scratch_buf1.va =
+ dma_alloc_coherent(dev->hw->device,
+ hmc_fpm_misc->fw_scratch_buf1.size,
+ &hmc_fpm_misc->fw_scratch_buf1.pa,
+ GFP_KERNEL);
+
+ if (!hmc_fpm_misc->fw_scratch_buf1.va) {
+ hmc_fpm_misc->fw_scratch_buf1.size = 0;
+ dma_free_coherent(dev->hw->device,
+ hmc_fpm_misc->fw_scratch_buf0.size,
+ hmc_fpm_misc->fw_scratch_buf0.va,
+ hmc_fpm_misc->fw_scratch_buf0.pa);
+ hmc_fpm_misc->fw_scratch_buf0.va = NULL;
+ hmc_fpm_misc->fw_scratch_buf0.size = 0;
+ return -ENOMEM;
+ }
+ }
}
return 0;
@@ -4187,6 +4222,8 @@ static int irdma_sc_commit_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch,
hdr = FIELD_PREP(IRDMA_CQPSQ_BUFSIZE, IRDMA_COMMIT_FPM_BUF_SIZE) |
FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_COMMIT_FPM_VAL) |
+ FIELD_PREP(IRDMA_CQPSQ_CFPM_FW_SCRATCH_BUF_PRESENT,
+ cqp->dev->hmc_fpm_misc.fw_scratch_buf0.va != NULL) |
FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
dma_wmb(); /* make sure WQE is written before valid bit is set */
@@ -5034,7 +5071,9 @@ static void irdma_set_loc_mem(__le64 *buf)
for (offset = 0; offset < IRDMA_COMMIT_FPM_BUF_SIZE;
offset += sizeof(__le64)) {
- if (offset == IRDMA_PBLE_COMMIT_OFFSET)
+ if (offset == IRDMA_PBLE_COMMIT_OFFSET ||
+ offset == IRDMA_SCRATCH_BUF0_COMMIT_OFFSET ||
+ offset == IRDMA_SCRATCH_BUF1_COMMIT_OFFSET)
continue;
get_64bit_val(buf, offset, &temp);
if (temp)
@@ -5090,6 +5129,8 @@ static int irdma_sc_cfg_iw_fpm(struct irdma_sc_dev *dev, u8 hmc_fn_id)
(u64)obj_info[IRDMA_HMC_IW_OOISC].cnt);
set_64bit_val(buf, 168,
(u64)obj_info[IRDMA_HMC_IW_OOISCFFL].cnt);
+ set_64bit_val(buf, 192, dev->hmc_fpm_misc.fw_scratch_buf0.pa);
+ set_64bit_val(buf, 200, dev->hmc_fpm_misc.fw_scratch_buf1.pa);
if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_3 &&
dev->hmc_fpm_misc.loc_mem_pages)
irdma_set_loc_mem(buf);
diff --git a/drivers/infiniband/hw/irdma/defs.h b/drivers/infiniband/hw/irdma/defs.h
index 983b22d7ae233..d6a3152959dde 100644
--- a/drivers/infiniband/hw/irdma/defs.h
+++ b/drivers/infiniband/hw/irdma/defs.h
@@ -133,6 +133,8 @@ enum irdma_protocol_used {
#define MAX_MR_PER_SD 0x8000
#define MAX_MR_SD_PER_FCN 0x80
#define IRDMA_PBLE_COMMIT_OFFSET 112
+#define IRDMA_SCRATCH_BUF0_COMMIT_OFFSET 192
+#define IRDMA_SCRATCH_BUF1_COMMIT_OFFSET 200
#define IRDMA_MAX_QUANTA_PER_WR 8
#define IRDMA_QP_SW_MAX_WQ_QUANTA 32768
@@ -658,6 +660,8 @@ enum irdma_cqp_op_type {
#define IRDMA_COMMIT_FPM_QPCNT GENMASK_ULL(20, 0)
#define IRDMA_COMMIT_FPM_BASE_S 32
#define IRDMA_CQPSQ_CFPM_HMCFNID GENMASK_ULL(15, 0)
+#define IRDMA_CQPSQ_CFPM_FW_SCRATCH_BUF_PRESENT_S 38
+#define IRDMA_CQPSQ_CFPM_FW_SCRATCH_BUF_PRESENT BIT_ULL(38)
#define IRDMA_CQPSQ_FWQE_AECODE GENMASK_ULL(15, 0)
#define IRDMA_CQPSQ_FWQE_AESOURCE GENMASK_ULL(19, 16)
diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c
index f4ae530f56db0..879fb632ee7a9 100644
--- a/drivers/infiniband/hw/irdma/hw.c
+++ b/drivers/infiniband/hw/irdma/hw.c
@@ -1693,6 +1693,8 @@ static int irdma_hmc_setup(struct irdma_pci_f *rf)
static void irdma_del_init_mem(struct irdma_pci_f *rf)
{
struct irdma_sc_dev *dev = &rf->sc_dev;
+ struct irdma_dma_mem *fw_scratch_buf0;
+ struct irdma_dma_mem *fw_scratch_buf1;
if (!rf->sc_dev.privileged)
irdma_vchnl_req_put_hmc_fcn(&rf->sc_dev);
@@ -1713,6 +1715,15 @@ static void irdma_del_init_mem(struct irdma_pci_f *rf)
rf->iw_msixtbl = NULL;
kfree(rf->hmc_info_mem);
rf->hmc_info_mem = NULL;
+
+ fw_scratch_buf0 = &dev->hmc_fpm_misc.fw_scratch_buf0;
+ fw_scratch_buf1 = &dev->hmc_fpm_misc.fw_scratch_buf1;
+ if (fw_scratch_buf0->va)
+ dma_free_coherent(dev->hw->device, fw_scratch_buf0->size,
+ fw_scratch_buf0->va, fw_scratch_buf0->pa);
+ if (fw_scratch_buf1->va)
+ dma_free_coherent(dev->hw->device, fw_scratch_buf1->size,
+ fw_scratch_buf1->va, fw_scratch_buf1->pa);
}
/**
diff --git a/drivers/infiniband/hw/irdma/type.h b/drivers/infiniband/hw/irdma/type.h
index da8c54d1f035a..5557d9338796f 100644
--- a/drivers/infiniband/hw/irdma/type.h
+++ b/drivers/infiniband/hw/irdma/type.h
@@ -622,6 +622,8 @@ struct irdma_hmc_fpm_misc {
u32 timer_bucket;
u32 rrf_block_size;
u32 ooiscf_block_size;
+ struct irdma_dma_mem fw_scratch_buf0;
+ struct irdma_dma_mem fw_scratch_buf1;
};
#define IRDMA_VCHNL_MAX_MSG_SIZE 512
diff --git a/drivers/infiniband/hw/irdma/user.h b/drivers/infiniband/hw/irdma/user.h
index 9eb7fd0b1cbf9..008af1acc9288 100644
--- a/drivers/infiniband/hw/irdma/user.h
+++ b/drivers/infiniband/hw/irdma/user.h
@@ -159,8 +159,8 @@ enum irdma_device_caps_const {
IRDMA_CEQE_SIZE = 1,
IRDMA_CQP_CTX_SIZE = 8,
IRDMA_SHADOW_AREA_SIZE = 8,
- IRDMA_QUERY_FPM_BUF_SIZE = 192,
- IRDMA_COMMIT_FPM_BUF_SIZE = 192,
+ IRDMA_QUERY_FPM_BUF_SIZE = 200,
+ IRDMA_COMMIT_FPM_BUF_SIZE = 208,
IRDMA_GATHER_STATS_BUF_SIZE = 1024,
IRDMA_MIN_IW_QP_ID = 0,
IRDMA_MAX_IW_QP_ID = 262143,
--
2.43.0
More information about the kernel-team
mailing list