Merge tag 'for-linus-fwctl' of git://git.kernel.org/pub/scm/linux/kernel/git/fwctl/fwctl

Pull fwctl updates from Jason Gunthorpe:

 - Support more commands in bnxt, this completes what they originally
   wanted to do

 - Rust bindings for fwctl. The Nova GPU is expected to use them next
   cycle

* tag 'for-linus-fwctl' of git://git.kernel.org/pub/scm/linux/kernel/git/fwctl/fwctl:
  rust: introduce abstractions for fwctl
  fwctl/bnxt: Add DMA buffer support for HWRM commands
  bnxt_en: Update bnxt firmware spec

10 files changed, 1610 insertions, 6 deletions

diff --git a/MAINTAINERS b/MAINTAINERS
index b08bbe1271f8..d2b7ed20b247 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -10730,12 +10730,15 @@ FWCTL SUBSYSTEM
 M:	Dave Jiang <dave.jiang@intel.com>
 M:	Jason Gunthorpe <jgg@nvidia.com>
 M:	Saeed Mahameed <saeedm@nvidia.com>
+M:	Zhi Wang <zhiw@nvidia.com> (RUST)
 R:	Jonathan Cameron <jic23@kernel.org>
 S:	Maintained
 F:	Documentation/userspace-api/fwctl/
 F:	drivers/fwctl/
 F:	include/linux/fwctl.h
 F:	include/uapi/fwctl/
+F:	rust/helpers/fwctl.c
+F:	rust/kernel/fwctl.rs
 
 FWCTL BNXT DRIVER
 M:	Pavan Chebbi <pavan.chebbi@broadcom.com>
diff --git a/drivers/fwctl/Kconfig b/drivers/fwctl/Kconfig
index d1b1925bdaec..cac38cf79f30 100644
--- a/drivers/fwctl/Kconfig
+++ b/drivers/fwctl/Kconfig
@@ -9,6 +9,18 @@ menuconfig FWCTL
 	  fit neatly into an existing subsystem.
 
 if FWCTL
+
+config RUST_FWCTL_ABSTRACTIONS
+	bool "Rust fwctl abstractions"
+	depends on RUST && FWCTL=y
+	help
+	  This enables the Rust abstractions for the fwctl device firmware
+	  access framework. It provides safe wrappers around struct fwctl_device
+	  and struct fwctl_uctx, allowing Rust drivers to register fwctl devices
+	  and implement their control and RPC logic in safe Rust.
+
+	  If unsure, say N.
+
 config FWCTL_BNXT
 	tristate "bnxt control fwctl driver"
 	depends on BNXT
diff --git a/drivers/fwctl/bnxt/main.c b/drivers/fwctl/bnxt/main.c
index 951c8ac2e0a1..49944e706baf 100644
--- a/drivers/fwctl/bnxt/main.c
+++ b/drivers/fwctl/bnxt/main.c
@@ -4,6 +4,7 @@
  */
 
 #include <linux/auxiliary_bus.h>
+#include <linux/dma-mapping.h>
 #include <linux/slab.h>
 #include <linux/pci.h>
 #include <linux/fwctl.h>
@@ -31,7 +32,8 @@ static int bnxtctl_open_uctx(struct fwctl_uctx *uctx)
 
 	bnxtctl_uctx->uctx_caps = BIT(FWCTL_BNXT_INLINE_COMMANDS) |
 				  BIT(FWCTL_BNXT_QUERY_COMMANDS) |
-				  BIT(FWCTL_BNXT_SEND_COMMANDS);
+				  BIT(FWCTL_BNXT_SEND_COMMANDS) |
+				  BIT(FWCTL_BNXT_DMA_COMMANDS);
 	return 0;
 }
 
@@ -55,18 +57,348 @@ static void *bnxtctl_info(struct fwctl_uctx *uctx, size_t *length)
 	return info;
 }
 
+#define BNXTCTL_MAX_DMA_FIELDS	4
+
+struct bnxtctl_dma_field {
+	size_t offset;		/* offsetof(hwrm_xxx_input, addr_field) */
+	enum dma_data_direction dir;
+	size_t len_offset;	/* offsetof(hwrm_xxx_input, len_field); 0 if the
+				 * command carries no transfer-length field
+				 */
+	u8     len_width;	/* byte width of the length field: 2 or 4 */
+	u8     len_unit;	/* bytes represented by one unit of the length field */
+	u32    buf_len;		/* for commands with no length in payload */
+};
+
+struct bnxtctl_cmd_dma_desc {
+	u16                      req_type;
+	u8                       num_fields;
+	u8                       scope_min;
+	size_t                   req_size;   /* sizeof(struct hwrm_xxx_input) */
+	struct bnxtctl_dma_field fields[BNXTCTL_MAX_DMA_FIELDS];
+};
+
+/* input struct has an addr/len pair, but len is multiplied by _unit */
+#define CMD_DATA_UNIT(_struct, _dir, _data, _len, _unit) \
+	{ .offset = offsetof(_struct, _data),            \
+	  .dir = _dir,                                   \
+	  .len_offset = offsetof(_struct, _len),         \
+	  .len_width = sizeof(((_struct *)0)->_len),     \
+	  .len_unit = _unit }
+
+/* input struct has an addr/len pair with byte length */
+#define CMD_DATA_SIMPLE(_struct, _dir, _data, _len) \
+	CMD_DATA_UNIT(_struct, _dir, _data, _len, 1)
+
+/* input struct has an addr but the length is fixed */
+#define CMD_DATA_FIXED(_struct, _dir, _data, _len) \
+	{ .offset = offsetof(_struct, _data), .dir = _dir, .buf_len = _len }
+
+#define CMD_DMAS(_req_type, _scope_min, _struct, _num_fields, ...) \
+	{                                                          \
+		.req_type = _req_type,                             \
+		.scope_min = _scope_min,                           \
+		.req_size = sizeof(_struct),                       \
+		.num_fields = _num_fields,                         \
+		.fields = { __VA_ARGS__ },                         \
+	}
+
+#define CMD_DMA_LEN(_req_type, _scope_min, _dir, _struct, _data, _len) \
+	CMD_DMAS(_req_type, _scope_min, _struct, 1,                    \
+		 CMD_DATA_SIMPLE(_struct, _dir, _data, _len))
+
+/*
+ * Per-command DMA buffer descriptor table for HWRM commands that
+ * carry __le64 DMA address fields in their input
+ */
+static const struct bnxtctl_cmd_dma_desc bnxtctl_dma_cmds[] = {
+	CMD_DMA_LEN(HWRM_NVM_SET_VARIABLE, FWCTL_RPC_CONFIGURATION,
+		    DMA_TO_DEVICE,
+		    struct hwrm_nvm_set_variable_input, src_data_addr,
+		    data_len),
+	CMD_DMA_LEN(HWRM_NVM_GET_VARIABLE, FWCTL_RPC_CONFIGURATION,
+		    DMA_FROM_DEVICE,
+		    struct hwrm_nvm_get_variable_input, dest_data_addr,
+		    data_len),
+	CMD_DMA_LEN(HWRM_NVM_READ, FWCTL_RPC_DEBUG_READ_ONLY,
+		    DMA_FROM_DEVICE, struct hwrm_nvm_read_input,
+		    host_dest_addr, len),
+	CMD_DMAS(HWRM_NVM_GET_DIR_ENTRIES, FWCTL_RPC_DEBUG_READ_ONLY,
+		 struct hwrm_nvm_get_dir_entries_input, 1,
+		 CMD_DATA_FIXED(struct hwrm_nvm_get_dir_entries_input,
+				DMA_FROM_DEVICE, host_dest_addr,
+				FWCTL_BNXT_MAX_DMABUF)),
+	CMD_DMA_LEN(HWRM_NVM_WRITE, FWCTL_RPC_DEBUG_WRITE,
+		    DMA_TO_DEVICE, struct hwrm_nvm_write_input,
+		    host_src_addr, dir_data_length),
+	CMD_DMA_LEN(HWRM_NVM_MODIFY, FWCTL_RPC_DEBUG_WRITE,
+		    DMA_TO_DEVICE, struct hwrm_nvm_modify_input,
+		    host_src_addr, len),
+	CMD_DMA_LEN(HWRM_NVM_RAW_WRITE_BLK, FWCTL_RPC_DEBUG_WRITE_FULL,
+		    DMA_TO_DEVICE,
+		    struct hwrm_nvm_raw_write_blk_input, host_src_addr, len),
+	CMD_DMA_LEN(HWRM_NVM_RAW_DUMP, FWCTL_RPC_DEBUG_READ_ONLY,
+		    DMA_FROM_DEVICE, struct hwrm_nvm_raw_dump_input,
+		    host_dest_addr, len),
+
+	CMD_DMA_LEN(HWRM_FW_GET_STRUCTURED_DATA, FWCTL_RPC_DEBUG_READ_ONLY,
+		    DMA_FROM_DEVICE,
+		    struct hwrm_fw_get_structured_data_input, dest_data_addr,
+		    data_len),
+	CMD_DMA_LEN(HWRM_FW_SET_STRUCTURED_DATA, FWCTL_RPC_DEBUG_WRITE,
+		    DMA_TO_DEVICE,
+		    struct hwrm_fw_set_structured_data_input, src_data_addr,
+		    data_len),
+	CMD_DMA_LEN(HWRM_FW_LIVEPATCH, FWCTL_RPC_DEBUG_WRITE_FULL,
+		    DMA_TO_DEVICE, struct hwrm_fw_livepatch_input,
+		    host_addr, patch_len),
+
+	CMD_DMA_LEN(HWRM_DBG_COREDUMP_LIST, FWCTL_RPC_DEBUG_READ_ONLY,
+		    DMA_FROM_DEVICE,
+		    struct hwrm_dbg_coredump_list_input, host_dest_addr,
+		    host_buf_len),
+	CMD_DMA_LEN(HWRM_DBG_COREDUMP_RETRIEVE, FWCTL_RPC_DEBUG_READ_ONLY,
+		    DMA_FROM_DEVICE,
+		    struct hwrm_dbg_coredump_retrieve_input, host_dest_addr,
+		    host_buf_len),
+	/* read_len32 counts 32-bit words, not bytes (see bnxt_dbg_hwrm_rd_reg()). */
+	CMD_DMAS(HWRM_DBG_READ_DIRECT, FWCTL_RPC_DEBUG_READ_ONLY,
+		 struct hwrm_dbg_read_direct_input, 1,
+		 CMD_DATA_UNIT(struct hwrm_dbg_read_direct_input,
+			       DMA_FROM_DEVICE,
+			       host_dest_addr, read_len32, 4)),
+	CMD_DMA_LEN(HWRM_DBG_READ_INDIRECT, FWCTL_RPC_DEBUG_READ_ONLY,
+		    DMA_FROM_DEVICE,
+		    struct hwrm_dbg_read_indirect_input, host_dest_addr,
+		    host_dest_addr_len),
+	CMD_DMA_LEN(HWRM_DBG_SERDES_TEST, FWCTL_RPC_DEBUG_READ_ONLY,
+		    DMA_FROM_DEVICE,
+		    struct hwrm_dbg_serdes_test_input, resp_data_addr,
+		    data_len),
+	CMD_DMA_LEN(HWRM_DBG_TOKEN_CFG, FWCTL_RPC_DEBUG_WRITE_FULL,
+		    DMA_TO_DEVICE, struct hwrm_dbg_token_cfg_input,
+		    host_src_addr, dbg_token_len),
+
+	CMD_DMA_LEN(HWRM_QUEUE_DSCP2PRI_QCFG, FWCTL_RPC_DEBUG_READ_ONLY,
+		    DMA_FROM_DEVICE,
+		    struct hwrm_queue_dscp2pri_qcfg_input, dest_data_addr,
+		    dest_data_buffer_size),
+
+	CMD_DMAS(HWRM_PORT_QSTATS, FWCTL_RPC_DEBUG_READ_ONLY,
+		 struct hwrm_port_qstats_input, 2,
+		 CMD_DATA_FIXED(struct hwrm_port_qstats_input,
+				DMA_FROM_DEVICE, tx_stat_host_addr,
+				sizeof(struct tx_port_stats)),
+		 CMD_DATA_FIXED(struct hwrm_port_qstats_input,
+				DMA_FROM_DEVICE, rx_stat_host_addr,
+				sizeof(struct rx_port_stats))),
+	CMD_DMAS(HWRM_PORT_QSTATS_EXT, FWCTL_RPC_DEBUG_READ_ONLY,
+		 struct hwrm_port_qstats_ext_input, 2,
+		 CMD_DATA_SIMPLE(struct hwrm_port_qstats_ext_input,
+				 DMA_FROM_DEVICE, tx_stat_host_addr,
+				 tx_stat_size),
+		 CMD_DATA_SIMPLE(struct hwrm_port_qstats_ext_input,
+				 DMA_FROM_DEVICE, rx_stat_host_addr,
+				 rx_stat_size)),
+	CMD_DMAS(HWRM_PORT_QSTATS_EXT_PFC_ADV, FWCTL_RPC_DEBUG_READ_ONLY,
+		 struct hwrm_port_qstats_ext_pfc_adv_input, 2,
+		 CMD_DATA_SIMPLE(struct hwrm_port_qstats_ext_pfc_adv_input,
+				 DMA_FROM_DEVICE,
+				 tx_pfc_adv_stat_host_addr, pfc_adv_stat_size),
+		 CMD_DATA_SIMPLE(struct hwrm_port_qstats_ext_pfc_adv_input,
+				 DMA_FROM_DEVICE,
+				 rx_pfc_adv_stat_host_addr, pfc_adv_stat_size)),
+	CMD_DMA_LEN(HWRM_PCIE_QSTATS, FWCTL_RPC_DEBUG_READ_ONLY,
+		    DMA_FROM_DEVICE, struct hwrm_pcie_qstats_input,
+		    pcie_stat_host_addr, pcie_stat_size),
+	CMD_DMA_LEN(HWRM_STAT_GENERIC_QSTATS, FWCTL_RPC_DEBUG_READ_ONLY,
+		    DMA_FROM_DEVICE,
+		    struct hwrm_stat_generic_qstats_input,
+		    generic_stat_host_addr, generic_stat_size),
+	CMD_DMA_LEN(HWRM_STAT_QUERY_ROCE_STATS, FWCTL_RPC_DEBUG_READ_ONLY,
+		    DMA_FROM_DEVICE,
+		    struct hwrm_stat_query_roce_stats_input,
+		    roce_stat_host_addr, roce_stat_size),
+	CMD_DMA_LEN(HWRM_STAT_QUERY_ROCE_STATS_EXT, FWCTL_RPC_DEBUG_READ_ONLY,
+		    DMA_FROM_DEVICE,
+		    struct hwrm_stat_query_roce_stats_ext_input,
+		    roce_stat_host_addr, roce_stat_size),
+
+	CMD_DMA_LEN(HWRM_PORT_EVENTS_LOG, FWCTL_RPC_DEBUG_READ_ONLY,
+		    DMA_FROM_DEVICE,
+		    struct hwrm_port_events_log_input, host_dest_addr,
+		    host_dest_addr_len),
+	CMD_DMA_LEN(HWRM_PORT_PRBS_TEST, FWCTL_RPC_DEBUG_READ_ONLY,
+		    DMA_FROM_DEVICE,
+		    struct hwrm_port_prbs_test_input, resp_data_addr, data_len),
+	CMD_DMA_LEN(HWRM_PORT_DSC_DUMP, FWCTL_RPC_DEBUG_READ_ONLY,
+		    DMA_FROM_DEVICE, struct hwrm_port_dsc_dump_input,
+		    resp_data_addr, data_len),
+
+	/* num_fids counts 16-bit FIDs, not bytes. */
+	CMD_DMAS(HWRM_SCH_GRP_CFG, FWCTL_RPC_DEBUG_WRITE,
+		 struct hwrm_sch_grp_cfg_input, 1,
+		 CMD_DATA_UNIT(struct hwrm_sch_grp_cfg_input,
+			       DMA_TO_DEVICE, fid_table_addr,
+			       num_fids, 2)),
+	CMD_DMA_LEN(HWRM_SCH_GRP_QCFG, FWCTL_RPC_DEBUG_READ_ONLY,
+		    DMA_FROM_DEVICE, struct hwrm_sch_grp_qcfg_input,
+		    fid_table_addr, fid_table_len),
+
+	CMD_DMA_LEN(HWRM_SELFTEST_RETRIEVE_SERDES_DATA,
+		    FWCTL_RPC_DEBUG_READ_ONLY, DMA_FROM_DEVICE,
+		    struct hwrm_selftest_retrieve_serdes_data_input,
+		    resp_data_addr, data_len),
+
+	CMD_DMAS(HWRM_DBG_PTRACE, FWCTL_RPC_DEBUG_WRITE,
+		 struct hwrm_dbg_ptrace_input, 2,
+		 CMD_DATA_SIMPLE(struct hwrm_dbg_ptrace_input,
+				 DMA_TO_DEVICE, pdi_cmd_buf_addr,
+				 pdi_req_buf_len),
+		 CMD_DATA_SIMPLE(struct hwrm_dbg_ptrace_input,
+				 DMA_FROM_DEVICE, pdi_resp_buf_addr,
+				 pdi_req_buf_len)),
+};
+
+#undef CMD_DATA_UNIT
+#undef CMD_DATA_SIMPLE
+#undef CMD_DATA_FIXED
+#undef CMD_DMAS
+#undef CMD_DMA_LEN
+
+static const struct bnxtctl_cmd_dma_desc *
+bnxtctl_find_dma_desc(u16 req_type)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(bnxtctl_dma_cmds); i++)
+		if (bnxtctl_dma_cmds[i].req_type == req_type)
+			return &bnxtctl_dma_cmds[i];
+	return NULL;
+}
+
+static void bnxtctl_extract_and_zero_dma_fields(void *cmd,
+						const struct bnxtctl_cmd_dma_desc *desc,
+						u64 *user_addrs)
+{
+	unsigned int i;
+
+	for (i = 0; i < desc->num_fields; i++) {
+		__le32 *field = cmd + desc->fields[i].offset;
+
+		user_addrs[i] = le32_to_cpu(field[0]) |
+				((u64)le32_to_cpu(field[1]) << 32);
+		field[0] = 0;
+		field[1] = 0;
+	}
+}
+
+static u32 bnxtctl_read_len_field(void *cmd, const struct bnxtctl_dma_field *f)
+{
+	if (f->len_width == 2)
+		return le16_to_cpup((__le16 *)(cmd + f->len_offset));
+	return le32_to_cpup((__le32 *)(cmd + f->len_offset));
+}
+
+static int bnxtctl_check_dma_lens(void *cmd, const struct bnxtctl_cmd_dma_desc *desc,
+				  u32 *lens)
+{
+	unsigned int i;
+
+	for (i = 0; i < desc->num_fields; i++) {
+		const struct bnxtctl_dma_field *f = &desc->fields[i];
+		u64 len;
+
+		if (f->len_offset) {
+			if (check_mul_overflow(bnxtctl_read_len_field(cmd, f),
+					       f->len_unit, &len))
+				return -EINVAL;
+		} else {
+			len = f->buf_len;
+		}
+
+		if (!len || len > FWCTL_BNXT_MAX_DMABUF)
+			return -EINVAL;
+
+		lens[i] = len;
+	}
+	return 0;
+}
+
+static int bnxtctl_map_dma_bufs(struct device *dev, void *cmd,
+				const struct bnxtctl_cmd_dma_desc *desc,
+				const u64 *user_addrs, const u32 *lens,
+				void **kbufs, dma_addr_t *dma_addrs,
+				unsigned int *num_mapped)
+{
+	unsigned int i;
+
+	*num_mapped = 0;
+	for (i = 0; i < desc->num_fields; i++) {
+		const struct bnxtctl_dma_field *f = &desc->fields[i];
+		__le32 *field;
+
+		kbufs[i] = dma_alloc_coherent(dev, lens[i],
+					      &dma_addrs[i], GFP_KERNEL);
+		if (!kbufs[i])
+			return -ENOMEM;
+
+		if (f->dir == DMA_TO_DEVICE &&
+		    copy_from_user(kbufs[i], u64_to_user_ptr(user_addrs[i]),
+				   lens[i])) {
+			dma_free_coherent(dev, lens[i], kbufs[i],
+					  dma_addrs[i]);
+			kbufs[i] = NULL;
+			return -EFAULT;
+		}
+
+		(*num_mapped)++;
+
+		field = cmd + f->offset;
+		field[0] = cpu_to_le32(lower_32_bits(dma_addrs[i]));
+		field[1] = cpu_to_le32(upper_32_bits(dma_addrs[i]));
+	}
+	return 0;
+}
+
+static int bnxtctl_unmap_dma_bufs(struct device *dev,
+				  const struct bnxtctl_cmd_dma_desc *desc,
+				  const u64 *user_addrs, const u32 *lens,
+				  void **kbufs, dma_addr_t *dma_addrs,
+				  unsigned int num_mapped)
+{
+	unsigned int i;
+	int rc = 0;
+
+	for (i = 0; i < num_mapped; i++) {
+		if (desc->fields[i].dir == DMA_FROM_DEVICE &&
+		    copy_to_user(u64_to_user_ptr(user_addrs[i]),
+				 kbufs[i], lens[i]))
+			rc = -EFAULT;
+
+		dma_free_coherent(dev, lens[i], kbufs[i], dma_addrs[i]);
+	}
+	return rc;
+}
+
 /* Caller must hold edev->en_dev_lock */
 static bool bnxtctl_validate_rpc(struct bnxt_en_dev *edev,
 				 struct bnxt_fw_msg *hwrm_in,
 				 enum fwctl_rpc_scope scope)
 {
 	struct input *req = (struct input *)hwrm_in->msg;
+	u16 req_type = le16_to_cpu(req->req_type);
+	const struct bnxtctl_cmd_dma_desc *desc;
 
 	lockdep_assert_held(&edev->en_dev_lock);
 	if (edev->flags & BNXT_EN_FLAG_ULP_STOPPED)
 		return false;
 
-	switch (le16_to_cpu(req->req_type)) {
+	desc = bnxtctl_find_dma_desc(req_type);
+	if (desc)
+		return scope >= desc->scope_min;
+
+	switch (req_type) {
 	case HWRM_FUNC_RESET:
 	case HWRM_PORT_CLR_STATS:
 	case HWRM_FW_RESET:
@@ -138,6 +470,7 @@ static bool bnxtctl_validate_rpc(struct bnxt_en_dev *edev,
 	case HWRM_NVM_GET_DEV_INFO:
 	case HWRM_NVM_GET_DIR_INFO:
 	case HWRM_SELFTEST_QLIST:
+	case HWRM_DBG_COREDUMP_INITIATE:
 		return scope >= FWCTL_RPC_DEBUG_READ_ONLY;
 
 	case HWRM_PORT_PHY_I2C_WRITE:
@@ -162,6 +495,15 @@ static unsigned int bnxtctl_get_timeout(struct input *req)
 	case HWRM_NVM_VERIFY_UPDATE:
 	case HWRM_NVM_ERASE_DIR_ENTRY:
 	case HWRM_NVM_MOD_DIR_ENTRY:
+	case HWRM_NVM_WRITE:
+	case HWRM_FW_SYNC:
+	case HWRM_DBG_COREDUMP_LIST:
+	case HWRM_DBG_COREDUMP_RETRIEVE:
+	case HWRM_DBG_COREDUMP_INITIATE:
+	case HWRM_SELFTEST_RETRIEVE_SERDES_DATA:
+	case HWRM_DBG_SERDES_TEST:
+	case HWRM_NVM_RAW_WRITE_BLK:
+	case HWRM_FW_HEALTH_CHECK:
 		return BNXTCTL_HWRM_CMD_TIMEOUT_LONG;
 	case HWRM_FUNC_RESET:
 		return BNXTCTL_HWRM_CMD_TIMEOUT_MEDM;
@@ -177,7 +519,15 @@ static void *bnxtctl_fw_rpc(struct fwctl_uctx *uctx,
 	struct bnxtctl_dev *bnxtctl =
 		container_of(uctx->fwctl, struct bnxtctl_dev, fwctl);
 	struct bnxt_en_dev *edev = bnxtctl->aux_priv->edev;
-	struct bnxt_fw_msg rpc_in = {0};
+	dma_addr_t dma_addrs[BNXTCTL_MAX_DMA_FIELDS];
+	void *kbufs[BNXTCTL_MAX_DMA_FIELDS] = {};
+	const struct bnxtctl_cmd_dma_desc *desc;
+	u64 user_addrs[BNXTCTL_MAX_DMA_FIELDS];
+	struct device *dev = &edev->pdev->dev;
+	u32 dma_lens[BNXTCTL_MAX_DMA_FIELDS];
+	struct bnxt_fw_msg rpc_in = {};
+	unsigned int num_mapped = 0;
+	struct input *req = in;
 	int rc;
 
 	if (in_len < sizeof(struct input) || in_len > HWRM_MAX_REQ_LEN)
@@ -186,9 +536,22 @@ static void *bnxtctl_fw_rpc(struct fwctl_uctx *uctx,
 	if (*out_len < sizeof(struct output))
 		return ERR_PTR(-EINVAL);
 
+	desc = bnxtctl_find_dma_desc(le16_to_cpu(req->req_type));
+
+	if (desc) {
+		if (in_len != desc->req_size)
+			return ERR_PTR(-EINVAL);
+
+		rc = bnxtctl_check_dma_lens(in, desc, dma_lens);
+		if (rc)
+			return ERR_PTR(rc);
+
+		bnxtctl_extract_and_zero_dma_fields(in, desc, user_addrs);
+	}
+
 	rpc_in.msg = in;
 	rpc_in.msg_len = in_len;
-	rpc_in.resp = kzalloc(*out_len, GFP_KERNEL);
+	rpc_in.resp = kvzalloc(*out_len, GFP_KERNEL);
 	if (!rpc_in.resp)
 		return ERR_PTR(-ENOMEM);
 
@@ -198,10 +561,21 @@ static void *bnxtctl_fw_rpc(struct fwctl_uctx *uctx,
 	guard(mutex)(&edev->en_dev_lock);
 
 	if (!bnxtctl_validate_rpc(edev, &rpc_in, scope)) {
-		kfree(rpc_in.resp);
+		kvfree(rpc_in.resp);
 		return ERR_PTR(-EPERM);
 	}
 
+	if (desc) {
+		rc = bnxtctl_map_dma_bufs(dev, in, desc, user_addrs, dma_lens,
+					  kbufs, dma_addrs, &num_mapped);
+		if (rc) {
+			bnxtctl_unmap_dma_bufs(dev, desc, user_addrs, dma_lens,
+					       kbufs, dma_addrs, num_mapped);
+			kvfree(rpc_in.resp);
+			return ERR_PTR(rc);
+		}
+	}
+
 	rc = bnxt_send_msg(edev, &rpc_in);
 	if (rc) {
 		struct output *resp = rpc_in.resp;
@@ -216,6 +590,18 @@ static void *bnxtctl_fw_rpc(struct fwctl_uctx *uctx,
 			resp->error_code = cpu_to_le16(rc);
 	}
 
+	if (desc) {
+		int unmap_rc;
+
+		unmap_rc = bnxtctl_unmap_dma_bufs(dev, desc, user_addrs,
+						  dma_lens, kbufs, dma_addrs,
+						  num_mapped);
+		if (unmap_rc) {
+			kvfree(rpc_in.resp);
+			return ERR_PTR(unmap_rc);
+		}
+	}
+
 	return rpc_in.resp;
 }
 
diff --git a/include/linux/bnxt/hsi.h b/include/linux/bnxt/hsi.h
index 74a6bf278d88..2f17dbd02185 100644
--- a/include/linux/bnxt/hsi.h
+++ b/include/linux/bnxt/hsi.h
@@ -191,6 +191,8 @@ struct cmd_nums {
 	#define HWRM_PORT_EVENTS_LOG                      0x67UL
 	#define HWRM_VNIC_RSS_COS_LB_CTX_ALLOC            0x70UL
 	#define HWRM_VNIC_RSS_COS_LB_CTX_FREE             0x71UL
+	#define HWRM_SCH_GRP_CFG                          0x73UL
+	#define HWRM_SCH_GRP_QCFG                         0x74UL
 	#define HWRM_QUEUE_MPLS_QCAPS                     0x80UL
 	#define HWRM_QUEUE_MPLSTC2PRI_QCFG                0x81UL
 	#define HWRM_QUEUE_MPLSTC2PRI_CFG                 0x82UL
@@ -4911,6 +4913,29 @@ struct hwrm_port_phy_qcfg_output {
 	u8	valid;
 };
 
+/* hwrm_port_events_log_input (size:256b/32B) */
+struct hwrm_port_events_log_input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le64	host_dest_addr;
+	__le32	host_dest_addr_len;
+	u8	unused_0[4];
+};
+
+/* hwrm_port_events_log_output (size:128b/16B) */
+struct hwrm_port_events_log_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	data_len;
+	u8	unused_0[5];
+	u8	valid;
+};
+
 /* hwrm_port_mac_cfg_input (size:448b/56B) */
 struct hwrm_port_mac_cfg_input {
 	__le16	req_type;
@@ -5418,6 +5443,40 @@ struct port_stats_ecn {
 	__le64	mark_cnt_cos7;
 };
 
+/* port_stats_ext_pfc_adv (size:1536b/192B) */
+struct port_stats_ext_pfc_adv {
+	__le64	pfc_min_duration_time[8];
+	__le64	pfc_max_duration_time[8];
+	__le64	pfc_weighted_duration_time[8];
+};
+
+/* hwrm_port_qstats_ext_pfc_adv_input (size:320b/40B) */
+struct hwrm_port_qstats_ext_pfc_adv_input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	port_id;
+	__le16	pfc_adv_stat_size;
+	u8	flags;
+	#define PORT_QSTATS_EXT_PFC_ADV_REQ_FLAGS_COUNTER_MASK     0x1UL
+	u8	unused_0[3];
+	__le64	tx_pfc_adv_stat_host_addr;
+	__le64	rx_pfc_adv_stat_host_addr;
+};
+
+/* hwrm_port_qstats_ext_pfc_adv_output (size:128b/16B) */
+struct hwrm_port_qstats_ext_pfc_adv_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	pfc_adv_stat_size;
+	u8	unused_0[5];
+	u8	valid;
+};
+
 /* hwrm_port_clr_stats_input (size:192b/24B) */
 struct hwrm_port_clr_stats_input {
 	__le16	req_type;
@@ -6095,6 +6154,61 @@ struct hwrm_port_led_qcaps_output {
 	u8	valid;
 };
 
+/* hwrm_port_prbs_test_input (size:384b/48B) */
+struct hwrm_port_prbs_test_input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le64	resp_data_addr;
+	__le16	data_len;
+	__le16	flags;
+	#define PORT_PRBS_TEST_REQ_FLAGS_INTERNAL     0x1UL
+	__le32	unused_1;
+	__le16	port_id;
+	__le16	poly;
+	#define PORT_PRBS_TEST_REQ_POLY_PRBS7   0x0UL
+	#define PORT_PRBS_TEST_REQ_POLY_PRBS9   0x1UL
+	#define PORT_PRBS_TEST_REQ_POLY_PRBS11  0x2UL
+	#define PORT_PRBS_TEST_REQ_POLY_PRBS15  0x3UL
+	#define PORT_PRBS_TEST_REQ_POLY_PRBS23  0x4UL
+	#define PORT_PRBS_TEST_REQ_POLY_PRBS31  0x5UL
+	#define PORT_PRBS_TEST_REQ_POLY_PRBS58  0x6UL
+	#define PORT_PRBS_TEST_REQ_POLY_PRBS49  0x7UL
+	#define PORT_PRBS_TEST_REQ_POLY_PRBS10  0x8UL
+	#define PORT_PRBS_TEST_REQ_POLY_PRBS20  0x9UL
+	#define PORT_PRBS_TEST_REQ_POLY_PRBS13  0xaUL
+	#define PORT_PRBS_TEST_REQ_POLY_INVALID 0xffUL
+	#define PORT_PRBS_TEST_REQ_POLY_LAST   PORT_PRBS_TEST_REQ_POLY_INVALID
+	__le16	prbs_config;
+	#define PORT_PRBS_TEST_REQ_PRBS_CONFIG_START_STOP            0x1UL
+	#define PORT_PRBS_TEST_REQ_PRBS_CONFIG_TX_LANE_MAP_VALID     0x2UL
+	#define PORT_PRBS_TEST_REQ_PRBS_CONFIG_RX_LANE_MAP_VALID     0x4UL
+	#define PORT_PRBS_TEST_REQ_PRBS_CONFIG_FEC_STAT_T0_T7        0x8UL
+	#define PORT_PRBS_TEST_REQ_PRBS_CONFIG_FEC_STAT_T8_T15       0x10UL
+	#define PORT_PRBS_TEST_REQ_PRBS_CONFIG_T_CODE                0x20UL
+	__le16	timeout;
+	__le32	tx_lane_map;
+	__le32	rx_lane_map;
+};
+
+/* hwrm_port_prbs_test_output (size:128b/16B) */
+struct hwrm_port_prbs_test_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	total_data_len;
+	u8	ber_format;
+	#define PORT_PRBS_TEST_RESP_BER_FORMAT_PRBS 0x0UL
+	#define PORT_PRBS_TEST_RESP_BER_FORMAT_FEC  0x1UL
+	#define PORT_PRBS_TEST_RESP_BER_FORMAT_LAST PORT_PRBS_TEST_RESP_BER_FORMAT_FEC
+	u8	unused_0;
+	u8	unused_1[3];
+	u8	valid;
+};
+
 /* hwrm_port_phy_fdrstat_input (size:192b/24B) */
 struct hwrm_port_phy_fdrstat_input {
 	__le16	req_type;
@@ -6147,6 +6261,54 @@ struct hwrm_port_phy_fdrstat_cmd_err {
 	u8	unused_0[7];
 };
 
+/* hwrm_port_dsc_dump_input (size:320b/40B) */
+struct hwrm_port_dsc_dump_input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le64	resp_data_addr;
+	__le16	data_len;
+	__le16	unused_0;
+	__le32	data_offset;
+	__le16	port_id;
+	__le16	diag_level;
+	#define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_LANE       0x0UL
+	#define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_CORE       0x1UL
+	#define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_EVENT      0x2UL
+	#define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_EYE        0x3UL
+	#define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_REG_CORE   0x4UL
+	#define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_REG_LANE   0x5UL
+	#define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_UC_CORE    0x6UL
+	#define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_UC_LANE    0x7UL
+	#define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_LANE_DEBUG 0x8UL
+	#define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_BER_VERT   0x9UL
+	#define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_BER_HORZ   0xaUL
+	#define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_EVENT_SAFE 0xbUL
+	#define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_TIMESTAMP  0xcUL
+	#define PORT_DSC_DUMP_REQ_DIAG_LEVEL_LAST                PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_TIMESTAMP
+	__le16	lane_number;
+	__le16	dsc_dump_config;
+	#define PORT_DSC_DUMP_REQ_DSC_DUMP_CONFIG_START_RETRIEVE     0x1UL
+	#define PORT_DSC_DUMP_REQ_DSC_DUMP_CONFIG_BIG_BUFFER         0x2UL
+	#define PORT_DSC_DUMP_REQ_DSC_DUMP_CONFIG_DEFER_CLOSE        0x4UL
+};
+
+/* hwrm_port_dsc_dump_output (size:128b/16B) */
+struct hwrm_port_dsc_dump_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	total_data_len;
+	__le16	total_data_len_high;
+	u8	unused_1[2];
+	u8	flags;
+	#define PORT_DSC_DUMP_RESP_FLAGS_BIG_BUFFER     0x1UL
+	u8	valid;
+};
+
 /* hwrm_port_mac_qcaps_input (size:192b/24B) */
 struct hwrm_port_mac_qcaps_input {
 	__le16	req_type;
@@ -9559,6 +9721,62 @@ struct hwrm_stat_generic_qstats_output {
 	u8	valid;
 };
 
+/* hwrm_stat_query_roce_stats_input (size:256b/32B) */
+struct hwrm_stat_query_roce_stats_input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	roce_stat_size;
+	u8	flags;
+	#define STAT_QUERY_ROCE_STATS_REQ_FLAGS_PORT_AGGREGATED     0x1UL
+	u8	port_id;
+	u8	unused_0[4];
+	__le64	roce_stat_host_addr;
+};
+
+/* hwrm_stat_query_roce_stats_output (size:128b/16B) */
+struct hwrm_stat_query_roce_stats_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	roce_stat_size;
+	u8	flags;
+	#define STAT_QUERY_ROCE_STATS_RESP_FLAGS_PORT_AGGREGATED     0x1UL
+	u8	unused_0[4];
+	u8	valid;
+};
+
+/* hwrm_stat_query_roce_stats_ext_input (size:256b/32B) */
+struct hwrm_stat_query_roce_stats_ext_input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	roce_stat_size;
+	u8	flags;
+	#define STAT_QUERY_ROCE_STATS_EXT_REQ_FLAGS_PORT_AGGREGATED     0x1UL
+	u8	port_id;
+	u8	unused_0[4];
+	__le64	roce_stat_host_addr;
+};
+
+/* hwrm_stat_query_roce_stats_ext_output (size:128b/16B) */
+struct hwrm_stat_query_roce_stats_ext_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	roce_stat_size;
+	u8	flags;
+	#define STAT_QUERY_ROCE_STATS_EXT_RESP_FLAGS_PORT_AGGREGATED     0x1UL
+	u8	unused_0[4];
+	u8	valid;
+};
+
 /* generic_sw_hw_stats (size:1472b/184B) */
 struct generic_sw_hw_stats {
 	__le64	pcie_statistics_tx_tlp;
@@ -10207,6 +10425,57 @@ struct hwrm_dbg_read_direct_output {
 	u8	valid;
 };
 
+/* hwrm_dbg_read_indirect_input (size:640b/80B) */
+struct hwrm_dbg_read_indirect_input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le64	host_dest_addr;
+	__le32	host_dest_addr_len;
+	u8	indirect_access_type;
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_TE_MGMT_FILTERS_L2         0x0UL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_TE_MGMT_FILTERS_L3L4       0x1UL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_RE_MGMT_FILTERS_L2         0x2UL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_RE_MGMT_FILTERS_L3L4       0x3UL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_STAT_CTXS                  0x4UL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CFA_TX_L2_TCAM             0x5UL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CFA_RX_L2_TCAM             0x6UL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CFA_TX_IPV6_SUBNET_TCAM    0x7UL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CFA_RX_IPV6_SUBNET_TCAM    0x8UL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CFA_TX_SRC_PROPERTIES_TCAM 0x9UL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CFA_RX_SRC_PROPERTIES_TCAM 0xaUL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CFA_VEB_LOOKUP_TCAM        0xbUL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CFA_TX_PROFILE_LOOKUP_TCAM 0xcUL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CFA_RX_PROFILE_LOOKUP_TCAM 0xdUL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CFA_TX_LOOKUP_TCAM         0xeUL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CFA_RX_LOOKUP_TCAM         0xfUL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_MHB                        0x10UL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_PCIE_GBL                   0x11UL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_MULTI_HOST_SOC             0x12UL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_PCIE_PRIVATE               0x13UL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_HOST_DMA                   0x14UL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_SOC_ELOG                   0x15UL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CTX                        0x16UL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_STATS                      0x17UL
+	#define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_LAST      DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_STATS
+	u8	unused_0[3];
+	__le32	start_index;
+	__le32	num_of_entries;
+	__le32	opaque[10];
+};
+
+/* hwrm_dbg_read_indirect_output (size:128b/16B) */
+struct hwrm_dbg_read_indirect_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
 /* hwrm_dbg_qcaps_input (size:192b/24B) */
 struct hwrm_dbg_qcaps_input {
 	__le16	req_type;
@@ -10518,6 +10787,154 @@ struct hwrm_dbg_log_buffer_flush_output {
 	u8	valid;
 };
 
+/* hwrm_dbg_serdes_test_input (size:320b/40B) */
+struct hwrm_dbg_serdes_test_input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le64	resp_data_addr;
+	__le32	resp_data_offset;
+	__le16	data_len;
+	u8	flags;
+	#define DBG_SERDES_TEST_REQ_FLAGS_UNUSED_TEST_MASK         0x7UL
+	#define DBG_SERDES_TEST_REQ_FLAGS_UNUSED_TEST_SFT          0
+	#define DBG_SERDES_TEST_REQ_FLAGS_EYE_PROJECTION           0x8UL
+	#define DBG_SERDES_TEST_REQ_FLAGS_PCIE_SERDES_TEST         0x10UL
+	#define DBG_SERDES_TEST_REQ_FLAGS_ETHERNET_SERDES_TEST     0x20UL
+	u8	options;
+	#define DBG_SERDES_TEST_REQ_OPTIONS_LANE_NO_MASK          0xfUL
+	#define DBG_SERDES_TEST_REQ_OPTIONS_LANE_NO_SFT           0
+	#define DBG_SERDES_TEST_REQ_OPTIONS_DIRECTION             0x10UL
+	#define DBG_SERDES_TEST_REQ_OPTIONS_DIRECTION_HORIZONTAL    (0x0UL << 4)
+	#define DBG_SERDES_TEST_REQ_OPTIONS_DIRECTION_VERTICAL      (0x1UL << 4)
+	#define DBG_SERDES_TEST_REQ_OPTIONS_DIRECTION_LAST         DBG_SERDES_TEST_REQ_OPTIONS_DIRECTION_VERTICAL
+	#define DBG_SERDES_TEST_REQ_OPTIONS_PROJ_TYPE             0x20UL
+	#define DBG_SERDES_TEST_REQ_OPTIONS_PROJ_TYPE_LEFT_TOP      (0x0UL << 5)
+	#define DBG_SERDES_TEST_REQ_OPTIONS_PROJ_TYPE_RIGHT_BOTTOM  (0x1UL << 5)
+	#define DBG_SERDES_TEST_REQ_OPTIONS_PROJ_TYPE_LAST         DBG_SERDES_TEST_REQ_OPTIONS_PROJ_TYPE_RIGHT_BOTTOM
+	#define DBG_SERDES_TEST_REQ_OPTIONS_RSVD_MASK             0xc0UL
+	#define DBG_SERDES_TEST_REQ_OPTIONS_RSVD_SFT              6
+	u8	targetBER;
+	#define DBG_SERDES_TEST_REQ_TARGETBER_BER_1E8  0x0UL
+	#define DBG_SERDES_TEST_REQ_TARGETBER_BER_1E9  0x1UL
+	#define DBG_SERDES_TEST_REQ_TARGETBER_BER_1E10 0x2UL
+	#define DBG_SERDES_TEST_REQ_TARGETBER_BER_1E11 0x3UL
+	#define DBG_SERDES_TEST_REQ_TARGETBER_BER_1E12 0x4UL
+	#define DBG_SERDES_TEST_REQ_TARGETBER_LAST    DBG_SERDES_TEST_REQ_TARGETBER_BER_1E12
+	u8	action;
+	#define DBG_SERDES_TEST_REQ_ACTION_SYNCHRONOUS 0x0UL
+	#define DBG_SERDES_TEST_REQ_ACTION_START       0x1UL
+	#define DBG_SERDES_TEST_REQ_ACTION_PROGRESS    0x2UL
+	#define DBG_SERDES_TEST_REQ_ACTION_STOP        0x3UL
+	#define DBG_SERDES_TEST_REQ_ACTION_LAST       DBG_SERDES_TEST_REQ_ACTION_STOP
+	u8	unused[6];
+};
+
+/* hwrm_dbg_serdes_test_output (size:192b/24B) */
+struct hwrm_dbg_serdes_test_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	total_data_len;
+	__le16	copied_data_len;
+	__le16	progress_percent;
+	__le16	timeout;
+	u8	flags;
+	#define DBG_SERDES_TEST_RESP_FLAGS_BIT_COUNT_TYPE                0x1UL
+	#define DBG_SERDES_TEST_RESP_FLAGS_BIT_COUNT_TYPE_BIT_COUNT_TOTAL  (0x0UL << 0)
+	#define DBG_SERDES_TEST_RESP_FLAGS_BIT_COUNT_TYPE_BIT_COUNT_POW2   (0x1UL << 0)
+	#define DBG_SERDES_TEST_RESP_FLAGS_BIT_COUNT_TYPE_LAST            DBG_SERDES_TEST_RESP_FLAGS_BIT_COUNT_TYPE_BIT_COUNT_POW2
+	#define DBG_SERDES_TEST_RESP_FLAGS_RSVD_MASK                     0xfeUL
+	#define DBG_SERDES_TEST_RESP_FLAGS_RSVD_SFT                      1
+	u8	unused_0;
+	__le16	hdr_size;
+	u8	unused_1[3];
+	u8	valid;
+};
+
+/* hwrm_dbg_ptrace_input (size:320b/40B) */
+struct hwrm_dbg_ptrace_input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le32	pdi_cmd_buf_addr[2];
+	__le32	pdi_resp_buf_addr[2];
+	__le32	pdi_req_buf_len;
+	__le16	seq_no;
+	__le16	flags;
+	#define DBG_PTRACE_REQ_FLAGS_SELECT_IN        0x1UL
+	#define DBG_PTRACE_REQ_FLAGS_SELECT_OUT       0x2UL
+	#define DBG_PTRACE_REQ_FLAGS_GLOBAL_START     0x4UL
+	#define DBG_PTRACE_REQ_FLAGS_GLOBAL_STOP      0x8UL
+};
+
+/* hwrm_dbg_ptrace_output (size:128b/16B) */
+struct hwrm_dbg_ptrace_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	flags;
+	#define DBG_PTRACE_RESP_FLAGS_MORE     0x1UL
+	__le16	data_len;
+	u8	unused_0[3];
+	u8	valid;
+};
+
+/* hwrm_dbg_token_cfg_input (size:256b/32B) */
+struct hwrm_dbg_token_cfg_input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	u8	flags;
+	#define DBG_TOKEN_CFG_REQ_FLAGS_ENABLE     0x1UL
+	u8	unused_0[3];
+	__le32	dbg_token_len;
+	__le64	host_src_addr;
+};
+
+/* hwrm_dbg_token_cfg_output (size:128b/16B) */
+struct hwrm_dbg_token_cfg_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_nvm_raw_write_blk_input (size:320b/40B) */
+struct hwrm_nvm_raw_write_blk_input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le64	host_src_addr;
+	__le32	dest_addr;
+	__le32	len;
+	u8	flags;
+	#define NVM_RAW_WRITE_BLK_REQ_FLAGS_SECURITY_SOC_NVM     0x1UL
+	u8	unused_0[7];
+};
+
+/* hwrm_nvm_raw_write_blk_output (size:128b/16B) */
+struct hwrm_nvm_raw_write_blk_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
 /* hwrm_nvm_read_input (size:320b/40B) */
 struct hwrm_nvm_read_input {
 	__le16	req_type;
@@ -10543,6 +10960,31 @@ struct hwrm_nvm_read_output {
 	u8	valid;
 };
 
+/* hwrm_nvm_raw_dump_input (size:320b/40B) */
+struct hwrm_nvm_raw_dump_input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le64	host_dest_addr;
+	__le32	offset;
+	__le32	len;
+	u8	flags;
+	#define NVM_RAW_DUMP_REQ_FLAGS_SECURITY_SOC_NVM     0x1UL
+	u8	unused_0[7];
+};
+
+/* hwrm_nvm_raw_dump_output (size:128b/16B) */
+struct hwrm_nvm_raw_dump_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
 /* hwrm_nvm_get_dir_entries_input (size:192b/24B) */
 struct hwrm_nvm_get_dir_entries_input {
 	__le16	req_type;
@@ -11166,6 +11608,149 @@ struct hwrm_selftest_irq_output {
 	u8	valid;
 };
 
+/* hwrm_selftest_retrieve_serdes_data_input (size:320b/40B) */
+struct hwrm_selftest_retrieve_serdes_data_input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le64	resp_data_addr;
+	__le32	resp_data_offset;
+	__le16	data_len;
+	u8	flags;
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_FLAGS_UNUSED_TEST_MASK         0x7UL
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_FLAGS_UNUSED_TEST_SFT          0
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_FLAGS_EYE_PROJECTION           0x8UL
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_FLAGS_PCIE_SERDES_TEST         0x10UL
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_FLAGS_ETHERNET_SERDES_TEST     0x20UL
+	u8	options;
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_PCIE_LANE_NO_MASK     0xfUL
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_PCIE_LANE_NO_SFT      0
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_DIRECTION             0x10UL
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_DIRECTION_HORIZONTAL    (0x0UL << 4)
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_DIRECTION_VERTICAL      (0x1UL << 4)
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_DIRECTION_LAST         SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_DIRECTION_VERTICAL
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_PROJ_TYPE             0x20UL
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_PROJ_TYPE_LEFT_TOP      (0x0UL << 5)
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_PROJ_TYPE_RIGHT_BOTTOM  (0x1UL << 5)
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_PROJ_TYPE_LAST         SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_PROJ_TYPE_RIGHT_BOTTOM
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_RSVD_MASK             0xc0UL
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_RSVD_SFT              6
+	u8	targetBER;
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_TARGETBER_BER_1E8  0x0UL
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_TARGETBER_BER_1E9  0x1UL
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_TARGETBER_BER_1E10 0x2UL
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_TARGETBER_BER_1E11 0x3UL
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_TARGETBER_BER_1E12 0x4UL
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_TARGETBER_LAST    SELFTEST_RETRIEVE_SERDES_DATA_REQ_TARGETBER_BER_1E12
+	u8	action;
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_ACTION_SYNCHRONOUS 0x0UL
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_ACTION_START       0x1UL
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_ACTION_PROGRESS    0x2UL
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_ACTION_STOP        0x3UL
+	#define SELFTEST_RETRIEVE_SERDES_DATA_REQ_ACTION_LAST       SELFTEST_RETRIEVE_SERDES_DATA_REQ_ACTION_STOP
+	u8	unused[6];
+};
+
+/* hwrm_selftest_retrieve_serdes_data_output (size:192b/24B) */
+struct hwrm_selftest_retrieve_serdes_data_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le16	total_data_len;
+	__le16	copied_data_len;
+	__le16	progress_percent;
+	__le16	timeout;
+	u8	flags;
+	#define SELFTEST_RETRIEVE_SERDES_DATA_RESP_FLAGS_BIT_COUNT_TYPE                0x1UL
+	#define SELFTEST_RETRIEVE_SERDES_DATA_RESP_FLAGS_BIT_COUNT_TYPE_BIT_COUNT_TOTAL  (0x0UL << 0)
+	#define SELFTEST_RETRIEVE_SERDES_DATA_RESP_FLAGS_BIT_COUNT_TYPE_BIT_COUNT_POW2   (0x1UL << 0)
+	#define SELFTEST_RETRIEVE_SERDES_DATA_RESP_FLAGS_BIT_COUNT_TYPE_LAST            SELFTEST_RETRIEVE_SERDES_DATA_RESP_FLAGS_BIT_COUNT_TYPE_BIT_COUNT_POW2
+	#define SELFTEST_RETRIEVE_SERDES_DATA_RESP_FLAGS_RSVD_MASK                     0xfeUL
+	#define SELFTEST_RETRIEVE_SERDES_DATA_RESP_FLAGS_RSVD_SFT                      1
+	u8	unused_0;
+	__le16	hdr_size;
+	u8	unused_1[3];
+	u8	valid;
+};
+
+/* hwrm_sch_grp_cfg_input (size:704b/88B) */
+struct hwrm_sch_grp_cfg_input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	sch_grp_id;
+	__le16	num_fids;
+	__le32	enables;
+	#define SCH_GRP_CFG_REQ_ENABLES_QUEUE_ID0_VALID     0x1UL
+	#define SCH_GRP_CFG_REQ_ENABLES_QUEUE_ID1_VALID     0x2UL
+	#define SCH_GRP_CFG_REQ_ENABLES_QUEUE_ID2_VALID     0x4UL
+	#define SCH_GRP_CFG_REQ_ENABLES_QUEUEID3_VALID      0x8UL
+	#define SCH_GRP_CFG_REQ_ENABLES_QUEUE_ID4_VALID     0x10UL
+	#define SCH_GRP_CFG_REQ_ENABLES_QUEUE_ID5_VALID     0x20UL
+	#define SCH_GRP_CFG_REQ_ENABLES_QUEUE_ID6_VALID     0x40UL
+	#define SCH_GRP_CFG_REQ_ENABLES_QUEUE_ID7_VALID     0x80UL
+	#define SCH_GRP_CFG_REQ_ENABLES_MAX_BW              0x100UL
+	#define SCH_GRP_CFG_REQ_ENABLES_FID_MAP             0x200UL
+	__le64	fid_table_addr;
+	__le32	max_bw;
+	u8	unused_0[4];
+	u8	queue_id[8];
+	u8	queue_tsa_assign[8];
+	#define SCH_GRP_CFG_REQ_QUEUE_TSA_ASSIGN_SP   0x0UL
+	#define SCH_GRP_CFG_REQ_QUEUE_TSA_ASSIGN_ETS  0x1UL
+	#define SCH_GRP_CFG_REQ_QUEUE_TSA_ASSIGN_LAST SCH_GRP_CFG_REQ_QUEUE_TSA_ASSIGN_ETS
+	__le32	queue_min_bw_percent[8];
+};
+
+/* hwrm_sch_grp_cfg_output (size:128b/16B) */
+struct hwrm_sch_grp_cfg_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	u8	unused_0[7];
+	u8	valid;
+};
+
+/* hwrm_sch_grp_qcfg_input (size:256b/32B) */
+struct hwrm_sch_grp_qcfg_input {
+	__le16	req_type;
+	__le16	cmpl_ring;
+	__le16	seq_id;
+	__le16	target_id;
+	__le64	resp_addr;
+	__le16	sch_grp_id;
+	__le16	fid_table_len;
+	u8	unused_0[4];
+	__le64	fid_table_addr;
+};
+
+/* hwrm_sch_grp_qcfg_output (size:576b/72B) */
+struct hwrm_sch_grp_qcfg_output {
+	__le16	error_code;
+	__le16	req_type;
+	__le16	seq_id;
+	__le16	resp_len;
+	__le32	max_bw;
+	u8	unused_0[2];
+	__le16	num_fids;
+	u8	queue_id[8];
+	#define SCH_GRP_QCFG_RESP_QUEUE_ID_INVALID_QUEUE_ID 0xffUL
+	#define SCH_GRP_QCFG_RESP_QUEUE_ID_LAST            SCH_GRP_QCFG_RESP_QUEUE_ID_INVALID_QUEUE_ID
+	u8	queue_tsa_assign[8];
+	#define SCH_GRP_QCFG_RESP_QUEUE_TSA_ASSIGN_SP   0x0UL
+	#define SCH_GRP_QCFG_RESP_QUEUE_TSA_ASSIGN_ETS  0x1UL
+	#define SCH_GRP_QCFG_RESP_QUEUE_TSA_ASSIGN_LAST SCH_GRP_QCFG_RESP_QUEUE_TSA_ASSIGN_ETS
+	__le32	queue_min_bw_percent[8];
+	u8	unused_1[7];
+	u8	valid;
+};
+
 /* dbc_dbc (size:64b/8B) */
 struct dbc_dbc {
 	__le32	index;
diff --git a/include/uapi/fwctl/bnxt.h b/include/uapi/fwctl/bnxt.h
index 32e0bfb9a836..398c1ad0cc0a 100644
--- a/include/uapi/fwctl/bnxt.h
+++ b/include/uapi/fwctl/bnxt.h
@@ -12,6 +12,7 @@ enum fwctl_bnxt_commands {
 	FWCTL_BNXT_INLINE_COMMANDS = 0,
 	FWCTL_BNXT_QUERY_COMMANDS,
 	FWCTL_BNXT_SEND_COMMANDS,
+	FWCTL_BNXT_DMA_COMMANDS,
 };
 
 /**
@@ -23,4 +24,7 @@ enum fwctl_bnxt_commands {
 struct fwctl_info_bnxt {
 	__u32 uctx_caps;
 };
+
+#define FWCTL_BNXT_MAX_DMABUF	0x10000   /* 64 KiB */
+
 #endif
diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h
index fe7c505da236..82d2faf839f2 100644
--- a/rust/bindings/bindings_helper.h
+++ b/rust/bindings/bindings_helper.h
@@ -60,6 +60,7 @@
 #include <linux/fdtable.h>
 #include <linux/file.h>
 #include <linux/firmware.h>
+#include <linux/fwctl.h>
 #include <linux/fs.h>
 #include <linux/i2c.h>
 #include <linux/interrupt.h>
diff --git a/rust/helpers/fwctl.c b/rust/helpers/fwctl.c
new file mode 100644
index 000000000000..c7eecd4336a7
--- /dev/null
+++ b/rust/helpers/fwctl.c
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/fwctl.h>
+
+#if IS_ENABLED(CONFIG_RUST_FWCTL_ABSTRACTIONS)
+
+__rust_helper struct fwctl_device *rust_helper_fwctl_get(struct fwctl_device *fwctl)
+{
+	return fwctl_get(fwctl);
+}
+
+__rust_helper void rust_helper_fwctl_put(struct fwctl_device *fwctl)
+{
+	fwctl_put(fwctl);
+}
+
+#endif
diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c
index 198c5f807eba..e05cde02c95d 100644
--- a/rust/helpers/helpers.c
+++ b/rust/helpers/helpers.c
@@ -63,11 +63,12 @@
 #include "drm.c"
 #include "drm_gpuvm.c"
 #include "err.c"
-#include "irq.c"
 #include "fs.c"
+#include "fwctl.c"
 #include "gpu.c"
 #include "interrupt.c"
 #include "io.c"
+#include "irq.c"
 #include "jump_label.c"
 #include "kunit.c"
 #include "list.c"
diff --git a/rust/kernel/fwctl.rs b/rust/kernel/fwctl.rs
new file mode 100644
index 000000000000..f29244fb0d1d
--- /dev/null
+++ b/rust/kernel/fwctl.rs
@@ -0,0 +1,593 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+//! Abstractions for the fwctl subsystem.
+//!
+//! C header: `include/linux/fwctl.h`
+
+use crate::{
+    bindings,
+    container_of,
+    device,
+    prelude::*,
+    sync::aref::{
+        ARef,
+        AlwaysRefCounted, //
+    },
+    types::Opaque, //
+};
+use core::{
+    alloc::Layout,
+    cell::UnsafeCell,
+    marker::PhantomData,
+    ptr::NonNull,
+    slice, //
+};
+
+/// Returns a kmalloc-compatible allocation size for `T`.
+const fn kmalloc_aligned_size<T>() -> usize {
+    Layout::new::<T>().pad_to_align().size()
+}
+
+/// Represents a fwctl device type.
+///
+/// Corresponds to the C `enum fwctl_device_type`. All non-error UAPI values are represented so
+/// Rust drivers can select a device type without passing an untyped integer, while
+/// `FWCTL_DEVICE_TYPE_ERROR` remains unrepresentable.
+#[repr(u32)]
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
+pub enum DeviceType {
+    /// Mellanox ConnectX (mlx5) device.
+    Mlx5 = bindings::fwctl_device_type_FWCTL_DEVICE_TYPE_MLX5,
+    /// CXL (Compute Express Link) device.
+    Cxl = bindings::fwctl_device_type_FWCTL_DEVICE_TYPE_CXL,
+    /// AMD/Pensando PDS device.
+    Pds = bindings::fwctl_device_type_FWCTL_DEVICE_TYPE_PDS,
+    /// Broadcom NetXtreme (bnxt) device.
+    Bnxt = bindings::fwctl_device_type_FWCTL_DEVICE_TYPE_BNXT,
+}
+
+/// Scope of access for an RPC request.
+///
+/// Corresponds to the C `enum fwctl_rpc_scope`.
+#[repr(u32)]
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
+pub enum RpcScope {
+    /// Read/write access to device configuration.
+    Configuration = bindings::fwctl_rpc_scope_FWCTL_RPC_CONFIGURATION,
+    /// Read-only access to debug information.
+    DebugReadOnly = bindings::fwctl_rpc_scope_FWCTL_RPC_DEBUG_READ_ONLY,
+    /// Write access to lockdown-compatible debug information.
+    DebugWrite = bindings::fwctl_rpc_scope_FWCTL_RPC_DEBUG_WRITE,
+    /// Full read/write access to all debug information (requires `CAP_SYS_RAWIO`).
+    DebugWriteFull = bindings::fwctl_rpc_scope_FWCTL_RPC_DEBUG_WRITE_FULL,
+}
+
+impl TryFrom<u32> for RpcScope {
+    type Error = Error;
+
+    #[inline]
+    fn try_from(value: u32) -> Result<Self, Error> {
+        match value {
+            v if v == Self::Configuration as u32 => Ok(Self::Configuration),
+            v if v == Self::DebugReadOnly as u32 => Ok(Self::DebugReadOnly),
+            v if v == Self::DebugWrite as u32 => Ok(Self::DebugWrite),
+            v if v == Self::DebugWriteFull as u32 => Ok(Self::DebugWriteFull),
+            _ => Err(EINVAL),
+        }
+    }
+}
+
+/// Response from a [`Operations::fw_rpc`] call.
+pub enum FwRpcResponse {
+    /// Reuse the input buffer as the output, with the given output length.
+    ///
+    /// The callback returns `EINVAL` if the output length exceeds the input buffer length.
+    InPlace(usize),
+    /// Return a newly allocated buffer as the output.
+    NewBuffer(KVVec<u8>),
+}
+
+/// Trait implemented by each Rust driver that integrates with the fwctl subsystem.
+///
+/// The implementing type **is** the per-FD user context: one instance is
+/// created for each `open()` call and dropped when the FD is closed.
+///
+/// Each implementation corresponds to a specific device type and provides the
+/// vtable used by the core `fwctl` layer to manage per-FD user contexts and
+/// handle RPC requests.
+pub trait Operations: Sized + Send + Sync + 'static {
+    /// Data owned by the [`Registration`] and accessible during callbacks.
+    ///
+    /// The lifetime `'a` is tied to the [`Registration`] scope (which lives within the parent bus
+    /// device binding scope). Drivers use it to store references to resources bound to this scope,
+    /// such as PCI BARs or typed bus device references.
+    type RegistrationData<'a>: Send + Sync + 'a
+    where
+        Self: 'a;
+
+    /// fwctl device type identifier.
+    const DEVICE_TYPE: DeviceType;
+
+    /// Called when a new user context is opened.
+    ///
+    /// Returns a [`PinInit`] initializer for `Self`. The instance is dropped
+    /// automatically when the FD is closed (after [`close`](Self::close)).
+    fn open<'a>(
+        device: &Device<Self>,
+        reg_data: &Self::RegistrationData<'a>,
+    ) -> impl PinInit<Self, Error>;
+
+    /// Called when the user context is closed.
+    ///
+    /// The driver may perform additional cleanup here that requires access
+    /// to the owning [`Device`]. `Self` is dropped automatically after this
+    /// returns.
+    fn close<'a>(
+        _this: Pin<&mut Self>,
+        _device: &Device<Self>,
+        _reg_data: &Self::RegistrationData<'a>,
+    ) {
+    }
+
+    /// Return device information to userspace.
+    ///
+    /// The default implementation returns no device-specific data.
+    fn info<'a>(
+        _this: Pin<&Self>,
+        _device: &Device<Self>,
+        _reg_data: &Self::RegistrationData<'a>,
+    ) -> Result<KVec<u8>, Error> {
+        Ok(KVec::new())
+    }
+
+    /// Handle a userspace RPC request.
+    ///
+    /// `max_output_len` is the size of the userspace output buffer. A driver may return a larger
+    /// response to report the required size; the fwctl core copies only the bytes that fit and
+    /// reports the full response length to userspace.
+    fn fw_rpc<'a>(
+        this: Pin<&Self>,
+        device: &Device<Self>,
+        reg_data: &Self::RegistrationData<'a>,
+        scope: RpcScope,
+        rpc_buf: &mut [u8],
+        max_output_len: usize,
+    ) -> Result<FwRpcResponse, Error>;
+}
+
+/// A fwctl device.
+///
+/// `#[repr(C)]` with the `fwctl_device` at offset 0, matching the C `fwctl_alloc_device()` layout
+/// convention. Contains a pointer to the [`Registration`]'s data, set at registration time and
+/// cleared on unregistration.
+///
+/// # Invariants
+///
+/// - `dev` is embedded at offset 0 and is initialised by fwctl.
+/// - The fwctl refcount owns the allocation lifetime.
+/// - `registration_data` is either [`NonNull::dangling()`] (before registration / after
+///   unregistration) or points to valid data owned by the [`Registration`].
+#[repr(C)]
+pub struct Device<T: Operations> {
+    dev: Opaque<bindings::fwctl_device>,
+    registration_data: UnsafeCell<NonNull<T::RegistrationData<'static>>>,
+}
+
+impl<T: Operations> Device<T> {
+    /// Allocate a new fwctl device.
+    ///
+    /// Returns an [`ARef`] that can be passed to [`Registration::new()`]
+    /// to make the device visible to userspace.
+    pub fn new(parent: &device::Device<device::Bound>) -> Result<ARef<Self>> {
+        const_assert!(
+            core::mem::offset_of!(Self, dev) == 0,
+            "struct fwctl_device must be at offset 0"
+        );
+
+        let size = kmalloc_aligned_size::<Self>();
+        let ops = core::ptr::from_ref::<bindings::fwctl_ops>(&VTable::<T>::VTABLE).cast_mut();
+
+        // SAFETY: `ops` is static, `parent` is bound, and `size` is padded so the allocation made
+        // by `_fwctl_alloc_device` satisfies the size and alignment required by `Device<T>`.
+        let raw = unsafe { bindings::_fwctl_alloc_device(parent.as_raw(), ops, size) };
+        let this = NonNull::new(raw.cast::<Self>()).ok_or(ENOMEM)?;
+
+        // INVARIANT: Set `registration_data` to dangling (no registration yet).
+        // SAFETY: `this` points to the allocation just returned by fwctl.
+        unsafe {
+            (&raw mut (*this.as_ptr()).registration_data)
+                .write(UnsafeCell::new(NonNull::dangling()));
+        };
+
+        // SAFETY: `this` owns the initial reference.
+        Ok(unsafe { ARef::from_raw(this) })
+    }
+
+    /// Returns the underlying `fwctl_device` pointer.
+    #[inline]
+    fn as_raw(&self) -> *mut bindings::fwctl_device {
+        self.dev.get()
+    }
+
+    /// Borrows a Rust fwctl device from its raw C pointer.
+    ///
+    /// # Safety
+    ///
+    /// `ptr` must point to a valid `fwctl_device` embedded in a [`Device<T>`].
+    #[inline]
+    unsafe fn from_raw<'a>(ptr: *mut bindings::fwctl_device) -> &'a Self {
+        // SAFETY: The caller upholds the offset-0 `Device<T>` invariant.
+        unsafe { &*ptr.cast() }
+    }
+
+    /// Invokes `f` with the registration data.
+    ///
+    /// The higher-ranked callback prevents the erased registration lifetime from escaping and
+    /// permits registration data that is invariant over its lifetime parameter.
+    ///
+    /// # Safety
+    ///
+    /// The caller must ensure that the device is registered and that this is called from a fwctl
+    /// callback protected by `registration_lock`.
+    #[inline]
+    unsafe fn with_registration_data<R>(
+        &self,
+        f: impl for<'a> FnOnce(&Device<T>, &'a T::RegistrationData<'a>) -> R,
+    ) -> R {
+        // SAFETY: Caller guarantees the device is registered, so the pointer is valid.
+        // Lifetimes do not affect layout. The higher-ranked callback prevents the shortened
+        // lifetime from escaping or being selected by the caller.
+        let reg_data = unsafe {
+            (*self.registration_data.get())
+                .cast::<T::RegistrationData<'_>>()
+                .as_ref()
+        };
+
+        f(self, reg_data)
+    }
+}
+
+impl<T: Operations> AsRef<device::Device> for Device<T> {
+    #[inline]
+    fn as_ref(&self) -> &device::Device {
+        // SAFETY: `self` contains a live fwctl_device.
+        let dev = unsafe { &raw mut (*self.as_raw()).dev };
+        // SAFETY: The embedded device is initialised by fwctl.
+        unsafe { device::Device::from_raw(dev) }
+    }
+}
+
+// SAFETY: `fwctl_get` increments the refcount of a valid fwctl_device.
+// `fwctl_put` decrements it and frees the device when it reaches zero.
+unsafe impl<T: Operations> AlwaysRefCounted for Device<T> {
+    #[inline]
+    fn inc_ref(&self) {
+        // SAFETY: `self` holds a live reference.
+        unsafe { bindings::fwctl_get(self.as_raw()) };
+    }
+
+    #[inline]
+    unsafe fn dec_ref(obj: NonNull<Self>) {
+        // SAFETY: The caller owns a live reference.
+        unsafe { bindings::fwctl_put(obj.cast().as_ptr()) };
+    }
+}
+
+// SAFETY: `Device<T>` is refcounted by the fwctl core and may be released from any thread.
+unsafe impl<T: Operations> Send for Device<T> {}
+
+// SAFETY: Shared access to the embedded `fwctl_device` is protected by the fwctl core. The
+// `registration_data` field is only mutated before registration and after unregistration (both
+// single-threaded with respect to callbacks).
+unsafe impl<T: Operations> Sync for Device<T> {}
+
+/// A registered fwctl device.
+///
+/// Owns the [`RegistrationData`](Operations::RegistrationData) made available to driver callbacks.
+/// The parent device lifetime ensures that [`fwctl_unregister`] runs before the parent driver
+/// unbinds.
+///
+/// On drop the device is unregistered (all user contexts are closed and `ops` is set to `NULL`)
+/// and the registration data is dropped.
+///
+/// [`fwctl_unregister`]: srctree/drivers/fwctl/main.c
+pub struct Registration<'a, T: Operations> {
+    dev: ARef<Device<T>>,
+    _reg_data: Pin<KBox<T::RegistrationData<'a>>>,
+}
+
+impl<'a, T: Operations> Registration<'a, T> {
+    /// Register a previously allocated fwctl device with the given registration data.
+    ///
+    /// The `reg_data` is owned by the registration and accessible during callbacks.
+    ///
+    /// # Safety
+    ///
+    /// Callers must not `mem::forget()` the returned [`Registration`] or otherwise prevent its
+    /// [`Drop`] implementation from running, since `fwctl_unregister` must be called before the
+    /// parent device is unbound.
+    ///
+    /// `dev` must be an unregistered [`Device`] that is not associated with any live
+    /// [`Registration`], and no other thread may attempt to register the same device concurrently.
+    pub unsafe fn new(
+        parent: &'a device::Device<device::Bound>,
+        dev: &Device<T>,
+        reg_data: impl PinInit<T::RegistrationData<'a>, Error>,
+    ) -> Result<Self> {
+        let actual_parent = dev.as_ref().parent().ok_or(EINVAL)?;
+        let parent_device: &device::Device = parent;
+        if !core::ptr::eq(actual_parent, parent_device) {
+            return Err(EINVAL);
+        }
+
+        let reg_data: Pin<KBox<T::RegistrationData<'a>>> = KBox::pin_init(reg_data, GFP_KERNEL)?;
+
+        // Store the registration data pointer in the device before registration, so that it is
+        // visible once callbacks can be invoked. The `'static` type is only an erased storage
+        // handle; callbacks access the pointer through a higher-ranked closure.
+        let ptr: NonNull<T::RegistrationData<'static>> =
+            NonNull::from(Pin::get_ref(reg_data.as_ref())).cast();
+
+        // SAFETY: No concurrent access; the device is not yet registered.
+        unsafe { *dev.registration_data.get() = ptr };
+
+        // SAFETY: `dev` is a valid fwctl_device backed by an ARef.
+        let ret = unsafe { bindings::fwctl_register(dev.as_raw()) };
+        if ret != 0 {
+            // SAFETY: No concurrent readers; registration failed.
+            unsafe { *dev.registration_data.get() = NonNull::dangling() };
+            return Err(Error::from_errno(ret));
+        }
+
+        Ok(Self {
+            dev: dev.into(),
+            _reg_data: reg_data,
+        })
+    }
+}
+
+impl<T: Operations> Drop for Registration<'_, T> {
+    fn drop(&mut self) {
+        // SAFETY: The Registration lifetime guarantees that the parent device is still bound.
+        // `fwctl_unregister` takes the write lock, closes all user contexts, and sets ops=NULL.
+        // After it returns, no callbacks can be running or will run.
+        unsafe { bindings::fwctl_unregister(self.dev.as_raw()) };
+
+        // SAFETY: `fwctl_unregister` guarantees no concurrent readers.
+        unsafe { *self.dev.registration_data.get() = NonNull::dangling() };
+
+        // `self._reg_data` is dropped here, after callbacks have stopped.
+    }
+}
+
+/// Internal per-FD user context wrapping `struct fwctl_uctx` and `T`.
+///
+/// Not exposed to drivers; they work with `&T` / `Pin<&mut T>` directly.
+#[repr(C)]
+#[pin_data]
+struct UserCtx<T: Operations> {
+    #[pin]
+    fwctl_uctx: Opaque<bindings::fwctl_uctx>,
+    #[pin]
+    uctx: T,
+}
+
+impl<T: Operations> UserCtx<T> {
+    /// Borrows a pinned Rust user context from its raw C pointer.
+    ///
+    /// # Safety
+    ///
+    /// `ptr` must point to a `fwctl_uctx` embedded in a live, pinned `UserCtx<T>` that remains
+    /// valid and does not move for the duration of `'a`.
+    #[inline]
+    unsafe fn from_raw<'a>(ptr: *mut bindings::fwctl_uctx) -> Pin<&'a Self> {
+        // SAFETY: The caller upholds the `UserCtx<T>` embedding, lifetime, and pinning invariants.
+        unsafe { Pin::new_unchecked(&*container_of!(Opaque::cast_from(ptr), Self, fwctl_uctx)) }
+    }
+
+    /// Mutably borrows a pinned Rust user context from its raw C pointer.
+    ///
+    /// # Safety
+    ///
+    /// - `ptr` must point to a `fwctl_uctx` embedded in a live, pinned `UserCtx<T>` that remains
+    ///   valid and does not move for the duration of `'a`.
+    /// - The caller must ensure exclusive access to the `UserCtx<T>` for the duration of `'a`.
+    #[inline]
+    unsafe fn from_raw_mut<'a>(ptr: *mut bindings::fwctl_uctx) -> Pin<&'a mut Self> {
+        // SAFETY: The caller upholds the embedding, lifetime, pinning, and exclusivity invariants.
+        unsafe {
+            Pin::new_unchecked(
+                &mut *container_of!(Opaque::cast_from(ptr), Self, fwctl_uctx).cast_mut(),
+            )
+        }
+    }
+
+    /// Returns a reference to the fwctl [`Device`] that owns this context.
+    #[inline]
+    fn device(self: Pin<&Self>) -> &Device<T> {
+        // SAFETY: fwctl initialises this pointer before any driver callback.
+        let raw_fwctl = unsafe { (*self.fwctl_uctx.get()).fwctl };
+        // SAFETY: Rust fwctl devices use the offset-0 `Device<T>` layout.
+        unsafe { Device::from_raw(raw_fwctl) }
+    }
+
+    /// Returns a pinned reference to the driver context.
+    #[inline]
+    fn uctx(self: Pin<&Self>) -> Pin<&T> {
+        ::pin_init::assert_pinned!(UserCtx<T>, uctx, T, inline);
+
+        // SAFETY: `uctx` is structurally pinned.
+        unsafe { self.map_unchecked(|ctx| &ctx.uctx) }
+    }
+}
+
+/// Static vtable mapping Rust trait methods to C callbacks.
+struct VTable<T: Operations>(PhantomData<T>);
+
+impl<T: Operations> VTable<T> {
+    /// The fwctl operations vtable for this driver type.
+    const VTABLE: bindings::fwctl_ops = bindings::fwctl_ops {
+        // CAST: `DeviceType` has the same `u32` representation as the C enum field.
+        device_type: T::DEVICE_TYPE as u32,
+        uctx_size: kmalloc_aligned_size::<UserCtx<T>>(),
+        open_uctx: Some(Self::open_uctx_callback),
+        close_uctx: Some(Self::close_uctx_callback),
+        info: Some(Self::info_callback),
+        fw_rpc: Some(Self::fw_rpc_callback),
+    };
+
+    /// Initialises a newly opened Rust user context.
+    ///
+    /// # Safety
+    ///
+    /// `uctx` must be a valid `fwctl_uctx` embedded in a `UserCtx<T>` with
+    /// sufficient allocated space for the uctx field.
+    unsafe extern "C" fn open_uctx_callback(uctx: *mut bindings::fwctl_uctx) -> ffi::c_int {
+        const_assert!(
+            core::mem::offset_of!(UserCtx<T>, fwctl_uctx) == 0,
+            "struct fwctl_uctx must be at offset 0"
+        );
+
+        // SAFETY: fwctl sets this pointer before calling `open_uctx`.
+        let raw_fwctl = unsafe { (*uctx).fwctl };
+        // SAFETY: Rust fwctl devices use the offset-0 `Device<T>` layout.
+        let device = unsafe { Device::<T>::from_raw(raw_fwctl) };
+
+        let uctx_offset = core::mem::offset_of!(UserCtx<T>, uctx);
+        // SAFETY: `uctx_size` reserves space for the full `UserCtx<T>`.
+        let uctx_ptr: *mut T = unsafe { uctx.byte_add(uctx_offset).cast() };
+
+        // SAFETY: `open_uctx` is called under `registration_lock` read, so the device is
+        // registered. `uctx_ptr` addresses the uninitialised pinned context reserved by
+        // `uctx_size`.
+        unsafe {
+            device.with_registration_data(|device, reg_data| {
+                match pin_init::raw_try_init(uctx_ptr, T::open(device, reg_data)) {
+                    Ok(()) => 0,
+                    Err(e) => e.to_errno(),
+                }
+            })
+        }
+    }
+
+    /// Closes and drops an opened Rust user context.
+    ///
+    /// # Safety
+    ///
+    /// `uctx` must point to a fully initialised `UserCtx<T>`.
+    unsafe extern "C" fn close_uctx_callback(uctx: *mut bindings::fwctl_uctx) {
+        // SAFETY: fwctl keeps the owning device live for this callback.
+        let device = unsafe { Device::<T>::from_raw((*uctx).fwctl) };
+
+        // SAFETY: close is called for an opened Rust user context.
+        let mut ctx = unsafe { UserCtx::<T>::from_raw_mut(uctx) };
+
+        // SAFETY: `close_uctx` is called under `registration_lock` write (from
+        // `fwctl_unregister`) or read (from `fwctl_fops_release`), so the device is registered.
+        unsafe {
+            device.with_registration_data(|device, reg_data| {
+                T::close(ctx.as_mut().project().uctx, device, reg_data);
+            });
+        }
+
+        // SAFETY: close is the last callback before fwctl frees the allocation.
+        unsafe { core::ptr::drop_in_place(ctx.project().uctx.get_unchecked_mut()) };
+    }
+
+    /// Returns device-specific information for an opened Rust user context.
+    ///
+    /// # Safety
+    ///
+    /// - `uctx` must point to a fully initialised `UserCtx<T>`.
+    /// - `length` must be a valid pointer.
+    unsafe extern "C" fn info_callback(
+        uctx: *mut bindings::fwctl_uctx,
+        length: *mut usize,
+    ) -> *mut ffi::c_void {
+        // SAFETY: info is called for an opened Rust user context.
+        let ctx = unsafe { UserCtx::<T>::from_raw(uctx) };
+        let device = ctx.device();
+
+        // SAFETY: `info` is called under `registration_lock` read, so the device is registered.
+        let result = unsafe {
+            device.with_registration_data(|device, reg_data| T::info(ctx.uctx(), device, reg_data))
+        };
+
+        match result {
+            Ok(kvec) if kvec.is_empty() => {
+                // SAFETY: `length` is a valid out-parameter.
+                unsafe { *length = 0 };
+                // Return NULL for empty data; kfree(NULL) is safe.
+                core::ptr::null_mut()
+            }
+            Ok(kvec) => {
+                let (ptr, len, _cap) = kvec.into_raw_parts();
+                // SAFETY: `length` is a valid out-parameter.
+                unsafe { *length = len };
+                ptr.cast::<ffi::c_void>()
+            }
+            Err(e) => Error::to_ptr(e),
+        }
+    }
+
+    /// Dispatches a firmware RPC for an opened Rust user context.
+    ///
+    /// # Safety
+    ///
+    /// - `uctx` must point to a fully initialised `UserCtx<T>`.
+    /// - `rpc_in` must be valid, initialised, and exclusively accessible for `in_len` bytes.
+    /// - `out_len` must be valid for reading and writing an initialised `usize`.
+    unsafe extern "C" fn fw_rpc_callback(
+        uctx: *mut bindings::fwctl_uctx,
+        scope: u32,
+        rpc_in: *mut ffi::c_void,
+        in_len: usize,
+        out_len: *mut usize,
+    ) -> *mut ffi::c_void {
+        let scope = match RpcScope::try_from(scope) {
+            Ok(s) => s,
+            Err(e) => return Error::to_ptr(e),
+        };
+
+        // SAFETY: `out_len` points to an initialised `usize` supplied by fwctl.
+        let max_output_len = unsafe { *out_len };
+
+        // SAFETY: RPC is called for an opened Rust user context.
+        let ctx = unsafe { UserCtx::<T>::from_raw(uctx) };
+        let device = ctx.device();
+
+        // SAFETY: fwctl passes an exclusively owned buffer that is valid and initialised for
+        // `in_len` bytes. It remains live for the duration of this callback.
+        let rpc_buf = unsafe { slice::from_raw_parts_mut(rpc_in.cast::<u8>(), in_len) };
+
+        // SAFETY: `fw_rpc` is called under `registration_lock` read, so the device is registered.
+        let result = unsafe {
+            device.with_registration_data(|device, reg_data| {
+                T::fw_rpc(ctx.uctx(), device, reg_data, scope, rpc_buf, max_output_len)
+            })
+        };
+
+        let (response, response_len) = match result {
+            Ok(FwRpcResponse::InPlace(len)) => {
+                if len > in_len {
+                    return Error::to_ptr(EINVAL);
+                }
+
+                (rpc_in, len)
+            }
+            Ok(FwRpcResponse::NewBuffer(kvec)) if kvec.is_empty() => {
+                // Return NULL for empty data; kvfree(NULL) is safe.
+                (core::ptr::null_mut(), 0)
+            }
+            Ok(FwRpcResponse::NewBuffer(kvec)) => {
+                let (ptr, len, _cap) = kvec.into_raw_parts();
+                (ptr.cast::<ffi::c_void>(), len)
+            }
+            Err(e) => return Error::to_ptr(e),
+        };
+
+        // SAFETY: `out_len` is a valid out-parameter.
+        unsafe { *out_len = response_len };
+        response
+    }
+}
diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs
index abb208518413..4d5c96ddc49c 100644
--- a/rust/kernel/lib.rs
+++ b/rust/kernel/lib.rs
@@ -76,6 +76,8 @@ pub mod faux;
 pub mod firmware;
 pub mod fmt;
 pub mod fs;
+#[cfg(CONFIG_RUST_FWCTL_ABSTRACTIONS)]
+pub mod fwctl;
 #[cfg(CONFIG_GPU_BUDDY = "y")]
 pub mod gpu;
 #[cfg(CONFIG_I2C = "y")]