Commit c82e9aa0 authored by Eli Cohen's avatar Eli Cohen Committed by David S. Miller
Browse files

mlx4_core: resource tracking for HCA resources used by guests



The resource tracker is used to track usage of HCA resources by the different
guests.

Virtual functions (VFs) are attached to guest operating systems but
resources are allocated from the same pool and are assigned to VFs. It is
essential that hostile/buggy guests not be able to affect the operation of
other VFs, possibly attached to other guest OSs since ConnectX firmware is not
tolerant to misuse of resources.

The resource tracker module associates each resource with a VF and maintains
state information for the allocated object. It also defines allowed state
transitions and enforces them.

Relationships between resources are also referred to. For example, CQs are
pointed to by QPs, so it is forbidden to destroy a CQ if a QP refers to it.

ICM memory is always accessible through the primary function and hence it is
allocated by the owner of the primary function.

When a guest dies, an FLR is generated for all the VFs it owns and all the
resources it used are freed.

The tracked resource types are: QPs, CQs, SRQs, MPTs, MTTs, MACs, RES_EQs,
and XRCDNs.
Signed-off-by: default avatarEli Cohen <eli@mellanox.co.il>
Signed-off-by: default avatarJack Morgenstein <jackm@dev.mellanox.co.il>
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent acba2420
obj-$(CONFIG_MLX4_CORE) += mlx4_core.o
mlx4_core-y := alloc.o catas.o cmd.o cq.o eq.o fw.o icm.o intf.o main.o mcg.o \
mr.o pd.o port.o profile.o qp.o reset.o sense.o srq.o
mr.o pd.o port.o profile.o qp.o reset.o sense.o srq.o resource_tracker.o
obj-$(CONFIG_MLX4_EN) += mlx4_en.o
......
......@@ -172,12 +172,6 @@ static void mlx4_comm_cmd_post(struct mlx4_dev *dev, u8 cmd, u16 param)
mmiowb();
}
/* dummy procedure for this patch */
int mlx4_GEN_EQE(struct mlx4_dev *dev, int slave, struct mlx4_eqe *eqe)
{
return 0;
}
static int mlx4_comm_cmd_poll(struct mlx4_dev *dev, u8 cmd, u16 param,
unsigned long timeout)
{
......@@ -614,6 +608,403 @@ static struct mlx4_cmd_info cmd_info[] = {
.verify = NULL,
.wrapper = NULL
},
{
.opcode = MLX4_CMD_QUERY_FUNC_CAP,
.has_inbox = false,
.has_outbox = true,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_QUERY_FUNC_CAP_wrapper
},
{
.opcode = MLX4_CMD_QUERY_ADAPTER,
.has_inbox = false,
.has_outbox = true,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = NULL
},
{
.opcode = MLX4_CMD_INIT_PORT,
.has_inbox = false,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_INIT_PORT_wrapper
},
{
.opcode = MLX4_CMD_CLOSE_PORT,
.has_inbox = false,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_CLOSE_PORT_wrapper
},
{
.opcode = MLX4_CMD_QUERY_PORT,
.has_inbox = false,
.has_outbox = true,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_QUERY_PORT_wrapper
},
{
.opcode = MLX4_CMD_MAP_EQ,
.has_inbox = false,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_MAP_EQ_wrapper
},
{
.opcode = MLX4_CMD_SW2HW_EQ,
.has_inbox = true,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = true,
.verify = NULL,
.wrapper = mlx4_SW2HW_EQ_wrapper
},
{
.opcode = MLX4_CMD_HW_HEALTH_CHECK,
.has_inbox = false,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = NULL
},
{
.opcode = MLX4_CMD_NOP,
.has_inbox = false,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = NULL
},
{
.opcode = MLX4_CMD_ALLOC_RES,
.has_inbox = false,
.has_outbox = false,
.out_is_imm = true,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_ALLOC_RES_wrapper
},
{
.opcode = MLX4_CMD_FREE_RES,
.has_inbox = false,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_FREE_RES_wrapper
},
{
.opcode = MLX4_CMD_SW2HW_MPT,
.has_inbox = true,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = true,
.verify = NULL,
.wrapper = mlx4_SW2HW_MPT_wrapper
},
{
.opcode = MLX4_CMD_QUERY_MPT,
.has_inbox = false,
.has_outbox = true,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_QUERY_MPT_wrapper
},
{
.opcode = MLX4_CMD_HW2SW_MPT,
.has_inbox = false,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_HW2SW_MPT_wrapper
},
{
.opcode = MLX4_CMD_READ_MTT,
.has_inbox = false,
.has_outbox = true,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = NULL
},
{
.opcode = MLX4_CMD_WRITE_MTT,
.has_inbox = true,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_WRITE_MTT_wrapper
},
{
.opcode = MLX4_CMD_SYNC_TPT,
.has_inbox = true,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = NULL
},
{
.opcode = MLX4_CMD_HW2SW_EQ,
.has_inbox = false,
.has_outbox = true,
.out_is_imm = false,
.encode_slave_id = true,
.verify = NULL,
.wrapper = mlx4_HW2SW_EQ_wrapper
},
{
.opcode = MLX4_CMD_QUERY_EQ,
.has_inbox = false,
.has_outbox = true,
.out_is_imm = false,
.encode_slave_id = true,
.verify = NULL,
.wrapper = mlx4_QUERY_EQ_wrapper
},
{
.opcode = MLX4_CMD_SW2HW_CQ,
.has_inbox = true,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = true,
.verify = NULL,
.wrapper = mlx4_SW2HW_CQ_wrapper
},
{
.opcode = MLX4_CMD_HW2SW_CQ,
.has_inbox = false,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_HW2SW_CQ_wrapper
},
{
.opcode = MLX4_CMD_QUERY_CQ,
.has_inbox = false,
.has_outbox = true,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_QUERY_CQ_wrapper
},
{
.opcode = MLX4_CMD_MODIFY_CQ,
.has_inbox = true,
.has_outbox = false,
.out_is_imm = true,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_MODIFY_CQ_wrapper
},
{
.opcode = MLX4_CMD_SW2HW_SRQ,
.has_inbox = true,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = true,
.verify = NULL,
.wrapper = mlx4_SW2HW_SRQ_wrapper
},
{
.opcode = MLX4_CMD_HW2SW_SRQ,
.has_inbox = false,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_HW2SW_SRQ_wrapper
},
{
.opcode = MLX4_CMD_QUERY_SRQ,
.has_inbox = false,
.has_outbox = true,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_QUERY_SRQ_wrapper
},
{
.opcode = MLX4_CMD_ARM_SRQ,
.has_inbox = false,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_ARM_SRQ_wrapper
},
{
.opcode = MLX4_CMD_RST2INIT_QP,
.has_inbox = true,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = true,
.verify = NULL,
.wrapper = mlx4_RST2INIT_QP_wrapper
},
{
.opcode = MLX4_CMD_INIT2INIT_QP,
.has_inbox = true,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_GEN_QP_wrapper
},
{
.opcode = MLX4_CMD_INIT2RTR_QP,
.has_inbox = true,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_INIT2RTR_QP_wrapper
},
{
.opcode = MLX4_CMD_RTR2RTS_QP,
.has_inbox = true,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_GEN_QP_wrapper
},
{
.opcode = MLX4_CMD_RTS2RTS_QP,
.has_inbox = true,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_GEN_QP_wrapper
},
{
.opcode = MLX4_CMD_SQERR2RTS_QP,
.has_inbox = true,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_GEN_QP_wrapper
},
{
.opcode = MLX4_CMD_2ERR_QP,
.has_inbox = false,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_GEN_QP_wrapper
},
{
.opcode = MLX4_CMD_RTS2SQD_QP,
.has_inbox = false,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_GEN_QP_wrapper
},
{
.opcode = MLX4_CMD_SQD2SQD_QP,
.has_inbox = true,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_GEN_QP_wrapper
},
{
.opcode = MLX4_CMD_SQD2RTS_QP,
.has_inbox = true,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_GEN_QP_wrapper
},
{
.opcode = MLX4_CMD_2RST_QP,
.has_inbox = false,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_2RST_QP_wrapper
},
{
.opcode = MLX4_CMD_QUERY_QP,
.has_inbox = false,
.has_outbox = true,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_GEN_QP_wrapper
},
{
.opcode = MLX4_CMD_SUSPEND_QP,
.has_inbox = false,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_GEN_QP_wrapper
},
{
.opcode = MLX4_CMD_UNSUSPEND_QP,
.has_inbox = false,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_GEN_QP_wrapper
},
{
.opcode = MLX4_CMD_QUERY_IF_STAT,
.has_inbox = false,
.has_outbox = true,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_QUERY_IF_STAT_wrapper
},
/* Native multicast commands are not available for guests */
{
.opcode = MLX4_CMD_QP_ATTACH,
.has_inbox = true,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = mlx4_QP_ATTACH_wrapper
},
{
.opcode = MLX4_CMD_INFORM_FLR_DONE,
.has_inbox = false,
.has_outbox = false,
.out_is_imm = false,
.encode_slave_id = false,
.verify = NULL,
.wrapper = NULL
},
};
static int mlx4_master_process_vhcr(struct mlx4_dev *dev, int slave,
......@@ -877,6 +1268,8 @@ static void mlx4_master_do_cmd(struct mlx4_dev *dev, int slave, u8 cmd,
return;
reset_slave:
/* cleanup any slave resources */
mlx4_delete_all_resources_for_slave(dev, slave);
spin_lock(&priv->mfunc.master.slave_state_lock);
if (!slave_state[slave].is_slave_going_down)
slave_state[slave].last_cmd = MLX4_COMM_CMD_RESET;
......
......@@ -44,27 +44,6 @@
#include "mlx4.h"
#include "icm.h"
struct mlx4_cq_context {
__be32 flags;
u16 reserved1[3];
__be16 page_offset;
__be32 logsize_usrpage;
__be16 cq_period;
__be16 cq_max_count;
u8 reserved2[3];
u8 comp_eqn;
u8 log_page_size;
u8 reserved3[2];
u8 mtt_base_addr_h;
__be32 mtt_base_addr_l;
__be32 last_notified_index;
__be32 solicit_producer_index;
__be32 consumer_index;
__be32 producer_index;
u32 reserved4[2];
__be64 db_rec_addr;
};
#define MLX4_CQ_STATUS_OK ( 0 << 28)
#define MLX4_CQ_STATUS_OVERFLOW ( 9 << 28)
#define MLX4_CQ_STATUS_WRITE_FAIL (10 << 28)
......@@ -189,7 +168,7 @@ int mlx4_cq_resize(struct mlx4_dev *dev, struct mlx4_cq *cq,
}
EXPORT_SYMBOL_GPL(mlx4_cq_resize);
static int __mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn)
int __mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn)
{
struct mlx4_priv *priv = mlx4_priv(dev);
struct mlx4_cq_table *cq_table = &priv->cq_table;
......@@ -235,7 +214,7 @@ static int mlx4_cq_alloc_icm(struct mlx4_dev *dev, int *cqn)
return __mlx4_cq_alloc_icm(dev, cqn);
}
static void __mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn)
void __mlx4_cq_free_icm(struct mlx4_dev *dev, int cqn)
{
struct mlx4_priv *priv = mlx4_priv(dev);
struct mlx4_cq_table *cq_table = &priv->cq_table;
......
......@@ -53,30 +53,6 @@ enum {
MLX4_EQ_ENTRY_SIZE = 0x20
};
/*
* Must be packed because start is 64 bits but only aligned to 32 bits.
*/
struct mlx4_eq_context {
__be32 flags;
u16 reserved1[3];
__be16 page_offset;
u8 log_eq_size;
u8 reserved2[4];
u8 eq_period;
u8 reserved3;
u8 eq_max_count;
u8 reserved4[3];
u8 intr;
u8 log_page_size;
u8 reserved5[2];
u8 mtt_base_addr_h;
__be32 mtt_base_addr_l;
u32 reserved6[2];
__be32 consumer_index;
__be32 producer_index;
u32 reserved7[4];
};
#define MLX4_EQ_STATUS_OK ( 0 << 28)
#define MLX4_EQ_STATUS_WRITE_FAIL (10 << 28)
#define MLX4_EQ_OWNER_SW ( 0 << 24)
......@@ -135,19 +111,6 @@ static struct mlx4_eqe *next_slave_event_eqe(struct mlx4_slave_event_eq *slave_e
eqe : NULL;
}
/* dummies for now */
void mlx4_delete_all_resources_for_slave(struct mlx4_dev *dev, int slave)
{
}
int mlx4_get_slave_from_resource_id(struct mlx4_dev *dev,
enum mlx4_resource type,
int res_id, int *slave)
{
return -ENOENT;
}
/* end dummies */
void mlx4_gen_slave_eqe(struct work_struct *work)
{
struct mlx4_mfunc_master_ctx *master =
......
......@@ -224,6 +224,91 @@ struct mlx4_icm_table {
struct mlx4_icm **icm;
};
/*
* Must be packed because mtt_seg is 64 bits but only aligned to 32 bits.
*/
struct mlx4_mpt_entry {
__be32 flags;
__be32 qpn;
__be32 key;
__be32 pd_flags;
__be64 start;
__be64 length;
__be32 lkey;
__be32 win_cnt;
u8 reserved1[3];
u8 mtt_rep;
__be64 mtt_seg;
__be32 mtt_sz;
__be32 entity_size;
__be32 first_byte_offset;
} __packed;
/*
* Must be packed because start is 64 bits but only aligned to 32 bits.
*/
struct mlx4_eq_context {
__be32 flags;
u16 reserved1[3];
__be16 page_offset;
u8 log_eq_size;
u8 reserved2[4];
u8 eq_period;
u8 reserved3;
u8 eq_max_count;
u8 reserved4[3];
u8 intr;
u8 log_page_size;
u8 reserved5[2];
u8 mtt_base_addr_h;
__be32 mtt_base_addr_l;
u32 reserved6[2];
__be32 consumer_index;
__be32 producer_index;
u32 reserved7[4];
};
struct mlx4_cq_context {
__be32 flags;
u16 reserved1[3];
__be16 page_offset;
__be32 logsize_usrpage;
__be16 cq_period;
__be16 cq_max_count;
u8 reserved2[3];
u8 comp_eqn;
u8 log_page_size;
u8 reserved3[2];
u8 mtt_base_addr_h;
__be32 mtt_base_addr_l;
__be32 last_notified_index;
__be32 solicit_producer_index;
__be32 consumer_index;
__be32 producer_index;
u32 reserved4[2];
__be64 db_rec_addr;
};
struct mlx4_srq_context {
__be32 state_logsize_srqn;
u8 logstride;
u8 reserved1;
__be16 xrcd;
__be32 pg_offset_cqn;
u32 reserved2;
u8 log_page_size;