iv/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mlx4_core: Multiple port type support

Browse Source 
Multi-protocol adapters support different port types.  Each consumer
of mlx4_core queries for supported port types; in particular mlx4_ib
can no longer assume that all physical ports belong to it.  Port type
is configured through a sysfs interface.  When the type of a port is
changed, all mlx4 interfaces are unregistered, and then registered
again with the new port types.

Signed-off-by: Yevgeny Petrilin <yevgenyp@mellanox.co.il>
Signed-off-by: Roland Dreier <rolandd@cisco.com>
This commit is contained in:
Yevgeny Petrilin 2008-10-22 15:38:42 -07:00 committed by Roland Dreier
parent 2a2336f822
commit 7ff93f8b7e
9 changed files with 266 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
drivers/infiniband/hw/mlx4/mad.c
View File

@ -298,7 +298,7 @@ int mlx4_ib_mad_init(struct mlx4_ib_dev *dev)
int p, q;
int ret;
for (p = 0; p < dev->dev->caps.num_ports; ++p)
for (p = 0; p < dev->num_ports; ++p)
for (q = 0; q <= 1; ++q) {
agent = ib_register_mad_agent(&dev->ib_dev, p + 1,
q ? IB_QPT_GSI : IB_QPT_SMI,
@ -314,7 +314,7 @@ int mlx4_ib_mad_init(struct mlx4_ib_dev *dev)
return 0;
err:
for (p = 0; p < dev->dev->caps.num_ports; ++p)
for (p = 0; p < dev->num_ports; ++p)
for (q = 0; q <= 1; ++q)
if (dev->send_agent[p][q])
ib_unregister_mad_agent(dev->send_agent[p][q]);
@ -327,7 +327,7 @@ void mlx4_ib_mad_cleanup(struct mlx4_ib_dev *dev)
struct ib_mad_agent *agent;
int p, q;
for (p = 0; p < dev->dev->caps.num_ports; ++p) {
for (p = 0; p < dev->num_ports; ++p) {
for (q = 0; q <= 1; ++q) {
agent = dev->send_agent[p][q];
dev->send_agent[p][q] = NULL;

11
drivers/infiniband/hw/mlx4/main.c
View File

@ -574,7 +574,10 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
ibdev->ib_dev.owner = THIS_MODULE;
ibdev->ib_dev.node_type = RDMA_NODE_IB_CA;
ibdev->ib_dev.local_dma_lkey = dev->caps.reserved_lkey;
ibdev->ib_dev.phys_port_cnt = dev->caps.num_ports;
ibdev->num_ports = 0;
mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB)
ibdev->num_ports++;
ibdev->ib_dev.phys_port_cnt = ibdev->num_ports;
ibdev->ib_dev.num_comp_vectors = 1;
ibdev->ib_dev.dma_device = &dev->pdev->dev;
@ -691,7 +694,7 @@ static void mlx4_ib_remove(struct mlx4_dev *dev, void *ibdev_ptr)
struct mlx4_ib_dev *ibdev = ibdev_ptr;
int p;
for (p = 1; p <= dev->caps.num_ports; ++p)
for (p = 1; p <= ibdev->num_ports; ++p)
mlx4_CLOSE_PORT(dev, p);
mlx4_ib_mad_cleanup(ibdev);
@ -706,6 +709,10 @@ static void mlx4_ib_event(struct mlx4_dev *dev, void *ibdev_ptr,
enum mlx4_dev_event event, int port)
{
struct ib_event ibev;
struct mlx4_ib_dev *ibdev = to_mdev((struct ib_device *) ibdev_ptr);
if (port > ibdev->num_ports)
return;
switch (event) {
case MLX4_DEV_EVENT_PORT_UP:

1
drivers/infiniband/hw/mlx4/mlx4_ib.h
View File

@ -162,6 +162,7 @@ struct mlx4_ib_ah {
struct mlx4_ib_dev {
struct ib_device ib_dev;
struct mlx4_dev *dev;
int num_ports;
void __iomem *uar_map;
struct mlx4_uar priv_uar;

4
drivers/net/mlx4/fw.c
View File

@ -88,6 +88,7 @@ static void dump_dev_cap_flags(struct mlx4_dev *dev, u32 flags)
[ 8] = "P_Key violation counter",
[ 9] = "Q_Key violation counter",
[10] = "VMM",
[12] = "DPDP",
[16] = "MW support",
[17] = "APM support",
[18] = "Atomic ops support",
@ -354,6 +355,7 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
dev_cap->max_pkeys[i] = 1 << (field & 0xf);
}
} else {
#define QUERY_PORT_SUPPORTED_TYPE_OFFSET 0x00
#define QUERY_PORT_MTU_OFFSET 0x01
#define QUERY_PORT_ETH_MTU_OFFSET 0x02
#define QUERY_PORT_WIDTH_OFFSET 0x06
@ -368,6 +370,8 @@ int mlx4_QUERY_DEV_CAP(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
if (err)
goto out;
MLX4_GET(field, outbox, QUERY_PORT_SUPPORTED_TYPE_OFFSET);
dev_cap->supported_port_types[i] = field & 3;
MLX4_GET(field, outbox, QUERY_PORT_MTU_OFFSET);
dev_cap->ib_mtu[i] = field & 0xf;
MLX4_GET(field, outbox, QUERY_PORT_WIDTH_OFFSET);

1
drivers/net/mlx4/fw.h
View File

@ -104,6 +104,7 @@ struct mlx4_dev_cap {
u32 reserved_lkey;
u64 max_icm_sz;
int max_gso_sz;
u8 supported_port_types[MLX4_MAX_PORTS + 1];
u8 log_max_macs[MLX4_MAX_PORTS + 1];
u8 log_max_vlans[MLX4_MAX_PORTS + 1];
};

211
drivers/net/mlx4/main.c
View File

@ -98,6 +98,44 @@ module_param_named(use_prio, use_prio, bool, 0444);
MODULE_PARM_DESC(use_prio, "Enable steering by VLAN priority on ETH ports "
"(0/1, default 0)");
static int mlx4_check_port_params(struct mlx4_dev *dev,
enum mlx4_port_type *port_type)
{
int i;
for (i = 0; i < dev->caps.num_ports - 1; i++) {
if (port_type[i] != port_type[i+1] &&
!(dev->caps.flags & MLX4_DEV_CAP_FLAG_DPDP)) {
mlx4_err(dev, "Only same port types supported "
"on this HCA, aborting.\n");
return -EINVAL;
}
}
if ((port_type[0] == MLX4_PORT_TYPE_ETH) &&
(port_type[1] == MLX4_PORT_TYPE_IB)) {
mlx4_err(dev, "eth-ib configuration is not supported.\n");
return -EINVAL;
}
for (i = 0; i < dev->caps.num_ports; i++) {
if (!(port_type[i] & dev->caps.supported_type[i+1])) {
mlx4_err(dev, "Requested port type for port %d is not "
"supported on this HCA\n", i + 1);
return -EINVAL;
}
}
return 0;
}
static void mlx4_set_port_mask(struct mlx4_dev *dev)
{
int i;
dev->caps.port_mask = 0;
for (i = 1; i <= dev->caps.num_ports; ++i)
if (dev->caps.port_type[i] == MLX4_PORT_TYPE_IB)
dev->caps.port_mask |= 1 << (i - 1);
}
static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
{
int err;
@ -139,6 +177,7 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
dev->caps.port_width_cap[i] = dev_cap->max_port_width[i];
dev->caps.eth_mtu_cap[i] = dev_cap->eth_mtu[i];
dev->caps.def_mac[i] = dev_cap->def_mac[i];
dev->caps.supported_type[i] = dev_cap->supported_port_types[i];
}
dev->caps.num_uars = dev_cap->uar_size / PAGE_SIZE;
@ -182,6 +221,11 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
dev->caps.log_num_prios = use_prio ? 3 : 0;
for (i = 1; i <= dev->caps.num_ports; ++i) {
if (dev->caps.supported_type[i] != MLX4_PORT_TYPE_ETH)
dev->caps.port_type[i] = MLX4_PORT_TYPE_IB;
else
dev->caps.port_type[i] = MLX4_PORT_TYPE_ETH;
if (dev->caps.log_num_macs > dev_cap->log_max_macs[i]) {
dev->caps.log_num_macs = dev_cap->log_max_macs[i];
mlx4_warn(dev, "Requested number of MACs is too much "
@ -196,6 +240,8 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
}
}
mlx4_set_port_mask(dev);
dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW] = dev_cap->reserved_qps;
dev->caps.reserved_qps_cnt[MLX4_QP_REGION_ETH_ADDR] =
dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FC_ADDR] =
@ -213,6 +259,95 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap)
return 0;
}
/*
* Change the port configuration of the device.
* Every user of this function must hold the port mutex.
*/
static int mlx4_change_port_types(struct mlx4_dev *dev,
enum mlx4_port_type *port_types)
{
int err = 0;
int change = 0;
int port;
for (port = 0; port < dev->caps.num_ports; port++) {
if (port_types[port] != dev->caps.port_type[port + 1]) {
change = 1;
dev->caps.port_type[port + 1] = port_types[port];
}
}
if (change) {
mlx4_unregister_device(dev);
for (port = 1; port <= dev->caps.num_ports; port++) {
mlx4_CLOSE_PORT(dev, port);
err = mlx4_SET_PORT(dev, port);
if (err) {
mlx4_err(dev, "Failed to set port %d, "
"aborting\n", port);
goto out;
}
}
mlx4_set_port_mask(dev);
err = mlx4_register_device(dev);
}
out:
return err;
}
static ssize_t show_port_type(struct device *dev,
struct device_attribute *attr,
char *buf)
{
struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info,
port_attr);
struct mlx4_dev *mdev = info->dev;
return sprintf(buf, "%s\n",
mdev->caps.port_type[info->port] == MLX4_PORT_TYPE_IB ?
"ib" : "eth");
}
static ssize_t set_port_type(struct device *dev,
struct device_attribute *attr,
const char *buf, size_t count)
{
struct mlx4_port_info *info = container_of(attr, struct mlx4_port_info,
port_attr);
struct mlx4_dev *mdev = info->dev;
struct mlx4_priv *priv = mlx4_priv(mdev);
enum mlx4_port_type types[MLX4_MAX_PORTS];
int i;
int err = 0;
if (!strcmp(buf, "ib\n"))
info->tmp_type = MLX4_PORT_TYPE_IB;
else if (!strcmp(buf, "eth\n"))
info->tmp_type = MLX4_PORT_TYPE_ETH;
else {
mlx4_err(mdev, "%s is not supported port type\n", buf);
return -EINVAL;
}
mutex_lock(&priv->port_mutex);
for (i = 0; i < mdev->caps.num_ports; i++)
types[i] = priv->port[i+1].tmp_type ? priv->port[i+1].tmp_type :
mdev->caps.port_type[i+1];
err = mlx4_check_port_params(mdev, types);
if (err)
goto out;
for (i = 1; i <= mdev->caps.num_ports; i++)
priv->port[i].tmp_type = 0;
err = mlx4_change_port_types(mdev, types);
out:
mutex_unlock(&priv->port_mutex);
return err ? err : count;
}
static int mlx4_load_fw(struct mlx4_dev *dev)
{
struct mlx4_priv *priv = mlx4_priv(dev);
@ -617,6 +752,7 @@ static int mlx4_setup_hca(struct mlx4_dev *dev)
{
struct mlx4_priv *priv = mlx4_priv(dev);
int err;
int port;
err = mlx4_init_uar_table(dev);
if (err) {
@ -715,8 +851,20 @@ static int mlx4_setup_hca(struct mlx4_dev *dev)
goto err_qp_table_free;
}
for (port = 1; port <= dev->caps.num_ports; port++) {
err = mlx4_SET_PORT(dev, port);
if (err) {
mlx4_err(dev, "Failed to set port %d, aborting\n",
port);
goto err_mcg_table_free;
}
}
return 0;
err_mcg_table_free:
mlx4_cleanup_mcg_table(dev);
err_qp_table_free:
mlx4_cleanup_qp_table(dev);
@ -780,14 +928,37 @@ no_msi:
priv->eq_table.eq[i].irq = dev->pdev->irq;
}
static void mlx4_init_port_info(struct mlx4_dev *dev, int port)
static int mlx4_init_port_info(struct mlx4_dev *dev, int port)
{
struct mlx4_port_info *info = &mlx4_priv(dev)->port[port];
int err = 0;
info->dev = dev;
info->port = port;
mlx4_init_mac_table(dev, &info->mac_table);
mlx4_init_vlan_table(dev, &info->vlan_table);
sprintf(info->dev_name, "mlx4_port%d", port);
info->port_attr.attr.name = info->dev_name;
info->port_attr.attr.mode = S_IRUGO | S_IWUSR;
info->port_attr.show = show_port_type;
info->port_attr.store = set_port_type;
err = device_create_file(&dev->pdev->dev, &info->port_attr);
if (err) {
mlx4_err(dev, "Failed to create file for port %d\n", port);
info->port = -1;
}
return err;
}
static void mlx4_cleanup_port_info(struct mlx4_port_info *info)
{
if (info->port < 0)
return;
device_remove_file(&info->dev->pdev->dev, &info->port_attr);
}
static int __mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
@ -870,6 +1041,8 @@ static int __mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
INIT_LIST_HEAD(&priv->ctx_list);
spin_lock_init(&priv->ctx_lock);
mutex_init(&priv->port_mutex);
INIT_LIST_HEAD(&priv->pgdir_list);
mutex_init(&priv->pgdir_mutex);
@ -905,18 +1078,24 @@ static int __mlx4_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
if (err)
goto err_close;
for (port = 1; port <= dev->caps.num_ports; port++)
mlx4_init_port_info(dev, port);
for (port = 1; port <= dev->caps.num_ports; port++) {
err = mlx4_init_port_info(dev, port);
if (err)
goto err_port;
}
err = mlx4_register_device(dev);
if (err)
goto err_cleanup;
goto err_port;
pci_set_drvdata(pdev, dev);
return 0;
err_cleanup:
err_port:
for (port = 1; port <= dev->caps.num_ports; port++)
mlx4_cleanup_port_info(&priv->port[port]);
mlx4_cleanup_mcg_table(dev);
mlx4_cleanup_qp_table(dev);
mlx4_cleanup_srq_table(dev);
@ -973,8 +1152,10 @@ static void mlx4_remove_one(struct pci_dev *pdev)
if (dev) {
mlx4_unregister_device(dev);
for (p = 1; p <= dev->caps.num_ports; ++p)
for (p = 1; p <= dev->caps.num_ports; p++) {
mlx4_cleanup_port_info(&priv->port[p]);
mlx4_CLOSE_PORT(dev, p);
}
mlx4_cleanup_mcg_table(dev);
mlx4_cleanup_qp_table(dev);
@ -1026,10 +1207,28 @@ static struct pci_driver mlx4_driver = {
.remove = __devexit_p(mlx4_remove_one)
};
static int __init mlx4_verify_params(void)
{
if ((log_num_mac < 0) || (log_num_mac > 7)) {
printk(KERN_WARNING "mlx4_core: bad num_mac: %d\n", log_num_mac);
return -1;
}
if ((log_num_vlan < 0) || (log_num_vlan > 7)) {
printk(KERN_WARNING "mlx4_core: bad num_vlan: %d\n", log_num_vlan);
return -1;
}
return 0;
}
static int __init mlx4_init(void)
{
int ret;
if (mlx4_verify_params())
return -EINVAL;
ret = mlx4_catas_init();
if (ret)
return ret;

6
drivers/net/mlx4/mlx4.h
View File

@ -277,6 +277,9 @@ struct mlx4_vlan_table {
struct mlx4_port_info {
struct mlx4_dev *dev;
int port;
char dev_name[16];
struct device_attribute port_attr;
enum mlx4_port_type tmp_type;
struct mlx4_mac_table mac_table;
struct mlx4_vlan_table vlan_table;
};
@ -310,6 +313,7 @@ struct mlx4_priv {
struct mlx4_uar driver_uar;
void __iomem *kar;
struct mlx4_port_info port[MLX4_MAX_PORTS + 1];
struct mutex port_mutex;
};
static inline struct mlx4_priv *mlx4_priv(struct mlx4_dev *dev)
@ -383,4 +387,6 @@ void mlx4_handle_catas_err(struct mlx4_dev *dev);
void mlx4_init_mac_table(struct mlx4_dev *dev, struct mlx4_mac_table *table);
void mlx4_init_vlan_table(struct mlx4_dev *dev, struct mlx4_vlan_table *table);
int mlx4_SET_PORT(struct mlx4_dev *dev, u8 port);
#endif /* MLX4_H */

23
drivers/net/mlx4/port.c
View File

@ -257,3 +257,26 @@ out:
mutex_unlock(&table->mutex);
}
EXPORT_SYMBOL_GPL(mlx4_unregister_vlan);
int mlx4_SET_PORT(struct mlx4_dev *dev, u8 port)
{
struct mlx4_cmd_mailbox *mailbox;
int err;
u8 is_eth = dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH;
mailbox = mlx4_alloc_cmd_mailbox(dev);
if (IS_ERR(mailbox))
return PTR_ERR(mailbox);
memset(mailbox->buf, 0, 256);
if (is_eth) {
((u8 *) mailbox->buf)[3] = 6;
((__be16 *) mailbox->buf)[4] = cpu_to_be16(1 << 15);
((__be16 *) mailbox->buf)[6] = cpu_to_be16(1 << 15);
}
err = mlx4_cmd(dev, mailbox->dma, port, is_eth, MLX4_CMD_SET_PORT,
MLX4_CMD_TIME_CLASS_B);
mlx4_free_cmd_mailbox(dev, mailbox);
return err;
}

14
include/linux/mlx4/device.h
View File

@ -60,6 +60,7 @@ enum {
MLX4_DEV_CAP_FLAG_IPOIB_CSUM = 1 << 7,
MLX4_DEV_CAP_FLAG_BAD_PKEY_CNTR = 1 << 8,
MLX4_DEV_CAP_FLAG_BAD_QKEY_CNTR = 1 << 9,
MLX4_DEV_CAP_FLAG_DPDP = 1 << 12,
MLX4_DEV_CAP_FLAG_MEM_WINDOW = 1 << 16,
MLX4_DEV_CAP_FLAG_APM = 1 << 17,
MLX4_DEV_CAP_FLAG_ATOMIC = 1 << 18,
@ -153,6 +154,11 @@ enum mlx4_qp_region {
MLX4_NUM_QP_REGION
};
enum mlx4_port_type {
MLX4_PORT_TYPE_IB = 1 << 0,
MLX4_PORT_TYPE_ETH = 1 << 1,
};
enum mlx4_special_vlan_idx {
MLX4_NO_VLAN_IDX = 0,
MLX4_VLAN_MISS_IDX,
@ -226,6 +232,9 @@ struct mlx4_caps {
int log_num_macs;
int log_num_vlans;
int log_num_prios;
enum mlx4_port_type port_type[MLX4_MAX_PORTS + 1];
u8 supported_type[MLX4_MAX_PORTS + 1];
u32 port_mask;
};
struct mlx4_buf_list {
@ -380,6 +389,11 @@ struct mlx4_init_port_param {
u64 si_guid;
};
#define mlx4_foreach_port(port, dev, type) \
for ((port) = 1; (port) <= (dev)->caps.num_ports; (port)++) \
if (((type) == MLX4_PORT_TYPE_IB ? (dev)->caps.port_mask : \
~(dev)->caps.port_mask) & 1 << ((port) - 1))
int mlx4_buf_alloc(struct mlx4_dev *dev, int size, int max_direct,
struct mlx4_buf *buf);
void mlx4_buf_free(struct mlx4_dev *dev, int size, struct mlx4_buf *buf);