diff options
Diffstat (limited to 'drivers/infiniband/hw/i40iw')
20 files changed, 840 insertions, 365 deletions
diff --git a/drivers/infiniband/hw/i40iw/i40iw.h b/drivers/infiniband/hw/i40iw/i40iw.h index 819767681..b738acdb9 100644 --- a/drivers/infiniband/hw/i40iw/i40iw.h +++ b/drivers/infiniband/hw/i40iw/i40iw.h @@ -50,8 +50,6 @@ #include <rdma/ib_pack.h> #include <rdma/rdma_cm.h> #include <rdma/iw_cm.h> -#include <rdma/iw_portmap.h> -#include <rdma/rdma_netlink.h> #include <crypto/hash.h> #include "i40iw_status.h" @@ -115,6 +113,8 @@ #define IW_HMC_OBJ_TYPE_NUM ARRAY_SIZE(iw_hmc_obj_types) #define IW_CFG_FPM_QP_COUNT 32768 +#define I40IW_MAX_PAGES_PER_FMR 512 +#define I40IW_MIN_PAGES_PER_FMR 1 #define I40IW_MTU_TO_MSS 40 #define I40IW_DEFAULT_MSS 1460 @@ -254,6 +254,7 @@ struct i40iw_device { u32 arp_table_size; u32 next_arp_index; spinlock_t resource_lock; /* hw resource access */ + spinlock_t qptable_lock; u32 vendor_id; u32 vendor_part_id; u32 of_device_registered; @@ -392,7 +393,7 @@ void i40iw_flush_wqes(struct i40iw_device *iwdev, void i40iw_manage_arp_cache(struct i40iw_device *iwdev, unsigned char *mac_addr, - __be32 *ip_addr, + u32 *ip_addr, bool ipv4, u32 action); @@ -550,7 +551,7 @@ enum i40iw_status_code i40iw_hw_flush_wqes(struct i40iw_device *iwdev, struct i40iw_qp_flush_info *info, bool wait); -void i40iw_copy_ip_ntohl(u32 *dst, u32 *src); +void i40iw_copy_ip_ntohl(u32 *dst, __be32 *src); struct ib_mr *i40iw_reg_phys_mr(struct ib_pd *ib_pd, u64 addr, u64 size, diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.c b/drivers/infiniband/hw/i40iw/i40iw_cm.c index 38f917a6c..d2fa72516 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_cm.c +++ b/drivers/infiniband/hw/i40iw/i40iw_cm.c @@ -771,6 +771,7 @@ static void i40iw_build_mpa_v2(struct i40iw_cm_node *cm_node, { struct ietf_mpa_v2 *mpa_frame = (struct ietf_mpa_v2 *)start_addr; struct ietf_rtr_msg *rtr_msg = &mpa_frame->rtr_msg; + u16 ctrl_ird, ctrl_ord; /* initialize the upper 5 bytes of the frame */ i40iw_build_mpa_v1(cm_node, start_addr, mpa_key); @@ -779,38 +780,38 @@ static void i40iw_build_mpa_v2(struct i40iw_cm_node *cm_node, /* initialize RTR msg */ if (cm_node->mpav2_ird_ord == IETF_NO_IRD_ORD) { - rtr_msg->ctrl_ird = IETF_NO_IRD_ORD; - rtr_msg->ctrl_ord = IETF_NO_IRD_ORD; + ctrl_ird = IETF_NO_IRD_ORD; + ctrl_ord = IETF_NO_IRD_ORD; } else { - rtr_msg->ctrl_ird = (cm_node->ird_size > IETF_NO_IRD_ORD) ? + ctrl_ird = (cm_node->ird_size > IETF_NO_IRD_ORD) ? IETF_NO_IRD_ORD : cm_node->ird_size; - rtr_msg->ctrl_ord = (cm_node->ord_size > IETF_NO_IRD_ORD) ? + ctrl_ord = (cm_node->ord_size > IETF_NO_IRD_ORD) ? IETF_NO_IRD_ORD : cm_node->ord_size; } - rtr_msg->ctrl_ird |= IETF_PEER_TO_PEER; - rtr_msg->ctrl_ird |= IETF_FLPDU_ZERO_LEN; + ctrl_ird |= IETF_PEER_TO_PEER; + ctrl_ird |= IETF_FLPDU_ZERO_LEN; switch (mpa_key) { case MPA_KEY_REQUEST: - rtr_msg->ctrl_ord |= IETF_RDMA0_WRITE; - rtr_msg->ctrl_ord |= IETF_RDMA0_READ; + ctrl_ord |= IETF_RDMA0_WRITE; + ctrl_ord |= IETF_RDMA0_READ; break; case MPA_KEY_REPLY: switch (cm_node->send_rdma0_op) { case SEND_RDMA_WRITE_ZERO: - rtr_msg->ctrl_ord |= IETF_RDMA0_WRITE; + ctrl_ord |= IETF_RDMA0_WRITE; break; case SEND_RDMA_READ_ZERO: - rtr_msg->ctrl_ord |= IETF_RDMA0_READ; + ctrl_ord |= IETF_RDMA0_READ; break; } break; default: break; } - rtr_msg->ctrl_ird = htons(rtr_msg->ctrl_ird); - rtr_msg->ctrl_ord = htons(rtr_msg->ctrl_ord); + rtr_msg->ctrl_ird = htons(ctrl_ird); + rtr_msg->ctrl_ord = htons(ctrl_ord); } /** @@ -2107,7 +2108,7 @@ static bool i40iw_ipv6_is_loopback(u32 *loc_addr, u32 *rem_addr) struct in6_addr raddr6; i40iw_copy_ip_htonl(raddr6.in6_u.u6_addr32, rem_addr); - return (!memcmp(loc_addr, rem_addr, 16) || ipv6_addr_loopback(&raddr6)); + return !memcmp(loc_addr, rem_addr, 16) || ipv6_addr_loopback(&raddr6); } /** @@ -2160,7 +2161,7 @@ static struct i40iw_cm_node *i40iw_make_cm_node( cm_node->tcp_cntxt.rcv_wnd = I40IW_CM_DEFAULT_RCV_WND_SCALED >> I40IW_CM_DEFAULT_RCV_WND_SCALE; ts = current_kernel_time(); - cm_node->tcp_cntxt.loc_seq_num = htonl(ts.tv_nsec); + cm_node->tcp_cntxt.loc_seq_num = ts.tv_nsec; cm_node->tcp_cntxt.mss = iwdev->mss; cm_node->iwdev = iwdev; @@ -2234,7 +2235,7 @@ static void i40iw_rem_ref_cm_node(struct i40iw_cm_node *cm_node) if (cm_node->listener) { i40iw_dec_refcnt_listen(cm_core, cm_node->listener, 0, true); } else { - if (!i40iw_listen_port_in_use(cm_core, htons(cm_node->loc_port)) && + if (!i40iw_listen_port_in_use(cm_core, cm_node->loc_port) && cm_node->apbvt_set && cm_node->iwdev) { i40iw_manage_apbvt(cm_node->iwdev, cm_node->loc_port, @@ -2852,7 +2853,6 @@ static struct i40iw_cm_node *i40iw_create_cm_node( void *private_data, struct i40iw_cm_info *cm_info) { - int ret; struct i40iw_cm_node *cm_node; struct i40iw_cm_listener *loopback_remotelistener; struct i40iw_cm_node *loopback_remotenode; @@ -2922,30 +2922,6 @@ static struct i40iw_cm_node *i40iw_create_cm_node( memcpy(cm_node->pdata_buf, private_data, private_data_len); cm_node->state = I40IW_CM_STATE_SYN_SENT; - ret = i40iw_send_syn(cm_node, 0); - - if (ret) { - if (cm_node->ipv4) - i40iw_debug(cm_node->dev, - I40IW_DEBUG_CM, - "Api - connect() FAILED: dest addr=%pI4", - cm_node->rem_addr); - else - i40iw_debug(cm_node->dev, I40IW_DEBUG_CM, - "Api - connect() FAILED: dest addr=%pI6", - cm_node->rem_addr); - i40iw_rem_ref_cm_node(cm_node); - cm_node = NULL; - } - - if (cm_node) - i40iw_debug(cm_node->dev, - I40IW_DEBUG_CM, - "Api - connect(): port=0x%04x, cm_node=%p, cm_id = %p.\n", - cm_node->rem_port, - cm_node, - cm_node->cm_id); - return cm_node; } @@ -3266,11 +3242,13 @@ static void i40iw_init_tcp_ctx(struct i40iw_cm_node *cm_node, tcp_info->dest_ip_addr3 = cpu_to_le32(cm_node->rem_addr[0]); tcp_info->local_ipaddr3 = cpu_to_le32(cm_node->loc_addr[0]); - tcp_info->arp_idx = cpu_to_le32(i40iw_arp_table(iwqp->iwdev, - &tcp_info->dest_ip_addr3, - true, - NULL, - I40IW_ARP_RESOLVE)); + tcp_info->arp_idx = + cpu_to_le16((u16)i40iw_arp_table( + iwqp->iwdev, + &tcp_info->dest_ip_addr3, + true, + NULL, + I40IW_ARP_RESOLVE)); } else { tcp_info->src_port = cpu_to_le16(cm_node->loc_port); tcp_info->dst_port = cpu_to_le16(cm_node->rem_port); @@ -3282,12 +3260,13 @@ static void i40iw_init_tcp_ctx(struct i40iw_cm_node *cm_node, tcp_info->local_ipaddr1 = cpu_to_le32(cm_node->loc_addr[1]); tcp_info->local_ipaddr2 = cpu_to_le32(cm_node->loc_addr[2]); tcp_info->local_ipaddr3 = cpu_to_le32(cm_node->loc_addr[3]); - tcp_info->arp_idx = cpu_to_le32(i40iw_arp_table( - iwqp->iwdev, - &tcp_info->dest_ip_addr0, - false, - NULL, - I40IW_ARP_RESOLVE)); + tcp_info->arp_idx = + cpu_to_le16((u16)i40iw_arp_table( + iwqp->iwdev, + &tcp_info->dest_ip_addr0, + false, + NULL, + I40IW_ARP_RESOLVE)); } } @@ -3564,7 +3543,6 @@ int i40iw_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) struct i40iw_cm_node *cm_node; struct ib_qp_attr attr; int passive_state; - struct i40iw_ib_device *iwibdev; struct ib_mr *ibmr; struct i40iw_pd *iwpd; u16 buf_len = 0; @@ -3627,7 +3605,6 @@ int i40iw_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) !i40iw_ipv4_is_loopback(cm_node->loc_addr[0], cm_node->rem_addr[0])) || (!cm_node->ipv4 && !i40iw_ipv6_is_loopback(cm_node->loc_addr, cm_node->rem_addr))) { - iwibdev = iwdev->iwibdev; iwpd = iwqp->iwpd; tagged_offset = (uintptr_t)iwqp->ietf_mem.va; ibmr = i40iw_reg_phys_mr(&iwpd->ibpd, @@ -3752,6 +3729,7 @@ int i40iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) struct sockaddr_in *raddr; struct sockaddr_in6 *laddr6; struct sockaddr_in6 *raddr6; + bool qhash_set = false; int apbvt_set = 0; enum i40iw_status_code status; @@ -3810,6 +3788,7 @@ int i40iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) true); if (status) return -EINVAL; + qhash_set = true; } status = i40iw_manage_apbvt(iwdev, cm_info.loc_port, I40IW_MANAGE_APBVT_ADD); if (status) { @@ -3828,23 +3807,8 @@ int i40iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) conn_param->private_data_len, (void *)conn_param->private_data, &cm_info); - if (!cm_node) { - i40iw_manage_qhash(iwdev, - &cm_info, - I40IW_QHASH_TYPE_TCP_ESTABLISHED, - I40IW_QHASH_MANAGE_TYPE_DELETE, - NULL, - false); - - if (apbvt_set && !i40iw_listen_port_in_use(&iwdev->cm_core, - cm_info.loc_port)) - i40iw_manage_apbvt(iwdev, - cm_info.loc_port, - I40IW_MANAGE_APBVT_DEL); - cm_id->rem_ref(cm_id); - iwdev->cm_core.stats_connect_errs++; - return -ENOMEM; - } + if (!cm_node) + goto err; i40iw_record_ird_ord(cm_node, (u16)conn_param->ird, (u16)conn_param->ord); if (cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO && @@ -3852,12 +3816,54 @@ int i40iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) cm_node->ord_size = 1; cm_node->apbvt_set = apbvt_set; - cm_node->qhash_set = true; + cm_node->qhash_set = qhash_set; iwqp->cm_node = cm_node; cm_node->iwqp = iwqp; iwqp->cm_id = cm_id; i40iw_add_ref(&iwqp->ibqp); + + if (cm_node->state == I40IW_CM_STATE_SYN_SENT) { + if (i40iw_send_syn(cm_node, 0)) { + i40iw_rem_ref_cm_node(cm_node); + goto err; + } + } + + i40iw_debug(cm_node->dev, + I40IW_DEBUG_CM, + "Api - connect(): port=0x%04x, cm_node=%p, cm_id = %p.\n", + cm_node->rem_port, + cm_node, + cm_node->cm_id); return 0; + +err: + if (cm_node) { + if (cm_node->ipv4) + i40iw_debug(cm_node->dev, + I40IW_DEBUG_CM, + "Api - connect() FAILED: dest addr=%pI4", + cm_node->rem_addr); + else + i40iw_debug(cm_node->dev, I40IW_DEBUG_CM, + "Api - connect() FAILED: dest addr=%pI6", + cm_node->rem_addr); + } + i40iw_manage_qhash(iwdev, + &cm_info, + I40IW_QHASH_TYPE_TCP_ESTABLISHED, + I40IW_QHASH_MANAGE_TYPE_DELETE, + NULL, + false); + + if (apbvt_set && !i40iw_listen_port_in_use(&iwdev->cm_core, + cm_info.loc_port)) + i40iw_manage_apbvt(iwdev, + cm_info.loc_port, + I40IW_MANAGE_APBVT_DEL); + cm_id->rem_ref(cm_id); + iwdev->cm_core.stats_connect_errs++; + return -ENOMEM; } /** diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.h b/drivers/infiniband/hw/i40iw/i40iw_cm.h index 5f8ceb4a8..e9046d9f9 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_cm.h +++ b/drivers/infiniband/hw/i40iw/i40iw_cm.h @@ -1,6 +1,6 @@ /******************************************************************************* * -* Copyright (c) 2015 Intel Corporation. All rights reserved. +* Copyright (c) 2015-2016 Intel Corporation. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -291,8 +291,6 @@ struct i40iw_cm_listener { u8 loc_mac[ETH_ALEN]; u32 loc_addr[4]; u16 loc_port; - u32 map_loc_addr[4]; - u16 map_loc_port; struct iw_cm_id *cm_id; atomic_t ref_count; struct i40iw_device *iwdev; @@ -317,8 +315,6 @@ struct i40iw_kmem_info { struct i40iw_cm_node { u32 loc_addr[4], rem_addr[4]; u16 loc_port, rem_port; - u32 map_loc_addr[4], map_rem_addr[4]; - u16 map_loc_port, map_rem_port; u16 vlan_id; enum i40iw_cm_node_state state; u8 loc_mac[ETH_ALEN]; @@ -370,10 +366,6 @@ struct i40iw_cm_info { u16 rem_port; u32 loc_addr[4]; u32 rem_addr[4]; - u16 map_loc_port; - u16 map_rem_port; - u32 map_loc_addr[4]; - u32 map_rem_addr[4]; u16 vlan_id; int backlog; u16 user_pri; diff --git a/drivers/infiniband/hw/i40iw/i40iw_ctrl.c b/drivers/infiniband/hw/i40iw/i40iw_ctrl.c index f05802bf6..2c4b4d072 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_ctrl.c +++ b/drivers/infiniband/hw/i40iw/i40iw_ctrl.c @@ -114,16 +114,21 @@ static enum i40iw_status_code i40iw_cqp_poll_registers( * i40iw_sc_parse_fpm_commit_buf - parse fpm commit buffer * @buf: ptr to fpm commit buffer * @info: ptr to i40iw_hmc_obj_info struct + * @sd: number of SDs for HMC objects * * parses fpm commit info and copy base value * of hmc objects in hmc_info */ static enum i40iw_status_code i40iw_sc_parse_fpm_commit_buf( u64 *buf, - struct i40iw_hmc_obj_info *info) + struct i40iw_hmc_obj_info *info, + u32 *sd) { u64 temp; + u64 size; + u64 base = 0; u32 i, j; + u32 k = 0; u32 low; /* copy base values in obj_info */ @@ -131,10 +136,20 @@ static enum i40iw_status_code i40iw_sc_parse_fpm_commit_buf( i <= I40IW_HMC_IW_PBLE; i++, j += 8) { get_64bit_val(buf, j, &temp); info[i].base = RS_64_1(temp, 32) * 512; + if (info[i].base > base) { + base = info[i].base; + k = i; + } low = (u32)(temp); if (low) info[i].cnt = low; } + size = info[k].cnt * info[k].size + info[k].base; + if (size & 0x1FFFFF) + *sd = (u32)((size >> 21) + 1); /* add 1 for remainder */ + else + *sd = (u32)(size >> 21); + return 0; } @@ -2909,6 +2924,65 @@ static enum i40iw_status_code i40iw_sc_mw_alloc( } /** + * i40iw_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp + * @qp: sc qp struct + * @info: fast mr info + * @post_sq: flag for cqp db to ring + */ +enum i40iw_status_code i40iw_sc_mr_fast_register( + struct i40iw_sc_qp *qp, + struct i40iw_fast_reg_stag_info *info, + bool post_sq) +{ + u64 temp, header; + u64 *wqe; + u32 wqe_idx; + + wqe = i40iw_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx, I40IW_QP_WQE_MIN_SIZE, + 0, info->wr_id); + if (!wqe) + return I40IW_ERR_QP_TOOMANY_WRS_POSTED; + + i40iw_debug(qp->dev, I40IW_DEBUG_MR, "%s: wr_id[%llxh] wqe_idx[%04d] location[%p]\n", + __func__, info->wr_id, wqe_idx, + &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid); + temp = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? (uintptr_t)info->va : info->fbo; + set_64bit_val(wqe, 0, temp); + + temp = RS_64(info->first_pm_pbl_index >> 16, I40IWQPSQ_FIRSTPMPBLIDXHI); + set_64bit_val(wqe, + 8, + LS_64(temp, I40IWQPSQ_FIRSTPMPBLIDXHI) | + LS_64(info->reg_addr_pa >> I40IWQPSQ_PBLADDR_SHIFT, I40IWQPSQ_PBLADDR)); + + set_64bit_val(wqe, + 16, + info->total_len | + LS_64(info->first_pm_pbl_index, I40IWQPSQ_FIRSTPMPBLIDXLO)); + + header = LS_64(info->stag_key, I40IWQPSQ_STAGKEY) | + LS_64(info->stag_idx, I40IWQPSQ_STAGINDEX) | + LS_64(I40IWQP_OP_FAST_REGISTER, I40IWQPSQ_OPCODE) | + LS_64(info->chunk_size, I40IWQPSQ_LPBLSIZE) | + LS_64(info->page_size, I40IWQPSQ_HPAGESIZE) | + LS_64(info->access_rights, I40IWQPSQ_STAGRIGHTS) | + LS_64(info->addr_type, I40IWQPSQ_VABASEDTO) | + LS_64(info->read_fence, I40IWQPSQ_READFENCE) | + LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) | + LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) | + LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID); + + i40iw_insert_wqe_hdr(wqe, header); + + i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "FAST_REG WQE", + wqe, I40IW_QP_WQE_MIN_SIZE); + + if (post_sq) + i40iw_qp_post_wr(&qp->qp_uk); + return 0; +} + +/** * i40iw_sc_send_lsmm - send last streaming mode message * @qp: sc qp struct * @lsmm_buf: buffer with lsmm message @@ -3147,7 +3221,7 @@ enum i40iw_status_code i40iw_sc_init_iw_hmc(struct i40iw_sc_dev *dev, u8 hmc_fn_ i40iw_cqp_commit_fpm_values_cmd(dev, &query_fpm_mem, hmc_fn_id); /* parse the fpm_commit_buf and fill hmc obj info */ - i40iw_sc_parse_fpm_commit_buf((u64 *)query_fpm_mem.va, hmc_info->hmc_obj); + i40iw_sc_parse_fpm_commit_buf((u64 *)query_fpm_mem.va, hmc_info->hmc_obj, &hmc_info->sd_table.sd_cnt); mem_size = sizeof(struct i40iw_hmc_sd_entry) * (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index); ret_code = i40iw_allocate_virt_mem(dev->hw, &virt_mem, mem_size); @@ -3221,7 +3295,9 @@ static enum i40iw_status_code i40iw_sc_configure_iw_fpm(struct i40iw_sc_dev *dev /* parse the fpm_commit_buf and fill hmc obj info */ if (!ret_code) - ret_code = i40iw_sc_parse_fpm_commit_buf(dev->fpm_commit_buf, hmc_info->hmc_obj); + ret_code = i40iw_sc_parse_fpm_commit_buf(dev->fpm_commit_buf, + hmc_info->hmc_obj, + &hmc_info->sd_table.sd_cnt); i40iw_debug_buf(dev, I40IW_DEBUG_HMC, "COMMIT FPM BUFFER", commit_fpm_mem.va, I40IW_COMMIT_FPM_BUF_SIZE); @@ -3469,6 +3545,40 @@ static bool i40iw_ring_full(struct i40iw_sc_cqp *cqp) } /** + * i40iw_est_sd - returns approximate number of SDs for HMC + * @dev: sc device struct + * @hmc_info: hmc structure, size and count for HMC objects + */ +static u64 i40iw_est_sd(struct i40iw_sc_dev *dev, struct i40iw_hmc_info *hmc_info) +{ + int i; + u64 size = 0; + u64 sd; + + for (i = I40IW_HMC_IW_QP; i < I40IW_HMC_IW_PBLE; i++) + size += hmc_info->hmc_obj[i].cnt * hmc_info->hmc_obj[i].size; + + if (dev->is_pf) + size += hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt * hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].size; + + if (size & 0x1FFFFF) + sd = (size >> 21) + 1; /* add 1 for remainder */ + else + sd = size >> 21; + + if (!dev->is_pf) { + /* 2MB alignment for VF PBLE HMC */ + size = hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt * hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].size; + if (size & 0x1FFFFF) + sd += (size >> 21) + 1; /* add 1 for remainder */ + else + sd += size >> 21; + } + + return sd; +} + +/** * i40iw_config_fpm_values - configure HMC objects * @dev: sc device struct * @qp_count: desired qp count @@ -3479,7 +3589,7 @@ enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_ u32 i, mem_size; u32 qpwantedoriginal, qpwanted, mrwanted, pblewanted; u32 powerof2; - u64 sd_needed, bytes_needed; + u64 sd_needed; u32 loop_count = 0; struct i40iw_hmc_info *hmc_info; @@ -3497,23 +3607,15 @@ enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_ return ret_code; } - bytes_needed = 0; - for (i = I40IW_HMC_IW_QP; i < I40IW_HMC_IW_MAX; i++) { + for (i = I40IW_HMC_IW_QP; i < I40IW_HMC_IW_MAX; i++) hmc_info->hmc_obj[i].cnt = hmc_info->hmc_obj[i].max_cnt; - bytes_needed += - (hmc_info->hmc_obj[i].max_cnt) * (hmc_info->hmc_obj[i].size); - i40iw_debug(dev, I40IW_DEBUG_HMC, - "%s i[%04d] max_cnt[0x%04X] size[0x%04llx]\n", - __func__, i, hmc_info->hmc_obj[i].max_cnt, - hmc_info->hmc_obj[i].size); - } - sd_needed = (bytes_needed / I40IW_HMC_DIRECT_BP_SIZE) + 1; /* round up */ + sd_needed = i40iw_est_sd(dev, hmc_info); i40iw_debug(dev, I40IW_DEBUG_HMC, "%s: FW initial max sd_count[%08lld] first_sd_index[%04d]\n", __func__, sd_needed, hmc_info->first_sd_index); i40iw_debug(dev, I40IW_DEBUG_HMC, - "%s: bytes_needed=0x%llx sd count %d where max sd is %d\n", - __func__, bytes_needed, hmc_info->sd_table.sd_cnt, + "%s: sd count %d where max sd is %d\n", + __func__, hmc_info->sd_table.sd_cnt, hmc_fpm_misc->max_sds); qpwanted = min(qp_count, hmc_info->hmc_obj[I40IW_HMC_IW_QP].max_cnt); @@ -3555,11 +3657,7 @@ enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_ hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt = pblewanted; /* How much memory is needed for all the objects. */ - bytes_needed = 0; - for (i = I40IW_HMC_IW_QP; i < I40IW_HMC_IW_MAX; i++) - bytes_needed += - (hmc_info->hmc_obj[i].cnt) * (hmc_info->hmc_obj[i].size); - sd_needed = (bytes_needed / I40IW_HMC_DIRECT_BP_SIZE) + 1; + sd_needed = i40iw_est_sd(dev, hmc_info); if ((loop_count > 1000) || ((!(loop_count % 10)) && (qpwanted > qpwantedoriginal * 2 / 3))) { @@ -3580,15 +3678,7 @@ enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_ pblewanted -= FPM_MULTIPLIER * 1000; } while (sd_needed > hmc_fpm_misc->max_sds && loop_count < 2000); - bytes_needed = 0; - for (i = I40IW_HMC_IW_QP; i < I40IW_HMC_IW_MAX; i++) { - bytes_needed += (hmc_info->hmc_obj[i].cnt) * (hmc_info->hmc_obj[i].size); - i40iw_debug(dev, I40IW_DEBUG_HMC, - "%s i[%04d] cnt[0x%04x] size[0x%04llx]\n", - __func__, i, hmc_info->hmc_obj[i].cnt, - hmc_info->hmc_obj[i].size); - } - sd_needed = (bytes_needed / I40IW_HMC_DIRECT_BP_SIZE) + 1; /* round up not truncate. */ + sd_needed = i40iw_est_sd(dev, hmc_info); i40iw_debug(dev, I40IW_DEBUG_HMC, "loop_cnt=%d, sd_needed=%lld, qpcnt = %d, cqcnt=%d, mrcnt=%d, pblecnt=%d\n", @@ -3606,8 +3696,6 @@ enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_ return ret_code; } - hmc_info->sd_table.sd_cnt = (u32)sd_needed; - mem_size = sizeof(struct i40iw_hmc_sd_entry) * (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index + 1); ret_code = i40iw_allocate_virt_mem(dev->hw, &virt_mem, mem_size); @@ -3911,11 +3999,11 @@ enum i40iw_status_code i40iw_process_bh(struct i40iw_sc_dev *dev) */ static u32 i40iw_iwarp_opcode(struct i40iw_aeqe_info *info, u8 *pkt) { - u16 *mpa; + __be16 *mpa; u32 opcode = 0xffffffff; if (info->q2_data_written) { - mpa = (u16 *)pkt; + mpa = (__be16 *)pkt; opcode = ntohs(mpa[1]) & 0xf; } return opcode; @@ -3977,7 +4065,7 @@ static int i40iw_bld_terminate_hdr(struct i40iw_sc_qp *qp, if (info->q2_data_written) { /* Use data from offending packet to fill in ddp & rdma hdrs */ pkt = i40iw_locate_mpa(pkt); - ddp_seg_len = ntohs(*(u16 *)pkt); + ddp_seg_len = ntohs(*(__be16 *)pkt); if (ddp_seg_len) { copy_len = 2; termhdr->hdrct = DDP_LEN_FLAG; @@ -4188,13 +4276,13 @@ void i40iw_terminate_connection(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info * void i40iw_terminate_received(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info) { u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET; - u32 *mpa; + __be32 *mpa; u8 ddp_ctl; u8 rdma_ctl; u16 aeq_id = 0; struct i40iw_terminate_hdr *termhdr; - mpa = (u32 *)i40iw_locate_mpa(pkt); + mpa = (__be32 *)i40iw_locate_mpa(pkt); if (info->q2_data_written) { /* did not validate the frame - do it now */ ddp_ctl = (ntohl(mpa[0]) >> 8) & 0xff; @@ -4559,17 +4647,18 @@ static struct i40iw_pd_ops iw_pd_ops = { }; static struct i40iw_priv_qp_ops iw_priv_qp_ops = { - i40iw_sc_qp_init, - i40iw_sc_qp_create, - i40iw_sc_qp_modify, - i40iw_sc_qp_destroy, - i40iw_sc_qp_flush_wqes, - i40iw_sc_qp_upload_context, - i40iw_sc_qp_setctx, - i40iw_sc_send_lsmm, - i40iw_sc_send_lsmm_nostag, - i40iw_sc_send_rtt, - i40iw_sc_post_wqe0, + .qp_init = i40iw_sc_qp_init, + .qp_create = i40iw_sc_qp_create, + .qp_modify = i40iw_sc_qp_modify, + .qp_destroy = i40iw_sc_qp_destroy, + .qp_flush_wqes = i40iw_sc_qp_flush_wqes, + .qp_upload_context = i40iw_sc_qp_upload_context, + .qp_setctx = i40iw_sc_qp_setctx, + .qp_send_lsmm = i40iw_sc_send_lsmm, + .qp_send_lsmm_nostag = i40iw_sc_send_lsmm_nostag, + .qp_send_rtt = i40iw_sc_send_rtt, + .qp_post_wqe0 = i40iw_sc_post_wqe0, + .iw_mr_fast_register = i40iw_sc_mr_fast_register }; static struct i40iw_priv_cq_ops iw_priv_cq_ops = { diff --git a/drivers/infiniband/hw/i40iw/i40iw_d.h b/drivers/infiniband/hw/i40iw/i40iw_d.h index aab88d65f..bd942da91 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_d.h +++ b/drivers/infiniband/hw/i40iw/i40iw_d.h @@ -1290,7 +1290,7 @@ /* wqe size considering 32 bytes per wqe*/ #define I40IWQP_SW_MIN_WQSIZE 4 /* 128 bytes */ -#define I40IWQP_SW_MAX_WQSIZE 16384 /* 524288 bytes */ +#define I40IWQP_SW_MAX_WQSIZE 2048 /* 2048 bytes */ #define I40IWQP_OP_RDMA_WRITE 0 #define I40IWQP_OP_RDMA_READ 1 @@ -1512,6 +1512,8 @@ enum i40iw_alignment { I40IW_SD_BUF_ALIGNMENT = 0x100 }; +#define I40IW_WQE_SIZE_64 64 + #define I40IW_QP_WQE_MIN_SIZE 32 #define I40IW_QP_WQE_MAX_SIZE 128 diff --git a/drivers/infiniband/hw/i40iw/i40iw_hw.c b/drivers/infiniband/hw/i40iw/i40iw_hw.c index 9fd302425..3ee0cad96 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_hw.c +++ b/drivers/infiniband/hw/i40iw/i40iw_hw.c @@ -106,7 +106,9 @@ u32 i40iw_initialize_hw_resources(struct i40iw_device *iwdev) set_bit(2, iwdev->allocated_pds); spin_lock_init(&iwdev->resource_lock); - mrdrvbits = 24 - get_count_order(iwdev->max_mr); + spin_lock_init(&iwdev->qptable_lock); + /* stag index mask has a minimum of 14 bits */ + mrdrvbits = 24 - max(get_count_order(iwdev->max_mr), 14); iwdev->mr_stagmask = ~(((1 << mrdrvbits) - 1) << (32 - mrdrvbits)); return 0; } @@ -301,11 +303,15 @@ void i40iw_process_aeq(struct i40iw_device *iwdev) "%s ae_id = 0x%x bool qp=%d qp_id = %d\n", __func__, info->ae_id, info->qp, info->qp_cq_id); if (info->qp) { + spin_lock_irqsave(&iwdev->qptable_lock, flags); iwqp = iwdev->qp_table[info->qp_cq_id]; if (!iwqp) { + spin_unlock_irqrestore(&iwdev->qptable_lock, flags); i40iw_pr_err("qp_id %d is already freed\n", info->qp_cq_id); continue; } + i40iw_add_ref(&iwqp->ibqp); + spin_unlock_irqrestore(&iwdev->qptable_lock, flags); qp = &iwqp->sc_qp; spin_lock_irqsave(&iwqp->lock, flags); iwqp->hw_tcp_state = info->tcp_state; @@ -411,6 +417,8 @@ void i40iw_process_aeq(struct i40iw_device *iwdev) i40iw_terminate_connection(qp, info); break; } + if (info->qp) + i40iw_rem_ref(&iwqp->ibqp); } while (1); if (aeqcnt) @@ -460,7 +468,7 @@ int i40iw_manage_apbvt(struct i40iw_device *iwdev, u16 accel_local_port, bool ad */ void i40iw_manage_arp_cache(struct i40iw_device *iwdev, unsigned char *mac_addr, - __be32 *ip_addr, + u32 *ip_addr, bool ipv4, u32 action) { @@ -481,7 +489,7 @@ void i40iw_manage_arp_cache(struct i40iw_device *iwdev, cqp_info->cqp_cmd = OP_ADD_ARP_CACHE_ENTRY; info = &cqp_info->in.u.add_arp_cache_entry.info; memset(info, 0, sizeof(*info)); - info->arp_index = cpu_to_le32(arp_index); + info->arp_index = cpu_to_le16((u16)arp_index); info->permanent = true; ether_addr_copy(info->mac_addr, mac_addr); cqp_info->in.u.add_arp_cache_entry.scratch = (uintptr_t)cqp_request; diff --git a/drivers/infiniband/hw/i40iw/i40iw_main.c b/drivers/infiniband/hw/i40iw/i40iw_main.c index 90e5af217..6e9081380 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_main.c +++ b/drivers/infiniband/hw/i40iw/i40iw_main.c @@ -270,7 +270,6 @@ static void i40iw_disable_irq(struct i40iw_sc_dev *dev, i40iw_wr32(dev->hw, I40E_PFINT_DYN_CTLN(msix_vec->idx - 1), 0); else i40iw_wr32(dev->hw, I40E_VFINT_DYN_CTLN1(msix_vec->idx - 1), 0); - synchronize_irq(msix_vec->irq); free_irq(msix_vec->irq, dev_id); } @@ -601,8 +600,7 @@ static enum i40iw_status_code i40iw_create_cqp(struct i40iw_device *iwdev) cqp_init_info.scratch_array = cqp->scratch_array; status = dev->cqp_ops->cqp_init(dev->cqp, &cqp_init_info); if (status) { - i40iw_pr_err("cqp init status %d maj_err %d min_err %d\n", - status, maj_err, min_err); + i40iw_pr_err("cqp init status %d\n", status); goto exit; } status = dev->cqp_ops->cqp_create(dev->cqp, true, &maj_err, &min_err); @@ -1147,10 +1145,7 @@ static enum i40iw_status_code i40iw_alloc_set_mac_ipaddr(struct i40iw_device *iw if (!status) { status = i40iw_add_mac_ipaddr_entry(iwdev, macaddr, (u8)iwdev->mac_ip_table_idx); - if (!status) - status = i40iw_add_mac_ipaddr_entry(iwdev, macaddr, - (u8)iwdev->mac_ip_table_idx); - else + if (status) i40iw_del_macip_entry(iwdev, (u8)iwdev->mac_ip_table_idx); } return status; @@ -1165,7 +1160,7 @@ static void i40iw_add_ipv6_addr(struct i40iw_device *iwdev) struct net_device *ip_dev; struct inet6_dev *idev; struct inet6_ifaddr *ifp; - __be32 local_ipaddr6[4]; + u32 local_ipaddr6[4]; rcu_read_lock(); for_each_netdev_rcu(&init_net, ip_dev) { @@ -1512,6 +1507,7 @@ static enum i40iw_status_code i40iw_setup_init_state(struct i40iw_handler *hdl, I40IW_HMC_PROFILE_DEFAULT; iwdev->max_rdma_vfs = (iwdev->resource_profile != I40IW_HMC_PROFILE_DEFAULT) ? max_rdma_vfs : 0; + iwdev->max_enabled_vfs = iwdev->max_rdma_vfs; iwdev->netdev = ldev->netdev; hdl->client = client; iwdev->mss = (!ldev->params.mtu) ? I40IW_DEFAULT_MSS : ldev->params.mtu - I40IW_MTU_TO_MSS; @@ -1531,7 +1527,10 @@ static enum i40iw_status_code i40iw_setup_init_state(struct i40iw_handler *hdl, goto exit; iwdev->obj_next = iwdev->obj_mem; iwdev->push_mode = push_mode; + init_waitqueue_head(&iwdev->vchnl_waitq); + init_waitqueue_head(&dev->vf_reqs); + status = i40iw_initialize_dev(iwdev, ldev); exit: if (status) { @@ -1710,7 +1709,6 @@ static void i40iw_vf_reset(struct i40e_info *ldev, struct i40e_client *client, u for (i = 0; i < I40IW_MAX_PE_ENABLED_VF_COUNT; i++) { if (!dev->vf_dev[i] || (dev->vf_dev[i]->vf_id != vf_id)) continue; - /* free all resources allocated on behalf of vf */ tmp_vfdev = dev->vf_dev[i]; spin_lock_irqsave(&dev->dev_pestat.stats_lock, flags); @@ -1819,8 +1817,6 @@ static int i40iw_virtchnl_receive(struct i40e_info *ldev, dev = &hdl->device.sc_dev; iwdev = dev->back_dev; - i40iw_debug(dev, I40IW_DEBUG_VIRT, "msg %p, message length %u\n", msg, len); - if (dev->vchnl_if.vchnl_recv) { ret_code = dev->vchnl_if.vchnl_recv(dev, vf_id, msg, len); if (!dev->is_pf) { @@ -1832,6 +1828,39 @@ static int i40iw_virtchnl_receive(struct i40e_info *ldev, } /** + * i40iw_vf_clear_to_send - wait to send virtual channel message + * @dev: iwarp device * + * Wait for until virtual channel is clear + * before sending the next message + * + * Returns false if error + * Returns true if clear to send + */ +bool i40iw_vf_clear_to_send(struct i40iw_sc_dev *dev) +{ + struct i40iw_device *iwdev; + wait_queue_t wait; + + iwdev = dev->back_dev; + + if (!wq_has_sleeper(&dev->vf_reqs) && + (atomic_read(&iwdev->vchnl_msgs) == 0)) + return true; /* virtual channel is clear */ + + init_wait(&wait); + add_wait_queue_exclusive(&dev->vf_reqs, &wait); + + if (!wait_event_timeout(dev->vf_reqs, + (atomic_read(&iwdev->vchnl_msgs) == 0), + I40IW_VCHNL_EVENT_TIMEOUT)) + dev->vchnl_up = false; + + remove_wait_queue(&dev->vf_reqs, &wait); + + return dev->vchnl_up; +} + +/** * i40iw_virtchnl_send - send a message through the virtual channel * @dev: iwarp device * @vf_id: virtual function id associated with the message @@ -1848,22 +1877,20 @@ static enum i40iw_status_code i40iw_virtchnl_send(struct i40iw_sc_dev *dev, { struct i40iw_device *iwdev; struct i40e_info *ldev; - enum i40iw_status_code ret_code = I40IW_ERR_BAD_PTR; if (!dev || !dev->back_dev) - return ret_code; + return I40IW_ERR_BAD_PTR; iwdev = dev->back_dev; ldev = iwdev->ldev; if (ldev && ldev->ops && ldev->ops->virtchnl_send) - ret_code = ldev->ops->virtchnl_send(ldev, &i40iw_client, vf_id, msg, len); - - return ret_code; + return ldev->ops->virtchnl_send(ldev, &i40iw_client, vf_id, msg, len); + return I40IW_ERR_BAD_PTR; } /* client interface functions */ -static struct i40e_client_ops i40e_ops = { +static const struct i40e_client_ops i40e_ops = { .open = i40iw_open, .close = i40iw_close, .l2_param_change = i40iw_l2param_change, diff --git a/drivers/infiniband/hw/i40iw/i40iw_osdep.h b/drivers/infiniband/hw/i40iw/i40iw_osdep.h index 7e2049351..80f422bf3 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_osdep.h +++ b/drivers/infiniband/hw/i40iw/i40iw_osdep.h @@ -172,6 +172,7 @@ struct i40iw_hw; u8 __iomem *i40iw_get_hw_addr(void *dev); void i40iw_ieq_mpa_crc_ae(struct i40iw_sc_dev *dev, struct i40iw_sc_qp *qp); enum i40iw_status_code i40iw_vf_wait_vchnl_resp(struct i40iw_sc_dev *dev); +bool i40iw_vf_clear_to_send(struct i40iw_sc_dev *dev); enum i40iw_status_code i40iw_ieq_check_mpacrc(struct shash_desc *desc, void *addr, u32 length, u32 value); struct i40iw_sc_qp *i40iw_ieq_get_qp(struct i40iw_sc_dev *dev, struct i40iw_puda_buf *buf); diff --git a/drivers/infiniband/hw/i40iw/i40iw_pble.c b/drivers/infiniband/hw/i40iw/i40iw_pble.c index ded853d2f..85993dc44 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_pble.c +++ b/drivers/infiniband/hw/i40iw/i40iw_pble.c @@ -404,13 +404,14 @@ static enum i40iw_status_code add_pble_pool(struct i40iw_sc_dev *dev, sd_entry->u.pd_table.pd_page_addr.pa : sd_entry->u.bp.addr.pa; if (sd_entry->valid) return 0; - if (dev->is_pf) + if (dev->is_pf) { ret_code = i40iw_hmc_sd_one(dev, hmc_info->hmc_fn_id, sd_reg_val, idx->sd_idx, sd_entry->entry_type, true); - if (ret_code) { - i40iw_pr_err("cqp cmd failed for sd (pbles)\n"); - goto error; + if (ret_code) { + i40iw_pr_err("cqp cmd failed for sd (pbles)\n"); + goto error; + } } sd_entry->valid = true; diff --git a/drivers/infiniband/hw/i40iw/i40iw_puda.c b/drivers/infiniband/hw/i40iw/i40iw_puda.c index 8eb400d8a..e9c6e82af 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_puda.c +++ b/drivers/infiniband/hw/i40iw/i40iw_puda.c @@ -1194,7 +1194,7 @@ static enum i40iw_status_code i40iw_ieq_process_buf(struct i40iw_puda_rsrc *ieq, ioffset = (u16)(buf->data - (u8 *)buf->mem.va); while (datalen) { - fpdu_len = i40iw_ieq_get_fpdu_length(ntohs(*(u16 *)datap)); + fpdu_len = i40iw_ieq_get_fpdu_length(ntohs(*(__be16 *)datap)); if (fpdu_len > pfpdu->max_fpdu_data) { i40iw_debug(ieq->dev, I40IW_DEBUG_IEQ, "%s: error bad fpdu_len\n", __func__); diff --git a/drivers/infiniband/hw/i40iw/i40iw_status.h b/drivers/infiniband/hw/i40iw/i40iw_status.h index b0110c15e..91c421762 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_status.h +++ b/drivers/infiniband/hw/i40iw/i40iw_status.h @@ -95,6 +95,7 @@ enum i40iw_status_code { I40IW_ERR_INVALID_MAC_ADDR = -65, I40IW_ERR_BAD_STAG = -66, I40IW_ERR_CQ_COMPL_ERROR = -67, + I40IW_ERR_QUEUE_DESTROYED = -68 }; #endif diff --git a/drivers/infiniband/hw/i40iw/i40iw_type.h b/drivers/infiniband/hw/i40iw/i40iw_type.h index edb3a8c82..16cc61720 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_type.h +++ b/drivers/infiniband/hw/i40iw/i40iw_type.h @@ -479,16 +479,17 @@ struct i40iw_sc_dev { struct i40iw_virt_mem ieq_mem; struct i40iw_puda_rsrc *ieq; - struct i40iw_vf_cqp_ops *iw_vf_cqp_ops; + const struct i40iw_vf_cqp_ops *iw_vf_cqp_ops; struct i40iw_hmc_fpm_misc hmc_fpm_misc; u16 qs_handle; - u32 debug_mask; + u32 debug_mask; u16 exception_lan_queue; u8 hmc_fn_id; bool is_pf; bool vchnl_up; u8 vf_id; + wait_queue_head_t vf_reqs; u64 cqp_cmd_stats[OP_SIZE_CQP_STAT_ARRAY]; struct i40iw_vchnl_vf_msg_buffer vchnl_vf_msg_buf; u8 hw_rev; @@ -889,8 +890,8 @@ struct i40iw_qhash_table_info { u32 qp_num; u32 dest_ip[4]; u32 src_ip[4]; - u32 dest_port; - u32 src_port; + u16 dest_port; + u16 src_port; }; struct i40iw_local_mac_ipaddr_entry_info { @@ -1040,6 +1041,9 @@ struct i40iw_priv_qp_ops { void (*qp_send_lsmm_nostag)(struct i40iw_sc_qp *, void *, u32); void (*qp_send_rtt)(struct i40iw_sc_qp *, bool); enum i40iw_status_code (*qp_post_wqe0)(struct i40iw_sc_qp *, u8); + enum i40iw_status_code (*iw_mr_fast_register)(struct i40iw_sc_qp *, + struct i40iw_fast_reg_stag_info *, + bool); }; struct i40iw_priv_cq_ops { @@ -1108,7 +1112,7 @@ struct i40iw_hmc_ops { enum i40iw_status_code (*parse_fpm_query_buf)(u64 *, struct i40iw_hmc_info *, struct i40iw_hmc_fpm_misc *); enum i40iw_status_code (*configure_iw_fpm)(struct i40iw_sc_dev *, u8); - enum i40iw_status_code (*parse_fpm_commit_buf)(u64 *, struct i40iw_hmc_obj_info *); + enum i40iw_status_code (*parse_fpm_commit_buf)(u64 *, struct i40iw_hmc_obj_info *, u32 *sd); enum i40iw_status_code (*create_hmc_object)(struct i40iw_sc_dev *dev, struct i40iw_hmc_create_obj_info *); enum i40iw_status_code (*del_hmc_object)(struct i40iw_sc_dev *dev, diff --git a/drivers/infiniband/hw/i40iw/i40iw_uk.c b/drivers/infiniband/hw/i40iw/i40iw_uk.c index f78c3dc8b..e35faea88 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_uk.c +++ b/drivers/infiniband/hw/i40iw/i40iw_uk.c @@ -56,6 +56,9 @@ static enum i40iw_status_code i40iw_nop_1(struct i40iw_qp_uk *qp) wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring); wqe = qp->sq_base[wqe_idx].elem; + + qp->sq_wrtrk_array[wqe_idx].wqe_size = I40IW_QP_WQE_MIN_SIZE; + peek_head = (qp->sq_ring.head + 1) % qp->sq_ring.size; wqe_0 = qp->sq_base[peek_head].elem; if (peek_head) @@ -130,7 +133,10 @@ static void i40iw_qp_ring_push_db(struct i40iw_qp_uk *qp, u32 wqe_idx) */ u64 *i40iw_qp_get_next_send_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx, - u8 wqe_size) + u8 wqe_size, + u32 total_size, + u64 wr_id + ) { u64 *wqe = NULL; u64 wqe_ptr; @@ -159,6 +165,17 @@ u64 *i40iw_qp_get_next_send_wqe(struct i40iw_qp_uk *qp, if (!*wqe_idx) qp->swqe_polarity = !qp->swqe_polarity; } + + if (((*wqe_idx & 3) == 1) && (wqe_size == I40IW_WQE_SIZE_64)) { + i40iw_nop_1(qp); + I40IW_RING_MOVE_HEAD(qp->sq_ring, ret_code); + if (ret_code) + return NULL; + *wqe_idx = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring); + if (!*wqe_idx) + qp->swqe_polarity = !qp->swqe_polarity; + } + for (i = 0; i < wqe_size / I40IW_QP_WQE_MIN_SIZE; i++) { I40IW_RING_MOVE_HEAD(qp->sq_ring, ret_code); if (ret_code) @@ -169,8 +186,15 @@ u64 *i40iw_qp_get_next_send_wqe(struct i40iw_qp_uk *qp, peek_head = I40IW_RING_GETCURRENT_HEAD(qp->sq_ring); wqe_0 = qp->sq_base[peek_head].elem; - if (peek_head & 0x3) - wqe_0[3] = LS_64(!qp->swqe_polarity, I40IWQPSQ_VALID); + + if (((peek_head & 3) == 1) || ((peek_head & 3) == 3)) { + if (RS_64(wqe_0[3], I40IWQPSQ_VALID) != !qp->swqe_polarity) + wqe_0[3] = LS_64(!qp->swqe_polarity, I40IWQPSQ_VALID); + } + + qp->sq_wrtrk_array[*wqe_idx].wrid = wr_id; + qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size; + qp->sq_wrtrk_array[*wqe_idx].wqe_size = wqe_size; return wqe; } @@ -249,12 +273,9 @@ static enum i40iw_status_code i40iw_rdma_write(struct i40iw_qp_uk *qp, if (ret_code) return ret_code; - wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size); + wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, total_size, info->wr_id); if (!wqe) return I40IW_ERR_QP_TOOMANY_WRS_POSTED; - - qp->sq_wrtrk_array[wqe_idx].wrid = info->wr_id; - qp->sq_wrtrk_array[wqe_idx].wr_len = total_size; set_64bit_val(wqe, 16, LS_64(op_info->rem_addr.tag_off, I40IWQPSQ_FRAG_TO)); if (!op_info->rem_addr.stag) @@ -309,12 +330,9 @@ static enum i40iw_status_code i40iw_rdma_read(struct i40iw_qp_uk *qp, ret_code = i40iw_fragcnt_to_wqesize_sq(1, &wqe_size); if (ret_code) return ret_code; - wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size); + wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, op_info->lo_addr.len, info->wr_id); if (!wqe) return I40IW_ERR_QP_TOOMANY_WRS_POSTED; - - qp->sq_wrtrk_array[wqe_idx].wrid = info->wr_id; - qp->sq_wrtrk_array[wqe_idx].wr_len = op_info->lo_addr.len; local_fence |= info->local_fence; set_64bit_val(wqe, 16, LS_64(op_info->rem_addr.tag_off, I40IWQPSQ_FRAG_TO)); @@ -366,13 +384,11 @@ static enum i40iw_status_code i40iw_send(struct i40iw_qp_uk *qp, if (ret_code) return ret_code; - wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size); + wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, total_size, info->wr_id); if (!wqe) return I40IW_ERR_QP_TOOMANY_WRS_POSTED; read_fence |= info->read_fence; - qp->sq_wrtrk_array[wqe_idx].wrid = info->wr_id; - qp->sq_wrtrk_array[wqe_idx].wr_len = total_size; set_64bit_val(wqe, 16, 0); header = LS_64(stag_to_inv, I40IWQPSQ_REMSTAG) | LS_64(info->op_type, I40IWQPSQ_OPCODE) | @@ -427,13 +443,11 @@ static enum i40iw_status_code i40iw_inline_rdma_write(struct i40iw_qp_uk *qp, if (ret_code) return ret_code; - wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size); + wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, op_info->len, info->wr_id); if (!wqe) return I40IW_ERR_QP_TOOMANY_WRS_POSTED; read_fence |= info->read_fence; - qp->sq_wrtrk_array[wqe_idx].wrid = info->wr_id; - qp->sq_wrtrk_array[wqe_idx].wr_len = op_info->len; set_64bit_val(wqe, 16, LS_64(op_info->rem_addr.tag_off, I40IWQPSQ_FRAG_TO)); @@ -507,14 +521,11 @@ static enum i40iw_status_code i40iw_inline_send(struct i40iw_qp_uk *qp, if (ret_code) return ret_code; - wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size); + wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, wqe_size, op_info->len, info->wr_id); if (!wqe) return I40IW_ERR_QP_TOOMANY_WRS_POSTED; read_fence |= info->read_fence; - - qp->sq_wrtrk_array[wqe_idx].wrid = info->wr_id; - qp->sq_wrtrk_array[wqe_idx].wr_len = op_info->len; header = LS_64(stag_to_inv, I40IWQPSQ_REMSTAG) | LS_64(info->op_type, I40IWQPSQ_OPCODE) | LS_64(op_info->len, I40IWQPSQ_INLINEDATALEN) | @@ -574,12 +585,9 @@ static enum i40iw_status_code i40iw_stag_local_invalidate(struct i40iw_qp_uk *qp op_info = &info->op.inv_local_stag; local_fence = info->local_fence; - wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE); + wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE, 0, info->wr_id); if (!wqe) return I40IW_ERR_QP_TOOMANY_WRS_POSTED; - - qp->sq_wrtrk_array[wqe_idx].wrid = info->wr_id; - qp->sq_wrtrk_array[wqe_idx].wr_len = 0; set_64bit_val(wqe, 0, 0); set_64bit_val(wqe, 8, LS_64(op_info->target_stag, I40IWQPSQ_LOCSTAG)); @@ -619,12 +627,9 @@ static enum i40iw_status_code i40iw_mw_bind(struct i40iw_qp_uk *qp, op_info = &info->op.bind_window; local_fence |= info->local_fence; - wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE); + wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE, 0, info->wr_id); if (!wqe) return I40IW_ERR_QP_TOOMANY_WRS_POSTED; - - qp->sq_wrtrk_array[wqe_idx].wrid = info->wr_id; - qp->sq_wrtrk_array[wqe_idx].wr_len = 0; set_64bit_val(wqe, 0, (uintptr_t)op_info->va); set_64bit_val(wqe, 8, LS_64(op_info->mr_stag, I40IWQPSQ_PARENTMRSTAG) | @@ -760,7 +765,7 @@ static enum i40iw_status_code i40iw_cq_poll_completion(struct i40iw_cq_uk *cq, enum i40iw_status_code ret_code2 = 0; bool move_cq_head = true; u8 polarity; - u8 addl_frag_cnt, addl_wqes = 0; + u8 addl_wqes = 0; if (cq->avoid_mem_cflct) cqe = (u64 *)I40IW_GET_CURRENT_EXTENDED_CQ_ELEMENT(cq); @@ -797,6 +802,10 @@ static enum i40iw_status_code i40iw_cq_poll_completion(struct i40iw_cq_uk *cq, info->is_srq = (bool)RS_64(qword3, I40IWCQ_SRQ); qp = (struct i40iw_qp_uk *)(unsigned long)comp_ctx; + if (!qp) { + ret_code = I40IW_ERR_QUEUE_DESTROYED; + goto exit; + } wqe_idx = (u32)RS_64(qword3, I40IW_CQ_WQEIDX); info->qp_handle = (i40iw_qp_handle)(unsigned long)qp; @@ -827,11 +836,8 @@ static enum i40iw_status_code i40iw_cq_poll_completion(struct i40iw_cq_uk *cq, info->op_type = (u8)RS_64(qword3, I40IWCQ_OP); sw_wqe = qp->sq_base[wqe_idx].elem; get_64bit_val(sw_wqe, 24, &wqe_qword); - addl_frag_cnt = - (u8)RS_64(wqe_qword, I40IWQPSQ_ADDFRAGCNT); - i40iw_fragcnt_to_wqesize_sq(addl_frag_cnt + 1, &addl_wqes); - addl_wqes = (addl_wqes / I40IW_QP_WQE_MIN_SIZE); + addl_wqes = qp->sq_wrtrk_array[wqe_idx].wqe_size / I40IW_QP_WQE_MIN_SIZE; I40IW_RING_SET_TAIL(qp->sq_ring, (wqe_idx + addl_wqes)); } else { do { @@ -843,9 +849,7 @@ static enum i40iw_status_code i40iw_cq_poll_completion(struct i40iw_cq_uk *cq, get_64bit_val(sw_wqe, 24, &wqe_qword); op_type = (u8)RS_64(wqe_qword, I40IWQPSQ_OPCODE); info->op_type = op_type; - addl_frag_cnt = (u8)RS_64(wqe_qword, I40IWQPSQ_ADDFRAGCNT); - i40iw_fragcnt_to_wqesize_sq(addl_frag_cnt + 1, &addl_wqes); - addl_wqes = (addl_wqes / I40IW_QP_WQE_MIN_SIZE); + addl_wqes = qp->sq_wrtrk_array[tail].wqe_size / I40IW_QP_WQE_MIN_SIZE; I40IW_RING_SET_TAIL(qp->sq_ring, (tail + addl_wqes)); if (op_type != I40IWQP_OP_NOP) { info->wr_id = qp->sq_wrtrk_array[tail].wrid; @@ -859,6 +863,7 @@ static enum i40iw_status_code i40iw_cq_poll_completion(struct i40iw_cq_uk *cq, ret_code = 0; +exit: if (!ret_code && (info->comp_status == I40IW_COMPL_STATUS_FLUSHED)) if (pring && (I40IW_RING_MORE_WORK(*pring))) @@ -893,19 +898,21 @@ static enum i40iw_status_code i40iw_cq_poll_completion(struct i40iw_cq_uk *cq, * i40iw_get_wqe_shift - get shift count for maximum wqe size * @wqdepth: depth of wq required. * @sge: Maximum Scatter Gather Elements wqe + * @inline_data: Maximum inline data size * @shift: Returns the shift needed based on sge * - * Shift can be used to left shift the wqe size based on sge. - * If sge, == 1, shift =0 (wqe_size of 32 bytes), for sge=2 and 3, shift =1 - * (64 bytes wqes) and 2 otherwise (128 bytes wqe). + * Shift can be used to left shift the wqe size based on number of SGEs and inlind data size. + * For 1 SGE or inline data <= 16, shift = 0 (wqe size of 32 bytes). + * For 2 or 3 SGEs or inline data <= 48, shift = 1 (wqe size of 64 bytes). + * Shift of 2 otherwise (wqe size of 128 bytes). */ -enum i40iw_status_code i40iw_get_wqe_shift(u32 wqdepth, u8 sge, u8 *shift) +enum i40iw_status_code i40iw_get_wqe_shift(u32 wqdepth, u32 sge, u32 inline_data, u8 *shift) { u32 size; *shift = 0; - if (sge > 1) - *shift = (sge < 4) ? 1 : 2; + if (sge > 1 || inline_data > 16) + *shift = (sge < 4 && inline_data <= 48) ? 1 : 2; /* check if wqdepth is multiple of 2 or not */ @@ -968,11 +975,11 @@ enum i40iw_status_code i40iw_qp_uk_init(struct i40iw_qp_uk *qp, if (info->max_rq_frag_cnt > I40IW_MAX_WQ_FRAGMENT_COUNT) return I40IW_ERR_INVALID_FRAG_COUNT; - ret_code = i40iw_get_wqe_shift(info->sq_size, info->max_sq_frag_cnt, &sqshift); + ret_code = i40iw_get_wqe_shift(info->sq_size, info->max_sq_frag_cnt, info->max_inline_data, &sqshift); if (ret_code) return ret_code; - ret_code = i40iw_get_wqe_shift(info->rq_size, info->max_rq_frag_cnt, &rqshift); + ret_code = i40iw_get_wqe_shift(info->rq_size, info->max_rq_frag_cnt, 0, &rqshift); if (ret_code) return ret_code; @@ -1097,12 +1104,9 @@ enum i40iw_status_code i40iw_nop(struct i40iw_qp_uk *qp, u64 header, *wqe; u32 wqe_idx; - wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE); + wqe = i40iw_qp_get_next_send_wqe(qp, &wqe_idx, I40IW_QP_WQE_MIN_SIZE, 0, wr_id); if (!wqe) return I40IW_ERR_QP_TOOMANY_WRS_POSTED; - - qp->sq_wrtrk_array[wqe_idx].wrid = wr_id; - qp->sq_wrtrk_array[wqe_idx].wr_len = 0; set_64bit_val(wqe, 0, 0); set_64bit_val(wqe, 8, 0); set_64bit_val(wqe, 16, 0); @@ -1125,7 +1129,7 @@ enum i40iw_status_code i40iw_nop(struct i40iw_qp_uk *qp, * @frag_cnt: number of fragments * @wqe_size: size of sq wqe returned */ -enum i40iw_status_code i40iw_fragcnt_to_wqesize_sq(u8 frag_cnt, u8 *wqe_size) +enum i40iw_status_code i40iw_fragcnt_to_wqesize_sq(u32 frag_cnt, u8 *wqe_size) { switch (frag_cnt) { case 0: @@ -1156,7 +1160,7 @@ enum i40iw_status_code i40iw_fragcnt_to_wqesize_sq(u8 frag_cnt, u8 *wqe_size) * @frag_cnt: number of fragments * @wqe_size: size of rq wqe returned */ -enum i40iw_status_code i40iw_fragcnt_to_wqesize_rq(u8 frag_cnt, u8 *wqe_size) +enum i40iw_status_code i40iw_fragcnt_to_wqesize_rq(u32 frag_cnt, u8 *wqe_size) { switch (frag_cnt) { case 0: diff --git a/drivers/infiniband/hw/i40iw/i40iw_user.h b/drivers/infiniband/hw/i40iw/i40iw_user.h index 5cd971bb8..4627646fe 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_user.h +++ b/drivers/infiniband/hw/i40iw/i40iw_user.h @@ -61,7 +61,7 @@ enum i40iw_device_capabilities_const { I40IW_MAX_CQ_SIZE = 1048575, I40IW_MAX_AEQ_ALLOCATE_COUNT = 255, I40IW_DB_ID_ZERO = 0, - I40IW_MAX_WQ_FRAGMENT_COUNT = 6, + I40IW_MAX_WQ_FRAGMENT_COUNT = 3, I40IW_MAX_SGE_RD = 1, I40IW_MAX_OUTBOUND_MESSAGE_SIZE = 2147483647, I40IW_MAX_INBOUND_MESSAGE_SIZE = 2147483647, @@ -70,8 +70,8 @@ enum i40iw_device_capabilities_const { I40IW_MAX_VF_FPM_ID = 47, I40IW_MAX_VF_PER_PF = 127, I40IW_MAX_SQ_PAYLOAD_SIZE = 2145386496, - I40IW_MAX_INLINE_DATA_SIZE = 112, - I40IW_MAX_PUSHMODE_INLINE_DATA_SIZE = 112, + I40IW_MAX_INLINE_DATA_SIZE = 48, + I40IW_MAX_PUSHMODE_INLINE_DATA_SIZE = 48, I40IW_MAX_IRD_SIZE = 32, I40IW_QPCTX_ENCD_MAXIRD = 3, I40IW_MAX_WQ_ENTRIES = 2048, @@ -102,6 +102,8 @@ enum i40iw_device_capabilities_const { #define I40IW_STAG_INDEX_FROM_STAG(stag) (((stag) && 0xFFFFFF00) >> 8) +#define I40IW_MAX_MR_SIZE 0x10000000000L + struct i40iw_qp_uk; struct i40iw_cq_uk; struct i40iw_srq_uk; @@ -198,7 +200,7 @@ enum i40iw_completion_notify { struct i40iw_post_send { i40iw_sgl sg_list; - u8 num_sges; + u32 num_sges; }; struct i40iw_post_inline_send { @@ -220,7 +222,7 @@ struct i40iw_post_inline_send_w_inv { struct i40iw_rdma_write { i40iw_sgl lo_sg_list; - u8 num_lo_sges; + u32 num_lo_sges; struct i40iw_sge rem_addr; }; @@ -345,7 +347,9 @@ struct i40iw_dev_uk { struct i40iw_sq_uk_wr_trk_info { u64 wrid; - u64 wr_len; + u32 wr_len; + u8 wqe_size; + u8 reserved[3]; }; struct i40iw_qp_quanta { @@ -367,6 +371,8 @@ struct i40iw_qp_uk { u32 qp_id; u32 sq_size; u32 rq_size; + u32 max_sq_frag_cnt; + u32 max_rq_frag_cnt; struct i40iw_qp_uk_ops ops; bool use_srq; u8 swqe_polarity; @@ -374,8 +380,6 @@ struct i40iw_qp_uk { u8 rwqe_polarity; u8 rq_wqe_size; u8 rq_wqe_size_multiplier; - u8 max_sq_frag_cnt; - u8 max_rq_frag_cnt; bool deferred_flag; }; @@ -404,8 +408,9 @@ struct i40iw_qp_uk_init_info { u32 qp_id; u32 sq_size; u32 rq_size; - u8 max_sq_frag_cnt; - u8 max_rq_frag_cnt; + u32 max_sq_frag_cnt; + u32 max_rq_frag_cnt; + u32 max_inline_data; }; @@ -422,7 +427,10 @@ void i40iw_device_init_uk(struct i40iw_dev_uk *dev); void i40iw_qp_post_wr(struct i40iw_qp_uk *qp); u64 *i40iw_qp_get_next_send_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx, - u8 wqe_size); + u8 wqe_size, + u32 total_size, + u64 wr_id + ); u64 *i40iw_qp_get_next_recv_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx); u64 *i40iw_qp_get_next_srq_wqe(struct i40iw_srq_uk *srq, u32 *wqe_idx); @@ -434,9 +442,9 @@ enum i40iw_status_code i40iw_qp_uk_init(struct i40iw_qp_uk *qp, void i40iw_clean_cq(void *queue, struct i40iw_cq_uk *cq); enum i40iw_status_code i40iw_nop(struct i40iw_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq); -enum i40iw_status_code i40iw_fragcnt_to_wqesize_sq(u8 frag_cnt, u8 *wqe_size); -enum i40iw_status_code i40iw_fragcnt_to_wqesize_rq(u8 frag_cnt, u8 *wqe_size); +enum i40iw_status_code i40iw_fragcnt_to_wqesize_sq(u32 frag_cnt, u8 *wqe_size); +enum i40iw_status_code i40iw_fragcnt_to_wqesize_rq(u32 frag_cnt, u8 *wqe_size); enum i40iw_status_code i40iw_inline_data_size_to_wqesize(u32 data_size, u8 *wqe_size); -enum i40iw_status_code i40iw_get_wqe_shift(u32 wqdepth, u8 sge, u8 *shift); +enum i40iw_status_code i40iw_get_wqe_shift(u32 wqdepth, u32 sge, u32 inline_data, u8 *shift); #endif diff --git a/drivers/infiniband/hw/i40iw/i40iw_utils.c b/drivers/infiniband/hw/i40iw/i40iw_utils.c index 1ceec81bd..0e8db0a35 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_utils.c +++ b/drivers/infiniband/hw/i40iw/i40iw_utils.c @@ -59,7 +59,7 @@ * @action: modify, delete or add */ int i40iw_arp_table(struct i40iw_device *iwdev, - __be32 *ip_addr, + u32 *ip_addr, bool ipv4, u8 *mac_addr, u32 action) @@ -152,7 +152,7 @@ int i40iw_inetaddr_event(struct notifier_block *notifier, struct net_device *upper_dev; struct i40iw_device *iwdev; struct i40iw_handler *hdl; - __be32 local_ipaddr; + u32 local_ipaddr; hdl = i40iw_find_netdev(event_netdev); if (!hdl) @@ -167,11 +167,10 @@ int i40iw_inetaddr_event(struct notifier_block *notifier, switch (event) { case NETDEV_DOWN: if (upper_dev) - local_ipaddr = - ((struct in_device *)upper_dev->ip_ptr)->ifa_list->ifa_address; + local_ipaddr = ntohl( + ((struct in_device *)upper_dev->ip_ptr)->ifa_list->ifa_address); else - local_ipaddr = ifa->ifa_address; - local_ipaddr = ntohl(local_ipaddr); + local_ipaddr = ntohl(ifa->ifa_address); i40iw_manage_arp_cache(iwdev, netdev->dev_addr, &local_ipaddr, @@ -180,11 +179,10 @@ int i40iw_inetaddr_event(struct notifier_block *notifier, return NOTIFY_OK; case NETDEV_UP: if (upper_dev) - local_ipaddr = - ((struct in_device *)upper_dev->ip_ptr)->ifa_list->ifa_address; + local_ipaddr = ntohl( + ((struct in_device *)upper_dev->ip_ptr)->ifa_list->ifa_address); else - local_ipaddr = ifa->ifa_address; - local_ipaddr = ntohl(local_ipaddr); + local_ipaddr = ntohl(ifa->ifa_address); i40iw_manage_arp_cache(iwdev, netdev->dev_addr, &local_ipaddr, @@ -194,12 +192,11 @@ int i40iw_inetaddr_event(struct notifier_block *notifier, case NETDEV_CHANGEADDR: /* Add the address to the IP table */ if (upper_dev) - local_ipaddr = - ((struct in_device *)upper_dev->ip_ptr)->ifa_list->ifa_address; + local_ipaddr = ntohl( + ((struct in_device *)upper_dev->ip_ptr)->ifa_list->ifa_address); else - local_ipaddr = ifa->ifa_address; + local_ipaddr = ntohl(ifa->ifa_address); - local_ipaddr = ntohl(local_ipaddr); i40iw_manage_arp_cache(iwdev, netdev->dev_addr, &local_ipaddr, @@ -227,7 +224,7 @@ int i40iw_inet6addr_event(struct notifier_block *notifier, struct net_device *netdev; struct i40iw_device *iwdev; struct i40iw_handler *hdl; - __be32 local_ipaddr6[4]; + u32 local_ipaddr6[4]; hdl = i40iw_find_netdev(event_netdev); if (!hdl) @@ -506,14 +503,19 @@ void i40iw_rem_ref(struct ib_qp *ibqp) struct cqp_commands_info *cqp_info; struct i40iw_device *iwdev; u32 qp_num; + unsigned long flags; iwqp = to_iwqp(ibqp); - if (!atomic_dec_and_test(&iwqp->refcount)) + iwdev = iwqp->iwdev; + spin_lock_irqsave(&iwdev->qptable_lock, flags); + if (!atomic_dec_and_test(&iwqp->refcount)) { + spin_unlock_irqrestore(&iwdev->qptable_lock, flags); return; + } - iwdev = iwqp->iwdev; qp_num = iwqp->ibqp.qp_num; iwdev->qp_table[qp_num] = NULL; + spin_unlock_irqrestore(&iwdev->qptable_lock, flags); cqp_request = i40iw_get_cqp_request(&iwdev->cqp, false); if (!cqp_request) return; @@ -985,21 +987,24 @@ enum i40iw_status_code i40iw_cqp_commit_fpm_values_cmd(struct i40iw_sc_dev *dev, enum i40iw_status_code i40iw_vf_wait_vchnl_resp(struct i40iw_sc_dev *dev) { struct i40iw_device *iwdev = dev->back_dev; - enum i40iw_status_code err_code = 0; int timeout_ret; i40iw_debug(dev, I40IW_DEBUG_VIRT, "%s[%u] dev %p, iwdev %p\n", __func__, __LINE__, dev, iwdev); - atomic_add(2, &iwdev->vchnl_msgs); + + atomic_set(&iwdev->vchnl_msgs, 2); timeout_ret = wait_event_timeout(iwdev->vchnl_waitq, (atomic_read(&iwdev->vchnl_msgs) == 1), I40IW_VCHNL_EVENT_TIMEOUT); atomic_dec(&iwdev->vchnl_msgs); if (!timeout_ret) { i40iw_pr_err("virt channel completion timeout = 0x%x\n", timeout_ret); - err_code = I40IW_ERR_TIMEOUT; + atomic_set(&iwdev->vchnl_msgs, 0); + dev->vchnl_up = false; + return I40IW_ERR_TIMEOUT; } - return err_code; + wake_up(&dev->vf_reqs); + return 0; } /** diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c index 1fe3b84a0..283b64c94 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c +++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c @@ -63,8 +63,8 @@ static int i40iw_query_device(struct ib_device *ibdev, ether_addr_copy((u8 *)&props->sys_image_guid, iwdev->netdev->dev_addr); props->fw_ver = I40IW_FW_VERSION; props->device_cap_flags = iwdev->device_cap_flags; - props->vendor_id = iwdev->vendor_id; - props->vendor_part_id = iwdev->vendor_part_id; + props->vendor_id = iwdev->ldev->pcidev->vendor; + props->vendor_part_id = iwdev->ldev->pcidev->device; props->hw_ver = (u32)iwdev->sc_dev.hw_rev; props->max_mr_size = I40IW_MAX_OUTBOUND_MESSAGE_SIZE; props->max_qp = iwdev->max_qp; @@ -74,11 +74,12 @@ static int i40iw_query_device(struct ib_device *ibdev, props->max_cqe = iwdev->max_cqe; props->max_mr = iwdev->max_mr; props->max_pd = iwdev->max_pd; - props->max_sge_rd = 1; + props->max_sge_rd = I40IW_MAX_SGE_RD; props->max_qp_rd_atom = I40IW_MAX_IRD_SIZE; props->max_qp_init_rd_atom = props->max_qp_rd_atom; props->atomic_cap = IB_ATOMIC_NONE; props->max_map_per_fmr = 1; + props->max_fast_reg_page_list_len = I40IW_MAX_PAGES_PER_FMR; return 0; } @@ -120,7 +121,7 @@ static int i40iw_query_port(struct ib_device *ibdev, props->pkey_tbl_len = 1; props->active_width = IB_WIDTH_4X; props->active_speed = 1; - props->max_msg_sz = 0x80000000; + props->max_msg_sz = I40IW_MAX_OUTBOUND_MESSAGE_SIZE; return 0; } @@ -437,7 +438,6 @@ void i40iw_free_qp_resources(struct i40iw_device *iwdev, kfree(iwqp->kqp.wrid_mem); iwqp->kqp.wrid_mem = NULL; kfree(iwqp->allocated_buffer); - iwqp->allocated_buffer = NULL; } /** @@ -521,14 +521,12 @@ static int i40iw_setup_kmode_qp(struct i40iw_device *iwdev, enum i40iw_status_code status; struct i40iw_qp_uk_init_info *ukinfo = &info->qp_uk_init_info; - ukinfo->max_sq_frag_cnt = I40IW_MAX_WQ_FRAGMENT_COUNT; - sq_size = i40iw_qp_roundup(ukinfo->sq_size + 1); rq_size = i40iw_qp_roundup(ukinfo->rq_size + 1); - status = i40iw_get_wqe_shift(sq_size, ukinfo->max_sq_frag_cnt, &sqshift); + status = i40iw_get_wqe_shift(sq_size, ukinfo->max_sq_frag_cnt, ukinfo->max_inline_data, &sqshift); if (!status) - status = i40iw_get_wqe_shift(rq_size, ukinfo->max_rq_frag_cnt, &rqshift); + status = i40iw_get_wqe_shift(rq_size, ukinfo->max_rq_frag_cnt, 0, &rqshift); if (status) return -ENOSYS; @@ -609,6 +607,9 @@ static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd, if (init_attr->cap.max_inline_data > I40IW_MAX_INLINE_DATA_SIZE) init_attr->cap.max_inline_data = I40IW_MAX_INLINE_DATA_SIZE; + if (init_attr->cap.max_send_sge > I40IW_MAX_WQ_FRAGMENT_COUNT) + init_attr->cap.max_send_sge = I40IW_MAX_WQ_FRAGMENT_COUNT; + memset(&init_info, 0, sizeof(init_info)); sq_size = init_attr->cap.max_send_wr; @@ -618,6 +619,7 @@ static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd, init_info.qp_uk_init_info.rq_size = rq_size; init_info.qp_uk_init_info.max_sq_frag_cnt = init_attr->cap.max_send_sge; init_info.qp_uk_init_info.max_rq_frag_cnt = init_attr->cap.max_recv_sge; + init_info.qp_uk_init_info.max_inline_data = init_attr->cap.max_inline_data; mem = kzalloc(sizeof(*iwqp), GFP_KERNEL); if (!mem) @@ -722,8 +724,10 @@ static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd, iwarp_info = &iwqp->iwarp_info; iwarp_info->rd_enable = true; iwarp_info->wr_rdresp_en = true; - if (!iwqp->user_mode) + if (!iwqp->user_mode) { + iwarp_info->fast_reg_en = true; iwarp_info->priv_mode_en = true; + } iwarp_info->ddp_ver = 1; iwarp_info->rdmap_ver = 1; @@ -784,6 +788,8 @@ static struct ib_qp *i40iw_create_qp(struct ib_pd *ibpd, return ERR_PTR(err_code); } } + init_completion(&iwqp->sq_drained); + init_completion(&iwqp->rq_drained); return &iwqp->ibqp; error: @@ -1444,6 +1450,167 @@ static int i40iw_handle_q_mem(struct i40iw_device *iwdev, } /** + * i40iw_hw_alloc_stag - cqp command to allocate stag + * @iwdev: iwarp device + * @iwmr: iwarp mr pointer + */ +static int i40iw_hw_alloc_stag(struct i40iw_device *iwdev, struct i40iw_mr *iwmr) +{ + struct i40iw_allocate_stag_info *info; + struct i40iw_pd *iwpd = to_iwpd(iwmr->ibmr.pd); + enum i40iw_status_code status; + int err = 0; + struct i40iw_cqp_request *cqp_request; + struct cqp_commands_info *cqp_info; + + cqp_request = i40iw_get_cqp_request(&iwdev->cqp, true); + if (!cqp_request) + return -ENOMEM; + + cqp_info = &cqp_request->info; + info = &cqp_info->in.u.alloc_stag.info; + memset(info, 0, sizeof(*info)); + info->page_size = PAGE_SIZE; + info->stag_idx = iwmr->stag >> I40IW_CQPSQ_STAG_IDX_SHIFT; + info->pd_id = iwpd->sc_pd.pd_id; + info->total_len = iwmr->length; + info->remote_access = true; + cqp_info->cqp_cmd = OP_ALLOC_STAG; + cqp_info->post_sq = 1; + cqp_info->in.u.alloc_stag.dev = &iwdev->sc_dev; + cqp_info->in.u.alloc_stag.scratch = (uintptr_t)cqp_request; + + status = i40iw_handle_cqp_op(iwdev, cqp_request); + if (status) { + err = -ENOMEM; + i40iw_pr_err("CQP-OP MR Reg fail"); + } + return err; +} + +/** + * i40iw_alloc_mr - register stag for fast memory registration + * @pd: ibpd pointer + * @mr_type: memory for stag registrion + * @max_num_sg: man number of pages + */ +static struct ib_mr *i40iw_alloc_mr(struct ib_pd *pd, + enum ib_mr_type mr_type, + u32 max_num_sg) +{ + struct i40iw_pd *iwpd = to_iwpd(pd); + struct i40iw_device *iwdev = to_iwdev(pd->device); + struct i40iw_pble_alloc *palloc; + struct i40iw_pbl *iwpbl; + struct i40iw_mr *iwmr; + enum i40iw_status_code status; + u32 stag; + int err_code = -ENOMEM; + + iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL); + if (!iwmr) + return ERR_PTR(-ENOMEM); + + stag = i40iw_create_stag(iwdev); + if (!stag) { + err_code = -EOVERFLOW; + goto err; + } + iwmr->stag = stag; + iwmr->ibmr.rkey = stag; + iwmr->ibmr.lkey = stag; + iwmr->ibmr.pd = pd; + iwmr->ibmr.device = pd->device; + iwpbl = &iwmr->iwpbl; + iwpbl->iwmr = iwmr; + iwmr->type = IW_MEMREG_TYPE_MEM; + palloc = &iwpbl->pble_alloc; + iwmr->page_cnt = max_num_sg; + mutex_lock(&iwdev->pbl_mutex); + status = i40iw_get_pble(&iwdev->sc_dev, iwdev->pble_rsrc, palloc, iwmr->page_cnt); + mutex_unlock(&iwdev->pbl_mutex); + if (status) + goto err1; + + if (palloc->level != I40IW_LEVEL_1) + goto err2; + err_code = i40iw_hw_alloc_stag(iwdev, iwmr); + if (err_code) + goto err2; + iwpbl->pbl_allocated = true; + i40iw_add_pdusecount(iwpd); + return &iwmr->ibmr; +err2: + i40iw_free_pble(iwdev->pble_rsrc, palloc); +err1: + i40iw_free_stag(iwdev, stag); +err: + kfree(iwmr); + return ERR_PTR(err_code); +} + +/** + * i40iw_set_page - populate pbl list for fmr + * @ibmr: ib mem to access iwarp mr pointer + * @addr: page dma address fro pbl list + */ +static int i40iw_set_page(struct ib_mr *ibmr, u64 addr) +{ + struct i40iw_mr *iwmr = to_iwmr(ibmr); + struct i40iw_pbl *iwpbl = &iwmr->iwpbl; + struct i40iw_pble_alloc *palloc = &iwpbl->pble_alloc; + u64 *pbl; + + if (unlikely(iwmr->npages == iwmr->page_cnt)) + return -ENOMEM; + + pbl = (u64 *)palloc->level1.addr; + pbl[iwmr->npages++] = cpu_to_le64(addr); + return 0; +} + +/** + * i40iw_map_mr_sg - map of sg list for fmr + * @ibmr: ib mem to access iwarp mr pointer + * @sg: scatter gather list for fmr + * @sg_nents: number of sg pages + */ +static int i40iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, + int sg_nents, unsigned int *sg_offset) +{ + struct i40iw_mr *iwmr = to_iwmr(ibmr); + + iwmr->npages = 0; + return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, i40iw_set_page); +} + +/** + * i40iw_drain_sq - drain the send queue + * @ibqp: ib qp pointer + */ +static void i40iw_drain_sq(struct ib_qp *ibqp) +{ + struct i40iw_qp *iwqp = to_iwqp(ibqp); + struct i40iw_sc_qp *qp = &iwqp->sc_qp; + + if (I40IW_RING_MORE_WORK(qp->qp_uk.sq_ring)) + wait_for_completion(&iwqp->sq_drained); +} + +/** + * i40iw_drain_rq - drain the receive queue + * @ibqp: ib qp pointer + */ +static void i40iw_drain_rq(struct ib_qp *ibqp) +{ + struct i40iw_qp *iwqp = to_iwqp(ibqp); + struct i40iw_sc_qp *qp = &iwqp->sc_qp; + + if (I40IW_RING_MORE_WORK(qp->qp_uk.rq_ring)) + wait_for_completion(&iwqp->rq_drained); +} + +/** * i40iw_hwreg_mr - send cqp command for memory registration * @iwdev: iwarp device * @iwmr: iwarp mr pointer @@ -1526,14 +1693,16 @@ static struct ib_mr *i40iw_reg_user_mr(struct ib_pd *pd, struct i40iw_mr *iwmr; struct ib_umem *region; struct i40iw_mem_reg_req req; - u32 pbl_depth = 0; + u64 pbl_depth = 0; u32 stag = 0; u16 access; - u32 region_length; + u64 region_length; bool use_pbles = false; unsigned long flags; int err = -ENOSYS; + if (length > I40IW_MAX_MR_SIZE) + return ERR_PTR(-EINVAL); region = ib_umem_get(pd->uobject->context, start, length, acc, 0); if (IS_ERR(region)) return (struct ib_mr *)region; @@ -1564,7 +1733,7 @@ static struct ib_mr *i40iw_reg_user_mr(struct ib_pd *pd, palloc = &iwpbl->pble_alloc; iwmr->type = req.reg_type; - iwmr->page_cnt = pbl_depth; + iwmr->page_cnt = (u32)pbl_depth; switch (req.reg_type) { case IW_MEMREG_TYPE_QP: @@ -1881,12 +2050,14 @@ static int i40iw_post_send(struct ib_qp *ibqp, enum i40iw_status_code ret; int err = 0; unsigned long flags; + bool inv_stag; iwqp = (struct i40iw_qp *)ibqp; ukqp = &iwqp->sc_qp.qp_uk; spin_lock_irqsave(&iwqp->lock, flags); while (ib_wr) { + inv_stag = false; memset(&info, 0, sizeof(info)); info.wr_id = (u64)(ib_wr->wr_id); if ((ib_wr->send_flags & IB_SEND_SIGNALED) || iwqp->sig_all) @@ -1896,19 +2067,28 @@ static int i40iw_post_send(struct ib_qp *ibqp, switch (ib_wr->opcode) { case IB_WR_SEND: - if (ib_wr->send_flags & IB_SEND_SOLICITED) - info.op_type = I40IW_OP_TYPE_SEND_SOL; - else - info.op_type = I40IW_OP_TYPE_SEND; + /* fall-through */ + case IB_WR_SEND_WITH_INV: + if (ib_wr->opcode == IB_WR_SEND) { + if (ib_wr->send_flags & IB_SEND_SOLICITED) + info.op_type = I40IW_OP_TYPE_SEND_SOL; + else + info.op_type = I40IW_OP_TYPE_SEND; + } else { + if (ib_wr->send_flags & IB_SEND_SOLICITED) + info.op_type = I40IW_OP_TYPE_SEND_SOL_INV; + else + info.op_type = I40IW_OP_TYPE_SEND_INV; + } if (ib_wr->send_flags & IB_SEND_INLINE) { info.op.inline_send.data = (void *)(unsigned long)ib_wr->sg_list[0].addr; info.op.inline_send.len = ib_wr->sg_list[0].length; - ret = ukqp->ops.iw_inline_send(ukqp, &info, rdma_wr(ib_wr)->rkey, false); + ret = ukqp->ops.iw_inline_send(ukqp, &info, ib_wr->ex.invalidate_rkey, false); } else { info.op.send.num_sges = ib_wr->num_sge; info.op.send.sg_list = (struct i40iw_sge *)ib_wr->sg_list; - ret = ukqp->ops.iw_send(ukqp, &info, rdma_wr(ib_wr)->rkey, false); + ret = ukqp->ops.iw_send(ukqp, &info, ib_wr->ex.invalidate_rkey, false); } if (ret) @@ -1936,7 +2116,14 @@ static int i40iw_post_send(struct ib_qp *ibqp, if (ret) err = -EIO; break; + case IB_WR_RDMA_READ_WITH_INV: + inv_stag = true; + /* fall-through*/ case IB_WR_RDMA_READ: + if (ib_wr->num_sge > I40IW_MAX_SGE_RD) { + err = -EINVAL; + break; + } info.op_type = I40IW_OP_TYPE_RDMA_READ; info.op.rdma_read.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr; info.op.rdma_read.rem_addr.stag = rdma_wr(ib_wr)->rkey; @@ -1944,10 +2131,52 @@ static int i40iw_post_send(struct ib_qp *ibqp, info.op.rdma_read.lo_addr.tag_off = ib_wr->sg_list->addr; info.op.rdma_read.lo_addr.stag = ib_wr->sg_list->lkey; info.op.rdma_read.lo_addr.len = ib_wr->sg_list->length; - ret = ukqp->ops.iw_rdma_read(ukqp, &info, false, false); + ret = ukqp->ops.iw_rdma_read(ukqp, &info, inv_stag, false); if (ret) err = -EIO; break; + case IB_WR_LOCAL_INV: + info.op_type = I40IW_OP_TYPE_INV_STAG; + info.op.inv_local_stag.target_stag = ib_wr->ex.invalidate_rkey; + ret = ukqp->ops.iw_stag_local_invalidate(ukqp, &info, true); + if (ret) + err = -EIO; + break; + case IB_WR_REG_MR: + { + struct i40iw_mr *iwmr = to_iwmr(reg_wr(ib_wr)->mr); + int page_shift = ilog2(reg_wr(ib_wr)->mr->page_size); + int flags = reg_wr(ib_wr)->access; + struct i40iw_pble_alloc *palloc = &iwmr->iwpbl.pble_alloc; + struct i40iw_sc_dev *dev = &iwqp->iwdev->sc_dev; + struct i40iw_fast_reg_stag_info info; + + memset(&info, 0, sizeof(info)); + info.access_rights = I40IW_ACCESS_FLAGS_LOCALREAD; + info.access_rights |= i40iw_get_user_access(flags); + info.stag_key = reg_wr(ib_wr)->key & 0xff; + info.stag_idx = reg_wr(ib_wr)->key >> 8; + info.wr_id = ib_wr->wr_id; + + info.addr_type = I40IW_ADDR_TYPE_VA_BASED; + info.va = (void *)(uintptr_t)iwmr->ibmr.iova; + info.total_len = iwmr->ibmr.length; + info.reg_addr_pa = *(u64 *)palloc->level1.addr; + info.first_pm_pbl_index = palloc->level1.idx; + info.local_fence = ib_wr->send_flags & IB_SEND_FENCE; + info.signaled = ib_wr->send_flags & IB_SEND_SIGNALED; + + if (iwmr->npages > I40IW_MIN_PAGES_PER_FMR) + info.chunk_size = 1; + + if (page_shift == 21) + info.page_size = 1; /* 2M page */ + + ret = dev->iw_priv_qp_ops->iw_mr_fast_register(&iwqp->sc_qp, &info, true); + if (ret) + err = -EIO; + break; + } default: err = -EINVAL; i40iw_pr_err(" upost_send bad opcode = 0x%x\n", @@ -2027,6 +2256,7 @@ static int i40iw_poll_cq(struct ib_cq *ibcq, enum i40iw_status_code ret; struct i40iw_cq_uk *ukcq; struct i40iw_sc_qp *qp; + struct i40iw_qp *iwqp; unsigned long flags; iwcq = (struct i40iw_cq *)ibcq; @@ -2037,6 +2267,8 @@ static int i40iw_poll_cq(struct ib_cq *ibcq, ret = ukcq->ops.iw_cq_poll_completion(ukcq, &cq_poll_info, true); if (ret == I40IW_ERR_QUEUE_EMPTY) { break; + } else if (ret == I40IW_ERR_QUEUE_DESTROYED) { + continue; } else if (ret) { if (!cqe_count) cqe_count = -1; @@ -2044,10 +2276,12 @@ static int i40iw_poll_cq(struct ib_cq *ibcq, } entry->wc_flags = 0; entry->wr_id = cq_poll_info.wr_id; - if (!cq_poll_info.error) - entry->status = IB_WC_SUCCESS; - else + if (cq_poll_info.error) { entry->status = IB_WC_WR_FLUSH_ERR; + entry->vendor_err = cq_poll_info.major_err << 16 | cq_poll_info.minor_err; + } else { + entry->status = IB_WC_SUCCESS; + } switch (cq_poll_info.op_type) { case I40IW_OP_TYPE_RDMA_WRITE: @@ -2071,12 +2305,17 @@ static int i40iw_poll_cq(struct ib_cq *ibcq, break; } - entry->vendor_err = - cq_poll_info.major_err << 16 | cq_poll_info.minor_err; entry->ex.imm_data = 0; qp = (struct i40iw_sc_qp *)cq_poll_info.qp_handle; entry->qp = (struct ib_qp *)qp->back_qp; entry->src_qp = cq_poll_info.qp_id; + iwqp = (struct i40iw_qp *)qp->back_qp; + if (iwqp->iwarp_state > I40IW_QP_STATE_RTS) { + if (!I40IW_RING_MORE_WORK(qp->qp_uk.sq_ring)) + complete(&iwqp->sq_drained); + if (!I40IW_RING_MORE_WORK(qp->qp_uk.rq_ring)) + complete(&iwqp->rq_drained); + } entry->byte_len = cq_poll_info.bytes_xfered; entry++; cqe_count++; @@ -2095,13 +2334,16 @@ static int i40iw_req_notify_cq(struct ib_cq *ibcq, { struct i40iw_cq *iwcq; struct i40iw_cq_uk *ukcq; - enum i40iw_completion_notify cq_notify = IW_CQ_COMPL_SOLICITED; + unsigned long flags; + enum i40iw_completion_notify cq_notify = IW_CQ_COMPL_EVENT; iwcq = (struct i40iw_cq *)ibcq; ukcq = &iwcq->sc_cq.cq_uk; - if (notify_flags == IB_CQ_NEXT_COMP) - cq_notify = IW_CQ_COMPL_EVENT; + if (notify_flags == IB_CQ_SOLICITED) + cq_notify = IW_CQ_COMPL_SOLICITED; + spin_lock_irqsave(&iwcq->lock, flags); ukcq->ops.iw_cq_request_notification(ukcq, cq_notify); + spin_unlock_irqrestore(&iwcq->lock, flags); return 0; } @@ -2129,62 +2371,130 @@ static int i40iw_port_immutable(struct ib_device *ibdev, u8 port_num, return 0; } +static const char * const i40iw_hw_stat_names[] = { + // 32bit names + [I40IW_HW_STAT_INDEX_IP4RXDISCARD] = "ip4InDiscards", + [I40IW_HW_STAT_INDEX_IP4RXTRUNC] = "ip4InTruncatedPkts", + [I40IW_HW_STAT_INDEX_IP4TXNOROUTE] = "ip4OutNoRoutes", + [I40IW_HW_STAT_INDEX_IP6RXDISCARD] = "ip6InDiscards", + [I40IW_HW_STAT_INDEX_IP6RXTRUNC] = "ip6InTruncatedPkts", + [I40IW_HW_STAT_INDEX_IP6TXNOROUTE] = "ip6OutNoRoutes", + [I40IW_HW_STAT_INDEX_TCPRTXSEG] = "tcpRetransSegs", + [I40IW_HW_STAT_INDEX_TCPRXOPTERR] = "tcpInOptErrors", + [I40IW_HW_STAT_INDEX_TCPRXPROTOERR] = "tcpInProtoErrors", + // 64bit names + [I40IW_HW_STAT_INDEX_IP4RXOCTS + I40IW_HW_STAT_INDEX_MAX_32] = + "ip4InOctets", + [I40IW_HW_STAT_INDEX_IP4RXPKTS + I40IW_HW_STAT_INDEX_MAX_32] = + "ip4InPkts", + [I40IW_HW_STAT_INDEX_IP4RXFRAGS + I40IW_HW_STAT_INDEX_MAX_32] = + "ip4InReasmRqd", + [I40IW_HW_STAT_INDEX_IP4RXMCPKTS + I40IW_HW_STAT_INDEX_MAX_32] = + "ip4InMcastPkts", + [I40IW_HW_STAT_INDEX_IP4TXOCTS + I40IW_HW_STAT_INDEX_MAX_32] = + "ip4OutOctets", + [I40IW_HW_STAT_INDEX_IP4TXPKTS + I40IW_HW_STAT_INDEX_MAX_32] = + "ip4OutPkts", + [I40IW_HW_STAT_INDEX_IP4TXFRAGS + I40IW_HW_STAT_INDEX_MAX_32] = + "ip4OutSegRqd", + [I40IW_HW_STAT_INDEX_IP4TXMCPKTS + I40IW_HW_STAT_INDEX_MAX_32] = + "ip4OutMcastPkts", + [I40IW_HW_STAT_INDEX_IP6RXOCTS + I40IW_HW_STAT_INDEX_MAX_32] = + "ip6InOctets", + [I40IW_HW_STAT_INDEX_IP6RXPKTS + I40IW_HW_STAT_INDEX_MAX_32] = + "ip6InPkts", + [I40IW_HW_STAT_INDEX_IP6RXFRAGS + I40IW_HW_STAT_INDEX_MAX_32] = + "ip6InReasmRqd", + [I40IW_HW_STAT_INDEX_IP6RXMCPKTS + I40IW_HW_STAT_INDEX_MAX_32] = + "ip6InMcastPkts", + [I40IW_HW_STAT_INDEX_IP6TXOCTS + I40IW_HW_STAT_INDEX_MAX_32] = + "ip6OutOctets", + [I40IW_HW_STAT_INDEX_IP6TXPKTS + I40IW_HW_STAT_INDEX_MAX_32] = + "ip6OutPkts", + [I40IW_HW_STAT_INDEX_IP6TXFRAGS + I40IW_HW_STAT_INDEX_MAX_32] = + "ip6OutSegRqd", + [I40IW_HW_STAT_INDEX_IP6TXMCPKTS + I40IW_HW_STAT_INDEX_MAX_32] = + "ip6OutMcastPkts", + [I40IW_HW_STAT_INDEX_TCPRXSEGS + I40IW_HW_STAT_INDEX_MAX_32] = + "tcpInSegs", + [I40IW_HW_STAT_INDEX_TCPTXSEG + I40IW_HW_STAT_INDEX_MAX_32] = + "tcpOutSegs", + [I40IW_HW_STAT_INDEX_RDMARXRDS + I40IW_HW_STAT_INDEX_MAX_32] = + "iwInRdmaReads", + [I40IW_HW_STAT_INDEX_RDMARXSNDS + I40IW_HW_STAT_INDEX_MAX_32] = + "iwInRdmaSends", + [I40IW_HW_STAT_INDEX_RDMARXWRS + I40IW_HW_STAT_INDEX_MAX_32] = + "iwInRdmaWrites", + [I40IW_HW_STAT_INDEX_RDMATXRDS + I40IW_HW_STAT_INDEX_MAX_32] = + "iwOutRdmaReads", + [I40IW_HW_STAT_INDEX_RDMATXSNDS + I40IW_HW_STAT_INDEX_MAX_32] = + "iwOutRdmaSends", + [I40IW_HW_STAT_INDEX_RDMATXWRS + I40IW_HW_STAT_INDEX_MAX_32] = + "iwOutRdmaWrites", + [I40IW_HW_STAT_INDEX_RDMAVBND + I40IW_HW_STAT_INDEX_MAX_32] = + "iwRdmaBnd", + [I40IW_HW_STAT_INDEX_RDMAVINV + I40IW_HW_STAT_INDEX_MAX_32] = + "iwRdmaInv" +}; + +/** + * i40iw_alloc_hw_stats - Allocate a hw stats structure + * @ibdev: device pointer from stack + * @port_num: port number + */ +static struct rdma_hw_stats *i40iw_alloc_hw_stats(struct ib_device *ibdev, + u8 port_num) +{ + struct i40iw_device *iwdev = to_iwdev(ibdev); + struct i40iw_sc_dev *dev = &iwdev->sc_dev; + int num_counters = I40IW_HW_STAT_INDEX_MAX_32 + + I40IW_HW_STAT_INDEX_MAX_64; + unsigned long lifespan = RDMA_HW_STATS_DEFAULT_LIFESPAN; + + BUILD_BUG_ON(ARRAY_SIZE(i40iw_hw_stat_names) != + (I40IW_HW_STAT_INDEX_MAX_32 + + I40IW_HW_STAT_INDEX_MAX_64)); + + /* + * PFs get the default update lifespan, but VFs only update once + * per second + */ + if (!dev->is_pf) + lifespan = 1000; + return rdma_alloc_hw_stats_struct(i40iw_hw_stat_names, num_counters, + lifespan); +} + /** - * i40iw_get_protocol_stats - Populates the rdma_stats structure - * @ibdev: ib dev struct - * @stats: iw protocol stats struct + * i40iw_get_hw_stats - Populates the rdma_hw_stats structure + * @ibdev: device pointer from stack + * @stats: stats pointer from stack + * @port_num: port number + * @index: which hw counter the stack is requesting we update */ -static int i40iw_get_protocol_stats(struct ib_device *ibdev, - union rdma_protocol_stats *stats) +static int i40iw_get_hw_stats(struct ib_device *ibdev, + struct rdma_hw_stats *stats, + u8 port_num, int index) { struct i40iw_device *iwdev = to_iwdev(ibdev); struct i40iw_sc_dev *dev = &iwdev->sc_dev; struct i40iw_dev_pestat *devstat = &dev->dev_pestat; struct i40iw_dev_hw_stats *hw_stats = &devstat->hw_stats; - struct timespec curr_time; - static struct timespec last_rd_time = {0, 0}; - enum i40iw_status_code status = 0; unsigned long flags; - curr_time = current_kernel_time(); - memset(stats, 0, sizeof(*stats)); - if (dev->is_pf) { spin_lock_irqsave(&devstat->stats_lock, flags); devstat->ops.iw_hw_stat_read_all(devstat, &devstat->hw_stats); spin_unlock_irqrestore(&devstat->stats_lock, flags); } else { - if (((u64)curr_time.tv_sec - (u64)last_rd_time.tv_sec) > 1) - status = i40iw_vchnl_vf_get_pe_stats(dev, - &devstat->hw_stats); - - if (status) + if (i40iw_vchnl_vf_get_pe_stats(dev, &devstat->hw_stats)) return -ENOSYS; } - stats->iw.ipInReceives = hw_stats->stat_value_64[I40IW_HW_STAT_INDEX_IP4RXPKTS] + - hw_stats->stat_value_64[I40IW_HW_STAT_INDEX_IP6RXPKTS]; - stats->iw.ipInTruncatedPkts = hw_stats->stat_value_32[I40IW_HW_STAT_INDEX_IP4RXTRUNC] + - hw_stats->stat_value_32[I40IW_HW_STAT_INDEX_IP6RXTRUNC]; - stats->iw.ipInDiscards = hw_stats->stat_value_32[I40IW_HW_STAT_INDEX_IP4RXDISCARD] + - hw_stats->stat_value_32[I40IW_HW_STAT_INDEX_IP6RXDISCARD]; - stats->iw.ipOutNoRoutes = hw_stats->stat_value_32[I40IW_HW_STAT_INDEX_IP4TXNOROUTE] + - hw_stats->stat_value_32[I40IW_HW_STAT_INDEX_IP6TXNOROUTE]; - stats->iw.ipReasmReqds = hw_stats->stat_value_64[I40IW_HW_STAT_INDEX_IP4RXFRAGS] + - hw_stats->stat_value_64[I40IW_HW_STAT_INDEX_IP6RXFRAGS]; - stats->iw.ipFragCreates = hw_stats->stat_value_64[I40IW_HW_STAT_INDEX_IP4TXFRAGS] + - hw_stats->stat_value_64[I40IW_HW_STAT_INDEX_IP6TXFRAGS]; - stats->iw.ipInMcastPkts = hw_stats->stat_value_64[I40IW_HW_STAT_INDEX_IP4RXMCPKTS] + - hw_stats->stat_value_64[I40IW_HW_STAT_INDEX_IP6RXMCPKTS]; - stats->iw.ipOutMcastPkts = hw_stats->stat_value_64[I40IW_HW_STAT_INDEX_IP4TXMCPKTS] + - hw_stats->stat_value_64[I40IW_HW_STAT_INDEX_IP6TXMCPKTS]; - stats->iw.tcpOutSegs = hw_stats->stat_value_64[I40IW_HW_STAT_INDEX_TCPTXSEG]; - stats->iw.tcpInSegs = hw_stats->stat_value_64[I40IW_HW_STAT_INDEX_TCPRXSEGS]; - stats->iw.tcpRetransSegs = hw_stats->stat_value_32[I40IW_HW_STAT_INDEX_TCPRTXSEG]; - - last_rd_time = curr_time; - return 0; + memcpy(&stats->value[0], &hw_stats, sizeof(*hw_stats)); + + return stats->num_counters; } /** @@ -2323,10 +2633,15 @@ static struct i40iw_ib_device *i40iw_init_rdma_device(struct i40iw_device *iwdev iwibdev->ibdev.get_dma_mr = i40iw_get_dma_mr; iwibdev->ibdev.reg_user_mr = i40iw_reg_user_mr; iwibdev->ibdev.dereg_mr = i40iw_dereg_mr; - iwibdev->ibdev.get_protocol_stats = i40iw_get_protocol_stats; + iwibdev->ibdev.alloc_hw_stats = i40iw_alloc_hw_stats; + iwibdev->ibdev.get_hw_stats = i40iw_get_hw_stats; iwibdev->ibdev.query_device = i40iw_query_device; iwibdev->ibdev.create_ah = i40iw_create_ah; iwibdev->ibdev.destroy_ah = i40iw_destroy_ah; + iwibdev->ibdev.drain_sq = i40iw_drain_sq; + iwibdev->ibdev.drain_rq = i40iw_drain_rq; + iwibdev->ibdev.alloc_mr = i40iw_alloc_mr; + iwibdev->ibdev.map_mr_sg = i40iw_map_mr_sg; iwibdev->ibdev.iwcm = kzalloc(sizeof(*iwibdev->ibdev.iwcm), GFP_KERNEL); if (!iwibdev->ibdev.iwcm) { ib_dealloc_device(&iwibdev->ibdev); diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.h b/drivers/infiniband/hw/i40iw/i40iw_verbs.h index 1101f7708..0069be8a5 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_verbs.h +++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.h @@ -92,6 +92,7 @@ struct i40iw_mr { struct ib_umem *region; u16 type; u32 page_cnt; + u32 npages; u32 stag; u64 length; u64 pgaddrmem[MAX_SAVE_PAGE_ADDRS]; @@ -169,5 +170,7 @@ struct i40iw_qp { struct i40iw_pbl *iwpbl; struct i40iw_dma_mem q2_ctx_mem; struct i40iw_dma_mem ietf_mem; + struct completion sq_drained; + struct completion rq_drained; }; #endif diff --git a/drivers/infiniband/hw/i40iw/i40iw_vf.c b/drivers/infiniband/hw/i40iw/i40iw_vf.c index cb0f18340..e33d48109 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_vf.c +++ b/drivers/infiniband/hw/i40iw/i40iw_vf.c @@ -80,6 +80,6 @@ enum i40iw_status_code i40iw_manage_vf_pble_bp(struct i40iw_sc_cqp *cqp, return 0; } -struct i40iw_vf_cqp_ops iw_vf_cqp_ops = { +const struct i40iw_vf_cqp_ops iw_vf_cqp_ops = { i40iw_manage_vf_pble_bp }; diff --git a/drivers/infiniband/hw/i40iw/i40iw_vf.h b/drivers/infiniband/hw/i40iw/i40iw_vf.h index f649f3a62..4359559ec 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_vf.h +++ b/drivers/infiniband/hw/i40iw/i40iw_vf.h @@ -57,6 +57,6 @@ enum i40iw_status_code i40iw_manage_vf_pble_bp(struct i40iw_sc_cqp *cqp, u64 scratch, bool post_sq); -extern struct i40iw_vf_cqp_ops iw_vf_cqp_ops; +extern const struct i40iw_vf_cqp_ops iw_vf_cqp_ops; #endif diff --git a/drivers/infiniband/hw/i40iw/i40iw_virtchnl.c b/drivers/infiniband/hw/i40iw/i40iw_virtchnl.c index 6b68f7890..3041003c9 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_virtchnl.c +++ b/drivers/infiniband/hw/i40iw/i40iw_virtchnl.c @@ -254,7 +254,7 @@ static void vchnl_pf_send_get_hmc_fcn_resp(struct i40iw_sc_dev *dev, static void vchnl_pf_send_get_pe_stats_resp(struct i40iw_sc_dev *dev, u32 vf_id, struct i40iw_virtchnl_op_buf *vchnl_msg, - struct i40iw_dev_hw_stats hw_stats) + struct i40iw_dev_hw_stats *hw_stats) { enum i40iw_status_code ret_code; u8 resp_buffer[sizeof(struct i40iw_virtchnl_resp_buf) + sizeof(struct i40iw_dev_hw_stats) - 1]; @@ -264,7 +264,7 @@ static void vchnl_pf_send_get_pe_stats_resp(struct i40iw_sc_dev *dev, vchnl_msg_resp->iw_chnl_op_ctx = vchnl_msg->iw_chnl_op_ctx; vchnl_msg_resp->iw_chnl_buf_len = sizeof(resp_buffer); vchnl_msg_resp->iw_op_ret_code = I40IW_SUCCESS; - *((struct i40iw_dev_hw_stats *)vchnl_msg_resp->iw_chnl_buf) = hw_stats; + *((struct i40iw_dev_hw_stats *)vchnl_msg_resp->iw_chnl_buf) = *hw_stats; ret_code = dev->vchnl_if.vchnl_send(dev, vf_id, resp_buffer, sizeof(resp_buffer)); if (ret_code) i40iw_debug(dev, I40IW_DEBUG_VIRT, @@ -437,11 +437,9 @@ enum i40iw_status_code i40iw_vchnl_recv_pf(struct i40iw_sc_dev *dev, vchnl_pf_send_get_ver_resp(dev, vf_id, vchnl_msg); return I40IW_SUCCESS; } - for (iw_vf_idx = 0; iw_vf_idx < I40IW_MAX_PE_ENABLED_VF_COUNT; - iw_vf_idx++) { + for (iw_vf_idx = 0; iw_vf_idx < I40IW_MAX_PE_ENABLED_VF_COUNT; iw_vf_idx++) { if (!dev->vf_dev[iw_vf_idx]) { - if (first_avail_iw_vf == - I40IW_MAX_PE_ENABLED_VF_COUNT) + if (first_avail_iw_vf == I40IW_MAX_PE_ENABLED_VF_COUNT) first_avail_iw_vf = iw_vf_idx; continue; } @@ -541,7 +539,7 @@ enum i40iw_status_code i40iw_vchnl_recv_pf(struct i40iw_sc_dev *dev, devstat->ops.iw_hw_stat_read_all(devstat, &devstat->hw_stats); spin_unlock_irqrestore(&dev->dev_pestat.stats_lock, flags); vf_dev->msg_count--; - vchnl_pf_send_get_pe_stats_resp(dev, vf_id, vchnl_msg, devstat->hw_stats); + vchnl_pf_send_get_pe_stats_resp(dev, vf_id, vchnl_msg, &devstat->hw_stats); break; default: i40iw_debug(dev, I40IW_DEBUG_VIRT, @@ -596,23 +594,25 @@ enum i40iw_status_code i40iw_vchnl_vf_get_ver(struct i40iw_sc_dev *dev, struct i40iw_virtchnl_req vchnl_req; enum i40iw_status_code ret_code; + if (!i40iw_vf_clear_to_send(dev)) + return I40IW_ERR_TIMEOUT; memset(&vchnl_req, 0, sizeof(vchnl_req)); vchnl_req.dev = dev; vchnl_req.parm = vchnl_ver; vchnl_req.parm_len = sizeof(*vchnl_ver); vchnl_req.vchnl_msg = &dev->vchnl_vf_msg_buf.vchnl_msg; + ret_code = vchnl_vf_send_get_ver_req(dev, &vchnl_req); - if (!ret_code) { - ret_code = i40iw_vf_wait_vchnl_resp(dev); - if (!ret_code) - ret_code = vchnl_req.ret_code; - else - dev->vchnl_up = false; - } else { + if (ret_code) { i40iw_debug(dev, I40IW_DEBUG_VIRT, "%s Send message failed 0x%0x\n", __func__, ret_code); + return ret_code; } - return ret_code; + ret_code = i40iw_vf_wait_vchnl_resp(dev); + if (ret_code) + return ret_code; + else + return vchnl_req.ret_code; } /** @@ -626,23 +626,25 @@ enum i40iw_status_code i40iw_vchnl_vf_get_hmc_fcn(struct i40iw_sc_dev *dev, struct i40iw_virtchnl_req vchnl_req; enum i40iw_status_code ret_code; + if (!i40iw_vf_clear_to_send(dev)) + return I40IW_ERR_TIMEOUT; memset(&vchnl_req, 0, sizeof(vchnl_req)); vchnl_req.dev = dev; vchnl_req.parm = hmc_fcn; vchnl_req.parm_len = sizeof(*hmc_fcn); vchnl_req.vchnl_msg = &dev->vchnl_vf_msg_buf.vchnl_msg; + ret_code = vchnl_vf_send_get_hmc_fcn_req(dev, &vchnl_req); - if (!ret_code) { - ret_code = i40iw_vf_wait_vchnl_resp(dev); - if (!ret_code) - ret_code = vchnl_req.ret_code; - else - dev->vchnl_up = false; - } else { + if (ret_code) { i40iw_debug(dev, I40IW_DEBUG_VIRT, "%s Send message failed 0x%0x\n", __func__, ret_code); + return ret_code; } - return ret_code; + ret_code = i40iw_vf_wait_vchnl_resp(dev); + if (ret_code) + return ret_code; + else + return vchnl_req.ret_code; } /** @@ -660,25 +662,27 @@ enum i40iw_status_code i40iw_vchnl_vf_add_hmc_objs(struct i40iw_sc_dev *dev, struct i40iw_virtchnl_req vchnl_req; enum i40iw_status_code ret_code; + if (!i40iw_vf_clear_to_send(dev)) + return I40IW_ERR_TIMEOUT; memset(&vchnl_req, 0, sizeof(vchnl_req)); vchnl_req.dev = dev; vchnl_req.vchnl_msg = &dev->vchnl_vf_msg_buf.vchnl_msg; + ret_code = vchnl_vf_send_add_hmc_objs_req(dev, &vchnl_req, rsrc_type, start_index, rsrc_count); - if (!ret_code) { - ret_code = i40iw_vf_wait_vchnl_resp(dev); - if (!ret_code) - ret_code = vchnl_req.ret_code; - else - dev->vchnl_up = false; - } else { + if (ret_code) { i40iw_debug(dev, I40IW_DEBUG_VIRT, "%s Send message failed 0x%0x\n", __func__, ret_code); + return ret_code; } - return ret_code; + ret_code = i40iw_vf_wait_vchnl_resp(dev); + if (ret_code) + return ret_code; + else + return vchnl_req.ret_code; } /** @@ -696,25 +700,27 @@ enum i40iw_status_code i40iw_vchnl_vf_del_hmc_obj(struct i40iw_sc_dev *dev, struct i40iw_virtchnl_req vchnl_req; enum i40iw_status_code ret_code; + if (!i40iw_vf_clear_to_send(dev)) + return I40IW_ERR_TIMEOUT; memset(&vchnl_req, 0, sizeof(vchnl_req)); vchnl_req.dev = dev; vchnl_req.vchnl_msg = &dev->vchnl_vf_msg_buf.vchnl_msg; + ret_code = vchnl_vf_send_del_hmc_objs_req(dev, &vchnl_req, rsrc_type, start_index, rsrc_count); - if (!ret_code) { - ret_code = i40iw_vf_wait_vchnl_resp(dev); - if (!ret_code) - ret_code = vchnl_req.ret_code; - else - dev->vchnl_up = false; - } else { + if (ret_code) { i40iw_debug(dev, I40IW_DEBUG_VIRT, "%s Send message failed 0x%0x\n", __func__, ret_code); + return ret_code; } - return ret_code; + ret_code = i40iw_vf_wait_vchnl_resp(dev); + if (ret_code) + return ret_code; + else + return vchnl_req.ret_code; } /** @@ -728,21 +734,23 @@ enum i40iw_status_code i40iw_vchnl_vf_get_pe_stats(struct i40iw_sc_dev *dev, struct i40iw_virtchnl_req vchnl_req; enum i40iw_status_code ret_code; + if (!i40iw_vf_clear_to_send(dev)) + return I40IW_ERR_TIMEOUT; memset(&vchnl_req, 0, sizeof(vchnl_req)); vchnl_req.dev = dev; vchnl_req.parm = hw_stats; vchnl_req.parm_len = sizeof(*hw_stats); vchnl_req.vchnl_msg = &dev->vchnl_vf_msg_buf.vchnl_msg; + ret_code = vchnl_vf_send_get_pe_stats_req(dev, &vchnl_req); - if (!ret_code) { - ret_code = i40iw_vf_wait_vchnl_resp(dev); - if (!ret_code) - ret_code = vchnl_req.ret_code; - else - dev->vchnl_up = false; - } else { + if (ret_code) { i40iw_debug(dev, I40IW_DEBUG_VIRT, "%s Send message failed 0x%0x\n", __func__, ret_code); + return ret_code; } - return ret_code; + ret_code = i40iw_vf_wait_vchnl_resp(dev); + if (ret_code) + return ret_code; + else + return vchnl_req.ret_code; } |