summaryrefslogtreecommitdiff
path: root/drivers/infiniband/hw/mlx5/mr.c
diff options
context:
space:
mode:
authorAndré Fabian Silva Delgado <emulatorman@parabola.nu>2016-06-10 05:30:17 -0300
committerAndré Fabian Silva Delgado <emulatorman@parabola.nu>2016-06-10 05:30:17 -0300
commitd635711daa98be86d4c7fd01499c34f566b54ccb (patch)
treeaa5cc3760a27c3d57146498cb82fa549547de06c /drivers/infiniband/hw/mlx5/mr.c
parentc91265cd0efb83778f015b4d4b1129bd2cfd075e (diff)
Linux-libre 4.6.2-gnu
Diffstat (limited to 'drivers/infiniband/hw/mlx5/mr.c')
-rw-r--r--drivers/infiniband/hw/mlx5/mr.c601
1 files changed, 471 insertions, 130 deletions
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index 6000f7aee..4d5bff151 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -40,6 +40,7 @@
#include <rdma/ib_umem_odp.h>
#include <rdma/ib_verbs.h>
#include "mlx5_ib.h"
+#include "user.h"
enum {
MAX_PENDING_REG_MR = 8,
@@ -57,7 +58,7 @@ static int clean_mr(struct mlx5_ib_mr *mr);
static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
{
- int err = mlx5_core_destroy_mkey(dev->mdev, &mr->mmr);
+ int err = mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey);
#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
/* Wait until all page fault handlers using the mr complete. */
@@ -77,6 +78,40 @@ static int order2idx(struct mlx5_ib_dev *dev, int order)
return order - cache->ent[0].order;
}
+static bool use_umr_mtt_update(struct mlx5_ib_mr *mr, u64 start, u64 length)
+{
+ return ((u64)1 << mr->order) * MLX5_ADAPTER_PAGE_SIZE >=
+ length + (start & (MLX5_ADAPTER_PAGE_SIZE - 1));
+}
+
+#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
+static void update_odp_mr(struct mlx5_ib_mr *mr)
+{
+ if (mr->umem->odp_data) {
+ /*
+ * This barrier prevents the compiler from moving the
+ * setting of umem->odp_data->private to point to our
+ * MR, before reg_umr finished, to ensure that the MR
+ * initialization have finished before starting to
+ * handle invalidations.
+ */
+ smp_wmb();
+ mr->umem->odp_data->private = mr;
+ /*
+ * Make sure we will see the new
+ * umem->odp_data->private value in the invalidation
+ * routines, before we can get page faults on the
+ * MR. Page faults can happen once we put the MR in
+ * the tree, below this line. Without the barrier,
+ * there can be a fault handling and an invalidation
+ * before umem->odp_data->private == mr is visible to
+ * the invalidation handler.
+ */
+ smp_wmb();
+ }
+}
+#endif
+
static void reg_mr_callback(int status, void *context)
{
struct mlx5_ib_mr *mr = context;
@@ -86,7 +121,7 @@ static void reg_mr_callback(int status, void *context)
struct mlx5_cache_ent *ent = &cache->ent[c];
u8 key;
unsigned long flags;
- struct mlx5_mr_table *table = &dev->mdev->priv.mr_table;
+ struct mlx5_mkey_table *table = &dev->mdev->priv.mkey_table;
int err;
spin_lock_irqsave(&ent->lock, flags);
@@ -113,7 +148,7 @@ static void reg_mr_callback(int status, void *context)
spin_lock_irqsave(&dev->mdev->priv.mkey_lock, flags);
key = dev->mdev->priv.mkey_key++;
spin_unlock_irqrestore(&dev->mdev->priv.mkey_lock, flags);
- mr->mmr.key = mlx5_idx_to_mkey(be32_to_cpu(mr->out.mkey) & 0xffffff) | key;
+ mr->mmkey.key = mlx5_idx_to_mkey(be32_to_cpu(mr->out.mkey) & 0xffffff) | key;
cache->last_add = jiffies;
@@ -124,10 +159,10 @@ static void reg_mr_callback(int status, void *context)
spin_unlock_irqrestore(&ent->lock, flags);
write_lock_irqsave(&table->lock, flags);
- err = radix_tree_insert(&table->tree, mlx5_base_mkey(mr->mmr.key),
- &mr->mmr);
+ err = radix_tree_insert(&table->tree, mlx5_base_mkey(mr->mmkey.key),
+ &mr->mmkey);
if (err)
- pr_err("Error inserting to mr tree. 0x%x\n", -err);
+ pr_err("Error inserting to mkey tree. 0x%x\n", -err);
write_unlock_irqrestore(&table->lock, flags);
}
@@ -168,7 +203,7 @@ static int add_keys(struct mlx5_ib_dev *dev, int c, int num)
spin_lock_irq(&ent->lock);
ent->pending++;
spin_unlock_irq(&ent->lock);
- err = mlx5_core_create_mkey(dev->mdev, &mr->mmr, in,
+ err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in,
sizeof(*in), reg_mr_callback,
mr, &mr->out);
if (err) {
@@ -657,14 +692,14 @@ struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc)
seg->qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
seg->start_addr = 0;
- err = mlx5_core_create_mkey(mdev, &mr->mmr, in, sizeof(*in), NULL, NULL,
+ err = mlx5_core_create_mkey(mdev, &mr->mmkey, in, sizeof(*in), NULL, NULL,
NULL);
if (err)
goto err_in;
kfree(in);
- mr->ibmr.lkey = mr->mmr.key;
- mr->ibmr.rkey = mr->mmr.key;
+ mr->ibmr.lkey = mr->mmkey.key;
+ mr->ibmr.rkey = mr->mmkey.key;
mr->umem = NULL;
return &mr->ibmr;
@@ -693,10 +728,40 @@ static int use_umr(int order)
return order <= MLX5_MAX_UMR_SHIFT;
}
-static void prep_umr_reg_wqe(struct ib_pd *pd, struct ib_send_wr *wr,
- struct ib_sge *sg, u64 dma, int n, u32 key,
- int page_shift, u64 virt_addr, u64 len,
- int access_flags)
+static int dma_map_mr_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
+ int npages, int page_shift, int *size,
+ __be64 **mr_pas, dma_addr_t *dma)
+{
+ __be64 *pas;
+ struct device *ddev = dev->ib_dev.dma_device;
+
+ /*
+ * UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes.
+ * To avoid copying garbage after the pas array, we allocate
+ * a little more.
+ */
+ *size = ALIGN(sizeof(u64) * npages, MLX5_UMR_MTT_ALIGNMENT);
+ *mr_pas = kmalloc(*size + MLX5_UMR_ALIGN - 1, GFP_KERNEL);
+ if (!(*mr_pas))
+ return -ENOMEM;
+
+ pas = PTR_ALIGN(*mr_pas, MLX5_UMR_ALIGN);
+ mlx5_ib_populate_pas(dev, umem, page_shift, pas, MLX5_IB_MTT_PRESENT);
+ /* Clear padding after the actual pages. */
+ memset(pas + npages, 0, *size - npages * sizeof(u64));
+
+ *dma = dma_map_single(ddev, pas, *size, DMA_TO_DEVICE);
+ if (dma_mapping_error(ddev, *dma)) {
+ kfree(*mr_pas);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static void prep_umr_wqe_common(struct ib_pd *pd, struct ib_send_wr *wr,
+ struct ib_sge *sg, u64 dma, int n, u32 key,
+ int page_shift)
{
struct mlx5_ib_dev *dev = to_mdev(pd->device);
struct mlx5_umr_wr *umrwr = umr_wr(wr);
@@ -706,7 +771,6 @@ static void prep_umr_reg_wqe(struct ib_pd *pd, struct ib_send_wr *wr,
sg->lkey = dev->umrc.pd->local_dma_lkey;
wr->next = NULL;
- wr->send_flags = 0;
wr->sg_list = sg;
if (n)
wr->num_sge = 1;
@@ -718,6 +782,19 @@ static void prep_umr_reg_wqe(struct ib_pd *pd, struct ib_send_wr *wr,
umrwr->npages = n;
umrwr->page_shift = page_shift;
umrwr->mkey = key;
+}
+
+static void prep_umr_reg_wqe(struct ib_pd *pd, struct ib_send_wr *wr,
+ struct ib_sge *sg, u64 dma, int n, u32 key,
+ int page_shift, u64 virt_addr, u64 len,
+ int access_flags)
+{
+ struct mlx5_umr_wr *umrwr = umr_wr(wr);
+
+ prep_umr_wqe_common(pd, wr, sg, dma, n, key, page_shift);
+
+ wr->send_flags = 0;
+
umrwr->target.virt_addr = virt_addr;
umrwr->length = len;
umrwr->access_flags = access_flags;
@@ -734,26 +811,45 @@ static void prep_umr_unreg_wqe(struct mlx5_ib_dev *dev,
umrwr->mkey = key;
}
-void mlx5_umr_cq_handler(struct ib_cq *cq, void *cq_context)
+static struct ib_umem *mr_umem_get(struct ib_pd *pd, u64 start, u64 length,
+ int access_flags, int *npages,
+ int *page_shift, int *ncont, int *order)
{
- struct mlx5_ib_umr_context *context;
- struct ib_wc wc;
- int err;
-
- while (1) {
- err = ib_poll_cq(cq, 1, &wc);
- if (err < 0) {
- pr_warn("poll cq error %d\n", err);
- return;
- }
- if (err == 0)
- break;
+ struct mlx5_ib_dev *dev = to_mdev(pd->device);
+ struct ib_umem *umem = ib_umem_get(pd->uobject->context, start, length,
+ access_flags, 0);
+ if (IS_ERR(umem)) {
+ mlx5_ib_err(dev, "umem get failed (%ld)\n", PTR_ERR(umem));
+ return (void *)umem;
+ }
- context = (struct mlx5_ib_umr_context *) (unsigned long) wc.wr_id;
- context->status = wc.status;
- complete(&context->done);
+ mlx5_ib_cont_pages(umem, start, npages, page_shift, ncont, order);
+ if (!*npages) {
+ mlx5_ib_warn(dev, "avoid zero region\n");
+ ib_umem_release(umem);
+ return ERR_PTR(-EINVAL);
}
- ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
+
+ mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n",
+ *npages, *ncont, *order, *page_shift);
+
+ return umem;
+}
+
+static void mlx5_ib_umr_done(struct ib_cq *cq, struct ib_wc *wc)
+{
+ struct mlx5_ib_umr_context *context =
+ container_of(wc->wr_cqe, struct mlx5_ib_umr_context, cqe);
+
+ context->status = wc->status;
+ complete(&context->done);
+}
+
+static inline void mlx5_ib_init_umr_context(struct mlx5_ib_umr_context *context)
+{
+ context->cqe.done = mlx5_ib_umr_done;
+ context->status = -1;
+ init_completion(&context->done);
}
static struct mlx5_ib_mr *reg_umr(struct ib_pd *pd, struct ib_umem *umem,
@@ -764,13 +860,12 @@ static struct mlx5_ib_mr *reg_umr(struct ib_pd *pd, struct ib_umem *umem,
struct device *ddev = dev->ib_dev.dma_device;
struct umr_common *umrc = &dev->umrc;
struct mlx5_ib_umr_context umr_context;
- struct mlx5_umr_wr umrwr;
+ struct mlx5_umr_wr umrwr = {};
struct ib_send_wr *bad;
struct mlx5_ib_mr *mr;
struct ib_sge sg;
int size;
__be64 *mr_pas;
- __be64 *pas;
dma_addr_t dma;
int err = 0;
int i;
@@ -790,33 +885,17 @@ static struct mlx5_ib_mr *reg_umr(struct ib_pd *pd, struct ib_umem *umem,
if (!mr)
return ERR_PTR(-EAGAIN);
- /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes.
- * To avoid copying garbage after the pas array, we allocate
- * a little more. */
- size = ALIGN(sizeof(u64) * npages, MLX5_UMR_MTT_ALIGNMENT);
- mr_pas = kmalloc(size + MLX5_UMR_ALIGN - 1, GFP_KERNEL);
- if (!mr_pas) {
- err = -ENOMEM;
+ err = dma_map_mr_pas(dev, umem, npages, page_shift, &size, &mr_pas,
+ &dma);
+ if (err)
goto free_mr;
- }
- pas = PTR_ALIGN(mr_pas, MLX5_UMR_ALIGN);
- mlx5_ib_populate_pas(dev, umem, page_shift, pas, MLX5_IB_MTT_PRESENT);
- /* Clear padding after the actual pages. */
- memset(pas + npages, 0, size - npages * sizeof(u64));
-
- dma = dma_map_single(ddev, pas, size, DMA_TO_DEVICE);
- if (dma_mapping_error(ddev, dma)) {
- err = -ENOMEM;
- goto free_pas;
- }
+ mlx5_ib_init_umr_context(&umr_context);
- memset(&umrwr, 0, sizeof(umrwr));
- umrwr.wr.wr_id = (u64)(unsigned long)&umr_context;
- prep_umr_reg_wqe(pd, &umrwr.wr, &sg, dma, npages, mr->mmr.key,
+ umrwr.wr.wr_cqe = &umr_context.cqe;
+ prep_umr_reg_wqe(pd, &umrwr.wr, &sg, dma, npages, mr->mmkey.key,
page_shift, virt_addr, len, access_flags);
- mlx5_ib_init_umr_context(&umr_context);
down(&umrc->sem);
err = ib_post_send(umrc->qp, &umrwr.wr, &bad);
if (err) {
@@ -830,9 +909,9 @@ static struct mlx5_ib_mr *reg_umr(struct ib_pd *pd, struct ib_umem *umem,
}
}
- mr->mmr.iova = virt_addr;
- mr->mmr.size = len;
- mr->mmr.pd = to_mpd(pd)->pdn;
+ mr->mmkey.iova = virt_addr;
+ mr->mmkey.size = len;
+ mr->mmkey.pd = to_mpd(pd)->pdn;
mr->live = 1;
@@ -840,7 +919,6 @@ unmap_dma:
up(&umrc->sem);
dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE);
-free_pas:
kfree(mr_pas);
free_mr:
@@ -929,8 +1007,10 @@ int mlx5_ib_update_mtt(struct mlx5_ib_mr *mr, u64 start_page_index, int npages,
dma_sync_single_for_device(ddev, dma, size, DMA_TO_DEVICE);
+ mlx5_ib_init_umr_context(&umr_context);
+
memset(&wr, 0, sizeof(wr));
- wr.wr.wr_id = (u64)(unsigned long)&umr_context;
+ wr.wr.wr_cqe = &umr_context.cqe;
sg.addr = dma;
sg.length = ALIGN(npages * sizeof(u64),
@@ -944,10 +1024,9 @@ int mlx5_ib_update_mtt(struct mlx5_ib_mr *mr, u64 start_page_index, int npages,
wr.wr.opcode = MLX5_IB_WR_UMR;
wr.npages = sg.length / sizeof(u64);
wr.page_shift = PAGE_SHIFT;
- wr.mkey = mr->mmr.key;
+ wr.mkey = mr->mmkey.key;
wr.target.offset = start_page_index;
- mlx5_ib_init_umr_context(&umr_context);
down(&umrc->sem);
err = ib_post_send(umrc->qp, &wr.wr, &bad);
if (err) {
@@ -974,10 +1053,14 @@ free_pas:
}
#endif
-static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, u64 virt_addr,
- u64 length, struct ib_umem *umem,
- int npages, int page_shift,
- int access_flags)
+/*
+ * If ibmr is NULL it will be allocated by reg_create.
+ * Else, the given ibmr will be used.
+ */
+static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,
+ u64 virt_addr, u64 length,
+ struct ib_umem *umem, int npages,
+ int page_shift, int access_flags)
{
struct mlx5_ib_dev *dev = to_mdev(pd->device);
struct mlx5_create_mkey_mbox_in *in;
@@ -986,7 +1069,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, u64 virt_addr,
int err;
bool pg_cap = !!(MLX5_CAP_GEN(dev->mdev, pg));
- mr = kzalloc(sizeof(*mr), GFP_KERNEL);
+ mr = ibmr ? to_mmr(ibmr) : kzalloc(sizeof(*mr), GFP_KERNEL);
if (!mr)
return ERR_PTR(-ENOMEM);
@@ -1013,7 +1096,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, u64 virt_addr,
in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
in->xlat_oct_act_size = cpu_to_be32(get_octo_len(virt_addr, length,
1 << page_shift));
- err = mlx5_core_create_mkey(dev->mdev, &mr->mmr, in, inlen, NULL,
+ err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen, NULL,
NULL, NULL);
if (err) {
mlx5_ib_warn(dev, "create mkey failed\n");
@@ -1024,7 +1107,7 @@ static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, u64 virt_addr,
mr->live = 1;
kvfree(in);
- mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmr.key);
+ mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmkey.key);
return mr;
@@ -1032,11 +1115,23 @@ err_2:
kvfree(in);
err_1:
- kfree(mr);
+ if (!ibmr)
+ kfree(mr);
return ERR_PTR(err);
}
+static void set_mr_fileds(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr,
+ int npages, u64 length, int access_flags)
+{
+ mr->npages = npages;
+ atomic_add(npages, &dev->mdev->priv.reg_pages);
+ mr->ibmr.lkey = mr->mmkey.key;
+ mr->ibmr.rkey = mr->mmkey.key;
+ mr->ibmr.length = length;
+ mr->access_flags = access_flags;
+}
+
struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
u64 virt_addr, int access_flags,
struct ib_udata *udata)
@@ -1052,22 +1147,11 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
start, virt_addr, length, access_flags);
- umem = ib_umem_get(pd->uobject->context, start, length, access_flags,
- 0);
- if (IS_ERR(umem)) {
- mlx5_ib_dbg(dev, "umem get failed (%ld)\n", PTR_ERR(umem));
- return (void *)umem;
- }
+ umem = mr_umem_get(pd, start, length, access_flags, &npages,
+ &page_shift, &ncont, &order);
- mlx5_ib_cont_pages(umem, start, &npages, &page_shift, &ncont, &order);
- if (!npages) {
- mlx5_ib_warn(dev, "avoid zero region\n");
- err = -EINVAL;
- goto error;
- }
-
- mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n",
- npages, ncont, order, page_shift);
+ if (IS_ERR(umem))
+ return (void *)umem;
if (use_umr(order)) {
mr = reg_umr(pd, umem, virt_addr, length, ncont, page_shift,
@@ -1083,45 +1167,21 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
}
if (!mr)
- mr = reg_create(pd, virt_addr, length, umem, ncont, page_shift,
- access_flags);
+ mr = reg_create(NULL, pd, virt_addr, length, umem, ncont,
+ page_shift, access_flags);
if (IS_ERR(mr)) {
err = PTR_ERR(mr);
goto error;
}
- mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmr.key);
+ mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmkey.key);
mr->umem = umem;
- mr->npages = npages;
- atomic_add(npages, &dev->mdev->priv.reg_pages);
- mr->ibmr.lkey = mr->mmr.key;
- mr->ibmr.rkey = mr->mmr.key;
+ set_mr_fileds(dev, mr, npages, length, access_flags);
#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
- if (umem->odp_data) {
- /*
- * This barrier prevents the compiler from moving the
- * setting of umem->odp_data->private to point to our
- * MR, before reg_umr finished, to ensure that the MR
- * initialization have finished before starting to
- * handle invalidations.
- */
- smp_wmb();
- mr->umem->odp_data->private = mr;
- /*
- * Make sure we will see the new
- * umem->odp_data->private value in the invalidation
- * routines, before we can get page faults on the
- * MR. Page faults can happen once we put the MR in
- * the tree, below this line. Without the barrier,
- * there can be a fault handling and an invalidation
- * before umem->odp_data->private == mr is visible to
- * the invalidation handler.
- */
- smp_wmb();
- }
+ update_odp_mr(mr);
#endif
return &mr->ibmr;
@@ -1135,15 +1195,15 @@ static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
{
struct umr_common *umrc = &dev->umrc;
struct mlx5_ib_umr_context umr_context;
- struct mlx5_umr_wr umrwr;
+ struct mlx5_umr_wr umrwr = {};
struct ib_send_wr *bad;
int err;
- memset(&umrwr.wr, 0, sizeof(umrwr));
- umrwr.wr.wr_id = (u64)(unsigned long)&umr_context;
- prep_umr_unreg_wqe(dev, &umrwr.wr, mr->mmr.key);
-
mlx5_ib_init_umr_context(&umr_context);
+
+ umrwr.wr.wr_cqe = &umr_context.cqe;
+ prep_umr_unreg_wqe(dev, &umrwr.wr, mr->mmkey.key);
+
down(&umrc->sem);
err = ib_post_send(umrc->qp, &umrwr.wr, &bad);
if (err) {
@@ -1165,6 +1225,167 @@ error:
return err;
}
+static int rereg_umr(struct ib_pd *pd, struct mlx5_ib_mr *mr, u64 virt_addr,
+ u64 length, int npages, int page_shift, int order,
+ int access_flags, int flags)
+{
+ struct mlx5_ib_dev *dev = to_mdev(pd->device);
+ struct device *ddev = dev->ib_dev.dma_device;
+ struct mlx5_ib_umr_context umr_context;
+ struct ib_send_wr *bad;
+ struct mlx5_umr_wr umrwr = {};
+ struct ib_sge sg;
+ struct umr_common *umrc = &dev->umrc;
+ dma_addr_t dma = 0;
+ __be64 *mr_pas = NULL;
+ int size;
+ int err;
+
+ mlx5_ib_init_umr_context(&umr_context);
+
+ umrwr.wr.wr_cqe = &umr_context.cqe;
+ umrwr.wr.send_flags = MLX5_IB_SEND_UMR_FAIL_IF_FREE;
+
+ if (flags & IB_MR_REREG_TRANS) {
+ err = dma_map_mr_pas(dev, mr->umem, npages, page_shift, &size,
+ &mr_pas, &dma);
+ if (err)
+ return err;
+
+ umrwr.target.virt_addr = virt_addr;
+ umrwr.length = length;
+ umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_TRANSLATION;
+ }
+
+ prep_umr_wqe_common(pd, &umrwr.wr, &sg, dma, npages, mr->mmkey.key,
+ page_shift);
+
+ if (flags & IB_MR_REREG_PD) {
+ umrwr.pd = pd;
+ umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_PD;
+ }
+
+ if (flags & IB_MR_REREG_ACCESS) {
+ umrwr.access_flags = access_flags;
+ umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_ACCESS;
+ }
+
+ /* post send request to UMR QP */
+ down(&umrc->sem);
+ err = ib_post_send(umrc->qp, &umrwr.wr, &bad);
+
+ if (err) {
+ mlx5_ib_warn(dev, "post send failed, err %d\n", err);
+ } else {
+ wait_for_completion(&umr_context.done);
+ if (umr_context.status != IB_WC_SUCCESS) {
+ mlx5_ib_warn(dev, "reg umr failed (%u)\n",
+ umr_context.status);
+ err = -EFAULT;
+ }
+ }
+
+ up(&umrc->sem);
+ if (flags & IB_MR_REREG_TRANS) {
+ dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE);
+ kfree(mr_pas);
+ }
+ return err;
+}
+
+int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
+ u64 length, u64 virt_addr, int new_access_flags,
+ struct ib_pd *new_pd, struct ib_udata *udata)
+{
+ struct mlx5_ib_dev *dev = to_mdev(ib_mr->device);
+ struct mlx5_ib_mr *mr = to_mmr(ib_mr);
+ struct ib_pd *pd = (flags & IB_MR_REREG_PD) ? new_pd : ib_mr->pd;
+ int access_flags = flags & IB_MR_REREG_ACCESS ?
+ new_access_flags :
+ mr->access_flags;
+ u64 addr = (flags & IB_MR_REREG_TRANS) ? virt_addr : mr->umem->address;
+ u64 len = (flags & IB_MR_REREG_TRANS) ? length : mr->umem->length;
+ int page_shift = 0;
+ int npages = 0;
+ int ncont = 0;
+ int order = 0;
+ int err;
+
+ mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
+ start, virt_addr, length, access_flags);
+
+ if (flags != IB_MR_REREG_PD) {
+ /*
+ * Replace umem. This needs to be done whether or not UMR is
+ * used.
+ */
+ flags |= IB_MR_REREG_TRANS;
+ ib_umem_release(mr->umem);
+ mr->umem = mr_umem_get(pd, addr, len, access_flags, &npages,
+ &page_shift, &ncont, &order);
+ if (IS_ERR(mr->umem)) {
+ err = PTR_ERR(mr->umem);
+ mr->umem = NULL;
+ return err;
+ }
+ }
+
+ if (flags & IB_MR_REREG_TRANS && !use_umr_mtt_update(mr, addr, len)) {
+ /*
+ * UMR can't be used - MKey needs to be replaced.
+ */
+ if (mr->umred) {
+ err = unreg_umr(dev, mr);
+ if (err)
+ mlx5_ib_warn(dev, "Failed to unregister MR\n");
+ } else {
+ err = destroy_mkey(dev, mr);
+ if (err)
+ mlx5_ib_warn(dev, "Failed to destroy MKey\n");
+ }
+ if (err)
+ return err;
+
+ mr = reg_create(ib_mr, pd, addr, len, mr->umem, ncont,
+ page_shift, access_flags);
+
+ if (IS_ERR(mr))
+ return PTR_ERR(mr);
+
+ mr->umred = 0;
+ } else {
+ /*
+ * Send a UMR WQE
+ */
+ err = rereg_umr(pd, mr, addr, len, npages, page_shift,
+ order, access_flags, flags);
+ if (err) {
+ mlx5_ib_warn(dev, "Failed to rereg UMR\n");
+ return err;
+ }
+ }
+
+ if (flags & IB_MR_REREG_PD) {
+ ib_mr->pd = pd;
+ mr->mmkey.pd = to_mpd(pd)->pdn;
+ }
+
+ if (flags & IB_MR_REREG_ACCESS)
+ mr->access_flags = access_flags;
+
+ if (flags & IB_MR_REREG_TRANS) {
+ atomic_sub(mr->npages, &dev->mdev->priv.reg_pages);
+ set_mr_fileds(dev, mr, npages, len, access_flags);
+ mr->mmkey.iova = addr;
+ mr->mmkey.size = len;
+ }
+#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
+ update_odp_mr(mr);
+#endif
+
+ return 0;
+}
+
static int
mlx5_alloc_priv_descs(struct ib_device *device,
struct mlx5_ib_mr *mr,
@@ -1236,7 +1457,7 @@ static int clean_mr(struct mlx5_ib_mr *mr)
err = destroy_mkey(dev, mr);
if (err) {
mlx5_ib_warn(dev, "failed to destroy mkey 0x%x (%d)\n",
- mr->mmr.key, err);
+ mr->mmkey.key, err);
return err;
}
} else {
@@ -1300,8 +1521,8 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
struct mlx5_ib_dev *dev = to_mdev(pd->device);
struct mlx5_create_mkey_mbox_in *in;
struct mlx5_ib_mr *mr;
- int access_mode, err;
- int ndescs = roundup(max_num_sg, 4);
+ int ndescs = ALIGN(max_num_sg, 4);
+ int err;
mr = kzalloc(sizeof(*mr), GFP_KERNEL);
if (!mr)
@@ -1319,7 +1540,7 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
in->seg.flags_pd = cpu_to_be32(to_mpd(pd)->pdn);
if (mr_type == IB_MR_TYPE_MEM_REG) {
- access_mode = MLX5_ACCESS_MODE_MTT;
+ mr->access_mode = MLX5_ACCESS_MODE_MTT;
in->seg.log2_page_size = PAGE_SHIFT;
err = mlx5_alloc_priv_descs(pd->device, mr,
@@ -1329,6 +1550,15 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
mr->desc_size = sizeof(u64);
mr->max_descs = ndescs;
+ } else if (mr_type == IB_MR_TYPE_SG_GAPS) {
+ mr->access_mode = MLX5_ACCESS_MODE_KLM;
+
+ err = mlx5_alloc_priv_descs(pd->device, mr,
+ ndescs, sizeof(struct mlx5_klm));
+ if (err)
+ goto err_free_in;
+ mr->desc_size = sizeof(struct mlx5_klm);
+ mr->max_descs = ndescs;
} else if (mr_type == IB_MR_TYPE_SIGNATURE) {
u32 psv_index[2];
@@ -1347,7 +1577,7 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
if (err)
goto err_free_sig;
- access_mode = MLX5_ACCESS_MODE_KLM;
+ mr->access_mode = MLX5_ACCESS_MODE_KLM;
mr->sig->psv_memory.psv_idx = psv_index[0];
mr->sig->psv_wire.psv_idx = psv_index[1];
@@ -1361,14 +1591,14 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
goto err_free_in;
}
- in->seg.flags = MLX5_PERM_UMR_EN | access_mode;
- err = mlx5_core_create_mkey(dev->mdev, &mr->mmr, in, sizeof(*in),
+ in->seg.flags = MLX5_PERM_UMR_EN | mr->access_mode;
+ err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, sizeof(*in),
NULL, NULL, NULL);
if (err)
goto err_destroy_psv;
- mr->ibmr.lkey = mr->mmr.key;
- mr->ibmr.rkey = mr->mmr.key;
+ mr->ibmr.lkey = mr->mmkey.key;
+ mr->ibmr.rkey = mr->mmkey.key;
mr->umem = NULL;
kfree(in);
@@ -1395,6 +1625,88 @@ err_free:
return ERR_PTR(err);
}
+struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
+ struct ib_udata *udata)
+{
+ struct mlx5_ib_dev *dev = to_mdev(pd->device);
+ struct mlx5_create_mkey_mbox_in *in = NULL;
+ struct mlx5_ib_mw *mw = NULL;
+ int ndescs;
+ int err;
+ struct mlx5_ib_alloc_mw req = {};
+ struct {
+ __u32 comp_mask;
+ __u32 response_length;
+ } resp = {};
+
+ err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req)));
+ if (err)
+ return ERR_PTR(err);
+
+ if (req.comp_mask || req.reserved1 || req.reserved2)
+ return ERR_PTR(-EOPNOTSUPP);
+
+ if (udata->inlen > sizeof(req) &&
+ !ib_is_udata_cleared(udata, sizeof(req),
+ udata->inlen - sizeof(req)))
+ return ERR_PTR(-EOPNOTSUPP);
+
+ ndescs = req.num_klms ? roundup(req.num_klms, 4) : roundup(1, 4);
+
+ mw = kzalloc(sizeof(*mw), GFP_KERNEL);
+ in = kzalloc(sizeof(*in), GFP_KERNEL);
+ if (!mw || !in) {
+ err = -ENOMEM;
+ goto free;
+ }
+
+ in->seg.status = MLX5_MKEY_STATUS_FREE;
+ in->seg.xlt_oct_size = cpu_to_be32(ndescs);
+ in->seg.flags_pd = cpu_to_be32(to_mpd(pd)->pdn);
+ in->seg.flags = MLX5_PERM_UMR_EN | MLX5_ACCESS_MODE_KLM |
+ MLX5_PERM_LOCAL_READ;
+ if (type == IB_MW_TYPE_2)
+ in->seg.flags_pd |= cpu_to_be32(MLX5_MKEY_REMOTE_INVAL);
+ in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
+
+ err = mlx5_core_create_mkey(dev->mdev, &mw->mmkey, in, sizeof(*in),
+ NULL, NULL, NULL);
+ if (err)
+ goto free;
+
+ mw->ibmw.rkey = mw->mmkey.key;
+
+ resp.response_length = min(offsetof(typeof(resp), response_length) +
+ sizeof(resp.response_length), udata->outlen);
+ if (resp.response_length) {
+ err = ib_copy_to_udata(udata, &resp, resp.response_length);
+ if (err) {
+ mlx5_core_destroy_mkey(dev->mdev, &mw->mmkey);
+ goto free;
+ }
+ }
+
+ kfree(in);
+ return &mw->ibmw;
+
+free:
+ kfree(mw);
+ kfree(in);
+ return ERR_PTR(err);
+}
+
+int mlx5_ib_dealloc_mw(struct ib_mw *mw)
+{
+ struct mlx5_ib_mw *mmw = to_mmw(mw);
+ int err;
+
+ err = mlx5_core_destroy_mkey((to_mdev(mw->device))->mdev,
+ &mmw->mmkey);
+ if (!err)
+ kfree(mmw);
+ return err;
+}
+
int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
struct ib_mr_status *mr_status)
{
@@ -1436,6 +1748,32 @@ done:
return ret;
}
+static int
+mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr,
+ struct scatterlist *sgl,
+ unsigned short sg_nents)
+{
+ struct scatterlist *sg = sgl;
+ struct mlx5_klm *klms = mr->descs;
+ u32 lkey = mr->ibmr.pd->local_dma_lkey;
+ int i;
+
+ mr->ibmr.iova = sg_dma_address(sg);
+ mr->ibmr.length = 0;
+ mr->ndescs = sg_nents;
+
+ for_each_sg(sgl, sg, sg_nents, i) {
+ if (unlikely(i > mr->max_descs))
+ break;
+ klms[i].va = cpu_to_be64(sg_dma_address(sg));
+ klms[i].bcount = cpu_to_be32(sg_dma_len(sg));
+ klms[i].key = cpu_to_be32(lkey);
+ mr->ibmr.length += sg_dma_len(sg);
+ }
+
+ return i;
+}
+
static int mlx5_set_page(struct ib_mr *ibmr, u64 addr)
{
struct mlx5_ib_mr *mr = to_mmr(ibmr);
@@ -1463,7 +1801,10 @@ int mlx5_ib_map_mr_sg(struct ib_mr *ibmr,
mr->desc_size * mr->max_descs,
DMA_TO_DEVICE);
- n = ib_sg_to_pages(ibmr, sg, sg_nents, mlx5_set_page);
+ if (mr->access_mode == MLX5_ACCESS_MODE_KLM)
+ n = mlx5_ib_sg_to_klms(mr, sg, sg_nents);
+ else
+ n = ib_sg_to_pages(ibmr, sg, sg_nents, mlx5_set_page);
ib_dma_sync_single_for_device(ibmr->device, mr->desc_map,
mr->desc_size * mr->max_descs,