diff options
Diffstat (limited to 'drivers/staging/rdma/hfi1/pio.c')
| -rw-r--r-- | drivers/staging/rdma/hfi1/pio.c | 68 |
1 files changed, 51 insertions, 17 deletions
diff --git a/drivers/staging/rdma/hfi1/pio.c b/drivers/staging/rdma/hfi1/pio.c index e5c32db4b..b51a44163 100644 --- a/drivers/staging/rdma/hfi1/pio.c +++ b/drivers/staging/rdma/hfi1/pio.c @@ -660,6 +660,24 @@ void set_pio_integrity(struct send_context *sc) write_kctxt_csr(dd, hw_context, SC(CHECK_ENABLE), reg); } +static u32 get_buffers_allocated(struct send_context *sc) +{ + int cpu; + u32 ret = 0; + + for_each_possible_cpu(cpu) + ret += *per_cpu_ptr(sc->buffers_allocated, cpu); + return ret; +} + +static void reset_buffers_allocated(struct send_context *sc) +{ + int cpu; + + for_each_possible_cpu(cpu) + (*per_cpu_ptr(sc->buffers_allocated, cpu)) = 0; +} + /* * Allocate a NUMA relative send context structure of the given type along * with a HW context. @@ -668,7 +686,7 @@ struct send_context *sc_alloc(struct hfi1_devdata *dd, int type, uint hdrqentsize, int numa) { struct send_context_info *sci; - struct send_context *sc; + struct send_context *sc = NULL; dma_addr_t pa; unsigned long flags; u64 reg; @@ -686,10 +704,20 @@ struct send_context *sc_alloc(struct hfi1_devdata *dd, int type, if (!sc) return NULL; + sc->buffers_allocated = alloc_percpu(u32); + if (!sc->buffers_allocated) { + kfree(sc); + dd_dev_err(dd, + "Cannot allocate buffers_allocated per cpu counters\n" + ); + return NULL; + } + spin_lock_irqsave(&dd->sc_lock, flags); ret = sc_hw_alloc(dd, type, &sw_index, &hw_context); if (ret) { spin_unlock_irqrestore(&dd->sc_lock, flags); + free_percpu(sc->buffers_allocated); kfree(sc); return NULL; } @@ -705,7 +733,6 @@ struct send_context *sc_alloc(struct hfi1_devdata *dd, int type, spin_lock_init(&sc->credit_ctrl_lock); INIT_LIST_HEAD(&sc->piowait); INIT_WORK(&sc->halt_work, sc_halted); - atomic_set(&sc->buffers_allocated, 0); init_waitqueue_head(&sc->halt_wait); /* grouping is always single context for now */ @@ -815,15 +842,16 @@ struct send_context *sc_alloc(struct hfi1_devdata *dd, int type, } } - dd_dev_info(dd, - "Send context %u(%u) %s group %u credits %u credit_ctrl 0x%llx threshold %u\n", - sw_index, - hw_context, - sc_type_name(type), - sc->group, - sc->credits, - sc->credit_ctrl, - thresh); + hfi1_cdbg(PIO, + "Send context %u(%u) %s group %u credits %u credit_ctrl 0x%llx threshold %u\n", + sw_index, + hw_context, + sc_type_name(type), + sc->group, + sc->credits, + sc->credit_ctrl, + thresh); + return sc; } @@ -865,6 +893,7 @@ void sc_free(struct send_context *sc) spin_unlock_irqrestore(&dd->sc_lock, flags); kfree(sc->sr); + free_percpu(sc->buffers_allocated); kfree(sc); } @@ -1028,7 +1057,7 @@ int sc_restart(struct send_context *sc) /* kernel context */ loop = 0; while (1) { - count = atomic_read(&sc->buffers_allocated); + count = get_buffers_allocated(sc); if (count == 0) break; if (loop > 100) { @@ -1196,7 +1225,8 @@ int sc_enable(struct send_context *sc) sc->sr_head = 0; sc->sr_tail = 0; sc->flags = 0; - atomic_set(&sc->buffers_allocated, 0); + /* the alloc lock insures no fast path allocation */ + reset_buffers_allocated(sc); /* * Clear all per-context errors. Some of these will be set when @@ -1372,7 +1402,8 @@ retry: /* there is enough room */ - atomic_inc(&sc->buffers_allocated); + preempt_disable(); + this_cpu_inc(*sc->buffers_allocated); /* read this once */ head = sc->sr_head; @@ -1564,6 +1595,7 @@ void sc_release_update(struct send_context *sc) u64 hw_free; u32 head, tail; unsigned long old_free; + unsigned long free; unsigned long extra; unsigned long flags; int code; @@ -1578,7 +1610,7 @@ void sc_release_update(struct send_context *sc) extra = (((hw_free & CR_COUNTER_SMASK) >> CR_COUNTER_SHIFT) - (old_free & CR_COUNTER_MASK)) & CR_COUNTER_MASK; - sc->free = old_free + extra; + free = old_free + extra; trace_hfi1_piofree(sc, extra); /* call sent buffer callbacks */ @@ -1588,7 +1620,7 @@ void sc_release_update(struct send_context *sc) while (head != tail) { pbuf = &sc->sr[tail].pbuf; - if (sent_before(sc->free, pbuf->sent_at)) { + if (sent_before(free, pbuf->sent_at)) { /* not sent yet */ break; } @@ -1602,8 +1634,10 @@ void sc_release_update(struct send_context *sc) if (tail >= sc->sr_size) tail = 0; } - /* update tail, in case we moved it */ sc->sr_tail = tail; + /* make sure tail is updated before free */ + smp_wmb(); + sc->free = free; spin_unlock_irqrestore(&sc->release_lock, flags); sc_piobufavail(sc); } |