diff options
Diffstat (limited to 'drivers/staging/rdma/hfi1/driver.c')
-rw-r--r-- | drivers/staging/rdma/hfi1/driver.c | 215 |
1 files changed, 144 insertions, 71 deletions
diff --git a/drivers/staging/rdma/hfi1/driver.c b/drivers/staging/rdma/hfi1/driver.c index ce69141b5..8485de1fc 100644 --- a/drivers/staging/rdma/hfi1/driver.c +++ b/drivers/staging/rdma/hfi1/driver.c @@ -158,7 +158,7 @@ const char *get_unit_name(int unit) { static char iname[16]; - snprintf(iname, sizeof(iname), DRIVER_NAME"_%u", unit); + snprintf(iname, sizeof(iname), DRIVER_NAME "_%u", unit); return iname; } @@ -436,59 +436,58 @@ static inline void init_packet(struct hfi1_ctxtdata *rcd, #ifndef CONFIG_PRESCAN_RXQ static void prescan_rxq(struct hfi1_packet *packet) {} -#else /* CONFIG_PRESCAN_RXQ */ +#else /* !CONFIG_PRESCAN_RXQ */ static int prescan_receive_queue; static void process_ecn(struct hfi1_qp *qp, struct hfi1_ib_header *hdr, struct hfi1_other_headers *ohdr, - u64 rhf, struct ib_grh *grh) + u64 rhf, u32 bth1, struct ib_grh *grh) { struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); - u32 bth1; + u32 rqpn = 0; + u16 rlid; u8 sc5, svc_type; - int is_fecn, is_becn; switch (qp->ibqp.qp_type) { + case IB_QPT_SMI: + case IB_QPT_GSI: case IB_QPT_UD: + rlid = be16_to_cpu(hdr->lrh[3]); + rqpn = be32_to_cpu(ohdr->u.ud.deth[1]) & HFI1_QPN_MASK; svc_type = IB_CC_SVCTYPE_UD; break; - case IB_QPT_UC: /* LATER */ - case IB_QPT_RC: /* LATER */ + case IB_QPT_UC: + rlid = qp->remote_ah_attr.dlid; + rqpn = qp->remote_qpn; + svc_type = IB_CC_SVCTYPE_UC; + break; + case IB_QPT_RC: + rlid = qp->remote_ah_attr.dlid; + rqpn = qp->remote_qpn; + svc_type = IB_CC_SVCTYPE_RC; + break; default: return; } - is_fecn = (be32_to_cpu(ohdr->bth[1]) >> HFI1_FECN_SHIFT) & - HFI1_FECN_MASK; - is_becn = (be32_to_cpu(ohdr->bth[1]) >> HFI1_BECN_SHIFT) & - HFI1_BECN_MASK; - sc5 = (be16_to_cpu(hdr->lrh[0]) >> 12) & 0xf; if (rhf_dc_info(rhf)) sc5 |= 0x10; - if (is_fecn) { - u32 src_qpn = be32_to_cpu(ohdr->u.ud.deth[1]) & HFI1_QPN_MASK; + if (bth1 & HFI1_FECN_SMASK) { u16 pkey = (u16)be32_to_cpu(ohdr->bth[0]); u16 dlid = be16_to_cpu(hdr->lrh[1]); - u16 slid = be16_to_cpu(hdr->lrh[3]); - return_cnp(ibp, qp, src_qpn, pkey, dlid, slid, sc5, grh); + return_cnp(ibp, qp, rqpn, pkey, dlid, rlid, sc5, grh); } - if (is_becn) { + if (bth1 & HFI1_BECN_SMASK) { struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); - u32 lqpn = be32_to_cpu(ohdr->bth[1]) & HFI1_QPN_MASK; + u32 lqpn = bth1 & HFI1_QPN_MASK; u8 sl = ibp->sc_to_sl[sc5]; - process_becn(ppd, sl, 0, lqpn, 0, svc_type); + process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type); } - - /* turn off BECN, or FECN */ - bth1 = be32_to_cpu(ohdr->bth[1]); - bth1 &= ~(HFI1_FECN_MASK << HFI1_FECN_SHIFT); - bth1 &= ~(HFI1_BECN_MASK << HFI1_BECN_SHIFT); - ohdr->bth[1] = cpu_to_be32(bth1); } struct ps_mdata { @@ -508,38 +507,51 @@ static inline void init_ps_mdata(struct ps_mdata *mdata, mdata->rcd = rcd; mdata->rsize = packet->rsize; mdata->maxcnt = packet->maxcnt; + mdata->ps_head = packet->rhqoff; - if (rcd->ps_state.initialized == 0) { - mdata->ps_head = packet->rhqoff; - rcd->ps_state.initialized++; - } else - mdata->ps_head = rcd->ps_state.ps_head; - - if (HFI1_CAP_IS_KSET(DMA_RTAIL)) { - mdata->ps_tail = packet->hdrqtail; - mdata->ps_seq = 0; /* not used with DMA_RTAIL */ + if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) { + mdata->ps_tail = get_rcvhdrtail(rcd); + if (rcd->ctxt == HFI1_CTRL_CTXT) + mdata->ps_seq = rcd->seq_cnt; + else + mdata->ps_seq = 0; /* not used with DMA_RTAIL */ } else { mdata->ps_tail = 0; /* used only with DMA_RTAIL*/ mdata->ps_seq = rcd->seq_cnt; } } -static inline int ps_done(struct ps_mdata *mdata, u64 rhf) +static inline int ps_done(struct ps_mdata *mdata, u64 rhf, + struct hfi1_ctxtdata *rcd) { - if (HFI1_CAP_IS_KSET(DMA_RTAIL)) + if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) return mdata->ps_head == mdata->ps_tail; return mdata->ps_seq != rhf_rcv_seq(rhf); } -static inline void update_ps_mdata(struct ps_mdata *mdata) +static inline int ps_skip(struct ps_mdata *mdata, u64 rhf, + struct hfi1_ctxtdata *rcd) { - struct hfi1_ctxtdata *rcd = mdata->rcd; + /* + * Control context can potentially receive an invalid rhf. + * Drop such packets. + */ + if ((rcd->ctxt == HFI1_CTRL_CTXT) && (mdata->ps_head != mdata->ps_tail)) + return mdata->ps_seq != rhf_rcv_seq(rhf); + + return 0; +} +static inline void update_ps_mdata(struct ps_mdata *mdata, + struct hfi1_ctxtdata *rcd) +{ mdata->ps_head += mdata->rsize; - if (mdata->ps_head > mdata->maxcnt) + if (mdata->ps_head >= mdata->maxcnt) mdata->ps_head = 0; - rcd->ps_state.ps_head = mdata->ps_head; - if (!HFI1_CAP_IS_KSET(DMA_RTAIL)) { + + /* Control context must do seq counting */ + if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) || + (rcd->ctxt == HFI1_CTRL_CTXT)) { if (++mdata->ps_seq > 13) mdata->ps_seq = 1; } @@ -571,13 +583,16 @@ static void prescan_rxq(struct hfi1_packet *packet) struct hfi1_other_headers *ohdr; struct ib_grh *grh = NULL; u64 rhf = rhf_to_cpu(rhf_addr); - u32 etype = rhf_rcv_type(rhf), qpn; + u32 etype = rhf_rcv_type(rhf), qpn, bth1; int is_ecn = 0; u8 lnh; - if (ps_done(&mdata, rhf)) + if (ps_done(&mdata, rhf, rcd)) break; + if (ps_skip(&mdata, rhf, rcd)) + goto next; + if (etype != RHF_RCV_TYPE_IB) goto next; @@ -593,15 +608,13 @@ static void prescan_rxq(struct hfi1_packet *packet) } else goto next; /* just in case */ - is_ecn |= be32_to_cpu(ohdr->bth[1]) & - (HFI1_FECN_MASK << HFI1_FECN_SHIFT); - is_ecn |= be32_to_cpu(ohdr->bth[1]) & - (HFI1_BECN_MASK << HFI1_BECN_SHIFT); + bth1 = be32_to_cpu(ohdr->bth[1]); + is_ecn = !!(bth1 & (HFI1_FECN_SMASK | HFI1_BECN_SMASK)); if (!is_ecn) goto next; - qpn = be32_to_cpu(ohdr->bth[1]) & HFI1_QPN_MASK; + qpn = bth1 & HFI1_QPN_MASK; rcu_read_lock(); qp = hfi1_lookup_qpn(ibp, qpn); @@ -610,14 +623,44 @@ static void prescan_rxq(struct hfi1_packet *packet) goto next; } - process_ecn(qp, hdr, ohdr, rhf, grh); + process_ecn(qp, hdr, ohdr, rhf, bth1, grh); rcu_read_unlock(); + + /* turn off BECN, FECN */ + bth1 &= ~(HFI1_FECN_SMASK | HFI1_BECN_SMASK); + ohdr->bth[1] = cpu_to_be32(bth1); next: - update_ps_mdata(&mdata); + update_ps_mdata(&mdata, rcd); } } #endif /* CONFIG_PRESCAN_RXQ */ +static inline int skip_rcv_packet(struct hfi1_packet *packet, int thread) +{ + int ret = RCV_PKT_OK; + + /* Set up for the next packet */ + packet->rhqoff += packet->rsize; + if (packet->rhqoff >= packet->maxcnt) + packet->rhqoff = 0; + + packet->numpkt++; + if (unlikely((packet->numpkt & (MAX_PKT_RECV - 1)) == 0)) { + if (thread) { + cond_resched(); + } else { + ret = RCV_PKT_LIMIT; + this_cpu_inc(*packet->rcd->dd->rcv_limit); + } + } + + packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff + + packet->rcd->dd->rhf_offset; + packet->rhf = rhf_to_cpu(packet->rhf_addr); + + return ret; +} + static inline int process_rcv_packet(struct hfi1_packet *packet, int thread) { int ret = RCV_PKT_OK; @@ -721,8 +764,8 @@ static inline void process_rcv_qp_work(struct hfi1_packet *packet) */ list_for_each_entry_safe(qp, nqp, &rcd->qp_wait_list, rspwait) { list_del_init(&qp->rspwait); - if (qp->r_flags & HFI1_R_RSP_NAK) { - qp->r_flags &= ~HFI1_R_RSP_NAK; + if (qp->r_flags & HFI1_R_RSP_DEFERED_ACK) { + qp->r_flags &= ~HFI1_R_RSP_DEFERED_ACK; hfi1_send_rc_ack(rcd, qp, 0); } if (qp->r_flags & HFI1_R_RSP_SEND) { @@ -791,7 +834,6 @@ int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread) while (last == RCV_PKT_OK) { last = process_rcv_packet(&packet, thread); - hdrqtail = get_rcvhdrtail(rcd); if (packet.rhqoff == hdrqtail) last = RCV_PKT_DONE; process_rcv_update(last, &packet); @@ -806,7 +848,7 @@ static inline void set_all_nodma_rtail(struct hfi1_devdata *dd) { int i; - for (i = 0; i < dd->first_user_ctxt; i++) + for (i = HFI1_CTRL_CTXT + 1; i < dd->first_user_ctxt; i++) dd->rcd[i]->do_interrupt = &handle_receive_interrupt_nodma_rtail; } @@ -815,7 +857,7 @@ static inline void set_all_dma_rtail(struct hfi1_devdata *dd) { int i; - for (i = 0; i < dd->first_user_ctxt; i++) + for (i = HFI1_CTRL_CTXT + 1; i < dd->first_user_ctxt; i++) dd->rcd[i]->do_interrupt = &handle_receive_interrupt_dma_rtail; } @@ -831,12 +873,16 @@ int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread) { struct hfi1_devdata *dd = rcd->dd; u32 hdrqtail; - int last = RCV_PKT_OK, needset = 1; + int needset, last = RCV_PKT_OK; struct hfi1_packet packet; + int skip_pkt = 0; + + /* Control context will always use the slow path interrupt handler */ + needset = (rcd->ctxt == HFI1_CTRL_CTXT) ? 0 : 1; init_packet(rcd, &packet); - if (!HFI1_CAP_IS_KSET(DMA_RTAIL)) { + if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) { u32 seq = rhf_rcv_seq(packet.rhf); if (seq != rcd->seq_cnt) { @@ -851,6 +897,17 @@ int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread) goto bail; } smp_rmb(); /* prevent speculative reads of dma'ed hdrq */ + + /* + * Control context can potentially receive an invalid + * rhf. Drop such packets. + */ + if (rcd->ctxt == HFI1_CTRL_CTXT) { + u32 seq = rhf_rcv_seq(packet.rhf); + + if (seq != rcd->seq_cnt) + skip_pkt = 1; + } } prescan_rxq(&packet); @@ -868,11 +925,14 @@ int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread) dd->rhf_offset; packet.rhf = rhf_to_cpu(packet.rhf_addr); + } else if (skip_pkt) { + last = skip_rcv_packet(&packet, thread); + skip_pkt = 0; } else { last = process_rcv_packet(&packet, thread); } - if (!HFI1_CAP_IS_KSET(DMA_RTAIL)) { + if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) { u32 seq = rhf_rcv_seq(packet.rhf); if (++rcd->seq_cnt > 13) @@ -888,6 +948,19 @@ int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread) } else { if (packet.rhqoff == hdrqtail) last = RCV_PKT_DONE; + /* + * Control context can potentially receive an invalid + * rhf. Drop such packets. + */ + if (rcd->ctxt == HFI1_CTRL_CTXT) { + u32 seq = rhf_rcv_seq(packet.rhf); + + if (++rcd->seq_cnt > 13) + rcd->seq_cnt = 1; + if (!last && (seq != rcd->seq_cnt)) + skip_pkt = 1; + } + if (needset) { dd_dev_info(dd, "Switching to DMA_RTAIL\n"); @@ -1163,20 +1236,20 @@ void handle_eflags(struct hfi1_packet *packet) struct hfi1_ctxtdata *rcd = packet->rcd; u32 rte = rhf_rcv_type_err(packet->rhf); - dd_dev_err(rcd->dd, - "receive context %d: rhf 0x%016llx, errs [ %s%s%s%s%s%s%s%s] rte 0x%x\n", - rcd->ctxt, packet->rhf, - packet->rhf & RHF_K_HDR_LEN_ERR ? "k_hdr_len " : "", - packet->rhf & RHF_DC_UNC_ERR ? "dc_unc " : "", - packet->rhf & RHF_DC_ERR ? "dc " : "", - packet->rhf & RHF_TID_ERR ? "tid " : "", - packet->rhf & RHF_LEN_ERR ? "len " : "", - packet->rhf & RHF_ECC_ERR ? "ecc " : "", - packet->rhf & RHF_VCRC_ERR ? "vcrc " : "", - packet->rhf & RHF_ICRC_ERR ? "icrc " : "", - rte); - rcv_hdrerr(rcd, rcd->ppd, packet); + if (rhf_err_flags(packet->rhf)) + dd_dev_err(rcd->dd, + "receive context %d: rhf 0x%016llx, errs [ %s%s%s%s%s%s%s%s] rte 0x%x\n", + rcd->ctxt, packet->rhf, + packet->rhf & RHF_K_HDR_LEN_ERR ? "k_hdr_len " : "", + packet->rhf & RHF_DC_UNC_ERR ? "dc_unc " : "", + packet->rhf & RHF_DC_ERR ? "dc " : "", + packet->rhf & RHF_TID_ERR ? "tid " : "", + packet->rhf & RHF_LEN_ERR ? "len " : "", + packet->rhf & RHF_ECC_ERR ? "ecc " : "", + packet->rhf & RHF_VCRC_ERR ? "vcrc " : "", + packet->rhf & RHF_ICRC_ERR ? "icrc " : "", + rte); } /* |