diff options
Diffstat (limited to 'drivers/net/wireless/intel/iwlwifi/pcie')
| -rw-r--r-- | drivers/net/wireless/intel/iwlwifi/pcie/drv.c | 134 | ||||
| -rw-r--r-- | drivers/net/wireless/intel/iwlwifi/pcie/internal.h | 79 | ||||
| -rw-r--r-- | drivers/net/wireless/intel/iwlwifi/pcie/rx.c | 220 | ||||
| -rw-r--r-- | drivers/net/wireless/intel/iwlwifi/pcie/trans.c | 231 | ||||
| -rw-r--r-- | drivers/net/wireless/intel/iwlwifi/pcie/tx.c | 293 |
5 files changed, 568 insertions, 389 deletions
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c index 79d7cd7d4..13842ca12 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c @@ -433,6 +433,7 @@ static const struct pci_device_id iwl_hw_card_ids[] = { /* 8000 Series */ {IWL_PCI_DEVICE(0x24F3, 0x0010, iwl8260_2ac_cfg)}, {IWL_PCI_DEVICE(0x24F3, 0x1010, iwl8260_2ac_cfg)}, + {IWL_PCI_DEVICE(0x24F3, 0x10B0, iwl8260_2ac_cfg)}, {IWL_PCI_DEVICE(0x24F3, 0x0130, iwl8260_2ac_cfg)}, {IWL_PCI_DEVICE(0x24F3, 0x1130, iwl8260_2ac_cfg)}, {IWL_PCI_DEVICE(0x24F3, 0x0132, iwl8260_2ac_cfg)}, @@ -454,6 +455,8 @@ static const struct pci_device_id iwl_hw_card_ids[] = { {IWL_PCI_DEVICE(0x24F3, 0xD010, iwl8260_2ac_cfg)}, {IWL_PCI_DEVICE(0x24F3, 0xC050, iwl8260_2ac_cfg)}, {IWL_PCI_DEVICE(0x24F3, 0xD050, iwl8260_2ac_cfg)}, + {IWL_PCI_DEVICE(0x24F3, 0xD0B0, iwl8260_2ac_cfg)}, + {IWL_PCI_DEVICE(0x24F3, 0xB0B0, iwl8260_2ac_cfg)}, {IWL_PCI_DEVICE(0x24F3, 0x8010, iwl8260_2ac_cfg)}, {IWL_PCI_DEVICE(0x24F3, 0x8110, iwl8260_2ac_cfg)}, {IWL_PCI_DEVICE(0x24F3, 0x9010, iwl8260_2ac_cfg)}, @@ -481,6 +484,8 @@ static const struct pci_device_id iwl_hw_card_ids[] = { {IWL_PCI_DEVICE(0x24FD, 0x0010, iwl8265_2ac_cfg)}, {IWL_PCI_DEVICE(0x24FD, 0x0110, iwl8265_2ac_cfg)}, {IWL_PCI_DEVICE(0x24FD, 0x1110, iwl8265_2ac_cfg)}, + {IWL_PCI_DEVICE(0x24FD, 0x1130, iwl8265_2ac_cfg)}, + {IWL_PCI_DEVICE(0x24FD, 0x0130, iwl8265_2ac_cfg)}, {IWL_PCI_DEVICE(0x24FD, 0x1010, iwl8265_2ac_cfg)}, {IWL_PCI_DEVICE(0x24FD, 0x0050, iwl8265_2ac_cfg)}, {IWL_PCI_DEVICE(0x24FD, 0x0150, iwl8265_2ac_cfg)}, @@ -491,19 +496,29 @@ static const struct pci_device_id iwl_hw_card_ids[] = { {IWL_PCI_DEVICE(0x24FD, 0x0810, iwl8265_2ac_cfg)}, {IWL_PCI_DEVICE(0x24FD, 0x9110, iwl8265_2ac_cfg)}, {IWL_PCI_DEVICE(0x24FD, 0x8130, iwl8265_2ac_cfg)}, + {IWL_PCI_DEVICE(0x24FD, 0x0910, iwl8265_2ac_cfg)}, + {IWL_PCI_DEVICE(0x24FD, 0x0930, iwl8265_2ac_cfg)}, + {IWL_PCI_DEVICE(0x24FD, 0x0950, iwl8265_2ac_cfg)}, + {IWL_PCI_DEVICE(0x24FD, 0x0850, iwl8265_2ac_cfg)}, /* 9000 Series */ - {IWL_PCI_DEVICE(0x9DF0, 0x2A10, iwl5165_2ac_cfg)}, - {IWL_PCI_DEVICE(0x9DF0, 0x2010, iwl5165_2ac_cfg)}, + {IWL_PCI_DEVICE(0x2526, 0x0000, iwl9260_2ac_cfg)}, + {IWL_PCI_DEVICE(0x2526, 0x0010, iwl9260_2ac_cfg)}, {IWL_PCI_DEVICE(0x9DF0, 0x0A10, iwl9260_2ac_cfg)}, {IWL_PCI_DEVICE(0x9DF0, 0x0010, iwl9260_2ac_cfg)}, - {IWL_PCI_DEVICE(0x9DF0, 0x0000, iwl5165_2ac_cfg)}, - {IWL_PCI_DEVICE(0x9DF0, 0x0310, iwl5165_2ac_cfg)}, - {IWL_PCI_DEVICE(0x9DF0, 0x0510, iwl5165_2ac_cfg)}, - {IWL_PCI_DEVICE(0x9DF0, 0x0710, iwl5165_2ac_cfg)}, {IWL_PCI_DEVICE(0x9DF0, 0x0210, iwl9260_2ac_cfg)}, {IWL_PCI_DEVICE(0x9DF0, 0x0410, iwl9260_2ac_cfg)}, {IWL_PCI_DEVICE(0x9DF0, 0x0610, iwl9260_2ac_cfg)}, + {IWL_PCI_DEVICE(0x9DF0, 0x0310, iwl5165_2ac_cfg)}, + {IWL_PCI_DEVICE(0x9DF0, 0x0000, iwl5165_2ac_cfg)}, + {IWL_PCI_DEVICE(0x9DF0, 0x0510, iwl5165_2ac_cfg)}, + {IWL_PCI_DEVICE(0x9DF0, 0x2010, iwl5165_2ac_cfg)}, + {IWL_PCI_DEVICE(0x2526, 0x1420, iwl5165_2ac_cfg)}, + {IWL_PCI_DEVICE(0x9DF0, 0x0710, iwl5165_2ac_cfg)}, + {IWL_PCI_DEVICE(0x9DF0, 0x2A10, iwl5165_2ac_cfg)}, + +/* a000 Series */ + {IWL_PCI_DEVICE(0x2720, 0x0A10, iwla000_2ac_cfg)}, #endif /* CONFIG_IWLMVM */ {0} @@ -511,48 +526,64 @@ static const struct pci_device_id iwl_hw_card_ids[] = { MODULE_DEVICE_TABLE(pci, iwl_hw_card_ids); #ifdef CONFIG_ACPI -#define SPL_METHOD "SPLC" -#define SPL_DOMAINTYPE_MODULE BIT(0) -#define SPL_DOMAINTYPE_WIFI BIT(1) -#define SPL_DOMAINTYPE_WIGIG BIT(2) -#define SPL_DOMAINTYPE_RFEM BIT(3) +#define ACPI_SPLC_METHOD "SPLC" +#define ACPI_SPLC_DOMAIN_WIFI (0x07) -static u64 splx_get_pwr_limit(struct iwl_trans *trans, union acpi_object *splx) +static u64 splc_get_pwr_limit(struct iwl_trans *trans, union acpi_object *splc) { - union acpi_object *limits, *domain_type, *power_limit; - - if (splx->type != ACPI_TYPE_PACKAGE || - splx->package.count != 2 || - splx->package.elements[0].type != ACPI_TYPE_INTEGER || - splx->package.elements[0].integer.value != 0) { - IWL_ERR(trans, "Unsupported splx structure\n"); + union acpi_object *data_pkg, *dflt_pwr_limit; + int i; + + /* We need at least two elements, one for the revision and one + * for the data itself. Also check that the revision is + * supported (currently only revision 0). + */ + if (splc->type != ACPI_TYPE_PACKAGE || + splc->package.count < 2 || + splc->package.elements[0].type != ACPI_TYPE_INTEGER || + splc->package.elements[0].integer.value != 0) { + IWL_DEBUG_INFO(trans, + "Unsupported structure returned by the SPLC method. Ignoring.\n"); return 0; } - limits = &splx->package.elements[1]; - if (limits->type != ACPI_TYPE_PACKAGE || - limits->package.count < 2 || - limits->package.elements[0].type != ACPI_TYPE_INTEGER || - limits->package.elements[1].type != ACPI_TYPE_INTEGER) { - IWL_ERR(trans, "Invalid limits element\n"); - return 0; + /* loop through all the packages to find the one for WiFi */ + for (i = 1; i < splc->package.count; i++) { + union acpi_object *domain; + + data_pkg = &splc->package.elements[i]; + + /* Skip anything that is not a package with the right + * amount of elements (i.e. at least 2 integers). + */ + if (data_pkg->type != ACPI_TYPE_PACKAGE || + data_pkg->package.count < 2 || + data_pkg->package.elements[0].type != ACPI_TYPE_INTEGER || + data_pkg->package.elements[1].type != ACPI_TYPE_INTEGER) + continue; + + domain = &data_pkg->package.elements[0]; + if (domain->integer.value == ACPI_SPLC_DOMAIN_WIFI) + break; + + data_pkg = NULL; } - domain_type = &limits->package.elements[0]; - power_limit = &limits->package.elements[1]; - if (!(domain_type->integer.value & SPL_DOMAINTYPE_WIFI)) { - IWL_DEBUG_INFO(trans, "WiFi power is not limited\n"); + if (!data_pkg) { + IWL_DEBUG_INFO(trans, + "No element for the WiFi domain returned by the SPLC method.\n"); return 0; } - return power_limit->integer.value; + dflt_pwr_limit = &data_pkg->package.elements[1]; + return dflt_pwr_limit->integer.value; } static void set_dflt_pwr_limit(struct iwl_trans *trans, struct pci_dev *pdev) { acpi_handle pxsx_handle; acpi_handle handle; - struct acpi_buffer splx = {ACPI_ALLOCATE_BUFFER, NULL}; + struct acpi_buffer splc = {ACPI_ALLOCATE_BUFFER, NULL}; acpi_status status; pxsx_handle = ACPI_HANDLE(&pdev->dev); @@ -563,23 +594,24 @@ static void set_dflt_pwr_limit(struct iwl_trans *trans, struct pci_dev *pdev) } /* Get the method's handle */ - status = acpi_get_handle(pxsx_handle, (acpi_string)SPL_METHOD, &handle); + status = acpi_get_handle(pxsx_handle, (acpi_string)ACPI_SPLC_METHOD, + &handle); if (ACPI_FAILURE(status)) { - IWL_DEBUG_INFO(trans, "SPL method not found\n"); + IWL_DEBUG_INFO(trans, "SPLC method not found\n"); return; } /* Call SPLC with no arguments */ - status = acpi_evaluate_object(handle, NULL, NULL, &splx); + status = acpi_evaluate_object(handle, NULL, NULL, &splc); if (ACPI_FAILURE(status)) { IWL_ERR(trans, "SPLC invocation failed (0x%x)\n", status); return; } - trans->dflt_pwr_limit = splx_get_pwr_limit(trans, splx.pointer); + trans->dflt_pwr_limit = splc_get_pwr_limit(trans, splc.pointer); IWL_DEBUG_INFO(trans, "Default power limit set to %lld\n", trans->dflt_pwr_limit); - kfree(splx.pointer); + kfree(splc.pointer); } #else /* CONFIG_ACPI */ @@ -593,8 +625,8 @@ static int iwl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { const struct iwl_cfg *cfg = (struct iwl_cfg *)(ent->driver_data); const struct iwl_cfg *cfg_7265d __maybe_unused = NULL; + const struct iwl_cfg *cfg_9260lc __maybe_unused = NULL; struct iwl_trans *iwl_trans; - struct iwl_trans_pcie *trans_pcie; int ret; iwl_trans = iwl_trans_pcie_alloc(pdev, ent, cfg); @@ -620,15 +652,22 @@ static int iwl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) cfg = cfg_7265d; iwl_trans->cfg = cfg_7265d; } + + if (iwl_trans->cfg->rf_id) { + if (cfg == &iwl9260_2ac_cfg) + cfg_9260lc = &iwl9260lc_2ac_cfg; + if (cfg_9260lc && iwl_trans->hw_rf_id == CSR_HW_RF_ID_TYPE_LC) { + cfg = cfg_9260lc; + iwl_trans->cfg = cfg_9260lc; + } + } #endif pci_set_drvdata(pdev, iwl_trans); + iwl_trans->drv = iwl_drv_start(iwl_trans, cfg); - trans_pcie = IWL_TRANS_GET_PCIE_TRANS(iwl_trans); - trans_pcie->drv = iwl_drv_start(iwl_trans, cfg); - - if (IS_ERR(trans_pcie->drv)) { - ret = PTR_ERR(trans_pcie->drv); + if (IS_ERR(iwl_trans->drv)) { + ret = PTR_ERR(iwl_trans->drv); goto out_free_trans; } @@ -661,15 +700,13 @@ static int iwl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* The PCI device starts with a reference taken and we are * supposed to release it here. But to simplify the * interaction with the opmode, we don't do it now, but let - * the opmode release it when it's ready. To account for this - * reference, we start with ref_count set to 1. + * the opmode release it when it's ready. */ - trans_pcie->ref_count = 1; return 0; out_free_drv: - iwl_drv_stop(trans_pcie->drv); + iwl_drv_stop(iwl_trans->drv); out_free_trans: iwl_trans_pcie_free(iwl_trans); return ret; @@ -678,7 +715,6 @@ out_free_trans: static void iwl_pci_remove(struct pci_dev *pdev) { struct iwl_trans *trans = pci_get_drvdata(pdev); - struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); /* if RTPM was in use, restore it to the state before probe */ if (trans->runtime_pm_mode != IWL_PLAT_PM_MODE_DISABLED) { @@ -689,7 +725,7 @@ static void iwl_pci_remove(struct pci_dev *pdev) pm_runtime_forbid(trans->dev); } - iwl_drv_stop(trans_pcie->drv); + iwl_drv_stop(trans->drv); iwl_trans_pcie_free(trans); } diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/internal.h b/drivers/net/wireless/intel/iwlwifi/pcie/internal.h index dadafbdef..11e347dd4 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/internal.h +++ b/drivers/net/wireless/intel/iwlwifi/pcie/internal.h @@ -68,12 +68,14 @@ struct iwl_host_cmd; * struct iwl_rx_mem_buffer * @page_dma: bus address of rxb page * @page: driver's pointer to the rxb page + * @invalid: rxb is in driver ownership - not owned by HW * @vid: index of this rxb in the global table */ struct iwl_rx_mem_buffer { dma_addr_t page_dma; struct page *page; u16 vid; + bool invalid; struct list_head list; }; @@ -230,15 +232,16 @@ struct iwl_queue { #define TFD_CMD_SLOTS 32 /* - * The FH will write back to the first TB only, so we need - * to copy some data into the buffer regardless of whether - * it should be mapped or not. This indicates how big the - * first TB must be to include the scratch buffer. Since - * the scratch is 4 bytes at offset 12, it's 16 now. If we - * make it bigger then allocations will be bigger and copy - * slower, so that's probably not useful. + * The FH will write back to the first TB only, so we need to copy some data + * into the buffer regardless of whether it should be mapped or not. + * This indicates how big the first TB must be to include the scratch buffer + * and the assigned PN. + * Since PN location is 16 bytes at offset 24, it's 40 now. + * If we make it bigger then allocations will be bigger and copy slower, so + * that's probably not useful. */ -#define IWL_HCMD_SCRATCHBUF_SIZE 16 +#define IWL_FIRST_TB_SIZE 40 +#define IWL_FIRST_TB_SIZE_ALIGN ALIGN(IWL_FIRST_TB_SIZE, 64) struct iwl_pcie_txq_entry { struct iwl_device_cmd *cmd; @@ -248,20 +251,18 @@ struct iwl_pcie_txq_entry { struct iwl_cmd_meta meta; }; -struct iwl_pcie_txq_scratch_buf { - struct iwl_cmd_header hdr; - u8 buf[8]; - __le32 scratch; +struct iwl_pcie_first_tb_buf { + u8 buf[IWL_FIRST_TB_SIZE_ALIGN]; }; /** * struct iwl_txq - Tx Queue for DMA * @q: generic Rx/Tx queue descriptor * @tfds: transmit frame descriptors (DMA memory) - * @scratchbufs: start of command headers, including scratch buffers, for + * @first_tb_bufs: start of command headers, including scratch buffers, for * the writeback -- this is DMA memory and an array holding one buffer * for each command on the queue - * @scratchbufs_dma: DMA address for the scratchbufs start + * @first_tb_dma: DMA address for the first_tb_bufs start * @entries: transmit entries (driver state) * @lock: queue lock * @stuck_timer: timer that fires if queue gets stuck @@ -279,8 +280,8 @@ struct iwl_pcie_txq_scratch_buf { struct iwl_txq { struct iwl_queue q; struct iwl_tfd *tfds; - struct iwl_pcie_txq_scratch_buf *scratchbufs; - dma_addr_t scratchbufs_dma; + struct iwl_pcie_first_tb_buf *first_tb_bufs; + dma_addr_t first_tb_dma; struct iwl_pcie_txq_entry *entries; spinlock_t lock; unsigned long frozen_expiry_remainder; @@ -296,10 +297,10 @@ struct iwl_txq { }; static inline dma_addr_t -iwl_pcie_get_scratchbuf_dma(struct iwl_txq *txq, int idx) +iwl_pcie_get_first_tb_dma(struct iwl_txq *txq, int idx) { - return txq->scratchbufs_dma + - sizeof(struct iwl_pcie_txq_scratch_buf) * idx; + return txq->first_tb_dma + + sizeof(struct iwl_pcie_first_tb_buf) * idx; } struct iwl_tso_hdr_page { @@ -313,7 +314,6 @@ struct iwl_tso_hdr_page { * @rx_pool: initial pool of iwl_rx_mem_buffer for all the queues * @global_table: table mapping received VID from hw to rxb * @rba: allocator for RX replenishing - * @drv - pointer to iwl_drv * @trans: pointer to the generic transport area * @scd_base_addr: scheduler sram base address in SRAM * @scd_bc_tbls: pointer to the byte count table of the scheduler @@ -348,10 +348,9 @@ struct iwl_tso_hdr_page { struct iwl_trans_pcie { struct iwl_rxq *rxq; struct iwl_rx_mem_buffer rx_pool[RX_POOL_SIZE]; - struct iwl_rx_mem_buffer *global_table[MQ_RX_TABLE_SIZE]; + struct iwl_rx_mem_buffer *global_table[RX_POOL_SIZE]; struct iwl_rb_allocator rba; struct iwl_trans *trans; - struct iwl_drv *drv; struct net_device napi_dev; @@ -385,6 +384,8 @@ struct iwl_trans_pcie { wait_queue_head_t wait_command_queue; wait_queue_head_t d0i3_waitq; + u8 page_offs, dev_cmd_offs; + u8 cmd_queue; u8 cmd_fifo; unsigned int cmd_q_wdg_timeout; @@ -403,10 +404,6 @@ struct iwl_trans_pcie { bool cmd_hold_nic_awake; bool ref_cmd_in_flight; - /* protect ref counter */ - spinlock_t ref_lock; - u32 ref_count; - dma_addr_t fw_mon_phys; struct page *fw_mon_page; u32 fw_mon_size; @@ -475,6 +472,10 @@ void iwl_trans_pcie_txq_enable(struct iwl_trans *trans, int queue, u16 ssn, unsigned int wdg_timeout); void iwl_trans_pcie_txq_disable(struct iwl_trans *trans, int queue, bool configure_scd); +void iwl_trans_pcie_txq_set_shared_mode(struct iwl_trans *trans, u32 txq_id, + bool shared_mode); +void iwl_trans_pcie_log_scd_error(struct iwl_trans *trans, + struct iwl_txq *txq); int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb, struct iwl_device_cmd *dev_cmd, int txq_id); void iwl_pcie_txq_check_wrptrs(struct iwl_trans *trans); @@ -485,9 +486,6 @@ void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn, struct sk_buff_head *skbs); void iwl_trans_pcie_tx_reset(struct iwl_trans *trans); -void iwl_trans_pcie_ref(struct iwl_trans *trans); -void iwl_trans_pcie_unref(struct iwl_trans *trans); - static inline u16 iwl_pcie_tfd_tb_get_len(struct iwl_tfd *tfd, u8 idx) { struct iwl_tfd_tb *tb = &tfd->tbs[idx]; @@ -503,7 +501,7 @@ void iwl_pcie_dump_csr(struct iwl_trans *trans); /***************************************************** * Helpers ******************************************************/ -static inline void iwl_disable_interrupts(struct iwl_trans *trans) +static inline void _iwl_disable_interrupts(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); @@ -526,7 +524,16 @@ static inline void iwl_disable_interrupts(struct iwl_trans *trans) IWL_DEBUG_ISR(trans, "Disabled interrupts\n"); } -static inline void iwl_enable_interrupts(struct iwl_trans *trans) +static inline void iwl_disable_interrupts(struct iwl_trans *trans) +{ + struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); + + spin_lock(&trans_pcie->irq_lock); + _iwl_disable_interrupts(trans); + spin_unlock(&trans_pcie->irq_lock); +} + +static inline void _iwl_enable_interrupts(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); @@ -549,6 +556,14 @@ static inline void iwl_enable_interrupts(struct iwl_trans *trans) } } +static inline void iwl_enable_interrupts(struct iwl_trans *trans) +{ + struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); + + spin_lock(&trans_pcie->irq_lock); + _iwl_enable_interrupts(trans); + spin_unlock(&trans_pcie->irq_lock); +} static inline void iwl_enable_hw_int_msk_msix(struct iwl_trans *trans, u32 msk) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); @@ -680,4 +695,6 @@ static inline int iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans) int iwl_pci_fw_exit_d0i3(struct iwl_trans *trans); int iwl_pci_fw_enter_d0i3(struct iwl_trans *trans); +void iwl_pcie_enable_rx_wake(struct iwl_trans *trans, bool enable); + #endif /* __iwl_trans_int_pcie_h__ */ diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/rx.c b/drivers/net/wireless/intel/iwlwifi/pcie/rx.c index 4be3c35af..5c36e6d00 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/rx.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/rx.c @@ -161,20 +161,21 @@ static inline __le32 iwl_pcie_dma_addr2rbd_ptr(dma_addr_t dma_addr) return cpu_to_le32((u32)(dma_addr >> 8)); } -static void iwl_pcie_write_prph_64(struct iwl_trans *trans, u64 ofs, u64 val) -{ - iwl_write_prph(trans, ofs, val & 0xffffffff); - iwl_write_prph(trans, ofs + 4, val >> 32); -} - /* * iwl_pcie_rx_stop - stops the Rx DMA */ int iwl_pcie_rx_stop(struct iwl_trans *trans) { - iwl_write_direct32(trans, FH_MEM_RCSR_CHNL0_CONFIG_REG, 0); - return iwl_poll_direct_bit(trans, FH_MEM_RSSR_RX_STATUS_REG, - FH_RSSR_CHNL0_RX_STATUS_CHNL_IDLE, 1000); + if (trans->cfg->mq_rx_supported) { + iwl_write_prph(trans, RFH_RXF_DMA_CFG, 0); + return iwl_poll_prph_bit(trans, RFH_GEN_STATUS, + RXF_DMA_IDLE, RXF_DMA_IDLE, 1000); + } else { + iwl_write_direct32(trans, FH_MEM_RCSR_CHNL0_CONFIG_REG, 0); + return iwl_poll_direct_bit(trans, FH_MEM_RSSR_RX_STATUS_REG, + FH_RSSR_CHNL0_RX_STATUS_CHNL_IDLE, + 1000); + } } /* @@ -208,8 +209,8 @@ static void iwl_pcie_rxq_inc_wr_ptr(struct iwl_trans *trans, rxq->write_actual = round_down(rxq->write, 8); if (trans->cfg->mq_rx_supported) - iwl_write_prph(trans, RFH_Q_FRBDCB_WIDX(rxq->id), - rxq->write_actual); + iwl_write32(trans, RFH_Q_FRBDCB_WIDX_TRG(rxq->id), + rxq->write_actual); else iwl_write32(trans, FH_RSCSR_CHNL0_WPTR, rxq->write_actual); } @@ -232,10 +233,10 @@ static void iwl_pcie_rxq_check_wrptr(struct iwl_trans *trans) } /* - * iwl_pcie_rxq_mq_restock - restock implementation for multi-queue rx + * iwl_pcie_rxmq_restock - restock implementation for multi-queue rx */ -static void iwl_pcie_rxq_mq_restock(struct iwl_trans *trans, - struct iwl_rxq *rxq) +static void iwl_pcie_rxmq_restock(struct iwl_trans *trans, + struct iwl_rxq *rxq) { struct iwl_rx_mem_buffer *rxb; @@ -258,7 +259,7 @@ static void iwl_pcie_rxq_mq_restock(struct iwl_trans *trans, rxb = list_first_entry(&rxq->rx_free, struct iwl_rx_mem_buffer, list); list_del(&rxb->list); - + rxb->invalid = false; /* 12 first bits are expected to be empty */ WARN_ON(rxb->page_dma & DMA_BIT_MASK(12)); /* Point to Rx buffer via next RBD in circular buffer */ @@ -280,10 +281,10 @@ static void iwl_pcie_rxq_mq_restock(struct iwl_trans *trans, } /* - * iwl_pcie_rxq_sq_restock - restock implementation for single queue rx + * iwl_pcie_rxsq_restock - restock implementation for single queue rx */ -static void iwl_pcie_rxq_sq_restock(struct iwl_trans *trans, - struct iwl_rxq *rxq) +static void iwl_pcie_rxsq_restock(struct iwl_trans *trans, + struct iwl_rxq *rxq) { struct iwl_rx_mem_buffer *rxb; @@ -309,6 +310,7 @@ static void iwl_pcie_rxq_sq_restock(struct iwl_trans *trans, rxb = list_first_entry(&rxq->rx_free, struct iwl_rx_mem_buffer, list); list_del(&rxb->list); + rxb->invalid = false; /* Point to Rx buffer via next RBD in circular buffer */ bd[rxq->write] = iwl_pcie_dma_addr2rbd_ptr(rxb->page_dma); @@ -342,9 +344,9 @@ static void iwl_pcie_rxq_restock(struct iwl_trans *trans, struct iwl_rxq *rxq) { if (trans->cfg->mq_rx_supported) - iwl_pcie_rxq_mq_restock(trans, rxq); + iwl_pcie_rxmq_restock(trans, rxq); else - iwl_pcie_rxq_sq_restock(trans, rxq); + iwl_pcie_rxsq_restock(trans, rxq); } /* @@ -694,6 +696,7 @@ static void iwl_pcie_rx_hw_init(struct iwl_trans *trans, struct iwl_rxq *rxq) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); u32 rb_size; + unsigned long flags; const u32 rfdnlog = RX_QUEUE_SIZE_LOG; /* 256 RBDs */ switch (trans_pcie->rx_buf_size) { @@ -711,23 +714,26 @@ static void iwl_pcie_rx_hw_init(struct iwl_trans *trans, struct iwl_rxq *rxq) rb_size = FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_4K; } + if (!iwl_trans_grab_nic_access(trans, &flags)) + return; + /* Stop Rx DMA */ - iwl_write_direct32(trans, FH_MEM_RCSR_CHNL0_CONFIG_REG, 0); + iwl_write32(trans, FH_MEM_RCSR_CHNL0_CONFIG_REG, 0); /* reset and flush pointers */ - iwl_write_direct32(trans, FH_MEM_RCSR_CHNL0_RBDCB_WPTR, 0); - iwl_write_direct32(trans, FH_MEM_RCSR_CHNL0_FLUSH_RB_REQ, 0); - iwl_write_direct32(trans, FH_RSCSR_CHNL0_RDPTR, 0); + iwl_write32(trans, FH_MEM_RCSR_CHNL0_RBDCB_WPTR, 0); + iwl_write32(trans, FH_MEM_RCSR_CHNL0_FLUSH_RB_REQ, 0); + iwl_write32(trans, FH_RSCSR_CHNL0_RDPTR, 0); /* Reset driver's Rx queue write index */ - iwl_write_direct32(trans, FH_RSCSR_CHNL0_RBDCB_WPTR_REG, 0); + iwl_write32(trans, FH_RSCSR_CHNL0_RBDCB_WPTR_REG, 0); /* Tell device where to find RBD circular buffer in DRAM */ - iwl_write_direct32(trans, FH_RSCSR_CHNL0_RBDCB_BASE_REG, - (u32)(rxq->bd_dma >> 8)); + iwl_write32(trans, FH_RSCSR_CHNL0_RBDCB_BASE_REG, + (u32)(rxq->bd_dma >> 8)); /* Tell device where in DRAM to update its Rx status */ - iwl_write_direct32(trans, FH_RSCSR_CHNL0_STTS_WPTR_REG, - rxq->rb_stts_dma >> 4); + iwl_write32(trans, FH_RSCSR_CHNL0_STTS_WPTR_REG, + rxq->rb_stts_dma >> 4); /* Enable Rx DMA * FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY is set because of HW bug in @@ -737,13 +743,15 @@ static void iwl_pcie_rx_hw_init(struct iwl_trans *trans, struct iwl_rxq *rxq) * RB timeout 0x10 * 256 RBDs */ - iwl_write_direct32(trans, FH_MEM_RCSR_CHNL0_CONFIG_REG, - FH_RCSR_RX_CONFIG_CHNL_EN_ENABLE_VAL | - FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY | - FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_INT_HOST_VAL | - rb_size| - (RX_RB_TIMEOUT << FH_RCSR_RX_CONFIG_REG_IRQ_RBTH_POS)| - (rfdnlog << FH_RCSR_RX_CONFIG_RBDCB_SIZE_POS)); + iwl_write32(trans, FH_MEM_RCSR_CHNL0_CONFIG_REG, + FH_RCSR_RX_CONFIG_CHNL_EN_ENABLE_VAL | + FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY | + FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_INT_HOST_VAL | + rb_size | + (RX_RB_TIMEOUT << FH_RCSR_RX_CONFIG_REG_IRQ_RBTH_POS) | + (rfdnlog << FH_RCSR_RX_CONFIG_RBDCB_SIZE_POS)); + + iwl_trans_release_nic_access(trans, &flags); /* Set interrupt coalescing timer to default (2048 usecs) */ iwl_write8(trans, CSR_INT_COALESCING, IWL_HOST_INT_TIMEOUT_DEF); @@ -753,10 +761,28 @@ static void iwl_pcie_rx_hw_init(struct iwl_trans *trans, struct iwl_rxq *rxq) iwl_set_bit(trans, CSR_INT_COALESCING, IWL_HOST_INT_OPER_MODE); } +void iwl_pcie_enable_rx_wake(struct iwl_trans *trans, bool enable) +{ + /* + * Turn on the chicken-bits that cause MAC wakeup for RX-related + * values. + * This costs some power, but needed for W/A 9000 integrated A-step + * bug where shadow registers are not in the retention list and their + * value is lost when NIC powers down + */ + if (trans->cfg->integrated) { + iwl_set_bit(trans, CSR_MAC_SHADOW_REG_CTRL, + CSR_MAC_SHADOW_REG_CTRL_RX_WAKE); + iwl_set_bit(trans, CSR_MAC_SHADOW_REG_CTL2, + CSR_MAC_SHADOW_REG_CTL2_RX_WAKE); + } +} + static void iwl_pcie_rx_mq_hw_init(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); u32 rb_size, enabled = 0; + unsigned long flags; int i; switch (trans_pcie->rx_buf_size) { @@ -774,60 +800,70 @@ static void iwl_pcie_rx_mq_hw_init(struct iwl_trans *trans) rb_size = RFH_RXF_DMA_RB_SIZE_4K; } + if (!iwl_trans_grab_nic_access(trans, &flags)) + return; + /* Stop Rx DMA */ - iwl_write_prph(trans, RFH_RXF_DMA_CFG, 0); + iwl_write_prph_no_grab(trans, RFH_RXF_DMA_CFG, 0); /* disable free amd used rx queue operation */ - iwl_write_prph(trans, RFH_RXF_RXQ_ACTIVE, 0); + iwl_write_prph_no_grab(trans, RFH_RXF_RXQ_ACTIVE, 0); for (i = 0; i < trans->num_rx_queues; i++) { /* Tell device where to find RBD free table in DRAM */ - iwl_pcie_write_prph_64(trans, RFH_Q_FRBDCB_BA_LSB(i), - (u64)(trans_pcie->rxq[i].bd_dma)); + iwl_write_prph64_no_grab(trans, + RFH_Q_FRBDCB_BA_LSB(i), + trans_pcie->rxq[i].bd_dma); /* Tell device where to find RBD used table in DRAM */ - iwl_pcie_write_prph_64(trans, RFH_Q_URBDCB_BA_LSB(i), - (u64)(trans_pcie->rxq[i].used_bd_dma)); + iwl_write_prph64_no_grab(trans, + RFH_Q_URBDCB_BA_LSB(i), + trans_pcie->rxq[i].used_bd_dma); /* Tell device where in DRAM to update its Rx status */ - iwl_pcie_write_prph_64(trans, RFH_Q_URBD_STTS_WPTR_LSB(i), - trans_pcie->rxq[i].rb_stts_dma); + iwl_write_prph64_no_grab(trans, + RFH_Q_URBD_STTS_WPTR_LSB(i), + trans_pcie->rxq[i].rb_stts_dma); /* Reset device indice tables */ - iwl_write_prph(trans, RFH_Q_FRBDCB_WIDX(i), 0); - iwl_write_prph(trans, RFH_Q_FRBDCB_RIDX(i), 0); - iwl_write_prph(trans, RFH_Q_URBDCB_WIDX(i), 0); + iwl_write_prph_no_grab(trans, RFH_Q_FRBDCB_WIDX(i), 0); + iwl_write_prph_no_grab(trans, RFH_Q_FRBDCB_RIDX(i), 0); + iwl_write_prph_no_grab(trans, RFH_Q_URBDCB_WIDX(i), 0); enabled |= BIT(i) | BIT(i + 16); } - /* restock default queue */ - iwl_pcie_rxq_mq_restock(trans, &trans_pcie->rxq[0]); - /* * Enable Rx DMA - * Single frame mode * Rx buffer size 4 or 8k or 12k * Min RB size 4 or 8 * Drop frames that exceed RB size * 512 RBDs */ - iwl_write_prph(trans, RFH_RXF_DMA_CFG, - RFH_DMA_EN_ENABLE_VAL | - rb_size | RFH_RXF_DMA_SINGLE_FRAME_MASK | - RFH_RXF_DMA_MIN_RB_4_8 | - RFH_RXF_DMA_DROP_TOO_LARGE_MASK | - RFH_RXF_DMA_RBDCB_SIZE_512); + iwl_write_prph_no_grab(trans, RFH_RXF_DMA_CFG, + RFH_DMA_EN_ENABLE_VAL | rb_size | + RFH_RXF_DMA_MIN_RB_4_8 | + RFH_RXF_DMA_DROP_TOO_LARGE_MASK | + RFH_RXF_DMA_RBDCB_SIZE_512); /* * Activate DMA snooping. - * Set RX DMA chunk size to 64B + * Set RX DMA chunk size to 64B for IOSF and 128B for PCIe * Default queue is 0 */ - iwl_write_prph(trans, RFH_GEN_CFG, RFH_GEN_CFG_RFH_DMA_SNOOP | - (DEFAULT_RXQ_NUM << RFH_GEN_CFG_DEFAULT_RXQ_NUM_POS) | - RFH_GEN_CFG_SERVICE_DMA_SNOOP); + iwl_write_prph_no_grab(trans, RFH_GEN_CFG, RFH_GEN_CFG_RFH_DMA_SNOOP | + (DEFAULT_RXQ_NUM << + RFH_GEN_CFG_DEFAULT_RXQ_NUM_POS) | + RFH_GEN_CFG_SERVICE_DMA_SNOOP | + (trans->cfg->integrated ? + RFH_GEN_CFG_RB_CHUNK_SIZE_64 : + RFH_GEN_CFG_RB_CHUNK_SIZE_128) << + RFH_GEN_CFG_RB_CHUNK_SIZE_POS); /* Enable the relevant rx queues */ - iwl_write_prph(trans, RFH_RXF_RXQ_ACTIVE, enabled); + iwl_write_prph_no_grab(trans, RFH_RXF_RXQ_ACTIVE, enabled); + + iwl_trans_release_nic_access(trans, &flags); /* Set interrupt coalescing timer to default (2048 usecs) */ iwl_write8(trans, CSR_INT_COALESCING, IWL_HOST_INT_TIMEOUT_DEF); + + iwl_pcie_enable_rx_wake(trans, true); } static void iwl_pcie_rx_init_rxb_lists(struct iwl_rxq *rxq) @@ -908,6 +944,8 @@ int iwl_pcie_rx_init(struct iwl_trans *trans) allocator_pool_size = trans->num_rx_queues * (RX_CLAIM_REQ_ALLOC - RX_POST_REQ_ALLOC); num_alloc = queue_size + allocator_pool_size; + BUILD_BUG_ON(ARRAY_SIZE(trans_pcie->global_table) != + ARRAY_SIZE(trans_pcie->rx_pool)); for (i = 0; i < num_alloc; i++) { struct iwl_rx_mem_buffer *rxb = &trans_pcie->rx_pool[i]; @@ -916,16 +954,18 @@ int iwl_pcie_rx_init(struct iwl_trans *trans) else list_add(&rxb->list, &def_rxq->rx_used); trans_pcie->global_table[i] = rxb; - rxb->vid = (u16)i; + rxb->vid = (u16)(i + 1); + rxb->invalid = true; } iwl_pcie_rxq_alloc_rbs(trans, GFP_KERNEL, def_rxq); - if (trans->cfg->mq_rx_supported) { + + if (trans->cfg->mq_rx_supported) iwl_pcie_rx_mq_hw_init(trans); - } else { - iwl_pcie_rxq_sq_restock(trans, def_rxq); + else iwl_pcie_rx_hw_init(trans, def_rxq); - } + + iwl_pcie_rxq_restock(trans, def_rxq); spin_lock(&def_rxq->lock); iwl_pcie_rxq_inc_wr_ptr(trans, def_rxq); @@ -1064,6 +1104,9 @@ static void iwl_pcie_rx_handle_rb(struct iwl_trans *trans, if (pkt->len_n_flags == cpu_to_le32(FH_RSCSR_FRAME_INVALID)) break; + WARN_ON((le32_to_cpu(pkt->len_n_flags) & FH_RSCSR_RXQ_MASK) >> + FH_RSCSR_RXQ_POS != rxq->id); + IWL_DEBUG_RX(trans, "cmd at offset %d: %s (0x%.2x, seq 0x%x)\n", rxcb._offset, @@ -1201,10 +1244,19 @@ restart: */ u16 vid = le32_to_cpu(rxq->used_bd[i]) & 0x0FFF; - if (WARN(vid >= ARRAY_SIZE(trans_pcie->global_table), - "Invalid rxb index from HW %u\n", (u32)vid)) + if (WARN(!vid || + vid > ARRAY_SIZE(trans_pcie->global_table), + "Invalid rxb index from HW %u\n", (u32)vid)) { + iwl_force_nmi(trans); + goto out; + } + rxb = trans_pcie->global_table[vid - 1]; + if (WARN(rxb->invalid, + "Invalid rxb from HW %u\n", (u32)vid)) { + iwl_force_nmi(trans); goto out; - rxb = trans_pcie->global_table[vid]; + } + rxb->invalid = true; } else { rxb = rxq->queue[i]; rxq->queue[i] = NULL; @@ -1292,7 +1344,7 @@ static inline void iwl_pcie_clear_irq(struct iwl_trans *trans, * write 1 clear (W1C) register, meaning that it's being clear * by writing 1 to the bit. */ - iwl_write_direct32(trans, CSR_MSIX_AUTOMASK_ST_AD, BIT(entry->entry)); + iwl_write32(trans, CSR_MSIX_AUTOMASK_ST_AD, BIT(entry->entry)); } /* @@ -1484,7 +1536,7 @@ irqreturn_t iwl_pcie_irq_handler(int irq, void *dev_id) * have anything to service */ if (test_bit(STATUS_INT_ENABLED, &trans->status)) - iwl_enable_interrupts(trans); + _iwl_enable_interrupts(trans); spin_unlock(&trans_pcie->irq_lock); lock_map_release(&trans->sync_cmd_lockdep_map); return IRQ_NONE; @@ -1676,15 +1728,17 @@ irqreturn_t iwl_pcie_irq_handler(int irq, void *dev_id) inta & ~trans_pcie->inta_mask); } + spin_lock(&trans_pcie->irq_lock); + /* only Re-enable all interrupt if disabled by irq */ + if (test_bit(STATUS_INT_ENABLED, &trans->status)) + _iwl_enable_interrupts(trans); /* we are loading the firmware, enable FH_TX interrupt only */ - if (handled & CSR_INT_BIT_FH_TX) + else if (handled & CSR_INT_BIT_FH_TX) iwl_enable_fw_load_int(trans); - /* only Re-enable all interrupt if disabled by irq */ - else if (test_bit(STATUS_INT_ENABLED, &trans->status)) - iwl_enable_interrupts(trans); /* Re-enable RF_KILL if it occurred */ else if (handled & CSR_INT_BIT_RF_KILL) iwl_enable_rfkill_int(trans); + spin_unlock(&trans_pcie->irq_lock); out: lock_map_release(&trans->sync_cmd_lockdep_map); @@ -1748,7 +1802,7 @@ void iwl_pcie_reset_ict(struct iwl_trans *trans) return; spin_lock(&trans_pcie->irq_lock); - iwl_disable_interrupts(trans); + _iwl_disable_interrupts(trans); memset(trans_pcie->ict_tbl, 0, ICT_SIZE); @@ -1764,7 +1818,7 @@ void iwl_pcie_reset_ict(struct iwl_trans *trans) trans_pcie->use_ict = true; trans_pcie->ict_index = 0; iwl_write32(trans, CSR_INT, trans_pcie->inta_mask); - iwl_enable_interrupts(trans); + _iwl_enable_interrupts(trans); spin_unlock(&trans_pcie->irq_lock); } @@ -1805,19 +1859,19 @@ irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id) struct msix_entry *entry = dev_id; struct iwl_trans_pcie *trans_pcie = iwl_pcie_get_trans_pcie(entry); struct iwl_trans *trans = trans_pcie->trans; - struct isr_statistics *isr_stats = isr_stats = &trans_pcie->isr_stats; + struct isr_statistics *isr_stats = &trans_pcie->isr_stats; u32 inta_fh, inta_hw; lock_map_acquire(&trans->sync_cmd_lockdep_map); spin_lock(&trans_pcie->irq_lock); - inta_fh = iwl_read_direct32(trans, CSR_MSIX_FH_INT_CAUSES_AD); - inta_hw = iwl_read_direct32(trans, CSR_MSIX_HW_INT_CAUSES_AD); + inta_fh = iwl_read32(trans, CSR_MSIX_FH_INT_CAUSES_AD); + inta_hw = iwl_read32(trans, CSR_MSIX_HW_INT_CAUSES_AD); /* * Clear causes registers to avoid being handling the same cause. */ - iwl_write_direct32(trans, CSR_MSIX_FH_INT_CAUSES_AD, inta_fh); - iwl_write_direct32(trans, CSR_MSIX_HW_INT_CAUSES_AD, inta_hw); + iwl_write32(trans, CSR_MSIX_FH_INT_CAUSES_AD, inta_fh); + iwl_write32(trans, CSR_MSIX_HW_INT_CAUSES_AD, inta_hw); spin_unlock(&trans_pcie->irq_lock); if (unlikely(!(inta_fh | inta_hw))) { diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c index b2b79354d..74f2f035b 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c @@ -269,9 +269,8 @@ static int iwl_pcie_apm_init(struct iwl_trans *trans) iwl_pcie_apm_config(trans); /* Configure analog phase-lock-loop before activating to D0A */ - if (trans->cfg->base_params->pll_cfg_val) - iwl_set_bit(trans, CSR_ANA_PLL_CFG, - trans->cfg->base_params->pll_cfg_val); + if (trans->cfg->base_params->pll_cfg) + iwl_set_bit(trans, CSR_ANA_PLL_CFG, CSR50_ANA_PLL_CFG_VAL); /* * Set "initialization complete" bit to move adapter from @@ -361,8 +360,7 @@ static void iwl_pcie_apm_lp_xtal_enable(struct iwl_trans *trans) /* Reset entire device - do controller reset (results in SHRD_HW_RST) */ iwl_set_bit(trans, CSR_RESET, CSR_RESET_REG_FLAG_SW_RESET); - - udelay(10); + usleep_range(1000, 2000); /* * Set "initialization complete" bit to move adapter from @@ -408,8 +406,7 @@ static void iwl_pcie_apm_lp_xtal_enable(struct iwl_trans *trans) * SHRD_HW_RST). Turn MAC off before proceeding. */ iwl_set_bit(trans, CSR_RESET, CSR_RESET_REG_FLAG_SW_RESET); - - udelay(10); + usleep_range(1000, 2000); /* Enable LP XTAL by indirect access through CSR */ apmg_gp1_reg = iwl_trans_pcie_read_shr(trans, SHR_APMG_GP1_REG); @@ -506,8 +503,7 @@ static void iwl_pcie_apm_stop(struct iwl_trans *trans, bool op_mode_leave) /* Reset the entire device */ iwl_set_bit(trans, CSR_RESET, CSR_RESET_REG_FLAG_SW_RESET); - - udelay(10); + usleep_range(1000, 2000); /* * Clear "initialization complete" bit to move adapter from @@ -586,7 +582,7 @@ static int iwl_pcie_prepare_card_hw(struct iwl_trans *trans) iwl_set_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG, CSR_RESET_LINK_PWR_MGMT_DISABLED); - msleep(1); + usleep_range(1000, 2000); for (iter = 0; iter < 10; iter++) { /* If HW is not ready, prepare the conditions to check again */ @@ -612,18 +608,10 @@ static int iwl_pcie_prepare_card_hw(struct iwl_trans *trans) /* * ucode */ -static int iwl_pcie_load_firmware_chunk(struct iwl_trans *trans, u32 dst_addr, - dma_addr_t phy_addr, u32 byte_cnt) +static void iwl_pcie_load_firmware_chunk_fh(struct iwl_trans *trans, + u32 dst_addr, dma_addr_t phy_addr, + u32 byte_cnt) { - struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); - unsigned long flags; - int ret; - - trans_pcie->ucode_write_complete = false; - - if (!iwl_trans_grab_nic_access(trans, &flags)) - return -EIO; - iwl_write32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(FH_SRVC_CHNL), FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE); @@ -646,7 +634,50 @@ static int iwl_pcie_load_firmware_chunk(struct iwl_trans *trans, u32 dst_addr, FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE | FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_DISABLE | FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_ENDTFD); +} + +static void iwl_pcie_load_firmware_chunk_tfh(struct iwl_trans *trans, + u32 dst_addr, dma_addr_t phy_addr, + u32 byte_cnt) +{ + /* Stop DMA channel */ + iwl_write32(trans, TFH_SRV_DMA_CHNL0_CTRL, 0); + /* Configure SRAM address */ + iwl_write32(trans, TFH_SRV_DMA_CHNL0_SRAM_ADDR, + dst_addr); + + /* Configure DRAM address - 64 bit */ + iwl_write64(trans, TFH_SRV_DMA_CHNL0_DRAM_ADDR, phy_addr); + + /* Configure byte count to transfer */ + iwl_write32(trans, TFH_SRV_DMA_CHNL0_BC, byte_cnt); + + /* Enable the DRAM2SRAM to start */ + iwl_write32(trans, TFH_SRV_DMA_CHNL0_CTRL, TFH_SRV_DMA_SNOOP | + TFH_SRV_DMA_TO_DRIVER | + TFH_SRV_DMA_START); +} + +static int iwl_pcie_load_firmware_chunk(struct iwl_trans *trans, + u32 dst_addr, dma_addr_t phy_addr, + u32 byte_cnt) +{ + struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); + unsigned long flags; + int ret; + + trans_pcie->ucode_write_complete = false; + + if (!iwl_trans_grab_nic_access(trans, &flags)) + return -EIO; + + if (trans->cfg->use_tfh) + iwl_pcie_load_firmware_chunk_tfh(trans, dst_addr, phy_addr, + byte_cnt); + else + iwl_pcie_load_firmware_chunk_fh(trans, dst_addr, phy_addr, + byte_cnt); iwl_trans_release_nic_access(trans, &flags); ret = wait_event_timeout(trans_pcie->ucode_write_waitq, @@ -805,6 +836,8 @@ static int iwl_pcie_load_cpu_sections_8000(struct iwl_trans *trans, *first_ucode_section = last_read_idx; + iwl_enable_interrupts(trans); + if (cpu == 1) iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS, 0xFFFF); else @@ -984,6 +1017,8 @@ static int iwl_pcie_load_given_ucode(struct iwl_trans *trans, iwl_pcie_apply_destination(trans); } + iwl_enable_interrupts(trans); + /* release CPU reset */ iwl_write32(trans, CSR_RESET, 0); @@ -1037,9 +1072,7 @@ static void _iwl_trans_pcie_stop_device(struct iwl_trans *trans, bool low_power) was_hw_rfkill = iwl_is_rfkill_set(trans); /* tell the device to stop sending interrupts */ - spin_lock(&trans_pcie->irq_lock); iwl_disable_interrupts(trans); - spin_unlock(&trans_pcie->irq_lock); /* device going down, Stop using ICT table */ iwl_pcie_disable_ict(trans); @@ -1074,7 +1107,7 @@ static void _iwl_trans_pcie_stop_device(struct iwl_trans *trans, bool low_power) /* stop and reset the on-board processor */ iwl_write32(trans, CSR_RESET, CSR_RESET_REG_FLAG_SW_RESET); - udelay(20); + usleep_range(1000, 2000); /* * Upon stop, the APM issues an interrupt if HW RF kill is set. @@ -1083,9 +1116,7 @@ static void _iwl_trans_pcie_stop_device(struct iwl_trans *trans, bool low_power) * the time, unless the interrupt is ACKed even if the interrupt * should be masked. Re-ACK all the interrupts here. */ - spin_lock(&trans_pcie->irq_lock); iwl_disable_interrupts(trans); - spin_unlock(&trans_pcie->irq_lock); /* clear all status bits */ clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status); @@ -1219,7 +1250,6 @@ static int iwl_trans_pcie_start_fw(struct iwl_trans *trans, ret = iwl_pcie_load_given_ucode_8000(trans, fw); else ret = iwl_pcie_load_given_ucode(trans, fw); - iwl_enable_interrupts(trans); /* re-check RF-Kill state since we may have missed the interrupt */ hw_rfkill = iwl_is_rfkill_set(trans); @@ -1290,6 +1320,8 @@ static void iwl_trans_pcie_d3_suspend(struct iwl_trans *trans, bool test, iwl_clear_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE); + iwl_pcie_enable_rx_wake(trans, false); + if (reset) { /* * reset TX queues -- some of their registers reset during S3 @@ -1315,12 +1347,15 @@ static int iwl_trans_pcie_d3_resume(struct iwl_trans *trans, return 0; } + iwl_pcie_enable_rx_wake(trans, true); + /* * Also enables interrupts - none will happen as the device doesn't * know we're waking it up, only when the opmode actually tells it * after this call. */ iwl_pcie_reset_ict(trans); + iwl_enable_interrupts(trans); iwl_set_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); iwl_set_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE); @@ -1392,8 +1427,12 @@ static void iwl_pcie_init_msix(struct iwl_trans_pcie *trans_pcie) max_rx_vector = trans_pcie->allocated_vector - 1; - if (!trans_pcie->msix_enabled) + if (!trans_pcie->msix_enabled) { + if (trans->cfg->mq_rx_supported) + iwl_write_prph(trans, UREG_CHICK, + UREG_CHICK_MSI_ENABLE); return; + } iwl_write_prph(trans, UREG_CHICK, UREG_CHICK_MSIX_ENABLE); @@ -1434,7 +1473,7 @@ static void iwl_pcie_set_interrupt_capa(struct pci_dev *pdev, int ret, i; if (trans->cfg->mq_rx_supported) { - max_vector = min_t(u32, (num_possible_cpus() + 1), + max_vector = min_t(u32, (num_possible_cpus() + 2), IWL_MAX_RX_HW_QUEUES); for (i = 0; i < max_vector; i++) trans_pcie->msix_entries[i].entry = i; @@ -1465,7 +1504,7 @@ static void iwl_pcie_set_interrupt_capa(struct pci_dev *pdev, ret = pci_enable_msi(pdev); if (ret) { - dev_err(&pdev->dev, "pci_enable_msi failed(0X%x)\n", ret); + dev_err(&pdev->dev, "pci_enable_msi failed - %d\n", ret); /* enable rfkill interrupt: hw bug w/a */ pci_read_config_word(pdev, PCI_COMMAND, &pci_cmd); if (pci_cmd & PCI_COMMAND_INTX_DISABLE) { @@ -1499,8 +1538,8 @@ static int iwl_pcie_init_msix_handler(struct pci_dev *pdev, IWL_ERR(trans_pcie->trans, "Error allocating IRQ %d\n", i); for (j = 0; j < i; j++) - free_irq(trans_pcie->msix_entries[i].vector, - &trans_pcie->msix_entries[i]); + free_irq(trans_pcie->msix_entries[j].vector, + &trans_pcie->msix_entries[j]); pci_disable_msix(pdev); return ret; } @@ -1525,8 +1564,7 @@ static int _iwl_trans_pcie_start_hw(struct iwl_trans *trans, bool low_power) /* Reset the entire device */ iwl_write32(trans, CSR_RESET, CSR_RESET_REG_FLAG_SW_RESET); - - usleep_range(10, 15); + usleep_range(1000, 2000); iwl_pcie_apm_init(trans); @@ -1571,15 +1609,11 @@ static void iwl_trans_pcie_op_mode_leave(struct iwl_trans *trans) mutex_lock(&trans_pcie->mutex); /* disable interrupts - don't enable HW RF kill interrupt */ - spin_lock(&trans_pcie->irq_lock); iwl_disable_interrupts(trans); - spin_unlock(&trans_pcie->irq_lock); iwl_pcie_apm_stop(trans, true); - spin_lock(&trans_pcie->irq_lock); iwl_disable_interrupts(trans); - spin_unlock(&trans_pcie->irq_lock); iwl_pcie_disable_ict(trans); @@ -1643,6 +1677,9 @@ static void iwl_trans_pcie_configure(struct iwl_trans *trans, trans_pcie->scd_set_active = trans_cfg->scd_set_active; trans_pcie->sw_csum_tx = trans_cfg->sw_csum_tx; + trans_pcie->page_offs = trans_cfg->cb_data_offs; + trans_pcie->dev_cmd_offs = trans_cfg->cb_data_offs + sizeof(void *); + trans->command_groups = trans_cfg->command_groups; trans->command_groups_size = trans_cfg->command_groups_size; @@ -1694,6 +1731,7 @@ void iwl_trans_pcie_free(struct iwl_trans *trans) } free_percpu(trans_pcie->tso_hdr_page); + mutex_destroy(&trans_pcie->mutex); iwl_trans_free(trans); } @@ -1912,6 +1950,48 @@ static void iwl_trans_pcie_block_txq_ptrs(struct iwl_trans *trans, bool block) #define IWL_FLUSH_WAIT_MS 2000 +void iwl_trans_pcie_log_scd_error(struct iwl_trans *trans, struct iwl_txq *txq) +{ + struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); + u32 scd_sram_addr; + u8 buf[16]; + int cnt; + + IWL_ERR(trans, "Current SW read_ptr %d write_ptr %d\n", + txq->q.read_ptr, txq->q.write_ptr); + + scd_sram_addr = trans_pcie->scd_base_addr + + SCD_TX_STTS_QUEUE_OFFSET(txq->q.id); + iwl_trans_read_mem_bytes(trans, scd_sram_addr, buf, sizeof(buf)); + + iwl_print_hex_error(trans, buf, sizeof(buf)); + + for (cnt = 0; cnt < FH_TCSR_CHNL_NUM; cnt++) + IWL_ERR(trans, "FH TRBs(%d) = 0x%08x\n", cnt, + iwl_read_direct32(trans, FH_TX_TRB_REG(cnt))); + + for (cnt = 0; cnt < trans->cfg->base_params->num_of_queues; cnt++) { + u32 status = iwl_read_prph(trans, SCD_QUEUE_STATUS_BITS(cnt)); + u8 fifo = (status >> SCD_QUEUE_STTS_REG_POS_TXF) & 0x7; + bool active = !!(status & BIT(SCD_QUEUE_STTS_REG_POS_ACTIVE)); + u32 tbl_dw = + iwl_trans_read_mem32(trans, trans_pcie->scd_base_addr + + SCD_TRANS_TBL_OFFSET_QUEUE(cnt)); + + if (cnt & 0x1) + tbl_dw = (tbl_dw & 0xFFFF0000) >> 16; + else + tbl_dw = tbl_dw & 0x0000FFFF; + + IWL_ERR(trans, + "Q %d is %sactive and mapped to fifo %d ra_tid 0x%04x [%d,%d]\n", + cnt, active ? "" : "in", fifo, tbl_dw, + iwl_read_prph(trans, SCD_QUEUE_RDPTR(cnt)) & + (TFD_QUEUE_SIZE_MAX - 1), + iwl_read_prph(trans, SCD_QUEUE_WRPTR(cnt))); + } +} + static int iwl_trans_pcie_wait_txq_empty(struct iwl_trans *trans, u32 txq_bm) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); @@ -1919,8 +1999,6 @@ static int iwl_trans_pcie_wait_txq_empty(struct iwl_trans *trans, u32 txq_bm) struct iwl_queue *q; int cnt; unsigned long now = jiffies; - u32 scd_sram_addr; - u8 buf[16]; int ret = 0; /* waiting for all the tx frames complete might take a while */ @@ -1948,7 +2026,7 @@ static int iwl_trans_pcie_wait_txq_empty(struct iwl_trans *trans, u32 txq_bm) "WR pointer moved while flushing %d -> %d\n", wr_ptr, write_ptr)) return -ETIMEDOUT; - msleep(1); + usleep_range(1000, 2000); } if (q->read_ptr != q->write_ptr) { @@ -1960,42 +2038,8 @@ static int iwl_trans_pcie_wait_txq_empty(struct iwl_trans *trans, u32 txq_bm) IWL_DEBUG_TX_QUEUES(trans, "Queue %d is now empty.\n", cnt); } - if (!ret) - return 0; - - IWL_ERR(trans, "Current SW read_ptr %d write_ptr %d\n", - txq->q.read_ptr, txq->q.write_ptr); - - scd_sram_addr = trans_pcie->scd_base_addr + - SCD_TX_STTS_QUEUE_OFFSET(txq->q.id); - iwl_trans_read_mem_bytes(trans, scd_sram_addr, buf, sizeof(buf)); - - iwl_print_hex_error(trans, buf, sizeof(buf)); - - for (cnt = 0; cnt < FH_TCSR_CHNL_NUM; cnt++) - IWL_ERR(trans, "FH TRBs(%d) = 0x%08x\n", cnt, - iwl_read_direct32(trans, FH_TX_TRB_REG(cnt))); - - for (cnt = 0; cnt < trans->cfg->base_params->num_of_queues; cnt++) { - u32 status = iwl_read_prph(trans, SCD_QUEUE_STATUS_BITS(cnt)); - u8 fifo = (status >> SCD_QUEUE_STTS_REG_POS_TXF) & 0x7; - bool active = !!(status & BIT(SCD_QUEUE_STTS_REG_POS_ACTIVE)); - u32 tbl_dw = - iwl_trans_read_mem32(trans, trans_pcie->scd_base_addr + - SCD_TRANS_TBL_OFFSET_QUEUE(cnt)); - - if (cnt & 0x1) - tbl_dw = (tbl_dw & 0xFFFF0000) >> 16; - else - tbl_dw = tbl_dw & 0x0000FFFF; - - IWL_ERR(trans, - "Q %d is %sactive and mapped to fifo %d ra_tid 0x%04x [%d,%d]\n", - cnt, active ? "" : "in", fifo, tbl_dw, - iwl_read_prph(trans, SCD_QUEUE_RDPTR(cnt)) & - (TFD_QUEUE_SIZE_MAX - 1), - iwl_read_prph(trans, SCD_QUEUE_WRPTR(cnt))); - } + if (ret) + iwl_trans_pcie_log_scd_error(trans, txq); return ret; } @@ -2011,41 +2055,35 @@ static void iwl_trans_pcie_set_bits_mask(struct iwl_trans *trans, u32 reg, spin_unlock_irqrestore(&trans_pcie->reg_lock, flags); } -void iwl_trans_pcie_ref(struct iwl_trans *trans) +static void iwl_trans_pcie_ref(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); - unsigned long flags; if (iwlwifi_mod_params.d0i3_disable) return; - spin_lock_irqsave(&trans_pcie->ref_lock, flags); - IWL_DEBUG_RPM(trans, "ref_counter: %d\n", trans_pcie->ref_count); - trans_pcie->ref_count++; pm_runtime_get(&trans_pcie->pci_dev->dev); - spin_unlock_irqrestore(&trans_pcie->ref_lock, flags); + +#ifdef CONFIG_PM + IWL_DEBUG_RPM(trans, "runtime usage count: %d\n", + atomic_read(&trans_pcie->pci_dev->dev.power.usage_count)); +#endif /* CONFIG_PM */ } -void iwl_trans_pcie_unref(struct iwl_trans *trans) +static void iwl_trans_pcie_unref(struct iwl_trans *trans) { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); - unsigned long flags; if (iwlwifi_mod_params.d0i3_disable) return; - spin_lock_irqsave(&trans_pcie->ref_lock, flags); - IWL_DEBUG_RPM(trans, "ref_counter: %d\n", trans_pcie->ref_count); - if (WARN_ON_ONCE(trans_pcie->ref_count == 0)) { - spin_unlock_irqrestore(&trans_pcie->ref_lock, flags); - return; - } - trans_pcie->ref_count--; - pm_runtime_mark_last_busy(&trans_pcie->pci_dev->dev); pm_runtime_put_autosuspend(&trans_pcie->pci_dev->dev); - spin_unlock_irqrestore(&trans_pcie->ref_lock, flags); +#ifdef CONFIG_PM + IWL_DEBUG_RPM(trans, "runtime usage count: %d\n", + atomic_read(&trans_pcie->pci_dev->dev.power.usage_count)); +#endif /* CONFIG_PM */ } static const char *get_csr_string(int cmd) @@ -2750,6 +2788,8 @@ static const struct iwl_trans_ops trans_ops_pcie = { .txq_disable = iwl_trans_pcie_txq_disable, .txq_enable = iwl_trans_pcie_txq_enable, + .txq_set_shared_mode = iwl_trans_pcie_txq_set_shared_mode, + .wait_tx_queue_empty = iwl_trans_pcie_wait_txq_empty, .freeze_txq_timer = iwl_trans_pcie_freeze_txq_timer, .block_txq_ptrs = iwl_trans_pcie_block_txq_ptrs, @@ -2793,7 +2833,6 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev, trans_pcie->trans = trans; spin_lock_init(&trans_pcie->irq_lock); spin_lock_init(&trans_pcie->reg_lock); - spin_lock_init(&trans_pcie->ref_lock); mutex_init(&trans_pcie->mutex); init_waitqueue_head(&trans_pcie->ucode_write_waitq); trans_pcie->tso_hdr_page = alloc_percpu(struct iwl_tso_hdr_page); @@ -2912,6 +2951,8 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev, } } + trans->hw_rf_id = iwl_read32(trans, CSR_HW_RF_ID); + iwl_pcie_set_interrupt_capa(pdev, trans); trans->hw_id = (pdev->device << 16) + pdev->subsystem_device; snprintf(trans->hw_id_str, sizeof(trans->hw_id_str), diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/tx.c b/drivers/net/wireless/intel/iwlwifi/pcie/tx.c index 16ad820ca..478bba527 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/tx.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/tx.c @@ -32,6 +32,7 @@ #include <linux/ieee80211.h> #include <linux/slab.h> #include <linux/sched.h> +#include <linux/pm_runtime.h> #include <net/ip6_checksum.h> #include <net/tso.h> @@ -69,6 +70,7 @@ * Tx queue resumed. * ***************************************************/ + static int iwl_queue_space(const struct iwl_queue *q) { unsigned int max; @@ -153,10 +155,6 @@ static void iwl_pcie_txq_stuck_timer(unsigned long data) struct iwl_txq *txq = (void *)data; struct iwl_trans_pcie *trans_pcie = txq->trans_pcie; struct iwl_trans *trans = iwl_trans_pcie_get_trans(trans_pcie); - u32 scd_sram_addr = trans_pcie->scd_base_addr + - SCD_TX_STTS_QUEUE_OFFSET(txq->q.id); - u8 buf[16]; - int i; spin_lock(&txq->lock); /* check if triggered erroneously */ @@ -168,38 +166,8 @@ static void iwl_pcie_txq_stuck_timer(unsigned long data) IWL_ERR(trans, "Queue %d stuck for %u ms.\n", txq->q.id, jiffies_to_msecs(txq->wd_timeout)); - IWL_ERR(trans, "Current SW read_ptr %d write_ptr %d\n", - txq->q.read_ptr, txq->q.write_ptr); - - iwl_trans_read_mem_bytes(trans, scd_sram_addr, buf, sizeof(buf)); - - iwl_print_hex_error(trans, buf, sizeof(buf)); - - for (i = 0; i < FH_TCSR_CHNL_NUM; i++) - IWL_ERR(trans, "FH TRBs(%d) = 0x%08x\n", i, - iwl_read_direct32(trans, FH_TX_TRB_REG(i))); - for (i = 0; i < trans->cfg->base_params->num_of_queues; i++) { - u32 status = iwl_read_prph(trans, SCD_QUEUE_STATUS_BITS(i)); - u8 fifo = (status >> SCD_QUEUE_STTS_REG_POS_TXF) & 0x7; - bool active = !!(status & BIT(SCD_QUEUE_STTS_REG_POS_ACTIVE)); - u32 tbl_dw = - iwl_trans_read_mem32(trans, - trans_pcie->scd_base_addr + - SCD_TRANS_TBL_OFFSET_QUEUE(i)); - - if (i & 0x1) - tbl_dw = (tbl_dw & 0xFFFF0000) >> 16; - else - tbl_dw = tbl_dw & 0x0000FFFF; - - IWL_ERR(trans, - "Q %d is %sactive and mapped to fifo %d ra_tid 0x%04x [%d,%d]\n", - i, active ? "" : "in", fifo, tbl_dw, - iwl_read_prph(trans, SCD_QUEUE_RDPTR(i)) & - (TFD_QUEUE_SIZE_MAX - 1), - iwl_read_prph(trans, SCD_QUEUE_WRPTR(i))); - } + iwl_trans_pcie_log_scd_error(trans, txq); iwl_force_nmi(trans); } @@ -392,7 +360,7 @@ static void iwl_pcie_tfd_unmap(struct iwl_trans *trans, return; } - /* first TB is never freed - it's the scratchbuf data */ + /* first TB is never freed - it's the bidirectional DMA data */ for (i = 1; i < num_tbs; i++) { if (meta->flags & BIT(i + CMD_TB_BITMAP_POS)) @@ -490,7 +458,7 @@ static int iwl_pcie_txq_alloc(struct iwl_trans *trans, { struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); size_t tfd_sz = sizeof(struct iwl_tfd) * TFD_QUEUE_SIZE_MAX; - size_t scratchbuf_sz; + size_t tb0_buf_sz; int i; if (WARN_ON(txq->entries || txq->tfds)) @@ -525,17 +493,14 @@ static int iwl_pcie_txq_alloc(struct iwl_trans *trans, if (!txq->tfds) goto error; - BUILD_BUG_ON(IWL_HCMD_SCRATCHBUF_SIZE != sizeof(*txq->scratchbufs)); - BUILD_BUG_ON(offsetof(struct iwl_pcie_txq_scratch_buf, scratch) != - sizeof(struct iwl_cmd_header) + - offsetof(struct iwl_tx_cmd, scratch)); + BUILD_BUG_ON(IWL_FIRST_TB_SIZE_ALIGN != sizeof(*txq->first_tb_bufs)); - scratchbuf_sz = sizeof(*txq->scratchbufs) * slots_num; + tb0_buf_sz = sizeof(*txq->first_tb_bufs) * slots_num; - txq->scratchbufs = dma_alloc_coherent(trans->dev, scratchbuf_sz, - &txq->scratchbufs_dma, + txq->first_tb_bufs = dma_alloc_coherent(trans->dev, tb0_buf_sz, + &txq->first_tb_dma, GFP_KERNEL); - if (!txq->scratchbufs) + if (!txq->first_tb_bufs) goto err_free_tfds; txq->q.id = txq_id; @@ -557,6 +522,7 @@ error: static int iwl_pcie_txq_init(struct iwl_trans *trans, struct iwl_txq *txq, int slots_num, u32 txq_id) { + struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); int ret; txq->need_update = false; @@ -571,31 +537,65 @@ static int iwl_pcie_txq_init(struct iwl_trans *trans, struct iwl_txq *txq, return ret; spin_lock_init(&txq->lock); + + if (txq_id == trans_pcie->cmd_queue) { + static struct lock_class_key iwl_pcie_cmd_queue_lock_class; + + lockdep_set_class(&txq->lock, &iwl_pcie_cmd_queue_lock_class); + } + __skb_queue_head_init(&txq->overflow_q); /* * Tell nic where to find circular buffer of Tx Frame Descriptors for * given Tx queue, and enable the DMA channel used for that queue. * Circular buffer (TFD queue in DRAM) physical base address */ - iwl_write_direct32(trans, FH_MEM_CBBC_QUEUE(txq_id), - txq->q.dma_addr >> 8); + if (trans->cfg->use_tfh) + iwl_write_direct64(trans, + FH_MEM_CBBC_QUEUE(trans, txq_id), + txq->q.dma_addr); + else + iwl_write_direct32(trans, FH_MEM_CBBC_QUEUE(trans, txq_id), + txq->q.dma_addr >> 8); return 0; } -static void iwl_pcie_free_tso_page(struct sk_buff *skb) +static void iwl_pcie_free_tso_page(struct iwl_trans_pcie *trans_pcie, + struct sk_buff *skb) { - struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); + struct page **page_ptr; - if (info->driver_data[IWL_TRANS_FIRST_DRIVER_DATA]) { - struct page *page = - info->driver_data[IWL_TRANS_FIRST_DRIVER_DATA]; + page_ptr = (void *)((u8 *)skb->cb + trans_pcie->page_offs); - __free_page(page); - info->driver_data[IWL_TRANS_FIRST_DRIVER_DATA] = NULL; + if (*page_ptr) { + __free_page(*page_ptr); + *page_ptr = NULL; } } +static void iwl_pcie_clear_cmd_in_flight(struct iwl_trans *trans) +{ + struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); + + lockdep_assert_held(&trans_pcie->reg_lock); + + if (trans_pcie->ref_cmd_in_flight) { + trans_pcie->ref_cmd_in_flight = false; + IWL_DEBUG_RPM(trans, "clear ref_cmd_in_flight - unref\n"); + iwl_trans_unref(trans); + } + + if (!trans->cfg->base_params->apmg_wake_up_wa) + return; + if (WARN_ON(!trans_pcie->cmd_hold_nic_awake)) + return; + + trans_pcie->cmd_hold_nic_awake = false; + __iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL, + CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); +} + /* * iwl_pcie_txq_unmap - Unmap any remaining DMA mappings and free skb's */ @@ -616,10 +616,24 @@ static void iwl_pcie_txq_unmap(struct iwl_trans *trans, int txq_id) if (WARN_ON_ONCE(!skb)) continue; - iwl_pcie_free_tso_page(skb); + iwl_pcie_free_tso_page(trans_pcie, skb); } iwl_pcie_txq_free_tfd(trans, txq); q->read_ptr = iwl_queue_inc_wrap(q->read_ptr); + + if (q->read_ptr == q->write_ptr) { + unsigned long flags; + + spin_lock_irqsave(&trans_pcie->reg_lock, flags); + if (txq_id != trans_pcie->cmd_queue) { + IWL_DEBUG_RPM(trans, "Q %d - last tx freed\n", + q->id); + iwl_trans_unref(trans); + } else { + iwl_pcie_clear_cmd_in_flight(trans); + } + spin_unlock_irqrestore(&trans_pcie->reg_lock, flags); + } } txq->active = false; @@ -671,8 +685,8 @@ static void iwl_pcie_txq_free(struct iwl_trans *trans, int txq_id) txq->tfds = NULL; dma_free_coherent(dev, - sizeof(*txq->scratchbufs) * txq->q.n_window, - txq->scratchbufs, txq->scratchbufs_dma); + sizeof(*txq->first_tb_bufs) * txq->q.n_window, + txq->first_tb_bufs, txq->first_tb_dma); } kfree(txq->entries); @@ -749,9 +763,14 @@ void iwl_trans_pcie_tx_reset(struct iwl_trans *trans) for (txq_id = 0; txq_id < trans->cfg->base_params->num_of_queues; txq_id++) { struct iwl_txq *txq = &trans_pcie->txq[txq_id]; - - iwl_write_direct32(trans, FH_MEM_CBBC_QUEUE(txq_id), - txq->q.dma_addr >> 8); + if (trans->cfg->use_tfh) + iwl_write_direct64(trans, + FH_MEM_CBBC_QUEUE(trans, txq_id), + txq->q.dma_addr); + else + iwl_write_direct32(trans, + FH_MEM_CBBC_QUEUE(trans, txq_id), + txq->q.dma_addr >> 8); iwl_pcie_txq_unmap(trans, txq_id); txq->q.read_ptr = 0; txq->q.write_ptr = 0; @@ -959,6 +978,12 @@ int iwl_pcie_tx_init(struct iwl_trans *trans) } } + if (trans->cfg->use_tfh) + iwl_write_direct32(trans, TFH_TRANSFER_MODE, + TFH_TRANSFER_MAX_PENDING_REQ | + TFH_CHUNK_SIZE_128 | + TFH_CHUNK_SPLIT_MODE); + iwl_set_bits_prph(trans, SCD_GP_CTRL, SCD_GP_CTRL_AUTO_ACTIVE_MODE); if (trans->cfg->base_params->num_of_queues > 20) iwl_set_bits_prph(trans, SCD_GP_CTRL, @@ -1047,7 +1072,7 @@ void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn, if (WARN_ON_ONCE(!skb)) continue; - iwl_pcie_free_tso_page(skb); + iwl_pcie_free_tso_page(trans_pcie, skb); __skb_queue_tail(skbs, skb); @@ -1078,17 +1103,17 @@ void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn, while (!skb_queue_empty(&overflow_skbs)) { struct sk_buff *skb = __skb_dequeue(&overflow_skbs); - struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); - u8 dev_cmd_idx = IWL_TRANS_FIRST_DRIVER_DATA + 1; - struct iwl_device_cmd *dev_cmd = - info->driver_data[dev_cmd_idx]; + struct iwl_device_cmd *dev_cmd_ptr; + + dev_cmd_ptr = *(void **)((u8 *)skb->cb + + trans_pcie->dev_cmd_offs); /* * Note that we can very well be overflowing again. * In that case, iwl_queue_space will be small again * and we won't wake mac80211's queue. */ - iwl_trans_pcie_tx(trans, skb, dev_cmd, txq_id); + iwl_trans_pcie_tx(trans, skb, dev_cmd_ptr, txq_id); } spin_lock_bh(&txq->lock); @@ -1098,7 +1123,7 @@ void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn, if (q->read_ptr == q->write_ptr) { IWL_DEBUG_RPM(trans, "Q %d - last tx reclaimed\n", q->id); - iwl_trans_pcie_unref(trans); + iwl_trans_unref(trans); } out: @@ -1117,7 +1142,7 @@ static int iwl_pcie_set_cmd_in_flight(struct iwl_trans *trans, !trans_pcie->ref_cmd_in_flight) { trans_pcie->ref_cmd_in_flight = true; IWL_DEBUG_RPM(trans, "set ref_cmd_in_flight - ref\n"); - iwl_trans_pcie_ref(trans); + iwl_trans_ref(trans); } /* @@ -1148,29 +1173,6 @@ static int iwl_pcie_set_cmd_in_flight(struct iwl_trans *trans, return 0; } -static int iwl_pcie_clear_cmd_in_flight(struct iwl_trans *trans) -{ - struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); - - lockdep_assert_held(&trans_pcie->reg_lock); - - if (trans_pcie->ref_cmd_in_flight) { - trans_pcie->ref_cmd_in_flight = false; - IWL_DEBUG_RPM(trans, "clear ref_cmd_in_flight - unref\n"); - iwl_trans_pcie_unref(trans); - } - - if (trans->cfg->base_params->apmg_wake_up_wa) { - if (WARN_ON(!trans_pcie->cmd_hold_nic_awake)) - return 0; - - trans_pcie->cmd_hold_nic_awake = false; - __iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL, - CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ); - } - return 0; -} - /* * iwl_pcie_cmdq_reclaim - Reclaim TX command queue entries already Tx'd * @@ -1340,6 +1342,15 @@ void iwl_trans_pcie_txq_enable(struct iwl_trans *trans, int txq_id, u16 ssn, txq->active = true; } +void iwl_trans_pcie_txq_set_shared_mode(struct iwl_trans *trans, u32 txq_id, + bool shared_mode) +{ + struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); + struct iwl_txq *txq = &trans_pcie->txq[txq_id]; + + txq->ampdu = !shared_mode; +} + void iwl_trans_pcie_txq_disable(struct iwl_trans *trans, int txq_id, bool configure_scd) { @@ -1399,7 +1410,7 @@ static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans, void *dup_buf = NULL; dma_addr_t phys_addr; int idx; - u16 copy_size, cmd_size, scratch_size; + u16 copy_size, cmd_size, tb0_size; bool had_nocopy = false; u8 group_id = iwl_cmd_groupid(cmd->id); int i, ret; @@ -1430,9 +1441,9 @@ static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans, if (!cmd->len[i]) continue; - /* need at least IWL_HCMD_SCRATCHBUF_SIZE copied */ - if (copy_size < IWL_HCMD_SCRATCHBUF_SIZE) { - int copy = IWL_HCMD_SCRATCHBUF_SIZE - copy_size; + /* need at least IWL_FIRST_TB_SIZE copied */ + if (copy_size < IWL_FIRST_TB_SIZE) { + int copy = IWL_FIRST_TB_SIZE - copy_size; if (copy > cmdlen[i]) copy = cmdlen[i]; @@ -1553,8 +1564,8 @@ static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans, } /* - * Otherwise we need at least IWL_HCMD_SCRATCHBUF_SIZE copied - * in total (for the scratchbuf handling), but copy up to what + * Otherwise we need at least IWL_FIRST_TB_SIZE copied + * in total (for bi-directional DMA), but copy up to what * we can fit into the payload for debug dump purposes. */ copy = min_t(int, TFD_MAX_PAYLOAD_SIZE - cmd_pos, cmd->len[i]); @@ -1563,8 +1574,8 @@ static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans, cmd_pos += copy; /* However, treat copy_size the proper way, we need it below */ - if (copy_size < IWL_HCMD_SCRATCHBUF_SIZE) { - copy = IWL_HCMD_SCRATCHBUF_SIZE - copy_size; + if (copy_size < IWL_FIRST_TB_SIZE) { + copy = IWL_FIRST_TB_SIZE - copy_size; if (copy > cmd->len[i]) copy = cmd->len[i]; @@ -1579,18 +1590,18 @@ static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans, le16_to_cpu(out_cmd->hdr.sequence), cmd_size, q->write_ptr, idx, trans_pcie->cmd_queue); - /* start the TFD with the scratchbuf */ - scratch_size = min_t(int, copy_size, IWL_HCMD_SCRATCHBUF_SIZE); - memcpy(&txq->scratchbufs[q->write_ptr], &out_cmd->hdr, scratch_size); + /* start the TFD with the minimum copy bytes */ + tb0_size = min_t(int, copy_size, IWL_FIRST_TB_SIZE); + memcpy(&txq->first_tb_bufs[idx], &out_cmd->hdr, tb0_size); iwl_pcie_txq_build_tfd(trans, txq, - iwl_pcie_get_scratchbuf_dma(txq, q->write_ptr), - scratch_size, true); + iwl_pcie_get_first_tb_dma(txq, idx), + tb0_size, true); /* map first command fragment, if any remains */ - if (copy_size > scratch_size) { + if (copy_size > tb0_size) { phys_addr = dma_map_single(trans->dev, - ((u8 *)&out_cmd->hdr) + scratch_size, - copy_size - scratch_size, + ((u8 *)&out_cmd->hdr) + tb0_size, + copy_size - tb0_size, DMA_TO_DEVICE); if (dma_mapping_error(trans->dev, phys_addr)) { iwl_pcie_tfd_unmap(trans, out_meta, @@ -1600,7 +1611,7 @@ static int iwl_pcie_enqueue_hcmd(struct iwl_trans *trans, } iwl_pcie_txq_build_tfd(trans, txq, phys_addr, - copy_size - scratch_size, false); + copy_size - tb0_size, false); } /* map the remaining (adjusted) nocopy/dup fragments */ @@ -1786,6 +1797,16 @@ static int iwl_pcie_send_hcmd_sync(struct iwl_trans *trans, IWL_DEBUG_INFO(trans, "Setting HCMD_ACTIVE for command %s\n", iwl_get_cmd_string(trans, cmd->id)); + if (pm_runtime_suspended(&trans_pcie->pci_dev->dev)) { + ret = wait_event_timeout(trans_pcie->d0i3_waitq, + pm_runtime_active(&trans_pcie->pci_dev->dev), + msecs_to_jiffies(IWL_TRANS_IDLE_TIMEOUT)); + if (!ret) { + IWL_ERR(trans, "Timeout exiting D0i3 before hcmd\n"); + return -ETIMEDOUT; + } + } + cmd_idx = iwl_pcie_enqueue_hcmd(trans, cmd); if (cmd_idx < 0) { ret = cmd_idx; @@ -1935,7 +1956,7 @@ static int iwl_fill_data_tbs(struct iwl_trans *trans, struct sk_buff *skb, trace_iwlwifi_dev_tx(trans->dev, skb, &txq->tfds[txq->q.write_ptr], sizeof(struct iwl_tfd), - &dev_cmd->hdr, IWL_HCMD_SCRATCHBUF_SIZE + tb1_len, + &dev_cmd->hdr, IWL_FIRST_TB_SIZE + tb1_len, skb->data + hdr_len, tb2_len); trace_iwlwifi_dev_tx_data(trans->dev, skb, hdr_len, skb->len - hdr_len); @@ -1991,7 +2012,6 @@ static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb, struct iwl_cmd_meta *out_meta, struct iwl_device_cmd *dev_cmd, u16 tb1_len) { - struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); struct iwl_trans_pcie *trans_pcie = txq->trans_pcie; struct ieee80211_hdr *hdr = (void *)skb->data; unsigned int snap_ip_tcp_hdrlen, ip_hdrlen, total_len, hdr_room; @@ -2000,6 +2020,7 @@ static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb, u16 length, iv_len, amsdu_pad; u8 *start_hdr; struct iwl_tso_hdr_page *hdr_page; + struct page **page_ptr; int ret; struct tso_t tso; @@ -2011,7 +2032,7 @@ static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb, trace_iwlwifi_dev_tx(trans->dev, skb, &txq->tfds[txq->q.write_ptr], sizeof(struct iwl_tfd), - &dev_cmd->hdr, IWL_HCMD_SCRATCHBUF_SIZE + tb1_len, + &dev_cmd->hdr, IWL_FIRST_TB_SIZE + tb1_len, NULL, 0); ip_hdrlen = skb_transport_header(skb) - skb_network_header(skb); @@ -2030,7 +2051,8 @@ static int iwl_fill_data_tbs_amsdu(struct iwl_trans *trans, struct sk_buff *skb, get_page(hdr_page->page); start_hdr = hdr_page->pos; - info->driver_data[IWL_TRANS_FIRST_DRIVER_DATA] = hdr_page->page; + page_ptr = (void *)((u8 *)skb->cb + trans_pcie->page_offs); + *page_ptr = hdr_page->page; memcpy(hdr_page->pos, skb->data + hdr_len, iv_len); hdr_page->pos += iv_len; @@ -2197,6 +2219,7 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb, __le16 fc; u8 hdr_len; u16 wifi_seq; + bool amsdu; txq = &trans_pcie->txq[txq_id]; q = &txq->q; @@ -2239,10 +2262,12 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb, /* don't put the packet on the ring, if there is no room */ if (unlikely(iwl_queue_space(q) < 3)) { - struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); + struct iwl_device_cmd **dev_cmd_ptr; - info->driver_data[IWL_TRANS_FIRST_DRIVER_DATA + 1] = - dev_cmd; + dev_cmd_ptr = (void *)((u8 *)skb->cb + + trans_pcie->dev_cmd_offs); + + *dev_cmd_ptr = dev_cmd; __skb_queue_tail(&txq->overflow_q, skb); spin_unlock(&txq->lock); @@ -2269,7 +2294,7 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb, cpu_to_le16((u16)(QUEUE_TO_SEQ(txq_id) | INDEX_TO_SEQ(q->write_ptr))); - tb0_phys = iwl_pcie_get_scratchbuf_dma(txq, q->write_ptr); + tb0_phys = iwl_pcie_get_first_tb_dma(txq, q->write_ptr); scratch_phys = tb0_phys + sizeof(struct iwl_cmd_header) + offsetof(struct iwl_tx_cmd, scratch); @@ -2287,31 +2312,37 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb, * setup of the first TB) */ len = sizeof(struct iwl_tx_cmd) + sizeof(struct iwl_cmd_header) + - hdr_len - IWL_HCMD_SCRATCHBUF_SIZE; - tb1_len = ALIGN(len, 4); - - /* Tell NIC about any 2-byte padding after MAC header */ - if (tb1_len != len) - tx_cmd->tx_flags |= TX_CMD_FLG_MH_PAD_MSK; + hdr_len - IWL_FIRST_TB_SIZE; + /* do not align A-MSDU to dword as the subframe header aligns it */ + amsdu = ieee80211_is_data_qos(fc) && + (*ieee80211_get_qos_ctl(hdr) & + IEEE80211_QOS_CTL_A_MSDU_PRESENT); + if (trans_pcie->sw_csum_tx || !amsdu) { + tb1_len = ALIGN(len, 4); + /* Tell NIC about any 2-byte padding after MAC header */ + if (tb1_len != len) + tx_cmd->tx_flags |= TX_CMD_FLG_MH_PAD_MSK; + } else { + tb1_len = len; + } - /* The first TB points to the scratchbuf data - min_copy bytes */ - memcpy(&txq->scratchbufs[q->write_ptr], &dev_cmd->hdr, - IWL_HCMD_SCRATCHBUF_SIZE); + /* The first TB points to bi-directional DMA data */ + memcpy(&txq->first_tb_bufs[q->write_ptr], &dev_cmd->hdr, + IWL_FIRST_TB_SIZE); iwl_pcie_txq_build_tfd(trans, txq, tb0_phys, - IWL_HCMD_SCRATCHBUF_SIZE, true); + IWL_FIRST_TB_SIZE, true); /* there must be data left over for TB1 or this code must be changed */ - BUILD_BUG_ON(sizeof(struct iwl_tx_cmd) < IWL_HCMD_SCRATCHBUF_SIZE); + BUILD_BUG_ON(sizeof(struct iwl_tx_cmd) < IWL_FIRST_TB_SIZE); /* map the data for TB1 */ - tb1_addr = ((u8 *)&dev_cmd->hdr) + IWL_HCMD_SCRATCHBUF_SIZE; + tb1_addr = ((u8 *)&dev_cmd->hdr) + IWL_FIRST_TB_SIZE; tb1_phys = dma_map_single(trans->dev, tb1_addr, tb1_len, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(trans->dev, tb1_phys))) goto out_err; iwl_pcie_txq_build_tfd(trans, txq, tb1_phys, tb1_len, false); - if (ieee80211_is_data_qos(fc) && - (*ieee80211_get_qos_ctl(hdr) & IEEE80211_QOS_CTL_A_MSDU_PRESENT)) { + if (amsdu) { if (unlikely(iwl_fill_data_tbs_amsdu(trans, skb, txq, hdr_len, out_meta, dev_cmd, tb1_len))) @@ -2342,7 +2373,7 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb, txq->frozen_expiry_remainder = txq->wd_timeout; } IWL_DEBUG_RPM(trans, "Q: %d first tx - take ref\n", q->id); - iwl_trans_pcie_ref(trans); + iwl_trans_ref(trans); } /* Tell device the write index *just past* this latest filled TFD */ |