summaryrefslogtreecommitdiff
path: root/drivers/net/wireless/intel/iwlwifi/pcie
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/net/wireless/intel/iwlwifi/pcie')
-rw-r--r--drivers/net/wireless/intel/iwlwifi/pcie/drv.c213
-rw-r--r--drivers/net/wireless/intel/iwlwifi/pcie/internal.h142
-rw-r--r--drivers/net/wireless/intel/iwlwifi/pcie/rx.c858
-rw-r--r--drivers/net/wireless/intel/iwlwifi/pcie/trans.c428
-rw-r--r--drivers/net/wireless/intel/iwlwifi/pcie/tx.c87
5 files changed, 1337 insertions, 391 deletions
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
index 00335ea6b..79d7cd7d4 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
@@ -7,6 +7,7 @@
*
* Copyright(c) 2007 - 2014 Intel Corporation. All rights reserved.
* Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 Intel Deutschland GmbH
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of version 2 of the GNU General Public License as
@@ -66,6 +67,7 @@
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/module.h>
+#include <linux/pm_runtime.h>
#include <linux/pci.h>
#include <linux/pci-aspm.h>
#include <linux/acpi.h>
@@ -477,8 +479,18 @@ static const struct pci_device_id iwl_hw_card_ids[] = {
{IWL_PCI_DEVICE(0x24F3, 0x0930, iwl8260_2ac_cfg)},
{IWL_PCI_DEVICE(0x24F3, 0x0000, iwl8265_2ac_cfg)},
{IWL_PCI_DEVICE(0x24FD, 0x0010, iwl8265_2ac_cfg)},
+ {IWL_PCI_DEVICE(0x24FD, 0x0110, iwl8265_2ac_cfg)},
+ {IWL_PCI_DEVICE(0x24FD, 0x1110, iwl8265_2ac_cfg)},
+ {IWL_PCI_DEVICE(0x24FD, 0x1010, iwl8265_2ac_cfg)},
+ {IWL_PCI_DEVICE(0x24FD, 0x0050, iwl8265_2ac_cfg)},
+ {IWL_PCI_DEVICE(0x24FD, 0x0150, iwl8265_2ac_cfg)},
+ {IWL_PCI_DEVICE(0x24FD, 0x9010, iwl8265_2ac_cfg)},
+ {IWL_PCI_DEVICE(0x24FD, 0x8110, iwl8265_2ac_cfg)},
+ {IWL_PCI_DEVICE(0x24FD, 0x8050, iwl8265_2ac_cfg)},
{IWL_PCI_DEVICE(0x24FD, 0x8010, iwl8265_2ac_cfg)},
{IWL_PCI_DEVICE(0x24FD, 0x0810, iwl8265_2ac_cfg)},
+ {IWL_PCI_DEVICE(0x24FD, 0x9110, iwl8265_2ac_cfg)},
+ {IWL_PCI_DEVICE(0x24FD, 0x8130, iwl8265_2ac_cfg)},
/* 9000 Series */
{IWL_PCI_DEVICE(0x9DF0, 0x2A10, iwl5165_2ac_cfg)},
@@ -627,6 +639,33 @@ static int iwl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
if (ret)
goto out_free_drv;
+ /* if RTPM is in use, enable it in our device */
+ if (iwl_trans->runtime_pm_mode != IWL_PLAT_PM_MODE_DISABLED) {
+ /* We explicitly set the device to active here to
+ * clear contingent errors.
+ */
+ pm_runtime_set_active(&pdev->dev);
+
+ pm_runtime_set_autosuspend_delay(&pdev->dev,
+ iwlwifi_mod_params.d0i3_entry_delay);
+ pm_runtime_use_autosuspend(&pdev->dev);
+
+ /* We are not supposed to call pm_runtime_allow() by
+ * ourselves, but let userspace enable runtime PM via
+ * sysfs. However, since we don't enable this from
+ * userspace yet, we need to allow/forbid() ourselves.
+ */
+ pm_runtime_allow(&pdev->dev);
+ }
+
+ /* The PCI device starts with a reference taken and we are
+ * supposed to release it here. But to simplify the
+ * interaction with the opmode, we don't do it now, but let
+ * the opmode release it when it's ready. To account for this
+ * reference, we start with ref_count set to 1.
+ */
+ trans_pcie->ref_count = 1;
+
return 0;
out_free_drv:
@@ -641,7 +680,17 @@ static void iwl_pci_remove(struct pci_dev *pdev)
struct iwl_trans *trans = pci_get_drvdata(pdev);
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+ /* if RTPM was in use, restore it to the state before probe */
+ if (trans->runtime_pm_mode != IWL_PLAT_PM_MODE_DISABLED) {
+ /* We should not call forbid here, but we do for now.
+ * Check the comment to pm_runtime_allow() in
+ * iwl_pci_probe().
+ */
+ pm_runtime_forbid(trans->dev);
+ }
+
iwl_drv_stop(trans_pcie->drv);
+
iwl_trans_pcie_free(trans);
}
@@ -693,15 +742,173 @@ static int iwl_pci_resume(struct device *device)
return 0;
}
-static SIMPLE_DEV_PM_OPS(iwl_dev_pm_ops, iwl_pci_suspend, iwl_pci_resume);
+int iwl_pci_fw_enter_d0i3(struct iwl_trans *trans)
+{
+ struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+ int ret;
+
+ if (test_bit(STATUS_FW_ERROR, &trans->status))
+ return 0;
+
+ set_bit(STATUS_TRANS_GOING_IDLE, &trans->status);
+
+ /* config the fw */
+ ret = iwl_op_mode_enter_d0i3(trans->op_mode);
+ if (ret == 1) {
+ IWL_DEBUG_RPM(trans, "aborting d0i3 entrance\n");
+ clear_bit(STATUS_TRANS_GOING_IDLE, &trans->status);
+ return -EBUSY;
+ }
+ if (ret)
+ goto err;
+
+ ret = wait_event_timeout(trans_pcie->d0i3_waitq,
+ test_bit(STATUS_TRANS_IDLE, &trans->status),
+ msecs_to_jiffies(IWL_TRANS_IDLE_TIMEOUT));
+ if (!ret) {
+ IWL_ERR(trans, "Timeout entering D0i3\n");
+ ret = -ETIMEDOUT;
+ goto err;
+ }
+
+ clear_bit(STATUS_TRANS_GOING_IDLE, &trans->status);
+
+ return 0;
+err:
+ clear_bit(STATUS_TRANS_GOING_IDLE, &trans->status);
+ iwl_trans_fw_error(trans);
+ return ret;
+}
+
+int iwl_pci_fw_exit_d0i3(struct iwl_trans *trans)
+{
+ struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+ int ret;
+
+ /* sometimes a D0i3 entry is not followed through */
+ if (!test_bit(STATUS_TRANS_IDLE, &trans->status))
+ return 0;
+
+ /* config the fw */
+ ret = iwl_op_mode_exit_d0i3(trans->op_mode);
+ if (ret)
+ goto err;
+
+ /* we clear STATUS_TRANS_IDLE only when D0I3_END command is completed */
+
+ ret = wait_event_timeout(trans_pcie->d0i3_waitq,
+ !test_bit(STATUS_TRANS_IDLE, &trans->status),
+ msecs_to_jiffies(IWL_TRANS_IDLE_TIMEOUT));
+ if (!ret) {
+ IWL_ERR(trans, "Timeout exiting D0i3\n");
+ ret = -ETIMEDOUT;
+ goto err;
+ }
+
+ return 0;
+err:
+ clear_bit(STATUS_TRANS_IDLE, &trans->status);
+ iwl_trans_fw_error(trans);
+ return ret;
+}
+
+#ifdef CONFIG_IWLWIFI_PCIE_RTPM
+static int iwl_pci_runtime_suspend(struct device *device)
+{
+ struct pci_dev *pdev = to_pci_dev(device);
+ struct iwl_trans *trans = pci_get_drvdata(pdev);
+ int ret;
+
+ IWL_DEBUG_RPM(trans, "entering runtime suspend\n");
+
+ if (test_bit(STATUS_DEVICE_ENABLED, &trans->status)) {
+ ret = iwl_pci_fw_enter_d0i3(trans);
+ if (ret < 0)
+ return ret;
+ }
+
+ trans->system_pm_mode = IWL_PLAT_PM_MODE_D0I3;
+
+ iwl_trans_d3_suspend(trans, false, false);
+
+ return 0;
+}
+
+static int iwl_pci_runtime_resume(struct device *device)
+{
+ struct pci_dev *pdev = to_pci_dev(device);
+ struct iwl_trans *trans = pci_get_drvdata(pdev);
+ enum iwl_d3_status d3_status;
+
+ IWL_DEBUG_RPM(trans, "exiting runtime suspend (resume)\n");
+
+ iwl_trans_d3_resume(trans, &d3_status, false, false);
+
+ if (test_bit(STATUS_DEVICE_ENABLED, &trans->status))
+ return iwl_pci_fw_exit_d0i3(trans);
+
+ return 0;
+}
+
+static int iwl_pci_system_prepare(struct device *device)
+{
+ struct pci_dev *pdev = to_pci_dev(device);
+ struct iwl_trans *trans = pci_get_drvdata(pdev);
+
+ IWL_DEBUG_RPM(trans, "preparing for system suspend\n");
+
+ /* This is called before entering system suspend and before
+ * the runtime resume is called. Set the suspending flag to
+ * prevent the wakelock from being taken.
+ */
+ trans->suspending = true;
+
+ /* Wake the device up from runtime suspend before going to
+ * platform suspend. This is needed because we don't know
+ * whether wowlan any is set and, if it's not, mac80211 will
+ * disconnect (in which case, we can't be in D0i3).
+ */
+ pm_runtime_resume(device);
+
+ return 0;
+}
+
+static void iwl_pci_system_complete(struct device *device)
+{
+ struct pci_dev *pdev = to_pci_dev(device);
+ struct iwl_trans *trans = pci_get_drvdata(pdev);
+
+ IWL_DEBUG_RPM(trans, "completing system suspend\n");
+
+ /* This is called as a counterpart to the prepare op. It is
+ * called either when suspending fails or when suspend
+ * completed successfully. Now there's no risk of grabbing
+ * the wakelock anymore, so we can release the suspending
+ * flag.
+ */
+ trans->suspending = false;
+}
+#endif /* CONFIG_IWLWIFI_PCIE_RTPM */
+
+static const struct dev_pm_ops iwl_dev_pm_ops = {
+ SET_SYSTEM_SLEEP_PM_OPS(iwl_pci_suspend,
+ iwl_pci_resume)
+#ifdef CONFIG_IWLWIFI_PCIE_RTPM
+ SET_RUNTIME_PM_OPS(iwl_pci_runtime_suspend,
+ iwl_pci_runtime_resume,
+ NULL)
+ .prepare = iwl_pci_system_prepare,
+ .complete = iwl_pci_system_complete,
+#endif /* CONFIG_IWLWIFI_PCIE_RTPM */
+};
#define IWL_PM_OPS (&iwl_dev_pm_ops)
-#else
+#else /* CONFIG_PM_SLEEP */
#define IWL_PM_OPS NULL
-#endif
+#endif /* CONFIG_PM_SLEEP */
static struct pci_driver iwl_pci_driver = {
.name = DRV_NAME,
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/internal.h b/drivers/net/wireless/intel/iwlwifi/pcie/internal.h
index 73c95594e..dadafbdef 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/internal.h
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/internal.h
@@ -2,6 +2,7 @@
*
* Copyright(c) 2003 - 2015 Intel Corporation. All rights reserved.
* Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 Intel Deutschland GmbH
*
* Portions of this file are derived from the ipw3945 project, as well
* as portions of the ieee80211 subsystem header files.
@@ -56,17 +57,23 @@
#define RX_NUM_QUEUES 1
#define RX_POST_REQ_ALLOC 2
#define RX_CLAIM_REQ_ALLOC 8
-#define RX_POOL_SIZE ((RX_CLAIM_REQ_ALLOC - RX_POST_REQ_ALLOC) * RX_NUM_QUEUES)
-#define RX_LOW_WATERMARK 8
+#define RX_PENDING_WATERMARK 16
struct iwl_host_cmd;
/*This file includes the declaration that are internal to the
* trans_pcie layer */
+/**
+ * struct iwl_rx_mem_buffer
+ * @page_dma: bus address of rxb page
+ * @page: driver's pointer to the rxb page
+ * @vid: index of this rxb in the global table
+ */
struct iwl_rx_mem_buffer {
dma_addr_t page_dma;
struct page *page;
+ u16 vid;
struct list_head list;
};
@@ -90,8 +97,12 @@ struct isr_statistics {
/**
* struct iwl_rxq - Rx queue
- * @bd: driver's pointer to buffer of receive buffer descriptors (rbd)
+ * @id: queue index
+ * @bd: driver's pointer to buffer of receive buffer descriptors (rbd).
+ * Address size is 32 bit in pre-9000 devices and 64 bit in 9000 devices.
* @bd_dma: bus address of buffer of receive buffer descriptors (rbd)
+ * @ubd: driver's pointer to buffer of used receive buffer descriptors (rbd)
+ * @ubd_dma: physical address of buffer of used receive buffer descriptors (rbd)
* @read: Shared index to newest available Rx buffer
* @write: Shared index to oldest written Rx packet
* @free_count: Number of pre-allocated buffers in rx_free
@@ -103,32 +114,34 @@ struct isr_statistics {
* @rb_stts: driver's pointer to receive buffer status
* @rb_stts_dma: bus address of receive buffer status
* @lock:
- * @pool: initial pool of iwl_rx_mem_buffer for the queue
- * @queue: actual rx queue
+ * @queue: actual rx queue. Not used for multi-rx queue.
*
* NOTE: rx_free and rx_used are used as a FIFO for iwl_rx_mem_buffers
*/
struct iwl_rxq {
- __le32 *bd;
+ int id;
+ void *bd;
dma_addr_t bd_dma;
+ __le32 *used_bd;
+ dma_addr_t used_bd_dma;
u32 read;
u32 write;
u32 free_count;
u32 used_count;
u32 write_actual;
+ u32 queue_size;
struct list_head rx_free;
struct list_head rx_used;
bool need_update;
struct iwl_rb_status *rb_stts;
dma_addr_t rb_stts_dma;
spinlock_t lock;
- struct iwl_rx_mem_buffer pool[RX_QUEUE_SIZE];
+ struct napi_struct napi;
struct iwl_rx_mem_buffer *queue[RX_QUEUE_SIZE];
};
/**
* struct iwl_rb_allocator - Rx allocator
- * @pool: initial pool of allocator
* @req_pending: number of requests the allcator had not processed yet
* @req_ready: number of requests honored and ready for claiming
* @rbd_allocated: RBDs with pages allocated and ready to be handled to
@@ -140,7 +153,6 @@ struct iwl_rxq {
* @rx_alloc: work struct for background calls
*/
struct iwl_rb_allocator {
- struct iwl_rx_mem_buffer pool[RX_POOL_SIZE];
atomic_t req_pending;
atomic_t req_ready;
struct list_head rbd_allocated;
@@ -280,6 +292,7 @@ struct iwl_txq {
bool ampdu;
bool block;
unsigned long wd_timeout;
+ struct sk_buff_head overflow_q;
};
static inline dma_addr_t
@@ -297,6 +310,8 @@ struct iwl_tso_hdr_page {
/**
* struct iwl_trans_pcie - PCIe transport specific data
* @rxq: all the RX queue data
+ * @rx_pool: initial pool of iwl_rx_mem_buffer for all the queues
+ * @global_table: table mapping received VID from hw to rxb
* @rba: allocator for RX replenishing
* @drv - pointer to iwl_drv
* @trans: pointer to the generic transport area
@@ -321,15 +336,24 @@ struct iwl_tso_hdr_page {
* @fw_mon_phys: physical address of the buffer for the firmware monitor
* @fw_mon_page: points to the first page of the buffer for the firmware monitor
* @fw_mon_size: size of the buffer for the firmware monitor
+ * @msix_entries: array of MSI-X entries
+ * @msix_enabled: true if managed to enable MSI-X
+ * @allocated_vector: the number of interrupt vector allocated by the OS
+ * @default_irq_num: default irq for non rx interrupt
+ * @fh_init_mask: initial unmasked fh causes
+ * @hw_init_mask: initial unmasked hw causes
+ * @fh_mask: current unmasked fh causes
+ * @hw_mask: current unmasked hw causes
*/
struct iwl_trans_pcie {
- struct iwl_rxq rxq;
+ struct iwl_rxq *rxq;
+ struct iwl_rx_mem_buffer rx_pool[RX_POOL_SIZE];
+ struct iwl_rx_mem_buffer *global_table[MQ_RX_TABLE_SIZE];
struct iwl_rb_allocator rba;
struct iwl_trans *trans;
struct iwl_drv *drv;
struct net_device napi_dev;
- struct napi_struct napi;
struct __percpu iwl_tso_hdr_page *tso_hdr_page;
@@ -359,6 +383,7 @@ struct iwl_trans_pcie {
bool ucode_write_complete;
wait_queue_head_t ucode_write_waitq;
wait_queue_head_t wait_command_queue;
+ wait_queue_head_t d0i3_waitq;
u8 cmd_queue;
u8 cmd_fifo;
@@ -385,6 +410,15 @@ struct iwl_trans_pcie {
dma_addr_t fw_mon_phys;
struct page *fw_mon_page;
u32 fw_mon_size;
+
+ struct msix_entry msix_entries[IWL_MAX_RX_HW_QUEUES];
+ bool msix_enabled;
+ u32 allocated_vector;
+ u32 default_irq_num;
+ u32 fh_init_mask;
+ u32 hw_init_mask;
+ u32 fh_mask;
+ u32 hw_mask;
};
static inline struct iwl_trans_pcie *
@@ -413,7 +447,10 @@ void iwl_trans_pcie_free(struct iwl_trans *trans);
* RX
******************************************************/
int iwl_pcie_rx_init(struct iwl_trans *trans);
+irqreturn_t iwl_pcie_msix_isr(int irq, void *data);
irqreturn_t iwl_pcie_irq_handler(int irq, void *dev_id);
+irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id);
+irqreturn_t iwl_pcie_irq_rx_msix_handler(int irq, void *dev_id);
int iwl_pcie_rx_stop(struct iwl_trans *trans);
void iwl_pcie_rx_free(struct iwl_trans *trans);
@@ -468,15 +505,24 @@ void iwl_pcie_dump_csr(struct iwl_trans *trans);
******************************************************/
static inline void iwl_disable_interrupts(struct iwl_trans *trans)
{
- clear_bit(STATUS_INT_ENABLED, &trans->status);
-
- /* disable interrupts from uCode/NIC to host */
- iwl_write32(trans, CSR_INT_MASK, 0x00000000);
+ struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
- /* acknowledge/clear/reset any interrupts still pending
- * from uCode or flow handler (Rx/Tx DMA) */
- iwl_write32(trans, CSR_INT, 0xffffffff);
- iwl_write32(trans, CSR_FH_INT_STATUS, 0xffffffff);
+ clear_bit(STATUS_INT_ENABLED, &trans->status);
+ if (!trans_pcie->msix_enabled) {
+ /* disable interrupts from uCode/NIC to host */
+ iwl_write32(trans, CSR_INT_MASK, 0x00000000);
+
+ /* acknowledge/clear/reset any interrupts still pending
+ * from uCode or flow handler (Rx/Tx DMA) */
+ iwl_write32(trans, CSR_INT, 0xffffffff);
+ iwl_write32(trans, CSR_FH_INT_STATUS, 0xffffffff);
+ } else {
+ /* disable all the interrupt we might use */
+ iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD,
+ trans_pcie->fh_init_mask);
+ iwl_write32(trans, CSR_MSIX_HW_INT_MASK_AD,
+ trans_pcie->hw_init_mask);
+ }
IWL_DEBUG_ISR(trans, "Disabled interrupts\n");
}
@@ -486,8 +532,37 @@ static inline void iwl_enable_interrupts(struct iwl_trans *trans)
IWL_DEBUG_ISR(trans, "Enabling interrupts\n");
set_bit(STATUS_INT_ENABLED, &trans->status);
- trans_pcie->inta_mask = CSR_INI_SET_MASK;
- iwl_write32(trans, CSR_INT_MASK, trans_pcie->inta_mask);
+ if (!trans_pcie->msix_enabled) {
+ trans_pcie->inta_mask = CSR_INI_SET_MASK;
+ iwl_write32(trans, CSR_INT_MASK, trans_pcie->inta_mask);
+ } else {
+ /*
+ * fh/hw_mask keeps all the unmasked causes.
+ * Unlike msi, in msix cause is enabled when it is unset.
+ */
+ trans_pcie->hw_mask = trans_pcie->hw_init_mask;
+ trans_pcie->fh_mask = trans_pcie->fh_init_mask;
+ iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD,
+ ~trans_pcie->fh_mask);
+ iwl_write32(trans, CSR_MSIX_HW_INT_MASK_AD,
+ ~trans_pcie->hw_mask);
+ }
+}
+
+static inline void iwl_enable_hw_int_msk_msix(struct iwl_trans *trans, u32 msk)
+{
+ struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+
+ iwl_write32(trans, CSR_MSIX_HW_INT_MASK_AD, ~msk);
+ trans_pcie->hw_mask = msk;
+}
+
+static inline void iwl_enable_fh_int_msk_msix(struct iwl_trans *trans, u32 msk)
+{
+ struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+
+ iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD, ~msk);
+ trans_pcie->fh_mask = msk;
}
static inline void iwl_enable_fw_load_int(struct iwl_trans *trans)
@@ -495,8 +570,15 @@ static inline void iwl_enable_fw_load_int(struct iwl_trans *trans)
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
IWL_DEBUG_ISR(trans, "Enabling FW load interrupt\n");
- trans_pcie->inta_mask = CSR_INT_BIT_FH_TX;
- iwl_write32(trans, CSR_INT_MASK, trans_pcie->inta_mask);
+ if (!trans_pcie->msix_enabled) {
+ trans_pcie->inta_mask = CSR_INT_BIT_FH_TX;
+ iwl_write32(trans, CSR_INT_MASK, trans_pcie->inta_mask);
+ } else {
+ iwl_write32(trans, CSR_MSIX_HW_INT_MASK_AD,
+ trans_pcie->hw_init_mask);
+ iwl_enable_fh_int_msk_msix(trans,
+ MSIX_FH_INT_CAUSES_D2S_CH0_NUM);
+ }
}
static inline void iwl_enable_rfkill_int(struct iwl_trans *trans)
@@ -504,8 +586,15 @@ static inline void iwl_enable_rfkill_int(struct iwl_trans *trans)
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
IWL_DEBUG_ISR(trans, "Enabling rfkill interrupt\n");
- trans_pcie->inta_mask = CSR_INT_BIT_RF_KILL;
- iwl_write32(trans, CSR_INT_MASK, trans_pcie->inta_mask);
+ if (!trans_pcie->msix_enabled) {
+ trans_pcie->inta_mask = CSR_INT_BIT_RF_KILL;
+ iwl_write32(trans, CSR_INT_MASK, trans_pcie->inta_mask);
+ } else {
+ iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD,
+ trans_pcie->fh_init_mask);
+ iwl_enable_hw_int_msk_msix(trans,
+ MSIX_HW_INT_CAUSES_REG_RF_KILL);
+ }
}
static inline void iwl_wake_queue(struct iwl_trans *trans,
@@ -588,4 +677,7 @@ static inline int iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans)
}
#endif
+int iwl_pci_fw_exit_d0i3(struct iwl_trans *trans);
+int iwl_pci_fw_enter_d0i3(struct iwl_trans *trans);
+
#endif /* __iwl_trans_int_pcie_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/rx.c b/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
index 152cf9ad9..4be3c35af 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
@@ -2,6 +2,7 @@
*
* Copyright(c) 2003 - 2014 Intel Corporation. All rights reserved.
* Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 Intel Deutschland GmbH
*
* Portions of this file are derived from the ipw3945 project, as well
* as portions of the ieee80211 subsystem header files.
@@ -140,8 +141,8 @@
*/
static int iwl_rxq_space(const struct iwl_rxq *rxq)
{
- /* Make sure RX_QUEUE_SIZE is a power of 2 */
- BUILD_BUG_ON(RX_QUEUE_SIZE & (RX_QUEUE_SIZE - 1));
+ /* Make sure rx queue size is a power of 2 */
+ WARN_ON(rxq->queue_size & (rxq->queue_size - 1));
/*
* There can be up to (RX_QUEUE_SIZE - 1) free slots, to avoid ambiguity
@@ -149,7 +150,7 @@ static int iwl_rxq_space(const struct iwl_rxq *rxq)
* The following is equivalent to modulo by RX_QUEUE_SIZE and is well
* defined for negative dividends.
*/
- return (rxq->read - rxq->write - 1) & (RX_QUEUE_SIZE - 1);
+ return (rxq->read - rxq->write - 1) & (rxq->queue_size - 1);
}
/*
@@ -160,6 +161,12 @@ static inline __le32 iwl_pcie_dma_addr2rbd_ptr(dma_addr_t dma_addr)
return cpu_to_le32((u32)(dma_addr >> 8));
}
+static void iwl_pcie_write_prph_64(struct iwl_trans *trans, u64 ofs, u64 val)
+{
+ iwl_write_prph(trans, ofs, val & 0xffffffff);
+ iwl_write_prph(trans, ofs + 4, val >> 32);
+}
+
/*
* iwl_pcie_rx_stop - stops the Rx DMA
*/
@@ -173,10 +180,9 @@ int iwl_pcie_rx_stop(struct iwl_trans *trans)
/*
* iwl_pcie_rxq_inc_wr_ptr - Update the write pointer for the RX queue
*/
-static void iwl_pcie_rxq_inc_wr_ptr(struct iwl_trans *trans)
+static void iwl_pcie_rxq_inc_wr_ptr(struct iwl_trans *trans,
+ struct iwl_rxq *rxq)
{
- struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
- struct iwl_rxq *rxq = &trans_pcie->rxq;
u32 reg;
lockdep_assert_held(&rxq->lock);
@@ -201,41 +207,84 @@ static void iwl_pcie_rxq_inc_wr_ptr(struct iwl_trans *trans)
}
rxq->write_actual = round_down(rxq->write, 8);
- iwl_write32(trans, FH_RSCSR_CHNL0_WPTR, rxq->write_actual);
+ if (trans->cfg->mq_rx_supported)
+ iwl_write_prph(trans, RFH_Q_FRBDCB_WIDX(rxq->id),
+ rxq->write_actual);
+ else
+ iwl_write32(trans, FH_RSCSR_CHNL0_WPTR, rxq->write_actual);
}
static void iwl_pcie_rxq_check_wrptr(struct iwl_trans *trans)
{
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
- struct iwl_rxq *rxq = &trans_pcie->rxq;
+ int i;
- spin_lock(&rxq->lock);
+ for (i = 0; i < trans->num_rx_queues; i++) {
+ struct iwl_rxq *rxq = &trans_pcie->rxq[i];
+
+ if (!rxq->need_update)
+ continue;
+ spin_lock(&rxq->lock);
+ iwl_pcie_rxq_inc_wr_ptr(trans, rxq);
+ rxq->need_update = false;
+ spin_unlock(&rxq->lock);
+ }
+}
+
+/*
+ * iwl_pcie_rxq_mq_restock - restock implementation for multi-queue rx
+ */
+static void iwl_pcie_rxq_mq_restock(struct iwl_trans *trans,
+ struct iwl_rxq *rxq)
+{
+ struct iwl_rx_mem_buffer *rxb;
+
+ /*
+ * If the device isn't enabled - no need to try to add buffers...
+ * This can happen when we stop the device and still have an interrupt
+ * pending. We stop the APM before we sync the interrupts because we
+ * have to (see comment there). On the other hand, since the APM is
+ * stopped, we cannot access the HW (in particular not prph).
+ * So don't try to restock if the APM has been already stopped.
+ */
+ if (!test_bit(STATUS_DEVICE_ENABLED, &trans->status))
+ return;
- if (!rxq->need_update)
- goto exit_unlock;
+ spin_lock(&rxq->lock);
+ while (rxq->free_count) {
+ __le64 *bd = (__le64 *)rxq->bd;
- iwl_pcie_rxq_inc_wr_ptr(trans);
- rxq->need_update = false;
+ /* Get next free Rx buffer, remove from free list */
+ rxb = list_first_entry(&rxq->rx_free, struct iwl_rx_mem_buffer,
+ list);
+ list_del(&rxb->list);
- exit_unlock:
+ /* 12 first bits are expected to be empty */
+ WARN_ON(rxb->page_dma & DMA_BIT_MASK(12));
+ /* Point to Rx buffer via next RBD in circular buffer */
+ bd[rxq->write] = cpu_to_le64(rxb->page_dma | rxb->vid);
+ rxq->write = (rxq->write + 1) & MQ_RX_TABLE_MASK;
+ rxq->free_count--;
+ }
spin_unlock(&rxq->lock);
+
+ /*
+ * If we've added more space for the firmware to place data, tell it.
+ * Increment device's write pointer in multiples of 8.
+ */
+ if (rxq->write_actual != (rxq->write & ~0x7)) {
+ spin_lock(&rxq->lock);
+ iwl_pcie_rxq_inc_wr_ptr(trans, rxq);
+ spin_unlock(&rxq->lock);
+ }
}
/*
- * iwl_pcie_rxq_restock - refill RX queue from pre-allocated pool
- *
- * If there are slots in the RX queue that need to be restocked,
- * and we have free pre-allocated buffers, fill the ranks as much
- * as we can, pulling from rx_free.
- *
- * This moves the 'write' index forward to catch up with 'processed', and
- * also updates the memory address in the firmware to reference the new
- * target buffer.
+ * iwl_pcie_rxq_sq_restock - restock implementation for single queue rx
*/
-static void iwl_pcie_rxq_restock(struct iwl_trans *trans)
+static void iwl_pcie_rxq_sq_restock(struct iwl_trans *trans,
+ struct iwl_rxq *rxq)
{
- struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
- struct iwl_rxq *rxq = &trans_pcie->rxq;
struct iwl_rx_mem_buffer *rxb;
/*
@@ -251,6 +300,7 @@ static void iwl_pcie_rxq_restock(struct iwl_trans *trans)
spin_lock(&rxq->lock);
while ((iwl_rxq_space(rxq) > 0) && (rxq->free_count)) {
+ __le32 *bd = (__le32 *)rxq->bd;
/* The overwritten rxb must be a used one */
rxb = rxq->queue[rxq->write];
BUG_ON(rxb && rxb->page);
@@ -261,7 +311,7 @@ static void iwl_pcie_rxq_restock(struct iwl_trans *trans)
list_del(&rxb->list);
/* Point to Rx buffer via next RBD in circular buffer */
- rxq->bd[rxq->write] = iwl_pcie_dma_addr2rbd_ptr(rxb->page_dma);
+ bd[rxq->write] = iwl_pcie_dma_addr2rbd_ptr(rxb->page_dma);
rxq->queue[rxq->write] = rxb;
rxq->write = (rxq->write + 1) & RX_QUEUE_MASK;
rxq->free_count--;
@@ -272,12 +322,32 @@ static void iwl_pcie_rxq_restock(struct iwl_trans *trans)
* Increment device's write pointer in multiples of 8. */
if (rxq->write_actual != (rxq->write & ~0x7)) {
spin_lock(&rxq->lock);
- iwl_pcie_rxq_inc_wr_ptr(trans);
+ iwl_pcie_rxq_inc_wr_ptr(trans, rxq);
spin_unlock(&rxq->lock);
}
}
/*
+ * iwl_pcie_rxq_restock - refill RX queue from pre-allocated pool
+ *
+ * If there are slots in the RX queue that need to be restocked,
+ * and we have free pre-allocated buffers, fill the ranks as much
+ * as we can, pulling from rx_free.
+ *
+ * This moves the 'write' index forward to catch up with 'processed', and
+ * also updates the memory address in the firmware to reference the new
+ * target buffer.
+ */
+static
+void iwl_pcie_rxq_restock(struct iwl_trans *trans, struct iwl_rxq *rxq)
+{
+ if (trans->cfg->mq_rx_supported)
+ iwl_pcie_rxq_mq_restock(trans, rxq);
+ else
+ iwl_pcie_rxq_sq_restock(trans, rxq);
+}
+
+/*
* iwl_pcie_rx_alloc_page - allocates and returns a page.
*
*/
@@ -285,13 +355,9 @@ static struct page *iwl_pcie_rx_alloc_page(struct iwl_trans *trans,
gfp_t priority)
{
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
- struct iwl_rxq *rxq = &trans_pcie->rxq;
struct page *page;
gfp_t gfp_mask = priority;
- if (rxq->free_count > RX_LOW_WATERMARK)
- gfp_mask |= __GFP_NOWARN;
-
if (trans_pcie->rx_page_order > 0)
gfp_mask |= __GFP_COMP;
@@ -301,16 +367,13 @@ static struct page *iwl_pcie_rx_alloc_page(struct iwl_trans *trans,
if (net_ratelimit())
IWL_DEBUG_INFO(trans, "alloc_pages failed, order: %d\n",
trans_pcie->rx_page_order);
- /* Issue an error if the hardware has consumed more than half
- * of its free buffer list and we don't have enough
- * pre-allocated buffers.
+ /*
+ * Issue an error if we don't have enough pre-allocated
+ * buffers.
` */
- if (rxq->free_count <= RX_LOW_WATERMARK &&
- iwl_rxq_space(rxq) > (RX_QUEUE_SIZE / 2) &&
- net_ratelimit())
+ if (!(gfp_mask & __GFP_NOWARN) && net_ratelimit())
IWL_CRIT(trans,
- "Failed to alloc_pages with GFP_KERNEL. Only %u free buffers remaining.\n",
- rxq->free_count);
+ "Failed to alloc_pages\n");
return NULL;
}
return page;
@@ -325,10 +388,10 @@ static struct page *iwl_pcie_rx_alloc_page(struct iwl_trans *trans,
* iwl_pcie_rxq_restock. The latter function will update the HW to use the newly
* allocated buffers.
*/
-static void iwl_pcie_rxq_alloc_rbs(struct iwl_trans *trans, gfp_t priority)
+static void iwl_pcie_rxq_alloc_rbs(struct iwl_trans *trans, gfp_t priority,
+ struct iwl_rxq *rxq)
{
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
- struct iwl_rxq *rxq = &trans_pcie->rxq;
struct iwl_rx_mem_buffer *rxb;
struct page *page;
@@ -372,10 +435,6 @@ static void iwl_pcie_rxq_alloc_rbs(struct iwl_trans *trans, gfp_t priority)
__free_pages(page, trans_pcie->rx_page_order);
return;
}
- /* dma address must be no more than 36 bits */
- BUG_ON(rxb->page_dma & ~DMA_BIT_MASK(36));
- /* and also 256 byte aligned! */
- BUG_ON(rxb->page_dma & DMA_BIT_MASK(8));
spin_lock(&rxq->lock);
@@ -386,41 +445,24 @@ static void iwl_pcie_rxq_alloc_rbs(struct iwl_trans *trans, gfp_t priority)
}
}
-static void iwl_pcie_rxq_free_rbs(struct iwl_trans *trans)
+static void iwl_pcie_free_rbs_pool(struct iwl_trans *trans)
{
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
- struct iwl_rxq *rxq = &trans_pcie->rxq;
int i;
- lockdep_assert_held(&rxq->lock);
-
- for (i = 0; i < RX_QUEUE_SIZE; i++) {
- if (!rxq->pool[i].page)
+ for (i = 0; i < RX_POOL_SIZE; i++) {
+ if (!trans_pcie->rx_pool[i].page)
continue;
- dma_unmap_page(trans->dev, rxq->pool[i].page_dma,
+ dma_unmap_page(trans->dev, trans_pcie->rx_pool[i].page_dma,
PAGE_SIZE << trans_pcie->rx_page_order,
DMA_FROM_DEVICE);
- __free_pages(rxq->pool[i].page, trans_pcie->rx_page_order);
- rxq->pool[i].page = NULL;
+ __free_pages(trans_pcie->rx_pool[i].page,
+ trans_pcie->rx_page_order);
+ trans_pcie->rx_pool[i].page = NULL;
}
}
/*
- * iwl_pcie_rx_replenish - Move all used buffers from rx_used to rx_free
- *
- * When moving to rx_free an page is allocated for the slot.
- *
- * Also restock the Rx queue via iwl_pcie_rxq_restock.
- * This is called only during initialization
- */
-static void iwl_pcie_rx_replenish(struct iwl_trans *trans)
-{
- iwl_pcie_rxq_alloc_rbs(trans, GFP_KERNEL);
-
- iwl_pcie_rxq_restock(trans);
-}
-
-/*
* iwl_pcie_rx_allocator - Allocates pages in the background for RX queues
*
* Allocates for each received request 8 pages
@@ -444,6 +486,11 @@ static void iwl_pcie_rx_allocator(struct iwl_trans *trans)
while (pending) {
int i;
struct list_head local_allocated;
+ gfp_t gfp_mask = GFP_KERNEL;
+
+ /* Do not post a warning if there are only a few requests */
+ if (pending < RX_PENDING_WATERMARK)
+ gfp_mask |= __GFP_NOWARN;
INIT_LIST_HEAD(&local_allocated);
@@ -463,7 +510,7 @@ static void iwl_pcie_rx_allocator(struct iwl_trans *trans)
BUG_ON(rxb->page);
/* Alloc a new receive buffer */
- page = iwl_pcie_rx_alloc_page(trans, GFP_KERNEL);
+ page = iwl_pcie_rx_alloc_page(trans, gfp_mask);
if (!page)
continue;
rxb->page = page;
@@ -477,10 +524,6 @@ static void iwl_pcie_rx_allocator(struct iwl_trans *trans)
__free_pages(page, trans_pcie->rx_page_order);
continue;
}
- /* dma address must be no more than 36 bits */
- BUG_ON(rxb->page_dma & ~DMA_BIT_MASK(36));
- /* and also 256 byte aligned! */
- BUG_ON(rxb->page_dma & DMA_BIT_MASK(8));
/* move the allocated entry to the out list */
list_move(&rxb->list, &local_allocated);
@@ -512,40 +555,46 @@ static void iwl_pcie_rx_allocator(struct iwl_trans *trans)
}
/*
- * iwl_pcie_rx_allocator_get - Returns the pre-allocated pages
+ * iwl_pcie_rx_allocator_get - returns the pre-allocated pages
.*
.* Called by queue when the queue posted allocation request and
* has freed 8 RBDs in order to restock itself.
+ * This function directly moves the allocated RBs to the queue's ownership
+ * and updates the relevant counters.
*/
-static int iwl_pcie_rx_allocator_get(struct iwl_trans *trans,
- struct iwl_rx_mem_buffer
- *out[RX_CLAIM_REQ_ALLOC])
+static void iwl_pcie_rx_allocator_get(struct iwl_trans *trans,
+ struct iwl_rxq *rxq)
{
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
struct iwl_rb_allocator *rba = &trans_pcie->rba;
int i;
+ lockdep_assert_held(&rxq->lock);
+
/*
* atomic_dec_if_positive returns req_ready - 1 for any scenario.
* If req_ready is 0 atomic_dec_if_positive will return -1 and this
- * function will return -ENOMEM, as there are no ready requests.
+ * function will return early, as there are no ready requests.
* atomic_dec_if_positive will perofrm the *actual* decrement only if
* req_ready > 0, i.e. - there are ready requests and the function
* hands one request to the caller.
*/
if (atomic_dec_if_positive(&rba->req_ready) < 0)
- return -ENOMEM;
+ return;
spin_lock(&rba->lock);
for (i = 0; i < RX_CLAIM_REQ_ALLOC; i++) {
/* Get next free Rx buffer, remove it from free list */
- out[i] = list_first_entry(&rba->rbd_allocated,
- struct iwl_rx_mem_buffer, list);
- list_del(&out[i]->list);
+ struct iwl_rx_mem_buffer *rxb =
+ list_first_entry(&rba->rbd_allocated,
+ struct iwl_rx_mem_buffer, list);
+
+ list_move(&rxb->list, &rxq->rx_free);
}
spin_unlock(&rba->lock);
- return 0;
+ rxq->used_count -= RX_CLAIM_REQ_ALLOC;
+ rxq->free_count += RX_CLAIM_REQ_ALLOC;
}
static void iwl_pcie_rx_allocator_work(struct work_struct *data)
@@ -561,38 +610,83 @@ static void iwl_pcie_rx_allocator_work(struct work_struct *data)
static int iwl_pcie_rx_alloc(struct iwl_trans *trans)
{
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
- struct iwl_rxq *rxq = &trans_pcie->rxq;
struct iwl_rb_allocator *rba = &trans_pcie->rba;
struct device *dev = trans->dev;
+ int i;
+ int free_size = trans->cfg->mq_rx_supported ? sizeof(__le64) :
+ sizeof(__le32);
- memset(&trans_pcie->rxq, 0, sizeof(trans_pcie->rxq));
+ if (WARN_ON(trans_pcie->rxq))
+ return -EINVAL;
+
+ trans_pcie->rxq = kcalloc(trans->num_rx_queues, sizeof(struct iwl_rxq),
+ GFP_KERNEL);
+ if (!trans_pcie->rxq)
+ return -EINVAL;
- spin_lock_init(&rxq->lock);
spin_lock_init(&rba->lock);
- if (WARN_ON(rxq->bd || rxq->rb_stts))
- return -EINVAL;
+ for (i = 0; i < trans->num_rx_queues; i++) {
+ struct iwl_rxq *rxq = &trans_pcie->rxq[i];
- /* Allocate the circular buffer of Read Buffer Descriptors (RBDs) */
- rxq->bd = dma_zalloc_coherent(dev, sizeof(__le32) * RX_QUEUE_SIZE,
- &rxq->bd_dma, GFP_KERNEL);
- if (!rxq->bd)
- goto err_bd;
+ spin_lock_init(&rxq->lock);
+ if (trans->cfg->mq_rx_supported)
+ rxq->queue_size = MQ_RX_TABLE_SIZE;
+ else
+ rxq->queue_size = RX_QUEUE_SIZE;
- /*Allocate the driver's pointer to receive buffer status */
- rxq->rb_stts = dma_zalloc_coherent(dev, sizeof(*rxq->rb_stts),
- &rxq->rb_stts_dma, GFP_KERNEL);
- if (!rxq->rb_stts)
- goto err_rb_stts;
+ /*
+ * Allocate the circular buffer of Read Buffer Descriptors
+ * (RBDs)
+ */
+ rxq->bd = dma_zalloc_coherent(dev,
+ free_size * rxq->queue_size,
+ &rxq->bd_dma, GFP_KERNEL);
+ if (!rxq->bd)
+ goto err;
+
+ if (trans->cfg->mq_rx_supported) {
+ rxq->used_bd = dma_zalloc_coherent(dev,
+ sizeof(__le32) *
+ rxq->queue_size,
+ &rxq->used_bd_dma,
+ GFP_KERNEL);
+ if (!rxq->used_bd)
+ goto err;
+ }
+ /*Allocate the driver's pointer to receive buffer status */
+ rxq->rb_stts = dma_zalloc_coherent(dev, sizeof(*rxq->rb_stts),
+ &rxq->rb_stts_dma,
+ GFP_KERNEL);
+ if (!rxq->rb_stts)
+ goto err;
+ }
return 0;
-err_rb_stts:
- dma_free_coherent(dev, sizeof(__le32) * RX_QUEUE_SIZE,
- rxq->bd, rxq->bd_dma);
- rxq->bd_dma = 0;
- rxq->bd = NULL;
-err_bd:
+err:
+ for (i = 0; i < trans->num_rx_queues; i++) {
+ struct iwl_rxq *rxq = &trans_pcie->rxq[i];
+
+ if (rxq->bd)
+ dma_free_coherent(dev, free_size * rxq->queue_size,
+ rxq->bd, rxq->bd_dma);
+ rxq->bd_dma = 0;
+ rxq->bd = NULL;
+
+ if (rxq->rb_stts)
+ dma_free_coherent(trans->dev,
+ sizeof(struct iwl_rb_status),
+ rxq->rb_stts, rxq->rb_stts_dma);
+
+ if (rxq->used_bd)
+ dma_free_coherent(dev, sizeof(__le32) * rxq->queue_size,
+ rxq->used_bd, rxq->used_bd_dma);
+ rxq->used_bd_dma = 0;
+ rxq->used_bd = NULL;
+ }
+ kfree(trans_pcie->rxq);
+
return -ENOMEM;
}
@@ -659,65 +753,112 @@ static void iwl_pcie_rx_hw_init(struct iwl_trans *trans, struct iwl_rxq *rxq)
iwl_set_bit(trans, CSR_INT_COALESCING, IWL_HOST_INT_OPER_MODE);
}
-static void iwl_pcie_rx_init_rxb_lists(struct iwl_rxq *rxq)
+static void iwl_pcie_rx_mq_hw_init(struct iwl_trans *trans)
{
+ struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+ u32 rb_size, enabled = 0;
int i;
- lockdep_assert_held(&rxq->lock);
-
- INIT_LIST_HEAD(&rxq->rx_free);
- INIT_LIST_HEAD(&rxq->rx_used);
- rxq->free_count = 0;
- rxq->used_count = 0;
+ switch (trans_pcie->rx_buf_size) {
+ case IWL_AMSDU_4K:
+ rb_size = RFH_RXF_DMA_RB_SIZE_4K;
+ break;
+ case IWL_AMSDU_8K:
+ rb_size = RFH_RXF_DMA_RB_SIZE_8K;
+ break;
+ case IWL_AMSDU_12K:
+ rb_size = RFH_RXF_DMA_RB_SIZE_12K;
+ break;
+ default:
+ WARN_ON(1);
+ rb_size = RFH_RXF_DMA_RB_SIZE_4K;
+ }
- for (i = 0; i < RX_QUEUE_SIZE; i++)
- list_add(&rxq->pool[i].list, &rxq->rx_used);
-}
+ /* Stop Rx DMA */
+ iwl_write_prph(trans, RFH_RXF_DMA_CFG, 0);
+ /* disable free amd used rx queue operation */
+ iwl_write_prph(trans, RFH_RXF_RXQ_ACTIVE, 0);
+
+ for (i = 0; i < trans->num_rx_queues; i++) {
+ /* Tell device where to find RBD free table in DRAM */
+ iwl_pcie_write_prph_64(trans, RFH_Q_FRBDCB_BA_LSB(i),
+ (u64)(trans_pcie->rxq[i].bd_dma));
+ /* Tell device where to find RBD used table in DRAM */
+ iwl_pcie_write_prph_64(trans, RFH_Q_URBDCB_BA_LSB(i),
+ (u64)(trans_pcie->rxq[i].used_bd_dma));
+ /* Tell device where in DRAM to update its Rx status */
+ iwl_pcie_write_prph_64(trans, RFH_Q_URBD_STTS_WPTR_LSB(i),
+ trans_pcie->rxq[i].rb_stts_dma);
+ /* Reset device indice tables */
+ iwl_write_prph(trans, RFH_Q_FRBDCB_WIDX(i), 0);
+ iwl_write_prph(trans, RFH_Q_FRBDCB_RIDX(i), 0);
+ iwl_write_prph(trans, RFH_Q_URBDCB_WIDX(i), 0);
+
+ enabled |= BIT(i) | BIT(i + 16);
+ }
-static void iwl_pcie_rx_init_rba(struct iwl_rb_allocator *rba)
-{
- int i;
+ /* restock default queue */
+ iwl_pcie_rxq_mq_restock(trans, &trans_pcie->rxq[0]);
- lockdep_assert_held(&rba->lock);
+ /*
+ * Enable Rx DMA
+ * Single frame mode
+ * Rx buffer size 4 or 8k or 12k
+ * Min RB size 4 or 8
+ * Drop frames that exceed RB size
+ * 512 RBDs
+ */
+ iwl_write_prph(trans, RFH_RXF_DMA_CFG,
+ RFH_DMA_EN_ENABLE_VAL |
+ rb_size | RFH_RXF_DMA_SINGLE_FRAME_MASK |
+ RFH_RXF_DMA_MIN_RB_4_8 |
+ RFH_RXF_DMA_DROP_TOO_LARGE_MASK |
+ RFH_RXF_DMA_RBDCB_SIZE_512);
- INIT_LIST_HEAD(&rba->rbd_allocated);
- INIT_LIST_HEAD(&rba->rbd_empty);
+ /*
+ * Activate DMA snooping.
+ * Set RX DMA chunk size to 64B
+ * Default queue is 0
+ */
+ iwl_write_prph(trans, RFH_GEN_CFG, RFH_GEN_CFG_RFH_DMA_SNOOP |
+ (DEFAULT_RXQ_NUM << RFH_GEN_CFG_DEFAULT_RXQ_NUM_POS) |
+ RFH_GEN_CFG_SERVICE_DMA_SNOOP);
+ /* Enable the relevant rx queues */
+ iwl_write_prph(trans, RFH_RXF_RXQ_ACTIVE, enabled);
- for (i = 0; i < RX_POOL_SIZE; i++)
- list_add(&rba->pool[i].list, &rba->rbd_empty);
+ /* Set interrupt coalescing timer to default (2048 usecs) */
+ iwl_write8(trans, CSR_INT_COALESCING, IWL_HOST_INT_TIMEOUT_DEF);
}
-static void iwl_pcie_rx_free_rba(struct iwl_trans *trans)
+static void iwl_pcie_rx_init_rxb_lists(struct iwl_rxq *rxq)
{
- struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
- struct iwl_rb_allocator *rba = &trans_pcie->rba;
- int i;
+ lockdep_assert_held(&rxq->lock);
- lockdep_assert_held(&rba->lock);
+ INIT_LIST_HEAD(&rxq->rx_free);
+ INIT_LIST_HEAD(&rxq->rx_used);
+ rxq->free_count = 0;
+ rxq->used_count = 0;
+}
- for (i = 0; i < RX_POOL_SIZE; i++) {
- if (!rba->pool[i].page)
- continue;
- dma_unmap_page(trans->dev, rba->pool[i].page_dma,
- PAGE_SIZE << trans_pcie->rx_page_order,
- DMA_FROM_DEVICE);
- __free_pages(rba->pool[i].page, trans_pcie->rx_page_order);
- rba->pool[i].page = NULL;
- }
+static int iwl_pcie_dummy_napi_poll(struct napi_struct *napi, int budget)
+{
+ WARN_ON(1);
+ return 0;
}
int iwl_pcie_rx_init(struct iwl_trans *trans)
{
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
- struct iwl_rxq *rxq = &trans_pcie->rxq;
+ struct iwl_rxq *def_rxq;
struct iwl_rb_allocator *rba = &trans_pcie->rba;
- int i, err;
+ int i, err, queue_size, allocator_pool_size, num_alloc;
- if (!rxq->bd) {
+ if (!trans_pcie->rxq) {
err = iwl_pcie_rx_alloc(trans);
if (err)
return err;
}
+ def_rxq = trans_pcie->rxq;
if (!rba->alloc_wq)
rba->alloc_wq = alloc_workqueue("rb_allocator",
WQ_HIGHPRI | WQ_UNBOUND, 1);
@@ -726,34 +867,69 @@ int iwl_pcie_rx_init(struct iwl_trans *trans)
spin_lock(&rba->lock);
atomic_set(&rba->req_pending, 0);
atomic_set(&rba->req_ready, 0);
- /* free all first - we might be reconfigured for a different size */
- iwl_pcie_rx_free_rba(trans);
- iwl_pcie_rx_init_rba(rba);
+ INIT_LIST_HEAD(&rba->rbd_allocated);
+ INIT_LIST_HEAD(&rba->rbd_empty);
spin_unlock(&rba->lock);
- spin_lock(&rxq->lock);
-
/* free all first - we might be reconfigured for a different size */
- iwl_pcie_rxq_free_rbs(trans);
- iwl_pcie_rx_init_rxb_lists(rxq);
+ iwl_pcie_free_rbs_pool(trans);
for (i = 0; i < RX_QUEUE_SIZE; i++)
- rxq->queue[i] = NULL;
+ def_rxq->queue[i] = NULL;
- /* Set us so that we have processed and used all buffers, but have
- * not restocked the Rx queue with fresh buffers */
- rxq->read = rxq->write = 0;
- rxq->write_actual = 0;
- memset(rxq->rb_stts, 0, sizeof(*rxq->rb_stts));
- spin_unlock(&rxq->lock);
+ for (i = 0; i < trans->num_rx_queues; i++) {
+ struct iwl_rxq *rxq = &trans_pcie->rxq[i];
- iwl_pcie_rx_replenish(trans);
+ rxq->id = i;
- iwl_pcie_rx_hw_init(trans, rxq);
+ spin_lock(&rxq->lock);
+ /*
+ * Set read write pointer to reflect that we have processed
+ * and used all buffers, but have not restocked the Rx queue
+ * with fresh buffers
+ */
+ rxq->read = 0;
+ rxq->write = 0;
+ rxq->write_actual = 0;
+ memset(rxq->rb_stts, 0, sizeof(*rxq->rb_stts));
- spin_lock(&rxq->lock);
- iwl_pcie_rxq_inc_wr_ptr(trans);
- spin_unlock(&rxq->lock);
+ iwl_pcie_rx_init_rxb_lists(rxq);
+
+ if (!rxq->napi.poll)
+ netif_napi_add(&trans_pcie->napi_dev, &rxq->napi,
+ iwl_pcie_dummy_napi_poll, 64);
+
+ spin_unlock(&rxq->lock);
+ }
+
+ /* move the pool to the default queue and allocator ownerships */
+ queue_size = trans->cfg->mq_rx_supported ?
+ MQ_RX_NUM_RBDS : RX_QUEUE_SIZE;
+ allocator_pool_size = trans->num_rx_queues *
+ (RX_CLAIM_REQ_ALLOC - RX_POST_REQ_ALLOC);
+ num_alloc = queue_size + allocator_pool_size;
+ for (i = 0; i < num_alloc; i++) {
+ struct iwl_rx_mem_buffer *rxb = &trans_pcie->rx_pool[i];
+
+ if (i < allocator_pool_size)
+ list_add(&rxb->list, &rba->rbd_empty);
+ else
+ list_add(&rxb->list, &def_rxq->rx_used);
+ trans_pcie->global_table[i] = rxb;
+ rxb->vid = (u16)i;
+ }
+
+ iwl_pcie_rxq_alloc_rbs(trans, GFP_KERNEL, def_rxq);
+ if (trans->cfg->mq_rx_supported) {
+ iwl_pcie_rx_mq_hw_init(trans);
+ } else {
+ iwl_pcie_rxq_sq_restock(trans, def_rxq);
+ iwl_pcie_rx_hw_init(trans, def_rxq);
+ }
+
+ spin_lock(&def_rxq->lock);
+ iwl_pcie_rxq_inc_wr_ptr(trans, def_rxq);
+ spin_unlock(&def_rxq->lock);
return 0;
}
@@ -761,12 +937,16 @@ int iwl_pcie_rx_init(struct iwl_trans *trans)
void iwl_pcie_rx_free(struct iwl_trans *trans)
{
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
- struct iwl_rxq *rxq = &trans_pcie->rxq;
struct iwl_rb_allocator *rba = &trans_pcie->rba;
+ int free_size = trans->cfg->mq_rx_supported ? sizeof(__le64) :
+ sizeof(__le32);
+ int i;
- /*if rxq->bd is NULL, it means that nothing has been allocated,
- * exit now */
- if (!rxq->bd) {
+ /*
+ * if rxq is NULL, it means that nothing has been allocated,
+ * exit now
+ */
+ if (!trans_pcie->rxq) {
IWL_DEBUG_INFO(trans, "Free NULL rx context\n");
return;
}
@@ -777,27 +957,37 @@ void iwl_pcie_rx_free(struct iwl_trans *trans)
rba->alloc_wq = NULL;
}
- spin_lock(&rba->lock);
- iwl_pcie_rx_free_rba(trans);
- spin_unlock(&rba->lock);
-
- spin_lock(&rxq->lock);
- iwl_pcie_rxq_free_rbs(trans);
- spin_unlock(&rxq->lock);
-
- dma_free_coherent(trans->dev, sizeof(__le32) * RX_QUEUE_SIZE,
- rxq->bd, rxq->bd_dma);
- rxq->bd_dma = 0;
- rxq->bd = NULL;
-
- if (rxq->rb_stts)
- dma_free_coherent(trans->dev,
- sizeof(struct iwl_rb_status),
- rxq->rb_stts, rxq->rb_stts_dma);
- else
- IWL_DEBUG_INFO(trans, "Free rxq->rb_stts which is NULL\n");
- rxq->rb_stts_dma = 0;
- rxq->rb_stts = NULL;
+ iwl_pcie_free_rbs_pool(trans);
+
+ for (i = 0; i < trans->num_rx_queues; i++) {
+ struct iwl_rxq *rxq = &trans_pcie->rxq[i];
+
+ if (rxq->bd)
+ dma_free_coherent(trans->dev,
+ free_size * rxq->queue_size,
+ rxq->bd, rxq->bd_dma);
+ rxq->bd_dma = 0;
+ rxq->bd = NULL;
+
+ if (rxq->rb_stts)
+ dma_free_coherent(trans->dev,
+ sizeof(struct iwl_rb_status),
+ rxq->rb_stts, rxq->rb_stts_dma);
+ else
+ IWL_DEBUG_INFO(trans,
+ "Free rxq->rb_stts which is NULL\n");
+
+ if (rxq->used_bd)
+ dma_free_coherent(trans->dev,
+ sizeof(__le32) * rxq->queue_size,
+ rxq->used_bd, rxq->used_bd_dma);
+ rxq->used_bd_dma = 0;
+ rxq->used_bd = NULL;
+
+ if (rxq->napi.poll)
+ netif_napi_del(&rxq->napi);
+ }
+ kfree(trans_pcie->rxq);
}
/*
@@ -841,11 +1031,11 @@ static void iwl_pcie_rx_reuse_rbd(struct iwl_trans *trans,
}
static void iwl_pcie_rx_handle_rb(struct iwl_trans *trans,
+ struct iwl_rxq *rxq,
struct iwl_rx_mem_buffer *rxb,
bool emergency)
{
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
- struct iwl_rxq *rxq = &trans_pcie->rxq;
struct iwl_txq *txq = &trans_pcie->txq[trans_pcie->cmd_queue];
bool page_stolen = false;
int max_len = PAGE_SIZE << trans_pcie->rx_page_order;
@@ -911,7 +1101,12 @@ static void iwl_pcie_rx_handle_rb(struct iwl_trans *trans,
index = SEQ_TO_INDEX(sequence);
cmd_index = get_cmd_index(&txq->q, index);
- iwl_op_mode_rx(trans->op_mode, &trans_pcie->napi, &rxcb);
+ if (rxq->id == 0)
+ iwl_op_mode_rx(trans->op_mode, &rxq->napi,
+ &rxcb);
+ else
+ iwl_op_mode_rx_rss(trans->op_mode, &rxq->napi,
+ &rxcb, rxq->id);
if (reclaim) {
kzfree(txq->entries[cmd_index].free_buf);
@@ -972,11 +1167,11 @@ static void iwl_pcie_rx_handle_rb(struct iwl_trans *trans,
/*
* iwl_pcie_rx_handle - Main entry function for receiving responses from fw
*/
-static void iwl_pcie_rx_handle(struct iwl_trans *trans)
+static void iwl_pcie_rx_handle(struct iwl_trans *trans, int queue)
{
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
- struct iwl_rxq *rxq = &trans_pcie->rxq;
- u32 r, i, j, count = 0;
+ struct iwl_rxq *rxq = &trans_pcie->rxq[queue];
+ u32 r, i, count = 0;
bool emergency = false;
restart:
@@ -986,80 +1181,73 @@ restart:
r = le16_to_cpu(ACCESS_ONCE(rxq->rb_stts->closed_rb_num)) & 0x0FFF;
i = rxq->read;
+ /* W/A 9000 device step A0 wrap-around bug */
+ r &= (rxq->queue_size - 1);
+
/* Rx interrupt, but nothing sent from uCode */
if (i == r)
- IWL_DEBUG_RX(trans, "HW = SW = %d\n", r);
+ IWL_DEBUG_RX(trans, "Q %d: HW = SW = %d\n", rxq->id, r);
while (i != r) {
struct iwl_rx_mem_buffer *rxb;
- if (unlikely(rxq->used_count == RX_QUEUE_SIZE / 2))
+ if (unlikely(rxq->used_count == rxq->queue_size / 2))
emergency = true;
- rxb = rxq->queue[i];
- rxq->queue[i] = NULL;
+ if (trans->cfg->mq_rx_supported) {
+ /*
+ * used_bd is a 32 bit but only 12 are used to retrieve
+ * the vid
+ */
+ u16 vid = le32_to_cpu(rxq->used_bd[i]) & 0x0FFF;
+
+ if (WARN(vid >= ARRAY_SIZE(trans_pcie->global_table),
+ "Invalid rxb index from HW %u\n", (u32)vid))
+ goto out;
+ rxb = trans_pcie->global_table[vid];
+ } else {
+ rxb = rxq->queue[i];
+ rxq->queue[i] = NULL;
+ }
- IWL_DEBUG_RX(trans, "rxbuf: HW = %d, SW = %d\n", r, i);
- iwl_pcie_rx_handle_rb(trans, rxb, emergency);
+ IWL_DEBUG_RX(trans, "Q %d: HW = %d, SW = %d\n", rxq->id, r, i);
+ iwl_pcie_rx_handle_rb(trans, rxq, rxb, emergency);
- i = (i + 1) & RX_QUEUE_MASK;
+ i = (i + 1) & (rxq->queue_size - 1);
- /* If we have RX_CLAIM_REQ_ALLOC released rx buffers -
- * try to claim the pre-allocated buffers from the allocator */
- if (rxq->used_count >= RX_CLAIM_REQ_ALLOC) {
+ /*
+ * If we have RX_CLAIM_REQ_ALLOC released rx buffers -
+ * try to claim the pre-allocated buffers from the allocator.
+ * If not ready - will try to reclaim next time.
+ * There is no need to reschedule work - allocator exits only
+ * on success
+ */
+ if (rxq->used_count >= RX_CLAIM_REQ_ALLOC)
+ iwl_pcie_rx_allocator_get(trans, rxq);
+
+ if (rxq->used_count % RX_CLAIM_REQ_ALLOC == 0 && !emergency) {
struct iwl_rb_allocator *rba = &trans_pcie->rba;
- struct iwl_rx_mem_buffer *out[RX_CLAIM_REQ_ALLOC];
-
- if (rxq->used_count % RX_CLAIM_REQ_ALLOC == 0 &&
- !emergency) {
- /* Add the remaining 6 empty RBDs
- * for allocator use
- */
- spin_lock(&rba->lock);
- list_splice_tail_init(&rxq->rx_used,
- &rba->rbd_empty);
- spin_unlock(&rba->lock);
- }
- /* If not ready - continue, will try to reclaim later.
- * No need to reschedule work - allocator exits only on
- * success */
- if (!iwl_pcie_rx_allocator_get(trans, out)) {
- /* If success - then RX_CLAIM_REQ_ALLOC
- * buffers were retrieved and should be added
- * to free list */
- rxq->used_count -= RX_CLAIM_REQ_ALLOC;
- for (j = 0; j < RX_CLAIM_REQ_ALLOC; j++) {
- list_add_tail(&out[j]->list,
- &rxq->rx_free);
- rxq->free_count++;
- }
- }
- }
- if (emergency) {
+ /* Add the remaining empty RBDs for allocator use */
+ spin_lock(&rba->lock);
+ list_splice_tail_init(&rxq->rx_used, &rba->rbd_empty);
+ spin_unlock(&rba->lock);
+ } else if (emergency) {
count++;
if (count == 8) {
count = 0;
- if (rxq->used_count < RX_QUEUE_SIZE / 3)
+ if (rxq->used_count < rxq->queue_size / 3)
emergency = false;
+
+ rxq->read = i;
spin_unlock(&rxq->lock);
- iwl_pcie_rxq_alloc_rbs(trans, GFP_ATOMIC);
- spin_lock(&rxq->lock);
+ iwl_pcie_rxq_alloc_rbs(trans, GFP_ATOMIC, rxq);
+ iwl_pcie_rxq_restock(trans, rxq);
+ goto restart;
}
}
- /* handle restock for three cases, can be all of them at once:
- * - we just pulled buffers from the allocator
- * - we have 8+ unstolen pages accumulated
- * - we are in emergency and allocated buffers
- */
- if (rxq->free_count >= RX_CLAIM_REQ_ALLOC) {
- rxq->read = i;
- spin_unlock(&rxq->lock);
- iwl_pcie_rxq_restock(trans);
- goto restart;
- }
}
-
+out:
/* Backtrack one entry */
rxq->read = i;
spin_unlock(&rxq->lock);
@@ -1077,10 +1265,60 @@ restart:
* will be restocked by the next call of iwl_pcie_rxq_restock.
*/
if (unlikely(emergency && count))
- iwl_pcie_rxq_alloc_rbs(trans, GFP_ATOMIC);
+ iwl_pcie_rxq_alloc_rbs(trans, GFP_ATOMIC, rxq);
+
+ if (rxq->napi.poll)
+ napi_gro_flush(&rxq->napi, false);
+
+ iwl_pcie_rxq_restock(trans, rxq);
+}
+
+static struct iwl_trans_pcie *iwl_pcie_get_trans_pcie(struct msix_entry *entry)
+{
+ u8 queue = entry->entry;
+ struct msix_entry *entries = entry - queue;
+
+ return container_of(entries, struct iwl_trans_pcie, msix_entries[0]);
+}
- if (trans_pcie->napi.poll)
- napi_gro_flush(&trans_pcie->napi, false);
+static inline void iwl_pcie_clear_irq(struct iwl_trans *trans,
+ struct msix_entry *entry)
+{
+ /*
+ * Before sending the interrupt the HW disables it to prevent
+ * a nested interrupt. This is done by writing 1 to the corresponding
+ * bit in the mask register. After handling the interrupt, it should be
+ * re-enabled by clearing this bit. This register is defined as
+ * write 1 clear (W1C) register, meaning that it's being clear
+ * by writing 1 to the bit.
+ */
+ iwl_write_direct32(trans, CSR_MSIX_AUTOMASK_ST_AD, BIT(entry->entry));
+}
+
+/*
+ * iwl_pcie_rx_msix_handle - Main entry function for receiving responses from fw
+ * This interrupt handler should be used with RSS queue only.
+ */
+irqreturn_t iwl_pcie_irq_rx_msix_handler(int irq, void *dev_id)
+{
+ struct msix_entry *entry = dev_id;
+ struct iwl_trans_pcie *trans_pcie = iwl_pcie_get_trans_pcie(entry);
+ struct iwl_trans *trans = trans_pcie->trans;
+
+ if (WARN_ON(entry->entry >= trans->num_rx_queues))
+ return IRQ_NONE;
+
+ lock_map_acquire(&trans->sync_cmd_lockdep_map);
+
+ local_bh_disable();
+ iwl_pcie_rx_handle(trans, entry->entry);
+ local_bh_enable();
+
+ iwl_pcie_clear_irq(trans, entry);
+
+ lock_map_release(&trans->sync_cmd_lockdep_map);
+
+ return IRQ_HANDLED;
}
/*
@@ -1413,7 +1651,7 @@ irqreturn_t iwl_pcie_irq_handler(int irq, void *dev_id)
isr_stats->rx++;
local_bh_disable();
- iwl_pcie_rx_handle(trans);
+ iwl_pcie_rx_handle(trans, 0);
local_bh_enable();
}
@@ -1556,3 +1794,129 @@ irqreturn_t iwl_pcie_isr(int irq, void *data)
return IRQ_WAKE_THREAD;
}
+
+irqreturn_t iwl_pcie_msix_isr(int irq, void *data)
+{
+ return IRQ_WAKE_THREAD;
+}
+
+irqreturn_t iwl_pcie_irq_msix_handler(int irq, void *dev_id)
+{
+ struct msix_entry *entry = dev_id;
+ struct iwl_trans_pcie *trans_pcie = iwl_pcie_get_trans_pcie(entry);
+ struct iwl_trans *trans = trans_pcie->trans;
+ struct isr_statistics *isr_stats = isr_stats = &trans_pcie->isr_stats;
+ u32 inta_fh, inta_hw;
+
+ lock_map_acquire(&trans->sync_cmd_lockdep_map);
+
+ spin_lock(&trans_pcie->irq_lock);
+ inta_fh = iwl_read_direct32(trans, CSR_MSIX_FH_INT_CAUSES_AD);
+ inta_hw = iwl_read_direct32(trans, CSR_MSIX_HW_INT_CAUSES_AD);
+ /*
+ * Clear causes registers to avoid being handling the same cause.
+ */
+ iwl_write_direct32(trans, CSR_MSIX_FH_INT_CAUSES_AD, inta_fh);
+ iwl_write_direct32(trans, CSR_MSIX_HW_INT_CAUSES_AD, inta_hw);
+ spin_unlock(&trans_pcie->irq_lock);
+
+ if (unlikely(!(inta_fh | inta_hw))) {
+ IWL_DEBUG_ISR(trans, "Ignore interrupt, inta == 0\n");
+ lock_map_release(&trans->sync_cmd_lockdep_map);
+ return IRQ_NONE;
+ }
+
+ if (iwl_have_debug_level(IWL_DL_ISR))
+ IWL_DEBUG_ISR(trans, "ISR inta_fh 0x%08x, enabled 0x%08x\n",
+ inta_fh,
+ iwl_read32(trans, CSR_MSIX_FH_INT_MASK_AD));
+
+ /* This "Tx" DMA channel is used only for loading uCode */
+ if (inta_fh & MSIX_FH_INT_CAUSES_D2S_CH0_NUM) {
+ IWL_DEBUG_ISR(trans, "uCode load interrupt\n");
+ isr_stats->tx++;
+ /*
+ * Wake up uCode load routine,
+ * now that load is complete
+ */
+ trans_pcie->ucode_write_complete = true;
+ wake_up(&trans_pcie->ucode_write_waitq);
+ }
+
+ /* Error detected by uCode */
+ if ((inta_fh & MSIX_FH_INT_CAUSES_FH_ERR) ||
+ (inta_hw & MSIX_HW_INT_CAUSES_REG_SW_ERR)) {
+ IWL_ERR(trans,
+ "Microcode SW error detected. Restarting 0x%X.\n",
+ inta_fh);
+ isr_stats->sw++;
+ iwl_pcie_irq_handle_error(trans);
+ }
+
+ /* After checking FH register check HW register */
+ if (iwl_have_debug_level(IWL_DL_ISR))
+ IWL_DEBUG_ISR(trans,
+ "ISR inta_hw 0x%08x, enabled 0x%08x\n",
+ inta_hw,
+ iwl_read32(trans, CSR_MSIX_HW_INT_MASK_AD));
+
+ /* Alive notification via Rx interrupt will do the real work */
+ if (inta_hw & MSIX_HW_INT_CAUSES_REG_ALIVE) {
+ IWL_DEBUG_ISR(trans, "Alive interrupt\n");
+ isr_stats->alive++;
+ }
+
+ /* uCode wakes up after power-down sleep */
+ if (inta_hw & MSIX_HW_INT_CAUSES_REG_WAKEUP) {
+ IWL_DEBUG_ISR(trans, "Wakeup interrupt\n");
+ iwl_pcie_rxq_check_wrptr(trans);
+ iwl_pcie_txq_check_wrptrs(trans);
+
+ isr_stats->wakeup++;
+ }
+
+ /* Chip got too hot and stopped itself */
+ if (inta_hw & MSIX_HW_INT_CAUSES_REG_CT_KILL) {
+ IWL_ERR(trans, "Microcode CT kill error detected.\n");
+ isr_stats->ctkill++;
+ }
+
+ /* HW RF KILL switch toggled */
+ if (inta_hw & MSIX_HW_INT_CAUSES_REG_RF_KILL) {
+ bool hw_rfkill;
+
+ hw_rfkill = iwl_is_rfkill_set(trans);
+ IWL_WARN(trans, "RF_KILL bit toggled to %s.\n",
+ hw_rfkill ? "disable radio" : "enable radio");
+
+ isr_stats->rfkill++;
+
+ mutex_lock(&trans_pcie->mutex);
+ iwl_trans_pcie_rf_kill(trans, hw_rfkill);
+ mutex_unlock(&trans_pcie->mutex);
+ if (hw_rfkill) {
+ set_bit(STATUS_RFKILL, &trans->status);
+ if (test_and_clear_bit(STATUS_SYNC_HCMD_ACTIVE,
+ &trans->status))
+ IWL_DEBUG_RF_KILL(trans,
+ "Rfkill while SYNC HCMD in flight\n");
+ wake_up(&trans_pcie->wait_command_queue);
+ } else {
+ clear_bit(STATUS_RFKILL, &trans->status);
+ }
+ }
+
+ if (inta_hw & MSIX_HW_INT_CAUSES_REG_HW_ERR) {
+ IWL_ERR(trans,
+ "Hardware error detected. Restarting.\n");
+
+ isr_stats->hw++;
+ iwl_pcie_irq_handle_error(trans);
+ }
+
+ iwl_pcie_clear_irq(trans, entry);
+
+ lock_map_release(&trans->sync_cmd_lockdep_map);
+
+ return IRQ_HANDLED;
+}
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
index 1198caac3..b2b79354d 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c
@@ -72,6 +72,7 @@
#include <linux/bitops.h>
#include <linux/gfp.h>
#include <linux/vmalloc.h>
+#include <linux/pm_runtime.h>
#include "iwl-drv.h"
#include "iwl-trans.h"
@@ -615,38 +616,38 @@ static int iwl_pcie_load_firmware_chunk(struct iwl_trans *trans, u32 dst_addr,
dma_addr_t phy_addr, u32 byte_cnt)
{
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+ unsigned long flags;
int ret;
trans_pcie->ucode_write_complete = false;
- iwl_write_direct32(trans,
- FH_TCSR_CHNL_TX_CONFIG_REG(FH_SRVC_CHNL),
- FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE);
+ if (!iwl_trans_grab_nic_access(trans, &flags))
+ return -EIO;
+
+ iwl_write32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(FH_SRVC_CHNL),
+ FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE);
+
+ iwl_write32(trans, FH_SRVC_CHNL_SRAM_ADDR_REG(FH_SRVC_CHNL),
+ dst_addr);
- iwl_write_direct32(trans,
- FH_SRVC_CHNL_SRAM_ADDR_REG(FH_SRVC_CHNL),
- dst_addr);
+ iwl_write32(trans, FH_TFDIB_CTRL0_REG(FH_SRVC_CHNL),
+ phy_addr & FH_MEM_TFDIB_DRAM_ADDR_LSB_MSK);
- iwl_write_direct32(trans,
- FH_TFDIB_CTRL0_REG(FH_SRVC_CHNL),
- phy_addr & FH_MEM_TFDIB_DRAM_ADDR_LSB_MSK);
+ iwl_write32(trans, FH_TFDIB_CTRL1_REG(FH_SRVC_CHNL),
+ (iwl_get_dma_hi_addr(phy_addr)
+ << FH_MEM_TFDIB_REG1_ADDR_BITSHIFT) | byte_cnt);
- iwl_write_direct32(trans,
- FH_TFDIB_CTRL1_REG(FH_SRVC_CHNL),
- (iwl_get_dma_hi_addr(phy_addr)
- << FH_MEM_TFDIB_REG1_ADDR_BITSHIFT) | byte_cnt);
+ iwl_write32(trans, FH_TCSR_CHNL_TX_BUF_STS_REG(FH_SRVC_CHNL),
+ BIT(FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_NUM) |
+ BIT(FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_IDX) |
+ FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_VALID);
- iwl_write_direct32(trans,
- FH_TCSR_CHNL_TX_BUF_STS_REG(FH_SRVC_CHNL),
- 1 << FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_NUM |
- 1 << FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_IDX |
- FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_VALID);
+ iwl_write32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(FH_SRVC_CHNL),
+ FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE |
+ FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_DISABLE |
+ FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_ENDTFD);
- iwl_write_direct32(trans,
- FH_TCSR_CHNL_TX_CONFIG_REG(FH_SRVC_CHNL),
- FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE |
- FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_DISABLE |
- FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_ENDTFD);
+ iwl_trans_release_nic_access(trans, &flags);
ret = wait_event_timeout(trans_pcie->ucode_write_waitq,
trans_pcie->ucode_write_complete, 5 * HZ);
@@ -1122,6 +1123,20 @@ static void _iwl_trans_pcie_stop_device(struct iwl_trans *trans, bool low_power)
iwl_pcie_prepare_card_hw(trans);
}
+static void iwl_pcie_synchronize_irqs(struct iwl_trans *trans)
+{
+ struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+
+ if (trans_pcie->msix_enabled) {
+ int i;
+
+ for (i = 0; i < trans_pcie->allocated_vector; i++)
+ synchronize_irq(trans_pcie->msix_entries[i].vector);
+ } else {
+ synchronize_irq(trans_pcie->pci_dev->irq);
+ }
+}
+
static int iwl_trans_pcie_start_fw(struct iwl_trans *trans,
const struct fw_img *fw, bool run_in_rfkill)
{
@@ -1148,7 +1163,7 @@ static int iwl_trans_pcie_start_fw(struct iwl_trans *trans,
iwl_disable_interrupts(trans);
/* Make sure it finished running */
- synchronize_irq(trans_pcie->pci_dev->irq);
+ iwl_pcie_synchronize_irqs(trans);
mutex_lock(&trans_pcie->mutex);
@@ -1248,11 +1263,10 @@ void iwl_trans_pcie_rf_kill(struct iwl_trans *trans, bool state)
_iwl_trans_pcie_stop_device(trans, true);
}
-static void iwl_trans_pcie_d3_suspend(struct iwl_trans *trans, bool test)
+static void iwl_trans_pcie_d3_suspend(struct iwl_trans *trans, bool test,
+ bool reset)
{
- struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
-
- if (trans->system_pm_mode == IWL_PLAT_PM_MODE_D0I3) {
+ if (!reset) {
/* Enable persistence mode to avoid reset */
iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
CSR_HW_IF_CONFIG_REG_PERSIST_MODE);
@@ -1269,14 +1283,14 @@ static void iwl_trans_pcie_d3_suspend(struct iwl_trans *trans, bool test)
iwl_pcie_disable_ict(trans);
- synchronize_irq(trans_pcie->pci_dev->irq);
+ iwl_pcie_synchronize_irqs(trans);
iwl_clear_bit(trans, CSR_GP_CNTRL,
CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
iwl_clear_bit(trans, CSR_GP_CNTRL,
CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
- if (trans->system_pm_mode == IWL_PLAT_PM_MODE_D3) {
+ if (reset) {
/*
* reset TX queues -- some of their registers reset during S3
* so if we don't reset everything here the D3 image would try
@@ -1290,7 +1304,7 @@ static void iwl_trans_pcie_d3_suspend(struct iwl_trans *trans, bool test)
static int iwl_trans_pcie_d3_resume(struct iwl_trans *trans,
enum iwl_d3_status *status,
- bool test)
+ bool test, bool reset)
{
u32 val;
int ret;
@@ -1325,7 +1339,7 @@ static int iwl_trans_pcie_d3_resume(struct iwl_trans *trans,
iwl_pcie_set_pwr(trans, false);
- if (trans->system_pm_mode == IWL_PLAT_PM_MODE_D0I3) {
+ if (!reset) {
iwl_clear_bit(trans, CSR_GP_CNTRL,
CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
} else {
@@ -1348,6 +1362,153 @@ static int iwl_trans_pcie_d3_resume(struct iwl_trans *trans,
return 0;
}
+struct iwl_causes_list {
+ u32 cause_num;
+ u32 mask_reg;
+ u8 addr;
+};
+
+static struct iwl_causes_list causes_list[] = {
+ {MSIX_FH_INT_CAUSES_D2S_CH0_NUM, CSR_MSIX_FH_INT_MASK_AD, 0},
+ {MSIX_FH_INT_CAUSES_D2S_CH1_NUM, CSR_MSIX_FH_INT_MASK_AD, 0x1},
+ {MSIX_FH_INT_CAUSES_S2D, CSR_MSIX_FH_INT_MASK_AD, 0x3},
+ {MSIX_FH_INT_CAUSES_FH_ERR, CSR_MSIX_FH_INT_MASK_AD, 0x5},
+ {MSIX_HW_INT_CAUSES_REG_ALIVE, CSR_MSIX_HW_INT_MASK_AD, 0x10},
+ {MSIX_HW_INT_CAUSES_REG_WAKEUP, CSR_MSIX_HW_INT_MASK_AD, 0x11},
+ {MSIX_HW_INT_CAUSES_REG_CT_KILL, CSR_MSIX_HW_INT_MASK_AD, 0x16},
+ {MSIX_HW_INT_CAUSES_REG_RF_KILL, CSR_MSIX_HW_INT_MASK_AD, 0x17},
+ {MSIX_HW_INT_CAUSES_REG_PERIODIC, CSR_MSIX_HW_INT_MASK_AD, 0x18},
+ {MSIX_HW_INT_CAUSES_REG_SW_ERR, CSR_MSIX_HW_INT_MASK_AD, 0x29},
+ {MSIX_HW_INT_CAUSES_REG_SCD, CSR_MSIX_HW_INT_MASK_AD, 0x2A},
+ {MSIX_HW_INT_CAUSES_REG_FH_TX, CSR_MSIX_HW_INT_MASK_AD, 0x2B},
+ {MSIX_HW_INT_CAUSES_REG_HW_ERR, CSR_MSIX_HW_INT_MASK_AD, 0x2D},
+ {MSIX_HW_INT_CAUSES_REG_HAP, CSR_MSIX_HW_INT_MASK_AD, 0x2E},
+};
+
+static void iwl_pcie_init_msix(struct iwl_trans_pcie *trans_pcie)
+{
+ u32 val, max_rx_vector, i;
+ struct iwl_trans *trans = trans_pcie->trans;
+
+ max_rx_vector = trans_pcie->allocated_vector - 1;
+
+ if (!trans_pcie->msix_enabled)
+ return;
+
+ iwl_write_prph(trans, UREG_CHICK, UREG_CHICK_MSIX_ENABLE);
+
+ /*
+ * Each cause from the list above and the RX causes is represented as
+ * a byte in the IVAR table. We access the first (N - 1) bytes and map
+ * them to the (N - 1) vectors so these vectors will be used as rx
+ * vectors. Then access all non rx causes and map them to the
+ * default queue (N'th queue).
+ */
+ for (i = 0; i < max_rx_vector; i++) {
+ iwl_write8(trans, CSR_MSIX_RX_IVAR(i), MSIX_FH_INT_CAUSES_Q(i));
+ iwl_clear_bit(trans, CSR_MSIX_FH_INT_MASK_AD,
+ BIT(MSIX_FH_INT_CAUSES_Q(i)));
+ }
+
+ for (i = 0; i < ARRAY_SIZE(causes_list); i++) {
+ val = trans_pcie->default_irq_num |
+ MSIX_NON_AUTO_CLEAR_CAUSE;
+ iwl_write8(trans, CSR_MSIX_IVAR(causes_list[i].addr), val);
+ iwl_clear_bit(trans, causes_list[i].mask_reg,
+ causes_list[i].cause_num);
+ }
+ trans_pcie->fh_init_mask =
+ ~iwl_read32(trans, CSR_MSIX_FH_INT_MASK_AD);
+ trans_pcie->fh_mask = trans_pcie->fh_init_mask;
+ trans_pcie->hw_init_mask =
+ ~iwl_read32(trans, CSR_MSIX_HW_INT_MASK_AD);
+ trans_pcie->hw_mask = trans_pcie->hw_init_mask;
+}
+
+static void iwl_pcie_set_interrupt_capa(struct pci_dev *pdev,
+ struct iwl_trans *trans)
+{
+ struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+ u16 pci_cmd;
+ int max_vector;
+ int ret, i;
+
+ if (trans->cfg->mq_rx_supported) {
+ max_vector = min_t(u32, (num_possible_cpus() + 1),
+ IWL_MAX_RX_HW_QUEUES);
+ for (i = 0; i < max_vector; i++)
+ trans_pcie->msix_entries[i].entry = i;
+
+ ret = pci_enable_msix_range(pdev, trans_pcie->msix_entries,
+ MSIX_MIN_INTERRUPT_VECTORS,
+ max_vector);
+ if (ret > 1) {
+ IWL_DEBUG_INFO(trans,
+ "Enable MSI-X allocate %d interrupt vector\n",
+ ret);
+ trans_pcie->allocated_vector = ret;
+ trans_pcie->default_irq_num =
+ trans_pcie->allocated_vector - 1;
+ trans_pcie->trans->num_rx_queues =
+ trans_pcie->allocated_vector - 1;
+ trans_pcie->msix_enabled = true;
+
+ return;
+ }
+ IWL_DEBUG_INFO(trans,
+ "ret = %d %s move to msi mode\n", ret,
+ (ret == 1) ?
+ "can't allocate more than 1 interrupt vector" :
+ "failed to enable msi-x mode");
+ pci_disable_msix(pdev);
+ }
+
+ ret = pci_enable_msi(pdev);
+ if (ret) {
+ dev_err(&pdev->dev, "pci_enable_msi failed(0X%x)\n", ret);
+ /* enable rfkill interrupt: hw bug w/a */
+ pci_read_config_word(pdev, PCI_COMMAND, &pci_cmd);
+ if (pci_cmd & PCI_COMMAND_INTX_DISABLE) {
+ pci_cmd &= ~PCI_COMMAND_INTX_DISABLE;
+ pci_write_config_word(pdev, PCI_COMMAND, pci_cmd);
+ }
+ }
+}
+
+static int iwl_pcie_init_msix_handler(struct pci_dev *pdev,
+ struct iwl_trans_pcie *trans_pcie)
+{
+ int i, last_vector;
+
+ last_vector = trans_pcie->trans->num_rx_queues;
+
+ for (i = 0; i < trans_pcie->allocated_vector; i++) {
+ int ret;
+
+ ret = request_threaded_irq(trans_pcie->msix_entries[i].vector,
+ iwl_pcie_msix_isr,
+ (i == last_vector) ?
+ iwl_pcie_irq_msix_handler :
+ iwl_pcie_irq_rx_msix_handler,
+ IRQF_SHARED,
+ DRV_NAME,
+ &trans_pcie->msix_entries[i]);
+ if (ret) {
+ int j;
+
+ IWL_ERR(trans_pcie->trans,
+ "Error allocating IRQ %d\n", i);
+ for (j = 0; j < i; j++)
+ free_irq(trans_pcie->msix_entries[i].vector,
+ &trans_pcie->msix_entries[i]);
+ pci_disable_msix(pdev);
+ return ret;
+ }
+ }
+
+ return 0;
+}
+
static int _iwl_trans_pcie_start_hw(struct iwl_trans *trans, bool low_power)
{
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
@@ -1369,6 +1530,7 @@ static int _iwl_trans_pcie_start_hw(struct iwl_trans *trans, bool low_power)
iwl_pcie_apm_init(trans);
+ iwl_pcie_init_msix(trans_pcie);
/* From now on, the op_mode will be kept updated about RF kill state */
iwl_enable_rfkill_int(trans);
@@ -1383,6 +1545,10 @@ static int _iwl_trans_pcie_start_hw(struct iwl_trans *trans, bool low_power)
/* ... rfkill can call stop_device and set it false if needed */
iwl_trans_pcie_rf_kill(trans, hw_rfkill);
+ /* Make sure we sync here, because we'll need full access later */
+ if (low_power)
+ pm_runtime_resume(trans->dev);
+
return 0;
}
@@ -1419,7 +1585,7 @@ static void iwl_trans_pcie_op_mode_leave(struct iwl_trans *trans)
mutex_unlock(&trans_pcie->mutex);
- synchronize_irq(trans_pcie->pci_dev->irq);
+ iwl_pcie_synchronize_irqs(trans);
}
static void iwl_trans_pcie_write8(struct iwl_trans *trans, u32 ofs, u8 val)
@@ -1452,12 +1618,6 @@ static void iwl_trans_pcie_write_prph(struct iwl_trans *trans, u32 addr,
iwl_trans_pcie_write32(trans, HBUS_TARG_PRPH_WDAT, val);
}
-static int iwl_pcie_dummy_napi_poll(struct napi_struct *napi, int budget)
-{
- WARN_ON(1);
- return 0;
-}
-
static void iwl_trans_pcie_configure(struct iwl_trans *trans,
const struct iwl_trans_config *trans_cfg)
{
@@ -1486,19 +1646,13 @@ static void iwl_trans_pcie_configure(struct iwl_trans *trans,
trans->command_groups = trans_cfg->command_groups;
trans->command_groups_size = trans_cfg->command_groups_size;
- /* init ref_count to 1 (should be cleared when ucode is loaded) */
- trans_pcie->ref_count = 1;
-
/* Initialize NAPI here - it should be before registering to mac80211
* in the opmode but after the HW struct is allocated.
* As this function may be called again in some corner cases don't
* do anything if NAPI was already initialized.
*/
- if (!trans_pcie->napi.poll) {
+ if (trans_pcie->napi_dev.reg_state != NETREG_DUMMY)
init_dummy_netdev(&trans_pcie->napi_dev);
- netif_napi_add(&trans_pcie->napi_dev, &trans_pcie->napi,
- iwl_pcie_dummy_napi_poll, 64);
- }
}
void iwl_trans_pcie_free(struct iwl_trans *trans)
@@ -1506,22 +1660,29 @@ void iwl_trans_pcie_free(struct iwl_trans *trans)
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
int i;
- synchronize_irq(trans_pcie->pci_dev->irq);
+ iwl_pcie_synchronize_irqs(trans);
iwl_pcie_tx_free(trans);
iwl_pcie_rx_free(trans);
- free_irq(trans_pcie->pci_dev->irq, trans);
- iwl_pcie_free_ict(trans);
+ if (trans_pcie->msix_enabled) {
+ for (i = 0; i < trans_pcie->allocated_vector; i++)
+ free_irq(trans_pcie->msix_entries[i].vector,
+ &trans_pcie->msix_entries[i]);
+
+ pci_disable_msix(trans_pcie->pci_dev);
+ trans_pcie->msix_enabled = false;
+ } else {
+ free_irq(trans_pcie->pci_dev->irq, trans);
+
+ iwl_pcie_free_ict(trans);
- pci_disable_msi(trans_pcie->pci_dev);
+ pci_disable_msi(trans_pcie->pci_dev);
+ }
iounmap(trans_pcie->hw_base);
pci_release_regions(trans_pcie->pci_dev);
pci_disable_device(trans_pcie->pci_dev);
- if (trans_pcie->napi.poll)
- netif_napi_del(&trans_pcie->napi);
-
iwl_pcie_free_fw_monitor(trans);
for_each_possible_cpu(i) {
@@ -1861,6 +2022,7 @@ void iwl_trans_pcie_ref(struct iwl_trans *trans)
spin_lock_irqsave(&trans_pcie->ref_lock, flags);
IWL_DEBUG_RPM(trans, "ref_counter: %d\n", trans_pcie->ref_count);
trans_pcie->ref_count++;
+ pm_runtime_get(&trans_pcie->pci_dev->dev);
spin_unlock_irqrestore(&trans_pcie->ref_lock, flags);
}
@@ -1879,6 +2041,10 @@ void iwl_trans_pcie_unref(struct iwl_trans *trans)
return;
}
trans_pcie->ref_count--;
+
+ pm_runtime_mark_last_busy(&trans_pcie->pci_dev->dev);
+ pm_runtime_put_autosuspend(&trans_pcie->pci_dev->dev);
+
spin_unlock_irqrestore(&trans_pcie->ref_lock, flags);
}
@@ -2031,29 +2197,48 @@ static ssize_t iwl_dbgfs_rx_queue_read(struct file *file,
{
struct iwl_trans *trans = file->private_data;
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
- struct iwl_rxq *rxq = &trans_pcie->rxq;
- char buf[256];
- int pos = 0;
- const size_t bufsz = sizeof(buf);
-
- pos += scnprintf(buf + pos, bufsz - pos, "read: %u\n",
- rxq->read);
- pos += scnprintf(buf + pos, bufsz - pos, "write: %u\n",
- rxq->write);
- pos += scnprintf(buf + pos, bufsz - pos, "write_actual: %u\n",
- rxq->write_actual);
- pos += scnprintf(buf + pos, bufsz - pos, "need_update: %d\n",
- rxq->need_update);
- pos += scnprintf(buf + pos, bufsz - pos, "free_count: %u\n",
- rxq->free_count);
- if (rxq->rb_stts) {
- pos += scnprintf(buf + pos, bufsz - pos, "closed_rb_num: %u\n",
- le16_to_cpu(rxq->rb_stts->closed_rb_num) & 0x0FFF);
- } else {
- pos += scnprintf(buf + pos, bufsz - pos,
- "closed_rb_num: Not Allocated\n");
+ char *buf;
+ int pos = 0, i, ret;
+ size_t bufsz = sizeof(buf);
+
+ bufsz = sizeof(char) * 121 * trans->num_rx_queues;
+
+ if (!trans_pcie->rxq)
+ return -EAGAIN;
+
+ buf = kzalloc(bufsz, GFP_KERNEL);
+ if (!buf)
+ return -ENOMEM;
+
+ for (i = 0; i < trans->num_rx_queues && pos < bufsz; i++) {
+ struct iwl_rxq *rxq = &trans_pcie->rxq[i];
+
+ pos += scnprintf(buf + pos, bufsz - pos, "queue#: %2d\n",
+ i);
+ pos += scnprintf(buf + pos, bufsz - pos, "\tread: %u\n",
+ rxq->read);
+ pos += scnprintf(buf + pos, bufsz - pos, "\twrite: %u\n",
+ rxq->write);
+ pos += scnprintf(buf + pos, bufsz - pos, "\twrite_actual: %u\n",
+ rxq->write_actual);
+ pos += scnprintf(buf + pos, bufsz - pos, "\tneed_update: %2d\n",
+ rxq->need_update);
+ pos += scnprintf(buf + pos, bufsz - pos, "\tfree_count: %u\n",
+ rxq->free_count);
+ if (rxq->rb_stts) {
+ pos += scnprintf(buf + pos, bufsz - pos,
+ "\tclosed_rb_num: %u\n",
+ le16_to_cpu(rxq->rb_stts->closed_rb_num) &
+ 0x0FFF);
+ } else {
+ pos += scnprintf(buf + pos, bufsz - pos,
+ "\tclosed_rb_num: Not Allocated\n");
+ }
}
- return simple_read_from_buffer(user_buf, count, ppos, buf, pos);
+ ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos);
+ kfree(buf);
+
+ return ret;
}
static ssize_t iwl_dbgfs_interrupt_read(struct file *file,
@@ -2218,7 +2403,8 @@ static u32 iwl_trans_pcie_dump_rbs(struct iwl_trans *trans,
{
struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
int max_len = PAGE_SIZE << trans_pcie->rx_page_order;
- struct iwl_rxq *rxq = &trans_pcie->rxq;
+ /* Dump RBs is supported only for pre-9000 devices (1 queue) */
+ struct iwl_rxq *rxq = &trans_pcie->rxq[0];
u32 i, r, j, rb_len = 0;
spin_lock(&rxq->lock);
@@ -2413,7 +2599,8 @@ static struct iwl_trans_dump_data
u32 len, num_rbs;
u32 monitor_len;
int i, ptr;
- bool dump_rbs = test_bit(STATUS_FW_ERROR, &trans->status);
+ bool dump_rbs = test_bit(STATUS_FW_ERROR, &trans->status) &&
+ !trans->cfg->mq_rx_supported;
/* transport dump header */
len = sizeof(*dump_data);
@@ -2468,11 +2655,12 @@ static struct iwl_trans_dump_data
len += sizeof(*data) + (FH_MEM_UPPER_BOUND - FH_MEM_LOWER_BOUND);
if (dump_rbs) {
+ /* Dump RBs is supported only for pre-9000 devices (1 queue) */
+ struct iwl_rxq *rxq = &trans_pcie->rxq[0];
/* RBs */
- num_rbs = le16_to_cpu(ACCESS_ONCE(
- trans_pcie->rxq.rb_stts->closed_rb_num))
+ num_rbs = le16_to_cpu(ACCESS_ONCE(rxq->rb_stts->closed_rb_num))
& 0x0FFF;
- num_rbs = (num_rbs - trans_pcie->rxq.read) & RX_QUEUE_MASK;
+ num_rbs = (num_rbs - rxq->read) & RX_QUEUE_MASK;
len += num_rbs * (sizeof(*data) +
sizeof(struct iwl_fw_error_dump_rb) +
(PAGE_SIZE << trans_pcie->rx_page_order));
@@ -2523,6 +2711,22 @@ static struct iwl_trans_dump_data
return dump_data;
}
+#ifdef CONFIG_PM_SLEEP
+static int iwl_trans_pcie_suspend(struct iwl_trans *trans)
+{
+ if (trans->runtime_pm_mode == IWL_PLAT_PM_MODE_D0I3)
+ return iwl_pci_fw_enter_d0i3(trans);
+
+ return 0;
+}
+
+static void iwl_trans_pcie_resume(struct iwl_trans *trans)
+{
+ if (trans->runtime_pm_mode == IWL_PLAT_PM_MODE_D0I3)
+ iwl_pci_fw_exit_d0i3(trans);
+}
+#endif /* CONFIG_PM_SLEEP */
+
static const struct iwl_trans_ops trans_ops_pcie = {
.start_hw = iwl_trans_pcie_start_hw,
.op_mode_leave = iwl_trans_pcie_op_mode_leave,
@@ -2533,6 +2737,11 @@ static const struct iwl_trans_ops trans_ops_pcie = {
.d3_suspend = iwl_trans_pcie_d3_suspend,
.d3_resume = iwl_trans_pcie_d3_resume,
+#ifdef CONFIG_PM_SLEEP
+ .suspend = iwl_trans_pcie_suspend,
+ .resume = iwl_trans_pcie_resume,
+#endif /* CONFIG_PM_SLEEP */
+
.send_cmd = iwl_trans_pcie_send_hcmd,
.tx = iwl_trans_pcie_tx,
@@ -2570,8 +2779,7 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
{
struct iwl_trans_pcie *trans_pcie;
struct iwl_trans *trans;
- u16 pci_cmd;
- int ret;
+ int ret, addr_size;
trans = iwl_trans_alloc(sizeof(struct iwl_trans_pcie),
&pdev->dev, cfg, &trans_ops_pcie, 0);
@@ -2609,11 +2817,17 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
PCIE_LINK_STATE_CLKPM);
}
+ if (cfg->mq_rx_supported)
+ addr_size = 64;
+ else
+ addr_size = 36;
+
pci_set_master(pdev);
- ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(36));
+ ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(addr_size));
if (!ret)
- ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(36));
+ ret = pci_set_consistent_dma_mask(pdev,
+ DMA_BIT_MASK(addr_size));
if (ret) {
ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
if (!ret)
@@ -2647,17 +2861,6 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
trans_pcie->pci_dev = pdev;
iwl_disable_interrupts(trans);
- ret = pci_enable_msi(pdev);
- if (ret) {
- dev_err(&pdev->dev, "pci_enable_msi failed(0X%x)\n", ret);
- /* enable rfkill interrupt: hw bug w/a */
- pci_read_config_word(pdev, PCI_COMMAND, &pci_cmd);
- if (pci_cmd & PCI_COMMAND_INTX_DISABLE) {
- pci_cmd &= ~PCI_COMMAND_INTX_DISABLE;
- pci_write_config_word(pdev, PCI_COMMAND, pci_cmd);
- }
- }
-
trans->hw_rev = iwl_read32(trans, CSR_HW_REV);
/*
* In the 8000 HW family the format of the 4 bytes of CSR_HW_REV have
@@ -2709,6 +2912,7 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
}
}
+ iwl_pcie_set_interrupt_capa(pdev, trans);
trans->hw_id = (pdev->device << 16) + pdev->subsystem_device;
snprintf(trans->hw_id_str, sizeof(trans->hw_id_str),
"PCI ID: 0x%04X:0x%04X", pdev->device, pdev->subsystem_device);
@@ -2716,19 +2920,31 @@ struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
/* Initialize the wait queue for commands */
init_waitqueue_head(&trans_pcie->wait_command_queue);
- ret = iwl_pcie_alloc_ict(trans);
- if (ret)
- goto out_pci_disable_msi;
+ init_waitqueue_head(&trans_pcie->d0i3_waitq);
- ret = request_threaded_irq(pdev->irq, iwl_pcie_isr,
- iwl_pcie_irq_handler,
- IRQF_SHARED, DRV_NAME, trans);
- if (ret) {
- IWL_ERR(trans, "Error allocating IRQ %d\n", pdev->irq);
- goto out_free_ict;
- }
+ if (trans_pcie->msix_enabled) {
+ if (iwl_pcie_init_msix_handler(pdev, trans_pcie))
+ goto out_pci_release_regions;
+ } else {
+ ret = iwl_pcie_alloc_ict(trans);
+ if (ret)
+ goto out_pci_disable_msi;
- trans_pcie->inta_mask = CSR_INI_SET_MASK;
+ ret = request_threaded_irq(pdev->irq, iwl_pcie_isr,
+ iwl_pcie_irq_handler,
+ IRQF_SHARED, DRV_NAME, trans);
+ if (ret) {
+ IWL_ERR(trans, "Error allocating IRQ %d\n", pdev->irq);
+ goto out_free_ict;
+ }
+ trans_pcie->inta_mask = CSR_INI_SET_MASK;
+ }
+
+#ifdef CONFIG_IWLWIFI_PCIE_RTPM
+ trans->runtime_pm_mode = IWL_PLAT_PM_MODE_D0I3;
+#else
+ trans->runtime_pm_mode = IWL_PLAT_PM_MODE_DISABLED;
+#endif /* CONFIG_IWLWIFI_PCIE_RTPM */
return trans;
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/tx.c b/drivers/net/wireless/intel/iwlwifi/pcie/tx.c
index 5262028b5..16ad820ca 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/tx.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/tx.c
@@ -1,7 +1,8 @@
/******************************************************************************
*
* Copyright(c) 2003 - 2014 Intel Corporation. All rights reserved.
- * Copyright(c) 2013 - 2014 Intel Mobile Communications GmbH
+ * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
+ * Copyright(c) 2016 Intel Deutschland GmbH
*
* Portions of this file are derived from the ipw3945 project, as well
* as portions of the ieee80211 subsystem header files.
@@ -33,7 +34,6 @@
#include <linux/sched.h>
#include <net/ip6_checksum.h>
#include <net/tso.h>
-#include <net/ip6_checksum.h>
#include "iwl-debug.h"
#include "iwl-csr.h"
@@ -571,6 +571,7 @@ static int iwl_pcie_txq_init(struct iwl_trans *trans, struct iwl_txq *txq,
return ret;
spin_lock_init(&txq->lock);
+ __skb_queue_head_init(&txq->overflow_q);
/*
* Tell nic where to find circular buffer of Tx Frame Descriptors for
@@ -621,6 +622,13 @@ static void iwl_pcie_txq_unmap(struct iwl_trans *trans, int txq_id)
q->read_ptr = iwl_queue_inc_wrap(q->read_ptr);
}
txq->active = false;
+
+ while (!skb_queue_empty(&txq->overflow_q)) {
+ struct sk_buff *skb = __skb_dequeue(&txq->overflow_q);
+
+ iwl_op_mode_free_skb(trans->op_mode, skb);
+ }
+
spin_unlock_bh(&txq->lock);
/* just in case - this queue may have been stopped */
@@ -1052,8 +1060,41 @@ void iwl_trans_pcie_reclaim(struct iwl_trans *trans, int txq_id, int ssn,
iwl_pcie_txq_progress(txq);
- if (iwl_queue_space(&txq->q) > txq->q.low_mark)
- iwl_wake_queue(trans, txq);
+ if (iwl_queue_space(&txq->q) > txq->q.low_mark &&
+ test_bit(txq_id, trans_pcie->queue_stopped)) {
+ struct sk_buff_head overflow_skbs;
+
+ __skb_queue_head_init(&overflow_skbs);
+ skb_queue_splice_init(&txq->overflow_q, &overflow_skbs);
+
+ /*
+ * This is tricky: we are in reclaim path which is non
+ * re-entrant, so noone will try to take the access the
+ * txq data from that path. We stopped tx, so we can't
+ * have tx as well. Bottom line, we can unlock and re-lock
+ * later.
+ */
+ spin_unlock_bh(&txq->lock);
+
+ while (!skb_queue_empty(&overflow_skbs)) {
+ struct sk_buff *skb = __skb_dequeue(&overflow_skbs);
+ struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+ u8 dev_cmd_idx = IWL_TRANS_FIRST_DRIVER_DATA + 1;
+ struct iwl_device_cmd *dev_cmd =
+ info->driver_data[dev_cmd_idx];
+
+ /*
+ * Note that we can very well be overflowing again.
+ * In that case, iwl_queue_space will be small again
+ * and we won't wake mac80211's queue.
+ */
+ iwl_trans_pcie_tx(trans, skb, dev_cmd, txq_id);
+ }
+ spin_lock_bh(&txq->lock);
+
+ if (iwl_queue_space(&txq->q) > txq->q.low_mark)
+ iwl_wake_queue(trans, txq);
+ }
if (q->read_ptr == q->write_ptr) {
IWL_DEBUG_RPM(trans, "Q %d - last tx reclaimed\n", q->id);
@@ -1686,6 +1727,20 @@ void iwl_pcie_hcmd_complete(struct iwl_trans *trans,
wake_up(&trans_pcie->wait_command_queue);
}
+ if (meta->flags & CMD_MAKE_TRANS_IDLE) {
+ IWL_DEBUG_INFO(trans, "complete %s - mark trans as idle\n",
+ iwl_get_cmd_string(trans, cmd->hdr.cmd));
+ set_bit(STATUS_TRANS_IDLE, &trans->status);
+ wake_up(&trans_pcie->d0i3_waitq);
+ }
+
+ if (meta->flags & CMD_WAKE_UP_TRANS) {
+ IWL_DEBUG_INFO(trans, "complete %s - clear trans idle flag\n",
+ iwl_get_cmd_string(trans, cmd->hdr.cmd));
+ clear_bit(STATUS_TRANS_IDLE, &trans->status);
+ wake_up(&trans_pcie->d0i3_waitq);
+ }
+
meta->flags = 0;
spin_unlock_bh(&txq->lock);
@@ -2161,6 +2216,8 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
csum = skb_checksum(skb, offs, skb->len - offs, 0);
*(__sum16 *)(skb->data + csum_offs) = csum_fold(csum);
+
+ skb->ip_summed = CHECKSUM_UNNECESSARY;
}
if (skb_is_nonlinear(skb) &&
@@ -2177,6 +2234,22 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
spin_lock(&txq->lock);
+ if (iwl_queue_space(q) < q->high_mark) {
+ iwl_stop_queue(trans, txq);
+
+ /* don't put the packet on the ring, if there is no room */
+ if (unlikely(iwl_queue_space(q) < 3)) {
+ struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+
+ info->driver_data[IWL_TRANS_FIRST_DRIVER_DATA + 1] =
+ dev_cmd;
+ __skb_queue_tail(&txq->overflow_q, skb);
+
+ spin_unlock(&txq->lock);
+ return 0;
+ }
+ }
+
/* In AGG mode, the index in the ring must correspond to the WiFi
* sequence number. This is a HW requirements to help the SCD to parse
* the BA.
@@ -2281,12 +2354,6 @@ int iwl_trans_pcie_tx(struct iwl_trans *trans, struct sk_buff *skb,
* At this point the frame is "transmitted" successfully
* and we will get a TX status notification eventually.
*/
- if (iwl_queue_space(q) < q->high_mark) {
- if (wait_write_ptr)
- iwl_pcie_txq_inc_wr_ptr(trans, txq);
- else
- iwl_stop_queue(trans, txq);
- }
spin_unlock(&txq->lock);
return 0;
out_err: