diff options
author | André Fabian Silva Delgado <emulatorman@parabola.nu> | 2016-06-10 05:30:17 -0300 |
---|---|---|
committer | André Fabian Silva Delgado <emulatorman@parabola.nu> | 2016-06-10 05:30:17 -0300 |
commit | d635711daa98be86d4c7fd01499c34f566b54ccb (patch) | |
tree | aa5cc3760a27c3d57146498cb82fa549547de06c /drivers/target/target_core_user.c | |
parent | c91265cd0efb83778f015b4d4b1129bd2cfd075e (diff) |
Linux-libre 4.6.2-gnu
Diffstat (limited to 'drivers/target/target_core_user.c')
-rw-r--r-- | drivers/target/target_core_user.c | 267 |
1 files changed, 155 insertions, 112 deletions
diff --git a/drivers/target/target_core_user.c b/drivers/target/target_core_user.c index 94f5154ac..62bf4fe57 100644 --- a/drivers/target/target_core_user.c +++ b/drivers/target/target_core_user.c @@ -26,6 +26,7 @@ #include <linux/vmalloc.h> #include <linux/uio_driver.h> #include <linux/stringify.h> +#include <linux/bitops.h> #include <net/genetlink.h> #include <scsi/scsi_common.h> #include <scsi/scsi_proto.h> @@ -63,8 +64,11 @@ #define TCMU_TIME_OUT (30 * MSEC_PER_SEC) +#define DATA_BLOCK_BITS 256 +#define DATA_BLOCK_SIZE 4096 + #define CMDR_SIZE (16 * 4096) -#define DATA_SIZE (257 * 4096) +#define DATA_SIZE (DATA_BLOCK_BITS * DATA_BLOCK_SIZE) #define TCMU_RING_SIZE (CMDR_SIZE + DATA_SIZE) @@ -93,12 +97,11 @@ struct tcmu_dev { u32 cmdr_size; u32 cmdr_last_cleaned; /* Offset of data ring from start of mb */ + /* Must add data_off and mb_addr to get the address */ size_t data_off; size_t data_size; - /* Ring head + tail values. */ - /* Must add data_off and mb_addr to get the address */ - size_t data_head; - size_t data_tail; + + DECLARE_BITMAP(data_bitmap, DATA_BLOCK_BITS); wait_queue_head_t wait_cmdr; /* TODO should this be a mutex? */ @@ -122,9 +125,9 @@ struct tcmu_cmd { uint16_t cmd_id; - /* Can't use se_cmd->data_length when cleaning up expired cmds, because if + /* Can't use se_cmd when cleaning up expired cmds, because if cmd has been completed then accessing se_cmd is off limits */ - size_t data_length; + DECLARE_BITMAP(data_bitmap, DATA_BLOCK_BITS); unsigned long deadline; @@ -168,13 +171,6 @@ static struct tcmu_cmd *tcmu_alloc_cmd(struct se_cmd *se_cmd) tcmu_cmd->se_cmd = se_cmd; tcmu_cmd->tcmu_dev = udev; - tcmu_cmd->data_length = se_cmd->data_length; - - if (se_cmd->se_cmd_flags & SCF_BIDI) { - BUG_ON(!(se_cmd->t_bidi_data_sg && se_cmd->t_bidi_data_nents)); - tcmu_cmd->data_length += se_cmd->t_bidi_data_sg->length; - } - tcmu_cmd->deadline = jiffies + msecs_to_jiffies(TCMU_TIME_OUT); idr_preload(GFP_KERNEL); @@ -231,105 +227,126 @@ static inline size_t head_to_end(size_t head, size_t size) return size - head; } +static inline void new_iov(struct iovec **iov, int *iov_cnt, + struct tcmu_dev *udev) +{ + struct iovec *iovec; + + if (*iov_cnt != 0) + (*iov)++; + (*iov_cnt)++; + + iovec = *iov; + memset(iovec, 0, sizeof(struct iovec)); +} + #define UPDATE_HEAD(head, used, size) smp_store_release(&head, ((head % size) + used) % size) +/* offset is relative to mb_addr */ +static inline size_t get_block_offset(struct tcmu_dev *dev, + int block, int remaining) +{ + return dev->data_off + block * DATA_BLOCK_SIZE + + DATA_BLOCK_SIZE - remaining; +} + +static inline size_t iov_tail(struct tcmu_dev *udev, struct iovec *iov) +{ + return (size_t)iov->iov_base + iov->iov_len; +} + static void alloc_and_scatter_data_area(struct tcmu_dev *udev, struct scatterlist *data_sg, unsigned int data_nents, struct iovec **iov, int *iov_cnt, bool copy_data) { - int i; + int i, block; + int block_remaining = 0; void *from, *to; - size_t copy_bytes; + size_t copy_bytes, to_offset; struct scatterlist *sg; for_each_sg(data_sg, sg, data_nents, i) { - copy_bytes = min_t(size_t, sg->length, - head_to_end(udev->data_head, udev->data_size)); + int sg_remaining = sg->length; from = kmap_atomic(sg_page(sg)) + sg->offset; - to = (void *) udev->mb_addr + udev->data_off + udev->data_head; - - if (copy_data) { - memcpy(to, from, copy_bytes); - tcmu_flush_dcache_range(to, copy_bytes); - } - - /* Even iov_base is relative to mb_addr */ - (*iov)->iov_len = copy_bytes; - (*iov)->iov_base = (void __user *) udev->data_off + - udev->data_head; - (*iov_cnt)++; - (*iov)++; - - UPDATE_HEAD(udev->data_head, copy_bytes, udev->data_size); - - /* Uh oh, we wrapped the buffer. Must split sg across 2 iovs. */ - if (sg->length != copy_bytes) { - void *from_skip = from + copy_bytes; - - copy_bytes = sg->length - copy_bytes; - - (*iov)->iov_len = copy_bytes; - (*iov)->iov_base = (void __user *) udev->data_off + - udev->data_head; - + while (sg_remaining > 0) { + if (block_remaining == 0) { + block = find_first_zero_bit(udev->data_bitmap, + DATA_BLOCK_BITS); + block_remaining = DATA_BLOCK_SIZE; + set_bit(block, udev->data_bitmap); + } + copy_bytes = min_t(size_t, sg_remaining, + block_remaining); + to_offset = get_block_offset(udev, block, + block_remaining); + to = (void *)udev->mb_addr + to_offset; + if (*iov_cnt != 0 && + to_offset == iov_tail(udev, *iov)) { + (*iov)->iov_len += copy_bytes; + } else { + new_iov(iov, iov_cnt, udev); + (*iov)->iov_base = (void __user *) to_offset; + (*iov)->iov_len = copy_bytes; + } if (copy_data) { - to = (void *) udev->mb_addr + - udev->data_off + udev->data_head; - memcpy(to, from_skip, copy_bytes); + memcpy(to, from + sg->length - sg_remaining, + copy_bytes); tcmu_flush_dcache_range(to, copy_bytes); } - - (*iov_cnt)++; - (*iov)++; - - UPDATE_HEAD(udev->data_head, - copy_bytes, udev->data_size); + sg_remaining -= copy_bytes; + block_remaining -= copy_bytes; } - kunmap_atomic(from - sg->offset); } } -static void gather_and_free_data_area(struct tcmu_dev *udev, - struct scatterlist *data_sg, unsigned int data_nents) +static void free_data_area(struct tcmu_dev *udev, struct tcmu_cmd *cmd) { - int i; + bitmap_xor(udev->data_bitmap, udev->data_bitmap, cmd->data_bitmap, + DATA_BLOCK_BITS); +} + +static void gather_data_area(struct tcmu_dev *udev, unsigned long *cmd_bitmap, + struct scatterlist *data_sg, unsigned int data_nents) +{ + int i, block; + int block_remaining = 0; void *from, *to; - size_t copy_bytes; + size_t copy_bytes, from_offset; struct scatterlist *sg; - /* It'd be easier to look at entry's iovec again, but UAM */ for_each_sg(data_sg, sg, data_nents, i) { - copy_bytes = min_t(size_t, sg->length, - head_to_end(udev->data_tail, udev->data_size)); - + int sg_remaining = sg->length; to = kmap_atomic(sg_page(sg)) + sg->offset; - WARN_ON(sg->length + sg->offset > PAGE_SIZE); - from = (void *) udev->mb_addr + - udev->data_off + udev->data_tail; - tcmu_flush_dcache_range(from, copy_bytes); - memcpy(to, from, copy_bytes); - - UPDATE_HEAD(udev->data_tail, copy_bytes, udev->data_size); - - /* Uh oh, wrapped the data buffer for this sg's data */ - if (sg->length != copy_bytes) { - void *to_skip = to + copy_bytes; - - from = (void *) udev->mb_addr + - udev->data_off + udev->data_tail; - WARN_ON(udev->data_tail); - copy_bytes = sg->length - copy_bytes; + while (sg_remaining > 0) { + if (block_remaining == 0) { + block = find_first_bit(cmd_bitmap, + DATA_BLOCK_BITS); + block_remaining = DATA_BLOCK_SIZE; + clear_bit(block, cmd_bitmap); + } + copy_bytes = min_t(size_t, sg_remaining, + block_remaining); + from_offset = get_block_offset(udev, block, + block_remaining); + from = (void *) udev->mb_addr + from_offset; tcmu_flush_dcache_range(from, copy_bytes); - memcpy(to_skip, from, copy_bytes); + memcpy(to + sg->length - sg_remaining, from, + copy_bytes); - UPDATE_HEAD(udev->data_tail, - copy_bytes, udev->data_size); + sg_remaining -= copy_bytes; + block_remaining -= copy_bytes; } kunmap_atomic(to - sg->offset); } } +static inline size_t spc_bitmap_free(unsigned long *bitmap) +{ + return DATA_BLOCK_SIZE * (DATA_BLOCK_BITS - + bitmap_weight(bitmap, DATA_BLOCK_BITS)); +} + /* * We can't queue a command until we have space available on the cmd ring *and* * space available on the data ring. @@ -339,9 +356,8 @@ static void gather_and_free_data_area(struct tcmu_dev *udev, static bool is_ring_space_avail(struct tcmu_dev *udev, size_t cmd_size, size_t data_needed) { struct tcmu_mailbox *mb = udev->mb_addr; - size_t space; + size_t space, cmd_needed; u32 cmd_head; - size_t cmd_needed; tcmu_flush_dcache_range(mb, sizeof(*mb)); @@ -363,10 +379,10 @@ static bool is_ring_space_avail(struct tcmu_dev *udev, size_t cmd_size, size_t d return false; } - space = spc_free(udev->data_head, udev->data_tail, udev->data_size); + space = spc_bitmap_free(udev->data_bitmap); if (space < data_needed) { - pr_debug("no data space: %zu %zu %zu\n", udev->data_head, - udev->data_tail, udev->data_size); + pr_debug("no data space: only %zu available, but ask for %zu\n", + space, data_needed); return false; } @@ -385,6 +401,8 @@ static int tcmu_queue_cmd_ring(struct tcmu_cmd *tcmu_cmd) uint32_t cmd_head; uint64_t cdb_off; bool copy_to_data_area; + size_t data_length; + DECLARE_BITMAP(old_bitmap, DATA_BLOCK_BITS); if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags)) return -EINVAL; @@ -393,12 +411,12 @@ static int tcmu_queue_cmd_ring(struct tcmu_cmd *tcmu_cmd) * Must be a certain minimum size for response sense info, but * also may be larger if the iov array is large. * - * iovs = sgl_nents+1, for end-of-ring case, plus another 1 - * b/c size == offsetof one-past-element. + * We prepare way too many iovs for potential uses here, because it's + * expensive to tell how many regions are freed in the bitmap */ base_command_size = max(offsetof(struct tcmu_cmd_entry, - req.iov[se_cmd->t_bidi_data_nents + - se_cmd->t_data_nents + 2]), + req.iov[se_cmd->t_bidi_data_nents + + se_cmd->t_data_nents]), sizeof(struct tcmu_cmd_entry)); command_size = base_command_size + round_up(scsi_command_size(se_cmd->t_task_cdb), TCMU_OP_ALIGN_SIZE); @@ -409,13 +427,18 @@ static int tcmu_queue_cmd_ring(struct tcmu_cmd *tcmu_cmd) mb = udev->mb_addr; cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */ + data_length = se_cmd->data_length; + if (se_cmd->se_cmd_flags & SCF_BIDI) { + BUG_ON(!(se_cmd->t_bidi_data_sg && se_cmd->t_bidi_data_nents)); + data_length += se_cmd->t_bidi_data_sg->length; + } if ((command_size > (udev->cmdr_size / 2)) - || tcmu_cmd->data_length > (udev->data_size - 1)) + || data_length > udev->data_size) pr_warn("TCMU: Request of size %zu/%zu may be too big for %u/%zu " - "cmd/data ring buffers\n", command_size, tcmu_cmd->data_length, + "cmd/data ring buffers\n", command_size, data_length, udev->cmdr_size, udev->data_size); - while (!is_ring_space_avail(udev, command_size, tcmu_cmd->data_length)) { + while (!is_ring_space_avail(udev, command_size, data_length)) { int ret; DEFINE_WAIT(__wait); @@ -462,6 +485,8 @@ static int tcmu_queue_cmd_ring(struct tcmu_cmd *tcmu_cmd) entry->hdr.kflags = 0; entry->hdr.uflags = 0; + bitmap_copy(old_bitmap, udev->data_bitmap, DATA_BLOCK_BITS); + /* * Fix up iovecs, and handle if allocation in data ring wrapped. */ @@ -480,6 +505,10 @@ static int tcmu_queue_cmd_ring(struct tcmu_cmd *tcmu_cmd) se_cmd->t_bidi_data_nents, &iov, &iov_cnt, false); entry->req.iov_bidi_cnt = iov_cnt; + /* cmd's data_bitmap is what changed in process */ + bitmap_xor(tcmu_cmd->data_bitmap, old_bitmap, udev->data_bitmap, + DATA_BLOCK_BITS); + /* All offsets relative to mb_addr, not start of entry! */ cdb_off = CMDR_OFF + cmd_head + base_command_size; memcpy((void *) mb + cdb_off, se_cmd->t_task_cdb, scsi_command_size(se_cmd->t_task_cdb)); @@ -530,35 +559,42 @@ static void tcmu_handle_completion(struct tcmu_cmd *cmd, struct tcmu_cmd_entry * struct tcmu_dev *udev = cmd->tcmu_dev; if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) { - /* cmd has been completed already from timeout, just reclaim data - ring space */ - UPDATE_HEAD(udev->data_tail, cmd->data_length, udev->data_size); + /* + * cmd has been completed already from timeout, just reclaim + * data ring space and free cmd + */ + free_data_area(udev, cmd); + + kmem_cache_free(tcmu_cmd_cache, cmd); return; } if (entry->hdr.uflags & TCMU_UFLAG_UNKNOWN_OP) { - UPDATE_HEAD(udev->data_tail, cmd->data_length, udev->data_size); + free_data_area(udev, cmd); pr_warn("TCMU: Userspace set UNKNOWN_OP flag on se_cmd %p\n", cmd->se_cmd); entry->rsp.scsi_status = SAM_STAT_CHECK_CONDITION; } else if (entry->rsp.scsi_status == SAM_STAT_CHECK_CONDITION) { memcpy(se_cmd->sense_buffer, entry->rsp.sense_buffer, se_cmd->scsi_sense_length); - - UPDATE_HEAD(udev->data_tail, cmd->data_length, udev->data_size); + free_data_area(udev, cmd); } else if (se_cmd->se_cmd_flags & SCF_BIDI) { - /* Discard data_out buffer */ - UPDATE_HEAD(udev->data_tail, - (size_t)se_cmd->t_data_sg->length, udev->data_size); + DECLARE_BITMAP(bitmap, DATA_BLOCK_BITS); - /* Get Data-In buffer */ - gather_and_free_data_area(udev, + /* Get Data-In buffer before clean up */ + bitmap_copy(bitmap, cmd->data_bitmap, DATA_BLOCK_BITS); + gather_data_area(udev, bitmap, se_cmd->t_bidi_data_sg, se_cmd->t_bidi_data_nents); + free_data_area(udev, cmd); } else if (se_cmd->data_direction == DMA_FROM_DEVICE) { - gather_and_free_data_area(udev, + DECLARE_BITMAP(bitmap, DATA_BLOCK_BITS); + + bitmap_copy(bitmap, cmd->data_bitmap, DATA_BLOCK_BITS); + gather_data_area(udev, bitmap, se_cmd->t_data_sg, se_cmd->t_data_nents); + free_data_area(udev, cmd); } else if (se_cmd->data_direction == DMA_TO_DEVICE) { - UPDATE_HEAD(udev->data_tail, cmd->data_length, udev->data_size); + free_data_area(udev, cmd); } else if (se_cmd->data_direction != DMA_NONE) { pr_warn("TCMU: data direction was %d!\n", se_cmd->data_direction); @@ -894,11 +930,13 @@ static int tcmu_configure_device(struct se_device *dev) mb = udev->mb_addr; mb->version = TCMU_MAILBOX_VERSION; + mb->flags = TCMU_MAILBOX_FLAG_CAP_OOOC; mb->cmdr_off = CMDR_OFF; mb->cmdr_size = udev->cmdr_size; WARN_ON(!PAGE_ALIGNED(udev->data_off)); WARN_ON(udev->data_size % PAGE_SIZE); + WARN_ON(udev->data_size % DATA_BLOCK_SIZE); info->version = __stringify(TCMU_MAILBOX_VERSION); @@ -942,12 +980,12 @@ err_vzalloc: return ret; } -static int tcmu_check_pending_cmd(int id, void *p, void *data) +static int tcmu_check_and_free_pending_cmd(struct tcmu_cmd *cmd) { - struct tcmu_cmd *cmd = p; - - if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) + if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) { + kmem_cache_free(tcmu_cmd_cache, cmd); return 0; + } return -EINVAL; } @@ -962,6 +1000,8 @@ static void tcmu_dev_call_rcu(struct rcu_head *p) static void tcmu_free_device(struct se_device *dev) { struct tcmu_dev *udev = TCMU_DEV(dev); + struct tcmu_cmd *cmd; + bool all_expired = true; int i; del_timer_sync(&udev->timeout); @@ -970,10 +1010,13 @@ static void tcmu_free_device(struct se_device *dev) /* Upper layer should drain all requests before calling this */ spin_lock_irq(&udev->commands_lock); - i = idr_for_each(&udev->commands, tcmu_check_pending_cmd, NULL); + idr_for_each_entry(&udev->commands, cmd, i) { + if (tcmu_check_and_free_pending_cmd(cmd) != 0) + all_expired = false; + } idr_destroy(&udev->commands); spin_unlock_irq(&udev->commands_lock); - WARN_ON(i); + WARN_ON(!all_expired); /* Device was configured */ if (udev->uio_info.uio_dev) { |