/* * kernel/power/tuxonice_bio.c * * Copyright (C) 2004-2015 Nigel Cunningham (nigel at nigelcunningham com au) * * Distributed under GPLv2. * * This file contains block io functions for TuxOnIce. These are * used by the swapwriter and it is planned that they will also * be used by the NFSwriter. * */ #include #include #include #include #include #include #include "tuxonice.h" #include "tuxonice_sysfs.h" #include "tuxonice_modules.h" #include "tuxonice_prepare_image.h" #include "tuxonice_bio.h" #include "tuxonice_ui.h" #include "tuxonice_alloc.h" #include "tuxonice_io.h" #include "tuxonice_builtin.h" #include "tuxonice_bio_internal.h" #define MEMORY_ONLY 1 #define THROTTLE_WAIT 2 /* #define MEASURE_MUTEX_CONTENTION */ #ifndef MEASURE_MUTEX_CONTENTION #define my_mutex_lock(index, the_lock) mutex_lock(the_lock) #define my_mutex_unlock(index, the_lock) mutex_unlock(the_lock) #else unsigned long mutex_times[2][2][NR_CPUS]; #define my_mutex_lock(index, the_lock) do { \ int have_mutex; \ have_mutex = mutex_trylock(the_lock); \ if (!have_mutex) { \ mutex_lock(the_lock); \ mutex_times[index][0][smp_processor_id()]++; \ } else { \ mutex_times[index][1][smp_processor_id()]++; \ } #define my_mutex_unlock(index, the_lock) \ mutex_unlock(the_lock); \ } while (0) #endif static int page_idx, reset_idx; static int target_outstanding_io = 1024; static int max_outstanding_writes, max_outstanding_reads; static struct page *bio_queue_head, *bio_queue_tail; static atomic_t toi_bio_queue_size; static DEFINE_SPINLOCK(bio_queue_lock); static int free_mem_throttle, throughput_throttle; int more_readahead = 1; static struct page *readahead_list_head, *readahead_list_tail; static struct page *waiting_on; static atomic_t toi_io_in_progress, toi_io_done; static DECLARE_WAIT_QUEUE_HEAD(num_in_progress_wait); int current_stream; /* Not static, so that the allocators can setup and complete * writing the header */ char *toi_writer_buffer; int toi_writer_buffer_posn; static DEFINE_MUTEX(toi_bio_mutex); static DEFINE_MUTEX(toi_bio_readahead_mutex); static struct task_struct *toi_queue_flusher; static int toi_bio_queue_flush_pages(int dedicated_thread); struct toi_module_ops toi_blockwriter_ops; struct toi_incremental_image_pointer toi_inc_ptr[2][2]; #define TOTAL_OUTSTANDING_IO (atomic_read(&toi_io_in_progress) + \ atomic_read(&toi_bio_queue_size)) unsigned long raw_pages_allocd, header_pages_reserved; static int toi_rw_buffer(int writing, char *buffer, int buffer_size, int no_readahead); /** * set_free_mem_throttle - set the point where we pause to avoid oom. * * Initially, this value is zero, but when we first fail to allocate memory, * we set it (plus a buffer) and thereafter throttle i/o once that limit is * reached. **/ static void set_free_mem_throttle(void) { int new_throttle = nr_free_buffer_pages() + 256; if (new_throttle > free_mem_throttle) free_mem_throttle = new_throttle; } #define NUM_REASONS 7 static atomic_t reasons[NUM_REASONS]; static char *reason_name[NUM_REASONS] = { "readahead not ready", "bio allocation", "synchronous I/O", "toi_bio_get_new_page", "memory low", "readahead buffer allocation", "throughput_throttle", }; /* User Specified Parameters. */ unsigned long resume_firstblock; dev_t resume_dev_t; struct block_device *resume_block_device; static atomic_t resume_bdev_open_count; struct block_device *header_block_device; /** * toi_open_bdev: Open a bdev at resume time. * * index: The swap index. May be MAX_SWAPFILES for the resume_dev_t * (the user can have resume= pointing at a swap partition/file that isn't * swapon'd when they hibernate. MAX_SWAPFILES+1 for the first page of the * header. It will be from a swap partition that was enabled when we hibernated, * but we don't know it's real index until we read that first page. * dev_t: The device major/minor. * display_errs: Whether to try to do this quietly. * * We stored a dev_t in the image header. Open the matching device without * requiring /dev/ in most cases and record the details needed * to close it later and avoid duplicating work. */ struct block_device *toi_open_bdev(char *uuid, dev_t default_device, int display_errs) { struct block_device *bdev; dev_t device = default_device; char buf[32]; int retried = 0; retry: if (uuid) { struct fs_info seek; strncpy((char *) &seek.uuid, uuid, 16); seek.dev_t = 0; seek.last_mount_size = 0; device = blk_lookup_fs_info(&seek); if (!device) { device = default_device; printk(KERN_DEBUG "Unable to resolve uuid. Falling back" " to dev_t.\n"); } else printk(KERN_DEBUG "Resolved uuid to device %s.\n", format_dev_t(buf, device)); } if (!device) { printk(KERN_ERR "TuxOnIce attempting to open a " "blank dev_t!\n"); dump_stack(); return NULL; } bdev = toi_open_by_devnum(device); if (IS_ERR(bdev) || !bdev) { if (!retried) { retried = 1; wait_for_device_probe(); goto retry; } if (display_errs) toi_early_boot_message(1, TOI_CONTINUE_REQ, "Failed to get access to block device " "\"%x\" (error %d).\n Maybe you need " "to run mknod and/or lvmsetup in an " "initrd/ramfs?", device, bdev); return ERR_PTR(-EINVAL); } toi_message(TOI_BIO, TOI_VERBOSE, 0, "TuxOnIce got bdev %p for dev_t %x.", bdev, device); return bdev; } static void toi_bio_reserve_header_space(unsigned long request) { header_pages_reserved = request; } /** * do_bio_wait - wait for some TuxOnIce I/O to complete * @reason: The array index of the reason we're waiting. * * Wait for a particular page of I/O if we're after a particular page. * If we're not after a particular page, wait instead for all in flight * I/O to be completed or for us to have enough free memory to be able * to submit more I/O. * * If we wait, we also update our statistics regarding why we waited. **/ static void do_bio_wait(int reason) { struct page *was_waiting_on = waiting_on; /* On SMP, waiting_on can be reset, so we make a copy */ if (was_waiting_on) { wait_on_page_locked(was_waiting_on); atomic_inc(&reasons[reason]); } else { atomic_inc(&reasons[reason]); wait_event(num_in_progress_wait, !atomic_read(&toi_io_in_progress) || nr_free_buffer_pages() > free_mem_throttle); } } /** * throttle_if_needed - wait for I/O completion if throttle points are reached * @flags: What to check and how to act. * * Check whether we need to wait for some I/O to complete. We always check * whether we have enough memory available, but may also (depending upon * @reason) check if the throughput throttle limit has been reached. **/ static int throttle_if_needed(int flags) { int free_pages = nr_free_buffer_pages(); /* Getting low on memory and I/O is in progress? */ while (unlikely(free_pages < free_mem_throttle) && atomic_read(&toi_io_in_progress) && !test_result_state(TOI_ABORTED)) { if (!(flags & THROTTLE_WAIT)) return -ENOMEM; do_bio_wait(4); free_pages = nr_free_buffer_pages(); } while (!(flags & MEMORY_ONLY) && throughput_throttle && TOTAL_OUTSTANDING_IO >= throughput_throttle && !test_result_state(TOI_ABORTED)) { int result = toi_bio_queue_flush_pages(0); if (result) return result; atomic_inc(&reasons[6]); wait_event(num_in_progress_wait, !atomic_read(&toi_io_in_progress) || TOTAL_OUTSTANDING_IO < throughput_throttle); } return 0; } /** * update_throughput_throttle - update the raw throughput throttle * @jif_index: The number of times this function has been called. * * This function is called four times per second by the core, and used to limit * the amount of I/O we submit at once, spreading out our waiting through the * whole job and letting userui get an opportunity to do its work. * * We don't start limiting I/O until 1/4s has gone so that we get a * decent sample for our initial limit, and keep updating it because * throughput may vary (on rotating media, eg) with our block number. * * We throttle to 1/10s worth of I/O. **/ static void update_throughput_throttle(int jif_index) { int done = atomic_read(&toi_io_done); throughput_throttle = done * 2 / 5 / jif_index; } /** * toi_finish_all_io - wait for all outstanding i/o to complete * * Flush any queued but unsubmitted I/O and wait for it all to complete. **/ static int toi_finish_all_io(void) { int result = toi_bio_queue_flush_pages(0); toi_bio_queue_flusher_should_finish = 1; wake_up(&toi_io_queue_flusher); wait_event(num_in_progress_wait, !TOTAL_OUTSTANDING_IO); return result; } /** * toi_end_bio - bio completion function. * @bio: bio that has completed. * * Function called by the block driver from interrupt context when I/O is * completed. If we were writing the page, we want to free it and will have * set bio->bi_private to the parameter we should use in telling the page * allocation accounting code what the page was allocated for. If we're * reading the page, it will be in the singly linked list made from * page->private pointers. **/ static void toi_end_bio(struct bio *bio) { struct page *page = bio->bi_io_vec[0].bv_page; BUG_ON(bio->bi_error); unlock_page(page); bio_put(bio); if (waiting_on == page) waiting_on = NULL; put_page(page); if (bio->bi_private) toi__free_page((int) ((unsigned long) bio->bi_private) , page); bio_put(bio); atomic_dec(&toi_io_in_progress); atomic_inc(&toi_io_done); wake_up(&num_in_progress_wait); } /** * submit - submit BIO request * @writing: READ or WRITE. * @dev: The block device we're using. * @first_block: The first sector we're using. * @page: The page being used for I/O. * @free_group: If writing, the group that was used in allocating the page * and which will be used in freeing the page from the completion * routine. * * Based on Patrick Mochell's pmdisk code from long ago: "Straight from the * textbook - allocate and initialize the bio. If we're writing, make sure * the page is marked as dirty. Then submit it and carry on." * * If we're just testing the speed of our own code, we fake having done all * the hard work and all toi_end_bio immediately. **/ static int submit(int writing, struct block_device *dev, sector_t first_block, struct page *page, int free_group) { struct bio *bio = NULL; int cur_outstanding_io, result; /* * Shouldn't throttle if reading - can deadlock in the single * threaded case as pages are only freed when we use the * readahead. */ if (writing) { result = throttle_if_needed(MEMORY_ONLY | THROTTLE_WAIT); if (result) return result; } while (!bio) { bio = bio_alloc(TOI_ATOMIC_GFP, 1); if (!bio) { set_free_mem_throttle(); do_bio_wait(1); } } bio->bi_bdev = dev; bio->bi_iter.bi_sector = first_block; bio->bi_private = (void *) ((unsigned long) free_group); bio->bi_end_io = toi_end_bio; bio_set_flag(bio, BIO_TOI); if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) { printk(KERN_DEBUG "ERROR: adding page to bio at %lld\n", (unsigned long long) first_block); bio_put(bio); return -EFAULT; } bio_get(bio); cur_outstanding_io = atomic_add_return(1, &toi_io_in_progress); if (writing) { if (cur_outstanding_io > max_outstanding_writes) max_outstanding_writes = cur_outstanding_io; } else { if (cur_outstanding_io > max_outstanding_reads) max_outstanding_reads = cur_outstanding_io; } /* Still read the header! */ if (unlikely(test_action_state(TOI_TEST_BIO) && writing)) { /* Fake having done the hard work */ bio->bi_error = 0; toi_end_bio(bio); } else submit_bio(writing | REQ_SYNC, bio); return 0; } /** * toi_do_io: Prepare to do some i/o on a page and submit or batch it. * * @writing: Whether reading or writing. * @bdev: The block device which we're using. * @block0: The first sector we're reading or writing. * @page: The page on which I/O is being done. * @readahead_index: If doing readahead, the index (reset this flag when done). * @syncio: Whether the i/o is being done synchronously. * * Prepare and start a read or write operation. * * Note that we always work with our own page. If writing, we might be given a * compression buffer that will immediately be used to start compressing the * next page. For reading, we do readahead and therefore don't know the final * address where the data needs to go. **/ int toi_do_io(int writing, struct block_device *bdev, long block0, struct page *page, int is_readahead, int syncio, int free_group) { page->private = 0; /* Do here so we don't race against toi_bio_get_next_page_read */ lock_page(page); if (is_readahead) { if (readahead_list_head) readahead_list_tail->private = (unsigned long) page; else readahead_list_head = page; readahead_list_tail = page; } /* Done before submitting to avoid races. */ if (syncio) waiting_on = page; /* Submit the page */ get_page(page); if (submit(writing, bdev, block0, page, free_group)) return -EFAULT; if (syncio) do_bio_wait(2); return 0; } /** * toi_bdev_page_io - simpler interface to do directly i/o on a single page * @writing: Whether reading or writing. * @bdev: Block device on which we're operating. * @pos: Sector at which page to read or write starts. * @page: Page to be read/written. * * A simple interface to submit a page of I/O and wait for its completion. * The caller must free the page used. **/ static int toi_bdev_page_io(int writing, struct block_device *bdev, long pos, struct page *page) { return toi_do_io(writing, bdev, pos, page, 0, 1, 0); } /** * toi_bio_memory_needed - report the amount of memory needed for block i/o * * We want to have at least enough memory so as to have target_outstanding_io * or more transactions on the fly at once. If we can do more, fine. **/ static int toi_bio_memory_needed(void) { return target_outstanding_io * (PAGE_SIZE + sizeof(struct request) + sizeof(struct bio)); } /** * toi_bio_print_debug_stats - put out debugging info in the buffer provided * @buffer: A buffer of size @size into which text should be placed. * @size: The size of @buffer. * * Fill a buffer with debugging info. This is used for both our debug_info sysfs * entry and for recording the same info in dmesg. **/ static int toi_bio_print_debug_stats(char *buffer, int size) { int len = 0; if (toiActiveAllocator != &toi_blockwriter_ops) { len = scnprintf(buffer, size, "- Block I/O inactive.\n"); return len; } len = scnprintf(buffer, size, "- Block I/O active.\n"); len += toi_bio_chains_debug_info(buffer + len, size - len); len += scnprintf(buffer + len, size - len, "- Max outstanding reads %d. Max writes %d.\n", max_outstanding_reads, max_outstanding_writes); len += scnprintf(buffer + len, size - len, " Memory_needed: %d x (%lu + %u + %u) = %d bytes.\n", target_outstanding_io, PAGE_SIZE, (unsigned int) sizeof(struct request), (unsigned int) sizeof(struct bio), toi_bio_memory_needed()); #ifdef MEASURE_MUTEX_CONTENTION { int i; len += scnprintf(buffer + len, size - len, " Mutex contention while reading:\n Contended Free\n"); for_each_online_cpu(i) len += scnprintf(buffer + len, size - len, " %9lu %9lu\n", mutex_times[0][0][i], mutex_times[0][1][i]); len += scnprintf(buffer + len, size - len, " Mutex contention while writing:\n Contended Free\n"); for_each_online_cpu(i) len += scnprintf(buffer + len, size - len, " %9lu %9lu\n", mutex_times[1][0][i], mutex_times[1][1][i]); } #endif return len + scnprintf(buffer + len, size - len, " Free mem throttle point reached %d.\n", free_mem_throttle); } static int total_header_bytes; static int unowned; void debug_broken_header(void) { printk(KERN_DEBUG "Image header too big for size allocated!\n"); print_toi_header_storage_for_modules(); printk(KERN_DEBUG "Page flags : %d.\n", toi_pageflags_space_needed()); printk(KERN_DEBUG "toi_header : %zu.\n", sizeof(struct toi_header)); printk(KERN_DEBUG "Total unowned : %d.\n", unowned); printk(KERN_DEBUG "Total used : %d (%ld pages).\n", total_header_bytes, DIV_ROUND_UP(total_header_bytes, PAGE_SIZE)); printk(KERN_DEBUG "Space needed now : %ld.\n", get_header_storage_needed(0)); dump_block_chains(); abort_hibernate(TOI_HEADER_TOO_BIG, "Header reservation too small."); } static int toi_bio_update_previous_inc_img_ptr(int stream) { int result; char * buffer = (char *) toi_get_zeroed_page(12, TOI_ATOMIC_GFP); struct page *page; struct toi_incremental_image_pointer *prev, *this; prev = &toi_inc_ptr[stream][0]; this = &toi_inc_ptr[stream][1]; if (!buffer) { // We're at the start of writing a pageset. Memory should not be that scarce. return -ENOMEM; } page = virt_to_page(buffer); result = toi_do_io(READ, prev->bdev, prev->block, page, 0, 1, 0); if (result) goto out; memcpy(buffer, (char *) this, sizeof(this->save)); result = toi_do_io(WRITE, prev->bdev, prev->block, page, 0, 0, 12); // If the IO is successfully submitted (!result), the page will be freed // asynchronously on completion. out: if (result) toi__free_page(12, virt_to_page(buffer)); return result; } /** * toi_rw_init_incremental - incremental image part of setting up to write new section */ static int toi_write_init_incremental(int stream) { int result = 0; // Remember the location of this block so we can link to it. toi_bio_store_inc_image_ptr(&toi_inc_ptr[stream][1]); // Update the pointer at the start of the last pageset with the same stream number. result = toi_bio_update_previous_inc_img_ptr(stream); if (result) return result; // Move the current to the previous slot. memcpy(&toi_inc_ptr[stream][0], &toi_inc_ptr[stream][1], sizeof(toi_inc_ptr[stream][1])); // Store a blank pointer at the start of this incremental pageset memset(&toi_inc_ptr[stream][1], 0, sizeof(toi_inc_ptr[stream][1])); result = toi_rw_buffer(WRITE, (char *) &toi_inc_ptr[stream][1], sizeof(toi_inc_ptr[stream][1]), 0); if (result) return result; // Serialise extent chains if this is an incremental pageset return toi_serialise_extent_chains(); } /** * toi_read_init_incremental - incremental image part of setting up to read new section */ static int toi_read_init_incremental(int stream) { int result; // Set our position to the start of the next pageset toi_bio_restore_inc_image_ptr(&toi_inc_ptr[stream][1]); // Read the start of the next incremental pageset (if any) result = toi_rw_buffer(READ, (char *) &toi_inc_ptr[stream][1], sizeof(toi_inc_ptr[stream][1]), 0); if (!result) result = toi_load_extent_chains(); return result; } /** * toi_rw_init - prepare to read or write a stream in the image * @writing: Whether reading or writing. * @stream number: Section of the image being processed. * * Prepare to read or write a section ('stream') in the image. **/ static int toi_rw_init(int writing, int stream_number) { if (stream_number) toi_extent_state_restore(stream_number); else toi_extent_state_goto_start(); if (writing) { reset_idx = 0; if (!current_stream) page_idx = 0; } else { reset_idx = 1; } atomic_set(&toi_io_done, 0); if (!toi_writer_buffer) toi_writer_buffer = (char *) toi_get_zeroed_page(11, TOI_ATOMIC_GFP); toi_writer_buffer_posn = writing ? 0 : PAGE_SIZE; current_stream = stream_number; more_readahead = 1; if (test_result_state(TOI_KEPT_IMAGE)) { int result; if (writing) { result = toi_write_init_incremental(stream_number); } else { result = toi_read_init_incremental(stream_number); } if (result) return result; } return toi_writer_buffer ? 0 : -ENOMEM; } /** * toi_bio_queue_write - queue a page for writing * @full_buffer: Pointer to a page to be queued * * Add a page to the queue to be submitted. If we're the queue flusher, * we'll do this once we've dropped toi_bio_mutex, so other threads can * continue to submit I/O while we're on the slow path doing the actual * submission. **/ static void toi_bio_queue_write(char **full_buffer) { struct page *page = virt_to_page(*full_buffer); unsigned long flags; *full_buffer = NULL; page->private = 0; spin_lock_irqsave(&bio_queue_lock, flags); if (!bio_queue_head) bio_queue_head = page; else bio_queue_tail->private = (unsigned long) page; bio_queue_tail = page; atomic_inc(&toi_bio_queue_size); spin_unlock_irqrestore(&bio_queue_lock, flags); wake_up(&toi_io_queue_flusher); } /** * toi_rw_cleanup - Cleanup after i/o. * @writing: Whether we were reading or writing. * * Flush all I/O and clean everything up after reading or writing a * section of the image. **/ static int toi_rw_cleanup(int writing) { int i, result = 0; toi_message(TOI_BIO, TOI_VERBOSE, 0, "toi_rw_cleanup."); if (writing) { if (toi_writer_buffer_posn && !test_result_state(TOI_ABORTED)) toi_bio_queue_write(&toi_writer_buffer); while (bio_queue_head && !result) result = toi_bio_queue_flush_pages(0); if (result) return result; if (current_stream == 2) toi_extent_state_save(1); else if (current_stream == 1) toi_extent_state_save(3); } result = toi_finish_all_io(); while (readahead_list_head) { void *next = (void *) readahead_list_head->private; toi__free_page(12, readahead_list_head); readahead_list_head = next; } readahead_list_tail = NULL; if (!current_stream) return result; for (i = 0; i < NUM_REASONS; i++) { if (!atomic_read(&reasons[i])) continue; printk(KERN_DEBUG "Waited for i/o due to %s %d times.\n", reason_name[i], atomic_read(&reasons[i])); atomic_set(&reasons[i], 0); } current_stream = 0; return result; } /** * toi_start_one_readahead - start one page of readahead * @dedicated_thread: Is this a thread dedicated to doing readahead? * * Start one new page of readahead. If this is being called by a thread * whose only just is to submit readahead, don't quit because we failed * to allocate a page. **/ static int toi_start_one_readahead(int dedicated_thread) { char *buffer = NULL; int oom = 0, result; result = throttle_if_needed(dedicated_thread ? THROTTLE_WAIT : 0); if (result) { printk("toi_start_one_readahead: throttle_if_needed returned %d.\n", result); return result; } mutex_lock(&toi_bio_readahead_mutex); while (!buffer) { buffer = (char *) toi_get_zeroed_page(12, TOI_ATOMIC_GFP); if (!buffer) { if (oom && !dedicated_thread) { mutex_unlock(&toi_bio_readahead_mutex); printk("toi_start_one_readahead: oom and !dedicated thread %d.\n", result); return -ENOMEM; } oom = 1; set_free_mem_throttle(); do_bio_wait(5); } } result = toi_bio_rw_page(READ, virt_to_page(buffer), 1, 0); if (result) { printk("toi_start_one_readahead: toi_bio_rw_page returned %d.\n", result); } if (result == -ENOSPC) toi__free_page(12, virt_to_page(buffer)); mutex_unlock(&toi_bio_readahead_mutex); if (result) { if (result == -ENOSPC) toi_message(TOI_BIO, TOI_VERBOSE, 0, "Last readahead page submitted."); else printk(KERN_DEBUG "toi_bio_rw_page returned %d.\n", result); } return result; } /** * toi_start_new_readahead - start new readahead * @dedicated_thread: Are we dedicated to this task? * * Start readahead of image pages. * * We can be called as a thread dedicated to this task (may be helpful on * systems with lots of CPUs), in which case we don't exit until there's no * more readahead. * * If this is not called by a dedicated thread, we top up our queue until * there's no more readahead to submit, we've submitted the number given * in target_outstanding_io or the number in progress exceeds the target * outstanding I/O value. * * No mutex needed because this is only ever called by the first cpu. **/ static int toi_start_new_readahead(int dedicated_thread) { int last_result, num_submitted = 0; /* Start a new readahead? */ if (!more_readahead) return 0; do { last_result = toi_start_one_readahead(dedicated_thread); if (last_result) { if (last_result == -ENOMEM || last_result == -ENOSPC) return 0; printk(KERN_DEBUG "Begin read chunk returned %d.\n", last_result); } else num_submitted++; } while (more_readahead && !last_result && (dedicated_thread || (num_submitted < target_outstanding_io && atomic_read(&toi_io_in_progress) < target_outstanding_io))); return last_result; } /** * bio_io_flusher - start the dedicated I/O flushing routine * @writing: Whether we're writing the image. **/ static int bio_io_flusher(int writing) { if (writing) return toi_bio_queue_flush_pages(1); else return toi_start_new_readahead(1); } /** * toi_bio_get_next_page_read - read a disk page, perhaps with readahead * @no_readahead: Whether we can use readahead * * Read a page from disk, submitting readahead and cleaning up finished i/o * while we wait for the page we're after. **/ static int toi_bio_get_next_page_read(int no_readahead) { char *virt; struct page *old_readahead_list_head; /* * When reading the second page of the header, we have to * delay submitting the read until after we've gotten the * extents out of the first page. */ if (unlikely(no_readahead)) { int result = toi_start_one_readahead(0); if (result) { printk(KERN_EMERG "No readahead and toi_start_one_readahead " "returned non-zero.\n"); return -EIO; } } if (unlikely(!readahead_list_head)) { /* * If the last page finishes exactly on the page * boundary, we will be called one extra time and * have no data to return. In this case, we should * not BUG(), like we used to! */ if (!more_readahead) { printk(KERN_EMERG "No more readahead.\n"); return -ENOSPC; } if (unlikely(toi_start_one_readahead(0))) { printk(KERN_EMERG "No readahead and " "toi_start_one_readahead returned non-zero.\n"); return -EIO; } } if (PageLocked(readahead_list_head)) { waiting_on = readahead_list_head; do_bio_wait(0); } virt = page_address(readahead_list_head); memcpy(toi_writer_buffer, virt, PAGE_SIZE); mutex_lock(&toi_bio_readahead_mutex); old_readahead_list_head = readahead_list_head; readahead_list_head = (struct page *) readahead_list_head->private; mutex_unlock(&toi_bio_readahead_mutex); toi__free_page(12, old_readahead_list_head); return 0; } /** * toi_bio_queue_flush_pages - flush the queue of pages queued for writing * @dedicated_thread: Whether we're a dedicated thread * * Flush the queue of pages ready to be written to disk. * * If we're a dedicated thread, stay in here until told to leave, * sleeping in wait_event. * * The first thread is normally the only one to come in here. Another * thread can enter this routine too, though, via throttle_if_needed. * Since that's the case, we must be careful to only have one thread * doing this work at a time. Otherwise we have a race and could save * pages out of order. * * If an error occurs, free all remaining pages without submitting them * for I/O. **/ int toi_bio_queue_flush_pages(int dedicated_thread) { unsigned long flags; int result = 0; static DEFINE_MUTEX(busy); if (!mutex_trylock(&busy)) return 0; top: spin_lock_irqsave(&bio_queue_lock, flags); while (bio_queue_head) { struct page *page = bio_queue_head; bio_queue_head = (struct page *) page->private; if (bio_queue_tail == page) bio_queue_tail = NULL; atomic_dec(&toi_bio_queue_size); spin_unlock_irqrestore(&bio_queue_lock, flags); /* Don't generate more error messages if already had one */ if (!result) result = toi_bio_rw_page(WRITE, page, 0, 11); /* * If writing the page failed, don't drop out. * Flush the rest of the queue too. */ if (result) toi__free_page(11 , page); spin_lock_irqsave(&bio_queue_lock, flags); } spin_unlock_irqrestore(&bio_queue_lock, flags); if (dedicated_thread) { wait_event(toi_io_queue_flusher, bio_queue_head || toi_bio_queue_flusher_should_finish); if (likely(!toi_bio_queue_flusher_should_finish)) goto top; toi_bio_queue_flusher_should_finish = 0; } mutex_unlock(&busy); return result; } /** * toi_bio_get_new_page - get a new page for I/O * @full_buffer: Pointer to a page to allocate. **/ static int toi_bio_get_new_page(char **full_buffer) { int result = throttle_if_needed(THROTTLE_WAIT); if (result) return result; while (!*full_buffer) { *full_buffer = (char *) toi_get_zeroed_page(11, TOI_ATOMIC_GFP); if (!*full_buffer) { set_free_mem_throttle(); do_bio_wait(3); } } return 0; } /** * toi_rw_buffer - combine smaller buffers into PAGE_SIZE I/O * @writing: Bool - whether writing (or reading). * @buffer: The start of the buffer to write or fill. * @buffer_size: The size of the buffer to write or fill. * @no_readahead: Don't try to start readhead (when getting extents). **/ static int toi_rw_buffer(int writing, char *buffer, int buffer_size, int no_readahead) { int bytes_left = buffer_size, result = 0; while (bytes_left) { char *source_start = buffer + buffer_size - bytes_left; char *dest_start = toi_writer_buffer + toi_writer_buffer_posn; int capacity = PAGE_SIZE - toi_writer_buffer_posn; char *to = writing ? dest_start : source_start; char *from = writing ? source_start : dest_start; if (bytes_left <= capacity) { memcpy(to, from, bytes_left); toi_writer_buffer_posn += bytes_left; return 0; } /* Complete this page and start a new one */ memcpy(to, from, capacity); bytes_left -= capacity; if (!writing) { /* * Perform actual I/O: * read readahead_list_head into toi_writer_buffer */ int result = toi_bio_get_next_page_read(no_readahead); if (result && bytes_left) { printk("toi_bio_get_next_page_read " "returned %d. Expecting to read %d bytes.\n", result, bytes_left); return result; } } else { toi_bio_queue_write(&toi_writer_buffer); result = toi_bio_get_new_page(&toi_writer_buffer); if (result) { printk(KERN_ERR "toi_bio_get_new_page returned " "%d.\n", result); return result; } } toi_writer_buffer_posn = 0; toi_cond_pause(0, NULL); } return 0; } /** * toi_bio_read_page - read a page of the image * @pfn: The pfn where the data belongs. * @buffer_page: The page containing the (possibly compressed) data. * @buf_size: The number of bytes on @buffer_page used (PAGE_SIZE). * * Read a (possibly compressed) page from the image, into buffer_page, * returning its pfn and the buffer size. **/ static int toi_bio_read_page(unsigned long *pfn, int buf_type, void *buffer_page, unsigned int *buf_size) { int result = 0; int this_idx; char *buffer_virt = TOI_MAP(buf_type, buffer_page); /* * Only call start_new_readahead if we don't have a dedicated thread * and we're the queue flusher. */ if (current == toi_queue_flusher && more_readahead && !test_action_state(TOI_NO_READAHEAD)) { int result2 = toi_start_new_readahead(0); if (result2) { printk(KERN_DEBUG "Queue flusher and " "toi_start_one_readahead returned non-zero.\n"); result = -EIO; goto out; } } my_mutex_lock(0, &toi_bio_mutex); /* * Structure in the image: * [destination pfn|page size|page data] * buf_size is PAGE_SIZE * We can validly find there's nothing to read in a multithreaded * situation. */ if (toi_rw_buffer(READ, (char *) &this_idx, sizeof(int), 0) || toi_rw_buffer(READ, (char *) pfn, sizeof(unsigned long), 0) || toi_rw_buffer(READ, (char *) buf_size, sizeof(int), 0) || toi_rw_buffer(READ, buffer_virt, *buf_size, 0)) { result = -ENODATA; goto out_unlock; } if (reset_idx) { page_idx = this_idx; reset_idx = 0; } else { page_idx++; if (!this_idx) result = -ENODATA; else if (page_idx != this_idx) printk(KERN_ERR "Got page index %d, expected %d.\n", this_idx, page_idx); } out_unlock: my_mutex_unlock(0, &toi_bio_mutex); out: TOI_UNMAP(buf_type, buffer_page); return result; } /** * toi_bio_write_page - write a page of the image * @pfn: The pfn where the data belongs. * @buffer_page: The page containing the (possibly compressed) data. * @buf_size: The number of bytes on @buffer_page used. * * Write a (possibly compressed) page to the image from the buffer, together * with it's index and buffer size. **/ static int toi_bio_write_page(unsigned long pfn, int buf_type, void *buffer_page, unsigned int buf_size) { char *buffer_virt; int result = 0, result2 = 0; if (unlikely(test_action_state(TOI_TEST_FILTER_SPEED))) return 0; my_mutex_lock(1, &toi_bio_mutex); if (test_result_state(TOI_ABORTED)) { my_mutex_unlock(1, &toi_bio_mutex); return 0; } buffer_virt = TOI_MAP(buf_type, buffer_page); page_idx++; /* * Structure in the image: * [destination pfn|page size|page data] * buf_size is PAGE_SIZE */ if (toi_rw_buffer(WRITE, (char *) &page_idx, sizeof(int), 0) || toi_rw_buffer(WRITE, (char *) &pfn, sizeof(unsigned long), 0) || toi_rw_buffer(WRITE, (char *) &buf_size, sizeof(int), 0) || toi_rw_buffer(WRITE, buffer_virt, buf_size, 0)) { printk(KERN_DEBUG "toi_rw_buffer returned non-zero to " "toi_bio_write_page.\n"); result = -EIO; } TOI_UNMAP(buf_type, buffer_page); my_mutex_unlock(1, &toi_bio_mutex); if (current == toi_queue_flusher) result2 = toi_bio_queue_flush_pages(0); return result ? result : result2; } /** * _toi_rw_header_chunk - read or write a portion of the image header * @writing: Whether reading or writing. * @owner: The module for which we're writing. * Used for confirming that modules * don't use more header space than they asked for. * @buffer: Address of the data to write. * @buffer_size: Size of the data buffer. * @no_readahead: Don't try to start readhead (when getting extents). * * Perform PAGE_SIZE I/O. Start readahead if needed. **/ static int _toi_rw_header_chunk(int writing, struct toi_module_ops *owner, char *buffer, int buffer_size, int no_readahead) { int result = 0; if (owner) { owner->header_used += buffer_size; toi_message(TOI_HEADER, TOI_LOW, 1, "Header: %s : %d bytes (%d/%d) from offset %d.", owner->name, buffer_size, owner->header_used, owner->header_requested, toi_writer_buffer_posn); if (owner->header_used > owner->header_requested && writing) { printk(KERN_EMERG "TuxOnIce module %s is using more " "header space (%u) than it requested (%u).\n", owner->name, owner->header_used, owner->header_requested); return buffer_size; } } else { unowned += buffer_size; toi_message(TOI_HEADER, TOI_LOW, 1, "Header: (No owner): %d bytes (%d total so far) from " "offset %d.", buffer_size, unowned, toi_writer_buffer_posn); } if (!writing && !no_readahead && more_readahead) { result = toi_start_new_readahead(0); toi_message(TOI_BIO, TOI_VERBOSE, 0, "Start new readahead " "returned %d.", result); } if (!result) { result = toi_rw_buffer(writing, buffer, buffer_size, no_readahead); toi_message(TOI_BIO, TOI_VERBOSE, 0, "rw_buffer returned " "%d.", result); } total_header_bytes += buffer_size; toi_message(TOI_BIO, TOI_VERBOSE, 0, "_toi_rw_header_chunk returning " "%d.", result); return result; } static int toi_rw_header_chunk(int writing, struct toi_module_ops *owner, char *buffer, int size) { return _toi_rw_header_chunk(writing, owner, buffer, size, 1); } static int toi_rw_header_chunk_noreadahead(int writing, struct toi_module_ops *owner, char *buffer, int size) { return _toi_rw_header_chunk(writing, owner, buffer, size, 1); } /** * toi_bio_storage_needed - get the amount of storage needed for my fns **/ static int toi_bio_storage_needed(void) { return sizeof(int) + PAGE_SIZE + toi_bio_devinfo_storage_needed(); } /** * toi_bio_save_config_info - save block I/O config to image header * @buf: PAGE_SIZE'd buffer into which data should be saved. **/ static int toi_bio_save_config_info(char *buf) { int *ints = (int *) buf; ints[0] = target_outstanding_io; return sizeof(int); } /** * toi_bio_load_config_info - restore block I/O config * @buf: Data to be reloaded. * @size: Size of the buffer saved. **/ static void toi_bio_load_config_info(char *buf, int size) { int *ints = (int *) buf; target_outstanding_io = ints[0]; } void close_resume_dev_t(int force) { if (!resume_block_device) return; if (force) atomic_set(&resume_bdev_open_count, 0); else atomic_dec(&resume_bdev_open_count); if (!atomic_read(&resume_bdev_open_count)) { toi_close_bdev(resume_block_device); resume_block_device = NULL; } } int open_resume_dev_t(int force, int quiet) { if (force) { close_resume_dev_t(1); atomic_set(&resume_bdev_open_count, 1); } else atomic_inc(&resume_bdev_open_count); if (resume_block_device) return 0; resume_block_device = toi_open_bdev(NULL, resume_dev_t, 0); if (IS_ERR(resume_block_device)) { if (!quiet) toi_early_boot_message(1, TOI_CONTINUE_REQ, "Failed to open device %x, where" " the header should be found.", resume_dev_t); resume_block_device = NULL; atomic_set(&resume_bdev_open_count, 0); return 1; } return 0; } /** * toi_bio_initialise - initialise bio code at start of some action * @starting_cycle: Whether starting a hibernation cycle, or just reading or * writing a sysfs value. **/ static int toi_bio_initialise(int starting_cycle) { int result; if (!starting_cycle || !resume_dev_t) return 0; max_outstanding_writes = 0; max_outstanding_reads = 0; current_stream = 0; toi_queue_flusher = current; #ifdef MEASURE_MUTEX_CONTENTION { int i, j, k; for (i = 0; i < 2; i++) for (j = 0; j < 2; j++) for_each_online_cpu(k) mutex_times[i][j][k] = 0; } #endif result = open_resume_dev_t(0, 1); if (result) return result; result = toi_bio_register_storage(); if (result) return result; return get_signature_page(); } static unsigned long raw_to_real(unsigned long raw) { unsigned long extra; extra = (raw * (sizeof(unsigned long) + sizeof(int)) + (PAGE_SIZE + sizeof(unsigned long) + sizeof(int) + 1)) / (PAGE_SIZE + sizeof(unsigned long) + sizeof(int)); return raw > extra ? raw - extra : 0; } static unsigned long toi_bio_storage_available(void) { unsigned long sum = 0; struct toi_module_ops *this_module; list_for_each_entry(this_module, &toi_modules, module_list) { if (!this_module->enabled || this_module->type != BIO_ALLOCATOR_MODULE) continue; toi_message(TOI_BIO, TOI_VERBOSE, 0, "Seeking storage " "available from %s.", this_module->name); sum += this_module->bio_allocator_ops->storage_available(); } toi_message(TOI_BIO, TOI_VERBOSE, 0, "Total storage available is %lu " "pages (%d header pages).", sum, header_pages_reserved); return sum > header_pages_reserved ? raw_to_real(sum - header_pages_reserved) : 0; } static unsigned long toi_bio_storage_allocated(void) { return raw_pages_allocd > header_pages_reserved ? raw_to_real(raw_pages_allocd - header_pages_reserved) : 0; } /* * If we have read part of the image, we might have filled memory with * data that should be zeroed out. */ static void toi_bio_noresume_reset(void) { toi_message(TOI_BIO, TOI_VERBOSE, 0, "toi_bio_noresume_reset."); toi_rw_cleanup(READ); free_all_bdev_info(); } /** * toi_bio_cleanup - cleanup after some action * @finishing_cycle: Whether completing a cycle. **/ static void toi_bio_cleanup(int finishing_cycle) { if (!finishing_cycle) return; if (toi_writer_buffer) { toi_free_page(11, (unsigned long) toi_writer_buffer); toi_writer_buffer = NULL; } forget_signature_page(); if (header_block_device && toi_sig_data && toi_sig_data->header_dev_t != resume_dev_t) toi_close_bdev(header_block_device); header_block_device = NULL; close_resume_dev_t(0); } static int toi_bio_write_header_init(void) { int result; toi_message(TOI_BIO, TOI_VERBOSE, 0, "toi_bio_write_header_init"); toi_rw_init(WRITE, 0); toi_writer_buffer_posn = 0; /* Info needed to bootstrap goes at the start of the header. * First we save the positions and devinfo, including the number * of header pages. Then we save the structs containing data needed * for reading the header pages back. * Note that even if header pages take more than one page, when we * read back the info, we will have restored the location of the * next header page by the time we go to use it. */ toi_message(TOI_BIO, TOI_VERBOSE, 0, "serialise extent chains."); result = toi_serialise_extent_chains(); if (result) return result; /* * Signature page hasn't been modified at this point. Write it in * the header so we can restore it later. */ toi_message(TOI_BIO, TOI_VERBOSE, 0, "serialise signature page."); return toi_rw_header_chunk_noreadahead(WRITE, &toi_blockwriter_ops, (char *) toi_cur_sig_page, PAGE_SIZE); } static int toi_bio_write_header_cleanup(void) { int result = 0; if (toi_writer_buffer_posn) toi_bio_queue_write(&toi_writer_buffer); result = toi_finish_all_io(); unowned = 0; total_header_bytes = 0; /* Set signature to save we have an image */ if (!result) result = toi_bio_mark_have_image(); return result; } /* * toi_bio_read_header_init() * * Description: * 1. Attempt to read the device specified with resume=. * 2. Check the contents of the swap header for our signature. * 3. Warn, ignore, reset and/or continue as appropriate. * 4. If continuing, read the toi_swap configuration section * of the header and set up block device info so we can read * the rest of the header & image. * * Returns: * May not return if user choose to reboot at a warning. * -EINVAL if cannot resume at this time. Booting should continue * normally. */ static int toi_bio_read_header_init(void) { int result = 0; char buf[32]; toi_writer_buffer_posn = 0; toi_message(TOI_BIO, TOI_VERBOSE, 0, "toi_bio_read_header_init"); if (!toi_sig_data) { printk(KERN_INFO "toi_bio_read_header_init called when we " "haven't verified there is an image!\n"); return -EINVAL; } /* * If the header is not on the resume_swap_dev_t, get the resume device * first. */ toi_message(TOI_BIO, TOI_VERBOSE, 0, "Header dev_t is %lx.", toi_sig_data->header_dev_t); if (toi_sig_data->have_uuid) { struct fs_info seek; dev_t device; strncpy((char *) seek.uuid, toi_sig_data->header_uuid, 16); seek.dev_t = toi_sig_data->header_dev_t; seek.last_mount_size = 0; device = blk_lookup_fs_info(&seek); if (device) { printk("Using dev_t %s, returned by blk_lookup_fs_info.\n", format_dev_t(buf, device)); toi_sig_data->header_dev_t = device; } } if (toi_sig_data->header_dev_t != resume_dev_t) { header_block_device = toi_open_bdev(NULL, toi_sig_data->header_dev_t, 1); if (IS_ERR(header_block_device)) return PTR_ERR(header_block_device); } else header_block_device = resume_block_device; if (!toi_writer_buffer) toi_writer_buffer = (char *) toi_get_zeroed_page(11, TOI_ATOMIC_GFP); more_readahead = 1; /* * Read toi_swap configuration. * Headerblock size taken into account already. */ result = toi_bio_ops.bdev_page_io(READ, header_block_device, toi_sig_data->first_header_block, virt_to_page((unsigned long) toi_writer_buffer)); if (result) return result; toi_message(TOI_BIO, TOI_VERBOSE, 0, "load extent chains."); result = toi_load_extent_chains(); toi_message(TOI_BIO, TOI_VERBOSE, 0, "load original signature page."); toi_orig_sig_page = (char *) toi_get_zeroed_page(38, TOI_ATOMIC_GFP); if (!toi_orig_sig_page) { printk(KERN_ERR "Failed to allocate memory for the current" " image signature.\n"); return -ENOMEM; } return toi_rw_header_chunk_noreadahead(READ, &toi_blockwriter_ops, (char *) toi_orig_sig_page, PAGE_SIZE); } static int toi_bio_read_header_cleanup(void) { toi_message(TOI_BIO, TOI_VERBOSE, 0, "toi_bio_read_header_cleanup."); return toi_rw_cleanup(READ); } /* Works only for digits and letters, but small and fast */ #define TOLOWER(x) ((x) | 0x20) /* * UUID must be 32 chars long. It may have dashes, but nothing * else. */ char *uuid_from_commandline(char *commandline) { int low = 0; char *result = NULL, *output, *ptr; if (strncmp(commandline, "UUID=", 5)) return NULL; result = kzalloc(17, GFP_KERNEL); if (!result) { printk("Failed to kzalloc UUID text memory.\n"); return NULL; } ptr = commandline + 5; output = result; while (*ptr && (output - result) < 16) { if (isxdigit(*ptr)) { int value = isdigit(*ptr) ? *ptr - '0' : TOLOWER(*ptr) - 'a' + 10; if (low) { *output += value; output++; } else { *output = value << 4; } low = !low; } else if (*ptr != '-') break; ptr++; } if ((output - result) < 16 || *ptr) { printk(KERN_DEBUG "Found resume=UUID=, but the value looks " "invalid.\n"); kfree(result); result = NULL; } return result; } #define retry_if_fails(command) \ do { \ command; \ if (!resume_dev_t && !waited_for_device_probe) { \ wait_for_device_probe(); \ command; \ waited_for_device_probe = 1; \ } \ } while(0) /** * try_to_open_resume_device: Try to parse and open resume= * * Any "swap:" has been stripped away and we just have the path to deal with. * We attempt to do name_to_dev_t, open and stat the file. Having opened the * file, get the struct block_device * to match. */ static int try_to_open_resume_device(char *commandline, int quiet) { struct kstat stat; int error = 0; char *uuid = uuid_from_commandline(commandline); int waited_for_device_probe = 0; resume_dev_t = MKDEV(0, 0); if (!strlen(commandline)) retry_if_fails(toi_bio_scan_for_image(quiet)); if (uuid) { struct fs_info seek; strncpy((char *) &seek.uuid, uuid, 16); seek.dev_t = resume_dev_t; seek.last_mount_size = 0; retry_if_fails(resume_dev_t = blk_lookup_fs_info(&seek)); kfree(uuid); } if (!resume_dev_t) retry_if_fails(resume_dev_t = name_to_dev_t(commandline)); if (!resume_dev_t) { struct file *file = filp_open(commandline, O_RDONLY|O_LARGEFILE, 0); if (!IS_ERR(file) && file) { vfs_getattr(&file->f_path, &stat); filp_close(file, NULL); } else error = vfs_stat(commandline, &stat); if (!error) resume_dev_t = stat.rdev; } if (!resume_dev_t) { if (quiet) return 1; if (test_toi_state(TOI_TRYING_TO_RESUME)) toi_early_boot_message(1, toi_translate_err_default, "Failed to translate \"%s\" into a device id.\n", commandline); else printk("TuxOnIce: Can't translate \"%s\" into a device " "id yet.\n", commandline); return 1; } return open_resume_dev_t(1, quiet); } /* * Parse Image Location * * Attempt to parse a resume= parameter. * Swap Writer accepts: * resume=[swap:|file:]DEVNAME[:FIRSTBLOCK][@BLOCKSIZE] * * Where: * DEVNAME is convertable to a dev_t by name_to_dev_t * FIRSTBLOCK is the location of the first block in the swap file * (specifying for a swap partition is nonsensical but not prohibited). * Data is validated by attempting to read a swap header from the * location given. Failure will result in toi_swap refusing to * save an image, and a reboot with correct parameters will be * necessary. */ static int toi_bio_parse_sig_location(char *commandline, int only_allocator, int quiet) { char *thischar, *devstart, *colon = NULL; int signature_found, result = -EINVAL, temp_result = 0; if (strncmp(commandline, "swap:", 5) && strncmp(commandline, "file:", 5)) { /* * Failing swap:, we'll take a simple resume=/dev/hda2, or a * blank value (scan) but fall through to other allocators * if /dev/ or UUID= isn't matched. */ if (strncmp(commandline, "/dev/", 5) && strncmp(commandline, "UUID=", 5) && strlen(commandline)) return 1; } else commandline += 5; devstart = commandline; thischar = commandline; while ((*thischar != ':') && (*thischar != '@') && ((thischar - commandline) < 250) && (*thischar)) thischar++; if (*thischar == ':') { colon = thischar; *colon = 0; thischar++; } while ((thischar - commandline) < 250 && *thischar) thischar++; if (colon) { unsigned long block; temp_result = kstrtoul(colon + 1, 0, &block); if (!temp_result) resume_firstblock = (int) block; } else resume_firstblock = 0; clear_toi_state(TOI_CAN_HIBERNATE); clear_toi_state(TOI_CAN_RESUME); if (!temp_result) temp_result = try_to_open_resume_device(devstart, quiet); if (colon) *colon = ':'; /* No error if we only scanned */ if (temp_result) return strlen(commandline) ? -EINVAL : 1; signature_found = toi_bio_image_exists(quiet); if (signature_found != -1) { result = 0; /* * TODO: If only file storage, CAN_HIBERNATE should only be * set if file allocator's target is valid. */ set_toi_state(TOI_CAN_HIBERNATE); set_toi_state(TOI_CAN_RESUME); } else if (!quiet) printk(KERN_ERR "TuxOnIce: Block I/O: No " "signature found at %s.\n", devstart); return result; } static void toi_bio_release_storage(void) { header_pages_reserved = 0; raw_pages_allocd = 0; free_all_bdev_info(); } /* toi_swap_remove_image * */ static int toi_bio_remove_image(void) { int result; toi_message(TOI_BIO, TOI_VERBOSE, 0, "toi_bio_remove_image."); result = toi_bio_restore_original_signature(); /* * We don't do a sanity check here: we want to restore the swap * whatever version of kernel made the hibernate image. * * We need to write swap, but swap may not be enabled so * we write the device directly * * If we don't have an current_signature_page, we didn't * read an image header, so don't change anything. */ toi_bio_release_storage(); return result; } struct toi_bio_ops toi_bio_ops = { .bdev_page_io = toi_bdev_page_io, .register_storage = toi_register_storage_chain, .free_storage = toi_bio_release_storage, }; static struct toi_sysfs_data sysfs_params[] = { SYSFS_INT("target_outstanding_io", SYSFS_RW, &target_outstanding_io, 0, 16384, 0, NULL), }; struct toi_module_ops toi_blockwriter_ops = { .type = WRITER_MODULE, .name = "block i/o", .directory = "block_io", .module = THIS_MODULE, .memory_needed = toi_bio_memory_needed, .print_debug_info = toi_bio_print_debug_stats, .storage_needed = toi_bio_storage_needed, .save_config_info = toi_bio_save_config_info, .load_config_info = toi_bio_load_config_info, .initialise = toi_bio_initialise, .cleanup = toi_bio_cleanup, .post_atomic_restore = toi_bio_chains_post_atomic, .rw_init = toi_rw_init, .rw_cleanup = toi_rw_cleanup, .read_page = toi_bio_read_page, .write_page = toi_bio_write_page, .rw_header_chunk = toi_rw_header_chunk, .rw_header_chunk_noreadahead = toi_rw_header_chunk_noreadahead, .io_flusher = bio_io_flusher, .update_throughput_throttle = update_throughput_throttle, .finish_all_io = toi_finish_all_io, .noresume_reset = toi_bio_noresume_reset, .storage_available = toi_bio_storage_available, .storage_allocated = toi_bio_storage_allocated, .reserve_header_space = toi_bio_reserve_header_space, .allocate_storage = toi_bio_allocate_storage, .free_unused_storage = toi_bio_free_unused_storage, .image_exists = toi_bio_image_exists, .mark_resume_attempted = toi_bio_mark_resume_attempted, .write_header_init = toi_bio_write_header_init, .write_header_cleanup = toi_bio_write_header_cleanup, .read_header_init = toi_bio_read_header_init, .read_header_cleanup = toi_bio_read_header_cleanup, .get_header_version = toi_bio_get_header_version, .remove_image = toi_bio_remove_image, .parse_sig_location = toi_bio_parse_sig_location, .sysfs_data = sysfs_params, .num_sysfs_entries = sizeof(sysfs_params) / sizeof(struct toi_sysfs_data), }; /** * toi_block_io_load - load time routine for block I/O module * * Register block i/o ops and sysfs entries. **/ static __init int toi_block_io_load(void) { return toi_register_module(&toi_blockwriter_ops); } late_initcall(toi_block_io_load);