From 57f0f512b273f60d52568b8c6b77e17f5636edc0 Mon Sep 17 00:00:00 2001
From: André Fabian Silva Delgado <emulatorman@parabola.nu>
Date: Wed, 5 Aug 2015 17:04:01 -0300
Subject: Initial import
---
drivers/crypto/nx/Kconfig | 26 +
drivers/crypto/nx/Makefile | 14 +
drivers/crypto/nx/nx-842.c | 1603 +++++++++++++++++++++++++++++++++++++++
drivers/crypto/nx/nx-aes-cbc.c | 150 ++++
drivers/crypto/nx/nx-aes-ccm.c | 604 +++++++++++++++
drivers/crypto/nx/nx-aes-ctr.c | 187 +++++
drivers/crypto/nx/nx-aes-ecb.c | 149 ++++
drivers/crypto/nx/nx-aes-gcm.c | 525 +++++++++++++
drivers/crypto/nx/nx-aes-xcbc.c | 378 +++++++++
drivers/crypto/nx/nx-sha256.c | 274 +++++++
drivers/crypto/nx/nx-sha512.c | 279 +++++++
drivers/crypto/nx/nx.c | 799 +++++++++++++++++++
drivers/crypto/nx/nx.h | 196 +++++
drivers/crypto/nx/nx_csbcpb.h | 205 +++++
drivers/crypto/nx/nx_debugfs.c | 103 +++
15 files changed, 5492 insertions(+)
create mode 100644 drivers/crypto/nx/Kconfig
create mode 100644 drivers/crypto/nx/Makefile
create mode 100644 drivers/crypto/nx/nx-842.c
create mode 100644 drivers/crypto/nx/nx-aes-cbc.c
create mode 100644 drivers/crypto/nx/nx-aes-ccm.c
create mode 100644 drivers/crypto/nx/nx-aes-ctr.c
create mode 100644 drivers/crypto/nx/nx-aes-ecb.c
create mode 100644 drivers/crypto/nx/nx-aes-gcm.c
create mode 100644 drivers/crypto/nx/nx-aes-xcbc.c
create mode 100644 drivers/crypto/nx/nx-sha256.c
create mode 100644 drivers/crypto/nx/nx-sha512.c
create mode 100644 drivers/crypto/nx/nx.c
create mode 100644 drivers/crypto/nx/nx.h
create mode 100644 drivers/crypto/nx/nx_csbcpb.h
create mode 100644 drivers/crypto/nx/nx_debugfs.c
(limited to 'drivers/crypto/nx')
diff --git a/drivers/crypto/nx/Kconfig b/drivers/crypto/nx/Kconfig
new file mode 100644
index 000000000..f82616621
--- /dev/null
+++ b/drivers/crypto/nx/Kconfig
@@ -0,0 +1,26 @@
+config CRYPTO_DEV_NX_ENCRYPT
+ tristate "Encryption acceleration support"
+ depends on PPC64 && IBMVIO
+ default y
+ select CRYPTO_AES
+ select CRYPTO_CBC
+ select CRYPTO_ECB
+ select CRYPTO_CCM
+ select CRYPTO_GCM
+ select CRYPTO_AUTHENC
+ select CRYPTO_XCBC
+ select CRYPTO_SHA256
+ select CRYPTO_SHA512
+ help
+ Support for Power7+ in-Nest encryption acceleration. This
+ module supports acceleration for AES and SHA2 algorithms. If you
+ choose 'M' here, this module will be called nx_crypto.
+
+config CRYPTO_DEV_NX_COMPRESS
+ tristate "Compression acceleration support"
+ depends on PPC64 && IBMVIO
+ default y
+ help
+ Support for Power7+ in-Nest compression acceleration. This
+ module supports acceleration for AES and SHA2 algorithms. If you
+ choose 'M' here, this module will be called nx_compress.
diff --git a/drivers/crypto/nx/Makefile b/drivers/crypto/nx/Makefile
new file mode 100644
index 000000000..bb770ea45
--- /dev/null
+++ b/drivers/crypto/nx/Makefile
@@ -0,0 +1,14 @@
+obj-$(CONFIG_CRYPTO_DEV_NX_ENCRYPT) += nx-crypto.o
+nx-crypto-objs := nx.o \
+ nx_debugfs.o \
+ nx-aes-cbc.o \
+ nx-aes-ecb.o \
+ nx-aes-gcm.o \
+ nx-aes-ccm.o \
+ nx-aes-ctr.o \
+ nx-aes-xcbc.o \
+ nx-sha256.o \
+ nx-sha512.o
+
+obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS) += nx-compress.o
+nx-compress-objs := nx-842.o
diff --git a/drivers/crypto/nx/nx-842.c b/drivers/crypto/nx/nx-842.c
new file mode 100644
index 000000000..887196e9b
--- /dev/null
+++ b/drivers/crypto/nx/nx-842.c
@@ -0,0 +1,1603 @@
+/*
+ * Driver for IBM Power 842 compression accelerator
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * Copyright (C) IBM Corporation, 2012
+ *
+ * Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
+ * Seth Jennings <sjenning@linux.vnet.ibm.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/nx842.h>
+#include <linux/of.h>
+#include <linux/slab.h>
+
+#include <asm/page.h>
+#include <asm/vio.h>
+
+#include "nx_csbcpb.h" /* struct nx_csbcpb */
+
+#define MODULE_NAME "nx-compress"
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Robert Jennings <rcj@linux.vnet.ibm.com>");
+MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors");
+
+#define SHIFT_4K 12
+#define SHIFT_64K 16
+#define SIZE_4K (1UL << SHIFT_4K)
+#define SIZE_64K (1UL << SHIFT_64K)
+
+/* IO buffer must be 128 byte aligned */
+#define IO_BUFFER_ALIGN 128
+
+struct nx842_header {
+ int blocks_nr; /* number of compressed blocks */
+ int offset; /* offset of the first block (from beginning of header) */
+ int sizes[0]; /* size of compressed blocks */
+};
+
+static inline int nx842_header_size(const struct nx842_header *hdr)
+{
+ return sizeof(struct nx842_header) +
+ hdr->blocks_nr * sizeof(hdr->sizes[0]);
+}
+
+/* Macros for fields within nx_csbcpb */
+/* Check the valid bit within the csbcpb valid field */
+#define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7))
+
+/* CE macros operate on the completion_extension field bits in the csbcpb.
+ * CE0 0=full completion, 1=partial completion
+ * CE1 0=CE0 indicates completion, 1=termination (output may be modified)
+ * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */
+#define NX842_CSBCPB_CE0(x) (x & BIT_MASK(7))
+#define NX842_CSBCPB_CE1(x) (x & BIT_MASK(6))
+#define NX842_CSBCPB_CE2(x) (x & BIT_MASK(5))
+
+/* The NX unit accepts data only on 4K page boundaries */
+#define NX842_HW_PAGE_SHIFT SHIFT_4K
+#define NX842_HW_PAGE_SIZE (ASM_CONST(1) << NX842_HW_PAGE_SHIFT)
+#define NX842_HW_PAGE_MASK (~(NX842_HW_PAGE_SIZE-1))
+
+enum nx842_status {
+ UNAVAILABLE,
+ AVAILABLE
+};
+
+struct ibm_nx842_counters {
+ atomic64_t comp_complete;
+ atomic64_t comp_failed;
+ atomic64_t decomp_complete;
+ atomic64_t decomp_failed;
+ atomic64_t swdecomp;
+ atomic64_t comp_times[32];
+ atomic64_t decomp_times[32];
+};
+
+static struct nx842_devdata {
+ struct vio_dev *vdev;
+ struct device *dev;
+ struct ibm_nx842_counters *counters;
+ unsigned int max_sg_len;
+ unsigned int max_sync_size;
+ unsigned int max_sync_sg;
+ enum nx842_status status;
+} __rcu *devdata;
+static DEFINE_SPINLOCK(devdata_mutex);
+
+#define NX842_COUNTER_INC(_x) \
+static inline void nx842_inc_##_x( \
+ const struct nx842_devdata *dev) { \
+ if (dev) \
+ atomic64_inc(&dev->counters->_x); \
+}
+NX842_COUNTER_INC(comp_complete);
+NX842_COUNTER_INC(comp_failed);
+NX842_COUNTER_INC(decomp_complete);
+NX842_COUNTER_INC(decomp_failed);
+NX842_COUNTER_INC(swdecomp);
+
+#define NX842_HIST_SLOTS 16
+
+static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time)
+{
+ int bucket = fls(time);
+
+ if (bucket)
+ bucket = min((NX842_HIST_SLOTS - 1), bucket - 1);
+
+ atomic64_inc(×[bucket]);
+}
+
+/* NX unit operation flags */
+#define NX842_OP_COMPRESS 0x0
+#define NX842_OP_CRC 0x1
+#define NX842_OP_DECOMPRESS 0x2
+#define NX842_OP_COMPRESS_CRC (NX842_OP_COMPRESS | NX842_OP_CRC)
+#define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC)
+#define NX842_OP_ASYNC (1<<23)
+#define NX842_OP_NOTIFY (1<<22)
+#define NX842_OP_NOTIFY_INT(x) ((x & 0xff)<<8)
+
+static unsigned long nx842_get_desired_dma(struct vio_dev *viodev)
+{
+ /* No use of DMA mappings within the driver. */
+ return 0;
+}
+
+struct nx842_slentry {
+ unsigned long ptr; /* Real address (use __pa()) */
+ unsigned long len;
+};
+
+/* pHyp scatterlist entry */
+struct nx842_scatterlist {
+ int entry_nr; /* number of slentries */
+ struct nx842_slentry *entries; /* ptr to array of slentries */
+};
+
+/* Does not include sizeof(entry_nr) in the size */
+static inline unsigned long nx842_get_scatterlist_size(
+ struct nx842_scatterlist *sl)
+{
+ return sl->entry_nr * sizeof(struct nx842_slentry);
+}
+
+static inline unsigned long nx842_get_pa(void *addr)
+{
+ if (is_vmalloc_addr(addr))
+ return page_to_phys(vmalloc_to_page(addr))
+ + offset_in_page(addr);
+ else
+ return __pa(addr);
+}
+
+static int nx842_build_scatterlist(unsigned long buf, int len,
+ struct nx842_scatterlist *sl)
+{
+ unsigned long nextpage;
+ struct nx842_slentry *entry;
+
+ sl->entry_nr = 0;
+
+ entry = sl->entries;
+ while (len) {
+ entry->ptr = nx842_get_pa((void *)buf);
+ nextpage = ALIGN(buf + 1, NX842_HW_PAGE_SIZE);
+ if (nextpage < buf + len) {
+ /* we aren't at the end yet */
+ if (IS_ALIGNED(buf, NX842_HW_PAGE_SIZE))
+ /* we are in the middle (or beginning) */
+ entry->len = NX842_HW_PAGE_SIZE;
+ else
+ /* we are at the beginning */
+ entry->len = nextpage - buf;
+ } else {
+ /* at the end */
+ entry->len = len;
+ }
+
+ len -= entry->len;
+ buf += entry->len;
+ sl->entry_nr++;
+ entry++;
+ }
+
+ return 0;
+}
+
+/*
+ * Working memory for software decompression
+ */
+struct sw842_fifo {
+ union {
+ char f8[256][8];
+ char f4[512][4];
+ };
+ char f2[256][2];
+ unsigned char f84_full;
+ unsigned char f2_full;
+ unsigned char f8_count;
+ unsigned char f2_count;
+ unsigned int f4_count;
+};
+
+/*
+ * Working memory for crypto API
+ */
+struct nx842_workmem {
+ char bounce[PAGE_SIZE]; /* bounce buffer for decompression input */
+ union {
+ /* hardware working memory */
+ struct {
+ /* scatterlist */
+ char slin[SIZE_4K];
+ char slout[SIZE_4K];
+ /* coprocessor status/parameter block */
+ struct nx_csbcpb csbcpb;
+ };
+ /* software working memory */
+ struct sw842_fifo swfifo; /* software decompression fifo */
+ };
+};
+
+int nx842_get_workmem_size(void)
+{
+ return sizeof(struct nx842_workmem) + NX842_HW_PAGE_SIZE;
+}
+EXPORT_SYMBOL_GPL(nx842_get_workmem_size);
+
+int nx842_get_workmem_size_aligned(void)
+{
+ return sizeof(struct nx842_workmem);
+}
+EXPORT_SYMBOL_GPL(nx842_get_workmem_size_aligned);
+
+static int nx842_validate_result(struct device *dev,
+ struct cop_status_block *csb)
+{
+ /* The csb must be valid after returning from vio_h_cop_sync */
+ if (!NX842_CSBCBP_VALID_CHK(csb->valid)) {
+ dev_err(dev, "%s: cspcbp not valid upon completion.\n",
+ __func__);
+ dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n",
+ csb->valid,
+ csb->crb_seq_number,
+ csb->completion_code,
+ csb->completion_extension);
+ dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n",
+ csb->processed_byte_count,
+ (unsigned long)csb->address);
+ return -EIO;
+ }
+
+ /* Check return values from the hardware in the CSB */
+ switch (csb->completion_code) {
+ case 0: /* Completed without error */
+ break;
+ case 64: /* Target bytes > Source bytes during compression */
+ case 13: /* Output buffer too small */
+ dev_dbg(dev, "%s: Compression output larger than input\n",
+ __func__);
+ return -ENOSPC;
+ case 66: /* Input data contains an illegal template field */
+ case 67: /* Template indicates data past the end of the input stream */
+ dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n",
+ __func__, csb->completion_code);
+ return -EINVAL;
+ default:
+ dev_dbg(dev, "%s: Unspecified error (code:%d)\n",
+ __func__, csb->completion_code);
+ return -EIO;
+ }
+
+ /* Hardware sanity check */
+ if (!NX842_CSBCPB_CE2(csb->completion_extension)) {
+ dev_err(dev, "%s: No error returned by hardware, but "
+ "data returned is unusable, contact support.\n"
+ "(Additional info: csbcbp->processed bytes "
+ "does not specify processed bytes for the "
+ "target buffer.)\n", __func__);
+ return -EIO;
+ }
+
+ return 0;
+}
+
+/**
+ * nx842_compress - Compress data using the 842 algorithm
+ *
+ * Compression provide by the NX842 coprocessor on IBM Power systems.
+ * The input buffer is compressed and the result is stored in the
+ * provided output buffer.
+ *
+ * Upon return from this function @outlen contains the length of the
+ * compressed data. If there is an error then @outlen will be 0 and an
+ * error will be specified by the return code from this function.
+ *
+ * @in: Pointer to input buffer, must be page aligned
+ * @inlen: Length of input buffer, must be PAGE_SIZE
+ * @out: Pointer to output buffer
+ * @outlen: Length of output buffer
+ * @wrkmem: ptr to buffer for working memory, size determined by
+ * nx842_get_workmem_size()
+ *
+ * Returns:
+ * 0 Success, output of length @outlen stored in the buffer at @out
+ * -ENOMEM Unable to allocate internal buffers
+ * -ENOSPC Output buffer is to small
+ * -EMSGSIZE XXX Difficult to describe this limitation
+ * -EIO Internal error
+ * -ENODEV Hardware unavailable
+ */
+int nx842_compress(const unsigned char *in, unsigned int inlen,
+ unsigned char *out, unsigned int *outlen, void *wmem)
+{
+ struct nx842_header *hdr;
+ struct nx842_devdata *local_devdata;
+ struct device *dev = NULL;
+ struct nx842_workmem *workmem;
+ struct nx842_scatterlist slin, slout;
+ struct nx_csbcpb *csbcpb;
+ int ret = 0, max_sync_size, i, bytesleft, size, hdrsize;
+ unsigned long inbuf, outbuf, padding;
+ struct vio_pfo_op op = {
+ .done = NULL,
+ .handle = 0,
+ .timeout = 0,
+ };
+ unsigned long start_time = get_tb();
+
+ /*
+ * Make sure input buffer is 64k page aligned. This is assumed since
+ * this driver is designed for page compression only (for now). This
+ * is very nice since we can now use direct DDE(s) for the input and
+ * the alignment is guaranteed.
+ */
+ inbuf = (unsigned long)in;
+ if (!IS_ALIGNED(inbuf, PAGE_SIZE) || inlen != PAGE_SIZE)
+ return -EINVAL;
+
+ rcu_read_lock();
+ local_devdata = rcu_dereference(devdata);
+ if (!local_devdata || !local_devdata->dev) {
+ rcu_read_unlock();
+ return -ENODEV;
+ }
+ max_sync_size = local_devdata->max_sync_size;
+ dev = local_devdata->dev;
+
+ /* Create the header */
+ hdr = (struct nx842_header *)out;
+ hdr->blocks_nr = PAGE_SIZE / max_sync_size;
+ hdrsize = nx842_header_size(hdr);
+ outbuf = (unsigned long)out + hdrsize;
+ bytesleft = *outlen - hdrsize;
+
+ /* Init scatterlist */
+ workmem = (struct nx842_workmem *)ALIGN((unsigned long)wmem,
+ NX842_HW_PAGE_SIZE);
+ slin.entries = (struct nx842_slentry *)workmem->slin;
+ slout.entries = (struct nx842_slentry *)workmem->slout;
+
+ /* Init operation */
+ op.flags = NX842_OP_COMPRESS;
+ csbcpb = &workmem->csbcpb;
+ memset(csbcpb, 0, sizeof(*csbcpb));
+ op.csbcpb = nx842_get_pa(csbcpb);
+ op.out = nx842_get_pa(slout.entries);
+
+ for (i = 0; i < hdr->blocks_nr; i++) {
+ /*
+ * Aligning the output blocks to 128 bytes does waste space,
+ * but it prevents the need for bounce buffers and memory
+ * copies. It also simplifies the code a lot. In the worst
+ * case (64k page, 4k max_sync_size), you lose up to
+ * (128*16)/64k = ~3% the compression factor. For 64k
+ * max_sync_size, the loss would be at most 128/64k = ~0.2%.
+ */
+ padding = ALIGN(outbuf, IO_BUFFER_ALIGN) - outbuf;
+ outbuf += padding;
+ bytesleft -= padding;
+ if (i == 0)
+ /* save offset into first block in header */
+ hdr->offset = padding + hdrsize;
+
+ if (bytesleft <= 0) {
+ ret = -ENOSPC;
+ goto unlock;
+ }
+
+ /*
+ * NOTE: If the default max_sync_size is changed from 4k
+ * to 64k, remove the "likely" case below, since a
+ * scatterlist will always be needed.
+ */
+ if (likely(max_sync_size == NX842_HW_PAGE_SIZE)) {
+ /* Create direct DDE */
+ op.in = nx842_get_pa((void *)inbuf);
+ op.inlen = max_sync_size;
+
+ } else {
+ /* Create indirect DDE (scatterlist) */
+ nx842_build_scatterlist(inbuf, max_sync_size, &slin);
+ op.in = nx842_get_pa(slin.entries);
+ op.inlen = -nx842_get_scatterlist_size(&slin);
+ }
+
+ /*
+ * If max_sync_size != NX842_HW_PAGE_SIZE, an indirect
+ * DDE is required for the outbuf.
+ * If max_sync_size == NX842_HW_PAGE_SIZE, outbuf must
+ * also be page aligned (1 in 128/4k=32 chance) in order
+ * to use a direct DDE.
+ * This is unlikely, just use an indirect DDE always.
+ */
+ nx842_build_scatterlist(outbuf,
+ min(bytesleft, max_sync_size), &slout);
+ /* op.out set before loop */
+ op.outlen = -nx842_get_scatterlist_size(&slout);
+
+ /* Send request to pHyp */
+ ret = vio_h_cop_sync(local_devdata->vdev, &op);
+
+ /* Check for pHyp error */
+ if (ret) {
+ dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
+ __func__, ret, op.hcall_err);
+ ret = -EIO;
+ goto unlock;
+ }
+
+ /* Check for hardware error */
+ ret = nx842_validate_result(dev, &csbcpb->csb);
+ if (ret && ret != -ENOSPC)
+ goto unlock;
+
+ /* Handle incompressible data */
+ if (unlikely(ret == -ENOSPC)) {
+ if (bytesleft < max_sync_size) {
+ /*
+ * Not enough space left in the output buffer
+ * to store uncompressed block
+ */
+ goto unlock;
+ } else {
+ /* Store incompressible block */
+ memcpy((void *)outbuf, (void *)inbuf,
+ max_sync_size);
+ hdr->sizes[i] = -max_sync_size;
+ outbuf += max_sync_size;
+ bytesleft -= max_sync_size;
+ /* Reset ret, incompressible data handled */
+ ret = 0;
+ }
+ } else {
+ /* Normal case, compression was successful */
+ size = csbcpb->csb.processed_byte_count;
+ dev_dbg(dev, "%s: processed_bytes=%d\n",
+ __func__, size);
+ hdr->sizes[i] = size;
+ outbuf += size;
+ bytesleft -= size;
+ }
+
+ inbuf += max_sync_size;
+ }
+
+ *outlen = (unsigned int)(outbuf - (unsigned long)out);
+
+unlock:
+ if (ret)
+ nx842_inc_comp_failed(local_devdata);
+ else {
+ nx842_inc_comp_complete(local_devdata);
+ ibm_nx842_incr_hist(local_devdata->counters->comp_times,
+ (get_tb() - start_time) / tb_ticks_per_usec);
+ }
+ rcu_read_unlock();
+ return ret;
+}
+EXPORT_SYMBOL_GPL(nx842_compress);
+
+static int sw842_decompress(const unsigned char *, int, unsigned char *, int *,
+ const void *);
+
+/**
+ * nx842_decompress - Decompress data using the 842 algorithm
+ *
+ * Decompression provide by the NX842 coprocessor on IBM Power systems.
+ * The input buffer is decompressed and the result is stored in the
+ * provided output buffer. The size allocated to the output buffer is
+ * provided by the caller of this function in @outlen. Upon return from
+ * this function @outlen contains the length of the decompressed data.
+ * If there is an error then @outlen will be 0 and an error will be
+ * specified by the return code from this function.
+ *
+ * @in: Pointer to input buffer, will use bounce buffer if not 128 byte
+ * aligned
+ * @inlen: Length of input buffer
+ * @out: Pointer to output buffer, must be page aligned
+ * @outlen: Length of output buffer, must be PAGE_SIZE
+ * @wrkmem: ptr to buffer for working memory, size determined by
+ * nx842_get_workmem_size()
+ *
+ * Returns:
+ * 0 Success, output of length @outlen stored in the buffer at @out
+ * -ENODEV Hardware decompression device is unavailable
+ * -ENOMEM Unable to allocate internal buffers
+ * -ENOSPC Output buffer is to small
+ * -EINVAL Bad input data encountered when attempting decompress
+ * -EIO Internal error
+ */
+int nx842_decompress(const unsigned char *in, unsigned int inlen,
+ unsigned char *out, unsigned int *outlen, void *wmem)
+{
+ struct nx842_header *hdr;
+ struct nx842_devdata *local_devdata;
+ struct device *dev = NULL;
+ struct nx842_workmem *workmem;
+ struct nx842_scatterlist slin, slout;
+ struct nx_csbcpb *csbcpb;
+ int ret = 0, i, size, max_sync_size;
+ unsigned long inbuf, outbuf;
+ struct vio_pfo_op op = {
+ .done = NULL,
+ .handle = 0,
+ .timeout = 0,
+ };
+ unsigned long start_time = get_tb();
+
+ /* Ensure page alignment and size */
+ outbuf = (unsigned long)out;
+ if (!IS_ALIGNED(outbuf, PAGE_SIZE) || *outlen != PAGE_SIZE)
+ return -EINVAL;
+
+ rcu_read_lock();
+ local_devdata = rcu_dereference(devdata);
+ if (local_devdata)
+ dev = local_devdata->dev;
+
+ /* Get header */
+ hdr = (struct nx842_header *)in;
+
+ workmem = (struct nx842_workmem *)ALIGN((unsigned long)wmem,
+ NX842_HW_PAGE_SIZE);
+
+ inbuf = (unsigned long)in + hdr->offset;
+ if (likely(!IS_ALIGNED(inbuf, IO_BUFFER_ALIGN))) {
+ /* Copy block(s) into bounce buffer for alignment */
+ memcpy(workmem->bounce, in + hdr->offset, inlen - hdr->offset);
+ inbuf = (unsigned long)workmem->bounce;
+ }
+
+ /* Init scatterlist */
+ slin.entries = (struct nx842_slentry *)workmem->slin;
+ slout.entries = (struct nx842_slentry *)workmem->slout;
+
+ /* Init operation */
+ op.flags = NX842_OP_DECOMPRESS;
+ csbcpb = &workmem->csbcpb;
+ memset(csbcpb, 0, sizeof(*csbcpb));
+ op.csbcpb = nx842_get_pa(csbcpb);
+
+ /*
+ * max_sync_size may have changed since compression,
+ * so we can't read it from the device info. We need
+ * to derive it from hdr->blocks_nr.
+ */
+ max_sync_size = PAGE_SIZE / hdr->blocks_nr;
+
+ for (i = 0; i < hdr->blocks_nr; i++) {
+ /* Skip padding */
+ inbuf = ALIGN(inbuf, IO_BUFFER_ALIGN);
+
+ if (hdr->sizes[i] < 0) {
+ /* Negative sizes indicate uncompressed data blocks */
+ size = abs(hdr->sizes[i]);
+ memcpy((void *)outbuf, (void *)inbuf, size);
+ outbuf += size;
+ inbuf += size;
+ continue;
+ }
+
+ if (!dev)
+ goto sw;
+
+ /*
+ * The better the compression, the more likely the "likely"
+ * case becomes.
+ */
+ if (likely((inbuf & NX842_HW_PAGE_MASK) ==
+ ((inbuf + hdr->sizes[i] - 1) & NX842_HW_PAGE_MASK))) {
+ /* Create direct DDE */
+ op.in = nx842_get_pa((void *)inbuf);
+ op.inlen = hdr->sizes[i];
+ } else {
+ /* Create indirect DDE (scatterlist) */
+ nx842_build_scatterlist(inbuf, hdr->sizes[i] , &slin);
+ op.in = nx842_get_pa(slin.entries);
+ op.inlen = -nx842_get_scatterlist_size(&slin);
+ }
+
+ /*
+ * NOTE: If the default max_sync_size is changed from 4k
+ * to 64k, remove the "likely" case below, since a
+ * scatterlist will always be needed.
+ */
+ if (likely(max_sync_size == NX842_HW_PAGE_SIZE)) {
+ /* Create direct DDE */
+ op.out = nx842_get_pa((void *)outbuf);
+ op.outlen = max_sync_size;
+ } else {
+ /* Create indirect DDE (scatterlist) */
+ nx842_build_scatterlist(outbuf, max_sync_size, &slout);
+ op.out = nx842_get_pa(slout.entries);
+ op.outlen = -nx842_get_scatterlist_size(&slout);
+ }
+
+ /* Send request to pHyp */
+ ret = vio_h_cop_sync(local_devdata->vdev, &op);
+
+ /* Check for pHyp error */
+ if (ret) {
+ dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
+ __func__, ret, op.hcall_err);
+ dev = NULL;
+ goto sw;
+ }
+
+ /* Check for hardware error */
+ ret = nx842_validate_result(dev, &csbcpb->csb);
+ if (ret) {
+ dev = NULL;
+ goto sw;
+ }
+
+ /* HW decompression success */
+ inbuf += hdr->sizes[i];
+ outbuf += csbcpb->csb.processed_byte_count;
+ continue;
+
+sw:
+ /* software decompression */
+ size = max_sync_size;
+ ret = sw842_decompress(
+ (unsigned char *)inbuf, hdr->sizes[i],
+ (unsigned char *)outbuf, &size, wmem);
+ if (ret)
+ pr_debug("%s: sw842_decompress failed with %d\n",
+ __func__, ret);
+
+ if (ret) {
+ if (ret != -ENOSPC && ret != -EINVAL &&
+ ret != -EMSGSIZE)
+ ret = -EIO;
+ goto unlock;
+ }
+
+ /* SW decompression success */
+ inbuf += hdr->sizes[i];
+ outbuf += size;
+ }
+
+ *outlen = (unsigned int)(outbuf - (unsigned long)out);
+
+unlock:
+ if (ret)
+ /* decompress fail */
+ nx842_inc_decomp_failed(local_devdata);
+ else {
+ if (!dev)
+ /* software decompress */
+ nx842_inc_swdecomp(local_devdata);
+ nx842_inc_decomp_complete(local_devdata);
+ ibm_nx842_incr_hist(local_devdata->counters->decomp_times,
+ (get_tb() - start_time) / tb_ticks_per_usec);
+ }
+
+ rcu_read_unlock();
+ return ret;
+}
+EXPORT_SYMBOL_GPL(nx842_decompress);
+
+/**
+ * nx842_OF_set_defaults -- Set default (disabled) values for devdata
+ *
+ * @devdata - struct nx842_devdata to update
+ *
+ * Returns:
+ * 0 on success
+ * -ENOENT if @devdata ptr is NULL
+ */
+static int nx842_OF_set_defaults(struct nx842_devdata *devdata)
+{
+ if (devdata) {
+ devdata->max_sync_size = 0;
+ devdata->max_sync_sg = 0;
+ devdata->max_sg_len = 0;
+ devdata->status = UNAVAILABLE;
+ return 0;
+ } else
+ return -ENOENT;
+}
+
+/**
+ * nx842_OF_upd_status -- Update the device info from OF status prop
+ *
+ * The status property indicates if the accelerator is enabled. If the
+ * device is in the OF tree it indicates that the hardware is present.
+ * The status field indicates if the device is enabled when the status
+ * is 'okay'. Otherwise the device driver will be disabled.
+ *
+ * @devdata - struct nx842_devdata to update
+ * @prop - struct property point containing the maxsyncop for the update
+ *
+ * Returns:
+ * 0 - Device is available
+ * -EINVAL - Device is not available
+ */
+static int nx842_OF_upd_status(struct nx842_devdata *devdata,
+ struct property *prop) {
+ int ret = 0;
+ const char *status = (const char *)prop->value;
+
+ if (!strncmp(status, "okay", (size_t)prop->length)) {
+ devdata->status = AVAILABLE;
+ } else {
+ dev_info(devdata->dev, "%s: status '%s' is not 'okay'\n",
+ __func__, status);
+ devdata->status = UNAVAILABLE;
+ }
+
+ return ret;
+}
+
+/**
+ * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop
+ *
+ * Definition of the 'ibm,max-sg-len' OF property:
+ * This field indicates the maximum byte length of a scatter list
+ * for the platform facility. It is a single cell encoded as with encode-int.
+ *
+ * Example:
+ * # od -x ibm,max-sg-len
+ * 0000000 0000 0ff0
+ *
+ * In this example, the maximum byte length of a scatter list is
+ * 0x0ff0 (4,080).
+ *
+ * @devdata - struct nx842_devdata to update
+ * @prop - struct property point containing the maxsyncop for the update
+ *
+ * Returns:
+ * 0 on success
+ * -EINVAL on failure
+ */
+static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata,
+ struct property *prop) {
+ int ret = 0;
+ const int *maxsglen = prop->value;
+
+ if (prop->length != sizeof(*maxsglen)) {
+ dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__);
+ dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__,
+ prop->length, sizeof(*maxsglen));
+ ret = -EINVAL;
+ } else {
+ devdata->max_sg_len = (unsigned int)min(*maxsglen,
+ (int)NX842_HW_PAGE_SIZE);
+ }
+
+ return ret;
+}
+
+/**
+ * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop
+ *
+ * Definition of the 'ibm,max-sync-cop' OF property:
+ * Two series of cells. The first series of cells represents the maximums
+ * that can be synchronously compressed. The second series of cells
+ * represents the maximums that can be synchronously decompressed.
+ * 1. The first cell in each series contains the count of the number of
+ * data length, scatter list elements pairs that follow – each being
+ * of the form
+ * a. One cell data byte length
+ * b. One cell total number of scatter list elements
+ *
+ * Example:
+ * # od -x ibm,max-sync-cop
+ * 0000000 0000 0001 0000 1000 0000 01fe 0000 0001
+ * 0000020 0000 1000 0000 01fe
+ *
+ * In this example, compression supports 0x1000 (4,096) data byte length
+ * and 0x1fe (510) total scatter list elements. Decompression supports
+ * 0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list
+ * elements.
+ *
+ * @devdata - struct nx842_devdata to update
+ * @prop - struct property point containing the maxsyncop for the update
+ *
+ * Returns:
+ * 0 on success
+ * -EINVAL on failure
+ */
+static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata,
+ struct property *prop) {
+ int ret = 0;
+ const struct maxsynccop_t {
+ int comp_elements;
+ int comp_data_limit;
+ int comp_sg_limit;
+ int decomp_elements;
+ int decomp_data_limit;
+ int decomp_sg_limit;
+ } *maxsynccop;
+
+ if (prop->length != sizeof(*maxsynccop)) {
+ dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__);
+ dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length,
+ sizeof(*maxsynccop));
+ ret = -EINVAL;
+ goto out;
+ }
+
+ maxsynccop = (const struct maxsynccop_t *)prop->value;
+
+ /* Use one limit rather than separate limits for compression and
+ * decompression. Set a maximum for this so as not to exceed the
+ * size that the header can support and round the value down to
+ * the hardware page size (4K) */
+ devdata->max_sync_size =
+ (unsigned int)min(maxsynccop->comp_data_limit,
+ maxsynccop->decomp_data_limit);
+
+ devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size,
+ SIZE_64K);
+
+ if (devdata->max_sync_size < SIZE_4K) {
+ dev_err(devdata->dev, "%s: hardware max data size (%u) is "
+ "less than the driver minimum, unable to use "
+ "the hardware device\n",
+ __func__, devdata->max_sync_size);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ devdata->max_sync_sg = (unsigned int)min(maxsynccop->comp_sg_limit,
+ maxsynccop->decomp_sg_limit);
+ if (devdata->max_sync_sg < 1) {
+ dev_err(devdata->dev, "%s: hardware max sg size (%u) is "
+ "less than the driver minimum, unable to use "
+ "the hardware device\n",
+ __func__, devdata->max_sync_sg);
+ ret = -EINVAL;
+ goto out;
+ }
+
+out:
+ return ret;
+}
+
+/**
+ *
+ * nx842_OF_upd -- Handle OF properties updates for the device.
+ *
+ * Set all properties from the OF tree. Optionally, a new property
+ * can be provided by the @new_prop pointer to overwrite an existing value.
+ * The device will remain disabled until all values are valid, this function
+ * will return an error for updates unless all values are valid.
+ *
+ * @new_prop: If not NULL, this property is being updated. If NULL, update
+ * all properties from the current values in the OF tree.
+ *
+ * Returns:
+ * 0 - Success
+ * -ENOMEM - Could not allocate memory for new devdata structure
+ * -EINVAL - property value not found, new_prop is not a recognized
+ * property for the device or property value is not valid.
+ * -ENODEV - Device is not available
+ */
+static int nx842_OF_upd(struct property *new_prop)
+{
+ struct nx842_devdata *old_devdata = NULL;
+ struct nx842_devdata *new_devdata = NULL;
+ struct device_node *of_node = NULL;
+ struct property *status = NULL;
+ struct property *maxsglen = NULL;
+ struct property *maxsyncop = NULL;
+ int ret = 0;
+ unsigned long flags;
+
+ spin_lock_irqsave(&devdata_mutex, flags);
+ old_devdata = rcu_dereference_check(devdata,
+ lockdep_is_held(&devdata_mutex));
+ if (old_devdata)
+ of_node = old_devdata->dev->of_node;
+
+ if (!old_devdata || !of_node) {
+ pr_err("%s: device is not available\n", __func__);
+ spin_unlock_irqrestore(&devdata_mutex, flags);
+ return -ENODEV;
+ }
+
+ new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
+ if (!new_devdata) {
+ dev_err(old_devdata->dev, "%s: Could not allocate memory for device data\n", __func__);
+ ret = -ENOMEM;
+ goto error_out;
+ }
+
+ memcpy(new_devdata, old_devdata, sizeof(*old_devdata));
+ new_devdata->counters = old_devdata->counters;
+
+ /* Set ptrs for existing properties */
+ status = of_find_property(of_node, "status", NULL);
+ maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL);
+ maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL);
+ if (!status || !maxsglen || !maxsyncop) {
+ dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__);
+ ret = -EINVAL;
+ goto error_out;
+ }
+
+ /*
+ * If this is a property update, there are only certain properties that
+ * we care about. Bail if it isn't in the below list
+ */
+ if (new_prop && (strncmp(new_prop->name, "status", new_prop->length) ||
+ strncmp(new_prop->name, "ibm,max-sg-len", new_prop->length) ||
+ strncmp(new_prop->name, "ibm,max-sync-cop", new_prop->length)))
+ goto out;
+
+ /* Perform property updates */
+ ret = nx842_OF_upd_status(new_devdata, status);
+ if (ret)
+ goto error_out;
+
+ ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen);
+ if (ret)
+ goto error_out;
+
+ ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop);
+ if (ret)
+ goto error_out;
+
+out:
+ dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n",
+ __func__, new_devdata->max_sync_size,
+ old_devdata->max_sync_size);
+ dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n",
+ __func__, new_devdata->max_sync_sg,
+ old_devdata->max_sync_sg);
+ dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n",
+ __func__, new_devdata->max_sg_len,
+ old_devdata->max_sg_len);
+
+ rcu_assign_pointer(devdata, new_devdata);
+ spin_unlock_irqrestore(&devdata_mutex, flags);
+ synchronize_rcu();
+ dev_set_drvdata(new_devdata->dev, new_devdata);
+ kfree(old_devdata);
+ return 0;
+
+error_out:
+ if (new_devdata) {
+ dev_info(old_devdata->dev, "%s: device disabled\n", __func__);
+ nx842_OF_set_defaults(new_devdata);
+ rcu_assign_pointer(devdata, new_devdata);
+ spin_unlock_irqrestore(&devdata_mutex, flags);
+ synchronize_rcu();
+ dev_set_drvdata(new_devdata->dev, new_devdata);
+ kfree(old_devdata);
+ } else {
+ dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__);
+ spin_unlock_irqrestore(&devdata_mutex, flags);
+ }
+
+ if (!ret)
+ ret = -EINVAL;
+ return ret;
+}
+
+/**
+ * nx842_OF_notifier - Process updates to OF properties for the device
+ *
+ * @np: notifier block
+ * @action: notifier action
+ * @update: struct pSeries_reconfig_prop_update pointer if action is
+ * PSERIES_UPDATE_PROPERTY
+ *
+ * Returns:
+ * NOTIFY_OK on success
+ * NOTIFY_BAD encoded with error number on failure, use
+ * notifier_to_errno() to decode this value
+ */
+static int nx842_OF_notifier(struct notifier_block *np, unsigned long action,
+ void *data)
+{
+ struct of_reconfig_data *upd = data;
+ struct nx842_devdata *local_devdata;
+ struct device_node *node = NULL;
+
+ rcu_read_lock();
+ local_devdata = rcu_dereference(devdata);
+ if (local_devdata)
+ node = local_devdata->dev->of_node;
+
+ if (local_devdata &&
+ action == OF_RECONFIG_UPDATE_PROPERTY &&
+ !strcmp(upd->dn->name, node->name)) {
+ rcu_read_unlock();
+ nx842_OF_upd(upd->prop);
+ } else
+ rcu_read_unlock();
+
+ return NOTIFY_OK;
+}
+
+static struct notifier_block nx842_of_nb = {
+ .notifier_call = nx842_OF_notifier,
+};
+
+#define nx842_counter_read(_name) \
+static ssize_t nx842_##_name##_show(struct device *dev, \
+ struct device_attribute *attr, \
+ char *buf) { \
+ struct nx842_devdata *local_devdata; \
+ int p = 0; \
+ rcu_read_lock(); \
+ local_devdata = rcu_dereference(devdata); \
+ if (local_devdata) \
+ p = snprintf(buf, PAGE_SIZE, "%ld\n", \
+ atomic64_read(&local_devdata->counters->_name)); \
+ rcu_read_unlock(); \
+ return p; \
+}
+
+#define NX842DEV_COUNTER_ATTR_RO(_name) \
+ nx842_counter_read(_name); \
+ static struct device_attribute dev_attr_##_name = __ATTR(_name, \
+ 0444, \
+ nx842_##_name##_show,\
+ NULL);
+
+NX842DEV_COUNTER_ATTR_RO(comp_complete);
+NX842DEV_COUNTER_ATTR_RO(comp_failed);
+NX842DEV_COUNTER_ATTR_RO(decomp_complete);
+NX842DEV_COUNTER_ATTR_RO(decomp_failed);
+NX842DEV_COUNTER_ATTR_RO(swdecomp);
+
+static ssize_t nx842_timehist_show(struct device *,
+ struct device_attribute *, char *);
+
+static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444,
+ nx842_timehist_show, NULL);
+static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times,
+ 0444, nx842_timehist_show, NULL);
+
+static ssize_t nx842_timehist_show(struct device *dev,
+ struct device_attribute *attr, char *buf) {
+ char *p = buf;
+ struct nx842_devdata *local_devdata;
+ atomic64_t *times;
+ int bytes_remain = PAGE_SIZE;
+ int bytes;
+ int i;
+
+ rcu_read_lock();
+ local_devdata = rcu_dereference(devdata);
+ if (!local_devdata) {
+ rcu_read_unlock();
+ return 0;
+ }
+
+ if (attr == &dev_attr_comp_times)
+ times = local_devdata->counters->comp_times;
+ else if (attr == &dev_attr_decomp_times)
+ times = local_devdata->counters->decomp_times;
+ else {
+ rcu_read_unlock();
+ return 0;
+ }
+
+ for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) {
+ bytes = snprintf(p, bytes_remain, "%u-%uus:\t%ld\n",
+ i ? (2<<(i-1)) : 0, (2<<i)-1,
+ atomic64_read(×[i]));
+ bytes_remain -= bytes;
+ p += bytes;
+ }
+ /* The last bucket holds everything over
+ * 2<<(NX842_HIST_SLOTS - 2) us */
+ bytes = snprintf(p, bytes_remain, "%uus - :\t%ld\n",
+ 2<<(NX842_HIST_SLOTS - 2),
+ atomic64_read(×[(NX842_HIST_SLOTS - 1)]));
+ p += bytes;
+
+ rcu_read_unlock();
+ return p - buf;
+}
+
+static struct attribute *nx842_sysfs_entries[] = {
+ &dev_attr_comp_complete.attr,
+ &dev_attr_comp_failed.attr,
+ &dev_attr_decomp_complete.attr,
+ &dev_attr_decomp_failed.attr,
+ &dev_attr_swdecomp.attr,
+ &dev_attr_comp_times.attr,
+ &dev_attr_decomp_times.attr,
+ NULL,
+};
+
+static struct attribute_group nx842_attribute_group = {
+ .name = NULL, /* put in device directory */
+ .attrs = nx842_sysfs_entries,
+};
+
+static int __init nx842_probe(struct vio_dev *viodev,
+ const struct vio_device_id *id)
+{
+ struct nx842_devdata *old_devdata, *new_devdata = NULL;
+ unsigned long flags;
+ int ret = 0;
+
+ spin_lock_irqsave(&devdata_mutex, flags);
+ old_devdata = rcu_dereference_check(devdata,
+ lockdep_is_held(&devdata_mutex));
+
+ if (old_devdata && old_devdata->vdev != NULL) {
+ dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__);
+ ret = -1;
+ goto error_unlock;
+ }
+
+ dev_set_drvdata(&viodev->dev, NULL);
+
+ new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
+ if (!new_devdata) {
+ dev_err(&viodev->dev, "%s: Could not allocate memory for device data\n", __func__);
+ ret = -ENOMEM;
+ goto error_unlock;
+ }
+
+ new_devdata->counters = kzalloc(sizeof(*new_devdata->counters),
+ GFP_NOFS);
+ if (!new_devdata->counters) {
+ dev_err(&viodev->dev, "%s: Could not allocate memory for performance counters\n", __func__);
+ ret = -ENOMEM;
+ goto error_unlock;
+ }
+
+ new_devdata->vdev = viodev;
+ new_devdata->dev = &viodev->dev;
+ nx842_OF_set_defaults(new_devdata);
+
+ rcu_assign_pointer(devdata, new_devdata);
+ spin_unlock_irqrestore(&devdata_mutex, flags);
+ synchronize_rcu();
+ kfree(old_devdata);
+
+ of_reconfig_notifier_register(&nx842_of_nb);
+
+ ret = nx842_OF_upd(NULL);
+ if (ret && ret != -ENODEV) {
+ dev_err(&viodev->dev, "could not parse device tree. %d\n", ret);
+ ret = -1;
+ goto error;
+ }
+
+ rcu_read_lock();
+ dev_set_drvdata(&viodev->dev, rcu_dereference(devdata));
+ rcu_read_unlock();
+
+ if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) {
+ dev_err(&viodev->dev, "could not create sysfs device attributes\n");
+ ret = -1;
+ goto error;
+ }
+
+ return 0;
+
+error_unlock:
+ spin_unlock_irqrestore(&devdata_mutex, flags);
+ if (new_devdata)
+ kfree(new_devdata->counters);
+ kfree(new_devdata);
+error:
+ return ret;
+}
+
+static int __exit nx842_remove(struct vio_dev *viodev)
+{
+ struct nx842_devdata *old_devdata;
+ unsigned long flags;
+
+ pr_info("Removing IBM Power 842 compression device\n");
+ sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group);
+
+ spin_lock_irqsave(&devdata_mutex, flags);
+ old_devdata = rcu_dereference_check(devdata,
+ lockdep_is_held(&devdata_mutex));
+ of_reconfig_notifier_unregister(&nx842_of_nb);
+ RCU_INIT_POINTER(devdata, NULL);
+ spin_unlock_irqrestore(&devdata_mutex, flags);
+ synchronize_rcu();
+ dev_set_drvdata(&viodev->dev, NULL);
+ if (old_devdata)
+ kfree(old_devdata->counters);
+ kfree(old_devdata);
+ return 0;
+}
+
+static struct vio_device_id nx842_driver_ids[] = {
+ {"ibm,compression-v1", "ibm,compression"},
+ {"", ""},
+};
+
+static struct vio_driver nx842_driver = {
+ .name = MODULE_NAME,
+ .probe = nx842_probe,
+ .remove = __exit_p(nx842_remove),
+ .get_desired_dma = nx842_get_desired_dma,
+ .id_table = nx842_driver_ids,
+};
+
+static int __init nx842_init(void)
+{
+ struct nx842_devdata *new_devdata;
+ pr_info("Registering IBM Power 842 compression driver\n");
+
+ RCU_INIT_POINTER(devdata, NULL);
+ new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL);
+ if (!new_devdata) {
+ pr_err("Could not allocate memory for device data\n");
+ return -ENOMEM;
+ }
+ new_devdata->status = UNAVAILABLE;
+ RCU_INIT_POINTER(devdata, new_devdata);
+
+ return vio_register_driver(&nx842_driver);
+}
+
+module_init(nx842_init);
+
+static void __exit nx842_exit(void)
+{
+ struct nx842_devdata *old_devdata;
+ unsigned long flags;
+
+ pr_info("Exiting IBM Power 842 compression driver\n");
+ spin_lock_irqsave(&devdata_mutex, flags);
+ old_devdata = rcu_dereference_check(devdata,
+ lockdep_is_held(&devdata_mutex));
+ RCU_INIT_POINTER(devdata, NULL);
+ spin_unlock_irqrestore(&devdata_mutex, flags);
+ synchronize_rcu();
+ if (old_devdata)
+ dev_set_drvdata(old_devdata->dev, NULL);
+ kfree(old_devdata);
+ vio_unregister_driver(&nx842_driver);
+}
+
+module_exit(nx842_exit);
+
+/*********************************
+ * 842 software decompressor
+*********************************/
+typedef int (*sw842_template_op)(const char **, int *, unsigned char **,
+ struct sw842_fifo *);
+
+static int sw842_data8(const char **, int *, unsigned char **,
+ struct sw842_fifo *);
+static int sw842_data4(const char **, int *, unsigned char **,
+ struct sw842_fifo *);
+static int sw842_data2(const char **, int *, unsigned char **,
+ struct sw842_fifo *);
+static int sw842_ptr8(const char **, int *, unsigned char **,
+ struct sw842_fifo *);
+static int sw842_ptr4(const char **, int *, unsigned char **,
+ struct sw842_fifo *);
+static int sw842_ptr2(const char **, int *, unsigned char **,
+ struct sw842_fifo *);
+
+/* special templates */
+#define SW842_TMPL_REPEAT 0x1B
+#define SW842_TMPL_ZEROS 0x1C
+#define SW842_TMPL_EOF 0x1E
+
+static sw842_template_op sw842_tmpl_ops[26][4] = {
+ { sw842_data8, NULL}, /* 0 (00000) */
+ { sw842_data4, sw842_data2, sw842_ptr2, NULL},
+ { sw842_data4, sw842_ptr2, sw842_data2, NULL},
+ { sw842_data4, sw842_ptr2, sw842_ptr2, NULL},
+ { sw842_data4, sw842_ptr4, NULL},
+ { sw842_data2, sw842_ptr2, sw842_data4, NULL},
+ { sw842_data2, sw842_ptr2, sw842_data2, sw842_ptr2},
+ { sw842_data2, sw842_ptr2, sw842_ptr2, sw842_data2},
+ { sw842_data2, sw842_ptr2, sw842_ptr2, sw842_ptr2,},
+ { sw842_data2, sw842_ptr2, sw842_ptr4, NULL},
+ { sw842_ptr2, sw842_data2, sw842_data4, NULL}, /* 10 (01010) */
+ { sw842_ptr2, sw842_data4, sw842_ptr2, NULL},
+ { sw842_ptr2, sw842_data2, sw842_ptr2, sw842_data2},
+ { sw842_ptr2, sw842_data2, sw842_ptr2, sw842_ptr2},
+ { sw842_ptr2, sw842_data2, sw842_ptr4, NULL},
+ { sw842_ptr2, sw842_ptr2, sw842_data4, NULL},
+ { sw842_ptr2, sw842_ptr2, sw842_data2, sw842_ptr2},
+ { sw842_ptr2, sw842_ptr2, sw842_ptr2, sw842_data2},
+ { sw842_ptr2, sw842_ptr2, sw842_ptr2, sw842_ptr2},
+ { sw842_ptr2, sw842_ptr2, sw842_ptr4, NULL},
+ { sw842_ptr4, sw842_data4, NULL}, /* 20 (10100) */
+ { sw842_ptr4, sw842_data2, sw842_ptr2, NULL},
+ { sw842_ptr4, sw842_ptr2, sw842_data2, NULL},
+ { sw842_ptr4, sw842_ptr2, sw842_ptr2, NULL},
+ { sw842_ptr4, sw842_ptr4, NULL},
+ { sw842_ptr8, NULL}
+};
+
+/* Software decompress helpers */
+
+static uint8_t sw842_get_byte(const char *buf, int bit)
+{
+ uint8_t tmpl;
+ uint16_t tmp;
+ tmp = htons(*(uint16_t *)(buf));
+ tmp = (uint16_t)(tmp << bit);
+ tmp = ntohs(tmp);
+ memcpy(&tmpl, &tmp, 1);
+ return tmpl;
+}
+
+static uint8_t sw842_get_template(const char **buf, int *bit)
+{
+ uint8_t byte;
+ byte = sw842_get_byte(*buf, *bit);
+ byte = byte >> 3;
+ byte &= 0x1F;
+ *buf += (*bit + 5) / 8;
+ *bit = (*bit + 5) % 8;
+ return byte;
+}
+
+/* repeat_count happens to be 5-bit too (like the template) */
+static uint8_t sw842_get_repeat_count(const char **buf, int *bit)
+{
+ uint8_t byte;
+ byte = sw842_get_byte(*buf, *bit);
+ byte = byte >> 2;
+ byte &= 0x3F;
+ *buf += (*bit + 6) / 8;
+ *bit = (*bit + 6) % 8;
+ return byte;
+}
+
+static uint8_t sw842_get_ptr2(const char **buf, int *bit)
+{
+ uint8_t ptr;
+ ptr = sw842_get_byte(*buf, *bit);
+ (*buf)++;
+ return ptr;
+}
+
+static uint16_t sw842_get_ptr4(const char **buf, int *bit,
+ struct sw842_fifo *fifo)
+{
+ uint16_t ptr;
+ ptr = htons(*(uint16_t *)(*buf));
+ ptr = (uint16_t)(ptr << *bit);
+ ptr = ptr >> 7;
+ ptr &= 0x01FF;
+ *buf += (*bit + 9) / 8;
+ *bit = (*bit + 9) % 8;
+ return ptr;
+}
+
+static uint8_t sw842_get_ptr8(const char **buf, int *bit,
+ struct sw842_fifo *fifo)
+{
+ return sw842_get_ptr2(buf, bit);
+}
+
+/* Software decompress template ops */
+
+static int sw842_data8(const char **inbuf, int *inbit,
+ unsigned char **outbuf, struct sw842_fifo *fifo)
+{
+ int ret;
+
+ ret = sw842_data4(inbuf, inbit, outbuf, fifo);
+ if (ret)
+ return ret;
+ ret = sw842_data4(inbuf, inbit, outbuf, fifo);
+ return ret;
+}
+
+static int sw842_data4(const char **inbuf, int *inbit,
+ unsigned char **outbuf, struct sw842_fifo *fifo)
+{
+ int ret;
+
+ ret = sw842_data2(inbuf, inbit, outbuf, fifo);
+ if (ret)
+ return ret;
+ ret = sw842_data2(inbuf, inbit, outbuf, fifo);
+ return ret;
+}
+
+static int sw842_data2(const char **inbuf, int *inbit,
+ unsigned char **outbuf, struct sw842_fifo *fifo)
+{
+ **outbuf = sw842_get_byte(*inbuf, *inbit);
+ (*inbuf)++;
+ (*outbuf)++;
+ **outbuf = sw842_get_byte(*inbuf, *inbit);
+ (*inbuf)++;
+ (*outbuf)++;
+ return 0;
+}
+
+static int sw842_ptr8(const char **inbuf, int *inbit,
+ unsigned char **outbuf, struct sw842_fifo *fifo)
+{
+ uint8_t ptr;
+ ptr = sw842_get_ptr8(inbuf, inbit, fifo);
+ if (!fifo->f84_full && (ptr >= fifo->f8_count))
+ return 1;
+ memcpy(*outbuf, fifo->f8[ptr], 8);
+ *outbuf += 8;
+ return 0;
+}
+
+static int sw842_ptr4(const char **inbuf, int *inbit,
+ unsigned char **outbuf, struct sw842_fifo *fifo)
+{
+ uint16_t ptr;
+ ptr = sw842_get_ptr4(inbuf, inbit, fifo);
+ if (!fifo->f84_full && (ptr >= fifo->f4_count))
+ return 1;
+ memcpy(*outbuf, fifo->f4[ptr], 4);
+ *outbuf += 4;
+ return 0;
+}
+
+static int sw842_ptr2(const char **inbuf, int *inbit,
+ unsigned char **outbuf, struct sw842_fifo *fifo)
+{
+ uint8_t ptr;
+ ptr = sw842_get_ptr2(inbuf, inbit);
+ if (!fifo->f2_full && (ptr >= fifo->f2_count))
+ return 1;
+ memcpy(*outbuf, fifo->f2[ptr], 2);
+ *outbuf += 2;
+ return 0;
+}
+
+static void sw842_copy_to_fifo(const char *buf, struct sw842_fifo *fifo)
+{
+ unsigned char initial_f2count = fifo->f2_count;
+
+ memcpy(fifo->f8[fifo->f8_count], buf, 8);
+ fifo->f4_count += 2;
+ fifo->f8_count += 1;
+
+ if (!fifo->f84_full && fifo->f4_count >= 512) {
+ fifo->f84_full = 1;
+ fifo->f4_count /= 512;
+ }
+
+ memcpy(fifo->f2[fifo->f2_count++], buf, 2);
+ memcpy(fifo->f2[fifo->f2_count++], buf + 2, 2);
+ memcpy(fifo->f2[fifo->f2_count++], buf + 4, 2);
+ memcpy(fifo->f2[fifo->f2_count++], buf + 6, 2);
+ if (fifo->f2_count < initial_f2count)
+ fifo->f2_full = 1;
+}
+
+static int sw842_decompress(const unsigned char *src, int srclen,
+ unsigned char *dst, int *destlen,
+ const void *wrkmem)
+{
+ uint8_t tmpl;
+ const char *inbuf;
+ int inbit = 0;
+ unsigned char *outbuf, *outbuf_end, *origbuf, *prevbuf;
+ const char *inbuf_end;
+ sw842_template_op op;
+ int opindex;
+ int i, repeat_count;
+ struct sw842_fifo *fifo;
+ int ret = 0;
+
+ fifo = &((struct nx842_workmem *)(wrkmem))->swfifo;
+ memset(fifo, 0, sizeof(*fifo));
+
+ origbuf = NULL;
+ inbuf = src;
+ inbuf_end = src + srclen;
+ outbuf = dst;
+ outbuf_end = dst + *destlen;
+
+ while ((tmpl = sw842_get_template(&inbuf, &inbit)) != SW842_TMPL_EOF) {
+ if (inbuf >= inbuf_end) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ opindex = 0;
+ prevbuf = origbuf;
+ origbuf = outbuf;
+ switch (tmpl) {
+ case SW842_TMPL_REPEAT:
+ if (prevbuf == NULL) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ repeat_count = sw842_get_repeat_count(&inbuf,
+ &inbit) + 1;
+
+ /* Did the repeat count advance past the end of input */
+ if (inbuf > inbuf_end) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ for (i = 0; i < repeat_count; i++) {
+ /* Would this overflow the output buffer */
+ if ((outbuf + 8) > outbuf_end) {
+ ret = -ENOSPC;
+ goto out;
+ }
+
+ memcpy(outbuf, prevbuf, 8);
+ sw842_copy_to_fifo(outbuf, fifo);
+ outbuf += 8;
+ }
+ break;
+
+ case SW842_TMPL_ZEROS:
+ /* Would this overflow the output buffer */
+ if ((outbuf + 8) > outbuf_end) {
+ ret = -ENOSPC;
+ goto out;
+ }
+
+ memset(outbuf, 0, 8);
+ sw842_copy_to_fifo(outbuf, fifo);
+ outbuf += 8;
+ break;
+
+ default:
+ if (tmpl > 25) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* Does this go past the end of the input buffer */
+ if ((inbuf + 2) > inbuf_end) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* Would this overflow the output buffer */
+ if ((outbuf + 8) > outbuf_end) {
+ ret = -ENOSPC;
+ goto out;
+ }
+
+ while (opindex < 4 &&
+ (op = sw842_tmpl_ops[tmpl][opindex++])
+ != NULL) {
+ ret = (*op)(&inbuf, &inbit, &outbuf, fifo);
+ if (ret) {
+ ret = -EINVAL;
+ goto out;
+ }
+ sw842_copy_to_fifo(origbuf, fifo);
+ }
+ }
+ }
+
+out:
+ if (!ret)
+ *destlen = (unsigned int)(outbuf - dst);
+ else
+ *destlen = 0;
+
+ return ret;
+}
diff --git a/drivers/crypto/nx/nx-aes-cbc.c b/drivers/crypto/nx/nx-aes-cbc.c
new file mode 100644
index 000000000..a066cc345
--- /dev/null
+++ b/drivers/crypto/nx/nx-aes-cbc.c
@@ -0,0 +1,150 @@
+/**
+ * AES CBC routines supporting the Power 7+ Nest Accelerators driver
+ *
+ * Copyright (C) 2011-2012 International Business Machines Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 only.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Author: Kent Yoder <yoder1@us.ibm.com>
+ */
+
+#include <crypto/aes.h>
+#include <crypto/algapi.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/crypto.h>
+#include <asm/vio.h>
+
+#include "nx_csbcpb.h"
+#include "nx.h"
+
+
+static int cbc_aes_nx_set_key(struct crypto_tfm *tfm,
+ const u8 *in_key,
+ unsigned int key_len)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm);
+ struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+
+ nx_ctx_init(nx_ctx, HCOP_FC_AES);
+
+ switch (key_len) {
+ case AES_KEYSIZE_128:
+ NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128);
+ nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128];
+ break;
+ case AES_KEYSIZE_192:
+ NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_192);
+ nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_192];
+ break;
+ case AES_KEYSIZE_256:
+ NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_256);
+ nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_256];
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_CBC;
+ memcpy(csbcpb->cpb.aes_cbc.key, in_key, key_len);
+
+ return 0;
+}
+
+static int cbc_aes_nx_crypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst,
+ struct scatterlist *src,
+ unsigned int nbytes,
+ int enc)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_blkcipher_ctx(desc->tfm);
+ struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ unsigned long irq_flags;
+ unsigned int processed = 0, to_process;
+ int rc;
+
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+ if (enc)
+ NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
+ else
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
+
+ do {
+ to_process = nbytes - processed;
+
+ rc = nx_build_sg_lists(nx_ctx, desc, dst, src, &to_process,
+ processed, csbcpb->cpb.aes_cbc.iv);
+ if (rc)
+ goto out;
+
+ if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ memcpy(desc->info, csbcpb->cpb.aes_cbc.cv, AES_BLOCK_SIZE);
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(csbcpb->csb.processed_byte_count,
+ &(nx_ctx->stats->aes_bytes));
+
+ processed += to_process;
+ } while (processed < nbytes);
+out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+ return rc;
+}
+
+static int cbc_aes_nx_encrypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst,
+ struct scatterlist *src,
+ unsigned int nbytes)
+{
+ return cbc_aes_nx_crypt(desc, dst, src, nbytes, 1);
+}
+
+static int cbc_aes_nx_decrypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst,
+ struct scatterlist *src,
+ unsigned int nbytes)
+{
+ return cbc_aes_nx_crypt(desc, dst, src, nbytes, 0);
+}
+
+struct crypto_alg nx_cbc_aes_alg = {
+ .cra_name = "cbc(aes)",
+ .cra_driver_name = "cbc-aes-nx",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct nx_crypto_ctx),
+ .cra_type = &crypto_blkcipher_type,
+ .cra_alignmask = 0xf,
+ .cra_module = THIS_MODULE,
+ .cra_init = nx_crypto_ctx_aes_cbc_init,
+ .cra_exit = nx_crypto_ctx_exit,
+ .cra_blkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = cbc_aes_nx_set_key,
+ .encrypt = cbc_aes_nx_encrypt,
+ .decrypt = cbc_aes_nx_decrypt,
+ }
+};
diff --git a/drivers/crypto/nx/nx-aes-ccm.c b/drivers/crypto/nx/nx-aes-ccm.c
new file mode 100644
index 000000000..67f80813a
--- /dev/null
+++ b/drivers/crypto/nx/nx-aes-ccm.c
@@ -0,0 +1,604 @@
+/**
+ * AES CCM routines supporting the Power 7+ Nest Accelerators driver
+ *
+ * Copyright (C) 2012 International Business Machines Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 only.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Author: Kent Yoder <yoder1@us.ibm.com>
+ */
+
+#include <crypto/internal/aead.h>
+#include <crypto/aes.h>
+#include <crypto/algapi.h>
+#include <crypto/scatterwalk.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/crypto.h>
+#include <asm/vio.h>
+
+#include "nx_csbcpb.h"
+#include "nx.h"
+
+
+static int ccm_aes_nx_set_key(struct crypto_aead *tfm,
+ const u8 *in_key,
+ unsigned int key_len)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base);
+ struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead;
+
+ nx_ctx_init(nx_ctx, HCOP_FC_AES);
+
+ switch (key_len) {
+ case AES_KEYSIZE_128:
+ NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128);
+ NX_CPB_SET_KEY_SIZE(csbcpb_aead, NX_KS_AES_128);
+ nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128];
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_CCM;
+ memcpy(csbcpb->cpb.aes_ccm.key, in_key, key_len);
+
+ csbcpb_aead->cpb.hdr.mode = NX_MODE_AES_CCA;
+ memcpy(csbcpb_aead->cpb.aes_cca.key, in_key, key_len);
+
+ return 0;
+
+}
+
+static int ccm4309_aes_nx_set_key(struct crypto_aead *tfm,
+ const u8 *in_key,
+ unsigned int key_len)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base);
+
+ if (key_len < 3)
+ return -EINVAL;
+
+ key_len -= 3;
+
+ memcpy(nx_ctx->priv.ccm.nonce, in_key + key_len, 3);
+
+ return ccm_aes_nx_set_key(tfm, in_key, key_len);
+}
+
+static int ccm_aes_nx_setauthsize(struct crypto_aead *tfm,
+ unsigned int authsize)
+{
+ switch (authsize) {
+ case 4:
+ case 6:
+ case 8:
+ case 10:
+ case 12:
+ case 14:
+ case 16:
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ crypto_aead_crt(tfm)->authsize = authsize;
+
+ return 0;
+}
+
+static int ccm4309_aes_nx_setauthsize(struct crypto_aead *tfm,
+ unsigned int authsize)
+{
+ switch (authsize) {
+ case 8:
+ case 12:
+ case 16:
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ crypto_aead_crt(tfm)->authsize = authsize;
+
+ return 0;
+}
+
+/* taken from crypto/ccm.c */
+static int set_msg_len(u8 *block, unsigned int msglen, int csize)
+{
+ __be32 data;
+
+ memset(block, 0, csize);
+ block += csize;
+
+ if (csize >= 4)
+ csize = 4;
+ else if (msglen > (unsigned int)(1 << (8 * csize)))
+ return -EOVERFLOW;
+
+ data = cpu_to_be32(msglen);
+ memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
+
+ return 0;
+}
+
+/* taken from crypto/ccm.c */
+static inline int crypto_ccm_check_iv(const u8 *iv)
+{
+ /* 2 <= L <= 8, so 1 <= L' <= 7. */
+ if (1 > iv[0] || iv[0] > 7)
+ return -EINVAL;
+
+ return 0;
+}
+
+/* based on code from crypto/ccm.c */
+static int generate_b0(u8 *iv, unsigned int assoclen, unsigned int authsize,
+ unsigned int cryptlen, u8 *b0)
+{
+ unsigned int l, lp, m = authsize;
+ int rc;
+
+ memcpy(b0, iv, 16);
+
+ lp = b0[0];
+ l = lp + 1;
+
+ /* set m, bits 3-5 */
+ *b0 |= (8 * ((m - 2) / 2));
+
+ /* set adata, bit 6, if associated data is used */
+ if (assoclen)
+ *b0 |= 64;
+
+ rc = set_msg_len(b0 + 16 - l, cryptlen, l);
+
+ return rc;
+}
+
+static int generate_pat(u8 *iv,
+ struct aead_request *req,
+ struct nx_crypto_ctx *nx_ctx,
+ unsigned int authsize,
+ unsigned int nbytes,
+ u8 *out)
+{
+ struct nx_sg *nx_insg = nx_ctx->in_sg;
+ struct nx_sg *nx_outsg = nx_ctx->out_sg;
+ unsigned int iauth_len = 0;
+ u8 tmp[16], *b1 = NULL, *b0 = NULL, *result = NULL;
+ int rc;
+ unsigned int max_sg_len;
+
+ /* zero the ctr value */
+ memset(iv + 15 - iv[0], 0, iv[0] + 1);
+
+ /* page 78 of nx_wb.pdf has,
+ * Note: RFC3610 allows the AAD data to be up to 2^64 -1 bytes
+ * in length. If a full message is used, the AES CCA implementation
+ * restricts the maximum AAD length to 2^32 -1 bytes.
+ * If partial messages are used, the implementation supports
+ * 2^64 -1 bytes maximum AAD length.
+ *
+ * However, in the cryptoapi's aead_request structure,
+ * assoclen is an unsigned int, thus it cannot hold a length
+ * value greater than 2^32 - 1.
+ * Thus the AAD is further constrained by this and is never
+ * greater than 2^32.
+ */
+
+ if (!req->assoclen) {
+ b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0;
+ } else if (req->assoclen <= 14) {
+ /* if associated data is 14 bytes or less, we do 1 GCM
+ * operation on 2 AES blocks, B0 (stored in the csbcpb) and B1,
+ * which is fed in through the source buffers here */
+ b0 = nx_ctx->csbcpb->cpb.aes_ccm.in_pat_or_b0;
+ b1 = nx_ctx->priv.ccm.iauth_tag;
+ iauth_len = req->assoclen;
+ } else if (req->assoclen <= 65280) {
+ /* if associated data is less than (2^16 - 2^8), we construct
+ * B1 differently and feed in the associated data to a CCA
+ * operation */
+ b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0;
+ b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1;
+ iauth_len = 14;
+ } else {
+ b0 = nx_ctx->csbcpb_aead->cpb.aes_cca.b0;
+ b1 = nx_ctx->csbcpb_aead->cpb.aes_cca.b1;
+ iauth_len = 10;
+ }
+
+ /* generate B0 */
+ rc = generate_b0(iv, req->assoclen, authsize, nbytes, b0);
+ if (rc)
+ return rc;
+
+ /* generate B1:
+ * add control info for associated data
+ * RFC 3610 and NIST Special Publication 800-38C
+ */
+ if (b1) {
+ memset(b1, 0, 16);
+ if (req->assoclen <= 65280) {
+ *(u16 *)b1 = (u16)req->assoclen;
+ scatterwalk_map_and_copy(b1 + 2, req->assoc, 0,
+ iauth_len, SCATTERWALK_FROM_SG);
+ } else {
+ *(u16 *)b1 = (u16)(0xfffe);
+ *(u32 *)&b1[2] = (u32)req->assoclen;
+ scatterwalk_map_and_copy(b1 + 6, req->assoc, 0,
+ iauth_len, SCATTERWALK_FROM_SG);
+ }
+ }
+
+ /* now copy any remaining AAD to scatterlist and call nx... */
+ if (!req->assoclen) {
+ return rc;
+ } else if (req->assoclen <= 14) {
+ unsigned int len = 16;
+
+ nx_insg = nx_build_sg_list(nx_insg, b1, &len, nx_ctx->ap->sglen);
+
+ if (len != 16)
+ return -EINVAL;
+
+ nx_outsg = nx_build_sg_list(nx_outsg, tmp, &len,
+ nx_ctx->ap->sglen);
+
+ if (len != 16)
+ return -EINVAL;
+
+ /* inlen should be negative, indicating to phyp that its a
+ * pointer to an sg list */
+ nx_ctx->op.inlen = (nx_ctx->in_sg - nx_insg) *
+ sizeof(struct nx_sg);
+ nx_ctx->op.outlen = (nx_ctx->out_sg - nx_outsg) *
+ sizeof(struct nx_sg);
+
+ NX_CPB_FDM(nx_ctx->csbcpb) |= NX_FDM_ENDE_ENCRYPT;
+ NX_CPB_FDM(nx_ctx->csbcpb) |= NX_FDM_INTERMEDIATE;
+
+ result = nx_ctx->csbcpb->cpb.aes_ccm.out_pat_or_mac;
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ return rc;
+
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes));
+
+ } else {
+ unsigned int processed = 0, to_process;
+
+ processed += iauth_len;
+
+ /* page_limit: number of sg entries that fit on one page */
+ max_sg_len = min_t(u64, nx_ctx->ap->sglen,
+ nx_driver.of.max_sg_len/sizeof(struct nx_sg));
+ max_sg_len = min_t(u64, max_sg_len,
+ nx_ctx->ap->databytelen/NX_PAGE_SIZE);
+
+ do {
+ to_process = min_t(u32, req->assoclen - processed,
+ nx_ctx->ap->databytelen);
+
+ nx_insg = nx_walk_and_build(nx_ctx->in_sg,
+ nx_ctx->ap->sglen,
+ req->assoc, processed,
+ &to_process);
+
+ if ((to_process + processed) < req->assoclen) {
+ NX_CPB_FDM(nx_ctx->csbcpb_aead) |=
+ NX_FDM_INTERMEDIATE;
+ } else {
+ NX_CPB_FDM(nx_ctx->csbcpb_aead) &=
+ ~NX_FDM_INTERMEDIATE;
+ }
+
+
+ nx_ctx->op_aead.inlen = (nx_ctx->in_sg - nx_insg) *
+ sizeof(struct nx_sg);
+
+ result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0;
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op_aead,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ return rc;
+
+ memcpy(nx_ctx->csbcpb_aead->cpb.aes_cca.b0,
+ nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0,
+ AES_BLOCK_SIZE);
+
+ NX_CPB_FDM(nx_ctx->csbcpb_aead) |= NX_FDM_CONTINUATION;
+
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(req->assoclen,
+ &(nx_ctx->stats->aes_bytes));
+
+ processed += to_process;
+ } while (processed < req->assoclen);
+
+ result = nx_ctx->csbcpb_aead->cpb.aes_cca.out_pat_or_b0;
+ }
+
+ memcpy(out, result, AES_BLOCK_SIZE);
+
+ return rc;
+}
+
+static int ccm_nx_decrypt(struct aead_request *req,
+ struct blkcipher_desc *desc)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
+ struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ unsigned int nbytes = req->cryptlen;
+ unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
+ struct nx_ccm_priv *priv = &nx_ctx->priv.ccm;
+ unsigned long irq_flags;
+ unsigned int processed = 0, to_process;
+ int rc = -1;
+
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+ nbytes -= authsize;
+
+ /* copy out the auth tag to compare with later */
+ scatterwalk_map_and_copy(priv->oauth_tag,
+ req->src, nbytes, authsize,
+ SCATTERWALK_FROM_SG);
+
+ rc = generate_pat(desc->info, req, nx_ctx, authsize, nbytes,
+ csbcpb->cpb.aes_ccm.in_pat_or_b0);
+ if (rc)
+ goto out;
+
+ do {
+
+ /* to_process: the AES_BLOCK_SIZE data chunk to process in this
+ * update. This value is bound by sg list limits.
+ */
+ to_process = nbytes - processed;
+
+ if ((to_process + processed) < nbytes)
+ NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ else
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+
+ NX_CPB_FDM(nx_ctx->csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
+
+ rc = nx_build_sg_lists(nx_ctx, desc, req->dst, req->src,
+ &to_process, processed,
+ csbcpb->cpb.aes_ccm.iv_or_ctr);
+ if (rc)
+ goto out;
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ /* for partial completion, copy following for next
+ * entry into loop...
+ */
+ memcpy(desc->info, csbcpb->cpb.aes_ccm.out_ctr, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_ccm.in_pat_or_b0,
+ csbcpb->cpb.aes_ccm.out_pat_or_mac, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_ccm.in_s0,
+ csbcpb->cpb.aes_ccm.out_s0, AES_BLOCK_SIZE);
+
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+ /* update stats */
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(csbcpb->csb.processed_byte_count,
+ &(nx_ctx->stats->aes_bytes));
+
+ processed += to_process;
+ } while (processed < nbytes);
+
+ rc = memcmp(csbcpb->cpb.aes_ccm.out_pat_or_mac, priv->oauth_tag,
+ authsize) ? -EBADMSG : 0;
+out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+ return rc;
+}
+
+static int ccm_nx_encrypt(struct aead_request *req,
+ struct blkcipher_desc *desc)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
+ struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ unsigned int nbytes = req->cryptlen;
+ unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
+ unsigned long irq_flags;
+ unsigned int processed = 0, to_process;
+ int rc = -1;
+
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+ rc = generate_pat(desc->info, req, nx_ctx, authsize, nbytes,
+ csbcpb->cpb.aes_ccm.in_pat_or_b0);
+ if (rc)
+ goto out;
+
+ do {
+ /* to process: the AES_BLOCK_SIZE data chunk to process in this
+ * update. This value is bound by sg list limits.
+ */
+ to_process = nbytes - processed;
+
+ if ((to_process + processed) < nbytes)
+ NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ else
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+
+ NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
+
+ rc = nx_build_sg_lists(nx_ctx, desc, req->dst, req->src,
+ &to_process, processed,
+ csbcpb->cpb.aes_ccm.iv_or_ctr);
+ if (rc)
+ goto out;
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ /* for partial completion, copy following for next
+ * entry into loop...
+ */
+ memcpy(desc->info, csbcpb->cpb.aes_ccm.out_ctr, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_ccm.in_pat_or_b0,
+ csbcpb->cpb.aes_ccm.out_pat_or_mac, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_ccm.in_s0,
+ csbcpb->cpb.aes_ccm.out_s0, AES_BLOCK_SIZE);
+
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+ /* update stats */
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(csbcpb->csb.processed_byte_count,
+ &(nx_ctx->stats->aes_bytes));
+
+ processed += to_process;
+
+ } while (processed < nbytes);
+
+ /* copy out the auth tag */
+ scatterwalk_map_and_copy(csbcpb->cpb.aes_ccm.out_pat_or_mac,
+ req->dst, nbytes, authsize,
+ SCATTERWALK_TO_SG);
+
+out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+ return rc;
+}
+
+static int ccm4309_aes_nx_encrypt(struct aead_request *req)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
+ struct blkcipher_desc desc;
+ u8 *iv = nx_ctx->priv.ccm.iv;
+
+ iv[0] = 3;
+ memcpy(iv + 1, nx_ctx->priv.ccm.nonce, 3);
+ memcpy(iv + 4, req->iv, 8);
+
+ desc.info = iv;
+ desc.tfm = (struct crypto_blkcipher *)req->base.tfm;
+
+ return ccm_nx_encrypt(req, &desc);
+}
+
+static int ccm_aes_nx_encrypt(struct aead_request *req)
+{
+ struct blkcipher_desc desc;
+ int rc;
+
+ desc.info = req->iv;
+ desc.tfm = (struct crypto_blkcipher *)req->base.tfm;
+
+ rc = crypto_ccm_check_iv(desc.info);
+ if (rc)
+ return rc;
+
+ return ccm_nx_encrypt(req, &desc);
+}
+
+static int ccm4309_aes_nx_decrypt(struct aead_request *req)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
+ struct blkcipher_desc desc;
+ u8 *iv = nx_ctx->priv.ccm.iv;
+
+ iv[0] = 3;
+ memcpy(iv + 1, nx_ctx->priv.ccm.nonce, 3);
+ memcpy(iv + 4, req->iv, 8);
+
+ desc.info = iv;
+ desc.tfm = (struct crypto_blkcipher *)req->base.tfm;
+
+ return ccm_nx_decrypt(req, &desc);
+}
+
+static int ccm_aes_nx_decrypt(struct aead_request *req)
+{
+ struct blkcipher_desc desc;
+ int rc;
+
+ desc.info = req->iv;
+ desc.tfm = (struct crypto_blkcipher *)req->base.tfm;
+
+ rc = crypto_ccm_check_iv(desc.info);
+ if (rc)
+ return rc;
+
+ return ccm_nx_decrypt(req, &desc);
+}
+
+/* tell the block cipher walk routines that this is a stream cipher by
+ * setting cra_blocksize to 1. Even using blkcipher_walk_virt_block
+ * during encrypt/decrypt doesn't solve this problem, because it calls
+ * blkcipher_walk_done under the covers, which doesn't use walk->blocksize,
+ * but instead uses this tfm->blocksize. */
+struct crypto_alg nx_ccm_aes_alg = {
+ .cra_name = "ccm(aes)",
+ .cra_driver_name = "ccm-aes-nx",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_TYPE_AEAD |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct nx_crypto_ctx),
+ .cra_type = &crypto_aead_type,
+ .cra_module = THIS_MODULE,
+ .cra_init = nx_crypto_ctx_aes_ccm_init,
+ .cra_exit = nx_crypto_ctx_exit,
+ .cra_aead = {
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = AES_BLOCK_SIZE,
+ .setkey = ccm_aes_nx_set_key,
+ .setauthsize = ccm_aes_nx_setauthsize,
+ .encrypt = ccm_aes_nx_encrypt,
+ .decrypt = ccm_aes_nx_decrypt,
+ }
+};
+
+struct crypto_alg nx_ccm4309_aes_alg = {
+ .cra_name = "rfc4309(ccm(aes))",
+ .cra_driver_name = "rfc4309-ccm-aes-nx",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_TYPE_AEAD |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct nx_crypto_ctx),
+ .cra_type = &crypto_nivaead_type,
+ .cra_module = THIS_MODULE,
+ .cra_init = nx_crypto_ctx_aes_ccm_init,
+ .cra_exit = nx_crypto_ctx_exit,
+ .cra_aead = {
+ .ivsize = 8,
+ .maxauthsize = AES_BLOCK_SIZE,
+ .setkey = ccm4309_aes_nx_set_key,
+ .setauthsize = ccm4309_aes_nx_setauthsize,
+ .encrypt = ccm4309_aes_nx_encrypt,
+ .decrypt = ccm4309_aes_nx_decrypt,
+ .geniv = "seqiv",
+ }
+};
diff --git a/drivers/crypto/nx/nx-aes-ctr.c b/drivers/crypto/nx/nx-aes-ctr.c
new file mode 100644
index 000000000..2617cd4d5
--- /dev/null
+++ b/drivers/crypto/nx/nx-aes-ctr.c
@@ -0,0 +1,187 @@
+/**
+ * AES CTR routines supporting the Power 7+ Nest Accelerators driver
+ *
+ * Copyright (C) 2011-2012 International Business Machines Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 only.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Author: Kent Yoder <yoder1@us.ibm.com>
+ */
+
+#include <crypto/aes.h>
+#include <crypto/ctr.h>
+#include <crypto/algapi.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/crypto.h>
+#include <asm/vio.h>
+
+#include "nx_csbcpb.h"
+#include "nx.h"
+
+
+static int ctr_aes_nx_set_key(struct crypto_tfm *tfm,
+ const u8 *in_key,
+ unsigned int key_len)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm);
+ struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+
+ nx_ctx_init(nx_ctx, HCOP_FC_AES);
+
+ switch (key_len) {
+ case AES_KEYSIZE_128:
+ NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128);
+ nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128];
+ break;
+ case AES_KEYSIZE_192:
+ NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_192);
+ nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_192];
+ break;
+ case AES_KEYSIZE_256:
+ NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_256);
+ nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_256];
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_CTR;
+ memcpy(csbcpb->cpb.aes_ctr.key, in_key, key_len);
+
+ return 0;
+}
+
+static int ctr3686_aes_nx_set_key(struct crypto_tfm *tfm,
+ const u8 *in_key,
+ unsigned int key_len)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm);
+
+ if (key_len < CTR_RFC3686_NONCE_SIZE)
+ return -EINVAL;
+
+ memcpy(nx_ctx->priv.ctr.iv,
+ in_key + key_len - CTR_RFC3686_NONCE_SIZE,
+ CTR_RFC3686_NONCE_SIZE);
+
+ key_len -= CTR_RFC3686_NONCE_SIZE;
+
+ return ctr_aes_nx_set_key(tfm, in_key, key_len);
+}
+
+static int ctr_aes_nx_crypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst,
+ struct scatterlist *src,
+ unsigned int nbytes)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_blkcipher_ctx(desc->tfm);
+ struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ unsigned long irq_flags;
+ unsigned int processed = 0, to_process;
+ int rc;
+
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+ do {
+ to_process = nbytes - processed;
+
+ rc = nx_build_sg_lists(nx_ctx, desc, dst, src, &to_process,
+ processed, csbcpb->cpb.aes_ctr.iv);
+ if (rc)
+ goto out;
+
+ if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ memcpy(desc->info, csbcpb->cpb.aes_cbc.cv, AES_BLOCK_SIZE);
+
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(csbcpb->csb.processed_byte_count,
+ &(nx_ctx->stats->aes_bytes));
+
+ processed += to_process;
+ } while (processed < nbytes);
+out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+ return rc;
+}
+
+static int ctr3686_aes_nx_crypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst,
+ struct scatterlist *src,
+ unsigned int nbytes)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_blkcipher_ctx(desc->tfm);
+ u8 *iv = nx_ctx->priv.ctr.iv;
+
+ memcpy(iv + CTR_RFC3686_NONCE_SIZE,
+ desc->info, CTR_RFC3686_IV_SIZE);
+ iv[12] = iv[13] = iv[14] = 0;
+ iv[15] = 1;
+
+ desc->info = nx_ctx->priv.ctr.iv;
+
+ return ctr_aes_nx_crypt(desc, dst, src, nbytes);
+}
+
+struct crypto_alg nx_ctr_aes_alg = {
+ .cra_name = "ctr(aes)",
+ .cra_driver_name = "ctr-aes-nx",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct nx_crypto_ctx),
+ .cra_type = &crypto_blkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_init = nx_crypto_ctx_aes_ctr_init,
+ .cra_exit = nx_crypto_ctx_exit,
+ .cra_blkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = ctr_aes_nx_set_key,
+ .encrypt = ctr_aes_nx_crypt,
+ .decrypt = ctr_aes_nx_crypt,
+ }
+};
+
+struct crypto_alg nx_ctr3686_aes_alg = {
+ .cra_name = "rfc3686(ctr(aes))",
+ .cra_driver_name = "rfc3686-ctr-aes-nx",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct nx_crypto_ctx),
+ .cra_type = &crypto_blkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_init = nx_crypto_ctx_aes_ctr_init,
+ .cra_exit = nx_crypto_ctx_exit,
+ .cra_blkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
+ .ivsize = CTR_RFC3686_IV_SIZE,
+ .geniv = "seqiv",
+ .setkey = ctr3686_aes_nx_set_key,
+ .encrypt = ctr3686_aes_nx_crypt,
+ .decrypt = ctr3686_aes_nx_crypt,
+ }
+};
diff --git a/drivers/crypto/nx/nx-aes-ecb.c b/drivers/crypto/nx/nx-aes-ecb.c
new file mode 100644
index 000000000..cfdde8b8b
--- /dev/null
+++ b/drivers/crypto/nx/nx-aes-ecb.c
@@ -0,0 +1,149 @@
+/**
+ * AES ECB routines supporting the Power 7+ Nest Accelerators driver
+ *
+ * Copyright (C) 2011-2012 International Business Machines Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 only.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Author: Kent Yoder <yoder1@us.ibm.com>
+ */
+
+#include <crypto/aes.h>
+#include <crypto/algapi.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/crypto.h>
+#include <asm/vio.h>
+
+#include "nx_csbcpb.h"
+#include "nx.h"
+
+
+static int ecb_aes_nx_set_key(struct crypto_tfm *tfm,
+ const u8 *in_key,
+ unsigned int key_len)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm);
+ struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
+
+ nx_ctx_init(nx_ctx, HCOP_FC_AES);
+
+ switch (key_len) {
+ case AES_KEYSIZE_128:
+ NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128);
+ nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128];
+ break;
+ case AES_KEYSIZE_192:
+ NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_192);
+ nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_192];
+ break;
+ case AES_KEYSIZE_256:
+ NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_256);
+ nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_256];
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_ECB;
+ memcpy(csbcpb->cpb.aes_ecb.key, in_key, key_len);
+
+ return 0;
+}
+
+static int ecb_aes_nx_crypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst,
+ struct scatterlist *src,
+ unsigned int nbytes,
+ int enc)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_blkcipher_ctx(desc->tfm);
+ struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ unsigned long irq_flags;
+ unsigned int processed = 0, to_process;
+ int rc;
+
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+ if (enc)
+ NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
+ else
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
+
+ do {
+ to_process = nbytes - processed;
+
+ rc = nx_build_sg_lists(nx_ctx, desc, dst, src, &to_process,
+ processed, NULL);
+ if (rc)
+ goto out;
+
+ if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(csbcpb->csb.processed_byte_count,
+ &(nx_ctx->stats->aes_bytes));
+
+ processed += to_process;
+ } while (processed < nbytes);
+
+out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+ return rc;
+}
+
+static int ecb_aes_nx_encrypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst,
+ struct scatterlist *src,
+ unsigned int nbytes)
+{
+ return ecb_aes_nx_crypt(desc, dst, src, nbytes, 1);
+}
+
+static int ecb_aes_nx_decrypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst,
+ struct scatterlist *src,
+ unsigned int nbytes)
+{
+ return ecb_aes_nx_crypt(desc, dst, src, nbytes, 0);
+}
+
+struct crypto_alg nx_ecb_aes_alg = {
+ .cra_name = "ecb(aes)",
+ .cra_driver_name = "ecb-aes-nx",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_alignmask = 0xf,
+ .cra_ctxsize = sizeof(struct nx_crypto_ctx),
+ .cra_type = &crypto_blkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_init = nx_crypto_ctx_aes_ecb_init,
+ .cra_exit = nx_crypto_ctx_exit,
+ .cra_blkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .setkey = ecb_aes_nx_set_key,
+ .encrypt = ecb_aes_nx_encrypt,
+ .decrypt = ecb_aes_nx_decrypt,
+ }
+};
diff --git a/drivers/crypto/nx/nx-aes-gcm.c b/drivers/crypto/nx/nx-aes-gcm.c
new file mode 100644
index 000000000..88c562434
--- /dev/null
+++ b/drivers/crypto/nx/nx-aes-gcm.c
@@ -0,0 +1,525 @@
+/**
+ * AES GCM routines supporting the Power 7+ Nest Accelerators driver
+ *
+ * Copyright (C) 2012 International Business Machines Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 only.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Author: Kent Yoder <yoder1@us.ibm.com>
+ */
+
+#include <crypto/internal/aead.h>
+#include <crypto/aes.h>
+#include <crypto/algapi.h>
+#include <crypto/scatterwalk.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/crypto.h>
+#include <asm/vio.h>
+
+#include "nx_csbcpb.h"
+#include "nx.h"
+
+
+static int gcm_aes_nx_set_key(struct crypto_aead *tfm,
+ const u8 *in_key,
+ unsigned int key_len)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base);
+ struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead;
+
+ nx_ctx_init(nx_ctx, HCOP_FC_AES);
+
+ switch (key_len) {
+ case AES_KEYSIZE_128:
+ NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128);
+ NX_CPB_SET_KEY_SIZE(csbcpb_aead, NX_KS_AES_128);
+ nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128];
+ break;
+ case AES_KEYSIZE_192:
+ NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_192);
+ NX_CPB_SET_KEY_SIZE(csbcpb_aead, NX_KS_AES_192);
+ nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_192];
+ break;
+ case AES_KEYSIZE_256:
+ NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_256);
+ NX_CPB_SET_KEY_SIZE(csbcpb_aead, NX_KS_AES_256);
+ nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_256];
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_GCM;
+ memcpy(csbcpb->cpb.aes_gcm.key, in_key, key_len);
+
+ csbcpb_aead->cpb.hdr.mode = NX_MODE_AES_GCA;
+ memcpy(csbcpb_aead->cpb.aes_gca.key, in_key, key_len);
+
+ return 0;
+}
+
+static int gcm4106_aes_nx_set_key(struct crypto_aead *tfm,
+ const u8 *in_key,
+ unsigned int key_len)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&tfm->base);
+ char *nonce = nx_ctx->priv.gcm.nonce;
+ int rc;
+
+ if (key_len < 4)
+ return -EINVAL;
+
+ key_len -= 4;
+
+ rc = gcm_aes_nx_set_key(tfm, in_key, key_len);
+ if (rc)
+ goto out;
+
+ memcpy(nonce, in_key + key_len, 4);
+out:
+ return rc;
+}
+
+static int gcm_aes_nx_setauthsize(struct crypto_aead *tfm,
+ unsigned int authsize)
+{
+ if (authsize > crypto_aead_alg(tfm)->maxauthsize)
+ return -EINVAL;
+
+ crypto_aead_crt(tfm)->authsize = authsize;
+
+ return 0;
+}
+
+static int gcm4106_aes_nx_setauthsize(struct crypto_aead *tfm,
+ unsigned int authsize)
+{
+ switch (authsize) {
+ case 8:
+ case 12:
+ case 16:
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ crypto_aead_crt(tfm)->authsize = authsize;
+
+ return 0;
+}
+
+static int nx_gca(struct nx_crypto_ctx *nx_ctx,
+ struct aead_request *req,
+ u8 *out)
+{
+ int rc;
+ struct nx_csbcpb *csbcpb_aead = nx_ctx->csbcpb_aead;
+ struct scatter_walk walk;
+ struct nx_sg *nx_sg = nx_ctx->in_sg;
+ unsigned int nbytes = req->assoclen;
+ unsigned int processed = 0, to_process;
+ unsigned int max_sg_len;
+
+ if (nbytes <= AES_BLOCK_SIZE) {
+ scatterwalk_start(&walk, req->assoc);
+ scatterwalk_copychunks(out, &walk, nbytes, SCATTERWALK_FROM_SG);
+ scatterwalk_done(&walk, SCATTERWALK_FROM_SG, 0);
+ return 0;
+ }
+
+ NX_CPB_FDM(csbcpb_aead) &= ~NX_FDM_CONTINUATION;
+
+ /* page_limit: number of sg entries that fit on one page */
+ max_sg_len = min_t(u64, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
+ max_sg_len = min_t(u64, max_sg_len,
+ nx_ctx->ap->databytelen/NX_PAGE_SIZE);
+
+ do {
+ /*
+ * to_process: the data chunk to process in this update.
+ * This value is bound by sg list limits.
+ */
+ to_process = min_t(u64, nbytes - processed,
+ nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+
+ nx_sg = nx_walk_and_build(nx_ctx->in_sg, max_sg_len,
+ req->assoc, processed, &to_process);
+
+ if ((to_process + processed) < nbytes)
+ NX_CPB_FDM(csbcpb_aead) |= NX_FDM_INTERMEDIATE;
+ else
+ NX_CPB_FDM(csbcpb_aead) &= ~NX_FDM_INTERMEDIATE;
+
+ nx_ctx->op_aead.inlen = (nx_ctx->in_sg - nx_sg)
+ * sizeof(struct nx_sg);
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op_aead,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ return rc;
+
+ memcpy(csbcpb_aead->cpb.aes_gca.in_pat,
+ csbcpb_aead->cpb.aes_gca.out_pat,
+ AES_BLOCK_SIZE);
+ NX_CPB_FDM(csbcpb_aead) |= NX_FDM_CONTINUATION;
+
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes));
+
+ processed += to_process;
+ } while (processed < nbytes);
+
+ memcpy(out, csbcpb_aead->cpb.aes_gca.out_pat, AES_BLOCK_SIZE);
+
+ return rc;
+}
+
+static int gmac(struct aead_request *req, struct blkcipher_desc *desc)
+{
+ int rc;
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
+ struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ struct nx_sg *nx_sg;
+ unsigned int nbytes = req->assoclen;
+ unsigned int processed = 0, to_process;
+ unsigned int max_sg_len;
+
+ /* Set GMAC mode */
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_GMAC;
+
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_CONTINUATION;
+
+ /* page_limit: number of sg entries that fit on one page */
+ max_sg_len = min_t(u64, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
+ max_sg_len = min_t(u64, max_sg_len,
+ nx_ctx->ap->databytelen/NX_PAGE_SIZE);
+
+ /* Copy IV */
+ memcpy(csbcpb->cpb.aes_gcm.iv_or_cnt, desc->info, AES_BLOCK_SIZE);
+
+ do {
+ /*
+ * to_process: the data chunk to process in this update.
+ * This value is bound by sg list limits.
+ */
+ to_process = min_t(u64, nbytes - processed,
+ nx_ctx->ap->databytelen);
+ to_process = min_t(u64, to_process,
+ NX_PAGE_SIZE * (max_sg_len - 1));
+
+ nx_sg = nx_walk_and_build(nx_ctx->in_sg, max_sg_len,
+ req->assoc, processed, &to_process);
+
+ if ((to_process + processed) < nbytes)
+ NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ else
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+
+ nx_ctx->op.inlen = (nx_ctx->in_sg - nx_sg)
+ * sizeof(struct nx_sg);
+
+ csbcpb->cpb.aes_gcm.bit_length_data = 0;
+ csbcpb->cpb.aes_gcm.bit_length_aad = 8 * nbytes;
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ memcpy(csbcpb->cpb.aes_gcm.in_pat_or_aad,
+ csbcpb->cpb.aes_gcm.out_pat_or_mac, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_gcm.in_s0,
+ csbcpb->cpb.aes_gcm.out_s0, AES_BLOCK_SIZE);
+
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(req->assoclen, &(nx_ctx->stats->aes_bytes));
+
+ processed += to_process;
+ } while (processed < nbytes);
+
+out:
+ /* Restore GCM mode */
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_GCM;
+ return rc;
+}
+
+static int gcm_empty(struct aead_request *req, struct blkcipher_desc *desc,
+ int enc)
+{
+ int rc;
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
+ struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ char out[AES_BLOCK_SIZE];
+ struct nx_sg *in_sg, *out_sg;
+ int len;
+
+ /* For scenarios where the input message is zero length, AES CTR mode
+ * may be used. Set the source data to be a single block (16B) of all
+ * zeros, and set the input IV value to be the same as the GMAC IV
+ * value. - nx_wb 4.8.1.3 */
+
+ /* Change to ECB mode */
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_ECB;
+ memcpy(csbcpb->cpb.aes_ecb.key, csbcpb->cpb.aes_gcm.key,
+ sizeof(csbcpb->cpb.aes_ecb.key));
+ if (enc)
+ NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
+ else
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
+
+ len = AES_BLOCK_SIZE;
+
+ /* Encrypt the counter/IV */
+ in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *) desc->info,
+ &len, nx_ctx->ap->sglen);
+
+ if (len != AES_BLOCK_SIZE)
+ return -EINVAL;
+
+ len = sizeof(out);
+ out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *) out, &len,
+ nx_ctx->ap->sglen);
+
+ if (len != sizeof(out))
+ return -EINVAL;
+
+ nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
+ nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+
+ /* Copy out the auth tag */
+ memcpy(csbcpb->cpb.aes_gcm.out_pat_or_mac, out,
+ crypto_aead_authsize(crypto_aead_reqtfm(req)));
+out:
+ /* Restore XCBC mode */
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_GCM;
+
+ /*
+ * ECB key uses the same region that GCM AAD and counter, so it's safe
+ * to just fill it with zeroes.
+ */
+ memset(csbcpb->cpb.aes_ecb.key, 0, sizeof(csbcpb->cpb.aes_ecb.key));
+
+ return rc;
+}
+
+static int gcm_aes_nx_crypt(struct aead_request *req, int enc)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
+ struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ struct blkcipher_desc desc;
+ unsigned int nbytes = req->cryptlen;
+ unsigned int processed = 0, to_process;
+ unsigned long irq_flags;
+ int rc = -EINVAL;
+
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+ desc.info = nx_ctx->priv.gcm.iv;
+ /* initialize the counter */
+ *(u32 *)(desc.info + NX_GCM_CTR_OFFSET) = 1;
+
+ if (nbytes == 0) {
+ if (req->assoclen == 0)
+ rc = gcm_empty(req, &desc, enc);
+ else
+ rc = gmac(req, &desc);
+ if (rc)
+ goto out;
+ else
+ goto mac;
+ }
+
+ /* Process associated data */
+ csbcpb->cpb.aes_gcm.bit_length_aad = req->assoclen * 8;
+ if (req->assoclen) {
+ rc = nx_gca(nx_ctx, req, csbcpb->cpb.aes_gcm.in_pat_or_aad);
+ if (rc)
+ goto out;
+ }
+
+ /* Set flags for encryption */
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_CONTINUATION;
+ if (enc) {
+ NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
+ } else {
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
+ nbytes -= crypto_aead_authsize(crypto_aead_reqtfm(req));
+ }
+
+ do {
+ to_process = nbytes - processed;
+
+ csbcpb->cpb.aes_gcm.bit_length_data = nbytes * 8;
+ desc.tfm = (struct crypto_blkcipher *) req->base.tfm;
+ rc = nx_build_sg_lists(nx_ctx, &desc, req->dst,
+ req->src, &to_process, processed,
+ csbcpb->cpb.aes_gcm.iv_or_cnt);
+
+ if (rc)
+ goto out;
+
+ if ((to_process + processed) < nbytes)
+ NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ else
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ memcpy(desc.info, csbcpb->cpb.aes_gcm.out_cnt, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_gcm.in_pat_or_aad,
+ csbcpb->cpb.aes_gcm.out_pat_or_mac, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_gcm.in_s0,
+ csbcpb->cpb.aes_gcm.out_s0, AES_BLOCK_SIZE);
+
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+ atomic64_add(csbcpb->csb.processed_byte_count,
+ &(nx_ctx->stats->aes_bytes));
+
+ processed += to_process;
+ } while (processed < nbytes);
+
+mac:
+ if (enc) {
+ /* copy out the auth tag */
+ scatterwalk_map_and_copy(csbcpb->cpb.aes_gcm.out_pat_or_mac,
+ req->dst, nbytes,
+ crypto_aead_authsize(crypto_aead_reqtfm(req)),
+ SCATTERWALK_TO_SG);
+ } else {
+ u8 *itag = nx_ctx->priv.gcm.iauth_tag;
+ u8 *otag = csbcpb->cpb.aes_gcm.out_pat_or_mac;
+
+ scatterwalk_map_and_copy(itag, req->src, nbytes,
+ crypto_aead_authsize(crypto_aead_reqtfm(req)),
+ SCATTERWALK_FROM_SG);
+ rc = memcmp(itag, otag,
+ crypto_aead_authsize(crypto_aead_reqtfm(req))) ?
+ -EBADMSG : 0;
+ }
+out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+ return rc;
+}
+
+static int gcm_aes_nx_encrypt(struct aead_request *req)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
+ char *iv = nx_ctx->priv.gcm.iv;
+
+ memcpy(iv, req->iv, 12);
+
+ return gcm_aes_nx_crypt(req, 1);
+}
+
+static int gcm_aes_nx_decrypt(struct aead_request *req)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
+ char *iv = nx_ctx->priv.gcm.iv;
+
+ memcpy(iv, req->iv, 12);
+
+ return gcm_aes_nx_crypt(req, 0);
+}
+
+static int gcm4106_aes_nx_encrypt(struct aead_request *req)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
+ char *iv = nx_ctx->priv.gcm.iv;
+ char *nonce = nx_ctx->priv.gcm.nonce;
+
+ memcpy(iv, nonce, NX_GCM4106_NONCE_LEN);
+ memcpy(iv + NX_GCM4106_NONCE_LEN, req->iv, 8);
+
+ return gcm_aes_nx_crypt(req, 1);
+}
+
+static int gcm4106_aes_nx_decrypt(struct aead_request *req)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(req->base.tfm);
+ char *iv = nx_ctx->priv.gcm.iv;
+ char *nonce = nx_ctx->priv.gcm.nonce;
+
+ memcpy(iv, nonce, NX_GCM4106_NONCE_LEN);
+ memcpy(iv + NX_GCM4106_NONCE_LEN, req->iv, 8);
+
+ return gcm_aes_nx_crypt(req, 0);
+}
+
+/* tell the block cipher walk routines that this is a stream cipher by
+ * setting cra_blocksize to 1. Even using blkcipher_walk_virt_block
+ * during encrypt/decrypt doesn't solve this problem, because it calls
+ * blkcipher_walk_done under the covers, which doesn't use walk->blocksize,
+ * but instead uses this tfm->blocksize. */
+struct crypto_alg nx_gcm_aes_alg = {
+ .cra_name = "gcm(aes)",
+ .cra_driver_name = "gcm-aes-nx",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_TYPE_AEAD,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct nx_crypto_ctx),
+ .cra_type = &crypto_aead_type,
+ .cra_module = THIS_MODULE,
+ .cra_init = nx_crypto_ctx_aes_gcm_init,
+ .cra_exit = nx_crypto_ctx_exit,
+ .cra_aead = {
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = AES_BLOCK_SIZE,
+ .setkey = gcm_aes_nx_set_key,
+ .setauthsize = gcm_aes_nx_setauthsize,
+ .encrypt = gcm_aes_nx_encrypt,
+ .decrypt = gcm_aes_nx_decrypt,
+ }
+};
+
+struct crypto_alg nx_gcm4106_aes_alg = {
+ .cra_name = "rfc4106(gcm(aes))",
+ .cra_driver_name = "rfc4106-gcm-aes-nx",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_TYPE_AEAD,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct nx_crypto_ctx),
+ .cra_type = &crypto_nivaead_type,
+ .cra_module = THIS_MODULE,
+ .cra_init = nx_crypto_ctx_aes_gcm_init,
+ .cra_exit = nx_crypto_ctx_exit,
+ .cra_aead = {
+ .ivsize = 8,
+ .maxauthsize = AES_BLOCK_SIZE,
+ .geniv = "seqiv",
+ .setkey = gcm4106_aes_nx_set_key,
+ .setauthsize = gcm4106_aes_nx_setauthsize,
+ .encrypt = gcm4106_aes_nx_encrypt,
+ .decrypt = gcm4106_aes_nx_decrypt,
+ }
+};
diff --git a/drivers/crypto/nx/nx-aes-xcbc.c b/drivers/crypto/nx/nx-aes-xcbc.c
new file mode 100644
index 000000000..8c2faffab
--- /dev/null
+++ b/drivers/crypto/nx/nx-aes-xcbc.c
@@ -0,0 +1,378 @@
+/**
+ * AES XCBC routines supporting the Power 7+ Nest Accelerators driver
+ *
+ * Copyright (C) 2011-2012 International Business Machines Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 only.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Author: Kent Yoder <yoder1@us.ibm.com>
+ */
+
+#include <crypto/internal/hash.h>
+#include <crypto/aes.h>
+#include <crypto/algapi.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/crypto.h>
+#include <asm/vio.h>
+
+#include "nx_csbcpb.h"
+#include "nx.h"
+
+
+struct xcbc_state {
+ u8 state[AES_BLOCK_SIZE];
+ unsigned int count;
+ u8 buffer[AES_BLOCK_SIZE];
+};
+
+static int nx_xcbc_set_key(struct crypto_shash *desc,
+ const u8 *in_key,
+ unsigned int key_len)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc);
+
+ switch (key_len) {
+ case AES_KEYSIZE_128:
+ nx_ctx->ap = &nx_ctx->props[NX_PROPS_AES_128];
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ memcpy(nx_ctx->priv.xcbc.key, in_key, key_len);
+
+ return 0;
+}
+
+/*
+ * Based on RFC 3566, for a zero-length message:
+ *
+ * n = 1
+ * K1 = E(K, 0x01010101010101010101010101010101)
+ * K3 = E(K, 0x03030303030303030303030303030303)
+ * E[0] = 0x00000000000000000000000000000000
+ * M[1] = 0x80000000000000000000000000000000 (0 length message with padding)
+ * E[1] = (K1, M[1] ^ E[0] ^ K3)
+ * Tag = M[1]
+ */
+static int nx_xcbc_empty(struct shash_desc *desc, u8 *out)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
+ struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ struct nx_sg *in_sg, *out_sg;
+ u8 keys[2][AES_BLOCK_SIZE];
+ u8 key[32];
+ int rc = 0;
+ int len;
+
+ /* Change to ECB mode */
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_ECB;
+ memcpy(key, csbcpb->cpb.aes_xcbc.key, AES_BLOCK_SIZE);
+ memcpy(csbcpb->cpb.aes_ecb.key, key, AES_BLOCK_SIZE);
+ NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
+
+ /* K1 and K3 base patterns */
+ memset(keys[0], 0x01, sizeof(keys[0]));
+ memset(keys[1], 0x03, sizeof(keys[1]));
+
+ len = sizeof(keys);
+ /* Generate K1 and K3 encrypting the patterns */
+ in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *) keys, &len,
+ nx_ctx->ap->sglen);
+
+ if (len != sizeof(keys))
+ return -EINVAL;
+
+ out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *) keys, &len,
+ nx_ctx->ap->sglen);
+
+ if (len != sizeof(keys))
+ return -EINVAL;
+
+ nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
+ nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+
+ /* XOr K3 with the padding for a 0 length message */
+ keys[1][0] ^= 0x80;
+
+ len = sizeof(keys[1]);
+
+ /* Encrypt the final result */
+ memcpy(csbcpb->cpb.aes_ecb.key, keys[0], AES_BLOCK_SIZE);
+ in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *) keys[1], &len,
+ nx_ctx->ap->sglen);
+
+ if (len != sizeof(keys[1]))
+ return -EINVAL;
+
+ len = AES_BLOCK_SIZE;
+ out_sg = nx_build_sg_list(nx_ctx->out_sg, out, &len,
+ nx_ctx->ap->sglen);
+
+ if (len != AES_BLOCK_SIZE)
+ return -EINVAL;
+
+ nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
+ nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+
+out:
+ /* Restore XCBC mode */
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_XCBC_MAC;
+ memcpy(csbcpb->cpb.aes_xcbc.key, key, AES_BLOCK_SIZE);
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_ENDE_ENCRYPT;
+
+ return rc;
+}
+
+static int nx_xcbc_init(struct shash_desc *desc)
+{
+ struct xcbc_state *sctx = shash_desc_ctx(desc);
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
+ struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ struct nx_sg *out_sg;
+ int len;
+
+ nx_ctx_init(nx_ctx, HCOP_FC_AES);
+
+ memset(sctx, 0, sizeof *sctx);
+
+ NX_CPB_SET_KEY_SIZE(csbcpb, NX_KS_AES_128);
+ csbcpb->cpb.hdr.mode = NX_MODE_AES_XCBC_MAC;
+
+ memcpy(csbcpb->cpb.aes_xcbc.key, nx_ctx->priv.xcbc.key, AES_BLOCK_SIZE);
+ memset(nx_ctx->priv.xcbc.key, 0, sizeof *nx_ctx->priv.xcbc.key);
+
+ len = AES_BLOCK_SIZE;
+ out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *)sctx->state,
+ &len, nx_ctx->ap->sglen);
+
+ if (len != AES_BLOCK_SIZE)
+ return -EINVAL;
+
+ nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
+
+ return 0;
+}
+
+static int nx_xcbc_update(struct shash_desc *desc,
+ const u8 *data,
+ unsigned int len)
+{
+ struct xcbc_state *sctx = shash_desc_ctx(desc);
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
+ struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ struct nx_sg *in_sg;
+ u32 to_process = 0, leftover, total;
+ unsigned int max_sg_len;
+ unsigned long irq_flags;
+ int rc = 0;
+ int data_len;
+
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+
+ total = sctx->count + len;
+
+ /* 2 cases for total data len:
+ * 1: <= AES_BLOCK_SIZE: copy into state, return 0
+ * 2: > AES_BLOCK_SIZE: process X blocks, copy in leftover
+ */
+ if (total <= AES_BLOCK_SIZE) {
+ memcpy(sctx->buffer + sctx->count, data, len);
+ sctx->count += len;
+ goto out;
+ }
+
+ in_sg = nx_ctx->in_sg;
+ max_sg_len = min_t(u64, nx_driver.of.max_sg_len/sizeof(struct nx_sg),
+ nx_ctx->ap->sglen);
+ max_sg_len = min_t(u64, max_sg_len,
+ nx_ctx->ap->databytelen/NX_PAGE_SIZE);
+
+ do {
+ to_process = total - to_process;
+ to_process = to_process & ~(AES_BLOCK_SIZE - 1);
+
+ leftover = total - to_process;
+
+ /* the hardware will not accept a 0 byte operation for this
+ * algorithm and the operation MUST be finalized to be correct.
+ * So if we happen to get an update that falls on a block sized
+ * boundary, we must save off the last block to finalize with
+ * later. */
+ if (!leftover) {
+ to_process -= AES_BLOCK_SIZE;
+ leftover = AES_BLOCK_SIZE;
+ }
+
+ if (sctx->count) {
+ data_len = sctx->count;
+ in_sg = nx_build_sg_list(nx_ctx->in_sg,
+ (u8 *) sctx->buffer,
+ &data_len,
+ max_sg_len);
+ if (data_len != sctx->count)
+ return -EINVAL;
+ }
+
+ data_len = to_process - sctx->count;
+ in_sg = nx_build_sg_list(in_sg,
+ (u8 *) data,
+ &data_len,
+ max_sg_len);
+
+ if (data_len != to_process - sctx->count)
+ return -EINVAL;
+
+ nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) *
+ sizeof(struct nx_sg);
+
+ /* we've hit the nx chip previously and we're updating again,
+ * so copy over the partial digest */
+ if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
+ memcpy(csbcpb->cpb.aes_xcbc.cv,
+ csbcpb->cpb.aes_xcbc.out_cv_mac,
+ AES_BLOCK_SIZE);
+ }
+
+ NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+
+ /* everything after the first update is continuation */
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+ total -= to_process;
+ data += to_process - sctx->count;
+ sctx->count = 0;
+ in_sg = nx_ctx->in_sg;
+ } while (leftover > AES_BLOCK_SIZE);
+
+ /* copy the leftover back into the state struct */
+ memcpy(sctx->buffer, data, leftover);
+ sctx->count = leftover;
+
+out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+ return rc;
+}
+
+static int nx_xcbc_final(struct shash_desc *desc, u8 *out)
+{
+ struct xcbc_state *sctx = shash_desc_ctx(desc);
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
+ struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+ struct nx_sg *in_sg, *out_sg;
+ unsigned long irq_flags;
+ int rc = 0;
+ int len;
+
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+ if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
+ /* we've hit the nx chip previously, now we're finalizing,
+ * so copy over the partial digest */
+ memcpy(csbcpb->cpb.aes_xcbc.cv,
+ csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE);
+ } else if (sctx->count == 0) {
+ /*
+ * we've never seen an update, so this is a 0 byte op. The
+ * hardware cannot handle a 0 byte op, so just ECB to
+ * generate the hash.
+ */
+ rc = nx_xcbc_empty(desc, out);
+ goto out;
+ }
+
+ /* final is represented by continuing the operation and indicating that
+ * this is not an intermediate operation */
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+
+ len = sctx->count;
+ in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)sctx->buffer,
+ &len, nx_ctx->ap->sglen);
+
+ if (len != sctx->count)
+ return -EINVAL;
+
+ len = AES_BLOCK_SIZE;
+ out_sg = nx_build_sg_list(nx_ctx->out_sg, out, &len,
+ nx_ctx->ap->sglen);
+
+ if (len != AES_BLOCK_SIZE)
+ return -EINVAL;
+
+ nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
+ nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg);
+
+ if (!nx_ctx->op.outlen) {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ atomic_inc(&(nx_ctx->stats->aes_ops));
+
+ memcpy(out, csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE);
+out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+ return rc;
+}
+
+struct shash_alg nx_shash_aes_xcbc_alg = {
+ .digestsize = AES_BLOCK_SIZE,
+ .init = nx_xcbc_init,
+ .update = nx_xcbc_update,
+ .final = nx_xcbc_final,
+ .setkey = nx_xcbc_set_key,
+ .descsize = sizeof(struct xcbc_state),
+ .statesize = sizeof(struct xcbc_state),
+ .base = {
+ .cra_name = "xcbc(aes)",
+ .cra_driver_name = "xcbc-aes-nx",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_TYPE_SHASH,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct nx_crypto_ctx),
+ .cra_init = nx_crypto_ctx_aes_xcbc_init,
+ .cra_exit = nx_crypto_ctx_exit,
+ }
+};
diff --git a/drivers/crypto/nx/nx-sha256.c b/drivers/crypto/nx/nx-sha256.c
new file mode 100644
index 000000000..23621da62
--- /dev/null
+++ b/drivers/crypto/nx/nx-sha256.c
@@ -0,0 +1,274 @@
+/**
+ * SHA-256 routines supporting the Power 7+ Nest Accelerators driver
+ *
+ * Copyright (C) 2011-2012 International Business Machines Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 only.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Author: Kent Yoder <yoder1@us.ibm.com>
+ */
+
+#include <crypto/internal/hash.h>
+#include <crypto/sha.h>
+#include <linux/module.h>
+#include <asm/vio.h>
+#include <asm/byteorder.h>
+
+#include "nx_csbcpb.h"
+#include "nx.h"
+
+
+static int nx_sha256_init(struct shash_desc *desc)
+{
+ struct sha256_state *sctx = shash_desc_ctx(desc);
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
+ int len;
+ int rc;
+
+ nx_ctx_init(nx_ctx, HCOP_FC_SHA);
+
+ memset(sctx, 0, sizeof *sctx);
+
+ nx_ctx->ap = &nx_ctx->props[NX_PROPS_SHA256];
+
+ NX_CPB_SET_DIGEST_SIZE(nx_ctx->csbcpb, NX_DS_SHA256);
+
+ len = SHA256_DIGEST_SIZE;
+ rc = nx_sha_build_sg_list(nx_ctx, nx_ctx->out_sg,
+ &nx_ctx->op.outlen,
+ &len,
+ (u8 *) sctx->state,
+ NX_DS_SHA256);
+
+ if (rc)
+ goto out;
+
+ sctx->state[0] = __cpu_to_be32(SHA256_H0);
+ sctx->state[1] = __cpu_to_be32(SHA256_H1);
+ sctx->state[2] = __cpu_to_be32(SHA256_H2);
+ sctx->state[3] = __cpu_to_be32(SHA256_H3);
+ sctx->state[4] = __cpu_to_be32(SHA256_H4);
+ sctx->state[5] = __cpu_to_be32(SHA256_H5);
+ sctx->state[6] = __cpu_to_be32(SHA256_H6);
+ sctx->state[7] = __cpu_to_be32(SHA256_H7);
+ sctx->count = 0;
+
+out:
+ return 0;
+}
+
+static int nx_sha256_update(struct shash_desc *desc, const u8 *data,
+ unsigned int len)
+{
+ struct sha256_state *sctx = shash_desc_ctx(desc);
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
+ struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
+ u64 to_process = 0, leftover, total;
+ unsigned long irq_flags;
+ int rc = 0;
+ int data_len;
+ u64 buf_len = (sctx->count % SHA256_BLOCK_SIZE);
+
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+ /* 2 cases for total data len:
+ * 1: < SHA256_BLOCK_SIZE: copy into state, return 0
+ * 2: >= SHA256_BLOCK_SIZE: process X blocks, copy in leftover
+ */
+ total = (sctx->count % SHA256_BLOCK_SIZE) + len;
+ if (total < SHA256_BLOCK_SIZE) {
+ memcpy(sctx->buf + buf_len, data, len);
+ sctx->count += len;
+ goto out;
+ }
+
+ memcpy(csbcpb->cpb.sha256.message_digest, sctx->state, SHA256_DIGEST_SIZE);
+ NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+ do {
+ /*
+ * to_process: the SHA256_BLOCK_SIZE data chunk to process in
+ * this update. This value is also restricted by the sg list
+ * limits.
+ */
+ to_process = total - to_process;
+ to_process = to_process & ~(SHA256_BLOCK_SIZE - 1);
+
+ if (buf_len) {
+ data_len = buf_len;
+ rc = nx_sha_build_sg_list(nx_ctx, nx_ctx->in_sg,
+ &nx_ctx->op.inlen,
+ &data_len,
+ (u8 *) sctx->buf,
+ NX_DS_SHA256);
+
+ if (rc || data_len != buf_len)
+ goto out;
+ }
+
+ data_len = to_process - buf_len;
+ rc = nx_sha_build_sg_list(nx_ctx, nx_ctx->in_sg,
+ &nx_ctx->op.inlen,
+ &data_len,
+ (u8 *) data,
+ NX_DS_SHA256);
+
+ if (rc)
+ goto out;
+
+ to_process = (data_len + buf_len);
+ leftover = total - to_process;
+
+ /*
+ * we've hit the nx chip previously and we're updating
+ * again, so copy over the partial digest.
+ */
+ memcpy(csbcpb->cpb.sha256.input_partial_digest,
+ csbcpb->cpb.sha256.message_digest,
+ SHA256_DIGEST_SIZE);
+
+ if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ atomic_inc(&(nx_ctx->stats->sha256_ops));
+
+ total -= to_process;
+ data += to_process - buf_len;
+ buf_len = 0;
+
+ } while (leftover >= SHA256_BLOCK_SIZE);
+
+ /* copy the leftover back into the state struct */
+ if (leftover)
+ memcpy(sctx->buf, data, leftover);
+
+ sctx->count += len;
+ memcpy(sctx->state, csbcpb->cpb.sha256.message_digest, SHA256_DIGEST_SIZE);
+out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+ return rc;
+}
+
+static int nx_sha256_final(struct shash_desc *desc, u8 *out)
+{
+ struct sha256_state *sctx = shash_desc_ctx(desc);
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
+ struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
+ unsigned long irq_flags;
+ int rc;
+ int len;
+
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+ /* final is represented by continuing the operation and indicating that
+ * this is not an intermediate operation */
+ if (sctx->count >= SHA256_BLOCK_SIZE) {
+ /* we've hit the nx chip previously, now we're finalizing,
+ * so copy over the partial digest */
+ memcpy(csbcpb->cpb.sha256.input_partial_digest, sctx->state, SHA256_DIGEST_SIZE);
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+ } else {
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_CONTINUATION;
+ }
+
+ csbcpb->cpb.sha256.message_bit_length = (u64) (sctx->count * 8);
+
+ len = sctx->count & (SHA256_BLOCK_SIZE - 1);
+ rc = nx_sha_build_sg_list(nx_ctx, nx_ctx->in_sg,
+ &nx_ctx->op.inlen,
+ &len,
+ (u8 *) sctx->buf,
+ NX_DS_SHA256);
+
+ if (rc || len != (sctx->count & (SHA256_BLOCK_SIZE - 1)))
+ goto out;
+
+ len = SHA256_DIGEST_SIZE;
+ rc = nx_sha_build_sg_list(nx_ctx, nx_ctx->out_sg,
+ &nx_ctx->op.outlen,
+ &len,
+ out,
+ NX_DS_SHA256);
+
+ if (rc || len != SHA256_DIGEST_SIZE)
+ goto out;
+
+ if (!nx_ctx->op.outlen) {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ atomic_inc(&(nx_ctx->stats->sha256_ops));
+
+ atomic64_add(sctx->count, &(nx_ctx->stats->sha256_bytes));
+ memcpy(out, csbcpb->cpb.sha256.message_digest, SHA256_DIGEST_SIZE);
+out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+ return rc;
+}
+
+static int nx_sha256_export(struct shash_desc *desc, void *out)
+{
+ struct sha256_state *sctx = shash_desc_ctx(desc);
+
+ memcpy(out, sctx, sizeof(*sctx));
+
+ return 0;
+}
+
+static int nx_sha256_import(struct shash_desc *desc, const void *in)
+{
+ struct sha256_state *sctx = shash_desc_ctx(desc);
+
+ memcpy(sctx, in, sizeof(*sctx));
+
+ return 0;
+}
+
+struct shash_alg nx_shash_sha256_alg = {
+ .digestsize = SHA256_DIGEST_SIZE,
+ .init = nx_sha256_init,
+ .update = nx_sha256_update,
+ .final = nx_sha256_final,
+ .export = nx_sha256_export,
+ .import = nx_sha256_import,
+ .descsize = sizeof(struct sha256_state),
+ .statesize = sizeof(struct sha256_state),
+ .base = {
+ .cra_name = "sha256",
+ .cra_driver_name = "sha256-nx",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_TYPE_SHASH,
+ .cra_blocksize = SHA256_BLOCK_SIZE,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct nx_crypto_ctx),
+ .cra_init = nx_crypto_ctx_sha_init,
+ .cra_exit = nx_crypto_ctx_exit,
+ }
+};
diff --git a/drivers/crypto/nx/nx-sha512.c b/drivers/crypto/nx/nx-sha512.c
new file mode 100644
index 000000000..b3adf1022
--- /dev/null
+++ b/drivers/crypto/nx/nx-sha512.c
@@ -0,0 +1,279 @@
+/**
+ * SHA-512 routines supporting the Power 7+ Nest Accelerators driver
+ *
+ * Copyright (C) 2011-2012 International Business Machines Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 only.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Author: Kent Yoder <yoder1@us.ibm.com>
+ */
+
+#include <crypto/internal/hash.h>
+#include <crypto/sha.h>
+#include <linux/module.h>
+#include <asm/vio.h>
+
+#include "nx_csbcpb.h"
+#include "nx.h"
+
+
+static int nx_sha512_init(struct shash_desc *desc)
+{
+ struct sha512_state *sctx = shash_desc_ctx(desc);
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
+ int len;
+ int rc;
+
+ nx_ctx_init(nx_ctx, HCOP_FC_SHA);
+
+ memset(sctx, 0, sizeof *sctx);
+
+ nx_ctx->ap = &nx_ctx->props[NX_PROPS_SHA512];
+
+ NX_CPB_SET_DIGEST_SIZE(nx_ctx->csbcpb, NX_DS_SHA512);
+
+ len = SHA512_DIGEST_SIZE;
+ rc = nx_sha_build_sg_list(nx_ctx, nx_ctx->out_sg,
+ &nx_ctx->op.outlen,
+ &len,
+ (u8 *)sctx->state,
+ NX_DS_SHA512);
+
+ if (rc || len != SHA512_DIGEST_SIZE)
+ goto out;
+
+ sctx->state[0] = __cpu_to_be64(SHA512_H0);
+ sctx->state[1] = __cpu_to_be64(SHA512_H1);
+ sctx->state[2] = __cpu_to_be64(SHA512_H2);
+ sctx->state[3] = __cpu_to_be64(SHA512_H3);
+ sctx->state[4] = __cpu_to_be64(SHA512_H4);
+ sctx->state[5] = __cpu_to_be64(SHA512_H5);
+ sctx->state[6] = __cpu_to_be64(SHA512_H6);
+ sctx->state[7] = __cpu_to_be64(SHA512_H7);
+ sctx->count[0] = 0;
+
+out:
+ return 0;
+}
+
+static int nx_sha512_update(struct shash_desc *desc, const u8 *data,
+ unsigned int len)
+{
+ struct sha512_state *sctx = shash_desc_ctx(desc);
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
+ struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
+ u64 to_process, leftover = 0, total;
+ unsigned long irq_flags;
+ int rc = 0;
+ int data_len;
+ u64 buf_len = (sctx->count[0] % SHA512_BLOCK_SIZE);
+
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+ /* 2 cases for total data len:
+ * 1: < SHA512_BLOCK_SIZE: copy into state, return 0
+ * 2: >= SHA512_BLOCK_SIZE: process X blocks, copy in leftover
+ */
+ total = (sctx->count[0] % SHA512_BLOCK_SIZE) + len;
+ if (total < SHA512_BLOCK_SIZE) {
+ memcpy(sctx->buf + buf_len, data, len);
+ sctx->count[0] += len;
+ goto out;
+ }
+
+ memcpy(csbcpb->cpb.sha512.message_digest, sctx->state, SHA512_DIGEST_SIZE);
+ NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+
+ do {
+ /*
+ * to_process: the SHA512_BLOCK_SIZE data chunk to process in
+ * this update. This value is also restricted by the sg list
+ * limits.
+ */
+ to_process = total - leftover;
+ to_process = to_process & ~(SHA512_BLOCK_SIZE - 1);
+ leftover = total - to_process;
+
+ if (buf_len) {
+ data_len = buf_len;
+ rc = nx_sha_build_sg_list(nx_ctx, nx_ctx->in_sg,
+ &nx_ctx->op.inlen,
+ &data_len,
+ (u8 *) sctx->buf,
+ NX_DS_SHA512);
+
+ if (rc || data_len != buf_len)
+ goto out;
+ }
+
+ data_len = to_process - buf_len;
+ rc = nx_sha_build_sg_list(nx_ctx, nx_ctx->in_sg,
+ &nx_ctx->op.inlen,
+ &data_len,
+ (u8 *) data,
+ NX_DS_SHA512);
+
+ if (rc || data_len != (to_process - buf_len))
+ goto out;
+
+ to_process = (data_len + buf_len);
+ leftover = total - to_process;
+
+ /*
+ * we've hit the nx chip previously and we're updating
+ * again, so copy over the partial digest.
+ */
+ memcpy(csbcpb->cpb.sha512.input_partial_digest,
+ csbcpb->cpb.sha512.message_digest,
+ SHA512_DIGEST_SIZE);
+
+ if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ atomic_inc(&(nx_ctx->stats->sha512_ops));
+
+ total -= to_process;
+ data += to_process - buf_len;
+ buf_len = 0;
+
+ } while (leftover >= SHA512_BLOCK_SIZE);
+
+ /* copy the leftover back into the state struct */
+ if (leftover)
+ memcpy(sctx->buf, data, leftover);
+ sctx->count[0] += len;
+ memcpy(sctx->state, csbcpb->cpb.sha512.message_digest, SHA512_DIGEST_SIZE);
+out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+ return rc;
+}
+
+static int nx_sha512_final(struct shash_desc *desc, u8 *out)
+{
+ struct sha512_state *sctx = shash_desc_ctx(desc);
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
+ struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
+ u64 count0;
+ unsigned long irq_flags;
+ int rc;
+ int len;
+
+ spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+ /* final is represented by continuing the operation and indicating that
+ * this is not an intermediate operation */
+ if (sctx->count[0] >= SHA512_BLOCK_SIZE) {
+ /* we've hit the nx chip previously, now we're finalizing,
+ * so copy over the partial digest */
+ memcpy(csbcpb->cpb.sha512.input_partial_digest, sctx->state,
+ SHA512_DIGEST_SIZE);
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+ NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
+ } else {
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_CONTINUATION;
+ }
+
+ NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE;
+
+ count0 = sctx->count[0] * 8;
+
+ csbcpb->cpb.sha512.message_bit_length_lo = count0;
+
+ len = sctx->count[0] & (SHA512_BLOCK_SIZE - 1);
+ rc = nx_sha_build_sg_list(nx_ctx, nx_ctx->in_sg,
+ &nx_ctx->op.inlen,
+ &len,
+ (u8 *)sctx->buf,
+ NX_DS_SHA512);
+
+ if (rc || len != (sctx->count[0] & (SHA512_BLOCK_SIZE - 1)))
+ goto out;
+
+ len = SHA512_DIGEST_SIZE;
+ rc = nx_sha_build_sg_list(nx_ctx, nx_ctx->out_sg,
+ &nx_ctx->op.outlen,
+ &len,
+ out,
+ NX_DS_SHA512);
+
+ if (rc)
+ goto out;
+
+ if (!nx_ctx->op.outlen) {
+ rc = -EINVAL;
+ goto out;
+ }
+
+ rc = nx_hcall_sync(nx_ctx, &nx_ctx->op,
+ desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP);
+ if (rc)
+ goto out;
+
+ atomic_inc(&(nx_ctx->stats->sha512_ops));
+ atomic64_add(sctx->count[0], &(nx_ctx->stats->sha512_bytes));
+
+ memcpy(out, csbcpb->cpb.sha512.message_digest, SHA512_DIGEST_SIZE);
+out:
+ spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
+ return rc;
+}
+
+static int nx_sha512_export(struct shash_desc *desc, void *out)
+{
+ struct sha512_state *sctx = shash_desc_ctx(desc);
+
+ memcpy(out, sctx, sizeof(*sctx));
+
+ return 0;
+}
+
+static int nx_sha512_import(struct shash_desc *desc, const void *in)
+{
+ struct sha512_state *sctx = shash_desc_ctx(desc);
+
+ memcpy(sctx, in, sizeof(*sctx));
+
+ return 0;
+}
+
+struct shash_alg nx_shash_sha512_alg = {
+ .digestsize = SHA512_DIGEST_SIZE,
+ .init = nx_sha512_init,
+ .update = nx_sha512_update,
+ .final = nx_sha512_final,
+ .export = nx_sha512_export,
+ .import = nx_sha512_import,
+ .descsize = sizeof(struct sha512_state),
+ .statesize = sizeof(struct sha512_state),
+ .base = {
+ .cra_name = "sha512",
+ .cra_driver_name = "sha512-nx",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_TYPE_SHASH,
+ .cra_blocksize = SHA512_BLOCK_SIZE,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct nx_crypto_ctx),
+ .cra_init = nx_crypto_ctx_sha_init,
+ .cra_exit = nx_crypto_ctx_exit,
+ }
+};
diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c
new file mode 100644
index 000000000..1da6dc59d
--- /dev/null
+++ b/drivers/crypto/nx/nx.c
@@ -0,0 +1,799 @@
+/**
+ * Routines supporting the Power 7+ Nest Accelerators driver
+ *
+ * Copyright (C) 2011-2012 International Business Machines Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 only.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Author: Kent Yoder <yoder1@us.ibm.com>
+ */
+
+#include <crypto/internal/hash.h>
+#include <crypto/hash.h>
+#include <crypto/aes.h>
+#include <crypto/sha.h>
+#include <crypto/algapi.h>
+#include <crypto/scatterwalk.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/types.h>
+#include <linux/mm.h>
+#include <linux/crypto.h>
+#include <linux/scatterlist.h>
+#include <linux/device.h>
+#include <linux/of.h>
+#include <asm/hvcall.h>
+#include <asm/vio.h>
+
+#include "nx_csbcpb.h"
+#include "nx.h"
+
+
+/**
+ * nx_hcall_sync - make an H_COP_OP hcall for the passed in op structure
+ *
+ * @nx_ctx: the crypto context handle
+ * @op: PFO operation struct to pass in
+ * @may_sleep: flag indicating the request can sleep
+ *
+ * Make the hcall, retrying while the hardware is busy. If we cannot yield
+ * the thread, limit the number of retries to 10 here.
+ */
+int nx_hcall_sync(struct nx_crypto_ctx *nx_ctx,
+ struct vio_pfo_op *op,
+ u32 may_sleep)
+{
+ int rc, retries = 10;
+ struct vio_dev *viodev = nx_driver.viodev;
+
+ atomic_inc(&(nx_ctx->stats->sync_ops));
+
+ do {
+ rc = vio_h_cop_sync(viodev, op);
+ } while (rc == -EBUSY && !may_sleep && retries--);
+
+ if (rc) {
+ dev_dbg(&viodev->dev, "vio_h_cop_sync failed: rc: %d "
+ "hcall rc: %ld\n", rc, op->hcall_err);
+ atomic_inc(&(nx_ctx->stats->errors));
+ atomic_set(&(nx_ctx->stats->last_error), op->hcall_err);
+ atomic_set(&(nx_ctx->stats->last_error_pid), current->pid);
+ }
+
+ return rc;
+}
+
+/**
+ * nx_build_sg_list - build an NX scatter list describing a single buffer
+ *
+ * @sg_head: pointer to the first scatter list element to build
+ * @start_addr: pointer to the linear buffer
+ * @len: length of the data at @start_addr
+ * @sgmax: the largest number of scatter list elements we're allowed to create
+ *
+ * This function will start writing nx_sg elements at @sg_head and keep
+ * writing them until all of the data from @start_addr is described or
+ * until sgmax elements have been written. Scatter list elements will be
+ * created such that none of the elements describes a buffer that crosses a 4K
+ * boundary.
+ */
+struct nx_sg *nx_build_sg_list(struct nx_sg *sg_head,
+ u8 *start_addr,
+ unsigned int *len,
+ u32 sgmax)
+{
+ unsigned int sg_len = 0;
+ struct nx_sg *sg;
+ u64 sg_addr = (u64)start_addr;
+ u64 end_addr;
+
+ /* determine the start and end for this address range - slightly
+ * different if this is in VMALLOC_REGION */
+ if (is_vmalloc_addr(start_addr))
+ sg_addr = page_to_phys(vmalloc_to_page(start_addr))
+ + offset_in_page(sg_addr);
+ else
+ sg_addr = __pa(sg_addr);
+
+ end_addr = sg_addr + *len;
+
+ /* each iteration will write one struct nx_sg element and add the
+ * length of data described by that element to sg_len. Once @len bytes
+ * have been described (or @sgmax elements have been written), the
+ * loop ends. min_t is used to ensure @end_addr falls on the same page
+ * as sg_addr, if not, we need to create another nx_sg element for the
+ * data on the next page.
+ *
+ * Also when using vmalloc'ed data, every time that a system page
+ * boundary is crossed the physical address needs to be re-calculated.
+ */
+ for (sg = sg_head; sg_len < *len; sg++) {
+ u64 next_page;
+
+ sg->addr = sg_addr;
+ sg_addr = min_t(u64, NX_PAGE_NUM(sg_addr + NX_PAGE_SIZE),
+ end_addr);
+
+ next_page = (sg->addr & PAGE_MASK) + PAGE_SIZE;
+ sg->len = min_t(u64, sg_addr, next_page) - sg->addr;
+ sg_len += sg->len;
+
+ if (sg_addr >= next_page &&
+ is_vmalloc_addr(start_addr + sg_len)) {
+ sg_addr = page_to_phys(vmalloc_to_page(
+ start_addr + sg_len));
+ end_addr = sg_addr + *len - sg_len;
+ }
+
+ if ((sg - sg_head) == sgmax) {
+ pr_err("nx: scatter/gather list overflow, pid: %d\n",
+ current->pid);
+ sg++;
+ break;
+ }
+ }
+ *len = sg_len;
+
+ /* return the moved sg_head pointer */
+ return sg;
+}
+
+/**
+ * nx_walk_and_build - walk a linux scatterlist and build an nx scatterlist
+ *
+ * @nx_dst: pointer to the first nx_sg element to write
+ * @sglen: max number of nx_sg entries we're allowed to write
+ * @sg_src: pointer to the source linux scatterlist to walk
+ * @start: number of bytes to fast-forward past at the beginning of @sg_src
+ * @src_len: number of bytes to walk in @sg_src
+ */
+struct nx_sg *nx_walk_and_build(struct nx_sg *nx_dst,
+ unsigned int sglen,
+ struct scatterlist *sg_src,
+ unsigned int start,
+ unsigned int *src_len)
+{
+ struct scatter_walk walk;
+ struct nx_sg *nx_sg = nx_dst;
+ unsigned int n, offset = 0, len = *src_len;
+ char *dst;
+
+ /* we need to fast forward through @start bytes first */
+ for (;;) {
+ scatterwalk_start(&walk, sg_src);
+
+ if (start < offset + sg_src->length)
+ break;
+
+ offset += sg_src->length;
+ sg_src = sg_next(sg_src);
+ }
+
+ /* start - offset is the number of bytes to advance in the scatterlist
+ * element we're currently looking at */
+ scatterwalk_advance(&walk, start - offset);
+
+ while (len && (nx_sg - nx_dst) < sglen) {
+ n = scatterwalk_clamp(&walk, len);
+ if (!n) {
+ /* In cases where we have scatterlist chain sg_next
+ * handles with it properly */
+ scatterwalk_start(&walk, sg_next(walk.sg));
+ n = scatterwalk_clamp(&walk, len);
+ }
+ dst = scatterwalk_map(&walk);
+
+ nx_sg = nx_build_sg_list(nx_sg, dst, &n, sglen - (nx_sg - nx_dst));
+ len -= n;
+
+ scatterwalk_unmap(dst);
+ scatterwalk_advance(&walk, n);
+ scatterwalk_done(&walk, SCATTERWALK_FROM_SG, len);
+ }
+ /* update to_process */
+ *src_len -= len;
+
+ /* return the moved destination pointer */
+ return nx_sg;
+}
+
+/**
+ * trim_sg_list - ensures the bound in sg list.
+ * @sg: sg list head
+ * @end: sg lisg end
+ * @delta: is the amount we need to crop in order to bound the list.
+ *
+ */
+static long int trim_sg_list(struct nx_sg *sg, struct nx_sg *end, unsigned int delta)
+{
+ while (delta && end > sg) {
+ struct nx_sg *last = end - 1;
+
+ if (last->len > delta) {
+ last->len -= delta;
+ delta = 0;
+ } else {
+ end--;
+ delta -= last->len;
+ }
+ }
+ return (sg - end) * sizeof(struct nx_sg);
+}
+
+/**
+ * nx_sha_build_sg_list - walk and build sg list to sha modes
+ * using right bounds and limits.
+ * @nx_ctx: NX crypto context for the lists we're building
+ * @nx_sg: current sg list in or out list
+ * @op_len: current op_len to be used in order to build a sg list
+ * @nbytes: number or bytes to be processed
+ * @offset: buf offset
+ * @mode: SHA256 or SHA512
+ */
+int nx_sha_build_sg_list(struct nx_crypto_ctx *nx_ctx,
+ struct nx_sg *nx_in_outsg,
+ s64 *op_len,
+ unsigned int *nbytes,
+ u8 *offset,
+ u32 mode)
+{
+ unsigned int delta = 0;
+ unsigned int total = *nbytes;
+ struct nx_sg *nx_insg = nx_in_outsg;
+ unsigned int max_sg_len;
+
+ max_sg_len = min_t(u64, nx_ctx->ap->sglen,
+ nx_driver.of.max_sg_len/sizeof(struct nx_sg));
+ max_sg_len = min_t(u64, max_sg_len,
+ nx_ctx->ap->databytelen/NX_PAGE_SIZE);
+
+ *nbytes = min_t(u64, *nbytes, nx_ctx->ap->databytelen);
+ nx_insg = nx_build_sg_list(nx_insg, offset, nbytes, max_sg_len);
+
+ switch (mode) {
+ case NX_DS_SHA256:
+ if (*nbytes < total)
+ delta = *nbytes - (*nbytes & ~(SHA256_BLOCK_SIZE - 1));
+ break;
+ case NX_DS_SHA512:
+ if (*nbytes < total)
+ delta = *nbytes - (*nbytes & ~(SHA512_BLOCK_SIZE - 1));
+ break;
+ default:
+ return -EINVAL;
+ }
+ *op_len = trim_sg_list(nx_in_outsg, nx_insg, delta);
+
+ return 0;
+}
+
+/**
+ * nx_build_sg_lists - walk the input scatterlists and build arrays of NX
+ * scatterlists based on them.
+ *
+ * @nx_ctx: NX crypto context for the lists we're building
+ * @desc: the block cipher descriptor for the operation
+ * @dst: destination scatterlist
+ * @src: source scatterlist
+ * @nbytes: length of data described in the scatterlists
+ * @offset: number of bytes to fast-forward past at the beginning of
+ * scatterlists.
+ * @iv: destination for the iv data, if the algorithm requires it
+ *
+ * This is common code shared by all the AES algorithms. It uses the block
+ * cipher walk routines to traverse input and output scatterlists, building
+ * corresponding NX scatterlists
+ */
+int nx_build_sg_lists(struct nx_crypto_ctx *nx_ctx,
+ struct blkcipher_desc *desc,
+ struct scatterlist *dst,
+ struct scatterlist *src,
+ unsigned int *nbytes,
+ unsigned int offset,
+ u8 *iv)
+{
+ unsigned int delta = 0;
+ unsigned int total = *nbytes;
+ struct nx_sg *nx_insg = nx_ctx->in_sg;
+ struct nx_sg *nx_outsg = nx_ctx->out_sg;
+ unsigned int max_sg_len;
+
+ max_sg_len = min_t(u64, nx_ctx->ap->sglen,
+ nx_driver.of.max_sg_len/sizeof(struct nx_sg));
+ max_sg_len = min_t(u64, max_sg_len,
+ nx_ctx->ap->databytelen/NX_PAGE_SIZE);
+
+ if (iv)
+ memcpy(iv, desc->info, AES_BLOCK_SIZE);
+
+ *nbytes = min_t(u64, *nbytes, nx_ctx->ap->databytelen);
+
+ nx_outsg = nx_walk_and_build(nx_outsg, max_sg_len, dst,
+ offset, nbytes);
+ nx_insg = nx_walk_and_build(nx_insg, max_sg_len, src,
+ offset, nbytes);
+
+ if (*nbytes < total)
+ delta = *nbytes - (*nbytes & ~(AES_BLOCK_SIZE - 1));
+
+ /* these lengths should be negative, which will indicate to phyp that
+ * the input and output parameters are scatterlists, not linear
+ * buffers */
+ nx_ctx->op.inlen = trim_sg_list(nx_ctx->in_sg, nx_insg, delta);
+ nx_ctx->op.outlen = trim_sg_list(nx_ctx->out_sg, nx_outsg, delta);
+
+ return 0;
+}
+
+/**
+ * nx_ctx_init - initialize an nx_ctx's vio_pfo_op struct
+ *
+ * @nx_ctx: the nx context to initialize
+ * @function: the function code for the op
+ */
+void nx_ctx_init(struct nx_crypto_ctx *nx_ctx, unsigned int function)
+{
+ spin_lock_init(&nx_ctx->lock);
+ memset(nx_ctx->kmem, 0, nx_ctx->kmem_len);
+ nx_ctx->csbcpb->csb.valid |= NX_CSB_VALID_BIT;
+
+ nx_ctx->op.flags = function;
+ nx_ctx->op.csbcpb = __pa(nx_ctx->csbcpb);
+ nx_ctx->op.in = __pa(nx_ctx->in_sg);
+ nx_ctx->op.out = __pa(nx_ctx->out_sg);
+
+ if (nx_ctx->csbcpb_aead) {
+ nx_ctx->csbcpb_aead->csb.valid |= NX_CSB_VALID_BIT;
+
+ nx_ctx->op_aead.flags = function;
+ nx_ctx->op_aead.csbcpb = __pa(nx_ctx->csbcpb_aead);
+ nx_ctx->op_aead.in = __pa(nx_ctx->in_sg);
+ nx_ctx->op_aead.out = __pa(nx_ctx->out_sg);
+ }
+}
+
+static void nx_of_update_status(struct device *dev,
+ struct property *p,
+ struct nx_of *props)
+{
+ if (!strncmp(p->value, "okay", p->length)) {
+ props->status = NX_WAITING;
+ props->flags |= NX_OF_FLAG_STATUS_SET;
+ } else {
+ dev_info(dev, "%s: status '%s' is not 'okay'\n", __func__,
+ (char *)p->value);
+ }
+}
+
+static void nx_of_update_sglen(struct device *dev,
+ struct property *p,
+ struct nx_of *props)
+{
+ if (p->length != sizeof(props->max_sg_len)) {
+ dev_err(dev, "%s: unexpected format for "
+ "ibm,max-sg-len property\n", __func__);
+ dev_dbg(dev, "%s: ibm,max-sg-len is %d bytes "
+ "long, expected %zd bytes\n", __func__,
+ p->length, sizeof(props->max_sg_len));
+ return;
+ }
+
+ props->max_sg_len = *(u32 *)p->value;
+ props->flags |= NX_OF_FLAG_MAXSGLEN_SET;
+}
+
+static void nx_of_update_msc(struct device *dev,
+ struct property *p,
+ struct nx_of *props)
+{
+ struct msc_triplet *trip;
+ struct max_sync_cop *msc;
+ unsigned int bytes_so_far, i, lenp;
+
+ msc = (struct max_sync_cop *)p->value;
+ lenp = p->length;
+
+ /* You can't tell if the data read in for this property is sane by its
+ * size alone. This is because there are sizes embedded in the data
+ * structure. The best we can do is check lengths as we parse and bail
+ * as soon as a length error is detected. */
+ bytes_so_far = 0;
+
+ while ((bytes_so_far + sizeof(struct max_sync_cop)) <= lenp) {
+ bytes_so_far += sizeof(struct max_sync_cop);
+
+ trip = msc->trip;
+
+ for (i = 0;
+ ((bytes_so_far + sizeof(struct msc_triplet)) <= lenp) &&
+ i < msc->triplets;
+ i++) {
+ if (msc->fc > NX_MAX_FC || msc->mode > NX_MAX_MODE) {
+ dev_err(dev, "unknown function code/mode "
+ "combo: %d/%d (ignored)\n", msc->fc,
+ msc->mode);
+ goto next_loop;
+ }
+
+ switch (trip->keybitlen) {
+ case 128:
+ case 160:
+ props->ap[msc->fc][msc->mode][0].databytelen =
+ trip->databytelen;
+ props->ap[msc->fc][msc->mode][0].sglen =
+ trip->sglen;
+ break;
+ case 192:
+ props->ap[msc->fc][msc->mode][1].databytelen =
+ trip->databytelen;
+ props->ap[msc->fc][msc->mode][1].sglen =
+ trip->sglen;
+ break;
+ case 256:
+ if (msc->fc == NX_FC_AES) {
+ props->ap[msc->fc][msc->mode][2].
+ databytelen = trip->databytelen;
+ props->ap[msc->fc][msc->mode][2].sglen =
+ trip->sglen;
+ } else if (msc->fc == NX_FC_AES_HMAC ||
+ msc->fc == NX_FC_SHA) {
+ props->ap[msc->fc][msc->mode][1].
+ databytelen = trip->databytelen;
+ props->ap[msc->fc][msc->mode][1].sglen =
+ trip->sglen;
+ } else {
+ dev_warn(dev, "unknown function "
+ "code/key bit len combo"
+ ": (%u/256)\n", msc->fc);
+ }
+ break;
+ case 512:
+ props->ap[msc->fc][msc->mode][2].databytelen =
+ trip->databytelen;
+ props->ap[msc->fc][msc->mode][2].sglen =
+ trip->sglen;
+ break;
+ default:
+ dev_warn(dev, "unknown function code/key bit "
+ "len combo: (%u/%u)\n", msc->fc,
+ trip->keybitlen);
+ break;
+ }
+next_loop:
+ bytes_so_far += sizeof(struct msc_triplet);
+ trip++;
+ }
+
+ msc = (struct max_sync_cop *)trip;
+ }
+
+ props->flags |= NX_OF_FLAG_MAXSYNCCOP_SET;
+}
+
+/**
+ * nx_of_init - read openFirmware values from the device tree
+ *
+ * @dev: device handle
+ * @props: pointer to struct to hold the properties values
+ *
+ * Called once at driver probe time, this function will read out the
+ * openFirmware properties we use at runtime. If all the OF properties are
+ * acceptable, when we exit this function props->flags will indicate that
+ * we're ready to register our crypto algorithms.
+ */
+static void nx_of_init(struct device *dev, struct nx_of *props)
+{
+ struct device_node *base_node = dev->of_node;
+ struct property *p;
+
+ p = of_find_property(base_node, "status", NULL);
+ if (!p)
+ dev_info(dev, "%s: property 'status' not found\n", __func__);
+ else
+ nx_of_update_status(dev, p, props);
+
+ p = of_find_property(base_node, "ibm,max-sg-len", NULL);
+ if (!p)
+ dev_info(dev, "%s: property 'ibm,max-sg-len' not found\n",
+ __func__);
+ else
+ nx_of_update_sglen(dev, p, props);
+
+ p = of_find_property(base_node, "ibm,max-sync-cop", NULL);
+ if (!p)
+ dev_info(dev, "%s: property 'ibm,max-sync-cop' not found\n",
+ __func__);
+ else
+ nx_of_update_msc(dev, p, props);
+}
+
+/**
+ * nx_register_algs - register algorithms with the crypto API
+ *
+ * Called from nx_probe()
+ *
+ * If all OF properties are in an acceptable state, the driver flags will
+ * indicate that we're ready and we'll create our debugfs files and register
+ * out crypto algorithms.
+ */
+static int nx_register_algs(void)
+{
+ int rc = -1;
+
+ if (nx_driver.of.flags != NX_OF_FLAG_MASK_READY)
+ goto out;
+
+ memset(&nx_driver.stats, 0, sizeof(struct nx_stats));
+
+ rc = NX_DEBUGFS_INIT(&nx_driver);
+ if (rc)
+ goto out;
+
+ nx_driver.of.status = NX_OKAY;
+
+ rc = crypto_register_alg(&nx_ecb_aes_alg);
+ if (rc)
+ goto out;
+
+ rc = crypto_register_alg(&nx_cbc_aes_alg);
+ if (rc)
+ goto out_unreg_ecb;
+
+ rc = crypto_register_alg(&nx_ctr_aes_alg);
+ if (rc)
+ goto out_unreg_cbc;
+
+ rc = crypto_register_alg(&nx_ctr3686_aes_alg);
+ if (rc)
+ goto out_unreg_ctr;
+
+ rc = crypto_register_alg(&nx_gcm_aes_alg);
+ if (rc)
+ goto out_unreg_ctr3686;
+
+ rc = crypto_register_alg(&nx_gcm4106_aes_alg);
+ if (rc)
+ goto out_unreg_gcm;
+
+ rc = crypto_register_alg(&nx_ccm_aes_alg);
+ if (rc)
+ goto out_unreg_gcm4106;
+
+ rc = crypto_register_alg(&nx_ccm4309_aes_alg);
+ if (rc)
+ goto out_unreg_ccm;
+
+ rc = crypto_register_shash(&nx_shash_sha256_alg);
+ if (rc)
+ goto out_unreg_ccm4309;
+
+ rc = crypto_register_shash(&nx_shash_sha512_alg);
+ if (rc)
+ goto out_unreg_s256;
+
+ rc = crypto_register_shash(&nx_shash_aes_xcbc_alg);
+ if (rc)
+ goto out_unreg_s512;
+
+ goto out;
+
+out_unreg_s512:
+ crypto_unregister_shash(&nx_shash_sha512_alg);
+out_unreg_s256:
+ crypto_unregister_shash(&nx_shash_sha256_alg);
+out_unreg_ccm4309:
+ crypto_unregister_alg(&nx_ccm4309_aes_alg);
+out_unreg_ccm:
+ crypto_unregister_alg(&nx_ccm_aes_alg);
+out_unreg_gcm4106:
+ crypto_unregister_alg(&nx_gcm4106_aes_alg);
+out_unreg_gcm:
+ crypto_unregister_alg(&nx_gcm_aes_alg);
+out_unreg_ctr3686:
+ crypto_unregister_alg(&nx_ctr3686_aes_alg);
+out_unreg_ctr:
+ crypto_unregister_alg(&nx_ctr_aes_alg);
+out_unreg_cbc:
+ crypto_unregister_alg(&nx_cbc_aes_alg);
+out_unreg_ecb:
+ crypto_unregister_alg(&nx_ecb_aes_alg);
+out:
+ return rc;
+}
+
+/**
+ * nx_crypto_ctx_init - create and initialize a crypto api context
+ *
+ * @nx_ctx: the crypto api context
+ * @fc: function code for the context
+ * @mode: the function code specific mode for this context
+ */
+static int nx_crypto_ctx_init(struct nx_crypto_ctx *nx_ctx, u32 fc, u32 mode)
+{
+ if (nx_driver.of.status != NX_OKAY) {
+ pr_err("Attempt to initialize NX crypto context while device "
+ "is not available!\n");
+ return -ENODEV;
+ }
+
+ /* we need an extra page for csbcpb_aead for these modes */
+ if (mode == NX_MODE_AES_GCM || mode == NX_MODE_AES_CCM)
+ nx_ctx->kmem_len = (5 * NX_PAGE_SIZE) +
+ sizeof(struct nx_csbcpb);
+ else
+ nx_ctx->kmem_len = (4 * NX_PAGE_SIZE) +
+ sizeof(struct nx_csbcpb);
+
+ nx_ctx->kmem = kmalloc(nx_ctx->kmem_len, GFP_KERNEL);
+ if (!nx_ctx->kmem)
+ return -ENOMEM;
+
+ /* the csbcpb and scatterlists must be 4K aligned pages */
+ nx_ctx->csbcpb = (struct nx_csbcpb *)(round_up((u64)nx_ctx->kmem,
+ (u64)NX_PAGE_SIZE));
+ nx_ctx->in_sg = (struct nx_sg *)((u8 *)nx_ctx->csbcpb + NX_PAGE_SIZE);
+ nx_ctx->out_sg = (struct nx_sg *)((u8 *)nx_ctx->in_sg + NX_PAGE_SIZE);
+
+ if (mode == NX_MODE_AES_GCM || mode == NX_MODE_AES_CCM)
+ nx_ctx->csbcpb_aead =
+ (struct nx_csbcpb *)((u8 *)nx_ctx->out_sg +
+ NX_PAGE_SIZE);
+
+ /* give each context a pointer to global stats and their OF
+ * properties */
+ nx_ctx->stats = &nx_driver.stats;
+ memcpy(nx_ctx->props, nx_driver.of.ap[fc][mode],
+ sizeof(struct alg_props) * 3);
+
+ return 0;
+}
+
+/* entry points from the crypto tfm initializers */
+int nx_crypto_ctx_aes_ccm_init(struct crypto_tfm *tfm)
+{
+ return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_AES,
+ NX_MODE_AES_CCM);
+}
+
+int nx_crypto_ctx_aes_gcm_init(struct crypto_tfm *tfm)
+{
+ return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_AES,
+ NX_MODE_AES_GCM);
+}
+
+int nx_crypto_ctx_aes_ctr_init(struct crypto_tfm *tfm)
+{
+ return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_AES,
+ NX_MODE_AES_CTR);
+}
+
+int nx_crypto_ctx_aes_cbc_init(struct crypto_tfm *tfm)
+{
+ return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_AES,
+ NX_MODE_AES_CBC);
+}
+
+int nx_crypto_ctx_aes_ecb_init(struct crypto_tfm *tfm)
+{
+ return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_AES,
+ NX_MODE_AES_ECB);
+}
+
+int nx_crypto_ctx_sha_init(struct crypto_tfm *tfm)
+{
+ return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_SHA, NX_MODE_SHA);
+}
+
+int nx_crypto_ctx_aes_xcbc_init(struct crypto_tfm *tfm)
+{
+ return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_AES,
+ NX_MODE_AES_XCBC_MAC);
+}
+
+/**
+ * nx_crypto_ctx_exit - destroy a crypto api context
+ *
+ * @tfm: the crypto transform pointer for the context
+ *
+ * As crypto API contexts are destroyed, this exit hook is called to free the
+ * memory associated with it.
+ */
+void nx_crypto_ctx_exit(struct crypto_tfm *tfm)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm);
+
+ kzfree(nx_ctx->kmem);
+ nx_ctx->csbcpb = NULL;
+ nx_ctx->csbcpb_aead = NULL;
+ nx_ctx->in_sg = NULL;
+ nx_ctx->out_sg = NULL;
+}
+
+static int nx_probe(struct vio_dev *viodev, const struct vio_device_id *id)
+{
+ dev_dbg(&viodev->dev, "driver probed: %s resource id: 0x%x\n",
+ viodev->name, viodev->resource_id);
+
+ if (nx_driver.viodev) {
+ dev_err(&viodev->dev, "%s: Attempt to register more than one "
+ "instance of the hardware\n", __func__);
+ return -EINVAL;
+ }
+
+ nx_driver.viodev = viodev;
+
+ nx_of_init(&viodev->dev, &nx_driver.of);
+
+ return nx_register_algs();
+}
+
+static int nx_remove(struct vio_dev *viodev)
+{
+ dev_dbg(&viodev->dev, "entering nx_remove for UA 0x%x\n",
+ viodev->unit_address);
+
+ if (nx_driver.of.status == NX_OKAY) {
+ NX_DEBUGFS_FINI(&nx_driver);
+
+ crypto_unregister_alg(&nx_ccm_aes_alg);
+ crypto_unregister_alg(&nx_ccm4309_aes_alg);
+ crypto_unregister_alg(&nx_gcm_aes_alg);
+ crypto_unregister_alg(&nx_gcm4106_aes_alg);
+ crypto_unregister_alg(&nx_ctr_aes_alg);
+ crypto_unregister_alg(&nx_ctr3686_aes_alg);
+ crypto_unregister_alg(&nx_cbc_aes_alg);
+ crypto_unregister_alg(&nx_ecb_aes_alg);
+ crypto_unregister_shash(&nx_shash_sha256_alg);
+ crypto_unregister_shash(&nx_shash_sha512_alg);
+ crypto_unregister_shash(&nx_shash_aes_xcbc_alg);
+ }
+
+ return 0;
+}
+
+
+/* module wide initialization/cleanup */
+static int __init nx_init(void)
+{
+ return vio_register_driver(&nx_driver.viodriver);
+}
+
+static void __exit nx_fini(void)
+{
+ vio_unregister_driver(&nx_driver.viodriver);
+}
+
+static struct vio_device_id nx_crypto_driver_ids[] = {
+ { "ibm,sym-encryption-v1", "ibm,sym-encryption" },
+ { "", "" }
+};
+MODULE_DEVICE_TABLE(vio, nx_crypto_driver_ids);
+
+/* driver state structure */
+struct nx_crypto_driver nx_driver = {
+ .viodriver = {
+ .id_table = nx_crypto_driver_ids,
+ .probe = nx_probe,
+ .remove = nx_remove,
+ .name = NX_NAME,
+ },
+};
+
+module_init(nx_init);
+module_exit(nx_fini);
+
+MODULE_AUTHOR("Kent Yoder <yoder1@us.ibm.com>");
+MODULE_DESCRIPTION(NX_STRING);
+MODULE_LICENSE("GPL");
+MODULE_VERSION(NX_VERSION);
diff --git a/drivers/crypto/nx/nx.h b/drivers/crypto/nx/nx.h
new file mode 100644
index 000000000..6c9ecaaea
--- /dev/null
+++ b/drivers/crypto/nx/nx.h
@@ -0,0 +1,196 @@
+
+#ifndef __NX_H__
+#define __NX_H__
+
+#define NX_NAME "nx-crypto"
+#define NX_STRING "IBM Power7+ Nest Accelerator Crypto Driver"
+#define NX_VERSION "1.0"
+
+static const char nx_driver_string[] = NX_STRING;
+static const char nx_driver_version[] = NX_VERSION;
+
+/* a scatterlist in the format PHYP is expecting */
+struct nx_sg {
+ u64 addr;
+ u32 rsvd;
+ u32 len;
+} __attribute((packed));
+
+#define NX_PAGE_SIZE (4096)
+#define NX_MAX_SG_ENTRIES (NX_PAGE_SIZE/(sizeof(struct nx_sg)))
+
+enum nx_status {
+ NX_DISABLED,
+ NX_WAITING,
+ NX_OKAY
+};
+
+/* msc_triplet and max_sync_cop are used only to assist in parsing the
+ * openFirmware property */
+struct msc_triplet {
+ u32 keybitlen;
+ u32 databytelen;
+ u32 sglen;
+} __packed;
+
+struct max_sync_cop {
+ u32 fc;
+ u32 mode;
+ u32 triplets;
+ struct msc_triplet trip[0];
+} __packed;
+
+struct alg_props {
+ u32 databytelen;
+ u32 sglen;
+};
+
+#define NX_OF_FLAG_MAXSGLEN_SET (1)
+#define NX_OF_FLAG_STATUS_SET (2)
+#define NX_OF_FLAG_MAXSYNCCOP_SET (4)
+#define NX_OF_FLAG_MASK_READY (NX_OF_FLAG_MAXSGLEN_SET | \
+ NX_OF_FLAG_STATUS_SET | \
+ NX_OF_FLAG_MAXSYNCCOP_SET)
+struct nx_of {
+ u32 flags;
+ u32 max_sg_len;
+ enum nx_status status;
+ struct alg_props ap[NX_MAX_FC][NX_MAX_MODE][3];
+};
+
+struct nx_stats {
+ atomic_t aes_ops;
+ atomic64_t aes_bytes;
+ atomic_t sha256_ops;
+ atomic64_t sha256_bytes;
+ atomic_t sha512_ops;
+ atomic64_t sha512_bytes;
+
+ atomic_t sync_ops;
+
+ atomic_t errors;
+ atomic_t last_error;
+ atomic_t last_error_pid;
+};
+
+struct nx_debugfs {
+ struct dentry *dfs_root;
+ struct dentry *dfs_aes_ops, *dfs_aes_bytes;
+ struct dentry *dfs_sha256_ops, *dfs_sha256_bytes;
+ struct dentry *dfs_sha512_ops, *dfs_sha512_bytes;
+ struct dentry *dfs_errors, *dfs_last_error, *dfs_last_error_pid;
+};
+
+struct nx_crypto_driver {
+ struct nx_stats stats;
+ struct nx_of of;
+ struct vio_dev *viodev;
+ struct vio_driver viodriver;
+ struct nx_debugfs dfs;
+};
+
+#define NX_GCM4106_NONCE_LEN (4)
+#define NX_GCM_CTR_OFFSET (12)
+struct nx_gcm_priv {
+ u8 iv[16];
+ u8 iauth_tag[16];
+ u8 nonce[NX_GCM4106_NONCE_LEN];
+};
+
+#define NX_CCM_AES_KEY_LEN (16)
+#define NX_CCM4309_AES_KEY_LEN (19)
+#define NX_CCM4309_NONCE_LEN (3)
+struct nx_ccm_priv {
+ u8 iv[16];
+ u8 b0[16];
+ u8 iauth_tag[16];
+ u8 oauth_tag[16];
+ u8 nonce[NX_CCM4309_NONCE_LEN];
+};
+
+struct nx_xcbc_priv {
+ u8 key[16];
+};
+
+struct nx_ctr_priv {
+ u8 iv[16];
+};
+
+struct nx_crypto_ctx {
+ spinlock_t lock; /* synchronize access to the context */
+ void *kmem; /* unaligned, kmalloc'd buffer */
+ size_t kmem_len; /* length of kmem */
+ struct nx_csbcpb *csbcpb; /* aligned page given to phyp @ hcall time */
+ struct vio_pfo_op op; /* operation struct with hcall parameters */
+ struct nx_csbcpb *csbcpb_aead; /* secondary csbcpb used by AEAD algs */
+ struct vio_pfo_op op_aead;/* operation struct for csbcpb_aead */
+
+ struct nx_sg *in_sg; /* aligned pointer into kmem to an sg list */
+ struct nx_sg *out_sg; /* aligned pointer into kmem to an sg list */
+
+ struct alg_props *ap; /* pointer into props based on our key size */
+ struct alg_props props[3];/* openFirmware properties for requests */
+ struct nx_stats *stats; /* pointer into an nx_crypto_driver for stats
+ reporting */
+
+ union {
+ struct nx_gcm_priv gcm;
+ struct nx_ccm_priv ccm;
+ struct nx_xcbc_priv xcbc;
+ struct nx_ctr_priv ctr;
+ } priv;
+};
+
+/* prototypes */
+int nx_crypto_ctx_aes_ccm_init(struct crypto_tfm *tfm);
+int nx_crypto_ctx_aes_gcm_init(struct crypto_tfm *tfm);
+int nx_crypto_ctx_aes_xcbc_init(struct crypto_tfm *tfm);
+int nx_crypto_ctx_aes_ctr_init(struct crypto_tfm *tfm);
+int nx_crypto_ctx_aes_cbc_init(struct crypto_tfm *tfm);
+int nx_crypto_ctx_aes_ecb_init(struct crypto_tfm *tfm);
+int nx_crypto_ctx_sha_init(struct crypto_tfm *tfm);
+void nx_crypto_ctx_exit(struct crypto_tfm *tfm);
+void nx_ctx_init(struct nx_crypto_ctx *nx_ctx, unsigned int function);
+int nx_hcall_sync(struct nx_crypto_ctx *ctx, struct vio_pfo_op *op,
+ u32 may_sleep);
+int nx_sha_build_sg_list(struct nx_crypto_ctx *, struct nx_sg *,
+ s64 *, unsigned int *, u8 *, u32);
+struct nx_sg *nx_build_sg_list(struct nx_sg *, u8 *, unsigned int *, u32);
+int nx_build_sg_lists(struct nx_crypto_ctx *, struct blkcipher_desc *,
+ struct scatterlist *, struct scatterlist *, unsigned int *,
+ unsigned int, u8 *);
+struct nx_sg *nx_walk_and_build(struct nx_sg *, unsigned int,
+ struct scatterlist *, unsigned int,
+ unsigned int *);
+
+#ifdef CONFIG_DEBUG_FS
+#define NX_DEBUGFS_INIT(drv) nx_debugfs_init(drv)
+#define NX_DEBUGFS_FINI(drv) nx_debugfs_fini(drv)
+
+int nx_debugfs_init(struct nx_crypto_driver *);
+void nx_debugfs_fini(struct nx_crypto_driver *);
+#else
+#define NX_DEBUGFS_INIT(drv) (0)
+#define NX_DEBUGFS_FINI(drv) (0)
+#endif
+
+#define NX_PAGE_NUM(x) ((u64)(x) & 0xfffffffffffff000ULL)
+
+extern struct crypto_alg nx_cbc_aes_alg;
+extern struct crypto_alg nx_ecb_aes_alg;
+extern struct crypto_alg nx_gcm_aes_alg;
+extern struct crypto_alg nx_gcm4106_aes_alg;
+extern struct crypto_alg nx_ctr_aes_alg;
+extern struct crypto_alg nx_ctr3686_aes_alg;
+extern struct crypto_alg nx_ccm_aes_alg;
+extern struct crypto_alg nx_ccm4309_aes_alg;
+extern struct shash_alg nx_shash_aes_xcbc_alg;
+extern struct shash_alg nx_shash_sha512_alg;
+extern struct shash_alg nx_shash_sha256_alg;
+
+extern struct nx_crypto_driver nx_driver;
+
+#define SCATTERWALK_TO_SG 1
+#define SCATTERWALK_FROM_SG 0
+
+#endif
diff --git a/drivers/crypto/nx/nx_csbcpb.h b/drivers/crypto/nx/nx_csbcpb.h
new file mode 100644
index 000000000..a304f956d
--- /dev/null
+++ b/drivers/crypto/nx/nx_csbcpb.h
@@ -0,0 +1,205 @@
+
+#ifndef __NX_CSBCPB_H__
+#define __NX_CSBCPB_H__
+
+struct cop_symcpb_aes_ecb {
+ u8 key[32];
+ u8 __rsvd[80];
+} __packed;
+
+struct cop_symcpb_aes_cbc {
+ u8 iv[16];
+ u8 key[32];
+ u8 cv[16];
+ u32 spbc;
+ u8 __rsvd[44];
+} __packed;
+
+struct cop_symcpb_aes_gca {
+ u8 in_pat[16];
+ u8 key[32];
+ u8 out_pat[16];
+ u32 spbc;
+ u8 __rsvd[44];
+} __packed;
+
+struct cop_symcpb_aes_gcm {
+ u8 in_pat_or_aad[16];
+ u8 iv_or_cnt[16];
+ u64 bit_length_aad;
+ u64 bit_length_data;
+ u8 in_s0[16];
+ u8 key[32];
+ u8 __rsvd1[16];
+ u8 out_pat_or_mac[16];
+ u8 out_s0[16];
+ u8 out_cnt[16];
+ u32 spbc;
+ u8 __rsvd2[12];
+} __packed;
+
+struct cop_symcpb_aes_ctr {
+ u8 iv[16];
+ u8 key[32];
+ u8 cv[16];
+ u32 spbc;
+ u8 __rsvd2[44];
+} __packed;
+
+struct cop_symcpb_aes_cca {
+ u8 b0[16];
+ u8 b1[16];
+ u8 key[16];
+ u8 out_pat_or_b0[16];
+ u32 spbc;
+ u8 __rsvd[44];
+} __packed;
+
+struct cop_symcpb_aes_ccm {
+ u8 in_pat_or_b0[16];
+ u8 iv_or_ctr[16];
+ u8 in_s0[16];
+ u8 key[16];
+ u8 __rsvd1[48];
+ u8 out_pat_or_mac[16];
+ u8 out_s0[16];
+ u8 out_ctr[16];
+ u32 spbc;
+ u8 __rsvd2[12];
+} __packed;
+
+struct cop_symcpb_aes_xcbc {
+ u8 cv[16];
+ u8 key[16];
+ u8 __rsvd1[16];
+ u8 out_cv_mac[16];
+ u32 spbc;
+ u8 __rsvd2[44];
+} __packed;
+
+struct cop_symcpb_sha256 {
+ u64 message_bit_length;
+ u64 __rsvd1;
+ u8 input_partial_digest[32];
+ u8 message_digest[32];
+ u32 spbc;
+ u8 __rsvd2[44];
+} __packed;
+
+struct cop_symcpb_sha512 {
+ u64 message_bit_length_hi;
+ u64 message_bit_length_lo;
+ u8 input_partial_digest[64];
+ u8 __rsvd1[32];
+ u8 message_digest[64];
+ u32 spbc;
+ u8 __rsvd2[76];
+} __packed;
+
+#define NX_FDM_INTERMEDIATE 0x01
+#define NX_FDM_CONTINUATION 0x02
+#define NX_FDM_ENDE_ENCRYPT 0x80
+
+#define NX_CPB_FDM(c) ((c)->cpb.hdr.fdm)
+#define NX_CPB_KS_DS(c) ((c)->cpb.hdr.ks_ds)
+
+#define NX_CPB_KEY_SIZE(c) (NX_CPB_KS_DS(c) >> 4)
+#define NX_CPB_SET_KEY_SIZE(c, x) NX_CPB_KS_DS(c) |= ((x) << 4)
+#define NX_CPB_SET_DIGEST_SIZE(c, x) NX_CPB_KS_DS(c) |= (x)
+
+struct cop_symcpb_header {
+ u8 mode;
+ u8 fdm;
+ u8 ks_ds;
+ u8 pad_byte;
+ u8 __rsvd[12];
+} __packed;
+
+struct cop_parameter_block {
+ struct cop_symcpb_header hdr;
+ union {
+ struct cop_symcpb_aes_ecb aes_ecb;
+ struct cop_symcpb_aes_cbc aes_cbc;
+ struct cop_symcpb_aes_gca aes_gca;
+ struct cop_symcpb_aes_gcm aes_gcm;
+ struct cop_symcpb_aes_cca aes_cca;
+ struct cop_symcpb_aes_ccm aes_ccm;
+ struct cop_symcpb_aes_ctr aes_ctr;
+ struct cop_symcpb_aes_xcbc aes_xcbc;
+ struct cop_symcpb_sha256 sha256;
+ struct cop_symcpb_sha512 sha512;
+ };
+} __packed;
+
+#define NX_CSB_VALID_BIT 0x80
+
+/* co-processor status block */
+struct cop_status_block {
+ u8 valid;
+ u8 crb_seq_number;
+ u8 completion_code;
+ u8 completion_extension;
+ u32 processed_byte_count;
+ u64 address;
+} __packed;
+
+/* Nest accelerator workbook section 4.4 */
+struct nx_csbcpb {
+ unsigned char __rsvd[112];
+ struct cop_status_block csb;
+ struct cop_parameter_block cpb;
+} __packed;
+
+/* nx_csbcpb related definitions */
+#define NX_MODE_AES_ECB 0
+#define NX_MODE_AES_CBC 1
+#define NX_MODE_AES_GMAC 2
+#define NX_MODE_AES_GCA 3
+#define NX_MODE_AES_GCM 4
+#define NX_MODE_AES_CCA 5
+#define NX_MODE_AES_CCM 6
+#define NX_MODE_AES_CTR 7
+#define NX_MODE_AES_XCBC_MAC 20
+#define NX_MODE_SHA 0
+#define NX_MODE_SHA_HMAC 1
+#define NX_MODE_AES_CBC_HMAC_ETA 8
+#define NX_MODE_AES_CBC_HMAC_ATE 9
+#define NX_MODE_AES_CBC_HMAC_EAA 10
+#define NX_MODE_AES_CTR_HMAC_ETA 12
+#define NX_MODE_AES_CTR_HMAC_ATE 13
+#define NX_MODE_AES_CTR_HMAC_EAA 14
+
+#define NX_FDM_CI_FULL 0
+#define NX_FDM_CI_FIRST 1
+#define NX_FDM_CI_LAST 2
+#define NX_FDM_CI_MIDDLE 3
+
+#define NX_FDM_PR_NONE 0
+#define NX_FDM_PR_PAD 1
+
+#define NX_KS_AES_128 1
+#define NX_KS_AES_192 2
+#define NX_KS_AES_256 3
+
+#define NX_DS_SHA256 2
+#define NX_DS_SHA512 3
+
+#define NX_FC_AES 0
+#define NX_FC_SHA 2
+#define NX_FC_AES_HMAC 6
+
+#define NX_MAX_FC (NX_FC_AES_HMAC + 1)
+#define NX_MAX_MODE (NX_MODE_AES_XCBC_MAC + 1)
+
+#define HCOP_FC_AES NX_FC_AES
+#define HCOP_FC_SHA NX_FC_SHA
+#define HCOP_FC_AES_HMAC NX_FC_AES_HMAC
+
+/* indices into the array of algorithm properties */
+#define NX_PROPS_AES_128 0
+#define NX_PROPS_AES_192 1
+#define NX_PROPS_AES_256 2
+#define NX_PROPS_SHA256 1
+#define NX_PROPS_SHA512 2
+
+#endif
diff --git a/drivers/crypto/nx/nx_debugfs.c b/drivers/crypto/nx/nx_debugfs.c
new file mode 100644
index 000000000..7ab2e8dcd
--- /dev/null
+++ b/drivers/crypto/nx/nx_debugfs.c
@@ -0,0 +1,103 @@
+/**
+ * debugfs routines supporting the Power 7+ Nest Accelerators driver
+ *
+ * Copyright (C) 2011-2012 International Business Machines Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 only.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ * Author: Kent Yoder <yoder1@us.ibm.com>
+ */
+
+#include <linux/device.h>
+#include <linux/kobject.h>
+#include <linux/string.h>
+#include <linux/debugfs.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/crypto.h>
+#include <crypto/hash.h>
+#include <asm/vio.h>
+
+#include "nx_csbcpb.h"
+#include "nx.h"
+
+#ifdef CONFIG_DEBUG_FS
+
+/*
+ * debugfs
+ *
+ * For documentation on these attributes, please see:
+ *
+ * Documentation/ABI/testing/debugfs-pfo-nx-crypto
+ */
+
+int nx_debugfs_init(struct nx_crypto_driver *drv)
+{
+ struct nx_debugfs *dfs = &drv->dfs;
+
+ dfs->dfs_root = debugfs_create_dir(NX_NAME, NULL);
+
+ dfs->dfs_aes_ops =
+ debugfs_create_u32("aes_ops",
+ S_IRUSR | S_IRGRP | S_IROTH,
+ dfs->dfs_root, (u32 *)&drv->stats.aes_ops);
+ dfs->dfs_sha256_ops =
+ debugfs_create_u32("sha256_ops",
+ S_IRUSR | S_IRGRP | S_IROTH,
+ dfs->dfs_root,
+ (u32 *)&drv->stats.sha256_ops);
+ dfs->dfs_sha512_ops =
+ debugfs_create_u32("sha512_ops",
+ S_IRUSR | S_IRGRP | S_IROTH,
+ dfs->dfs_root,
+ (u32 *)&drv->stats.sha512_ops);
+ dfs->dfs_aes_bytes =
+ debugfs_create_u64("aes_bytes",
+ S_IRUSR | S_IRGRP | S_IROTH,
+ dfs->dfs_root,
+ (u64 *)&drv->stats.aes_bytes);
+ dfs->dfs_sha256_bytes =
+ debugfs_create_u64("sha256_bytes",
+ S_IRUSR | S_IRGRP | S_IROTH,
+ dfs->dfs_root,
+ (u64 *)&drv->stats.sha256_bytes);
+ dfs->dfs_sha512_bytes =
+ debugfs_create_u64("sha512_bytes",
+ S_IRUSR | S_IRGRP | S_IROTH,
+ dfs->dfs_root,
+ (u64 *)&drv->stats.sha512_bytes);
+ dfs->dfs_errors =
+ debugfs_create_u32("errors",
+ S_IRUSR | S_IRGRP | S_IROTH,
+ dfs->dfs_root, (u32 *)&drv->stats.errors);
+ dfs->dfs_last_error =
+ debugfs_create_u32("last_error",
+ S_IRUSR | S_IRGRP | S_IROTH,
+ dfs->dfs_root,
+ (u32 *)&drv->stats.last_error);
+ dfs->dfs_last_error_pid =
+ debugfs_create_u32("last_error_pid",
+ S_IRUSR | S_IRGRP | S_IROTH,
+ dfs->dfs_root,
+ (u32 *)&drv->stats.last_error_pid);
+ return 0;
+}
+
+void
+nx_debugfs_fini(struct nx_crypto_driver *drv)
+{
+ debugfs_remove_recursive(drv->dfs.dfs_root);
+}
+
+#endif
--
cgit v1.2.3-54-g00ecf