summaryrefslogtreecommitdiff
path: root/drivers/crypto/nx
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/crypto/nx')
-rw-r--r--drivers/crypto/nx/Kconfig61
-rw-r--r--drivers/crypto/nx/Makefile9
-rw-r--r--drivers/crypto/nx/nx-842-crypto.c580
-rw-r--r--drivers/crypto/nx/nx-842-platform.c84
-rw-r--r--drivers/crypto/nx/nx-842-powernv.c637
-rw-r--r--drivers/crypto/nx/nx-842-pseries.c1140
-rw-r--r--drivers/crypto/nx/nx-842.c1610
-rw-r--r--drivers/crypto/nx/nx-842.h144
-rw-r--r--drivers/crypto/nx/nx-aes-gcm.c110
-rw-r--r--drivers/crypto/nx/nx-sha256.c27
-rw-r--r--drivers/crypto/nx/nx-sha512.c28
-rw-r--r--drivers/crypto/nx/nx.c169
-rw-r--r--drivers/crypto/nx/nx.h7
13 files changed, 2907 insertions, 1699 deletions
diff --git a/drivers/crypto/nx/Kconfig b/drivers/crypto/nx/Kconfig
index f82616621..e421c96c7 100644
--- a/drivers/crypto/nx/Kconfig
+++ b/drivers/crypto/nx/Kconfig
@@ -1,26 +1,55 @@
+
config CRYPTO_DEV_NX_ENCRYPT
- tristate "Encryption acceleration support"
- depends on PPC64 && IBMVIO
+ tristate "Encryption acceleration support on pSeries platform"
+ depends on PPC_PSERIES && IBMVIO && !CPU_LITTLE_ENDIAN
default y
select CRYPTO_AES
- select CRYPTO_CBC
- select CRYPTO_ECB
select CRYPTO_CCM
- select CRYPTO_GCM
- select CRYPTO_AUTHENC
- select CRYPTO_XCBC
- select CRYPTO_SHA256
- select CRYPTO_SHA512
help
- Support for Power7+ in-Nest encryption acceleration. This
- module supports acceleration for AES and SHA2 algorithms. If you
- choose 'M' here, this module will be called nx_crypto.
+ Support for PowerPC Nest (NX) encryption acceleration. This
+ module supports acceleration for AES and SHA2 algorithms on
+ the pSeries platform. If you choose 'M' here, this module
+ will be called nx_crypto.
config CRYPTO_DEV_NX_COMPRESS
tristate "Compression acceleration support"
- depends on PPC64 && IBMVIO
default y
help
- Support for Power7+ in-Nest compression acceleration. This
- module supports acceleration for AES and SHA2 algorithms. If you
- choose 'M' here, this module will be called nx_compress.
+ Support for PowerPC Nest (NX) compression acceleration. This
+ module supports acceleration for compressing memory with the 842
+ algorithm. One of the platform drivers must be selected also.
+ If you choose 'M' here, this module will be called nx_compress.
+
+if CRYPTO_DEV_NX_COMPRESS
+
+config CRYPTO_DEV_NX_COMPRESS_PSERIES
+ tristate "Compression acceleration support on pSeries platform"
+ depends on PPC_PSERIES && IBMVIO
+ default y
+ help
+ Support for PowerPC Nest (NX) compression acceleration. This
+ module supports acceleration for compressing memory with the 842
+ algorithm. This supports NX hardware on the pSeries platform.
+ If you choose 'M' here, this module will be called nx_compress_pseries.
+
+config CRYPTO_DEV_NX_COMPRESS_POWERNV
+ tristate "Compression acceleration support on PowerNV platform"
+ depends on PPC_POWERNV
+ default y
+ help
+ Support for PowerPC Nest (NX) compression acceleration. This
+ module supports acceleration for compressing memory with the 842
+ algorithm. This supports NX hardware on the PowerNV platform.
+ If you choose 'M' here, this module will be called nx_compress_powernv.
+
+config CRYPTO_DEV_NX_COMPRESS_CRYPTO
+ tristate "Compression acceleration cryptographic interface"
+ select CRYPTO_ALGAPI
+ select 842_DECOMPRESS
+ default y
+ help
+ Support for PowerPC Nest (NX) accelerators using the cryptographic
+ API. If you choose 'M' here, this module will be called
+ nx_compress_crypto.
+
+endif
diff --git a/drivers/crypto/nx/Makefile b/drivers/crypto/nx/Makefile
index bb770ea45..e1684f5ad 100644
--- a/drivers/crypto/nx/Makefile
+++ b/drivers/crypto/nx/Makefile
@@ -10,5 +10,12 @@ nx-crypto-objs := nx.o \
nx-sha256.o \
nx-sha512.o
-obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS) += nx-compress.o
+obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS) += nx-compress.o nx-compress-platform.o
+obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_PSERIES) += nx-compress-pseries.o
+obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_POWERNV) += nx-compress-powernv.o
+obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_CRYPTO) += nx-compress-crypto.o
nx-compress-objs := nx-842.o
+nx-compress-platform-objs := nx-842-platform.o
+nx-compress-pseries-objs := nx-842-pseries.o
+nx-compress-powernv-objs := nx-842-powernv.o
+nx-compress-crypto-objs := nx-842-crypto.o
diff --git a/drivers/crypto/nx/nx-842-crypto.c b/drivers/crypto/nx/nx-842-crypto.c
new file mode 100644
index 000000000..d53a1dcd7
--- /dev/null
+++ b/drivers/crypto/nx/nx-842-crypto.c
@@ -0,0 +1,580 @@
+/*
+ * Cryptographic API for the NX-842 hardware compression.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * Copyright (C) IBM Corporation, 2011-2015
+ *
+ * Original Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
+ * Seth Jennings <sjenning@linux.vnet.ibm.com>
+ *
+ * Rewrite: Dan Streetman <ddstreet@ieee.org>
+ *
+ * This is an interface to the NX-842 compression hardware in PowerPC
+ * processors. Most of the complexity of this drvier is due to the fact that
+ * the NX-842 compression hardware requires the input and output data buffers
+ * to be specifically aligned, to be a specific multiple in length, and within
+ * specific minimum and maximum lengths. Those restrictions, provided by the
+ * nx-842 driver via nx842_constraints, mean this driver must use bounce
+ * buffers and headers to correct misaligned in or out buffers, and to split
+ * input buffers that are too large.
+ *
+ * This driver will fall back to software decompression if the hardware
+ * decompression fails, so this driver's decompression should never fail as
+ * long as the provided compressed buffer is valid. Any compressed buffer
+ * created by this driver will have a header (except ones where the input
+ * perfectly matches the constraints); so users of this driver cannot simply
+ * pass a compressed buffer created by this driver over to the 842 software
+ * decompression library. Instead, users must use this driver to decompress;
+ * if the hardware fails or is unavailable, the compressed buffer will be
+ * parsed and the header removed, and the raw 842 buffer(s) passed to the 842
+ * software decompression library.
+ *
+ * This does not fall back to software compression, however, since the caller
+ * of this function is specifically requesting hardware compression; if the
+ * hardware compression fails, the caller can fall back to software
+ * compression, and the raw 842 compressed buffer that the software compressor
+ * creates can be passed to this driver for hardware decompression; any
+ * buffer without our specific header magic is assumed to be a raw 842 buffer
+ * and passed directly to the hardware. Note that the software compression
+ * library will produce a compressed buffer that is incompatible with the
+ * hardware decompressor if the original input buffer length is not a multiple
+ * of 8; if such a compressed buffer is passed to this driver for
+ * decompression, the hardware will reject it and this driver will then pass
+ * it over to the software library for decompression.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/crypto.h>
+#include <linux/vmalloc.h>
+#include <linux/sw842.h>
+#include <linux/ratelimit.h>
+
+#include "nx-842.h"
+
+/* The first 5 bits of this magic are 0x1f, which is an invalid 842 5-bit
+ * template (see lib/842/842.h), so this magic number will never appear at
+ * the start of a raw 842 compressed buffer. That is important, as any buffer
+ * passed to us without this magic is assumed to be a raw 842 compressed
+ * buffer, and passed directly to the hardware to decompress.
+ */
+#define NX842_CRYPTO_MAGIC (0xf842)
+#define NX842_CRYPTO_GROUP_MAX (0x20)
+#define NX842_CRYPTO_HEADER_SIZE(g) \
+ (sizeof(struct nx842_crypto_header) + \
+ sizeof(struct nx842_crypto_header_group) * (g))
+#define NX842_CRYPTO_HEADER_MAX_SIZE \
+ NX842_CRYPTO_HEADER_SIZE(NX842_CRYPTO_GROUP_MAX)
+
+/* bounce buffer size */
+#define BOUNCE_BUFFER_ORDER (2)
+#define BOUNCE_BUFFER_SIZE \
+ ((unsigned int)(PAGE_SIZE << BOUNCE_BUFFER_ORDER))
+
+/* try longer on comp because we can fallback to sw decomp if hw is busy */
+#define COMP_BUSY_TIMEOUT (250) /* ms */
+#define DECOMP_BUSY_TIMEOUT (50) /* ms */
+
+struct nx842_crypto_header_group {
+ __be16 padding; /* unused bytes at start of group */
+ __be32 compressed_length; /* compressed bytes in group */
+ __be32 uncompressed_length; /* bytes after decompression */
+} __packed;
+
+struct nx842_crypto_header {
+ __be16 magic; /* NX842_CRYPTO_MAGIC */
+ __be16 ignore; /* decompressed end bytes to ignore */
+ u8 groups; /* total groups in this header */
+ struct nx842_crypto_header_group group[];
+} __packed;
+
+struct nx842_crypto_param {
+ u8 *in;
+ unsigned int iremain;
+ u8 *out;
+ unsigned int oremain;
+ unsigned int ototal;
+};
+
+static int update_param(struct nx842_crypto_param *p,
+ unsigned int slen, unsigned int dlen)
+{
+ if (p->iremain < slen)
+ return -EOVERFLOW;
+ if (p->oremain < dlen)
+ return -ENOSPC;
+
+ p->in += slen;
+ p->iremain -= slen;
+ p->out += dlen;
+ p->oremain -= dlen;
+ p->ototal += dlen;
+
+ return 0;
+}
+
+struct nx842_crypto_ctx {
+ u8 *wmem;
+ u8 *sbounce, *dbounce;
+
+ struct nx842_crypto_header header;
+ struct nx842_crypto_header_group group[NX842_CRYPTO_GROUP_MAX];
+};
+
+static int nx842_crypto_init(struct crypto_tfm *tfm)
+{
+ struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ ctx->wmem = kmalloc(nx842_workmem_size(), GFP_KERNEL);
+ ctx->sbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER);
+ ctx->dbounce = (u8 *)__get_free_pages(GFP_KERNEL, BOUNCE_BUFFER_ORDER);
+ if (!ctx->wmem || !ctx->sbounce || !ctx->dbounce) {
+ kfree(ctx->wmem);
+ free_page((unsigned long)ctx->sbounce);
+ free_page((unsigned long)ctx->dbounce);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static void nx842_crypto_exit(struct crypto_tfm *tfm)
+{
+ struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ kfree(ctx->wmem);
+ free_page((unsigned long)ctx->sbounce);
+ free_page((unsigned long)ctx->dbounce);
+}
+
+static int read_constraints(struct nx842_constraints *c)
+{
+ int ret;
+
+ ret = nx842_constraints(c);
+ if (ret) {
+ pr_err_ratelimited("could not get nx842 constraints : %d\n",
+ ret);
+ return ret;
+ }
+
+ /* limit maximum, to always have enough bounce buffer to decompress */
+ if (c->maximum > BOUNCE_BUFFER_SIZE) {
+ c->maximum = BOUNCE_BUFFER_SIZE;
+ pr_info_once("limiting nx842 maximum to %x\n", c->maximum);
+ }
+
+ return 0;
+}
+
+static int nx842_crypto_add_header(struct nx842_crypto_header *hdr, u8 *buf)
+{
+ int s = NX842_CRYPTO_HEADER_SIZE(hdr->groups);
+
+ /* compress should have added space for header */
+ if (s > be16_to_cpu(hdr->group[0].padding)) {
+ pr_err("Internal error: no space for header\n");
+ return -EINVAL;
+ }
+
+ memcpy(buf, hdr, s);
+
+ print_hex_dump_debug("header ", DUMP_PREFIX_OFFSET, 16, 1, buf, s, 0);
+
+ return 0;
+}
+
+static int compress(struct nx842_crypto_ctx *ctx,
+ struct nx842_crypto_param *p,
+ struct nx842_crypto_header_group *g,
+ struct nx842_constraints *c,
+ u16 *ignore,
+ unsigned int hdrsize)
+{
+ unsigned int slen = p->iremain, dlen = p->oremain, tmplen;
+ unsigned int adj_slen = slen;
+ u8 *src = p->in, *dst = p->out;
+ int ret, dskip = 0;
+ ktime_t timeout;
+
+ if (p->iremain == 0)
+ return -EOVERFLOW;
+
+ if (p->oremain == 0 || hdrsize + c->minimum > dlen)
+ return -ENOSPC;
+
+ if (slen % c->multiple)
+ adj_slen = round_up(slen, c->multiple);
+ if (slen < c->minimum)
+ adj_slen = c->minimum;
+ if (slen > c->maximum)
+ adj_slen = slen = c->maximum;
+ if (adj_slen > slen || (u64)src % c->alignment) {
+ adj_slen = min(adj_slen, BOUNCE_BUFFER_SIZE);
+ slen = min(slen, BOUNCE_BUFFER_SIZE);
+ if (adj_slen > slen)
+ memset(ctx->sbounce + slen, 0, adj_slen - slen);
+ memcpy(ctx->sbounce, src, slen);
+ src = ctx->sbounce;
+ slen = adj_slen;
+ pr_debug("using comp sbounce buffer, len %x\n", slen);
+ }
+
+ dst += hdrsize;
+ dlen -= hdrsize;
+
+ if ((u64)dst % c->alignment) {
+ dskip = (int)(PTR_ALIGN(dst, c->alignment) - dst);
+ dst += dskip;
+ dlen -= dskip;
+ }
+ if (dlen % c->multiple)
+ dlen = round_down(dlen, c->multiple);
+ if (dlen < c->minimum) {
+nospc:
+ dst = ctx->dbounce;
+ dlen = min(p->oremain, BOUNCE_BUFFER_SIZE);
+ dlen = round_down(dlen, c->multiple);
+ dskip = 0;
+ pr_debug("using comp dbounce buffer, len %x\n", dlen);
+ }
+ if (dlen > c->maximum)
+ dlen = c->maximum;
+
+ tmplen = dlen;
+ timeout = ktime_add_ms(ktime_get(), COMP_BUSY_TIMEOUT);
+ do {
+ dlen = tmplen; /* reset dlen, if we're retrying */
+ ret = nx842_compress(src, slen, dst, &dlen, ctx->wmem);
+ /* possibly we should reduce the slen here, instead of
+ * retrying with the dbounce buffer?
+ */
+ if (ret == -ENOSPC && dst != ctx->dbounce)
+ goto nospc;
+ } while (ret == -EBUSY && ktime_before(ktime_get(), timeout));
+ if (ret)
+ return ret;
+
+ dskip += hdrsize;
+
+ if (dst == ctx->dbounce)
+ memcpy(p->out + dskip, dst, dlen);
+
+ g->padding = cpu_to_be16(dskip);
+ g->compressed_length = cpu_to_be32(dlen);
+ g->uncompressed_length = cpu_to_be32(slen);
+
+ if (p->iremain < slen) {
+ *ignore = slen - p->iremain;
+ slen = p->iremain;
+ }
+
+ pr_debug("compress slen %x ignore %x dlen %x padding %x\n",
+ slen, *ignore, dlen, dskip);
+
+ return update_param(p, slen, dskip + dlen);
+}
+
+static int nx842_crypto_compress(struct crypto_tfm *tfm,
+ const u8 *src, unsigned int slen,
+ u8 *dst, unsigned int *dlen)
+{
+ struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct nx842_crypto_header *hdr = &ctx->header;
+ struct nx842_crypto_param p;
+ struct nx842_constraints c;
+ unsigned int groups, hdrsize, h;
+ int ret, n;
+ bool add_header;
+ u16 ignore = 0;
+
+ p.in = (u8 *)src;
+ p.iremain = slen;
+ p.out = dst;
+ p.oremain = *dlen;
+ p.ototal = 0;
+
+ *dlen = 0;
+
+ ret = read_constraints(&c);
+ if (ret)
+ return ret;
+
+ groups = min_t(unsigned int, NX842_CRYPTO_GROUP_MAX,
+ DIV_ROUND_UP(p.iremain, c.maximum));
+ hdrsize = NX842_CRYPTO_HEADER_SIZE(groups);
+
+ /* skip adding header if the buffers meet all constraints */
+ add_header = (p.iremain % c.multiple ||
+ p.iremain < c.minimum ||
+ p.iremain > c.maximum ||
+ (u64)p.in % c.alignment ||
+ p.oremain % c.multiple ||
+ p.oremain < c.minimum ||
+ p.oremain > c.maximum ||
+ (u64)p.out % c.alignment);
+
+ hdr->magic = cpu_to_be16(NX842_CRYPTO_MAGIC);
+ hdr->groups = 0;
+ hdr->ignore = 0;
+
+ while (p.iremain > 0) {
+ n = hdr->groups++;
+ if (hdr->groups > NX842_CRYPTO_GROUP_MAX)
+ return -ENOSPC;
+
+ /* header goes before first group */
+ h = !n && add_header ? hdrsize : 0;
+
+ if (ignore)
+ pr_warn("interal error, ignore is set %x\n", ignore);
+
+ ret = compress(ctx, &p, &hdr->group[n], &c, &ignore, h);
+ if (ret)
+ return ret;
+ }
+
+ if (!add_header && hdr->groups > 1) {
+ pr_err("Internal error: No header but multiple groups\n");
+ return -EINVAL;
+ }
+
+ /* ignore indicates the input stream needed to be padded */
+ hdr->ignore = cpu_to_be16(ignore);
+ if (ignore)
+ pr_debug("marked %d bytes as ignore\n", ignore);
+
+ if (add_header)
+ ret = nx842_crypto_add_header(hdr, dst);
+ if (ret)
+ return ret;
+
+ *dlen = p.ototal;
+
+ pr_debug("compress total slen %x dlen %x\n", slen, *dlen);
+
+ return 0;
+}
+
+static int decompress(struct nx842_crypto_ctx *ctx,
+ struct nx842_crypto_param *p,
+ struct nx842_crypto_header_group *g,
+ struct nx842_constraints *c,
+ u16 ignore,
+ bool usehw)
+{
+ unsigned int slen = be32_to_cpu(g->compressed_length);
+ unsigned int required_len = be32_to_cpu(g->uncompressed_length);
+ unsigned int dlen = p->oremain, tmplen;
+ unsigned int adj_slen = slen;
+ u8 *src = p->in, *dst = p->out;
+ u16 padding = be16_to_cpu(g->padding);
+ int ret, spadding = 0, dpadding = 0;
+ ktime_t timeout;
+
+ if (!slen || !required_len)
+ return -EINVAL;
+
+ if (p->iremain <= 0 || padding + slen > p->iremain)
+ return -EOVERFLOW;
+
+ if (p->oremain <= 0 || required_len - ignore > p->oremain)
+ return -ENOSPC;
+
+ src += padding;
+
+ if (!usehw)
+ goto usesw;
+
+ if (slen % c->multiple)
+ adj_slen = round_up(slen, c->multiple);
+ if (slen < c->minimum)
+ adj_slen = c->minimum;
+ if (slen > c->maximum)
+ goto usesw;
+ if (slen < adj_slen || (u64)src % c->alignment) {
+ /* we can append padding bytes because the 842 format defines
+ * an "end" template (see lib/842/842_decompress.c) and will
+ * ignore any bytes following it.
+ */
+ if (slen < adj_slen)
+ memset(ctx->sbounce + slen, 0, adj_slen - slen);
+ memcpy(ctx->sbounce, src, slen);
+ src = ctx->sbounce;
+ spadding = adj_slen - slen;
+ slen = adj_slen;
+ pr_debug("using decomp sbounce buffer, len %x\n", slen);
+ }
+
+ if (dlen % c->multiple)
+ dlen = round_down(dlen, c->multiple);
+ if (dlen < required_len || (u64)dst % c->alignment) {
+ dst = ctx->dbounce;
+ dlen = min(required_len, BOUNCE_BUFFER_SIZE);
+ pr_debug("using decomp dbounce buffer, len %x\n", dlen);
+ }
+ if (dlen < c->minimum)
+ goto usesw;
+ if (dlen > c->maximum)
+ dlen = c->maximum;
+
+ tmplen = dlen;
+ timeout = ktime_add_ms(ktime_get(), DECOMP_BUSY_TIMEOUT);
+ do {
+ dlen = tmplen; /* reset dlen, if we're retrying */
+ ret = nx842_decompress(src, slen, dst, &dlen, ctx->wmem);
+ } while (ret == -EBUSY && ktime_before(ktime_get(), timeout));
+ if (ret) {
+usesw:
+ /* reset everything, sw doesn't have constraints */
+ src = p->in + padding;
+ slen = be32_to_cpu(g->compressed_length);
+ spadding = 0;
+ dst = p->out;
+ dlen = p->oremain;
+ dpadding = 0;
+ if (dlen < required_len) { /* have ignore bytes */
+ dst = ctx->dbounce;
+ dlen = BOUNCE_BUFFER_SIZE;
+ }
+ pr_info_ratelimited("using software 842 decompression\n");
+ ret = sw842_decompress(src, slen, dst, &dlen);
+ }
+ if (ret)
+ return ret;
+
+ slen -= spadding;
+
+ dlen -= ignore;
+ if (ignore)
+ pr_debug("ignoring last %x bytes\n", ignore);
+
+ if (dst == ctx->dbounce)
+ memcpy(p->out, dst, dlen);
+
+ pr_debug("decompress slen %x padding %x dlen %x ignore %x\n",
+ slen, padding, dlen, ignore);
+
+ return update_param(p, slen + padding, dlen);
+}
+
+static int nx842_crypto_decompress(struct crypto_tfm *tfm,
+ const u8 *src, unsigned int slen,
+ u8 *dst, unsigned int *dlen)
+{
+ struct nx842_crypto_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct nx842_crypto_header *hdr;
+ struct nx842_crypto_param p;
+ struct nx842_constraints c;
+ int n, ret, hdr_len;
+ u16 ignore = 0;
+ bool usehw = true;
+
+ p.in = (u8 *)src;
+ p.iremain = slen;
+ p.out = dst;
+ p.oremain = *dlen;
+ p.ototal = 0;
+
+ *dlen = 0;
+
+ if (read_constraints(&c))
+ usehw = false;
+
+ hdr = (struct nx842_crypto_header *)src;
+
+ /* If it doesn't start with our header magic number, assume it's a raw
+ * 842 compressed buffer and pass it directly to the hardware driver
+ */
+ if (be16_to_cpu(hdr->magic) != NX842_CRYPTO_MAGIC) {
+ struct nx842_crypto_header_group g = {
+ .padding = 0,
+ .compressed_length = cpu_to_be32(p.iremain),
+ .uncompressed_length = cpu_to_be32(p.oremain),
+ };
+
+ ret = decompress(ctx, &p, &g, &c, 0, usehw);
+ if (ret)
+ return ret;
+
+ *dlen = p.ototal;
+
+ return 0;
+ }
+
+ if (!hdr->groups) {
+ pr_err("header has no groups\n");
+ return -EINVAL;
+ }
+ if (hdr->groups > NX842_CRYPTO_GROUP_MAX) {
+ pr_err("header has too many groups %x, max %x\n",
+ hdr->groups, NX842_CRYPTO_GROUP_MAX);
+ return -EINVAL;
+ }
+
+ hdr_len = NX842_CRYPTO_HEADER_SIZE(hdr->groups);
+ if (hdr_len > slen)
+ return -EOVERFLOW;
+
+ memcpy(&ctx->header, src, hdr_len);
+ hdr = &ctx->header;
+
+ for (n = 0; n < hdr->groups; n++) {
+ /* ignore applies to last group */
+ if (n + 1 == hdr->groups)
+ ignore = be16_to_cpu(hdr->ignore);
+
+ ret = decompress(ctx, &p, &hdr->group[n], &c, ignore, usehw);
+ if (ret)
+ return ret;
+ }
+
+ *dlen = p.ototal;
+
+ pr_debug("decompress total slen %x dlen %x\n", slen, *dlen);
+
+ return 0;
+}
+
+static struct crypto_alg alg = {
+ .cra_name = "842",
+ .cra_driver_name = "842-nx",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
+ .cra_ctxsize = sizeof(struct nx842_crypto_ctx),
+ .cra_module = THIS_MODULE,
+ .cra_init = nx842_crypto_init,
+ .cra_exit = nx842_crypto_exit,
+ .cra_u = { .compress = {
+ .coa_compress = nx842_crypto_compress,
+ .coa_decompress = nx842_crypto_decompress } }
+};
+
+static int __init nx842_crypto_mod_init(void)
+{
+ return crypto_register_alg(&alg);
+}
+module_init(nx842_crypto_mod_init);
+
+static void __exit nx842_crypto_mod_exit(void)
+{
+ crypto_unregister_alg(&alg);
+}
+module_exit(nx842_crypto_mod_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("IBM PowerPC Nest (NX) 842 Hardware Compression Interface");
+MODULE_ALIAS_CRYPTO("842");
+MODULE_ALIAS_CRYPTO("842-nx");
+MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");
diff --git a/drivers/crypto/nx/nx-842-platform.c b/drivers/crypto/nx/nx-842-platform.c
new file mode 100644
index 000000000..664f13dd0
--- /dev/null
+++ b/drivers/crypto/nx/nx-842-platform.c
@@ -0,0 +1,84 @@
+
+#include "nx-842.h"
+
+/* this is needed, separate from the main nx-842.c driver, because that main
+ * driver loads the platform drivers during its init(), and it expects one
+ * (or none) of the platform drivers to set this pointer to its driver.
+ * That means this pointer can't be in the main nx-842 driver, because it
+ * wouldn't be accessible until after the main driver loaded, which wouldn't
+ * be possible as it's waiting for the platform driver to load. So place it
+ * here.
+ */
+static struct nx842_driver *driver;
+static DEFINE_SPINLOCK(driver_lock);
+
+struct nx842_driver *nx842_platform_driver(void)
+{
+ return driver;
+}
+EXPORT_SYMBOL_GPL(nx842_platform_driver);
+
+bool nx842_platform_driver_set(struct nx842_driver *_driver)
+{
+ bool ret = false;
+
+ spin_lock(&driver_lock);
+
+ if (!driver) {
+ driver = _driver;
+ ret = true;
+ } else
+ WARN(1, "can't set platform driver, already set to %s\n",
+ driver->name);
+
+ spin_unlock(&driver_lock);
+ return ret;
+}
+EXPORT_SYMBOL_GPL(nx842_platform_driver_set);
+
+/* only call this from the platform driver exit function */
+void nx842_platform_driver_unset(struct nx842_driver *_driver)
+{
+ spin_lock(&driver_lock);
+
+ if (driver == _driver)
+ driver = NULL;
+ else if (driver)
+ WARN(1, "can't unset platform driver %s, currently set to %s\n",
+ _driver->name, driver->name);
+ else
+ WARN(1, "can't unset platform driver, already unset\n");
+
+ spin_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(nx842_platform_driver_unset);
+
+bool nx842_platform_driver_get(void)
+{
+ bool ret = false;
+
+ spin_lock(&driver_lock);
+
+ if (driver)
+ ret = try_module_get(driver->owner);
+
+ spin_unlock(&driver_lock);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(nx842_platform_driver_get);
+
+void nx842_platform_driver_put(void)
+{
+ spin_lock(&driver_lock);
+
+ if (driver)
+ module_put(driver->owner);
+
+ spin_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(nx842_platform_driver_put);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");
+MODULE_DESCRIPTION("842 H/W Compression platform driver");
diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c
new file mode 100644
index 000000000..33b3b0abf
--- /dev/null
+++ b/drivers/crypto/nx/nx-842-powernv.c
@@ -0,0 +1,637 @@
+/*
+ * Driver for IBM PowerNV 842 compression accelerator
+ *
+ * Copyright (C) 2015 Dan Streetman, IBM Corp
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include "nx-842.h"
+
+#include <linux/timer.h>
+
+#include <asm/prom.h>
+#include <asm/icswx.h>
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");
+MODULE_DESCRIPTION("842 H/W Compression driver for IBM PowerNV processors");
+
+#define WORKMEM_ALIGN (CRB_ALIGN)
+#define CSB_WAIT_MAX (5000) /* ms */
+
+struct nx842_workmem {
+ /* Below fields must be properly aligned */
+ struct coprocessor_request_block crb; /* CRB_ALIGN align */
+ struct data_descriptor_entry ddl_in[DDL_LEN_MAX]; /* DDE_ALIGN align */
+ struct data_descriptor_entry ddl_out[DDL_LEN_MAX]; /* DDE_ALIGN align */
+ /* Above fields must be properly aligned */
+
+ ktime_t start;
+
+ char padding[WORKMEM_ALIGN]; /* unused, to allow alignment */
+} __packed __aligned(WORKMEM_ALIGN);
+
+struct nx842_coproc {
+ unsigned int chip_id;
+ unsigned int ct;
+ unsigned int ci;
+ struct list_head list;
+};
+
+/* no cpu hotplug on powernv, so this list never changes after init */
+static LIST_HEAD(nx842_coprocs);
+static unsigned int nx842_ct;
+
+/**
+ * setup_indirect_dde - Setup an indirect DDE
+ *
+ * The DDE is setup with the the DDE count, byte count, and address of
+ * first direct DDE in the list.
+ */
+static void setup_indirect_dde(struct data_descriptor_entry *dde,
+ struct data_descriptor_entry *ddl,
+ unsigned int dde_count, unsigned int byte_count)
+{
+ dde->flags = 0;
+ dde->count = dde_count;
+ dde->index = 0;
+ dde->length = cpu_to_be32(byte_count);
+ dde->address = cpu_to_be64(nx842_get_pa(ddl));
+}
+
+/**
+ * setup_direct_dde - Setup single DDE from buffer
+ *
+ * The DDE is setup with the buffer and length. The buffer must be properly
+ * aligned. The used length is returned.
+ * Returns:
+ * N Successfully set up DDE with N bytes
+ */
+static unsigned int setup_direct_dde(struct data_descriptor_entry *dde,
+ unsigned long pa, unsigned int len)
+{
+ unsigned int l = min_t(unsigned int, len, LEN_ON_PAGE(pa));
+
+ dde->flags = 0;
+ dde->count = 0;
+ dde->index = 0;
+ dde->length = cpu_to_be32(l);
+ dde->address = cpu_to_be64(pa);
+
+ return l;
+}
+
+/**
+ * setup_ddl - Setup DDL from buffer
+ *
+ * Returns:
+ * 0 Successfully set up DDL
+ */
+static int setup_ddl(struct data_descriptor_entry *dde,
+ struct data_descriptor_entry *ddl,
+ unsigned char *buf, unsigned int len,
+ bool in)
+{
+ unsigned long pa = nx842_get_pa(buf);
+ int i, ret, total_len = len;
+
+ if (!IS_ALIGNED(pa, DDE_BUFFER_ALIGN)) {
+ pr_debug("%s buffer pa 0x%lx not 0x%x-byte aligned\n",
+ in ? "input" : "output", pa, DDE_BUFFER_ALIGN);
+ return -EINVAL;
+ }
+
+ /* only need to check last mult; since buffer must be
+ * DDE_BUFFER_ALIGN aligned, and that is a multiple of
+ * DDE_BUFFER_SIZE_MULT, and pre-last page DDE buffers
+ * are guaranteed a multiple of DDE_BUFFER_SIZE_MULT.
+ */
+ if (len % DDE_BUFFER_LAST_MULT) {
+ pr_debug("%s buffer len 0x%x not a multiple of 0x%x\n",
+ in ? "input" : "output", len, DDE_BUFFER_LAST_MULT);
+ if (in)
+ return -EINVAL;
+ len = round_down(len, DDE_BUFFER_LAST_MULT);
+ }
+
+ /* use a single direct DDE */
+ if (len <= LEN_ON_PAGE(pa)) {
+ ret = setup_direct_dde(dde, pa, len);
+ WARN_ON(ret < len);
+ return 0;
+ }
+
+ /* use the DDL */
+ for (i = 0; i < DDL_LEN_MAX && len > 0; i++) {
+ ret = setup_direct_dde(&ddl[i], pa, len);
+ buf += ret;
+ len -= ret;
+ pa = nx842_get_pa(buf);
+ }
+
+ if (len > 0) {
+ pr_debug("0x%x total %s bytes 0x%x too many for DDL.\n",
+ total_len, in ? "input" : "output", len);
+ if (in)
+ return -EMSGSIZE;
+ total_len -= len;
+ }
+ setup_indirect_dde(dde, ddl, i, total_len);
+
+ return 0;
+}
+
+#define CSB_ERR(csb, msg, ...) \
+ pr_err("ERROR: " msg " : %02x %02x %02x %02x %08x\n", \
+ ##__VA_ARGS__, (csb)->flags, \
+ (csb)->cs, (csb)->cc, (csb)->ce, \
+ be32_to_cpu((csb)->count))
+
+#define CSB_ERR_ADDR(csb, msg, ...) \
+ CSB_ERR(csb, msg " at %lx", ##__VA_ARGS__, \
+ (unsigned long)be64_to_cpu((csb)->address))
+
+/**
+ * wait_for_csb
+ */
+static int wait_for_csb(struct nx842_workmem *wmem,
+ struct coprocessor_status_block *csb)
+{
+ ktime_t start = wmem->start, now = ktime_get();
+ ktime_t timeout = ktime_add_ms(start, CSB_WAIT_MAX);
+
+ while (!(ACCESS_ONCE(csb->flags) & CSB_V)) {
+ cpu_relax();
+ now = ktime_get();
+ if (ktime_after(now, timeout))
+ break;
+ }
+
+ /* hw has updated csb and output buffer */
+ barrier();
+
+ /* check CSB flags */
+ if (!(csb->flags & CSB_V)) {
+ CSB_ERR(csb, "CSB still not valid after %ld us, giving up",
+ (long)ktime_us_delta(now, start));
+ return -ETIMEDOUT;
+ }
+ if (csb->flags & CSB_F) {
+ CSB_ERR(csb, "Invalid CSB format");
+ return -EPROTO;
+ }
+ if (csb->flags & CSB_CH) {
+ CSB_ERR(csb, "Invalid CSB chaining state");
+ return -EPROTO;
+ }
+
+ /* verify CSB completion sequence is 0 */
+ if (csb->cs) {
+ CSB_ERR(csb, "Invalid CSB completion sequence");
+ return -EPROTO;
+ }
+
+ /* check CSB Completion Code */
+ switch (csb->cc) {
+ /* no error */
+ case CSB_CC_SUCCESS:
+ break;
+ case CSB_CC_TPBC_GT_SPBC:
+ /* not an error, but the compressed data is
+ * larger than the uncompressed data :(
+ */
+ break;
+
+ /* input data errors */
+ case CSB_CC_OPERAND_OVERLAP:
+ /* input and output buffers overlap */
+ CSB_ERR(csb, "Operand Overlap error");
+ return -EINVAL;
+ case CSB_CC_INVALID_OPERAND:
+ CSB_ERR(csb, "Invalid operand");
+ return -EINVAL;
+ case CSB_CC_NOSPC:
+ /* output buffer too small */
+ return -ENOSPC;
+ case CSB_CC_ABORT:
+ CSB_ERR(csb, "Function aborted");
+ return -EINTR;
+ case CSB_CC_CRC_MISMATCH:
+ CSB_ERR(csb, "CRC mismatch");
+ return -EINVAL;
+ case CSB_CC_TEMPL_INVALID:
+ CSB_ERR(csb, "Compressed data template invalid");
+ return -EINVAL;
+ case CSB_CC_TEMPL_OVERFLOW:
+ CSB_ERR(csb, "Compressed data template shows data past end");
+ return -EINVAL;
+
+ /* these should not happen */
+ case CSB_CC_INVALID_ALIGN:
+ /* setup_ddl should have detected this */
+ CSB_ERR_ADDR(csb, "Invalid alignment");
+ return -EINVAL;
+ case CSB_CC_DATA_LENGTH:
+ /* setup_ddl should have detected this */
+ CSB_ERR(csb, "Invalid data length");
+ return -EINVAL;
+ case CSB_CC_WR_TRANSLATION:
+ case CSB_CC_TRANSLATION:
+ case CSB_CC_TRANSLATION_DUP1:
+ case CSB_CC_TRANSLATION_DUP2:
+ case CSB_CC_TRANSLATION_DUP3:
+ case CSB_CC_TRANSLATION_DUP4:
+ case CSB_CC_TRANSLATION_DUP5:
+ case CSB_CC_TRANSLATION_DUP6:
+ /* should not happen, we use physical addrs */
+ CSB_ERR_ADDR(csb, "Translation error");
+ return -EPROTO;
+ case CSB_CC_WR_PROTECTION:
+ case CSB_CC_PROTECTION:
+ case CSB_CC_PROTECTION_DUP1:
+ case CSB_CC_PROTECTION_DUP2:
+ case CSB_CC_PROTECTION_DUP3:
+ case CSB_CC_PROTECTION_DUP4:
+ case CSB_CC_PROTECTION_DUP5:
+ case CSB_CC_PROTECTION_DUP6:
+ /* should not happen, we use physical addrs */
+ CSB_ERR_ADDR(csb, "Protection error");
+ return -EPROTO;
+ case CSB_CC_PRIVILEGE:
+ /* shouldn't happen, we're in HYP mode */
+ CSB_ERR(csb, "Insufficient Privilege error");
+ return -EPROTO;
+ case CSB_CC_EXCESSIVE_DDE:
+ /* shouldn't happen, setup_ddl doesn't use many dde's */
+ CSB_ERR(csb, "Too many DDEs in DDL");
+ return -EINVAL;
+ case CSB_CC_TRANSPORT:
+ /* shouldn't happen, we setup CRB correctly */
+ CSB_ERR(csb, "Invalid CRB");
+ return -EINVAL;
+ case CSB_CC_SEGMENTED_DDL:
+ /* shouldn't happen, setup_ddl creates DDL right */
+ CSB_ERR(csb, "Segmented DDL error");
+ return -EINVAL;
+ case CSB_CC_DDE_OVERFLOW:
+ /* shouldn't happen, setup_ddl creates DDL right */
+ CSB_ERR(csb, "DDE overflow error");
+ return -EINVAL;
+ case CSB_CC_SESSION:
+ /* should not happen with ICSWX */
+ CSB_ERR(csb, "Session violation error");
+ return -EPROTO;
+ case CSB_CC_CHAIN:
+ /* should not happen, we don't use chained CRBs */
+ CSB_ERR(csb, "Chained CRB error");
+ return -EPROTO;
+ case CSB_CC_SEQUENCE:
+ /* should not happen, we don't use chained CRBs */
+ CSB_ERR(csb, "CRB seqeunce number error");
+ return -EPROTO;
+ case CSB_CC_UNKNOWN_CODE:
+ CSB_ERR(csb, "Unknown subfunction code");
+ return -EPROTO;
+
+ /* hardware errors */
+ case CSB_CC_RD_EXTERNAL:
+ case CSB_CC_RD_EXTERNAL_DUP1:
+ case CSB_CC_RD_EXTERNAL_DUP2:
+ case CSB_CC_RD_EXTERNAL_DUP3:
+ CSB_ERR_ADDR(csb, "Read error outside coprocessor");
+ return -EPROTO;
+ case CSB_CC_WR_EXTERNAL:
+ CSB_ERR_ADDR(csb, "Write error outside coprocessor");
+ return -EPROTO;
+ case CSB_CC_INTERNAL:
+ CSB_ERR(csb, "Internal error in coprocessor");
+ return -EPROTO;
+ case CSB_CC_PROVISION:
+ CSB_ERR(csb, "Storage provision error");
+ return -EPROTO;
+ case CSB_CC_HW:
+ CSB_ERR(csb, "Correctable hardware error");
+ return -EPROTO;
+
+ default:
+ CSB_ERR(csb, "Invalid CC %d", csb->cc);
+ return -EPROTO;
+ }
+
+ /* check Completion Extension state */
+ if (csb->ce & CSB_CE_TERMINATION) {
+ CSB_ERR(csb, "CSB request was terminated");
+ return -EPROTO;
+ }
+ if (csb->ce & CSB_CE_INCOMPLETE) {
+ CSB_ERR(csb, "CSB request not complete");
+ return -EPROTO;
+ }
+ if (!(csb->ce & CSB_CE_TPBC)) {
+ CSB_ERR(csb, "TPBC not provided, unknown target length");
+ return -EPROTO;
+ }
+
+ /* successful completion */
+ pr_debug_ratelimited("Processed %u bytes in %lu us\n", csb->count,
+ (unsigned long)ktime_us_delta(now, start));
+
+ return 0;
+}
+
+/**
+ * nx842_powernv_function - compress/decompress data using the 842 algorithm
+ *
+ * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.
+ * This compresses or decompresses the provided input buffer into the provided
+ * output buffer.
+ *
+ * Upon return from this function @outlen contains the length of the
+ * output data. If there is an error then @outlen will be 0 and an
+ * error will be specified by the return code from this function.
+ *
+ * The @workmem buffer should only be used by one function call at a time.
+ *
+ * @in: input buffer pointer
+ * @inlen: input buffer size
+ * @out: output buffer pointer
+ * @outlenp: output buffer size pointer
+ * @workmem: working memory buffer pointer, size determined by
+ * nx842_powernv_driver.workmem_size
+ * @fc: function code, see CCW Function Codes in nx-842.h
+ *
+ * Returns:
+ * 0 Success, output of length @outlenp stored in the buffer at @out
+ * -ENODEV Hardware unavailable
+ * -ENOSPC Output buffer is to small
+ * -EMSGSIZE Input buffer too large
+ * -EINVAL buffer constraints do not fix nx842_constraints
+ * -EPROTO hardware error during operation
+ * -ETIMEDOUT hardware did not complete operation in reasonable time
+ * -EINTR operation was aborted
+ */
+static int nx842_powernv_function(const unsigned char *in, unsigned int inlen,
+ unsigned char *out, unsigned int *outlenp,
+ void *workmem, int fc)
+{
+ struct coprocessor_request_block *crb;
+ struct coprocessor_status_block *csb;
+ struct nx842_workmem *wmem;
+ int ret;
+ u64 csb_addr;
+ u32 ccw;
+ unsigned int outlen = *outlenp;
+
+ wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN);
+
+ *outlenp = 0;
+
+ /* shoudn't happen, we don't load without a coproc */
+ if (!nx842_ct) {
+ pr_err_ratelimited("coprocessor CT is 0");
+ return -ENODEV;
+ }
+
+ crb = &wmem->crb;
+ csb = &crb->csb;
+
+ /* Clear any previous values */
+ memset(crb, 0, sizeof(*crb));
+
+ /* set up DDLs */
+ ret = setup_ddl(&crb->source, wmem->ddl_in,
+ (unsigned char *)in, inlen, true);
+ if (ret)
+ return ret;
+ ret = setup_ddl(&crb->target, wmem->ddl_out,
+ out, outlen, false);
+ if (ret)
+ return ret;
+
+ /* set up CCW */
+ ccw = 0;
+ ccw = SET_FIELD(ccw, CCW_CT, nx842_ct);
+ ccw = SET_FIELD(ccw, CCW_CI_842, 0); /* use 0 for hw auto-selection */
+ ccw = SET_FIELD(ccw, CCW_FC_842, fc);
+
+ /* set up CRB's CSB addr */
+ csb_addr = nx842_get_pa(csb) & CRB_CSB_ADDRESS;
+ csb_addr |= CRB_CSB_AT; /* Addrs are phys */
+ crb->csb_addr = cpu_to_be64(csb_addr);
+
+ wmem->start = ktime_get();
+
+ /* do ICSWX */
+ ret = icswx(cpu_to_be32(ccw), crb);
+
+ pr_debug_ratelimited("icswx CR %x ccw %x crb->ccw %x\n", ret,
+ (unsigned int)ccw,
+ (unsigned int)be32_to_cpu(crb->ccw));
+
+ switch (ret) {
+ case ICSWX_INITIATED:
+ ret = wait_for_csb(wmem, csb);
+ break;
+ case ICSWX_BUSY:
+ pr_debug_ratelimited("842 Coprocessor busy\n");
+ ret = -EBUSY;
+ break;
+ case ICSWX_REJECTED:
+ pr_err_ratelimited("ICSWX rejected\n");
+ ret = -EPROTO;
+ break;
+ default:
+ pr_err_ratelimited("Invalid ICSWX return code %x\n", ret);
+ ret = -EPROTO;
+ break;
+ }
+
+ if (!ret)
+ *outlenp = be32_to_cpu(csb->count);
+
+ return ret;
+}
+
+/**
+ * nx842_powernv_compress - Compress data using the 842 algorithm
+ *
+ * Compression provided by the NX842 coprocessor on IBM PowerNV systems.
+ * The input buffer is compressed and the result is stored in the
+ * provided output buffer.
+ *
+ * Upon return from this function @outlen contains the length of the
+ * compressed data. If there is an error then @outlen will be 0 and an
+ * error will be specified by the return code from this function.
+ *
+ * @in: input buffer pointer
+ * @inlen: input buffer size
+ * @out: output buffer pointer
+ * @outlenp: output buffer size pointer
+ * @workmem: working memory buffer pointer, size determined by
+ * nx842_powernv_driver.workmem_size
+ *
+ * Returns: see @nx842_powernv_function()
+ */
+static int nx842_powernv_compress(const unsigned char *in, unsigned int inlen,
+ unsigned char *out, unsigned int *outlenp,
+ void *wmem)
+{
+ return nx842_powernv_function(in, inlen, out, outlenp,
+ wmem, CCW_FC_842_COMP_NOCRC);
+}
+
+/**
+ * nx842_powernv_decompress - Decompress data using the 842 algorithm
+ *
+ * Decompression provided by the NX842 coprocessor on IBM PowerNV systems.
+ * The input buffer is decompressed and the result is stored in the
+ * provided output buffer.
+ *
+ * Upon return from this function @outlen contains the length of the
+ * decompressed data. If there is an error then @outlen will be 0 and an
+ * error will be specified by the return code from this function.
+ *
+ * @in: input buffer pointer
+ * @inlen: input buffer size
+ * @out: output buffer pointer
+ * @outlenp: output buffer size pointer
+ * @workmem: working memory buffer pointer, size determined by
+ * nx842_powernv_driver.workmem_size
+ *
+ * Returns: see @nx842_powernv_function()
+ */
+static int nx842_powernv_decompress(const unsigned char *in, unsigned int inlen,
+ unsigned char *out, unsigned int *outlenp,
+ void *wmem)
+{
+ return nx842_powernv_function(in, inlen, out, outlenp,
+ wmem, CCW_FC_842_DECOMP_NOCRC);
+}
+
+static int __init nx842_powernv_probe(struct device_node *dn)
+{
+ struct nx842_coproc *coproc;
+ struct property *ct_prop, *ci_prop;
+ unsigned int ct, ci;
+ int chip_id;
+
+ chip_id = of_get_ibm_chip_id(dn);
+ if (chip_id < 0) {
+ pr_err("ibm,chip-id missing\n");
+ return -EINVAL;
+ }
+ ct_prop = of_find_property(dn, "ibm,842-coprocessor-type", NULL);
+ if (!ct_prop) {
+ pr_err("ibm,842-coprocessor-type missing\n");
+ return -EINVAL;
+ }
+ ct = be32_to_cpu(*(unsigned int *)ct_prop->value);
+ ci_prop = of_find_property(dn, "ibm,842-coprocessor-instance", NULL);
+ if (!ci_prop) {
+ pr_err("ibm,842-coprocessor-instance missing\n");
+ return -EINVAL;
+ }
+ ci = be32_to_cpu(*(unsigned int *)ci_prop->value);
+
+ coproc = kmalloc(sizeof(*coproc), GFP_KERNEL);
+ if (!coproc)
+ return -ENOMEM;
+
+ coproc->chip_id = chip_id;
+ coproc->ct = ct;
+ coproc->ci = ci;
+ INIT_LIST_HEAD(&coproc->list);
+ list_add(&coproc->list, &nx842_coprocs);
+
+ pr_info("coprocessor found on chip %d, CT %d CI %d\n", chip_id, ct, ci);
+
+ if (!nx842_ct)
+ nx842_ct = ct;
+ else if (nx842_ct != ct)
+ pr_err("NX842 chip %d, CT %d != first found CT %d\n",
+ chip_id, ct, nx842_ct);
+
+ return 0;
+}
+
+static struct nx842_constraints nx842_powernv_constraints = {
+ .alignment = DDE_BUFFER_ALIGN,
+ .multiple = DDE_BUFFER_LAST_MULT,
+ .minimum = DDE_BUFFER_LAST_MULT,
+ .maximum = (DDL_LEN_MAX - 1) * PAGE_SIZE,
+};
+
+static struct nx842_driver nx842_powernv_driver = {
+ .name = KBUILD_MODNAME,
+ .owner = THIS_MODULE,
+ .workmem_size = sizeof(struct nx842_workmem),
+ .constraints = &nx842_powernv_constraints,
+ .compress = nx842_powernv_compress,
+ .decompress = nx842_powernv_decompress,
+};
+
+static __init int nx842_powernv_init(void)
+{
+ struct device_node *dn;
+
+ /* verify workmem size/align restrictions */
+ BUILD_BUG_ON(WORKMEM_ALIGN % CRB_ALIGN);
+ BUILD_BUG_ON(CRB_ALIGN % DDE_ALIGN);
+ BUILD_BUG_ON(CRB_SIZE % DDE_ALIGN);
+ /* verify buffer size/align restrictions */
+ BUILD_BUG_ON(PAGE_SIZE % DDE_BUFFER_ALIGN);
+ BUILD_BUG_ON(DDE_BUFFER_ALIGN % DDE_BUFFER_SIZE_MULT);
+ BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT % DDE_BUFFER_LAST_MULT);
+
+ pr_info("loading\n");
+
+ for_each_compatible_node(dn, NULL, "ibm,power-nx")
+ nx842_powernv_probe(dn);
+
+ if (!nx842_ct) {
+ pr_err("no coprocessors found\n");
+ return -ENODEV;
+ }
+
+ if (!nx842_platform_driver_set(&nx842_powernv_driver)) {
+ struct nx842_coproc *coproc, *n;
+
+ list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
+ list_del(&coproc->list);
+ kfree(coproc);
+ }
+
+ return -EEXIST;
+ }
+
+ pr_info("loaded\n");
+
+ return 0;
+}
+module_init(nx842_powernv_init);
+
+static void __exit nx842_powernv_exit(void)
+{
+ struct nx842_coproc *coproc, *n;
+
+ nx842_platform_driver_unset(&nx842_powernv_driver);
+
+ list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
+ list_del(&coproc->list);
+ kfree(coproc);
+ }
+
+ pr_info("unloaded\n");
+}
+module_exit(nx842_powernv_exit);
diff --git a/drivers/crypto/nx/nx-842-pseries.c b/drivers/crypto/nx/nx-842-pseries.c
new file mode 100644
index 000000000..3040a6091
--- /dev/null
+++ b/drivers/crypto/nx/nx-842-pseries.c
@@ -0,0 +1,1140 @@
+/*
+ * Driver for IBM Power 842 compression accelerator
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * Copyright (C) IBM Corporation, 2012
+ *
+ * Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
+ * Seth Jennings <sjenning@linux.vnet.ibm.com>
+ */
+
+#include <asm/vio.h>
+
+#include "nx-842.h"
+#include "nx_csbcpb.h" /* struct nx_csbcpb */
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Robert Jennings <rcj@linux.vnet.ibm.com>");
+MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors");
+
+static struct nx842_constraints nx842_pseries_constraints = {
+ .alignment = DDE_BUFFER_ALIGN,
+ .multiple = DDE_BUFFER_LAST_MULT,
+ .minimum = DDE_BUFFER_LAST_MULT,
+ .maximum = PAGE_SIZE, /* dynamic, max_sync_size */
+};
+
+static int check_constraints(unsigned long buf, unsigned int *len, bool in)
+{
+ if (!IS_ALIGNED(buf, nx842_pseries_constraints.alignment)) {
+ pr_debug("%s buffer 0x%lx not aligned to 0x%x\n",
+ in ? "input" : "output", buf,
+ nx842_pseries_constraints.alignment);
+ return -EINVAL;
+ }
+ if (*len % nx842_pseries_constraints.multiple) {
+ pr_debug("%s buffer len 0x%x not multiple of 0x%x\n",
+ in ? "input" : "output", *len,
+ nx842_pseries_constraints.multiple);
+ if (in)
+ return -EINVAL;
+ *len = round_down(*len, nx842_pseries_constraints.multiple);
+ }
+ if (*len < nx842_pseries_constraints.minimum) {
+ pr_debug("%s buffer len 0x%x under minimum 0x%x\n",
+ in ? "input" : "output", *len,
+ nx842_pseries_constraints.minimum);
+ return -EINVAL;
+ }
+ if (*len > nx842_pseries_constraints.maximum) {
+ pr_debug("%s buffer len 0x%x over maximum 0x%x\n",
+ in ? "input" : "output", *len,
+ nx842_pseries_constraints.maximum);
+ if (in)
+ return -EINVAL;
+ *len = nx842_pseries_constraints.maximum;
+ }
+ return 0;
+}
+
+/* I assume we need to align the CSB? */
+#define WORKMEM_ALIGN (256)
+
+struct nx842_workmem {
+ /* scatterlist */
+ char slin[4096];
+ char slout[4096];
+ /* coprocessor status/parameter block */
+ struct nx_csbcpb csbcpb;
+
+ char padding[WORKMEM_ALIGN];
+} __aligned(WORKMEM_ALIGN);
+
+/* Macros for fields within nx_csbcpb */
+/* Check the valid bit within the csbcpb valid field */
+#define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7))
+
+/* CE macros operate on the completion_extension field bits in the csbcpb.
+ * CE0 0=full completion, 1=partial completion
+ * CE1 0=CE0 indicates completion, 1=termination (output may be modified)
+ * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */
+#define NX842_CSBCPB_CE0(x) (x & BIT_MASK(7))
+#define NX842_CSBCPB_CE1(x) (x & BIT_MASK(6))
+#define NX842_CSBCPB_CE2(x) (x & BIT_MASK(5))
+
+/* The NX unit accepts data only on 4K page boundaries */
+#define NX842_HW_PAGE_SIZE (4096)
+#define NX842_HW_PAGE_MASK (~(NX842_HW_PAGE_SIZE-1))
+
+enum nx842_status {
+ UNAVAILABLE,
+ AVAILABLE
+};
+
+struct ibm_nx842_counters {
+ atomic64_t comp_complete;
+ atomic64_t comp_failed;
+ atomic64_t decomp_complete;
+ atomic64_t decomp_failed;
+ atomic64_t swdecomp;
+ atomic64_t comp_times[32];
+ atomic64_t decomp_times[32];
+};
+
+static struct nx842_devdata {
+ struct vio_dev *vdev;
+ struct device *dev;
+ struct ibm_nx842_counters *counters;
+ unsigned int max_sg_len;
+ unsigned int max_sync_size;
+ unsigned int max_sync_sg;
+ enum nx842_status status;
+} __rcu *devdata;
+static DEFINE_SPINLOCK(devdata_mutex);
+
+#define NX842_COUNTER_INC(_x) \
+static inline void nx842_inc_##_x( \
+ const struct nx842_devdata *dev) { \
+ if (dev) \
+ atomic64_inc(&dev->counters->_x); \
+}
+NX842_COUNTER_INC(comp_complete);
+NX842_COUNTER_INC(comp_failed);
+NX842_COUNTER_INC(decomp_complete);
+NX842_COUNTER_INC(decomp_failed);
+NX842_COUNTER_INC(swdecomp);
+
+#define NX842_HIST_SLOTS 16
+
+static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time)
+{
+ int bucket = fls(time);
+
+ if (bucket)
+ bucket = min((NX842_HIST_SLOTS - 1), bucket - 1);
+
+ atomic64_inc(&times[bucket]);
+}
+
+/* NX unit operation flags */
+#define NX842_OP_COMPRESS 0x0
+#define NX842_OP_CRC 0x1
+#define NX842_OP_DECOMPRESS 0x2
+#define NX842_OP_COMPRESS_CRC (NX842_OP_COMPRESS | NX842_OP_CRC)
+#define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC)
+#define NX842_OP_ASYNC (1<<23)
+#define NX842_OP_NOTIFY (1<<22)
+#define NX842_OP_NOTIFY_INT(x) ((x & 0xff)<<8)
+
+static unsigned long nx842_get_desired_dma(struct vio_dev *viodev)
+{
+ /* No use of DMA mappings within the driver. */
+ return 0;
+}
+
+struct nx842_slentry {
+ __be64 ptr; /* Real address (use __pa()) */
+ __be64 len;
+};
+
+/* pHyp scatterlist entry */
+struct nx842_scatterlist {
+ int entry_nr; /* number of slentries */
+ struct nx842_slentry *entries; /* ptr to array of slentries */
+};
+
+/* Does not include sizeof(entry_nr) in the size */
+static inline unsigned long nx842_get_scatterlist_size(
+ struct nx842_scatterlist *sl)
+{
+ return sl->entry_nr * sizeof(struct nx842_slentry);
+}
+
+static int nx842_build_scatterlist(unsigned long buf, int len,
+ struct nx842_scatterlist *sl)
+{
+ unsigned long entrylen;
+ struct nx842_slentry *entry;
+
+ sl->entry_nr = 0;
+
+ entry = sl->entries;
+ while (len) {
+ entry->ptr = cpu_to_be64(nx842_get_pa((void *)buf));
+ entrylen = min_t(int, len,
+ LEN_ON_SIZE(buf, NX842_HW_PAGE_SIZE));
+ entry->len = cpu_to_be64(entrylen);
+
+ len -= entrylen;
+ buf += entrylen;
+
+ sl->entry_nr++;
+ entry++;
+ }
+
+ return 0;
+}
+
+static int nx842_validate_result(struct device *dev,
+ struct cop_status_block *csb)
+{
+ /* The csb must be valid after returning from vio_h_cop_sync */
+ if (!NX842_CSBCBP_VALID_CHK(csb->valid)) {
+ dev_err(dev, "%s: cspcbp not valid upon completion.\n",
+ __func__);
+ dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n",
+ csb->valid,
+ csb->crb_seq_number,
+ csb->completion_code,
+ csb->completion_extension);
+ dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n",
+ be32_to_cpu(csb->processed_byte_count),
+ (unsigned long)be64_to_cpu(csb->address));
+ return -EIO;
+ }
+
+ /* Check return values from the hardware in the CSB */
+ switch (csb->completion_code) {
+ case 0: /* Completed without error */
+ break;
+ case 64: /* Target bytes > Source bytes during compression */
+ case 13: /* Output buffer too small */
+ dev_dbg(dev, "%s: Compression output larger than input\n",
+ __func__);
+ return -ENOSPC;
+ case 66: /* Input data contains an illegal template field */
+ case 67: /* Template indicates data past the end of the input stream */
+ dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n",
+ __func__, csb->completion_code);
+ return -EINVAL;
+ default:
+ dev_dbg(dev, "%s: Unspecified error (code:%d)\n",
+ __func__, csb->completion_code);
+ return -EIO;
+ }
+
+ /* Hardware sanity check */
+ if (!NX842_CSBCPB_CE2(csb->completion_extension)) {
+ dev_err(dev, "%s: No error returned by hardware, but "
+ "data returned is unusable, contact support.\n"
+ "(Additional info: csbcbp->processed bytes "
+ "does not specify processed bytes for the "
+ "target buffer.)\n", __func__);
+ return -EIO;
+ }
+
+ return 0;
+}
+
+/**
+ * nx842_pseries_compress - Compress data using the 842 algorithm
+ *
+ * Compression provide by the NX842 coprocessor on IBM Power systems.
+ * The input buffer is compressed and the result is stored in the
+ * provided output buffer.
+ *
+ * Upon return from this function @outlen contains the length of the
+ * compressed data. If there is an error then @outlen will be 0 and an
+ * error will be specified by the return code from this function.
+ *
+ * @in: Pointer to input buffer
+ * @inlen: Length of input buffer
+ * @out: Pointer to output buffer
+ * @outlen: Length of output buffer
+ * @wrkmem: ptr to buffer for working memory, size determined by
+ * nx842_pseries_driver.workmem_size
+ *
+ * Returns:
+ * 0 Success, output of length @outlen stored in the buffer at @out
+ * -ENOMEM Unable to allocate internal buffers
+ * -ENOSPC Output buffer is to small
+ * -EIO Internal error
+ * -ENODEV Hardware unavailable
+ */
+static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen,
+ unsigned char *out, unsigned int *outlen,
+ void *wmem)
+{
+ struct nx842_devdata *local_devdata;
+ struct device *dev = NULL;
+ struct nx842_workmem *workmem;
+ struct nx842_scatterlist slin, slout;
+ struct nx_csbcpb *csbcpb;
+ int ret = 0, max_sync_size;
+ unsigned long inbuf, outbuf;
+ struct vio_pfo_op op = {
+ .done = NULL,
+ .handle = 0,
+ .timeout = 0,
+ };
+ unsigned long start = get_tb();
+
+ inbuf = (unsigned long)in;
+ if (check_constraints(inbuf, &inlen, true))
+ return -EINVAL;
+
+ outbuf = (unsigned long)out;
+ if (check_constraints(outbuf, outlen, false))
+ return -EINVAL;
+
+ rcu_read_lock();
+ local_devdata = rcu_dereference(devdata);
+ if (!local_devdata || !local_devdata->dev) {
+ rcu_read_unlock();
+ return -ENODEV;
+ }
+ max_sync_size = local_devdata->max_sync_size;
+ dev = local_devdata->dev;
+
+ /* Init scatterlist */
+ workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
+ slin.entries = (struct nx842_slentry *)workmem->slin;
+ slout.entries = (struct nx842_slentry *)workmem->slout;
+
+ /* Init operation */
+ op.flags = NX842_OP_COMPRESS;
+ csbcpb = &workmem->csbcpb;
+ memset(csbcpb, 0, sizeof(*csbcpb));
+ op.csbcpb = nx842_get_pa(csbcpb);
+
+ if ((inbuf & NX842_HW_PAGE_MASK) ==
+ ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
+ /* Create direct DDE */
+ op.in = nx842_get_pa((void *)inbuf);
+ op.inlen = inlen;
+ } else {
+ /* Create indirect DDE (scatterlist) */
+ nx842_build_scatterlist(inbuf, inlen, &slin);
+ op.in = nx842_get_pa(slin.entries);
+ op.inlen = -nx842_get_scatterlist_size(&slin);
+ }
+
+ if ((outbuf & NX842_HW_PAGE_MASK) ==
+ ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
+ /* Create direct DDE */
+ op.out = nx842_get_pa((void *)outbuf);
+ op.outlen = *outlen;
+ } else {
+ /* Create indirect DDE (scatterlist) */
+ nx842_build_scatterlist(outbuf, *outlen, &slout);
+ op.out = nx842_get_pa(slout.entries);
+ op.outlen = -nx842_get_scatterlist_size(&slout);
+ }
+
+ dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
+ __func__, (unsigned long)op.in, (long)op.inlen,
+ (unsigned long)op.out, (long)op.outlen);
+
+ /* Send request to pHyp */
+ ret = vio_h_cop_sync(local_devdata->vdev, &op);
+
+ /* Check for pHyp error */
+ if (ret) {
+ dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
+ __func__, ret, op.hcall_err);
+ ret = -EIO;
+ goto unlock;
+ }
+
+ /* Check for hardware error */
+ ret = nx842_validate_result(dev, &csbcpb->csb);
+ if (ret)
+ goto unlock;
+
+ *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
+ dev_dbg(dev, "%s: processed_bytes=%d\n", __func__, *outlen);
+
+unlock:
+ if (ret)
+ nx842_inc_comp_failed(local_devdata);
+ else {
+ nx842_inc_comp_complete(local_devdata);
+ ibm_nx842_incr_hist(local_devdata->counters->comp_times,
+ (get_tb() - start) / tb_ticks_per_usec);
+ }
+ rcu_read_unlock();
+ return ret;
+}
+
+/**
+ * nx842_pseries_decompress - Decompress data using the 842 algorithm
+ *
+ * Decompression provide by the NX842 coprocessor on IBM Power systems.
+ * The input buffer is decompressed and the result is stored in the
+ * provided output buffer. The size allocated to the output buffer is
+ * provided by the caller of this function in @outlen. Upon return from
+ * this function @outlen contains the length of the decompressed data.
+ * If there is an error then @outlen will be 0 and an error will be
+ * specified by the return code from this function.
+ *
+ * @in: Pointer to input buffer
+ * @inlen: Length of input buffer
+ * @out: Pointer to output buffer
+ * @outlen: Length of output buffer
+ * @wrkmem: ptr to buffer for working memory, size determined by
+ * nx842_pseries_driver.workmem_size
+ *
+ * Returns:
+ * 0 Success, output of length @outlen stored in the buffer at @out
+ * -ENODEV Hardware decompression device is unavailable
+ * -ENOMEM Unable to allocate internal buffers
+ * -ENOSPC Output buffer is to small
+ * -EINVAL Bad input data encountered when attempting decompress
+ * -EIO Internal error
+ */
+static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen,
+ unsigned char *out, unsigned int *outlen,
+ void *wmem)
+{
+ struct nx842_devdata *local_devdata;
+ struct device *dev = NULL;
+ struct nx842_workmem *workmem;
+ struct nx842_scatterlist slin, slout;
+ struct nx_csbcpb *csbcpb;
+ int ret = 0, max_sync_size;
+ unsigned long inbuf, outbuf;
+ struct vio_pfo_op op = {
+ .done = NULL,
+ .handle = 0,
+ .timeout = 0,
+ };
+ unsigned long start = get_tb();
+
+ /* Ensure page alignment and size */
+ inbuf = (unsigned long)in;
+ if (check_constraints(inbuf, &inlen, true))
+ return -EINVAL;
+
+ outbuf = (unsigned long)out;
+ if (check_constraints(outbuf, outlen, false))
+ return -EINVAL;
+
+ rcu_read_lock();
+ local_devdata = rcu_dereference(devdata);
+ if (!local_devdata || !local_devdata->dev) {
+ rcu_read_unlock();
+ return -ENODEV;
+ }
+ max_sync_size = local_devdata->max_sync_size;
+ dev = local_devdata->dev;
+
+ workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
+
+ /* Init scatterlist */
+ slin.entries = (struct nx842_slentry *)workmem->slin;
+ slout.entries = (struct nx842_slentry *)workmem->slout;
+
+ /* Init operation */
+ op.flags = NX842_OP_DECOMPRESS;
+ csbcpb = &workmem->csbcpb;
+ memset(csbcpb, 0, sizeof(*csbcpb));
+ op.csbcpb = nx842_get_pa(csbcpb);
+
+ if ((inbuf & NX842_HW_PAGE_MASK) ==
+ ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
+ /* Create direct DDE */
+ op.in = nx842_get_pa((void *)inbuf);
+ op.inlen = inlen;
+ } else {
+ /* Create indirect DDE (scatterlist) */
+ nx842_build_scatterlist(inbuf, inlen, &slin);
+ op.in = nx842_get_pa(slin.entries);
+ op.inlen = -nx842_get_scatterlist_size(&slin);
+ }
+
+ if ((outbuf & NX842_HW_PAGE_MASK) ==
+ ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
+ /* Create direct DDE */
+ op.out = nx842_get_pa((void *)outbuf);
+ op.outlen = *outlen;
+ } else {
+ /* Create indirect DDE (scatterlist) */
+ nx842_build_scatterlist(outbuf, *outlen, &slout);
+ op.out = nx842_get_pa(slout.entries);
+ op.outlen = -nx842_get_scatterlist_size(&slout);
+ }
+
+ dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
+ __func__, (unsigned long)op.in, (long)op.inlen,
+ (unsigned long)op.out, (long)op.outlen);
+
+ /* Send request to pHyp */
+ ret = vio_h_cop_sync(local_devdata->vdev, &op);
+
+ /* Check for pHyp error */
+ if (ret) {
+ dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
+ __func__, ret, op.hcall_err);
+ goto unlock;
+ }
+
+ /* Check for hardware error */
+ ret = nx842_validate_result(dev, &csbcpb->csb);
+ if (ret)
+ goto unlock;
+
+ *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
+
+unlock:
+ if (ret)
+ /* decompress fail */
+ nx842_inc_decomp_failed(local_devdata);
+ else {
+ nx842_inc_decomp_complete(local_devdata);
+ ibm_nx842_incr_hist(local_devdata->counters->decomp_times,
+ (get_tb() - start) / tb_ticks_per_usec);
+ }
+
+ rcu_read_unlock();
+ return ret;
+}
+
+/**
+ * nx842_OF_set_defaults -- Set default (disabled) values for devdata
+ *
+ * @devdata - struct nx842_devdata to update
+ *
+ * Returns:
+ * 0 on success
+ * -ENOENT if @devdata ptr is NULL
+ */
+static int nx842_OF_set_defaults(struct nx842_devdata *devdata)
+{
+ if (devdata) {
+ devdata->max_sync_size = 0;
+ devdata->max_sync_sg = 0;
+ devdata->max_sg_len = 0;
+ devdata->status = UNAVAILABLE;
+ return 0;
+ } else
+ return -ENOENT;
+}
+
+/**
+ * nx842_OF_upd_status -- Update the device info from OF status prop
+ *
+ * The status property indicates if the accelerator is enabled. If the
+ * device is in the OF tree it indicates that the hardware is present.
+ * The status field indicates if the device is enabled when the status
+ * is 'okay'. Otherwise the device driver will be disabled.
+ *
+ * @devdata - struct nx842_devdata to update
+ * @prop - struct property point containing the maxsyncop for the update
+ *
+ * Returns:
+ * 0 - Device is available
+ * -EINVAL - Device is not available
+ */
+static int nx842_OF_upd_status(struct nx842_devdata *devdata,
+ struct property *prop) {
+ int ret = 0;
+ const char *status = (const char *)prop->value;
+
+ if (!strncmp(status, "okay", (size_t)prop->length)) {
+ devdata->status = AVAILABLE;
+ } else {
+ dev_info(devdata->dev, "%s: status '%s' is not 'okay'\n",
+ __func__, status);
+ devdata->status = UNAVAILABLE;
+ }
+
+ return ret;
+}
+
+/**
+ * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop
+ *
+ * Definition of the 'ibm,max-sg-len' OF property:
+ * This field indicates the maximum byte length of a scatter list
+ * for the platform facility. It is a single cell encoded as with encode-int.
+ *
+ * Example:
+ * # od -x ibm,max-sg-len
+ * 0000000 0000 0ff0
+ *
+ * In this example, the maximum byte length of a scatter list is
+ * 0x0ff0 (4,080).
+ *
+ * @devdata - struct nx842_devdata to update
+ * @prop - struct property point containing the maxsyncop for the update
+ *
+ * Returns:
+ * 0 on success
+ * -EINVAL on failure
+ */
+static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata,
+ struct property *prop) {
+ int ret = 0;
+ const unsigned int maxsglen = of_read_number(prop->value, 1);
+
+ if (prop->length != sizeof(maxsglen)) {
+ dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__);
+ dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__,
+ prop->length, sizeof(maxsglen));
+ ret = -EINVAL;
+ } else {
+ devdata->max_sg_len = min_t(unsigned int,
+ maxsglen, NX842_HW_PAGE_SIZE);
+ }
+
+ return ret;
+}
+
+/**
+ * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop
+ *
+ * Definition of the 'ibm,max-sync-cop' OF property:
+ * Two series of cells. The first series of cells represents the maximums
+ * that can be synchronously compressed. The second series of cells
+ * represents the maximums that can be synchronously decompressed.
+ * 1. The first cell in each series contains the count of the number of
+ * data length, scatter list elements pairs that follow – each being
+ * of the form
+ * a. One cell data byte length
+ * b. One cell total number of scatter list elements
+ *
+ * Example:
+ * # od -x ibm,max-sync-cop
+ * 0000000 0000 0001 0000 1000 0000 01fe 0000 0001
+ * 0000020 0000 1000 0000 01fe
+ *
+ * In this example, compression supports 0x1000 (4,096) data byte length
+ * and 0x1fe (510) total scatter list elements. Decompression supports
+ * 0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list
+ * elements.
+ *
+ * @devdata - struct nx842_devdata to update
+ * @prop - struct property point containing the maxsyncop for the update
+ *
+ * Returns:
+ * 0 on success
+ * -EINVAL on failure
+ */
+static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata,
+ struct property *prop) {
+ int ret = 0;
+ unsigned int comp_data_limit, decomp_data_limit;
+ unsigned int comp_sg_limit, decomp_sg_limit;
+ const struct maxsynccop_t {
+ __be32 comp_elements;
+ __be32 comp_data_limit;
+ __be32 comp_sg_limit;
+ __be32 decomp_elements;
+ __be32 decomp_data_limit;
+ __be32 decomp_sg_limit;
+ } *maxsynccop;
+
+ if (prop->length != sizeof(*maxsynccop)) {
+ dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__);
+ dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length,
+ sizeof(*maxsynccop));
+ ret = -EINVAL;
+ goto out;
+ }
+
+ maxsynccop = (const struct maxsynccop_t *)prop->value;
+ comp_data_limit = be32_to_cpu(maxsynccop->comp_data_limit);
+ comp_sg_limit = be32_to_cpu(maxsynccop->comp_sg_limit);
+ decomp_data_limit = be32_to_cpu(maxsynccop->decomp_data_limit);
+ decomp_sg_limit = be32_to_cpu(maxsynccop->decomp_sg_limit);
+
+ /* Use one limit rather than separate limits for compression and
+ * decompression. Set a maximum for this so as not to exceed the
+ * size that the header can support and round the value down to
+ * the hardware page size (4K) */
+ devdata->max_sync_size = min(comp_data_limit, decomp_data_limit);
+
+ devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size,
+ 65536);
+
+ if (devdata->max_sync_size < 4096) {
+ dev_err(devdata->dev, "%s: hardware max data size (%u) is "
+ "less than the driver minimum, unable to use "
+ "the hardware device\n",
+ __func__, devdata->max_sync_size);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ nx842_pseries_constraints.maximum = devdata->max_sync_size;
+
+ devdata->max_sync_sg = min(comp_sg_limit, decomp_sg_limit);
+ if (devdata->max_sync_sg < 1) {
+ dev_err(devdata->dev, "%s: hardware max sg size (%u) is "
+ "less than the driver minimum, unable to use "
+ "the hardware device\n",
+ __func__, devdata->max_sync_sg);
+ ret = -EINVAL;
+ goto out;
+ }
+
+out:
+ return ret;
+}
+
+/**
+ *
+ * nx842_OF_upd -- Handle OF properties updates for the device.
+ *
+ * Set all properties from the OF tree. Optionally, a new property
+ * can be provided by the @new_prop pointer to overwrite an existing value.
+ * The device will remain disabled until all values are valid, this function
+ * will return an error for updates unless all values are valid.
+ *
+ * @new_prop: If not NULL, this property is being updated. If NULL, update
+ * all properties from the current values in the OF tree.
+ *
+ * Returns:
+ * 0 - Success
+ * -ENOMEM - Could not allocate memory for new devdata structure
+ * -EINVAL - property value not found, new_prop is not a recognized
+ * property for the device or property value is not valid.
+ * -ENODEV - Device is not available
+ */
+static int nx842_OF_upd(struct property *new_prop)
+{
+ struct nx842_devdata *old_devdata = NULL;
+ struct nx842_devdata *new_devdata = NULL;
+ struct device_node *of_node = NULL;
+ struct property *status = NULL;
+ struct property *maxsglen = NULL;
+ struct property *maxsyncop = NULL;
+ int ret = 0;
+ unsigned long flags;
+
+ spin_lock_irqsave(&devdata_mutex, flags);
+ old_devdata = rcu_dereference_check(devdata,
+ lockdep_is_held(&devdata_mutex));
+ if (old_devdata)
+ of_node = old_devdata->dev->of_node;
+
+ if (!old_devdata || !of_node) {
+ pr_err("%s: device is not available\n", __func__);
+ spin_unlock_irqrestore(&devdata_mutex, flags);
+ return -ENODEV;
+ }
+
+ new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
+ if (!new_devdata) {
+ dev_err(old_devdata->dev, "%s: Could not allocate memory for device data\n", __func__);
+ ret = -ENOMEM;
+ goto error_out;
+ }
+
+ memcpy(new_devdata, old_devdata, sizeof(*old_devdata));
+ new_devdata->counters = old_devdata->counters;
+
+ /* Set ptrs for existing properties */
+ status = of_find_property(of_node, "status", NULL);
+ maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL);
+ maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL);
+ if (!status || !maxsglen || !maxsyncop) {
+ dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__);
+ ret = -EINVAL;
+ goto error_out;
+ }
+
+ /*
+ * If this is a property update, there are only certain properties that
+ * we care about. Bail if it isn't in the below list
+ */
+ if (new_prop && (strncmp(new_prop->name, "status", new_prop->length) ||
+ strncmp(new_prop->name, "ibm,max-sg-len", new_prop->length) ||
+ strncmp(new_prop->name, "ibm,max-sync-cop", new_prop->length)))
+ goto out;
+
+ /* Perform property updates */
+ ret = nx842_OF_upd_status(new_devdata, status);
+ if (ret)
+ goto error_out;
+
+ ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen);
+ if (ret)
+ goto error_out;
+
+ ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop);
+ if (ret)
+ goto error_out;
+
+out:
+ dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n",
+ __func__, new_devdata->max_sync_size,
+ old_devdata->max_sync_size);
+ dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n",
+ __func__, new_devdata->max_sync_sg,
+ old_devdata->max_sync_sg);
+ dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n",
+ __func__, new_devdata->max_sg_len,
+ old_devdata->max_sg_len);
+
+ rcu_assign_pointer(devdata, new_devdata);
+ spin_unlock_irqrestore(&devdata_mutex, flags);
+ synchronize_rcu();
+ dev_set_drvdata(new_devdata->dev, new_devdata);
+ kfree(old_devdata);
+ return 0;
+
+error_out:
+ if (new_devdata) {
+ dev_info(old_devdata->dev, "%s: device disabled\n", __func__);
+ nx842_OF_set_defaults(new_devdata);
+ rcu_assign_pointer(devdata, new_devdata);
+ spin_unlock_irqrestore(&devdata_mutex, flags);
+ synchronize_rcu();
+ dev_set_drvdata(new_devdata->dev, new_devdata);
+ kfree(old_devdata);
+ } else {
+ dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__);
+ spin_unlock_irqrestore(&devdata_mutex, flags);
+ }
+
+ if (!ret)
+ ret = -EINVAL;
+ return ret;
+}
+
+/**
+ * nx842_OF_notifier - Process updates to OF properties for the device
+ *
+ * @np: notifier block
+ * @action: notifier action
+ * @update: struct pSeries_reconfig_prop_update pointer if action is
+ * PSERIES_UPDATE_PROPERTY
+ *
+ * Returns:
+ * NOTIFY_OK on success
+ * NOTIFY_BAD encoded with error number on failure, use
+ * notifier_to_errno() to decode this value
+ */
+static int nx842_OF_notifier(struct notifier_block *np, unsigned long action,
+ void *data)
+{
+ struct of_reconfig_data *upd = data;
+ struct nx842_devdata *local_devdata;
+ struct device_node *node = NULL;
+
+ rcu_read_lock();
+ local_devdata = rcu_dereference(devdata);
+ if (local_devdata)
+ node = local_devdata->dev->of_node;
+
+ if (local_devdata &&
+ action == OF_RECONFIG_UPDATE_PROPERTY &&
+ !strcmp(upd->dn->name, node->name)) {
+ rcu_read_unlock();
+ nx842_OF_upd(upd->prop);
+ } else
+ rcu_read_unlock();
+
+ return NOTIFY_OK;
+}
+
+static struct notifier_block nx842_of_nb = {
+ .notifier_call = nx842_OF_notifier,
+};
+
+#define nx842_counter_read(_name) \
+static ssize_t nx842_##_name##_show(struct device *dev, \
+ struct device_attribute *attr, \
+ char *buf) { \
+ struct nx842_devdata *local_devdata; \
+ int p = 0; \
+ rcu_read_lock(); \
+ local_devdata = rcu_dereference(devdata); \
+ if (local_devdata) \
+ p = snprintf(buf, PAGE_SIZE, "%ld\n", \
+ atomic64_read(&local_devdata->counters->_name)); \
+ rcu_read_unlock(); \
+ return p; \
+}
+
+#define NX842DEV_COUNTER_ATTR_RO(_name) \
+ nx842_counter_read(_name); \
+ static struct device_attribute dev_attr_##_name = __ATTR(_name, \
+ 0444, \
+ nx842_##_name##_show,\
+ NULL);
+
+NX842DEV_COUNTER_ATTR_RO(comp_complete);
+NX842DEV_COUNTER_ATTR_RO(comp_failed);
+NX842DEV_COUNTER_ATTR_RO(decomp_complete);
+NX842DEV_COUNTER_ATTR_RO(decomp_failed);
+NX842DEV_COUNTER_ATTR_RO(swdecomp);
+
+static ssize_t nx842_timehist_show(struct device *,
+ struct device_attribute *, char *);
+
+static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444,
+ nx842_timehist_show, NULL);
+static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times,
+ 0444, nx842_timehist_show, NULL);
+
+static ssize_t nx842_timehist_show(struct device *dev,
+ struct device_attribute *attr, char *buf) {
+ char *p = buf;
+ struct nx842_devdata *local_devdata;
+ atomic64_t *times;
+ int bytes_remain = PAGE_SIZE;
+ int bytes;
+ int i;
+
+ rcu_read_lock();
+ local_devdata = rcu_dereference(devdata);
+ if (!local_devdata) {
+ rcu_read_unlock();
+ return 0;
+ }
+
+ if (attr == &dev_attr_comp_times)
+ times = local_devdata->counters->comp_times;
+ else if (attr == &dev_attr_decomp_times)
+ times = local_devdata->counters->decomp_times;
+ else {
+ rcu_read_unlock();
+ return 0;
+ }
+
+ for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) {
+ bytes = snprintf(p, bytes_remain, "%u-%uus:\t%ld\n",
+ i ? (2<<(i-1)) : 0, (2<<i)-1,
+ atomic64_read(&times[i]));
+ bytes_remain -= bytes;
+ p += bytes;
+ }
+ /* The last bucket holds everything over
+ * 2<<(NX842_HIST_SLOTS - 2) us */
+ bytes = snprintf(p, bytes_remain, "%uus - :\t%ld\n",
+ 2<<(NX842_HIST_SLOTS - 2),
+ atomic64_read(&times[(NX842_HIST_SLOTS - 1)]));
+ p += bytes;
+
+ rcu_read_unlock();
+ return p - buf;
+}
+
+static struct attribute *nx842_sysfs_entries[] = {
+ &dev_attr_comp_complete.attr,
+ &dev_attr_comp_failed.attr,
+ &dev_attr_decomp_complete.attr,
+ &dev_attr_decomp_failed.attr,
+ &dev_attr_swdecomp.attr,
+ &dev_attr_comp_times.attr,
+ &dev_attr_decomp_times.attr,
+ NULL,
+};
+
+static struct attribute_group nx842_attribute_group = {
+ .name = NULL, /* put in device directory */
+ .attrs = nx842_sysfs_entries,
+};
+
+static struct nx842_driver nx842_pseries_driver = {
+ .name = KBUILD_MODNAME,
+ .owner = THIS_MODULE,
+ .workmem_size = sizeof(struct nx842_workmem),
+ .constraints = &nx842_pseries_constraints,
+ .compress = nx842_pseries_compress,
+ .decompress = nx842_pseries_decompress,
+};
+
+static int __init nx842_probe(struct vio_dev *viodev,
+ const struct vio_device_id *id)
+{
+ struct nx842_devdata *old_devdata, *new_devdata = NULL;
+ unsigned long flags;
+ int ret = 0;
+
+ spin_lock_irqsave(&devdata_mutex, flags);
+ old_devdata = rcu_dereference_check(devdata,
+ lockdep_is_held(&devdata_mutex));
+
+ if (old_devdata && old_devdata->vdev != NULL) {
+ dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__);
+ ret = -1;
+ goto error_unlock;
+ }
+
+ dev_set_drvdata(&viodev->dev, NULL);
+
+ new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
+ if (!new_devdata) {
+ dev_err(&viodev->dev, "%s: Could not allocate memory for device data\n", __func__);
+ ret = -ENOMEM;
+ goto error_unlock;
+ }
+
+ new_devdata->counters = kzalloc(sizeof(*new_devdata->counters),
+ GFP_NOFS);
+ if (!new_devdata->counters) {
+ dev_err(&viodev->dev, "%s: Could not allocate memory for performance counters\n", __func__);
+ ret = -ENOMEM;
+ goto error_unlock;
+ }
+
+ new_devdata->vdev = viodev;
+ new_devdata->dev = &viodev->dev;
+ nx842_OF_set_defaults(new_devdata);
+
+ rcu_assign_pointer(devdata, new_devdata);
+ spin_unlock_irqrestore(&devdata_mutex, flags);
+ synchronize_rcu();
+ kfree(old_devdata);
+
+ of_reconfig_notifier_register(&nx842_of_nb);
+
+ ret = nx842_OF_upd(NULL);
+ if (ret && ret != -ENODEV) {
+ dev_err(&viodev->dev, "could not parse device tree. %d\n", ret);
+ ret = -1;
+ goto error;
+ }
+
+ rcu_read_lock();
+ dev_set_drvdata(&viodev->dev, rcu_dereference(devdata));
+ rcu_read_unlock();
+
+ if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) {
+ dev_err(&viodev->dev, "could not create sysfs device attributes\n");
+ ret = -1;
+ goto error;
+ }
+
+ return 0;
+
+error_unlock:
+ spin_unlock_irqrestore(&devdata_mutex, flags);
+ if (new_devdata)
+ kfree(new_devdata->counters);
+ kfree(new_devdata);
+error:
+ return ret;
+}
+
+static int __exit nx842_remove(struct vio_dev *viodev)
+{
+ struct nx842_devdata *old_devdata;
+ unsigned long flags;
+
+ pr_info("Removing IBM Power 842 compression device\n");
+ sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group);
+
+ spin_lock_irqsave(&devdata_mutex, flags);
+ old_devdata = rcu_dereference_check(devdata,
+ lockdep_is_held(&devdata_mutex));
+ of_reconfig_notifier_unregister(&nx842_of_nb);
+ RCU_INIT_POINTER(devdata, NULL);
+ spin_unlock_irqrestore(&devdata_mutex, flags);
+ synchronize_rcu();
+ dev_set_drvdata(&viodev->dev, NULL);
+ if (old_devdata)
+ kfree(old_devdata->counters);
+ kfree(old_devdata);
+
+ return 0;
+}
+
+static struct vio_device_id nx842_vio_driver_ids[] = {
+ {"ibm,compression-v1", "ibm,compression"},
+ {"", ""},
+};
+
+static struct vio_driver nx842_vio_driver = {
+ .name = KBUILD_MODNAME,
+ .probe = nx842_probe,
+ .remove = __exit_p(nx842_remove),
+ .get_desired_dma = nx842_get_desired_dma,
+ .id_table = nx842_vio_driver_ids,
+};
+
+static int __init nx842_init(void)
+{
+ struct nx842_devdata *new_devdata;
+ int ret;
+
+ pr_info("Registering IBM Power 842 compression driver\n");
+
+ if (!of_find_compatible_node(NULL, NULL, "ibm,compression"))
+ return -ENODEV;
+
+ RCU_INIT_POINTER(devdata, NULL);
+ new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL);
+ if (!new_devdata) {
+ pr_err("Could not allocate memory for device data\n");
+ return -ENOMEM;
+ }
+ new_devdata->status = UNAVAILABLE;
+ RCU_INIT_POINTER(devdata, new_devdata);
+
+ ret = vio_register_driver(&nx842_vio_driver);
+ if (ret) {
+ pr_err("Could not register VIO driver %d\n", ret);
+
+ kfree(new_devdata);
+ return ret;
+ }
+
+ if (!nx842_platform_driver_set(&nx842_pseries_driver)) {
+ vio_unregister_driver(&nx842_vio_driver);
+ kfree(new_devdata);
+ return -EEXIST;
+ }
+
+ return 0;
+}
+
+module_init(nx842_init);
+
+static void __exit nx842_exit(void)
+{
+ struct nx842_devdata *old_devdata;
+ unsigned long flags;
+
+ pr_info("Exiting IBM Power 842 compression driver\n");
+ nx842_platform_driver_unset(&nx842_pseries_driver);
+ spin_lock_irqsave(&devdata_mutex, flags);
+ old_devdata = rcu_dereference_check(devdata,
+ lockdep_is_held(&devdata_mutex));
+ RCU_INIT_POINTER(devdata, NULL);
+ spin_unlock_irqrestore(&devdata_mutex, flags);
+ synchronize_rcu();
+ if (old_devdata && old_devdata->dev)
+ dev_set_drvdata(old_devdata->dev, NULL);
+ kfree(old_devdata);
+ vio_unregister_driver(&nx842_vio_driver);
+}
+
+module_exit(nx842_exit);
+
diff --git a/drivers/crypto/nx/nx-842.c b/drivers/crypto/nx/nx-842.c
index 887196e9b..6e5e0d60d 100644
--- a/drivers/crypto/nx/nx-842.c
+++ b/drivers/crypto/nx/nx-842.c
@@ -1,5 +1,10 @@
/*
- * Driver for IBM Power 842 compression accelerator
+ * Driver frontend for IBM Power 842 compression accelerator
+ *
+ * Copyright (C) 2015 Dan Streetman, IBM Corp
+ *
+ * Designer of the Power data compression engine:
+ * Bulent Abali <abali@us.ibm.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -10,1594 +15,89 @@
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- *
- * Copyright (C) IBM Corporation, 2012
- *
- * Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
- * Seth Jennings <sjenning@linux.vnet.ibm.com>
*/
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/nx842.h>
-#include <linux/of.h>
-#include <linux/slab.h>
-
-#include <asm/page.h>
-#include <asm/vio.h>
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-#include "nx_csbcpb.h" /* struct nx_csbcpb */
+#include "nx-842.h"
-#define MODULE_NAME "nx-compress"
MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Robert Jennings <rcj@linux.vnet.ibm.com>");
+MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");
MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors");
-#define SHIFT_4K 12
-#define SHIFT_64K 16
-#define SIZE_4K (1UL << SHIFT_4K)
-#define SIZE_64K (1UL << SHIFT_64K)
-
-/* IO buffer must be 128 byte aligned */
-#define IO_BUFFER_ALIGN 128
-
-struct nx842_header {
- int blocks_nr; /* number of compressed blocks */
- int offset; /* offset of the first block (from beginning of header) */
- int sizes[0]; /* size of compressed blocks */
-};
-
-static inline int nx842_header_size(const struct nx842_header *hdr)
-{
- return sizeof(struct nx842_header) +
- hdr->blocks_nr * sizeof(hdr->sizes[0]);
-}
-
-/* Macros for fields within nx_csbcpb */
-/* Check the valid bit within the csbcpb valid field */
-#define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7))
-
-/* CE macros operate on the completion_extension field bits in the csbcpb.
- * CE0 0=full completion, 1=partial completion
- * CE1 0=CE0 indicates completion, 1=termination (output may be modified)
- * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */
-#define NX842_CSBCPB_CE0(x) (x & BIT_MASK(7))
-#define NX842_CSBCPB_CE1(x) (x & BIT_MASK(6))
-#define NX842_CSBCPB_CE2(x) (x & BIT_MASK(5))
-
-/* The NX unit accepts data only on 4K page boundaries */
-#define NX842_HW_PAGE_SHIFT SHIFT_4K
-#define NX842_HW_PAGE_SIZE (ASM_CONST(1) << NX842_HW_PAGE_SHIFT)
-#define NX842_HW_PAGE_MASK (~(NX842_HW_PAGE_SIZE-1))
-
-enum nx842_status {
- UNAVAILABLE,
- AVAILABLE
-};
-
-struct ibm_nx842_counters {
- atomic64_t comp_complete;
- atomic64_t comp_failed;
- atomic64_t decomp_complete;
- atomic64_t decomp_failed;
- atomic64_t swdecomp;
- atomic64_t comp_times[32];
- atomic64_t decomp_times[32];
-};
-
-static struct nx842_devdata {
- struct vio_dev *vdev;
- struct device *dev;
- struct ibm_nx842_counters *counters;
- unsigned int max_sg_len;
- unsigned int max_sync_size;
- unsigned int max_sync_sg;
- enum nx842_status status;
-} __rcu *devdata;
-static DEFINE_SPINLOCK(devdata_mutex);
-
-#define NX842_COUNTER_INC(_x) \
-static inline void nx842_inc_##_x( \
- const struct nx842_devdata *dev) { \
- if (dev) \
- atomic64_inc(&dev->counters->_x); \
-}
-NX842_COUNTER_INC(comp_complete);
-NX842_COUNTER_INC(comp_failed);
-NX842_COUNTER_INC(decomp_complete);
-NX842_COUNTER_INC(decomp_failed);
-NX842_COUNTER_INC(swdecomp);
-
-#define NX842_HIST_SLOTS 16
-
-static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time)
-{
- int bucket = fls(time);
-
- if (bucket)
- bucket = min((NX842_HIST_SLOTS - 1), bucket - 1);
-
- atomic64_inc(&times[bucket]);
-}
-
-/* NX unit operation flags */
-#define NX842_OP_COMPRESS 0x0
-#define NX842_OP_CRC 0x1
-#define NX842_OP_DECOMPRESS 0x2
-#define NX842_OP_COMPRESS_CRC (NX842_OP_COMPRESS | NX842_OP_CRC)
-#define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC)
-#define NX842_OP_ASYNC (1<<23)
-#define NX842_OP_NOTIFY (1<<22)
-#define NX842_OP_NOTIFY_INT(x) ((x & 0xff)<<8)
-
-static unsigned long nx842_get_desired_dma(struct vio_dev *viodev)
-{
- /* No use of DMA mappings within the driver. */
- return 0;
-}
-
-struct nx842_slentry {
- unsigned long ptr; /* Real address (use __pa()) */
- unsigned long len;
-};
-
-/* pHyp scatterlist entry */
-struct nx842_scatterlist {
- int entry_nr; /* number of slentries */
- struct nx842_slentry *entries; /* ptr to array of slentries */
-};
-
-/* Does not include sizeof(entry_nr) in the size */
-static inline unsigned long nx842_get_scatterlist_size(
- struct nx842_scatterlist *sl)
-{
- return sl->entry_nr * sizeof(struct nx842_slentry);
-}
-
-static inline unsigned long nx842_get_pa(void *addr)
-{
- if (is_vmalloc_addr(addr))
- return page_to_phys(vmalloc_to_page(addr))
- + offset_in_page(addr);
- else
- return __pa(addr);
-}
-
-static int nx842_build_scatterlist(unsigned long buf, int len,
- struct nx842_scatterlist *sl)
-{
- unsigned long nextpage;
- struct nx842_slentry *entry;
-
- sl->entry_nr = 0;
-
- entry = sl->entries;
- while (len) {
- entry->ptr = nx842_get_pa((void *)buf);
- nextpage = ALIGN(buf + 1, NX842_HW_PAGE_SIZE);
- if (nextpage < buf + len) {
- /* we aren't at the end yet */
- if (IS_ALIGNED(buf, NX842_HW_PAGE_SIZE))
- /* we are in the middle (or beginning) */
- entry->len = NX842_HW_PAGE_SIZE;
- else
- /* we are at the beginning */
- entry->len = nextpage - buf;
- } else {
- /* at the end */
- entry->len = len;
- }
-
- len -= entry->len;
- buf += entry->len;
- sl->entry_nr++;
- entry++;
- }
-
- return 0;
-}
-
-/*
- * Working memory for software decompression
- */
-struct sw842_fifo {
- union {
- char f8[256][8];
- char f4[512][4];
- };
- char f2[256][2];
- unsigned char f84_full;
- unsigned char f2_full;
- unsigned char f8_count;
- unsigned char f2_count;
- unsigned int f4_count;
-};
-
-/*
- * Working memory for crypto API
+/**
+ * nx842_constraints
+ *
+ * This provides the driver's constraints. Different nx842 implementations
+ * may have varying requirements. The constraints are:
+ * @alignment: All buffers should be aligned to this
+ * @multiple: All buffer lengths should be a multiple of this
+ * @minimum: Buffer lengths must not be less than this amount
+ * @maximum: Buffer lengths must not be more than this amount
+ *
+ * The constraints apply to all buffers and lengths, both input and output,
+ * for both compression and decompression, except for the minimum which
+ * only applies to compression input and decompression output; the
+ * compressed data can be less than the minimum constraint. It can be
+ * assumed that compressed data will always adhere to the multiple
+ * constraint.
+ *
+ * The driver may succeed even if these constraints are violated;
+ * however the driver can return failure or suffer reduced performance
+ * if any constraint is not met.
*/
-struct nx842_workmem {
- char bounce[PAGE_SIZE]; /* bounce buffer for decompression input */
- union {
- /* hardware working memory */
- struct {
- /* scatterlist */
- char slin[SIZE_4K];
- char slout[SIZE_4K];
- /* coprocessor status/parameter block */
- struct nx_csbcpb csbcpb;
- };
- /* software working memory */
- struct sw842_fifo swfifo; /* software decompression fifo */
- };
-};
-
-int nx842_get_workmem_size(void)
-{
- return sizeof(struct nx842_workmem) + NX842_HW_PAGE_SIZE;
-}
-EXPORT_SYMBOL_GPL(nx842_get_workmem_size);
-
-int nx842_get_workmem_size_aligned(void)
-{
- return sizeof(struct nx842_workmem);
-}
-EXPORT_SYMBOL_GPL(nx842_get_workmem_size_aligned);
-
-static int nx842_validate_result(struct device *dev,
- struct cop_status_block *csb)
+int nx842_constraints(struct nx842_constraints *c)
{
- /* The csb must be valid after returning from vio_h_cop_sync */
- if (!NX842_CSBCBP_VALID_CHK(csb->valid)) {
- dev_err(dev, "%s: cspcbp not valid upon completion.\n",
- __func__);
- dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n",
- csb->valid,
- csb->crb_seq_number,
- csb->completion_code,
- csb->completion_extension);
- dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n",
- csb->processed_byte_count,
- (unsigned long)csb->address);
- return -EIO;
- }
-
- /* Check return values from the hardware in the CSB */
- switch (csb->completion_code) {
- case 0: /* Completed without error */
- break;
- case 64: /* Target bytes > Source bytes during compression */
- case 13: /* Output buffer too small */
- dev_dbg(dev, "%s: Compression output larger than input\n",
- __func__);
- return -ENOSPC;
- case 66: /* Input data contains an illegal template field */
- case 67: /* Template indicates data past the end of the input stream */
- dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n",
- __func__, csb->completion_code);
- return -EINVAL;
- default:
- dev_dbg(dev, "%s: Unspecified error (code:%d)\n",
- __func__, csb->completion_code);
- return -EIO;
- }
-
- /* Hardware sanity check */
- if (!NX842_CSBCPB_CE2(csb->completion_extension)) {
- dev_err(dev, "%s: No error returned by hardware, but "
- "data returned is unusable, contact support.\n"
- "(Additional info: csbcbp->processed bytes "
- "does not specify processed bytes for the "
- "target buffer.)\n", __func__);
- return -EIO;
- }
-
+ memcpy(c, nx842_platform_driver()->constraints, sizeof(*c));
return 0;
}
+EXPORT_SYMBOL_GPL(nx842_constraints);
/**
- * nx842_compress - Compress data using the 842 algorithm
- *
- * Compression provide by the NX842 coprocessor on IBM Power systems.
- * The input buffer is compressed and the result is stored in the
- * provided output buffer.
- *
- * Upon return from this function @outlen contains the length of the
- * compressed data. If there is an error then @outlen will be 0 and an
- * error will be specified by the return code from this function.
- *
- * @in: Pointer to input buffer, must be page aligned
- * @inlen: Length of input buffer, must be PAGE_SIZE
- * @out: Pointer to output buffer
- * @outlen: Length of output buffer
- * @wrkmem: ptr to buffer for working memory, size determined by
- * nx842_get_workmem_size()
+ * nx842_workmem_size
*
- * Returns:
- * 0 Success, output of length @outlen stored in the buffer at @out
- * -ENOMEM Unable to allocate internal buffers
- * -ENOSPC Output buffer is to small
- * -EMSGSIZE XXX Difficult to describe this limitation
- * -EIO Internal error
- * -ENODEV Hardware unavailable
+ * Get the amount of working memory the driver requires.
*/
-int nx842_compress(const unsigned char *in, unsigned int inlen,
- unsigned char *out, unsigned int *outlen, void *wmem)
+size_t nx842_workmem_size(void)
{
- struct nx842_header *hdr;
- struct nx842_devdata *local_devdata;
- struct device *dev = NULL;
- struct nx842_workmem *workmem;
- struct nx842_scatterlist slin, slout;
- struct nx_csbcpb *csbcpb;
- int ret = 0, max_sync_size, i, bytesleft, size, hdrsize;
- unsigned long inbuf, outbuf, padding;
- struct vio_pfo_op op = {
- .done = NULL,
- .handle = 0,
- .timeout = 0,
- };
- unsigned long start_time = get_tb();
-
- /*
- * Make sure input buffer is 64k page aligned. This is assumed since
- * this driver is designed for page compression only (for now). This
- * is very nice since we can now use direct DDE(s) for the input and
- * the alignment is guaranteed.
- */
- inbuf = (unsigned long)in;
- if (!IS_ALIGNED(inbuf, PAGE_SIZE) || inlen != PAGE_SIZE)
- return -EINVAL;
-
- rcu_read_lock();
- local_devdata = rcu_dereference(devdata);
- if (!local_devdata || !local_devdata->dev) {
- rcu_read_unlock();
- return -ENODEV;
- }
- max_sync_size = local_devdata->max_sync_size;
- dev = local_devdata->dev;
-
- /* Create the header */
- hdr = (struct nx842_header *)out;
- hdr->blocks_nr = PAGE_SIZE / max_sync_size;
- hdrsize = nx842_header_size(hdr);
- outbuf = (unsigned long)out + hdrsize;
- bytesleft = *outlen - hdrsize;
-
- /* Init scatterlist */
- workmem = (struct nx842_workmem *)ALIGN((unsigned long)wmem,
- NX842_HW_PAGE_SIZE);
- slin.entries = (struct nx842_slentry *)workmem->slin;
- slout.entries = (struct nx842_slentry *)workmem->slout;
-
- /* Init operation */
- op.flags = NX842_OP_COMPRESS;
- csbcpb = &workmem->csbcpb;
- memset(csbcpb, 0, sizeof(*csbcpb));
- op.csbcpb = nx842_get_pa(csbcpb);
- op.out = nx842_get_pa(slout.entries);
-
- for (i = 0; i < hdr->blocks_nr; i++) {
- /*
- * Aligning the output blocks to 128 bytes does waste space,
- * but it prevents the need for bounce buffers and memory
- * copies. It also simplifies the code a lot. In the worst
- * case (64k page, 4k max_sync_size), you lose up to
- * (128*16)/64k = ~3% the compression factor. For 64k
- * max_sync_size, the loss would be at most 128/64k = ~0.2%.
- */
- padding = ALIGN(outbuf, IO_BUFFER_ALIGN) - outbuf;
- outbuf += padding;
- bytesleft -= padding;
- if (i == 0)
- /* save offset into first block in header */
- hdr->offset = padding + hdrsize;
-
- if (bytesleft <= 0) {
- ret = -ENOSPC;
- goto unlock;
- }
-
- /*
- * NOTE: If the default max_sync_size is changed from 4k
- * to 64k, remove the "likely" case below, since a
- * scatterlist will always be needed.
- */
- if (likely(max_sync_size == NX842_HW_PAGE_SIZE)) {
- /* Create direct DDE */
- op.in = nx842_get_pa((void *)inbuf);
- op.inlen = max_sync_size;
-
- } else {
- /* Create indirect DDE (scatterlist) */
- nx842_build_scatterlist(inbuf, max_sync_size, &slin);
- op.in = nx842_get_pa(slin.entries);
- op.inlen = -nx842_get_scatterlist_size(&slin);
- }
-
- /*
- * If max_sync_size != NX842_HW_PAGE_SIZE, an indirect
- * DDE is required for the outbuf.
- * If max_sync_size == NX842_HW_PAGE_SIZE, outbuf must
- * also be page aligned (1 in 128/4k=32 chance) in order
- * to use a direct DDE.
- * This is unlikely, just use an indirect DDE always.
- */
- nx842_build_scatterlist(outbuf,
- min(bytesleft, max_sync_size), &slout);
- /* op.out set before loop */
- op.outlen = -nx842_get_scatterlist_size(&slout);
-
- /* Send request to pHyp */
- ret = vio_h_cop_sync(local_devdata->vdev, &op);
-
- /* Check for pHyp error */
- if (ret) {
- dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
- __func__, ret, op.hcall_err);
- ret = -EIO;
- goto unlock;
- }
-
- /* Check for hardware error */
- ret = nx842_validate_result(dev, &csbcpb->csb);
- if (ret && ret != -ENOSPC)
- goto unlock;
-
- /* Handle incompressible data */
- if (unlikely(ret == -ENOSPC)) {
- if (bytesleft < max_sync_size) {
- /*
- * Not enough space left in the output buffer
- * to store uncompressed block
- */
- goto unlock;
- } else {
- /* Store incompressible block */
- memcpy((void *)outbuf, (void *)inbuf,
- max_sync_size);
- hdr->sizes[i] = -max_sync_size;
- outbuf += max_sync_size;
- bytesleft -= max_sync_size;
- /* Reset ret, incompressible data handled */
- ret = 0;
- }
- } else {
- /* Normal case, compression was successful */
- size = csbcpb->csb.processed_byte_count;
- dev_dbg(dev, "%s: processed_bytes=%d\n",
- __func__, size);
- hdr->sizes[i] = size;
- outbuf += size;
- bytesleft -= size;
- }
-
- inbuf += max_sync_size;
- }
-
- *outlen = (unsigned int)(outbuf - (unsigned long)out);
-
-unlock:
- if (ret)
- nx842_inc_comp_failed(local_devdata);
- else {
- nx842_inc_comp_complete(local_devdata);
- ibm_nx842_incr_hist(local_devdata->counters->comp_times,
- (get_tb() - start_time) / tb_ticks_per_usec);
- }
- rcu_read_unlock();
- return ret;
+ return nx842_platform_driver()->workmem_size;
}
-EXPORT_SYMBOL_GPL(nx842_compress);
-
-static int sw842_decompress(const unsigned char *, int, unsigned char *, int *,
- const void *);
+EXPORT_SYMBOL_GPL(nx842_workmem_size);
-/**
- * nx842_decompress - Decompress data using the 842 algorithm
- *
- * Decompression provide by the NX842 coprocessor on IBM Power systems.
- * The input buffer is decompressed and the result is stored in the
- * provided output buffer. The size allocated to the output buffer is
- * provided by the caller of this function in @outlen. Upon return from
- * this function @outlen contains the length of the decompressed data.
- * If there is an error then @outlen will be 0 and an error will be
- * specified by the return code from this function.
- *
- * @in: Pointer to input buffer, will use bounce buffer if not 128 byte
- * aligned
- * @inlen: Length of input buffer
- * @out: Pointer to output buffer, must be page aligned
- * @outlen: Length of output buffer, must be PAGE_SIZE
- * @wrkmem: ptr to buffer for working memory, size determined by
- * nx842_get_workmem_size()
- *
- * Returns:
- * 0 Success, output of length @outlen stored in the buffer at @out
- * -ENODEV Hardware decompression device is unavailable
- * -ENOMEM Unable to allocate internal buffers
- * -ENOSPC Output buffer is to small
- * -EINVAL Bad input data encountered when attempting decompress
- * -EIO Internal error
- */
-int nx842_decompress(const unsigned char *in, unsigned int inlen,
- unsigned char *out, unsigned int *outlen, void *wmem)
+int nx842_compress(const unsigned char *in, unsigned int ilen,
+ unsigned char *out, unsigned int *olen, void *wmem)
{
- struct nx842_header *hdr;
- struct nx842_devdata *local_devdata;
- struct device *dev = NULL;
- struct nx842_workmem *workmem;
- struct nx842_scatterlist slin, slout;
- struct nx_csbcpb *csbcpb;
- int ret = 0, i, size, max_sync_size;
- unsigned long inbuf, outbuf;
- struct vio_pfo_op op = {
- .done = NULL,
- .handle = 0,
- .timeout = 0,
- };
- unsigned long start_time = get_tb();
-
- /* Ensure page alignment and size */
- outbuf = (unsigned long)out;
- if (!IS_ALIGNED(outbuf, PAGE_SIZE) || *outlen != PAGE_SIZE)
- return -EINVAL;
-
- rcu_read_lock();
- local_devdata = rcu_dereference(devdata);
- if (local_devdata)
- dev = local_devdata->dev;
-
- /* Get header */
- hdr = (struct nx842_header *)in;
-
- workmem = (struct nx842_workmem *)ALIGN((unsigned long)wmem,
- NX842_HW_PAGE_SIZE);
-
- inbuf = (unsigned long)in + hdr->offset;
- if (likely(!IS_ALIGNED(inbuf, IO_BUFFER_ALIGN))) {
- /* Copy block(s) into bounce buffer for alignment */
- memcpy(workmem->bounce, in + hdr->offset, inlen - hdr->offset);
- inbuf = (unsigned long)workmem->bounce;
- }
-
- /* Init scatterlist */
- slin.entries = (struct nx842_slentry *)workmem->slin;
- slout.entries = (struct nx842_slentry *)workmem->slout;
-
- /* Init operation */
- op.flags = NX842_OP_DECOMPRESS;
- csbcpb = &workmem->csbcpb;
- memset(csbcpb, 0, sizeof(*csbcpb));
- op.csbcpb = nx842_get_pa(csbcpb);
-
- /*
- * max_sync_size may have changed since compression,
- * so we can't read it from the device info. We need
- * to derive it from hdr->blocks_nr.
- */
- max_sync_size = PAGE_SIZE / hdr->blocks_nr;
-
- for (i = 0; i < hdr->blocks_nr; i++) {
- /* Skip padding */
- inbuf = ALIGN(inbuf, IO_BUFFER_ALIGN);
-
- if (hdr->sizes[i] < 0) {
- /* Negative sizes indicate uncompressed data blocks */
- size = abs(hdr->sizes[i]);
- memcpy((void *)outbuf, (void *)inbuf, size);
- outbuf += size;
- inbuf += size;
- continue;
- }
-
- if (!dev)
- goto sw;
-
- /*
- * The better the compression, the more likely the "likely"
- * case becomes.
- */
- if (likely((inbuf & NX842_HW_PAGE_MASK) ==
- ((inbuf + hdr->sizes[i] - 1) & NX842_HW_PAGE_MASK))) {
- /* Create direct DDE */
- op.in = nx842_get_pa((void *)inbuf);
- op.inlen = hdr->sizes[i];
- } else {
- /* Create indirect DDE (scatterlist) */
- nx842_build_scatterlist(inbuf, hdr->sizes[i] , &slin);
- op.in = nx842_get_pa(slin.entries);
- op.inlen = -nx842_get_scatterlist_size(&slin);
- }
-
- /*
- * NOTE: If the default max_sync_size is changed from 4k
- * to 64k, remove the "likely" case below, since a
- * scatterlist will always be needed.
- */
- if (likely(max_sync_size == NX842_HW_PAGE_SIZE)) {
- /* Create direct DDE */
- op.out = nx842_get_pa((void *)outbuf);
- op.outlen = max_sync_size;
- } else {
- /* Create indirect DDE (scatterlist) */
- nx842_build_scatterlist(outbuf, max_sync_size, &slout);
- op.out = nx842_get_pa(slout.entries);
- op.outlen = -nx842_get_scatterlist_size(&slout);
- }
-
- /* Send request to pHyp */
- ret = vio_h_cop_sync(local_devdata->vdev, &op);
-
- /* Check for pHyp error */
- if (ret) {
- dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
- __func__, ret, op.hcall_err);
- dev = NULL;
- goto sw;
- }
-
- /* Check for hardware error */
- ret = nx842_validate_result(dev, &csbcpb->csb);
- if (ret) {
- dev = NULL;
- goto sw;
- }
-
- /* HW decompression success */
- inbuf += hdr->sizes[i];
- outbuf += csbcpb->csb.processed_byte_count;
- continue;
-
-sw:
- /* software decompression */
- size = max_sync_size;
- ret = sw842_decompress(
- (unsigned char *)inbuf, hdr->sizes[i],
- (unsigned char *)outbuf, &size, wmem);
- if (ret)
- pr_debug("%s: sw842_decompress failed with %d\n",
- __func__, ret);
-
- if (ret) {
- if (ret != -ENOSPC && ret != -EINVAL &&
- ret != -EMSGSIZE)
- ret = -EIO;
- goto unlock;
- }
-
- /* SW decompression success */
- inbuf += hdr->sizes[i];
- outbuf += size;
- }
-
- *outlen = (unsigned int)(outbuf - (unsigned long)out);
-
-unlock:
- if (ret)
- /* decompress fail */
- nx842_inc_decomp_failed(local_devdata);
- else {
- if (!dev)
- /* software decompress */
- nx842_inc_swdecomp(local_devdata);
- nx842_inc_decomp_complete(local_devdata);
- ibm_nx842_incr_hist(local_devdata->counters->decomp_times,
- (get_tb() - start_time) / tb_ticks_per_usec);
- }
-
- rcu_read_unlock();
- return ret;
+ return nx842_platform_driver()->compress(in, ilen, out, olen, wmem);
}
-EXPORT_SYMBOL_GPL(nx842_decompress);
+EXPORT_SYMBOL_GPL(nx842_compress);
-/**
- * nx842_OF_set_defaults -- Set default (disabled) values for devdata
- *
- * @devdata - struct nx842_devdata to update
- *
- * Returns:
- * 0 on success
- * -ENOENT if @devdata ptr is NULL
- */
-static int nx842_OF_set_defaults(struct nx842_devdata *devdata)
+int nx842_decompress(const unsigned char *in, unsigned int ilen,
+ unsigned char *out, unsigned int *olen, void *wmem)
{
- if (devdata) {
- devdata->max_sync_size = 0;
- devdata->max_sync_sg = 0;
- devdata->max_sg_len = 0;
- devdata->status = UNAVAILABLE;
- return 0;
- } else
- return -ENOENT;
-}
-
-/**
- * nx842_OF_upd_status -- Update the device info from OF status prop
- *
- * The status property indicates if the accelerator is enabled. If the
- * device is in the OF tree it indicates that the hardware is present.
- * The status field indicates if the device is enabled when the status
- * is 'okay'. Otherwise the device driver will be disabled.
- *
- * @devdata - struct nx842_devdata to update
- * @prop - struct property point containing the maxsyncop for the update
- *
- * Returns:
- * 0 - Device is available
- * -EINVAL - Device is not available
- */
-static int nx842_OF_upd_status(struct nx842_devdata *devdata,
- struct property *prop) {
- int ret = 0;
- const char *status = (const char *)prop->value;
-
- if (!strncmp(status, "okay", (size_t)prop->length)) {
- devdata->status = AVAILABLE;
- } else {
- dev_info(devdata->dev, "%s: status '%s' is not 'okay'\n",
- __func__, status);
- devdata->status = UNAVAILABLE;
- }
-
- return ret;
-}
-
-/**
- * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop
- *
- * Definition of the 'ibm,max-sg-len' OF property:
- * This field indicates the maximum byte length of a scatter list
- * for the platform facility. It is a single cell encoded as with encode-int.
- *
- * Example:
- * # od -x ibm,max-sg-len
- * 0000000 0000 0ff0
- *
- * In this example, the maximum byte length of a scatter list is
- * 0x0ff0 (4,080).
- *
- * @devdata - struct nx842_devdata to update
- * @prop - struct property point containing the maxsyncop for the update
- *
- * Returns:
- * 0 on success
- * -EINVAL on failure
- */
-static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata,
- struct property *prop) {
- int ret = 0;
- const int *maxsglen = prop->value;
-
- if (prop->length != sizeof(*maxsglen)) {
- dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__);
- dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__,
- prop->length, sizeof(*maxsglen));
- ret = -EINVAL;
- } else {
- devdata->max_sg_len = (unsigned int)min(*maxsglen,
- (int)NX842_HW_PAGE_SIZE);
- }
-
- return ret;
-}
-
-/**
- * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop
- *
- * Definition of the 'ibm,max-sync-cop' OF property:
- * Two series of cells. The first series of cells represents the maximums
- * that can be synchronously compressed. The second series of cells
- * represents the maximums that can be synchronously decompressed.
- * 1. The first cell in each series contains the count of the number of
- * data length, scatter list elements pairs that follow – each being
- * of the form
- * a. One cell data byte length
- * b. One cell total number of scatter list elements
- *
- * Example:
- * # od -x ibm,max-sync-cop
- * 0000000 0000 0001 0000 1000 0000 01fe 0000 0001
- * 0000020 0000 1000 0000 01fe
- *
- * In this example, compression supports 0x1000 (4,096) data byte length
- * and 0x1fe (510) total scatter list elements. Decompression supports
- * 0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list
- * elements.
- *
- * @devdata - struct nx842_devdata to update
- * @prop - struct property point containing the maxsyncop for the update
- *
- * Returns:
- * 0 on success
- * -EINVAL on failure
- */
-static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata,
- struct property *prop) {
- int ret = 0;
- const struct maxsynccop_t {
- int comp_elements;
- int comp_data_limit;
- int comp_sg_limit;
- int decomp_elements;
- int decomp_data_limit;
- int decomp_sg_limit;
- } *maxsynccop;
-
- if (prop->length != sizeof(*maxsynccop)) {
- dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__);
- dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length,
- sizeof(*maxsynccop));
- ret = -EINVAL;
- goto out;
- }
-
- maxsynccop = (const struct maxsynccop_t *)prop->value;
-
- /* Use one limit rather than separate limits for compression and
- * decompression. Set a maximum for this so as not to exceed the
- * size that the header can support and round the value down to
- * the hardware page size (4K) */
- devdata->max_sync_size =
- (unsigned int)min(maxsynccop->comp_data_limit,
- maxsynccop->decomp_data_limit);
-
- devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size,
- SIZE_64K);
-
- if (devdata->max_sync_size < SIZE_4K) {
- dev_err(devdata->dev, "%s: hardware max data size (%u) is "
- "less than the driver minimum, unable to use "
- "the hardware device\n",
- __func__, devdata->max_sync_size);
- ret = -EINVAL;
- goto out;
- }
-
- devdata->max_sync_sg = (unsigned int)min(maxsynccop->comp_sg_limit,
- maxsynccop->decomp_sg_limit);
- if (devdata->max_sync_sg < 1) {
- dev_err(devdata->dev, "%s: hardware max sg size (%u) is "
- "less than the driver minimum, unable to use "
- "the hardware device\n",
- __func__, devdata->max_sync_sg);
- ret = -EINVAL;
- goto out;
- }
-
-out:
- return ret;
+ return nx842_platform_driver()->decompress(in, ilen, out, olen, wmem);
}
+EXPORT_SYMBOL_GPL(nx842_decompress);
-/**
- *
- * nx842_OF_upd -- Handle OF properties updates for the device.
- *
- * Set all properties from the OF tree. Optionally, a new property
- * can be provided by the @new_prop pointer to overwrite an existing value.
- * The device will remain disabled until all values are valid, this function
- * will return an error for updates unless all values are valid.
- *
- * @new_prop: If not NULL, this property is being updated. If NULL, update
- * all properties from the current values in the OF tree.
- *
- * Returns:
- * 0 - Success
- * -ENOMEM - Could not allocate memory for new devdata structure
- * -EINVAL - property value not found, new_prop is not a recognized
- * property for the device or property value is not valid.
- * -ENODEV - Device is not available
- */
-static int nx842_OF_upd(struct property *new_prop)
+static __init int nx842_init(void)
{
- struct nx842_devdata *old_devdata = NULL;
- struct nx842_devdata *new_devdata = NULL;
- struct device_node *of_node = NULL;
- struct property *status = NULL;
- struct property *maxsglen = NULL;
- struct property *maxsyncop = NULL;
- int ret = 0;
- unsigned long flags;
-
- spin_lock_irqsave(&devdata_mutex, flags);
- old_devdata = rcu_dereference_check(devdata,
- lockdep_is_held(&devdata_mutex));
- if (old_devdata)
- of_node = old_devdata->dev->of_node;
+ request_module("nx-compress-powernv");
+ request_module("nx-compress-pseries");
- if (!old_devdata || !of_node) {
- pr_err("%s: device is not available\n", __func__);
- spin_unlock_irqrestore(&devdata_mutex, flags);
- return -ENODEV;
- }
-
- new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
- if (!new_devdata) {
- dev_err(old_devdata->dev, "%s: Could not allocate memory for device data\n", __func__);
- ret = -ENOMEM;
- goto error_out;
- }
-
- memcpy(new_devdata, old_devdata, sizeof(*old_devdata));
- new_devdata->counters = old_devdata->counters;
-
- /* Set ptrs for existing properties */
- status = of_find_property(of_node, "status", NULL);
- maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL);
- maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL);
- if (!status || !maxsglen || !maxsyncop) {
- dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__);
- ret = -EINVAL;
- goto error_out;
- }
-
- /*
- * If this is a property update, there are only certain properties that
- * we care about. Bail if it isn't in the below list
+ /* we prevent loading if there's no platform driver, and we get the
+ * module that set it so it won't unload, so we don't need to check
+ * if it's set in any of the above functions
*/
- if (new_prop && (strncmp(new_prop->name, "status", new_prop->length) ||
- strncmp(new_prop->name, "ibm,max-sg-len", new_prop->length) ||
- strncmp(new_prop->name, "ibm,max-sync-cop", new_prop->length)))
- goto out;
-
- /* Perform property updates */
- ret = nx842_OF_upd_status(new_devdata, status);
- if (ret)
- goto error_out;
-
- ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen);
- if (ret)
- goto error_out;
-
- ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop);
- if (ret)
- goto error_out;
-
-out:
- dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n",
- __func__, new_devdata->max_sync_size,
- old_devdata->max_sync_size);
- dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n",
- __func__, new_devdata->max_sync_sg,
- old_devdata->max_sync_sg);
- dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n",
- __func__, new_devdata->max_sg_len,
- old_devdata->max_sg_len);
-
- rcu_assign_pointer(devdata, new_devdata);
- spin_unlock_irqrestore(&devdata_mutex, flags);
- synchronize_rcu();
- dev_set_drvdata(new_devdata->dev, new_devdata);
- kfree(old_devdata);
- return 0;
-
-error_out:
- if (new_devdata) {
- dev_info(old_devdata->dev, "%s: device disabled\n", __func__);
- nx842_OF_set_defaults(new_devdata);
- rcu_assign_pointer(devdata, new_devdata);
- spin_unlock_irqrestore(&devdata_mutex, flags);
- synchronize_rcu();
- dev_set_drvdata(new_devdata->dev, new_devdata);
- kfree(old_devdata);
- } else {
- dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__);
- spin_unlock_irqrestore(&devdata_mutex, flags);
- }
-
- if (!ret)
- ret = -EINVAL;
- return ret;
-}
-
-/**
- * nx842_OF_notifier - Process updates to OF properties for the device
- *
- * @np: notifier block
- * @action: notifier action
- * @update: struct pSeries_reconfig_prop_update pointer if action is
- * PSERIES_UPDATE_PROPERTY
- *
- * Returns:
- * NOTIFY_OK on success
- * NOTIFY_BAD encoded with error number on failure, use
- * notifier_to_errno() to decode this value
- */
-static int nx842_OF_notifier(struct notifier_block *np, unsigned long action,
- void *data)
-{
- struct of_reconfig_data *upd = data;
- struct nx842_devdata *local_devdata;
- struct device_node *node = NULL;
-
- rcu_read_lock();
- local_devdata = rcu_dereference(devdata);
- if (local_devdata)
- node = local_devdata->dev->of_node;
-
- if (local_devdata &&
- action == OF_RECONFIG_UPDATE_PROPERTY &&
- !strcmp(upd->dn->name, node->name)) {
- rcu_read_unlock();
- nx842_OF_upd(upd->prop);
- } else
- rcu_read_unlock();
-
- return NOTIFY_OK;
-}
-
-static struct notifier_block nx842_of_nb = {
- .notifier_call = nx842_OF_notifier,
-};
-
-#define nx842_counter_read(_name) \
-static ssize_t nx842_##_name##_show(struct device *dev, \
- struct device_attribute *attr, \
- char *buf) { \
- struct nx842_devdata *local_devdata; \
- int p = 0; \
- rcu_read_lock(); \
- local_devdata = rcu_dereference(devdata); \
- if (local_devdata) \
- p = snprintf(buf, PAGE_SIZE, "%ld\n", \
- atomic64_read(&local_devdata->counters->_name)); \
- rcu_read_unlock(); \
- return p; \
-}
-
-#define NX842DEV_COUNTER_ATTR_RO(_name) \
- nx842_counter_read(_name); \
- static struct device_attribute dev_attr_##_name = __ATTR(_name, \
- 0444, \
- nx842_##_name##_show,\
- NULL);
-
-NX842DEV_COUNTER_ATTR_RO(comp_complete);
-NX842DEV_COUNTER_ATTR_RO(comp_failed);
-NX842DEV_COUNTER_ATTR_RO(decomp_complete);
-NX842DEV_COUNTER_ATTR_RO(decomp_failed);
-NX842DEV_COUNTER_ATTR_RO(swdecomp);
-
-static ssize_t nx842_timehist_show(struct device *,
- struct device_attribute *, char *);
-
-static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444,
- nx842_timehist_show, NULL);
-static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times,
- 0444, nx842_timehist_show, NULL);
-
-static ssize_t nx842_timehist_show(struct device *dev,
- struct device_attribute *attr, char *buf) {
- char *p = buf;
- struct nx842_devdata *local_devdata;
- atomic64_t *times;
- int bytes_remain = PAGE_SIZE;
- int bytes;
- int i;
-
- rcu_read_lock();
- local_devdata = rcu_dereference(devdata);
- if (!local_devdata) {
- rcu_read_unlock();
- return 0;
- }
-
- if (attr == &dev_attr_comp_times)
- times = local_devdata->counters->comp_times;
- else if (attr == &dev_attr_decomp_times)
- times = local_devdata->counters->decomp_times;
- else {
- rcu_read_unlock();
- return 0;
- }
-
- for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) {
- bytes = snprintf(p, bytes_remain, "%u-%uus:\t%ld\n",
- i ? (2<<(i-1)) : 0, (2<<i)-1,
- atomic64_read(&times[i]));
- bytes_remain -= bytes;
- p += bytes;
- }
- /* The last bucket holds everything over
- * 2<<(NX842_HIST_SLOTS - 2) us */
- bytes = snprintf(p, bytes_remain, "%uus - :\t%ld\n",
- 2<<(NX842_HIST_SLOTS - 2),
- atomic64_read(&times[(NX842_HIST_SLOTS - 1)]));
- p += bytes;
-
- rcu_read_unlock();
- return p - buf;
-}
-
-static struct attribute *nx842_sysfs_entries[] = {
- &dev_attr_comp_complete.attr,
- &dev_attr_comp_failed.attr,
- &dev_attr_decomp_complete.attr,
- &dev_attr_decomp_failed.attr,
- &dev_attr_swdecomp.attr,
- &dev_attr_comp_times.attr,
- &dev_attr_decomp_times.attr,
- NULL,
-};
-
-static struct attribute_group nx842_attribute_group = {
- .name = NULL, /* put in device directory */
- .attrs = nx842_sysfs_entries,
-};
-
-static int __init nx842_probe(struct vio_dev *viodev,
- const struct vio_device_id *id)
-{
- struct nx842_devdata *old_devdata, *new_devdata = NULL;
- unsigned long flags;
- int ret = 0;
-
- spin_lock_irqsave(&devdata_mutex, flags);
- old_devdata = rcu_dereference_check(devdata,
- lockdep_is_held(&devdata_mutex));
-
- if (old_devdata && old_devdata->vdev != NULL) {
- dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__);
- ret = -1;
- goto error_unlock;
- }
-
- dev_set_drvdata(&viodev->dev, NULL);
-
- new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
- if (!new_devdata) {
- dev_err(&viodev->dev, "%s: Could not allocate memory for device data\n", __func__);
- ret = -ENOMEM;
- goto error_unlock;
- }
-
- new_devdata->counters = kzalloc(sizeof(*new_devdata->counters),
- GFP_NOFS);
- if (!new_devdata->counters) {
- dev_err(&viodev->dev, "%s: Could not allocate memory for performance counters\n", __func__);
- ret = -ENOMEM;
- goto error_unlock;
- }
-
- new_devdata->vdev = viodev;
- new_devdata->dev = &viodev->dev;
- nx842_OF_set_defaults(new_devdata);
-
- rcu_assign_pointer(devdata, new_devdata);
- spin_unlock_irqrestore(&devdata_mutex, flags);
- synchronize_rcu();
- kfree(old_devdata);
-
- of_reconfig_notifier_register(&nx842_of_nb);
-
- ret = nx842_OF_upd(NULL);
- if (ret && ret != -ENODEV) {
- dev_err(&viodev->dev, "could not parse device tree. %d\n", ret);
- ret = -1;
- goto error;
- }
-
- rcu_read_lock();
- dev_set_drvdata(&viodev->dev, rcu_dereference(devdata));
- rcu_read_unlock();
-
- if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) {
- dev_err(&viodev->dev, "could not create sysfs device attributes\n");
- ret = -1;
- goto error;
+ if (!nx842_platform_driver_get()) {
+ pr_err("no nx842 driver found.\n");
+ return -ENODEV;
}
return 0;
-
-error_unlock:
- spin_unlock_irqrestore(&devdata_mutex, flags);
- if (new_devdata)
- kfree(new_devdata->counters);
- kfree(new_devdata);
-error:
- return ret;
-}
-
-static int __exit nx842_remove(struct vio_dev *viodev)
-{
- struct nx842_devdata *old_devdata;
- unsigned long flags;
-
- pr_info("Removing IBM Power 842 compression device\n");
- sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group);
-
- spin_lock_irqsave(&devdata_mutex, flags);
- old_devdata = rcu_dereference_check(devdata,
- lockdep_is_held(&devdata_mutex));
- of_reconfig_notifier_unregister(&nx842_of_nb);
- RCU_INIT_POINTER(devdata, NULL);
- spin_unlock_irqrestore(&devdata_mutex, flags);
- synchronize_rcu();
- dev_set_drvdata(&viodev->dev, NULL);
- if (old_devdata)
- kfree(old_devdata->counters);
- kfree(old_devdata);
- return 0;
-}
-
-static struct vio_device_id nx842_driver_ids[] = {
- {"ibm,compression-v1", "ibm,compression"},
- {"", ""},
-};
-
-static struct vio_driver nx842_driver = {
- .name = MODULE_NAME,
- .probe = nx842_probe,
- .remove = __exit_p(nx842_remove),
- .get_desired_dma = nx842_get_desired_dma,
- .id_table = nx842_driver_ids,
-};
-
-static int __init nx842_init(void)
-{
- struct nx842_devdata *new_devdata;
- pr_info("Registering IBM Power 842 compression driver\n");
-
- RCU_INIT_POINTER(devdata, NULL);
- new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL);
- if (!new_devdata) {
- pr_err("Could not allocate memory for device data\n");
- return -ENOMEM;
- }
- new_devdata->status = UNAVAILABLE;
- RCU_INIT_POINTER(devdata, new_devdata);
-
- return vio_register_driver(&nx842_driver);
}
-
module_init(nx842_init);
static void __exit nx842_exit(void)
{
- struct nx842_devdata *old_devdata;
- unsigned long flags;
-
- pr_info("Exiting IBM Power 842 compression driver\n");
- spin_lock_irqsave(&devdata_mutex, flags);
- old_devdata = rcu_dereference_check(devdata,
- lockdep_is_held(&devdata_mutex));
- RCU_INIT_POINTER(devdata, NULL);
- spin_unlock_irqrestore(&devdata_mutex, flags);
- synchronize_rcu();
- if (old_devdata)
- dev_set_drvdata(old_devdata->dev, NULL);
- kfree(old_devdata);
- vio_unregister_driver(&nx842_driver);
+ nx842_platform_driver_put();
}
-
module_exit(nx842_exit);
-
-/*********************************
- * 842 software decompressor
-*********************************/
-typedef int (*sw842_template_op)(const char **, int *, unsigned char **,
- struct sw842_fifo *);
-
-static int sw842_data8(const char **, int *, unsigned char **,
- struct sw842_fifo *);
-static int sw842_data4(const char **, int *, unsigned char **,
- struct sw842_fifo *);
-static int sw842_data2(const char **, int *, unsigned char **,
- struct sw842_fifo *);
-static int sw842_ptr8(const char **, int *, unsigned char **,
- struct sw842_fifo *);
-static int sw842_ptr4(const char **, int *, unsigned char **,
- struct sw842_fifo *);
-static int sw842_ptr2(const char **, int *, unsigned char **,
- struct sw842_fifo *);
-
-/* special templates */
-#define SW842_TMPL_REPEAT 0x1B
-#define SW842_TMPL_ZEROS 0x1C
-#define SW842_TMPL_EOF 0x1E
-
-static sw842_template_op sw842_tmpl_ops[26][4] = {
- { sw842_data8, NULL}, /* 0 (00000) */
- { sw842_data4, sw842_data2, sw842_ptr2, NULL},
- { sw842_data4, sw842_ptr2, sw842_data2, NULL},
- { sw842_data4, sw842_ptr2, sw842_ptr2, NULL},
- { sw842_data4, sw842_ptr4, NULL},
- { sw842_data2, sw842_ptr2, sw842_data4, NULL},
- { sw842_data2, sw842_ptr2, sw842_data2, sw842_ptr2},
- { sw842_data2, sw842_ptr2, sw842_ptr2, sw842_data2},
- { sw842_data2, sw842_ptr2, sw842_ptr2, sw842_ptr2,},
- { sw842_data2, sw842_ptr2, sw842_ptr4, NULL},
- { sw842_ptr2, sw842_data2, sw842_data4, NULL}, /* 10 (01010) */
- { sw842_ptr2, sw842_data4, sw842_ptr2, NULL},
- { sw842_ptr2, sw842_data2, sw842_ptr2, sw842_data2},
- { sw842_ptr2, sw842_data2, sw842_ptr2, sw842_ptr2},
- { sw842_ptr2, sw842_data2, sw842_ptr4, NULL},
- { sw842_ptr2, sw842_ptr2, sw842_data4, NULL},
- { sw842_ptr2, sw842_ptr2, sw842_data2, sw842_ptr2},
- { sw842_ptr2, sw842_ptr2, sw842_ptr2, sw842_data2},
- { sw842_ptr2, sw842_ptr2, sw842_ptr2, sw842_ptr2},
- { sw842_ptr2, sw842_ptr2, sw842_ptr4, NULL},
- { sw842_ptr4, sw842_data4, NULL}, /* 20 (10100) */
- { sw842_ptr4, sw842_data2, sw842_ptr2, NULL},
- { sw842_ptr4, sw842_ptr2, sw842_data2, NULL},
- { sw842_ptr4, sw842_ptr2, sw842_ptr2, NULL},
- { sw842_ptr4, sw842_ptr4, NULL},
- { sw842_ptr8, NULL}
-};
-
-/* Software decompress helpers */
-
-static uint8_t sw842_get_byte(const char *buf, int bit)
-{
- uint8_t tmpl;
- uint16_t tmp;
- tmp = htons(*(uint16_t *)(buf));
- tmp = (uint16_t)(tmp << bit);
- tmp = ntohs(tmp);
- memcpy(&tmpl, &tmp, 1);
- return tmpl;
-}
-
-static uint8_t sw842_get_template(const char **buf, int *bit)
-{
- uint8_t byte;
- byte = sw842_get_byte(*buf, *bit);
- byte = byte >> 3;
- byte &= 0x1F;
- *buf += (*bit + 5) / 8;
- *bit = (*bit + 5) % 8;
- return byte;
-}
-
-/* repeat_count happens to be 5-bit too (like the template) */
-static uint8_t sw842_get_repeat_count(const char **buf, int *bit)
-{
- uint8_t byte;
- byte = sw842_get_byte(*buf, *bit);
- byte = byte >> 2;
- byte &= 0x3F;
- *buf += (*bit + 6) / 8;
- *bit = (*bit + 6) % 8;
- return byte;
-}
-
-static uint8_t sw842_get_ptr2(const char **buf, int *bit)
-{
- uint8_t ptr;
- ptr = sw842_get_byte(*buf, *bit);
- (*buf)++;
- return ptr;
-}
-
-static uint16_t sw842_get_ptr4(const char **buf, int *bit,
- struct sw842_fifo *fifo)
-{
- uint16_t ptr;
- ptr = htons(*(uint16_t *)(*buf));
- ptr = (uint16_t)(ptr << *bit);
- ptr = ptr >> 7;
- ptr &= 0x01FF;
- *buf += (*bit + 9) / 8;
- *bit = (*bit + 9) % 8;
- return ptr;
-}
-
-static uint8_t sw842_get_ptr8(const char **buf, int *bit,
- struct sw842_fifo *fifo)
-{
- return sw842_get_ptr2(buf, bit);
-}
-
-/* Software decompress template ops */
-
-static int sw842_data8(const char **inbuf, int *inbit,
- unsigned char **outbuf, struct sw842_fifo *fifo)
-{
- int ret;
-
- ret = sw842_data4(inbuf, inbit, outbuf, fifo);
- if (ret)
- return ret;
- ret = sw842_data4(inbuf, inbit, outbuf, fifo);
- return ret;
-}
-
-static int sw842_data4(const char **inbuf, int *inbit,
- unsigned char **outbuf, struct sw842_fifo *fifo)
-{
- int ret;
-
- ret = sw842_data2(inbuf, inbit, outbuf, fifo);
- if (ret)
- return ret;
- ret = sw842_data2(inbuf, inbit, outbuf, fifo);
- return ret;
-}
-
-static int sw842_data2(const char **inbuf, int *inbit,
- unsigned char **outbuf, struct sw842_fifo *fifo)
-{
- **outbuf = sw842_get_byte(*inbuf, *inbit);
- (*inbuf)++;
- (*outbuf)++;
- **outbuf = sw842_get_byte(*inbuf, *inbit);
- (*inbuf)++;
- (*outbuf)++;
- return 0;
-}
-
-static int sw842_ptr8(const char **inbuf, int *inbit,
- unsigned char **outbuf, struct sw842_fifo *fifo)
-{
- uint8_t ptr;
- ptr = sw842_get_ptr8(inbuf, inbit, fifo);
- if (!fifo->f84_full && (ptr >= fifo->f8_count))
- return 1;
- memcpy(*outbuf, fifo->f8[ptr], 8);
- *outbuf += 8;
- return 0;
-}
-
-static int sw842_ptr4(const char **inbuf, int *inbit,
- unsigned char **outbuf, struct sw842_fifo *fifo)
-{
- uint16_t ptr;
- ptr = sw842_get_ptr4(inbuf, inbit, fifo);
- if (!fifo->f84_full && (ptr >= fifo->f4_count))
- return 1;
- memcpy(*outbuf, fifo->f4[ptr], 4);
- *outbuf += 4;
- return 0;
-}
-
-static int sw842_ptr2(const char **inbuf, int *inbit,
- unsigned char **outbuf, struct sw842_fifo *fifo)
-{
- uint8_t ptr;
- ptr = sw842_get_ptr2(inbuf, inbit);
- if (!fifo->f2_full && (ptr >= fifo->f2_count))
- return 1;
- memcpy(*outbuf, fifo->f2[ptr], 2);
- *outbuf += 2;
- return 0;
-}
-
-static void sw842_copy_to_fifo(const char *buf, struct sw842_fifo *fifo)
-{
- unsigned char initial_f2count = fifo->f2_count;
-
- memcpy(fifo->f8[fifo->f8_count], buf, 8);
- fifo->f4_count += 2;
- fifo->f8_count += 1;
-
- if (!fifo->f84_full && fifo->f4_count >= 512) {
- fifo->f84_full = 1;
- fifo->f4_count /= 512;
- }
-
- memcpy(fifo->f2[fifo->f2_count++], buf, 2);
- memcpy(fifo->f2[fifo->f2_count++], buf + 2, 2);
- memcpy(fifo->f2[fifo->f2_count++], buf + 4, 2);
- memcpy(fifo->f2[fifo->f2_count++], buf + 6, 2);
- if (fifo->f2_count < initial_f2count)
- fifo->f2_full = 1;
-}
-
-static int sw842_decompress(const unsigned char *src, int srclen,
- unsigned char *dst, int *destlen,
- const void *wrkmem)
-{
- uint8_t tmpl;
- const char *inbuf;
- int inbit = 0;
- unsigned char *outbuf, *outbuf_end, *origbuf, *prevbuf;
- const char *inbuf_end;
- sw842_template_op op;
- int opindex;
- int i, repeat_count;
- struct sw842_fifo *fifo;
- int ret = 0;
-
- fifo = &((struct nx842_workmem *)(wrkmem))->swfifo;
- memset(fifo, 0, sizeof(*fifo));
-
- origbuf = NULL;
- inbuf = src;
- inbuf_end = src + srclen;
- outbuf = dst;
- outbuf_end = dst + *destlen;
-
- while ((tmpl = sw842_get_template(&inbuf, &inbit)) != SW842_TMPL_EOF) {
- if (inbuf >= inbuf_end) {
- ret = -EINVAL;
- goto out;
- }
-
- opindex = 0;
- prevbuf = origbuf;
- origbuf = outbuf;
- switch (tmpl) {
- case SW842_TMPL_REPEAT:
- if (prevbuf == NULL) {
- ret = -EINVAL;
- goto out;
- }
-
- repeat_count = sw842_get_repeat_count(&inbuf,
- &inbit) + 1;
-
- /* Did the repeat count advance past the end of input */
- if (inbuf > inbuf_end) {
- ret = -EINVAL;
- goto out;
- }
-
- for (i = 0; i < repeat_count; i++) {
- /* Would this overflow the output buffer */
- if ((outbuf + 8) > outbuf_end) {
- ret = -ENOSPC;
- goto out;
- }
-
- memcpy(outbuf, prevbuf, 8);
- sw842_copy_to_fifo(outbuf, fifo);
- outbuf += 8;
- }
- break;
-
- case SW842_TMPL_ZEROS:
- /* Would this overflow the output buffer */
- if ((outbuf + 8) > outbuf_end) {
- ret = -ENOSPC;
- goto out;
- }
-
- memset(outbuf, 0, 8);
- sw842_copy_to_fifo(outbuf, fifo);
- outbuf += 8;
- break;
-
- default:
- if (tmpl > 25) {
- ret = -EINVAL;
- goto out;
- }
-
- /* Does this go past the end of the input buffer */
- if ((inbuf + 2) > inbuf_end) {
- ret = -EINVAL;
- goto out;
- }
-
- /* Would this overflow the output buffer */
- if ((outbuf + 8) > outbuf_end) {
- ret = -ENOSPC;
- goto out;
- }
-
- while (opindex < 4 &&
- (op = sw842_tmpl_ops[tmpl][opindex++])
- != NULL) {
- ret = (*op)(&inbuf, &inbit, &outbuf, fifo);
- if (ret) {
- ret = -EINVAL;
- goto out;
- }
- sw842_copy_to_fifo(origbuf, fifo);
- }
- }
- }
-
-out:
- if (!ret)
- *destlen = (unsigned int)(outbuf - dst);
- else
- *destlen = 0;
-
- return ret;
-}
diff --git a/drivers/crypto/nx/nx-842.h b/drivers/crypto/nx/nx-842.h
new file mode 100644
index 000000000..ac0ea79d0
--- /dev/null
+++ b/drivers/crypto/nx/nx-842.h
@@ -0,0 +1,144 @@
+
+#ifndef __NX_842_H__
+#define __NX_842_H__
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/sw842.h>
+#include <linux/of.h>
+#include <linux/slab.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/ratelimit.h>
+
+/* Restrictions on Data Descriptor List (DDL) and Entry (DDE) buffers
+ *
+ * From NX P8 workbook, sec 4.9.1 "842 details"
+ * Each DDE buffer is 128 byte aligned
+ * Each DDE buffer size is a multiple of 32 bytes (except the last)
+ * The last DDE buffer size is a multiple of 8 bytes
+ */
+#define DDE_BUFFER_ALIGN (128)
+#define DDE_BUFFER_SIZE_MULT (32)
+#define DDE_BUFFER_LAST_MULT (8)
+
+/* Arbitrary DDL length limit
+ * Allows max buffer size of MAX-1 to MAX pages
+ * (depending on alignment)
+ */
+#define DDL_LEN_MAX (17)
+
+/* CCW 842 CI/FC masks
+ * NX P8 workbook, section 4.3.1, figure 4-6
+ * "CI/FC Boundary by NX CT type"
+ */
+#define CCW_CI_842 (0x00003ff8)
+#define CCW_FC_842 (0x00000007)
+
+/* CCW Function Codes (FC) for 842
+ * NX P8 workbook, section 4.9, table 4-28
+ * "Function Code Definitions for 842 Memory Compression"
+ */
+#define CCW_FC_842_COMP_NOCRC (0)
+#define CCW_FC_842_COMP_CRC (1)
+#define CCW_FC_842_DECOMP_NOCRC (2)
+#define CCW_FC_842_DECOMP_CRC (3)
+#define CCW_FC_842_MOVE (4)
+
+/* CSB CC Error Types for 842
+ * NX P8 workbook, section 4.10.3, table 4-30
+ * "Reported Error Types Summary Table"
+ */
+/* These are all duplicates of existing codes defined in icswx.h. */
+#define CSB_CC_TRANSLATION_DUP1 (80)
+#define CSB_CC_TRANSLATION_DUP2 (82)
+#define CSB_CC_TRANSLATION_DUP3 (84)
+#define CSB_CC_TRANSLATION_DUP4 (86)
+#define CSB_CC_TRANSLATION_DUP5 (92)
+#define CSB_CC_TRANSLATION_DUP6 (94)
+#define CSB_CC_PROTECTION_DUP1 (81)
+#define CSB_CC_PROTECTION_DUP2 (83)
+#define CSB_CC_PROTECTION_DUP3 (85)
+#define CSB_CC_PROTECTION_DUP4 (87)
+#define CSB_CC_PROTECTION_DUP5 (93)
+#define CSB_CC_PROTECTION_DUP6 (95)
+#define CSB_CC_RD_EXTERNAL_DUP1 (89)
+#define CSB_CC_RD_EXTERNAL_DUP2 (90)
+#define CSB_CC_RD_EXTERNAL_DUP3 (91)
+/* These are specific to NX */
+/* 842 codes */
+#define CSB_CC_TPBC_GT_SPBC (64) /* no error, but >1 comp ratio */
+#define CSB_CC_CRC_MISMATCH (65) /* decomp crc mismatch */
+#define CSB_CC_TEMPL_INVALID (66) /* decomp invalid template value */
+#define CSB_CC_TEMPL_OVERFLOW (67) /* decomp template shows data after end */
+/* sym crypt codes */
+#define CSB_CC_DECRYPT_OVERFLOW (64)
+/* asym crypt codes */
+#define CSB_CC_MINV_OVERFLOW (128)
+/* These are reserved for hypervisor use */
+#define CSB_CC_HYP_RESERVE_START (240)
+#define CSB_CC_HYP_RESERVE_END (253)
+#define CSB_CC_HYP_NO_HW (254)
+#define CSB_CC_HYP_HANG_ABORTED (255)
+
+/* CCB Completion Modes (CM) for 842
+ * NX P8 workbook, section 4.3, figure 4-5
+ * "CRB Details - Normal Cop_Req (CL=00, C=1)"
+ */
+#define CCB_CM_EXTRA_WRITE (CCB_CM0_ALL_COMPLETIONS & CCB_CM12_STORE)
+#define CCB_CM_INTERRUPT (CCB_CM0_ALL_COMPLETIONS & CCB_CM12_INTERRUPT)
+
+#define LEN_ON_SIZE(pa, size) ((size) - ((pa) & ((size) - 1)))
+#define LEN_ON_PAGE(pa) LEN_ON_SIZE(pa, PAGE_SIZE)
+
+static inline unsigned long nx842_get_pa(void *addr)
+{
+ if (!is_vmalloc_addr(addr))
+ return __pa(addr);
+
+ return page_to_phys(vmalloc_to_page(addr)) + offset_in_page(addr);
+}
+
+/* Get/Set bit fields */
+#define MASK_LSH(m) (__builtin_ffsl(m) - 1)
+#define GET_FIELD(v, m) (((v) & (m)) >> MASK_LSH(m))
+#define SET_FIELD(v, m, val) (((v) & ~(m)) | (((val) << MASK_LSH(m)) & (m)))
+
+struct nx842_constraints {
+ int alignment;
+ int multiple;
+ int minimum;
+ int maximum;
+};
+
+struct nx842_driver {
+ char *name;
+ struct module *owner;
+ size_t workmem_size;
+
+ struct nx842_constraints *constraints;
+
+ int (*compress)(const unsigned char *in, unsigned int in_len,
+ unsigned char *out, unsigned int *out_len,
+ void *wrkmem);
+ int (*decompress)(const unsigned char *in, unsigned int in_len,
+ unsigned char *out, unsigned int *out_len,
+ void *wrkmem);
+};
+
+struct nx842_driver *nx842_platform_driver(void);
+bool nx842_platform_driver_set(struct nx842_driver *driver);
+void nx842_platform_driver_unset(struct nx842_driver *driver);
+bool nx842_platform_driver_get(void);
+void nx842_platform_driver_put(void);
+
+size_t nx842_workmem_size(void);
+
+int nx842_constraints(struct nx842_constraints *constraints);
+
+int nx842_compress(const unsigned char *in, unsigned int in_len,
+ unsigned char *out, unsigned int *out_len, void *wrkmem);
+int nx842_decompress(const unsigned char *in, unsigned int in_len,
+ unsigned char *out, unsigned int *out_len, void *wrkmem);
+
+#endif /* __NX_842_H__ */
diff --git a/drivers/crypto/nx/nx-aes-gcm.c b/drivers/crypto/nx/nx-aes-gcm.c
index c6ebeb644..92c993f08 100644
--- a/drivers/crypto/nx/nx-aes-gcm.c
+++ b/drivers/crypto/nx/nx-aes-gcm.c
@@ -93,17 +93,6 @@ out:
return rc;
}
-static int gcm_aes_nx_setauthsize(struct crypto_aead *tfm,
- unsigned int authsize)
-{
- if (authsize > crypto_aead_alg(tfm)->maxauthsize)
- return -EINVAL;
-
- crypto_aead_crt(tfm)->authsize = authsize;
-
- return 0;
-}
-
static int gcm4106_aes_nx_setauthsize(struct crypto_aead *tfm,
unsigned int authsize)
{
@@ -116,8 +105,6 @@ static int gcm4106_aes_nx_setauthsize(struct crypto_aead *tfm,
return -EINVAL;
}
- crypto_aead_crt(tfm)->authsize = authsize;
-
return 0;
}
@@ -134,7 +121,7 @@ static int nx_gca(struct nx_crypto_ctx *nx_ctx,
unsigned int max_sg_len;
if (nbytes <= AES_BLOCK_SIZE) {
- scatterwalk_start(&walk, req->assoc);
+ scatterwalk_start(&walk, req->src);
scatterwalk_copychunks(out, &walk, nbytes, SCATTERWALK_FROM_SG);
scatterwalk_done(&walk, SCATTERWALK_FROM_SG, 0);
return 0;
@@ -159,7 +146,7 @@ static int nx_gca(struct nx_crypto_ctx *nx_ctx,
NX_PAGE_SIZE * (max_sg_len - 1));
nx_sg = nx_walk_and_build(nx_ctx->in_sg, max_sg_len,
- req->assoc, processed, &to_process);
+ req->src, processed, &to_process);
if ((to_process + processed) < nbytes)
NX_CPB_FDM(csbcpb_aead) |= NX_FDM_INTERMEDIATE;
@@ -225,7 +212,7 @@ static int gmac(struct aead_request *req, struct blkcipher_desc *desc)
NX_PAGE_SIZE * (max_sg_len - 1));
nx_sg = nx_walk_and_build(nx_ctx->in_sg, max_sg_len,
- req->assoc, processed, &to_process);
+ req->src, processed, &to_process);
if ((to_process + processed) < nbytes)
NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
@@ -378,7 +365,8 @@ static int gcm_aes_nx_crypt(struct aead_request *req, int enc)
csbcpb->cpb.aes_gcm.bit_length_data = nbytes * 8;
desc.tfm = (struct crypto_blkcipher *) req->base.tfm;
rc = nx_build_sg_lists(nx_ctx, &desc, req->dst,
- req->src, &to_process, processed,
+ req->src, &to_process,
+ processed + req->assoclen,
csbcpb->cpb.aes_gcm.iv_or_cnt);
if (rc)
@@ -413,17 +401,19 @@ static int gcm_aes_nx_crypt(struct aead_request *req, int enc)
mac:
if (enc) {
/* copy out the auth tag */
- scatterwalk_map_and_copy(csbcpb->cpb.aes_gcm.out_pat_or_mac,
- req->dst, nbytes,
- crypto_aead_authsize(crypto_aead_reqtfm(req)),
- SCATTERWALK_TO_SG);
+ scatterwalk_map_and_copy(
+ csbcpb->cpb.aes_gcm.out_pat_or_mac,
+ req->dst, req->assoclen + nbytes,
+ crypto_aead_authsize(crypto_aead_reqtfm(req)),
+ SCATTERWALK_TO_SG);
} else {
u8 *itag = nx_ctx->priv.gcm.iauth_tag;
u8 *otag = csbcpb->cpb.aes_gcm.out_pat_or_mac;
- scatterwalk_map_and_copy(itag, req->src, nbytes,
- crypto_aead_authsize(crypto_aead_reqtfm(req)),
- SCATTERWALK_FROM_SG);
+ scatterwalk_map_and_copy(
+ itag, req->src, req->assoclen + nbytes,
+ crypto_aead_authsize(crypto_aead_reqtfm(req)),
+ SCATTERWALK_FROM_SG);
rc = memcmp(itag, otag,
crypto_aead_authsize(crypto_aead_reqtfm(req))) ?
-EBADMSG : 0;
@@ -484,45 +474,39 @@ static int gcm4106_aes_nx_decrypt(struct aead_request *req)
* during encrypt/decrypt doesn't solve this problem, because it calls
* blkcipher_walk_done under the covers, which doesn't use walk->blocksize,
* but instead uses this tfm->blocksize. */
-struct crypto_alg nx_gcm_aes_alg = {
- .cra_name = "gcm(aes)",
- .cra_driver_name = "gcm-aes-nx",
- .cra_priority = 300,
- .cra_flags = CRYPTO_ALG_TYPE_AEAD,
- .cra_blocksize = 1,
- .cra_ctxsize = sizeof(struct nx_crypto_ctx),
- .cra_type = &crypto_aead_type,
- .cra_module = THIS_MODULE,
- .cra_init = nx_crypto_ctx_aes_gcm_init,
- .cra_exit = nx_crypto_ctx_exit,
- .cra_aead = {
- .ivsize = AES_BLOCK_SIZE,
- .maxauthsize = AES_BLOCK_SIZE,
- .setkey = gcm_aes_nx_set_key,
- .setauthsize = gcm_aes_nx_setauthsize,
- .encrypt = gcm_aes_nx_encrypt,
- .decrypt = gcm_aes_nx_decrypt,
- }
+struct aead_alg nx_gcm_aes_alg = {
+ .base = {
+ .cra_name = "gcm(aes)",
+ .cra_driver_name = "gcm-aes-nx",
+ .cra_priority = 300,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct nx_crypto_ctx),
+ .cra_module = THIS_MODULE,
+ },
+ .init = nx_crypto_ctx_aes_gcm_init,
+ .exit = nx_crypto_ctx_aead_exit,
+ .ivsize = 12,
+ .maxauthsize = AES_BLOCK_SIZE,
+ .setkey = gcm_aes_nx_set_key,
+ .encrypt = gcm_aes_nx_encrypt,
+ .decrypt = gcm_aes_nx_decrypt,
};
-struct crypto_alg nx_gcm4106_aes_alg = {
- .cra_name = "rfc4106(gcm(aes))",
- .cra_driver_name = "rfc4106-gcm-aes-nx",
- .cra_priority = 300,
- .cra_flags = CRYPTO_ALG_TYPE_AEAD,
- .cra_blocksize = 1,
- .cra_ctxsize = sizeof(struct nx_crypto_ctx),
- .cra_type = &crypto_nivaead_type,
- .cra_module = THIS_MODULE,
- .cra_init = nx_crypto_ctx_aes_gcm_init,
- .cra_exit = nx_crypto_ctx_exit,
- .cra_aead = {
- .ivsize = 8,
- .maxauthsize = AES_BLOCK_SIZE,
- .geniv = "seqiv",
- .setkey = gcm4106_aes_nx_set_key,
- .setauthsize = gcm4106_aes_nx_setauthsize,
- .encrypt = gcm4106_aes_nx_encrypt,
- .decrypt = gcm4106_aes_nx_decrypt,
- }
+struct aead_alg nx_gcm4106_aes_alg = {
+ .base = {
+ .cra_name = "rfc4106(gcm(aes))",
+ .cra_driver_name = "rfc4106-gcm-aes-nx",
+ .cra_priority = 300,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct nx_crypto_ctx),
+ .cra_module = THIS_MODULE,
+ },
+ .init = nx_crypto_ctx_aes_gcm_init,
+ .exit = nx_crypto_ctx_aead_exit,
+ .ivsize = 8,
+ .maxauthsize = AES_BLOCK_SIZE,
+ .setkey = gcm4106_aes_nx_set_key,
+ .setauthsize = gcm4106_aes_nx_setauthsize,
+ .encrypt = gcm4106_aes_nx_encrypt,
+ .decrypt = gcm4106_aes_nx_decrypt,
};
diff --git a/drivers/crypto/nx/nx-sha256.c b/drivers/crypto/nx/nx-sha256.c
index 08f8d5cd6..becb738c8 100644
--- a/drivers/crypto/nx/nx-sha256.c
+++ b/drivers/crypto/nx/nx-sha256.c
@@ -71,7 +71,6 @@ static int nx_sha256_update(struct shash_desc *desc, const u8 *data,
struct sha256_state *sctx = shash_desc_ctx(desc);
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
- struct nx_sg *in_sg;
struct nx_sg *out_sg;
u64 to_process = 0, leftover, total;
unsigned long irq_flags;
@@ -97,7 +96,6 @@ static int nx_sha256_update(struct shash_desc *desc, const u8 *data,
NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
- in_sg = nx_ctx->in_sg;
max_sg_len = min_t(u64, nx_ctx->ap->sglen,
nx_driver.of.max_sg_len/sizeof(struct nx_sg));
max_sg_len = min_t(u64, max_sg_len,
@@ -114,17 +112,12 @@ static int nx_sha256_update(struct shash_desc *desc, const u8 *data,
}
do {
- /*
- * to_process: the SHA256_BLOCK_SIZE data chunk to process in
- * this update. This value is also restricted by the sg list
- * limits.
- */
- to_process = total - to_process;
- to_process = to_process & ~(SHA256_BLOCK_SIZE - 1);
+ int used_sgs = 0;
+ struct nx_sg *in_sg = nx_ctx->in_sg;
if (buf_len) {
data_len = buf_len;
- in_sg = nx_build_sg_list(nx_ctx->in_sg,
+ in_sg = nx_build_sg_list(in_sg,
(u8 *) sctx->buf,
&data_len,
max_sg_len);
@@ -133,15 +126,27 @@ static int nx_sha256_update(struct shash_desc *desc, const u8 *data,
rc = -EINVAL;
goto out;
}
+ used_sgs = in_sg - nx_ctx->in_sg;
}
+ /* to_process: SHA256_BLOCK_SIZE aligned chunk to be
+ * processed in this iteration. This value is restricted
+ * by sg list limits and number of sgs we already used
+ * for leftover data. (see above)
+ * In ideal case, we could allow NX_PAGE_SIZE * max_sg_len,
+ * but because data may not be aligned, we need to account
+ * for that too. */
+ to_process = min_t(u64, total,
+ (max_sg_len - 1 - used_sgs) * NX_PAGE_SIZE);
+ to_process = to_process & ~(SHA256_BLOCK_SIZE - 1);
+
data_len = to_process - buf_len;
in_sg = nx_build_sg_list(in_sg, (u8 *) data,
&data_len, max_sg_len);
nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg);
- to_process = (data_len + buf_len);
+ to_process = data_len + buf_len;
leftover = total - to_process;
/*
diff --git a/drivers/crypto/nx/nx-sha512.c b/drivers/crypto/nx/nx-sha512.c
index aff0fe58e..b6e183d58 100644
--- a/drivers/crypto/nx/nx-sha512.c
+++ b/drivers/crypto/nx/nx-sha512.c
@@ -71,7 +71,6 @@ static int nx_sha512_update(struct shash_desc *desc, const u8 *data,
struct sha512_state *sctx = shash_desc_ctx(desc);
struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
- struct nx_sg *in_sg;
struct nx_sg *out_sg;
u64 to_process, leftover = 0, total;
unsigned long irq_flags;
@@ -97,7 +96,6 @@ static int nx_sha512_update(struct shash_desc *desc, const u8 *data,
NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
- in_sg = nx_ctx->in_sg;
max_sg_len = min_t(u64, nx_ctx->ap->sglen,
nx_driver.of.max_sg_len/sizeof(struct nx_sg));
max_sg_len = min_t(u64, max_sg_len,
@@ -114,18 +112,12 @@ static int nx_sha512_update(struct shash_desc *desc, const u8 *data,
}
do {
- /*
- * to_process: the SHA512_BLOCK_SIZE data chunk to process in
- * this update. This value is also restricted by the sg list
- * limits.
- */
- to_process = total - leftover;
- to_process = to_process & ~(SHA512_BLOCK_SIZE - 1);
- leftover = total - to_process;
+ int used_sgs = 0;
+ struct nx_sg *in_sg = nx_ctx->in_sg;
if (buf_len) {
data_len = buf_len;
- in_sg = nx_build_sg_list(nx_ctx->in_sg,
+ in_sg = nx_build_sg_list(in_sg,
(u8 *) sctx->buf,
&data_len, max_sg_len);
@@ -133,8 +125,20 @@ static int nx_sha512_update(struct shash_desc *desc, const u8 *data,
rc = -EINVAL;
goto out;
}
+ used_sgs = in_sg - nx_ctx->in_sg;
}
+ /* to_process: SHA512_BLOCK_SIZE aligned chunk to be
+ * processed in this iteration. This value is restricted
+ * by sg list limits and number of sgs we already used
+ * for leftover data. (see above)
+ * In ideal case, we could allow NX_PAGE_SIZE * max_sg_len,
+ * but because data may not be aligned, we need to account
+ * for that too. */
+ to_process = min_t(u64, total,
+ (max_sg_len - 1 - used_sgs) * NX_PAGE_SIZE);
+ to_process = to_process & ~(SHA512_BLOCK_SIZE - 1);
+
data_len = to_process - buf_len;
in_sg = nx_build_sg_list(in_sg, (u8 *) data,
&data_len, max_sg_len);
@@ -146,7 +150,7 @@ static int nx_sha512_update(struct shash_desc *desc, const u8 *data,
goto out;
}
- to_process = (data_len + buf_len);
+ to_process = data_len + buf_len;
leftover = total - to_process;
/*
diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c
index 737d33dc5..436971343 100644
--- a/drivers/crypto/nx/nx.c
+++ b/drivers/crypto/nx/nx.c
@@ -19,8 +19,8 @@
* Author: Kent Yoder <yoder1@us.ibm.com>
*/
+#include <crypto/internal/aead.h>
#include <crypto/internal/hash.h>
-#include <crypto/hash.h>
#include <crypto/aes.h>
#include <crypto/sha.h>
#include <crypto/algapi.h>
@@ -29,10 +29,10 @@
#include <linux/moduleparam.h>
#include <linux/types.h>
#include <linux/mm.h>
-#include <linux/crypto.h>
#include <linux/scatterlist.h>
#include <linux/device.h>
#include <linux/of.h>
+#include <linux/types.h>
#include <asm/hvcall.h>
#include <asm/vio.h>
@@ -399,6 +399,13 @@ static void nx_of_update_msc(struct device *dev,
goto next_loop;
}
+ if (!trip->sglen || trip->databytelen < NX_PAGE_SIZE) {
+ dev_warn(dev, "bogus sglen/databytelen: "
+ "%u/%u (ignored)\n", trip->sglen,
+ trip->databytelen);
+ goto next_loop;
+ }
+
switch (trip->keybitlen) {
case 128:
case 160:
@@ -491,6 +498,72 @@ static void nx_of_init(struct device *dev, struct nx_of *props)
nx_of_update_msc(dev, p, props);
}
+static bool nx_check_prop(struct device *dev, u32 fc, u32 mode, int slot)
+{
+ struct alg_props *props = &nx_driver.of.ap[fc][mode][slot];
+
+ if (!props->sglen || props->databytelen < NX_PAGE_SIZE) {
+ if (dev)
+ dev_warn(dev, "bogus sglen/databytelen for %u/%u/%u: "
+ "%u/%u (ignored)\n", fc, mode, slot,
+ props->sglen, props->databytelen);
+ return false;
+ }
+
+ return true;
+}
+
+static bool nx_check_props(struct device *dev, u32 fc, u32 mode)
+{
+ int i;
+
+ for (i = 0; i < 3; i++)
+ if (!nx_check_prop(dev, fc, mode, i))
+ return false;
+
+ return true;
+}
+
+static int nx_register_alg(struct crypto_alg *alg, u32 fc, u32 mode)
+{
+ return nx_check_props(&nx_driver.viodev->dev, fc, mode) ?
+ crypto_register_alg(alg) : 0;
+}
+
+static int nx_register_aead(struct aead_alg *alg, u32 fc, u32 mode)
+{
+ return nx_check_props(&nx_driver.viodev->dev, fc, mode) ?
+ crypto_register_aead(alg) : 0;
+}
+
+static int nx_register_shash(struct shash_alg *alg, u32 fc, u32 mode, int slot)
+{
+ return (slot >= 0 ? nx_check_prop(&nx_driver.viodev->dev,
+ fc, mode, slot) :
+ nx_check_props(&nx_driver.viodev->dev, fc, mode)) ?
+ crypto_register_shash(alg) : 0;
+}
+
+static void nx_unregister_alg(struct crypto_alg *alg, u32 fc, u32 mode)
+{
+ if (nx_check_props(NULL, fc, mode))
+ crypto_unregister_alg(alg);
+}
+
+static void nx_unregister_aead(struct aead_alg *alg, u32 fc, u32 mode)
+{
+ if (nx_check_props(NULL, fc, mode))
+ crypto_unregister_aead(alg);
+}
+
+static void nx_unregister_shash(struct shash_alg *alg, u32 fc, u32 mode,
+ int slot)
+{
+ if (slot >= 0 ? nx_check_prop(NULL, fc, mode, slot) :
+ nx_check_props(NULL, fc, mode))
+ crypto_unregister_shash(alg);
+}
+
/**
* nx_register_algs - register algorithms with the crypto API
*
@@ -515,72 +588,77 @@ static int nx_register_algs(void)
nx_driver.of.status = NX_OKAY;
- rc = crypto_register_alg(&nx_ecb_aes_alg);
+ rc = nx_register_alg(&nx_ecb_aes_alg, NX_FC_AES, NX_MODE_AES_ECB);
if (rc)
goto out;
- rc = crypto_register_alg(&nx_cbc_aes_alg);
+ rc = nx_register_alg(&nx_cbc_aes_alg, NX_FC_AES, NX_MODE_AES_CBC);
if (rc)
goto out_unreg_ecb;
- rc = crypto_register_alg(&nx_ctr_aes_alg);
+ rc = nx_register_alg(&nx_ctr_aes_alg, NX_FC_AES, NX_MODE_AES_CTR);
if (rc)
goto out_unreg_cbc;
- rc = crypto_register_alg(&nx_ctr3686_aes_alg);
+ rc = nx_register_alg(&nx_ctr3686_aes_alg, NX_FC_AES, NX_MODE_AES_CTR);
if (rc)
goto out_unreg_ctr;
- rc = crypto_register_alg(&nx_gcm_aes_alg);
+ rc = nx_register_aead(&nx_gcm_aes_alg, NX_FC_AES, NX_MODE_AES_GCM);
if (rc)
goto out_unreg_ctr3686;
- rc = crypto_register_alg(&nx_gcm4106_aes_alg);
+ rc = nx_register_aead(&nx_gcm4106_aes_alg, NX_FC_AES, NX_MODE_AES_GCM);
if (rc)
goto out_unreg_gcm;
- rc = crypto_register_alg(&nx_ccm_aes_alg);
+ rc = nx_register_alg(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM);
if (rc)
goto out_unreg_gcm4106;
- rc = crypto_register_alg(&nx_ccm4309_aes_alg);
+ rc = nx_register_alg(&nx_ccm4309_aes_alg, NX_FC_AES, NX_MODE_AES_CCM);
if (rc)
goto out_unreg_ccm;
- rc = crypto_register_shash(&nx_shash_sha256_alg);
+ rc = nx_register_shash(&nx_shash_sha256_alg, NX_FC_SHA, NX_MODE_SHA,
+ NX_PROPS_SHA256);
if (rc)
goto out_unreg_ccm4309;
- rc = crypto_register_shash(&nx_shash_sha512_alg);
+ rc = nx_register_shash(&nx_shash_sha512_alg, NX_FC_SHA, NX_MODE_SHA,
+ NX_PROPS_SHA512);
if (rc)
goto out_unreg_s256;
- rc = crypto_register_shash(&nx_shash_aes_xcbc_alg);
+ rc = nx_register_shash(&nx_shash_aes_xcbc_alg,
+ NX_FC_AES, NX_MODE_AES_XCBC_MAC, -1);
if (rc)
goto out_unreg_s512;
goto out;
out_unreg_s512:
- crypto_unregister_shash(&nx_shash_sha512_alg);
+ nx_unregister_shash(&nx_shash_sha512_alg, NX_FC_SHA, NX_MODE_SHA,
+ NX_PROPS_SHA512);
out_unreg_s256:
- crypto_unregister_shash(&nx_shash_sha256_alg);
+ nx_unregister_shash(&nx_shash_sha256_alg, NX_FC_SHA, NX_MODE_SHA,
+ NX_PROPS_SHA256);
out_unreg_ccm4309:
- crypto_unregister_alg(&nx_ccm4309_aes_alg);
+ nx_unregister_alg(&nx_ccm4309_aes_alg, NX_FC_AES, NX_MODE_AES_CCM);
out_unreg_ccm:
- crypto_unregister_alg(&nx_ccm_aes_alg);
+ nx_unregister_alg(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM);
out_unreg_gcm4106:
- crypto_unregister_alg(&nx_gcm4106_aes_alg);
+ nx_unregister_aead(&nx_gcm4106_aes_alg, NX_FC_AES, NX_MODE_AES_GCM);
out_unreg_gcm:
- crypto_unregister_alg(&nx_gcm_aes_alg);
+ nx_unregister_aead(&nx_gcm_aes_alg, NX_FC_AES, NX_MODE_AES_GCM);
out_unreg_ctr3686:
- crypto_unregister_alg(&nx_ctr3686_aes_alg);
+ nx_unregister_alg(&nx_ctr3686_aes_alg, NX_FC_AES, NX_MODE_AES_CTR);
out_unreg_ctr:
- crypto_unregister_alg(&nx_ctr_aes_alg);
+ nx_unregister_alg(&nx_ctr_aes_alg, NX_FC_AES, NX_MODE_AES_CTR);
out_unreg_cbc:
- crypto_unregister_alg(&nx_cbc_aes_alg);
+ nx_unregister_alg(&nx_cbc_aes_alg, NX_FC_AES, NX_MODE_AES_CBC);
out_unreg_ecb:
- crypto_unregister_alg(&nx_ecb_aes_alg);
+ nx_unregister_alg(&nx_ecb_aes_alg, NX_FC_AES, NX_MODE_AES_ECB);
out:
return rc;
}
@@ -635,15 +713,16 @@ static int nx_crypto_ctx_init(struct nx_crypto_ctx *nx_ctx, u32 fc, u32 mode)
/* entry points from the crypto tfm initializers */
int nx_crypto_ctx_aes_ccm_init(struct crypto_tfm *tfm)
{
- tfm->crt_aead.reqsize = sizeof(struct nx_ccm_rctx);
+ crypto_aead_set_reqsize(__crypto_aead_cast(tfm),
+ sizeof(struct nx_ccm_rctx));
return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_AES,
NX_MODE_AES_CCM);
}
-int nx_crypto_ctx_aes_gcm_init(struct crypto_tfm *tfm)
+int nx_crypto_ctx_aes_gcm_init(struct crypto_aead *tfm)
{
- tfm->crt_aead.reqsize = sizeof(struct nx_gcm_rctx);
- return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_AES,
+ crypto_aead_set_reqsize(tfm, sizeof(struct nx_gcm_rctx));
+ return nx_crypto_ctx_init(crypto_aead_ctx(tfm), NX_FC_AES,
NX_MODE_AES_GCM);
}
@@ -695,6 +774,13 @@ void nx_crypto_ctx_exit(struct crypto_tfm *tfm)
nx_ctx->out_sg = NULL;
}
+void nx_crypto_ctx_aead_exit(struct crypto_aead *tfm)
+{
+ struct nx_crypto_ctx *nx_ctx = crypto_aead_ctx(tfm);
+
+ kzfree(nx_ctx->kmem);
+}
+
static int nx_probe(struct vio_dev *viodev, const struct vio_device_id *id)
{
dev_dbg(&viodev->dev, "driver probed: %s resource id: 0x%x\n",
@@ -721,17 +807,24 @@ static int nx_remove(struct vio_dev *viodev)
if (nx_driver.of.status == NX_OKAY) {
NX_DEBUGFS_FINI(&nx_driver);
- crypto_unregister_alg(&nx_ccm_aes_alg);
- crypto_unregister_alg(&nx_ccm4309_aes_alg);
- crypto_unregister_alg(&nx_gcm_aes_alg);
- crypto_unregister_alg(&nx_gcm4106_aes_alg);
- crypto_unregister_alg(&nx_ctr_aes_alg);
- crypto_unregister_alg(&nx_ctr3686_aes_alg);
- crypto_unregister_alg(&nx_cbc_aes_alg);
- crypto_unregister_alg(&nx_ecb_aes_alg);
- crypto_unregister_shash(&nx_shash_sha256_alg);
- crypto_unregister_shash(&nx_shash_sha512_alg);
- crypto_unregister_shash(&nx_shash_aes_xcbc_alg);
+ nx_unregister_shash(&nx_shash_aes_xcbc_alg,
+ NX_FC_AES, NX_MODE_AES_XCBC_MAC, -1);
+ nx_unregister_shash(&nx_shash_sha512_alg,
+ NX_FC_SHA, NX_MODE_SHA, NX_PROPS_SHA256);
+ nx_unregister_shash(&nx_shash_sha256_alg,
+ NX_FC_SHA, NX_MODE_SHA, NX_PROPS_SHA512);
+ nx_unregister_alg(&nx_ccm4309_aes_alg,
+ NX_FC_AES, NX_MODE_AES_CCM);
+ nx_unregister_alg(&nx_ccm_aes_alg, NX_FC_AES, NX_MODE_AES_CCM);
+ nx_unregister_aead(&nx_gcm4106_aes_alg,
+ NX_FC_AES, NX_MODE_AES_GCM);
+ nx_unregister_aead(&nx_gcm_aes_alg,
+ NX_FC_AES, NX_MODE_AES_GCM);
+ nx_unregister_alg(&nx_ctr3686_aes_alg,
+ NX_FC_AES, NX_MODE_AES_CTR);
+ nx_unregister_alg(&nx_ctr_aes_alg, NX_FC_AES, NX_MODE_AES_CTR);
+ nx_unregister_alg(&nx_cbc_aes_alg, NX_FC_AES, NX_MODE_AES_CBC);
+ nx_unregister_alg(&nx_ecb_aes_alg, NX_FC_AES, NX_MODE_AES_ECB);
}
return 0;
diff --git a/drivers/crypto/nx/nx.h b/drivers/crypto/nx/nx.h
index c3ed83764..cdff03a42 100644
--- a/drivers/crypto/nx/nx.h
+++ b/drivers/crypto/nx/nx.h
@@ -151,13 +151,14 @@ struct nx_crypto_ctx {
/* prototypes */
int nx_crypto_ctx_aes_ccm_init(struct crypto_tfm *tfm);
-int nx_crypto_ctx_aes_gcm_init(struct crypto_tfm *tfm);
+int nx_crypto_ctx_aes_gcm_init(struct crypto_aead *tfm);
int nx_crypto_ctx_aes_xcbc_init(struct crypto_tfm *tfm);
int nx_crypto_ctx_aes_ctr_init(struct crypto_tfm *tfm);
int nx_crypto_ctx_aes_cbc_init(struct crypto_tfm *tfm);
int nx_crypto_ctx_aes_ecb_init(struct crypto_tfm *tfm);
int nx_crypto_ctx_sha_init(struct crypto_tfm *tfm);
void nx_crypto_ctx_exit(struct crypto_tfm *tfm);
+void nx_crypto_ctx_aead_exit(struct crypto_aead *tfm);
void nx_ctx_init(struct nx_crypto_ctx *nx_ctx, unsigned int function);
int nx_hcall_sync(struct nx_crypto_ctx *ctx, struct vio_pfo_op *op,
u32 may_sleep);
@@ -184,8 +185,8 @@ void nx_debugfs_fini(struct nx_crypto_driver *);
extern struct crypto_alg nx_cbc_aes_alg;
extern struct crypto_alg nx_ecb_aes_alg;
-extern struct crypto_alg nx_gcm_aes_alg;
-extern struct crypto_alg nx_gcm4106_aes_alg;
+extern struct aead_alg nx_gcm_aes_alg;
+extern struct aead_alg nx_gcm4106_aes_alg;
extern struct crypto_alg nx_ctr_aes_alg;
extern struct crypto_alg nx_ctr3686_aes_alg;
extern struct crypto_alg nx_ccm_aes_alg;