diff options
Diffstat (limited to 'drivers/crypto')
41 files changed, 1847 insertions, 1055 deletions
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 07d494276..477fffdb4 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -296,6 +296,7 @@ config CRYPTO_DEV_OMAP_AES depends on ARCH_OMAP2 || ARCH_OMAP3 || ARCH_OMAP2PLUS select CRYPTO_AES select CRYPTO_BLKCIPHER + select CRYPTO_ENGINE help OMAP processors have AES module accelerator. Select this if you want to use the OMAP module for AES algorithms. @@ -487,7 +488,7 @@ config CRYPTO_DEV_IMGTEC_HASH config CRYPTO_DEV_SUN4I_SS tristate "Support for Allwinner Security System cryptographic accelerator" - depends on ARCH_SUNXI + depends on ARCH_SUNXI && !64BIT select CRYPTO_MD5 select CRYPTO_SHA1 select CRYPTO_AES @@ -507,6 +508,10 @@ config CRYPTO_DEV_ROCKCHIP depends on OF && ARCH_ROCKCHIP select CRYPTO_AES select CRYPTO_DES + select CRYPTO_MD5 + select CRYPTO_SHA1 + select CRYPTO_SHA256 + select CRYPTO_HASH select CRYPTO_BLKCIPHER help diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index 7de007abe..e3d40a8df 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -369,12 +369,6 @@ static inline size_t atmel_aes_padlen(size_t len, size_t block_size) return len ? block_size - len : 0; } -static inline struct aead_request * -aead_request_cast(struct crypto_async_request *req) -{ - return container_of(req, struct aead_request, base); -} - static struct atmel_aes_dev *atmel_aes_find_dev(struct atmel_aes_base_ctx *ctx) { struct atmel_aes_dev *aes_dd = NULL; diff --git a/drivers/crypto/atmel-sha-regs.h b/drivers/crypto/atmel-sha-regs.h index 83b2d7425..e08897109 100644 --- a/drivers/crypto/atmel-sha-regs.h +++ b/drivers/crypto/atmel-sha-regs.h @@ -8,6 +8,8 @@ #define SHA_CR_START (1 << 0) #define SHA_CR_FIRST (1 << 4) #define SHA_CR_SWRST (1 << 8) +#define SHA_CR_WUIHV (1 << 12) +#define SHA_CR_WUIEHV (1 << 13) #define SHA_MR 0x04 #define SHA_MR_MODE_MASK (0x3 << 0) @@ -15,6 +17,8 @@ #define SHA_MR_MODE_AUTO 0x1 #define SHA_MR_MODE_PDC 0x2 #define SHA_MR_PROCDLY (1 << 4) +#define SHA_MR_UIHV (1 << 5) +#define SHA_MR_UIEHV (1 << 6) #define SHA_MR_ALGO_SHA1 (0 << 8) #define SHA_MR_ALGO_SHA256 (1 << 8) #define SHA_MR_ALGO_SHA384 (2 << 8) diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index 68d47a2da..97e34799e 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -53,6 +53,7 @@ #define SHA_FLAGS_FINUP BIT(16) #define SHA_FLAGS_SG BIT(17) +#define SHA_FLAGS_ALGO_MASK GENMASK(22, 18) #define SHA_FLAGS_SHA1 BIT(18) #define SHA_FLAGS_SHA224 BIT(19) #define SHA_FLAGS_SHA256 BIT(20) @@ -60,11 +61,12 @@ #define SHA_FLAGS_SHA512 BIT(22) #define SHA_FLAGS_ERROR BIT(23) #define SHA_FLAGS_PAD BIT(24) +#define SHA_FLAGS_RESTORE BIT(25) #define SHA_OP_UPDATE 1 #define SHA_OP_FINAL 2 -#define SHA_BUFFER_LEN PAGE_SIZE +#define SHA_BUFFER_LEN (PAGE_SIZE / 16) #define ATMEL_SHA_DMA_THRESHOLD 56 @@ -73,10 +75,15 @@ struct atmel_sha_caps { bool has_dualbuff; bool has_sha224; bool has_sha_384_512; + bool has_uihv; }; struct atmel_sha_dev; +/* + * .statesize = sizeof(struct atmel_sha_reqctx) must be <= PAGE_SIZE / 8 as + * tested by the ahash_prepare_alg() function. + */ struct atmel_sha_reqctx { struct atmel_sha_dev *dd; unsigned long flags; @@ -95,7 +102,7 @@ struct atmel_sha_reqctx { size_t block_size; - u8 buffer[0] __aligned(sizeof(u32)); + u8 buffer[SHA_BUFFER_LEN + SHA512_BLOCK_SIZE] __aligned(sizeof(u32)); }; struct atmel_sha_ctx { @@ -122,6 +129,7 @@ struct atmel_sha_dev { spinlock_t lock; int err; struct tasklet_struct done_task; + struct tasklet_struct queue_task; unsigned long flags; struct crypto_queue queue; @@ -317,7 +325,8 @@ static int atmel_sha_init(struct ahash_request *req) static void atmel_sha_write_ctrl(struct atmel_sha_dev *dd, int dma) { struct atmel_sha_reqctx *ctx = ahash_request_ctx(dd->req); - u32 valcr = 0, valmr = SHA_MR_MODE_AUTO; + u32 valmr = SHA_MR_MODE_AUTO; + unsigned int i, hashsize = 0; if (likely(dma)) { if (!dd->caps.has_dma) @@ -329,22 +338,62 @@ static void atmel_sha_write_ctrl(struct atmel_sha_dev *dd, int dma) atmel_sha_write(dd, SHA_IER, SHA_INT_DATARDY); } - if (ctx->flags & SHA_FLAGS_SHA1) + switch (ctx->flags & SHA_FLAGS_ALGO_MASK) { + case SHA_FLAGS_SHA1: valmr |= SHA_MR_ALGO_SHA1; - else if (ctx->flags & SHA_FLAGS_SHA224) + hashsize = SHA1_DIGEST_SIZE; + break; + + case SHA_FLAGS_SHA224: valmr |= SHA_MR_ALGO_SHA224; - else if (ctx->flags & SHA_FLAGS_SHA256) + hashsize = SHA256_DIGEST_SIZE; + break; + + case SHA_FLAGS_SHA256: valmr |= SHA_MR_ALGO_SHA256; - else if (ctx->flags & SHA_FLAGS_SHA384) + hashsize = SHA256_DIGEST_SIZE; + break; + + case SHA_FLAGS_SHA384: valmr |= SHA_MR_ALGO_SHA384; - else if (ctx->flags & SHA_FLAGS_SHA512) + hashsize = SHA512_DIGEST_SIZE; + break; + + case SHA_FLAGS_SHA512: valmr |= SHA_MR_ALGO_SHA512; + hashsize = SHA512_DIGEST_SIZE; + break; + + default: + break; + } /* Setting CR_FIRST only for the first iteration */ - if (!(ctx->digcnt[0] || ctx->digcnt[1])) - valcr = SHA_CR_FIRST; + if (!(ctx->digcnt[0] || ctx->digcnt[1])) { + atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST); + } else if (dd->caps.has_uihv && (ctx->flags & SHA_FLAGS_RESTORE)) { + const u32 *hash = (const u32 *)ctx->digest; + + /* + * Restore the hardware context: update the User Initialize + * Hash Value (UIHV) with the value saved when the latest + * 'update' operation completed on this very same crypto + * request. + */ + ctx->flags &= ~SHA_FLAGS_RESTORE; + atmel_sha_write(dd, SHA_CR, SHA_CR_WUIHV); + for (i = 0; i < hashsize / sizeof(u32); ++i) + atmel_sha_write(dd, SHA_REG_DIN(i), hash[i]); + atmel_sha_write(dd, SHA_CR, SHA_CR_FIRST); + valmr |= SHA_MR_UIHV; + } + /* + * WARNING: If the UIHV feature is not available, the hardware CANNOT + * process concurrent requests: the internal registers used to store + * the hash/digest are still set to the partial digest output values + * computed during the latest round. + */ - atmel_sha_write(dd, SHA_CR, valcr); atmel_sha_write(dd, SHA_MR, valmr); } @@ -713,23 +762,31 @@ static void atmel_sha_copy_hash(struct ahash_request *req) { struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); u32 *hash = (u32 *)ctx->digest; - int i; + unsigned int i, hashsize; - if (ctx->flags & SHA_FLAGS_SHA1) - for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) - hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); - else if (ctx->flags & SHA_FLAGS_SHA224) - for (i = 0; i < SHA224_DIGEST_SIZE / sizeof(u32); i++) - hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); - else if (ctx->flags & SHA_FLAGS_SHA256) - for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(u32); i++) - hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); - else if (ctx->flags & SHA_FLAGS_SHA384) - for (i = 0; i < SHA384_DIGEST_SIZE / sizeof(u32); i++) - hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); - else - for (i = 0; i < SHA512_DIGEST_SIZE / sizeof(u32); i++) - hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); + switch (ctx->flags & SHA_FLAGS_ALGO_MASK) { + case SHA_FLAGS_SHA1: + hashsize = SHA1_DIGEST_SIZE; + break; + + case SHA_FLAGS_SHA224: + case SHA_FLAGS_SHA256: + hashsize = SHA256_DIGEST_SIZE; + break; + + case SHA_FLAGS_SHA384: + case SHA_FLAGS_SHA512: + hashsize = SHA512_DIGEST_SIZE; + break; + + default: + /* Should not happen... */ + return; + } + + for (i = 0; i < hashsize / sizeof(u32); ++i) + hash[i] = atmel_sha_read(ctx->dd, SHA_REG_DIGEST(i)); + ctx->flags |= SHA_FLAGS_RESTORE; } static void atmel_sha_copy_ready_hash(struct ahash_request *req) @@ -788,7 +845,7 @@ static void atmel_sha_finish_req(struct ahash_request *req, int err) req->base.complete(&req->base, err); /* handle new request */ - tasklet_schedule(&dd->done_task); + tasklet_schedule(&dd->queue_task); } static int atmel_sha_hw_init(struct atmel_sha_dev *dd) @@ -922,36 +979,17 @@ static int atmel_sha_update(struct ahash_request *req) static int atmel_sha_final(struct ahash_request *req) { struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); - struct atmel_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm); - struct atmel_sha_dev *dd = tctx->dd; - - int err = 0; ctx->flags |= SHA_FLAGS_FINUP; if (ctx->flags & SHA_FLAGS_ERROR) return 0; /* uncompleted hash is not needed */ - if (ctx->bufcnt) { - return atmel_sha_enqueue(req, SHA_OP_FINAL); - } else if (!(ctx->flags & SHA_FLAGS_PAD)) { /* add padding */ - err = atmel_sha_hw_init(dd); - if (err) - goto err1; - - dd->flags |= SHA_FLAGS_BUSY; - err = atmel_sha_final_req(dd); - } else { + if (ctx->flags & SHA_FLAGS_PAD) /* copy ready hash (+ finalize hmac) */ return atmel_sha_finish(req); - } - -err1: - if (err != -EINPROGRESS) - /* done_task will not finish it, so do it here */ - atmel_sha_finish_req(req, err); - return err; + return atmel_sha_enqueue(req, SHA_OP_FINAL); } static int atmel_sha_finup(struct ahash_request *req) @@ -979,11 +1017,27 @@ static int atmel_sha_digest(struct ahash_request *req) return atmel_sha_init(req) ?: atmel_sha_finup(req); } + +static int atmel_sha_export(struct ahash_request *req, void *out) +{ + const struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + + memcpy(out, ctx, sizeof(*ctx)); + return 0; +} + +static int atmel_sha_import(struct ahash_request *req, const void *in) +{ + struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); + + memcpy(ctx, in, sizeof(*ctx)); + return 0; +} + static int atmel_sha_cra_init(struct crypto_tfm *tfm) { crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), - sizeof(struct atmel_sha_reqctx) + - SHA_BUFFER_LEN + SHA512_BLOCK_SIZE); + sizeof(struct atmel_sha_reqctx)); return 0; } @@ -995,8 +1049,11 @@ static struct ahash_alg sha_1_256_algs[] = { .final = atmel_sha_final, .finup = atmel_sha_finup, .digest = atmel_sha_digest, + .export = atmel_sha_export, + .import = atmel_sha_import, .halg = { .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct atmel_sha_reqctx), .base = { .cra_name = "sha1", .cra_driver_name = "atmel-sha1", @@ -1016,8 +1073,11 @@ static struct ahash_alg sha_1_256_algs[] = { .final = atmel_sha_final, .finup = atmel_sha_finup, .digest = atmel_sha_digest, + .export = atmel_sha_export, + .import = atmel_sha_import, .halg = { .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct atmel_sha_reqctx), .base = { .cra_name = "sha256", .cra_driver_name = "atmel-sha256", @@ -1039,8 +1099,11 @@ static struct ahash_alg sha_224_alg = { .final = atmel_sha_final, .finup = atmel_sha_finup, .digest = atmel_sha_digest, + .export = atmel_sha_export, + .import = atmel_sha_import, .halg = { .digestsize = SHA224_DIGEST_SIZE, + .statesize = sizeof(struct atmel_sha_reqctx), .base = { .cra_name = "sha224", .cra_driver_name = "atmel-sha224", @@ -1062,8 +1125,11 @@ static struct ahash_alg sha_384_512_algs[] = { .final = atmel_sha_final, .finup = atmel_sha_finup, .digest = atmel_sha_digest, + .export = atmel_sha_export, + .import = atmel_sha_import, .halg = { .digestsize = SHA384_DIGEST_SIZE, + .statesize = sizeof(struct atmel_sha_reqctx), .base = { .cra_name = "sha384", .cra_driver_name = "atmel-sha384", @@ -1083,8 +1149,11 @@ static struct ahash_alg sha_384_512_algs[] = { .final = atmel_sha_final, .finup = atmel_sha_finup, .digest = atmel_sha_digest, + .export = atmel_sha_export, + .import = atmel_sha_import, .halg = { .digestsize = SHA512_DIGEST_SIZE, + .statesize = sizeof(struct atmel_sha_reqctx), .base = { .cra_name = "sha512", .cra_driver_name = "atmel-sha512", @@ -1100,16 +1169,18 @@ static struct ahash_alg sha_384_512_algs[] = { }, }; +static void atmel_sha_queue_task(unsigned long data) +{ + struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data; + + atmel_sha_handle_queue(dd, NULL); +} + static void atmel_sha_done_task(unsigned long data) { struct atmel_sha_dev *dd = (struct atmel_sha_dev *)data; int err = 0; - if (!(SHA_FLAGS_BUSY & dd->flags)) { - atmel_sha_handle_queue(dd, NULL); - return; - } - if (SHA_FLAGS_CPU & dd->flags) { if (SHA_FLAGS_OUTPUT_READY & dd->flags) { dd->flags &= ~SHA_FLAGS_OUTPUT_READY; @@ -1272,14 +1343,23 @@ static void atmel_sha_get_cap(struct atmel_sha_dev *dd) dd->caps.has_dualbuff = 0; dd->caps.has_sha224 = 0; dd->caps.has_sha_384_512 = 0; + dd->caps.has_uihv = 0; /* keep only major version number */ switch (dd->hw_version & 0xff0) { + case 0x510: + dd->caps.has_dma = 1; + dd->caps.has_dualbuff = 1; + dd->caps.has_sha224 = 1; + dd->caps.has_sha_384_512 = 1; + dd->caps.has_uihv = 1; + break; case 0x420: dd->caps.has_dma = 1; dd->caps.has_dualbuff = 1; dd->caps.has_sha224 = 1; dd->caps.has_sha_384_512 = 1; + dd->caps.has_uihv = 1; break; case 0x410: dd->caps.has_dma = 1; @@ -1366,6 +1446,8 @@ static int atmel_sha_probe(struct platform_device *pdev) tasklet_init(&sha_dd->done_task, atmel_sha_done_task, (unsigned long)sha_dd); + tasklet_init(&sha_dd->queue_task, atmel_sha_queue_task, + (unsigned long)sha_dd); crypto_init_queue(&sha_dd->queue, ATMEL_SHA_QUEUE_LENGTH); @@ -1464,6 +1546,7 @@ err_sha_dma: iclk_unprepare: clk_unprepare(sha_dd->iclk); res_err: + tasklet_kill(&sha_dd->queue_task); tasklet_kill(&sha_dd->done_task); sha_dd_err: dev_err(dev, "initialization failed.\n"); @@ -1484,6 +1567,7 @@ static int atmel_sha_remove(struct platform_device *pdev) atmel_sha_unregister_algs(sha_dd); + tasklet_kill(&sha_dd->queue_task); tasklet_kill(&sha_dd->done_task); if (sha_dd->caps.has_dma) diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index 69d4a1326..44d30b45f 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -534,7 +534,7 @@ static int caam_probe(struct platform_device *pdev) * long pointers in master configuration register */ clrsetbits_32(&ctrl->mcr, MCFGR_AWCACHE_MASK, MCFGR_AWCACHE_CACH | - MCFGR_AWCACHE_BUFF | MCFGR_WDENABLE | + MCFGR_AWCACHE_BUFF | MCFGR_WDENABLE | MCFGR_LARGE_BURST | (sizeof(dma_addr_t) == sizeof(u64) ? MCFGR_LONG_PTR : 0)); /* diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index f7e0d8d4c..5ef4be22e 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -65,7 +65,7 @@ static int caam_reset_hw_jr(struct device *dev) /* * Shutdown JobR independent of platform property code */ -int caam_jr_shutdown(struct device *dev) +static int caam_jr_shutdown(struct device *dev) { struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); dma_addr_t inpbusaddr, outbusaddr; @@ -248,7 +248,7 @@ static void caam_jr_dequeue(unsigned long devarg) struct device *caam_jr_alloc(void) { struct caam_drv_private_jr *jrpriv, *min_jrpriv = NULL; - struct device *dev = NULL; + struct device *dev = ERR_PTR(-ENODEV); int min_tfm_cnt = INT_MAX; int tfm_cnt; diff --git a/drivers/crypto/caam/regs.h b/drivers/crypto/caam/regs.h index a8a799756..0ba9c4059 100644 --- a/drivers/crypto/caam/regs.h +++ b/drivers/crypto/caam/regs.h @@ -455,7 +455,8 @@ struct caam_ctrl { #define MCFGR_AXIPIPE_MASK (0xf << MCFGR_AXIPIPE_SHIFT) #define MCFGR_AXIPRI 0x00000008 /* Assert AXI priority sideband */ -#define MCFGR_BURST_64 0x00000001 /* Max burst size */ +#define MCFGR_LARGE_BURST 0x00000004 /* 128/256-byte burst size */ +#define MCFGR_BURST_64 0x00000001 /* 64-byte burst size */ /* JRSTART register offsets */ #define JRSTART_JR0_START 0x00000001 /* Start Job ring 0 */ diff --git a/drivers/crypto/ccp/Makefile b/drivers/crypto/ccp/Makefile index 55a1f3951..b750592cc 100644 --- a/drivers/crypto/ccp/Makefile +++ b/drivers/crypto/ccp/Makefile @@ -1,5 +1,5 @@ obj-$(CONFIG_CRYPTO_DEV_CCP_DD) += ccp.o -ccp-objs := ccp-dev.o ccp-ops.o ccp-platform.o +ccp-objs := ccp-dev.o ccp-ops.o ccp-dev-v3.o ccp-platform.o ccp-$(CONFIG_PCI) += ccp-pci.o obj-$(CONFIG_CRYPTO_DEV_CCP_CRYPTO) += ccp-crypto.o diff --git a/drivers/crypto/ccp/ccp-crypto-aes.c b/drivers/crypto/ccp/ccp-crypto-aes.c index 7984f9108..89291c150 100644 --- a/drivers/crypto/ccp/ccp-crypto-aes.c +++ b/drivers/crypto/ccp/ccp-crypto-aes.c @@ -1,7 +1,7 @@ /* * AMD Cryptographic Coprocessor (CCP) AES crypto API support * - * Copyright (C) 2013 Advanced Micro Devices, Inc. + * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. * * Author: Tom Lendacky <thomas.lendacky@amd.com> * @@ -259,6 +259,7 @@ static struct crypto_alg ccp_aes_rfc3686_defaults = { struct ccp_aes_def { enum ccp_aes_mode mode; + unsigned int version; const char *name; const char *driver_name; unsigned int blocksize; @@ -269,6 +270,7 @@ struct ccp_aes_def { static struct ccp_aes_def aes_algs[] = { { .mode = CCP_AES_MODE_ECB, + .version = CCP_VERSION(3, 0), .name = "ecb(aes)", .driver_name = "ecb-aes-ccp", .blocksize = AES_BLOCK_SIZE, @@ -277,6 +279,7 @@ static struct ccp_aes_def aes_algs[] = { }, { .mode = CCP_AES_MODE_CBC, + .version = CCP_VERSION(3, 0), .name = "cbc(aes)", .driver_name = "cbc-aes-ccp", .blocksize = AES_BLOCK_SIZE, @@ -285,6 +288,7 @@ static struct ccp_aes_def aes_algs[] = { }, { .mode = CCP_AES_MODE_CFB, + .version = CCP_VERSION(3, 0), .name = "cfb(aes)", .driver_name = "cfb-aes-ccp", .blocksize = AES_BLOCK_SIZE, @@ -293,6 +297,7 @@ static struct ccp_aes_def aes_algs[] = { }, { .mode = CCP_AES_MODE_OFB, + .version = CCP_VERSION(3, 0), .name = "ofb(aes)", .driver_name = "ofb-aes-ccp", .blocksize = 1, @@ -301,6 +306,7 @@ static struct ccp_aes_def aes_algs[] = { }, { .mode = CCP_AES_MODE_CTR, + .version = CCP_VERSION(3, 0), .name = "ctr(aes)", .driver_name = "ctr-aes-ccp", .blocksize = 1, @@ -309,6 +315,7 @@ static struct ccp_aes_def aes_algs[] = { }, { .mode = CCP_AES_MODE_CTR, + .version = CCP_VERSION(3, 0), .name = "rfc3686(ctr(aes))", .driver_name = "rfc3686-ctr-aes-ccp", .blocksize = 1, @@ -357,8 +364,11 @@ static int ccp_register_aes_alg(struct list_head *head, int ccp_register_aes_algs(struct list_head *head) { int i, ret; + unsigned int ccpversion = ccp_version(); for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { + if (aes_algs[i].version > ccpversion) + continue; ret = ccp_register_aes_alg(head, &aes_algs[i]); if (ret) return ret; diff --git a/drivers/crypto/ccp/ccp-crypto-sha.c b/drivers/crypto/ccp/ccp-crypto-sha.c index ab9945f2c..8f36af62f 100644 --- a/drivers/crypto/ccp/ccp-crypto-sha.c +++ b/drivers/crypto/ccp/ccp-crypto-sha.c @@ -1,7 +1,7 @@ /* * AMD Cryptographic Coprocessor (CCP) SHA crypto API support * - * Copyright (C) 2013 Advanced Micro Devices, Inc. + * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. * * Author: Tom Lendacky <thomas.lendacky@amd.com> * @@ -344,6 +344,7 @@ static void ccp_hmac_sha_cra_exit(struct crypto_tfm *tfm) } struct ccp_sha_def { + unsigned int version; const char *name; const char *drv_name; enum ccp_sha_type type; @@ -353,6 +354,7 @@ struct ccp_sha_def { static struct ccp_sha_def sha_algs[] = { { + .version = CCP_VERSION(3, 0), .name = "sha1", .drv_name = "sha1-ccp", .type = CCP_SHA_TYPE_1, @@ -360,6 +362,7 @@ static struct ccp_sha_def sha_algs[] = { .block_size = SHA1_BLOCK_SIZE, }, { + .version = CCP_VERSION(3, 0), .name = "sha224", .drv_name = "sha224-ccp", .type = CCP_SHA_TYPE_224, @@ -367,6 +370,7 @@ static struct ccp_sha_def sha_algs[] = { .block_size = SHA224_BLOCK_SIZE, }, { + .version = CCP_VERSION(3, 0), .name = "sha256", .drv_name = "sha256-ccp", .type = CCP_SHA_TYPE_256, @@ -483,8 +487,11 @@ static int ccp_register_sha_alg(struct list_head *head, int ccp_register_sha_algs(struct list_head *head) { int i, ret; + unsigned int ccpversion = ccp_version(); for (i = 0; i < ARRAY_SIZE(sha_algs); i++) { + if (sha_algs[i].version > ccpversion) + continue; ret = ccp_register_sha_alg(head, &sha_algs[i]); if (ret) return ret; diff --git a/drivers/crypto/ccp/ccp-dev-v3.c b/drivers/crypto/ccp/ccp-dev-v3.c new file mode 100644 index 000000000..7d5eab491 --- /dev/null +++ b/drivers/crypto/ccp/ccp-dev-v3.c @@ -0,0 +1,533 @@ +/* + * AMD Cryptographic Coprocessor (CCP) driver + * + * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/kthread.h> +#include <linux/interrupt.h> +#include <linux/ccp.h> + +#include "ccp-dev.h" + +static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count) +{ + struct ccp_cmd_queue *cmd_q = op->cmd_q; + struct ccp_device *ccp = cmd_q->ccp; + void __iomem *cr_addr; + u32 cr0, cmd; + unsigned int i; + int ret = 0; + + /* We could read a status register to see how many free slots + * are actually available, but reading that register resets it + * and you could lose some error information. + */ + cmd_q->free_slots--; + + cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT) + | (op->jobid << REQ0_JOBID_SHIFT) + | REQ0_WAIT_FOR_WRITE; + + if (op->soc) + cr0 |= REQ0_STOP_ON_COMPLETE + | REQ0_INT_ON_COMPLETE; + + if (op->ioc || !cmd_q->free_slots) + cr0 |= REQ0_INT_ON_COMPLETE; + + /* Start at CMD_REQ1 */ + cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR; + + mutex_lock(&ccp->req_mutex); + + /* Write CMD_REQ1 through CMD_REQx first */ + for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR) + iowrite32(*(cr + i), cr_addr); + + /* Tell the CCP to start */ + wmb(); + iowrite32(cr0, ccp->io_regs + CMD_REQ0); + + mutex_unlock(&ccp->req_mutex); + + if (cr0 & REQ0_INT_ON_COMPLETE) { + /* Wait for the job to complete */ + ret = wait_event_interruptible(cmd_q->int_queue, + cmd_q->int_rcvd); + if (ret || cmd_q->cmd_error) { + /* On error delete all related jobs from the queue */ + cmd = (cmd_q->id << DEL_Q_ID_SHIFT) + | op->jobid; + + iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); + + if (!ret) + ret = -EIO; + } else if (op->soc) { + /* Delete just head job from the queue on SoC */ + cmd = DEL_Q_ACTIVE + | (cmd_q->id << DEL_Q_ID_SHIFT) + | op->jobid; + + iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); + } + + cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status); + + cmd_q->int_rcvd = 0; + } + + return ret; +} + +static int ccp_perform_aes(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT) + | (op->u.aes.type << REQ1_AES_TYPE_SHIFT) + | (op->u.aes.mode << REQ1_AES_MODE_SHIFT) + | (op->u.aes.action << REQ1_AES_ACTION_SHIFT) + | (op->ksb_key << REQ1_KEY_KSB_SHIFT); + cr[1] = op->src.u.dma.length - 1; + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) + | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + cr[4] = ccp_addr_lo(&op->dst.u.dma); + cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->dst.u.dma); + + if (op->u.aes.mode == CCP_AES_MODE_CFB) + cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT); + + if (op->eom) + cr[0] |= REQ1_EOM; + + if (op->init) + cr[0] |= REQ1_INIT; + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static int ccp_perform_xts_aes(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT) + | (op->u.xts.action << REQ1_AES_ACTION_SHIFT) + | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT) + | (op->ksb_key << REQ1_KEY_KSB_SHIFT); + cr[1] = op->src.u.dma.length - 1; + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) + | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + cr[4] = ccp_addr_lo(&op->dst.u.dma); + cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->dst.u.dma); + + if (op->eom) + cr[0] |= REQ1_EOM; + + if (op->init) + cr[0] |= REQ1_INIT; + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static int ccp_perform_sha(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT) + | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT) + | REQ1_INIT; + cr[1] = op->src.u.dma.length - 1; + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) + | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + + if (op->eom) { + cr[0] |= REQ1_EOM; + cr[4] = lower_32_bits(op->u.sha.msg_bits); + cr[5] = upper_32_bits(op->u.sha.msg_bits); + } else { + cr[4] = 0; + cr[5] = 0; + } + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static int ccp_perform_rsa(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT) + | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT) + | (op->ksb_key << REQ1_KEY_KSB_SHIFT) + | REQ1_EOM; + cr[1] = op->u.rsa.input_len - 1; + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) + | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + cr[4] = ccp_addr_lo(&op->dst.u.dma); + cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->dst.u.dma); + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static int ccp_perform_passthru(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT) + | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT) + | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT); + + if (op->src.type == CCP_MEMTYPE_SYSTEM) + cr[1] = op->src.u.dma.length - 1; + else + cr[1] = op->dst.u.dma.length - 1; + + if (op->src.type == CCP_MEMTYPE_SYSTEM) { + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + + if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP) + cr[3] |= (op->ksb_key << REQ4_KSB_SHIFT); + } else { + cr[2] = op->src.u.ksb * CCP_KSB_BYTES; + cr[3] = (CCP_MEMTYPE_KSB << REQ4_MEMTYPE_SHIFT); + } + + if (op->dst.type == CCP_MEMTYPE_SYSTEM) { + cr[4] = ccp_addr_lo(&op->dst.u.dma); + cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->dst.u.dma); + } else { + cr[4] = op->dst.u.ksb * CCP_KSB_BYTES; + cr[5] = (CCP_MEMTYPE_KSB << REQ6_MEMTYPE_SHIFT); + } + + if (op->eom) + cr[0] |= REQ1_EOM; + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static int ccp_perform_ecc(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = REQ1_ECC_AFFINE_CONVERT + | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT) + | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT) + | REQ1_EOM; + cr[1] = op->src.u.dma.length - 1; + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + cr[4] = ccp_addr_lo(&op->dst.u.dma); + cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->dst.u.dma); + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait) +{ + struct ccp_device *ccp = container_of(rng, struct ccp_device, hwrng); + u32 trng_value; + int len = min_t(int, sizeof(trng_value), max); + + /* + * Locking is provided by the caller so we can update device + * hwrng-related fields safely + */ + trng_value = ioread32(ccp->io_regs + TRNG_OUT_REG); + if (!trng_value) { + /* Zero is returned if not data is available or if a + * bad-entropy error is present. Assume an error if + * we exceed TRNG_RETRIES reads of zero. + */ + if (ccp->hwrng_retries++ > TRNG_RETRIES) + return -EIO; + + return 0; + } + + /* Reset the counter and save the rng value */ + ccp->hwrng_retries = 0; + memcpy(data, &trng_value, len); + + return len; +} + +static int ccp_init(struct ccp_device *ccp) +{ + struct device *dev = ccp->dev; + struct ccp_cmd_queue *cmd_q; + struct dma_pool *dma_pool; + char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; + unsigned int qmr, qim, i; + int ret; + + /* Find available queues */ + qim = 0; + qmr = ioread32(ccp->io_regs + Q_MASK_REG); + for (i = 0; i < MAX_HW_QUEUES; i++) { + if (!(qmr & (1 << i))) + continue; + + /* Allocate a dma pool for this queue */ + snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d", + ccp->name, i); + dma_pool = dma_pool_create(dma_pool_name, dev, + CCP_DMAPOOL_MAX_SIZE, + CCP_DMAPOOL_ALIGN, 0); + if (!dma_pool) { + dev_err(dev, "unable to allocate dma pool\n"); + ret = -ENOMEM; + goto e_pool; + } + + cmd_q = &ccp->cmd_q[ccp->cmd_q_count]; + ccp->cmd_q_count++; + + cmd_q->ccp = ccp; + cmd_q->id = i; + cmd_q->dma_pool = dma_pool; + + /* Reserve 2 KSB regions for the queue */ + cmd_q->ksb_key = KSB_START + ccp->ksb_start++; + cmd_q->ksb_ctx = KSB_START + ccp->ksb_start++; + ccp->ksb_count -= 2; + + /* Preset some register values and masks that are queue + * number dependent + */ + cmd_q->reg_status = ccp->io_regs + CMD_Q_STATUS_BASE + + (CMD_Q_STATUS_INCR * i); + cmd_q->reg_int_status = ccp->io_regs + CMD_Q_INT_STATUS_BASE + + (CMD_Q_STATUS_INCR * i); + cmd_q->int_ok = 1 << (i * 2); + cmd_q->int_err = 1 << ((i * 2) + 1); + + cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status)); + + init_waitqueue_head(&cmd_q->int_queue); + + /* Build queue interrupt mask (two interrupts per queue) */ + qim |= cmd_q->int_ok | cmd_q->int_err; + +#ifdef CONFIG_ARM64 + /* For arm64 set the recommended queue cache settings */ + iowrite32(ccp->axcache, ccp->io_regs + CMD_Q_CACHE_BASE + + (CMD_Q_CACHE_INC * i)); +#endif + + dev_dbg(dev, "queue #%u available\n", i); + } + if (ccp->cmd_q_count == 0) { + dev_notice(dev, "no command queues available\n"); + ret = -EIO; + goto e_pool; + } + dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count); + + /* Disable and clear interrupts until ready */ + iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG); + for (i = 0; i < ccp->cmd_q_count; i++) { + cmd_q = &ccp->cmd_q[i]; + + ioread32(cmd_q->reg_int_status); + ioread32(cmd_q->reg_status); + } + iowrite32(qim, ccp->io_regs + IRQ_STATUS_REG); + + /* Request an irq */ + ret = ccp->get_irq(ccp); + if (ret) { + dev_err(dev, "unable to allocate an IRQ\n"); + goto e_pool; + } + + /* Initialize the queues used to wait for KSB space and suspend */ + init_waitqueue_head(&ccp->ksb_queue); + init_waitqueue_head(&ccp->suspend_queue); + + /* Create a kthread for each queue */ + for (i = 0; i < ccp->cmd_q_count; i++) { + struct task_struct *kthread; + + cmd_q = &ccp->cmd_q[i]; + + kthread = kthread_create(ccp_cmd_queue_thread, cmd_q, + "%s-q%u", ccp->name, cmd_q->id); + if (IS_ERR(kthread)) { + dev_err(dev, "error creating queue thread (%ld)\n", + PTR_ERR(kthread)); + ret = PTR_ERR(kthread); + goto e_kthread; + } + + cmd_q->kthread = kthread; + wake_up_process(kthread); + } + + /* Register the RNG */ + ccp->hwrng.name = ccp->rngname; + ccp->hwrng.read = ccp_trng_read; + ret = hwrng_register(&ccp->hwrng); + if (ret) { + dev_err(dev, "error registering hwrng (%d)\n", ret); + goto e_kthread; + } + + ccp_add_device(ccp); + + /* Enable interrupts */ + iowrite32(qim, ccp->io_regs + IRQ_MASK_REG); + + return 0; + +e_kthread: + for (i = 0; i < ccp->cmd_q_count; i++) + if (ccp->cmd_q[i].kthread) + kthread_stop(ccp->cmd_q[i].kthread); + + ccp->free_irq(ccp); + +e_pool: + for (i = 0; i < ccp->cmd_q_count; i++) + dma_pool_destroy(ccp->cmd_q[i].dma_pool); + + return ret; +} + +static void ccp_destroy(struct ccp_device *ccp) +{ + struct ccp_cmd_queue *cmd_q; + struct ccp_cmd *cmd; + unsigned int qim, i; + + /* Remove this device from the list of available units first */ + ccp_del_device(ccp); + + /* Unregister the RNG */ + hwrng_unregister(&ccp->hwrng); + + /* Stop the queue kthreads */ + for (i = 0; i < ccp->cmd_q_count; i++) + if (ccp->cmd_q[i].kthread) + kthread_stop(ccp->cmd_q[i].kthread); + + /* Build queue interrupt mask (two interrupt masks per queue) */ + qim = 0; + for (i = 0; i < ccp->cmd_q_count; i++) { + cmd_q = &ccp->cmd_q[i]; + qim |= cmd_q->int_ok | cmd_q->int_err; + } + + /* Disable and clear interrupts */ + iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG); + for (i = 0; i < ccp->cmd_q_count; i++) { + cmd_q = &ccp->cmd_q[i]; + + ioread32(cmd_q->reg_int_status); + ioread32(cmd_q->reg_status); + } + iowrite32(qim, ccp->io_regs + IRQ_STATUS_REG); + + ccp->free_irq(ccp); + + for (i = 0; i < ccp->cmd_q_count; i++) + dma_pool_destroy(ccp->cmd_q[i].dma_pool); + + /* Flush the cmd and backlog queue */ + while (!list_empty(&ccp->cmd)) { + /* Invoke the callback directly with an error code */ + cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); + list_del(&cmd->entry); + cmd->callback(cmd->data, -ENODEV); + } + while (!list_empty(&ccp->backlog)) { + /* Invoke the callback directly with an error code */ + cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry); + list_del(&cmd->entry); + cmd->callback(cmd->data, -ENODEV); + } +} + +static irqreturn_t ccp_irq_handler(int irq, void *data) +{ + struct device *dev = data; + struct ccp_device *ccp = dev_get_drvdata(dev); + struct ccp_cmd_queue *cmd_q; + u32 q_int, status; + unsigned int i; + + status = ioread32(ccp->io_regs + IRQ_STATUS_REG); + + for (i = 0; i < ccp->cmd_q_count; i++) { + cmd_q = &ccp->cmd_q[i]; + + q_int = status & (cmd_q->int_ok | cmd_q->int_err); + if (q_int) { + cmd_q->int_status = status; + cmd_q->q_status = ioread32(cmd_q->reg_status); + cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); + + /* On error, only save the first error value */ + if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error) + cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); + + cmd_q->int_rcvd = 1; + + /* Acknowledge the interrupt and wake the kthread */ + iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG); + wake_up_interruptible(&cmd_q->int_queue); + } + } + + return IRQ_HANDLED; +} + +static struct ccp_actions ccp3_actions = { + .perform_aes = ccp_perform_aes, + .perform_xts_aes = ccp_perform_xts_aes, + .perform_sha = ccp_perform_sha, + .perform_rsa = ccp_perform_rsa, + .perform_passthru = ccp_perform_passthru, + .perform_ecc = ccp_perform_ecc, + .init = ccp_init, + .destroy = ccp_destroy, + .irqhandler = ccp_irq_handler, +}; + +struct ccp_vdata ccpv3 = { + .version = CCP_VERSION(3, 0), + .perform = &ccp3_actions, +}; diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c index 861bacc1b..4dbc18727 100644 --- a/drivers/crypto/ccp/ccp-dev.c +++ b/drivers/crypto/ccp/ccp-dev.c @@ -1,7 +1,7 @@ /* * AMD Cryptographic Coprocessor (CCP) driver * - * Copyright (C) 2013 Advanced Micro Devices, Inc. + * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. * * Author: Tom Lendacky <thomas.lendacky@amd.com> * @@ -16,6 +16,8 @@ #include <linux/sched.h> #include <linux/interrupt.h> #include <linux/spinlock.h> +#include <linux/rwlock_types.h> +#include <linux/types.h> #include <linux/mutex.h> #include <linux/delay.h> #include <linux/hw_random.h> @@ -37,20 +39,107 @@ struct ccp_tasklet_data { struct ccp_cmd *cmd; }; -static struct ccp_device *ccp_dev; -static inline struct ccp_device *ccp_get_device(void) +/* List of CCPs, CCP count, read-write access lock, and access functions + * + * Lock structure: get ccp_unit_lock for reading whenever we need to + * examine the CCP list. While holding it for reading we can acquire + * the RR lock to update the round-robin next-CCP pointer. The unit lock + * must be acquired before the RR lock. + * + * If the unit-lock is acquired for writing, we have total control over + * the list, so there's no value in getting the RR lock. + */ +static DEFINE_RWLOCK(ccp_unit_lock); +static LIST_HEAD(ccp_units); + +/* Round-robin counter */ +static DEFINE_SPINLOCK(ccp_rr_lock); +static struct ccp_device *ccp_rr; + +/* Ever-increasing value to produce unique unit numbers */ +static atomic_t ccp_unit_ordinal; +unsigned int ccp_increment_unit_ordinal(void) { - return ccp_dev; + return atomic_inc_return(&ccp_unit_ordinal); } -static inline void ccp_add_device(struct ccp_device *ccp) +/** + * ccp_add_device - add a CCP device to the list + * + * @ccp: ccp_device struct pointer + * + * Put this CCP on the unit list, which makes it available + * for use. + * + * Returns zero if a CCP device is present, -ENODEV otherwise. + */ +void ccp_add_device(struct ccp_device *ccp) { - ccp_dev = ccp; + unsigned long flags; + + write_lock_irqsave(&ccp_unit_lock, flags); + list_add_tail(&ccp->entry, &ccp_units); + if (!ccp_rr) + /* We already have the list lock (we're first) so this + * pointer can't change on us. Set its initial value. + */ + ccp_rr = ccp; + write_unlock_irqrestore(&ccp_unit_lock, flags); } -static inline void ccp_del_device(struct ccp_device *ccp) +/** + * ccp_del_device - remove a CCP device from the list + * + * @ccp: ccp_device struct pointer + * + * Remove this unit from the list of devices. If the next device + * up for use is this one, adjust the pointer. If this is the last + * device, NULL the pointer. + */ +void ccp_del_device(struct ccp_device *ccp) { - ccp_dev = NULL; + unsigned long flags; + + write_lock_irqsave(&ccp_unit_lock, flags); + if (ccp_rr == ccp) { + /* ccp_unit_lock is read/write; any read access + * will be suspended while we make changes to the + * list and RR pointer. + */ + if (list_is_last(&ccp_rr->entry, &ccp_units)) + ccp_rr = list_first_entry(&ccp_units, struct ccp_device, + entry); + else + ccp_rr = list_next_entry(ccp_rr, entry); + } + list_del(&ccp->entry); + if (list_empty(&ccp_units)) + ccp_rr = NULL; + write_unlock_irqrestore(&ccp_unit_lock, flags); +} + +static struct ccp_device *ccp_get_device(void) +{ + unsigned long flags; + struct ccp_device *dp = NULL; + + /* We round-robin through the unit list. + * The (ccp_rr) pointer refers to the next unit to use. + */ + read_lock_irqsave(&ccp_unit_lock, flags); + if (!list_empty(&ccp_units)) { + spin_lock(&ccp_rr_lock); + dp = ccp_rr; + if (list_is_last(&ccp_rr->entry, &ccp_units)) + ccp_rr = list_first_entry(&ccp_units, struct ccp_device, + entry); + else + ccp_rr = list_next_entry(ccp_rr, entry); + spin_unlock(&ccp_rr_lock); + } + read_unlock_irqrestore(&ccp_unit_lock, flags); + + return dp; } /** @@ -60,14 +149,41 @@ static inline void ccp_del_device(struct ccp_device *ccp) */ int ccp_present(void) { - if (ccp_get_device()) - return 0; + unsigned long flags; + int ret; - return -ENODEV; + read_lock_irqsave(&ccp_unit_lock, flags); + ret = list_empty(&ccp_units); + read_unlock_irqrestore(&ccp_unit_lock, flags); + + return ret ? -ENODEV : 0; } EXPORT_SYMBOL_GPL(ccp_present); /** + * ccp_version - get the version of the CCP device + * + * Returns the version from the first unit on the list; + * otherwise a zero if no CCP device is present + */ +unsigned int ccp_version(void) +{ + struct ccp_device *dp; + unsigned long flags; + int ret = 0; + + read_lock_irqsave(&ccp_unit_lock, flags); + if (!list_empty(&ccp_units)) { + dp = list_first_entry(&ccp_units, struct ccp_device, entry); + ret = dp->vdata->version; + } + read_unlock_irqrestore(&ccp_unit_lock, flags); + + return ret; +} +EXPORT_SYMBOL_GPL(ccp_version); + +/** * ccp_enqueue_cmd - queue an operation for processing by the CCP * * @cmd: ccp_cmd struct to be processed @@ -221,7 +337,12 @@ static void ccp_do_cmd_complete(unsigned long data) complete(&tdata->completion); } -static int ccp_cmd_queue_thread(void *data) +/** + * ccp_cmd_queue_thread - create a kernel thread to manage a CCP queue + * + * @data: thread-specific data + */ +int ccp_cmd_queue_thread(void *data) { struct ccp_cmd_queue *cmd_q = (struct ccp_cmd_queue *)data; struct ccp_cmd *cmd; @@ -257,35 +378,6 @@ static int ccp_cmd_queue_thread(void *data) return 0; } -static int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait) -{ - struct ccp_device *ccp = container_of(rng, struct ccp_device, hwrng); - u32 trng_value; - int len = min_t(int, sizeof(trng_value), max); - - /* - * Locking is provided by the caller so we can update device - * hwrng-related fields safely - */ - trng_value = ioread32(ccp->io_regs + TRNG_OUT_REG); - if (!trng_value) { - /* Zero is returned if not data is available or if a - * bad-entropy error is present. Assume an error if - * we exceed TRNG_RETRIES reads of zero. - */ - if (ccp->hwrng_retries++ > TRNG_RETRIES) - return -EIO; - - return 0; - } - - /* Reset the counter and save the rng value */ - ccp->hwrng_retries = 0; - memcpy(data, &trng_value, len); - - return len; -} - /** * ccp_alloc_struct - allocate and initialize the ccp_device struct * @@ -309,253 +401,11 @@ struct ccp_device *ccp_alloc_struct(struct device *dev) ccp->ksb_count = KSB_COUNT; ccp->ksb_start = 0; - return ccp; -} - -/** - * ccp_init - initialize the CCP device - * - * @ccp: ccp_device struct - */ -int ccp_init(struct ccp_device *ccp) -{ - struct device *dev = ccp->dev; - struct ccp_cmd_queue *cmd_q; - struct dma_pool *dma_pool; - char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; - unsigned int qmr, qim, i; - int ret; - - /* Find available queues */ - qim = 0; - qmr = ioread32(ccp->io_regs + Q_MASK_REG); - for (i = 0; i < MAX_HW_QUEUES; i++) { - if (!(qmr & (1 << i))) - continue; - - /* Allocate a dma pool for this queue */ - snprintf(dma_pool_name, sizeof(dma_pool_name), "ccp_q%d", i); - dma_pool = dma_pool_create(dma_pool_name, dev, - CCP_DMAPOOL_MAX_SIZE, - CCP_DMAPOOL_ALIGN, 0); - if (!dma_pool) { - dev_err(dev, "unable to allocate dma pool\n"); - ret = -ENOMEM; - goto e_pool; - } - - cmd_q = &ccp->cmd_q[ccp->cmd_q_count]; - ccp->cmd_q_count++; - - cmd_q->ccp = ccp; - cmd_q->id = i; - cmd_q->dma_pool = dma_pool; - - /* Reserve 2 KSB regions for the queue */ - cmd_q->ksb_key = KSB_START + ccp->ksb_start++; - cmd_q->ksb_ctx = KSB_START + ccp->ksb_start++; - ccp->ksb_count -= 2; - - /* Preset some register values and masks that are queue - * number dependent - */ - cmd_q->reg_status = ccp->io_regs + CMD_Q_STATUS_BASE + - (CMD_Q_STATUS_INCR * i); - cmd_q->reg_int_status = ccp->io_regs + CMD_Q_INT_STATUS_BASE + - (CMD_Q_STATUS_INCR * i); - cmd_q->int_ok = 1 << (i * 2); - cmd_q->int_err = 1 << ((i * 2) + 1); - - cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status)); - - init_waitqueue_head(&cmd_q->int_queue); - - /* Build queue interrupt mask (two interrupts per queue) */ - qim |= cmd_q->int_ok | cmd_q->int_err; + ccp->ord = ccp_increment_unit_ordinal(); + snprintf(ccp->name, MAX_CCP_NAME_LEN, "ccp-%u", ccp->ord); + snprintf(ccp->rngname, MAX_CCP_NAME_LEN, "ccp-%u-rng", ccp->ord); -#ifdef CONFIG_ARM64 - /* For arm64 set the recommended queue cache settings */ - iowrite32(ccp->axcache, ccp->io_regs + CMD_Q_CACHE_BASE + - (CMD_Q_CACHE_INC * i)); -#endif - - dev_dbg(dev, "queue #%u available\n", i); - } - if (ccp->cmd_q_count == 0) { - dev_notice(dev, "no command queues available\n"); - ret = -EIO; - goto e_pool; - } - dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count); - - /* Disable and clear interrupts until ready */ - iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG); - for (i = 0; i < ccp->cmd_q_count; i++) { - cmd_q = &ccp->cmd_q[i]; - - ioread32(cmd_q->reg_int_status); - ioread32(cmd_q->reg_status); - } - iowrite32(qim, ccp->io_regs + IRQ_STATUS_REG); - - /* Request an irq */ - ret = ccp->get_irq(ccp); - if (ret) { - dev_err(dev, "unable to allocate an IRQ\n"); - goto e_pool; - } - - /* Initialize the queues used to wait for KSB space and suspend */ - init_waitqueue_head(&ccp->ksb_queue); - init_waitqueue_head(&ccp->suspend_queue); - - /* Create a kthread for each queue */ - for (i = 0; i < ccp->cmd_q_count; i++) { - struct task_struct *kthread; - - cmd_q = &ccp->cmd_q[i]; - - kthread = kthread_create(ccp_cmd_queue_thread, cmd_q, - "ccp-q%u", cmd_q->id); - if (IS_ERR(kthread)) { - dev_err(dev, "error creating queue thread (%ld)\n", - PTR_ERR(kthread)); - ret = PTR_ERR(kthread); - goto e_kthread; - } - - cmd_q->kthread = kthread; - wake_up_process(kthread); - } - - /* Register the RNG */ - ccp->hwrng.name = "ccp-rng"; - ccp->hwrng.read = ccp_trng_read; - ret = hwrng_register(&ccp->hwrng); - if (ret) { - dev_err(dev, "error registering hwrng (%d)\n", ret); - goto e_kthread; - } - - /* Make the device struct available before enabling interrupts */ - ccp_add_device(ccp); - - /* Enable interrupts */ - iowrite32(qim, ccp->io_regs + IRQ_MASK_REG); - - return 0; - -e_kthread: - for (i = 0; i < ccp->cmd_q_count; i++) - if (ccp->cmd_q[i].kthread) - kthread_stop(ccp->cmd_q[i].kthread); - - ccp->free_irq(ccp); - -e_pool: - for (i = 0; i < ccp->cmd_q_count; i++) - dma_pool_destroy(ccp->cmd_q[i].dma_pool); - - return ret; -} - -/** - * ccp_destroy - tear down the CCP device - * - * @ccp: ccp_device struct - */ -void ccp_destroy(struct ccp_device *ccp) -{ - struct ccp_cmd_queue *cmd_q; - struct ccp_cmd *cmd; - unsigned int qim, i; - - /* Remove general access to the device struct */ - ccp_del_device(ccp); - - /* Unregister the RNG */ - hwrng_unregister(&ccp->hwrng); - - /* Stop the queue kthreads */ - for (i = 0; i < ccp->cmd_q_count; i++) - if (ccp->cmd_q[i].kthread) - kthread_stop(ccp->cmd_q[i].kthread); - - /* Build queue interrupt mask (two interrupt masks per queue) */ - qim = 0; - for (i = 0; i < ccp->cmd_q_count; i++) { - cmd_q = &ccp->cmd_q[i]; - qim |= cmd_q->int_ok | cmd_q->int_err; - } - - /* Disable and clear interrupts */ - iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG); - for (i = 0; i < ccp->cmd_q_count; i++) { - cmd_q = &ccp->cmd_q[i]; - - ioread32(cmd_q->reg_int_status); - ioread32(cmd_q->reg_status); - } - iowrite32(qim, ccp->io_regs + IRQ_STATUS_REG); - - ccp->free_irq(ccp); - - for (i = 0; i < ccp->cmd_q_count; i++) - dma_pool_destroy(ccp->cmd_q[i].dma_pool); - - /* Flush the cmd and backlog queue */ - while (!list_empty(&ccp->cmd)) { - /* Invoke the callback directly with an error code */ - cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); - list_del(&cmd->entry); - cmd->callback(cmd->data, -ENODEV); - } - while (!list_empty(&ccp->backlog)) { - /* Invoke the callback directly with an error code */ - cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry); - list_del(&cmd->entry); - cmd->callback(cmd->data, -ENODEV); - } -} - -/** - * ccp_irq_handler - handle interrupts generated by the CCP device - * - * @irq: the irq associated with the interrupt - * @data: the data value supplied when the irq was created - */ -irqreturn_t ccp_irq_handler(int irq, void *data) -{ - struct device *dev = data; - struct ccp_device *ccp = dev_get_drvdata(dev); - struct ccp_cmd_queue *cmd_q; - u32 q_int, status; - unsigned int i; - - status = ioread32(ccp->io_regs + IRQ_STATUS_REG); - - for (i = 0; i < ccp->cmd_q_count; i++) { - cmd_q = &ccp->cmd_q[i]; - - q_int = status & (cmd_q->int_ok | cmd_q->int_err); - if (q_int) { - cmd_q->int_status = status; - cmd_q->q_status = ioread32(cmd_q->reg_status); - cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); - - /* On error, only save the first error value */ - if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error) - cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); - - cmd_q->int_rcvd = 1; - - /* Acknowledge the interrupt and wake the kthread */ - iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG); - wake_up_interruptible(&cmd_q->int_queue); - } - } - - return IRQ_HANDLED; + return ccp; } #ifdef CONFIG_PM @@ -577,41 +427,22 @@ bool ccp_queues_suspended(struct ccp_device *ccp) } #endif -#ifdef CONFIG_X86 -static const struct x86_cpu_id ccp_support[] = { - { X86_VENDOR_AMD, 22, }, - { }, -}; -#endif - static int __init ccp_mod_init(void) { #ifdef CONFIG_X86 - struct cpuinfo_x86 *cpuinfo = &boot_cpu_data; int ret; - if (!x86_match_cpu(ccp_support)) - return -ENODEV; - - switch (cpuinfo->x86) { - case 22: - if ((cpuinfo->x86_model < 48) || (cpuinfo->x86_model > 63)) - return -ENODEV; - - ret = ccp_pci_init(); - if (ret) - return ret; - - /* Don't leave the driver loaded if init failed */ - if (!ccp_get_device()) { - ccp_pci_exit(); - return -ENODEV; - } - - return 0; + ret = ccp_pci_init(); + if (ret) + return ret; - break; + /* Don't leave the driver loaded if init failed */ + if (ccp_present() != 0) { + ccp_pci_exit(); + return -ENODEV; } + + return 0; #endif #ifdef CONFIG_ARM64 @@ -622,7 +453,7 @@ static int __init ccp_mod_init(void) return ret; /* Don't leave the driver loaded if init failed */ - if (!ccp_get_device()) { + if (ccp_present() != 0) { ccp_platform_exit(); return -ENODEV; } @@ -636,13 +467,7 @@ static int __init ccp_mod_init(void) static void __exit ccp_mod_exit(void) { #ifdef CONFIG_X86 - struct cpuinfo_x86 *cpuinfo = &boot_cpu_data; - - switch (cpuinfo->x86) { - case 22: - ccp_pci_exit(); - break; - } + ccp_pci_exit(); #endif #ifdef CONFIG_ARM64 diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h index 6ff89031f..7745d0be4 100644 --- a/drivers/crypto/ccp/ccp-dev.h +++ b/drivers/crypto/ccp/ccp-dev.h @@ -1,7 +1,7 @@ /* * AMD Cryptographic Coprocessor (CCP) driver * - * Copyright (C) 2013 Advanced Micro Devices, Inc. + * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. * * Author: Tom Lendacky <thomas.lendacky@amd.com> * @@ -23,6 +23,7 @@ #include <linux/hw_random.h> #include <linux/bitops.h> +#define MAX_CCP_NAME_LEN 16 #define MAX_DMAPOOL_NAME_LEN 32 #define MAX_HW_QUEUES 5 @@ -140,6 +141,29 @@ #define CCP_ECC_RESULT_OFFSET 60 #define CCP_ECC_RESULT_SUCCESS 0x0001 +struct ccp_op; + +/* Structure for computation functions that are device-specific */ +struct ccp_actions { + int (*perform_aes)(struct ccp_op *); + int (*perform_xts_aes)(struct ccp_op *); + int (*perform_sha)(struct ccp_op *); + int (*perform_rsa)(struct ccp_op *); + int (*perform_passthru)(struct ccp_op *); + int (*perform_ecc)(struct ccp_op *); + int (*init)(struct ccp_device *); + void (*destroy)(struct ccp_device *); + irqreturn_t (*irqhandler)(int, void *); +}; + +/* Structure to hold CCP version-specific values */ +struct ccp_vdata { + unsigned int version; + struct ccp_actions *perform; +}; + +extern struct ccp_vdata ccpv3; + struct ccp_device; struct ccp_cmd; @@ -184,6 +208,13 @@ struct ccp_cmd_queue { } ____cacheline_aligned; struct ccp_device { + struct list_head entry; + + struct ccp_vdata *vdata; + unsigned int ord; + char name[MAX_CCP_NAME_LEN]; + char rngname[MAX_CCP_NAME_LEN]; + struct device *dev; /* @@ -258,18 +289,132 @@ struct ccp_device { unsigned int axcache; }; +enum ccp_memtype { + CCP_MEMTYPE_SYSTEM = 0, + CCP_MEMTYPE_KSB, + CCP_MEMTYPE_LOCAL, + CCP_MEMTYPE__LAST, +}; + +struct ccp_dma_info { + dma_addr_t address; + unsigned int offset; + unsigned int length; + enum dma_data_direction dir; +}; + +struct ccp_dm_workarea { + struct device *dev; + struct dma_pool *dma_pool; + unsigned int length; + + u8 *address; + struct ccp_dma_info dma; +}; + +struct ccp_sg_workarea { + struct scatterlist *sg; + int nents; + + struct scatterlist *dma_sg; + struct device *dma_dev; + unsigned int dma_count; + enum dma_data_direction dma_dir; + + unsigned int sg_used; + + u64 bytes_left; +}; + +struct ccp_data { + struct ccp_sg_workarea sg_wa; + struct ccp_dm_workarea dm_wa; +}; + +struct ccp_mem { + enum ccp_memtype type; + union { + struct ccp_dma_info dma; + u32 ksb; + } u; +}; + +struct ccp_aes_op { + enum ccp_aes_type type; + enum ccp_aes_mode mode; + enum ccp_aes_action action; +}; + +struct ccp_xts_aes_op { + enum ccp_aes_action action; + enum ccp_xts_aes_unit_size unit_size; +}; + +struct ccp_sha_op { + enum ccp_sha_type type; + u64 msg_bits; +}; + +struct ccp_rsa_op { + u32 mod_size; + u32 input_len; +}; + +struct ccp_passthru_op { + enum ccp_passthru_bitwise bit_mod; + enum ccp_passthru_byteswap byte_swap; +}; + +struct ccp_ecc_op { + enum ccp_ecc_function function; +}; + +struct ccp_op { + struct ccp_cmd_queue *cmd_q; + + u32 jobid; + u32 ioc; + u32 soc; + u32 ksb_key; + u32 ksb_ctx; + u32 init; + u32 eom; + + struct ccp_mem src; + struct ccp_mem dst; + + union { + struct ccp_aes_op aes; + struct ccp_xts_aes_op xts; + struct ccp_sha_op sha; + struct ccp_rsa_op rsa; + struct ccp_passthru_op passthru; + struct ccp_ecc_op ecc; + } u; +}; + +static inline u32 ccp_addr_lo(struct ccp_dma_info *info) +{ + return lower_32_bits(info->address + info->offset); +} + +static inline u32 ccp_addr_hi(struct ccp_dma_info *info) +{ + return upper_32_bits(info->address + info->offset) & 0x0000ffff; +} + int ccp_pci_init(void); void ccp_pci_exit(void); int ccp_platform_init(void); void ccp_platform_exit(void); +void ccp_add_device(struct ccp_device *ccp); +void ccp_del_device(struct ccp_device *ccp); + struct ccp_device *ccp_alloc_struct(struct device *dev); -int ccp_init(struct ccp_device *ccp); -void ccp_destroy(struct ccp_device *ccp); bool ccp_queues_suspended(struct ccp_device *ccp); - -irqreturn_t ccp_irq_handler(int irq, void *data); +int ccp_cmd_queue_thread(void *data); int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd); diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c index 6613aee79..eefdf595f 100644 --- a/drivers/crypto/ccp/ccp-ops.c +++ b/drivers/crypto/ccp/ccp-ops.c @@ -1,7 +1,7 @@ /* * AMD Cryptographic Coprocessor (CCP) driver * - * Copyright (C) 2013 Advanced Micro Devices, Inc. + * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. * * Author: Tom Lendacky <thomas.lendacky@amd.com> * @@ -13,124 +13,12 @@ #include <linux/module.h> #include <linux/kernel.h> #include <linux/pci.h> -#include <linux/pci_ids.h> -#include <linux/kthread.h> -#include <linux/sched.h> #include <linux/interrupt.h> -#include <linux/spinlock.h> -#include <linux/mutex.h> -#include <linux/delay.h> -#include <linux/ccp.h> -#include <linux/scatterlist.h> #include <crypto/scatterwalk.h> -#include <crypto/sha.h> +#include <linux/ccp.h> #include "ccp-dev.h" -enum ccp_memtype { - CCP_MEMTYPE_SYSTEM = 0, - CCP_MEMTYPE_KSB, - CCP_MEMTYPE_LOCAL, - CCP_MEMTYPE__LAST, -}; - -struct ccp_dma_info { - dma_addr_t address; - unsigned int offset; - unsigned int length; - enum dma_data_direction dir; -}; - -struct ccp_dm_workarea { - struct device *dev; - struct dma_pool *dma_pool; - unsigned int length; - - u8 *address; - struct ccp_dma_info dma; -}; - -struct ccp_sg_workarea { - struct scatterlist *sg; - int nents; - - struct scatterlist *dma_sg; - struct device *dma_dev; - unsigned int dma_count; - enum dma_data_direction dma_dir; - - unsigned int sg_used; - - u64 bytes_left; -}; - -struct ccp_data { - struct ccp_sg_workarea sg_wa; - struct ccp_dm_workarea dm_wa; -}; - -struct ccp_mem { - enum ccp_memtype type; - union { - struct ccp_dma_info dma; - u32 ksb; - } u; -}; - -struct ccp_aes_op { - enum ccp_aes_type type; - enum ccp_aes_mode mode; - enum ccp_aes_action action; -}; - -struct ccp_xts_aes_op { - enum ccp_aes_action action; - enum ccp_xts_aes_unit_size unit_size; -}; - -struct ccp_sha_op { - enum ccp_sha_type type; - u64 msg_bits; -}; - -struct ccp_rsa_op { - u32 mod_size; - u32 input_len; -}; - -struct ccp_passthru_op { - enum ccp_passthru_bitwise bit_mod; - enum ccp_passthru_byteswap byte_swap; -}; - -struct ccp_ecc_op { - enum ccp_ecc_function function; -}; - -struct ccp_op { - struct ccp_cmd_queue *cmd_q; - - u32 jobid; - u32 ioc; - u32 soc; - u32 ksb_key; - u32 ksb_ctx; - u32 init; - u32 eom; - - struct ccp_mem src; - struct ccp_mem dst; - - union { - struct ccp_aes_op aes; - struct ccp_xts_aes_op xts; - struct ccp_sha_op sha; - struct ccp_rsa_op rsa; - struct ccp_passthru_op passthru; - struct ccp_ecc_op ecc; - } u; -}; - /* SHA initial context values */ static const __be32 ccp_sha1_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = { cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1), @@ -152,253 +40,6 @@ static const __be32 ccp_sha256_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = { cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7), }; -static u32 ccp_addr_lo(struct ccp_dma_info *info) -{ - return lower_32_bits(info->address + info->offset); -} - -static u32 ccp_addr_hi(struct ccp_dma_info *info) -{ - return upper_32_bits(info->address + info->offset) & 0x0000ffff; -} - -static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count) -{ - struct ccp_cmd_queue *cmd_q = op->cmd_q; - struct ccp_device *ccp = cmd_q->ccp; - void __iomem *cr_addr; - u32 cr0, cmd; - unsigned int i; - int ret = 0; - - /* We could read a status register to see how many free slots - * are actually available, but reading that register resets it - * and you could lose some error information. - */ - cmd_q->free_slots--; - - cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT) - | (op->jobid << REQ0_JOBID_SHIFT) - | REQ0_WAIT_FOR_WRITE; - - if (op->soc) - cr0 |= REQ0_STOP_ON_COMPLETE - | REQ0_INT_ON_COMPLETE; - - if (op->ioc || !cmd_q->free_slots) - cr0 |= REQ0_INT_ON_COMPLETE; - - /* Start at CMD_REQ1 */ - cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR; - - mutex_lock(&ccp->req_mutex); - - /* Write CMD_REQ1 through CMD_REQx first */ - for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR) - iowrite32(*(cr + i), cr_addr); - - /* Tell the CCP to start */ - wmb(); - iowrite32(cr0, ccp->io_regs + CMD_REQ0); - - mutex_unlock(&ccp->req_mutex); - - if (cr0 & REQ0_INT_ON_COMPLETE) { - /* Wait for the job to complete */ - ret = wait_event_interruptible(cmd_q->int_queue, - cmd_q->int_rcvd); - if (ret || cmd_q->cmd_error) { - /* On error delete all related jobs from the queue */ - cmd = (cmd_q->id << DEL_Q_ID_SHIFT) - | op->jobid; - - iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); - - if (!ret) - ret = -EIO; - } else if (op->soc) { - /* Delete just head job from the queue on SoC */ - cmd = DEL_Q_ACTIVE - | (cmd_q->id << DEL_Q_ID_SHIFT) - | op->jobid; - - iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); - } - - cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status); - - cmd_q->int_rcvd = 0; - } - - return ret; -} - -static int ccp_perform_aes(struct ccp_op *op) -{ - u32 cr[6]; - - /* Fill out the register contents for REQ1 through REQ6 */ - cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT) - | (op->u.aes.type << REQ1_AES_TYPE_SHIFT) - | (op->u.aes.mode << REQ1_AES_MODE_SHIFT) - | (op->u.aes.action << REQ1_AES_ACTION_SHIFT) - | (op->ksb_key << REQ1_KEY_KSB_SHIFT); - cr[1] = op->src.u.dma.length - 1; - cr[2] = ccp_addr_lo(&op->src.u.dma); - cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) - | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) - | ccp_addr_hi(&op->src.u.dma); - cr[4] = ccp_addr_lo(&op->dst.u.dma); - cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) - | ccp_addr_hi(&op->dst.u.dma); - - if (op->u.aes.mode == CCP_AES_MODE_CFB) - cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT); - - if (op->eom) - cr[0] |= REQ1_EOM; - - if (op->init) - cr[0] |= REQ1_INIT; - - return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); -} - -static int ccp_perform_xts_aes(struct ccp_op *op) -{ - u32 cr[6]; - - /* Fill out the register contents for REQ1 through REQ6 */ - cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT) - | (op->u.xts.action << REQ1_AES_ACTION_SHIFT) - | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT) - | (op->ksb_key << REQ1_KEY_KSB_SHIFT); - cr[1] = op->src.u.dma.length - 1; - cr[2] = ccp_addr_lo(&op->src.u.dma); - cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) - | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) - | ccp_addr_hi(&op->src.u.dma); - cr[4] = ccp_addr_lo(&op->dst.u.dma); - cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) - | ccp_addr_hi(&op->dst.u.dma); - - if (op->eom) - cr[0] |= REQ1_EOM; - - if (op->init) - cr[0] |= REQ1_INIT; - - return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); -} - -static int ccp_perform_sha(struct ccp_op *op) -{ - u32 cr[6]; - - /* Fill out the register contents for REQ1 through REQ6 */ - cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT) - | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT) - | REQ1_INIT; - cr[1] = op->src.u.dma.length - 1; - cr[2] = ccp_addr_lo(&op->src.u.dma); - cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) - | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) - | ccp_addr_hi(&op->src.u.dma); - - if (op->eom) { - cr[0] |= REQ1_EOM; - cr[4] = lower_32_bits(op->u.sha.msg_bits); - cr[5] = upper_32_bits(op->u.sha.msg_bits); - } else { - cr[4] = 0; - cr[5] = 0; - } - - return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); -} - -static int ccp_perform_rsa(struct ccp_op *op) -{ - u32 cr[6]; - - /* Fill out the register contents for REQ1 through REQ6 */ - cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT) - | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT) - | (op->ksb_key << REQ1_KEY_KSB_SHIFT) - | REQ1_EOM; - cr[1] = op->u.rsa.input_len - 1; - cr[2] = ccp_addr_lo(&op->src.u.dma); - cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) - | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) - | ccp_addr_hi(&op->src.u.dma); - cr[4] = ccp_addr_lo(&op->dst.u.dma); - cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) - | ccp_addr_hi(&op->dst.u.dma); - - return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); -} - -static int ccp_perform_passthru(struct ccp_op *op) -{ - u32 cr[6]; - - /* Fill out the register contents for REQ1 through REQ6 */ - cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT) - | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT) - | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT); - - if (op->src.type == CCP_MEMTYPE_SYSTEM) - cr[1] = op->src.u.dma.length - 1; - else - cr[1] = op->dst.u.dma.length - 1; - - if (op->src.type == CCP_MEMTYPE_SYSTEM) { - cr[2] = ccp_addr_lo(&op->src.u.dma); - cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) - | ccp_addr_hi(&op->src.u.dma); - - if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP) - cr[3] |= (op->ksb_key << REQ4_KSB_SHIFT); - } else { - cr[2] = op->src.u.ksb * CCP_KSB_BYTES; - cr[3] = (CCP_MEMTYPE_KSB << REQ4_MEMTYPE_SHIFT); - } - - if (op->dst.type == CCP_MEMTYPE_SYSTEM) { - cr[4] = ccp_addr_lo(&op->dst.u.dma); - cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) - | ccp_addr_hi(&op->dst.u.dma); - } else { - cr[4] = op->dst.u.ksb * CCP_KSB_BYTES; - cr[5] = (CCP_MEMTYPE_KSB << REQ6_MEMTYPE_SHIFT); - } - - if (op->eom) - cr[0] |= REQ1_EOM; - - return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); -} - -static int ccp_perform_ecc(struct ccp_op *op) -{ - u32 cr[6]; - - /* Fill out the register contents for REQ1 through REQ6 */ - cr[0] = REQ1_ECC_AFFINE_CONVERT - | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT) - | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT) - | REQ1_EOM; - cr[1] = op->src.u.dma.length - 1; - cr[2] = ccp_addr_lo(&op->src.u.dma); - cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) - | ccp_addr_hi(&op->src.u.dma); - cr[4] = ccp_addr_lo(&op->dst.u.dma); - cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) - | ccp_addr_hi(&op->dst.u.dma); - - return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); -} - static u32 ccp_alloc_ksb(struct ccp_device *ccp, unsigned int count) { int start; @@ -837,7 +478,7 @@ static int ccp_copy_to_from_ksb(struct ccp_cmd_queue *cmd_q, op.u.passthru.byte_swap = byte_swap; - return ccp_perform_passthru(&op); + return cmd_q->ccp->vdata->perform->perform_passthru(&op); } static int ccp_copy_to_ksb(struct ccp_cmd_queue *cmd_q, @@ -969,7 +610,7 @@ static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, } } - ret = ccp_perform_aes(&op); + ret = cmd_q->ccp->vdata->perform->perform_aes(&op); if (ret) { cmd->engine_error = cmd_q->cmd_error; goto e_src; @@ -1131,7 +772,7 @@ static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) op.soc = 1; } - ret = ccp_perform_aes(&op); + ret = cmd_q->ccp->vdata->perform->perform_aes(&op); if (ret) { cmd->engine_error = cmd_q->cmd_error; goto e_dst; @@ -1296,7 +937,7 @@ static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, if (!src.sg_wa.bytes_left) op.eom = 1; - ret = ccp_perform_xts_aes(&op); + ret = cmd_q->ccp->vdata->perform->perform_xts_aes(&op); if (ret) { cmd->engine_error = cmd_q->cmd_error; goto e_dst; @@ -1453,7 +1094,7 @@ static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) if (sha->final && !src.sg_wa.bytes_left) op.eom = 1; - ret = ccp_perform_sha(&op); + ret = cmd_q->ccp->vdata->perform->perform_sha(&op); if (ret) { cmd->engine_error = cmd_q->cmd_error; goto e_data; @@ -1633,7 +1274,7 @@ static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) op.u.rsa.mod_size = rsa->key_size; op.u.rsa.input_len = i_len; - ret = ccp_perform_rsa(&op); + ret = cmd_q->ccp->vdata->perform->perform_rsa(&op); if (ret) { cmd->engine_error = cmd_q->cmd_error; goto e_dst; @@ -1758,7 +1399,7 @@ static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, op.dst.u.dma.offset = dst.sg_wa.sg_used; op.dst.u.dma.length = op.src.u.dma.length; - ret = ccp_perform_passthru(&op); + ret = cmd_q->ccp->vdata->perform->perform_passthru(&op); if (ret) { cmd->engine_error = cmd_q->cmd_error; goto e_dst; @@ -1870,7 +1511,7 @@ static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) op.u.ecc.function = cmd->u.ecc.function; - ret = ccp_perform_ecc(&op); + ret = cmd_q->ccp->vdata->perform->perform_ecc(&op); if (ret) { cmd->engine_error = cmd_q->cmd_error; goto e_dst; @@ -2034,7 +1675,7 @@ static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) op.u.ecc.function = cmd->u.ecc.function; - ret = ccp_perform_ecc(&op); + ret = cmd_q->ccp->vdata->perform->perform_ecc(&op); if (ret) { cmd->engine_error = cmd_q->cmd_error; goto e_dst; diff --git a/drivers/crypto/ccp/ccp-pci.c b/drivers/crypto/ccp/ccp-pci.c index 7690467c4..0bf262e36 100644 --- a/drivers/crypto/ccp/ccp-pci.c +++ b/drivers/crypto/ccp/ccp-pci.c @@ -1,7 +1,7 @@ /* * AMD Cryptographic Coprocessor (CCP) driver * - * Copyright (C) 2013 Advanced Micro Devices, Inc. + * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. * * Author: Tom Lendacky <thomas.lendacky@amd.com> * @@ -59,9 +59,11 @@ static int ccp_get_msix_irqs(struct ccp_device *ccp) ccp_pci->msix_count = ret; for (v = 0; v < ccp_pci->msix_count; v++) { /* Set the interrupt names and request the irqs */ - snprintf(ccp_pci->msix[v].name, name_len, "ccp-%u", v); + snprintf(ccp_pci->msix[v].name, name_len, "%s-%u", + ccp->name, v); ccp_pci->msix[v].vector = msix_entry[v].vector; - ret = request_irq(ccp_pci->msix[v].vector, ccp_irq_handler, + ret = request_irq(ccp_pci->msix[v].vector, + ccp->vdata->perform->irqhandler, 0, ccp_pci->msix[v].name, dev); if (ret) { dev_notice(dev, "unable to allocate MSI-X IRQ (%d)\n", @@ -94,7 +96,8 @@ static int ccp_get_msi_irq(struct ccp_device *ccp) return ret; ccp->irq = pdev->irq; - ret = request_irq(ccp->irq, ccp_irq_handler, 0, "ccp", dev); + ret = request_irq(ccp->irq, ccp->vdata->perform->irqhandler, 0, + ccp->name, dev); if (ret) { dev_notice(dev, "unable to allocate MSI IRQ (%d)\n", ret); goto e_msi; @@ -179,6 +182,12 @@ static int ccp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto e_err; ccp->dev_specific = ccp_pci; + ccp->vdata = (struct ccp_vdata *)id->driver_data; + if (!ccp->vdata || !ccp->vdata->version) { + ret = -ENODEV; + dev_err(dev, "missing driver data\n"); + goto e_err; + } ccp->get_irq = ccp_get_irqs; ccp->free_irq = ccp_free_irqs; @@ -221,7 +230,7 @@ static int ccp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) dev_set_drvdata(dev, ccp); - ret = ccp_init(ccp); + ret = ccp->vdata->perform->init(ccp); if (ret) goto e_iomap; @@ -251,7 +260,7 @@ static void ccp_pci_remove(struct pci_dev *pdev) if (!ccp) return; - ccp_destroy(ccp); + ccp->vdata->perform->destroy(ccp); pci_iounmap(pdev, ccp->io_map); @@ -312,7 +321,7 @@ static int ccp_pci_resume(struct pci_dev *pdev) #endif static const struct pci_device_id ccp_pci_table[] = { - { PCI_VDEVICE(AMD, 0x1537), }, + { PCI_VDEVICE(AMD, 0x1537), (kernel_ulong_t)&ccpv3 }, /* Last entry must be zero */ { 0, } }; diff --git a/drivers/crypto/ccp/ccp-platform.c b/drivers/crypto/ccp/ccp-platform.c index 66dd7c9d0..351f28d8c 100644 --- a/drivers/crypto/ccp/ccp-platform.c +++ b/drivers/crypto/ccp/ccp-platform.c @@ -1,7 +1,7 @@ /* * AMD Cryptographic Coprocessor (CCP) driver * - * Copyright (C) 2014 Advanced Micro Devices, Inc. + * Copyright (C) 2014,2016 Advanced Micro Devices, Inc. * * Author: Tom Lendacky <thomas.lendacky@amd.com> * @@ -32,6 +32,33 @@ struct ccp_platform { int coherent; }; +static const struct acpi_device_id ccp_acpi_match[]; +static const struct of_device_id ccp_of_match[]; + +static struct ccp_vdata *ccp_get_of_version(struct platform_device *pdev) +{ +#ifdef CONFIG_OF + const struct of_device_id *match; + + match = of_match_node(ccp_of_match, pdev->dev.of_node); + if (match && match->data) + return (struct ccp_vdata *)match->data; +#endif + return 0; +} + +static struct ccp_vdata *ccp_get_acpi_version(struct platform_device *pdev) +{ +#ifdef CONFIG_ACPI + const struct acpi_device_id *match; + + match = acpi_match_device(ccp_acpi_match, &pdev->dev); + if (match && match->driver_data) + return (struct ccp_vdata *)match->driver_data; +#endif + return 0; +} + static int ccp_get_irq(struct ccp_device *ccp) { struct device *dev = ccp->dev; @@ -43,7 +70,8 @@ static int ccp_get_irq(struct ccp_device *ccp) return ret; ccp->irq = ret; - ret = request_irq(ccp->irq, ccp_irq_handler, 0, "ccp", dev); + ret = request_irq(ccp->irq, ccp->vdata->perform->irqhandler, 0, + ccp->name, dev); if (ret) { dev_notice(dev, "unable to allocate IRQ (%d)\n", ret); return ret; @@ -106,6 +134,13 @@ static int ccp_platform_probe(struct platform_device *pdev) goto e_err; ccp->dev_specific = ccp_platform; + ccp->vdata = pdev->dev.of_node ? ccp_get_of_version(pdev) + : ccp_get_acpi_version(pdev); + if (!ccp->vdata || !ccp->vdata->version) { + ret = -ENODEV; + dev_err(dev, "missing driver data\n"); + goto e_err; + } ccp->get_irq = ccp_get_irqs; ccp->free_irq = ccp_free_irqs; @@ -137,7 +172,7 @@ static int ccp_platform_probe(struct platform_device *pdev) dev_set_drvdata(dev, ccp); - ret = ccp_init(ccp); + ret = ccp->vdata->perform->init(ccp); if (ret) goto e_err; @@ -155,7 +190,7 @@ static int ccp_platform_remove(struct platform_device *pdev) struct device *dev = &pdev->dev; struct ccp_device *ccp = dev_get_drvdata(dev); - ccp_destroy(ccp); + ccp->vdata->perform->destroy(ccp); dev_notice(dev, "disabled\n"); @@ -214,7 +249,7 @@ static int ccp_platform_resume(struct platform_device *pdev) #ifdef CONFIG_ACPI static const struct acpi_device_id ccp_acpi_match[] = { - { "AMDI0C00", 0 }, + { "AMDI0C00", (kernel_ulong_t)&ccpv3 }, { }, }; MODULE_DEVICE_TABLE(acpi, ccp_acpi_match); @@ -222,7 +257,8 @@ MODULE_DEVICE_TABLE(acpi, ccp_acpi_match); #ifdef CONFIG_OF static const struct of_device_id ccp_of_match[] = { - { .compatible = "amd,ccp-seattle-v1a" }, + { .compatible = "amd,ccp-seattle-v1a", + .data = (const void *)&ccpv3 }, { }, }; MODULE_DEVICE_TABLE(of, ccp_of_match); diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c index e52496a17..229693445 100644 --- a/drivers/crypto/ixp4xx_crypto.c +++ b/drivers/crypto/ixp4xx_crypto.c @@ -1031,6 +1031,18 @@ static int aead_perform(struct aead_request *req, int encrypt, BUG_ON(ivsize && !req->iv); memcpy(crypt->iv, req->iv, ivsize); + buf = chainup_buffers(dev, req->src, crypt->auth_len, + &src_hook, flags, src_direction); + req_ctx->src = src_hook.next; + crypt->src_buf = src_hook.phys_next; + if (!buf) + goto free_buf_src; + + lastlen = buf->buf_len; + if (lastlen >= authsize) + crypt->icv_rev_aes = buf->phys_addr + + buf->buf_len - authsize; + req_ctx->dst = NULL; if (req->src != req->dst) { @@ -1055,20 +1067,6 @@ static int aead_perform(struct aead_request *req, int encrypt, } } - buf = chainup_buffers(dev, req->src, crypt->auth_len, - &src_hook, flags, src_direction); - req_ctx->src = src_hook.next; - crypt->src_buf = src_hook.phys_next; - if (!buf) - goto free_buf_src; - - if (!encrypt || !req_ctx->dst) { - lastlen = buf->buf_len; - if (lastlen >= authsize) - crypt->icv_rev_aes = buf->phys_addr + - buf->buf_len - authsize; - } - if (unlikely(lastlen < authsize)) { /* The 12 hmac bytes are scattered, * we need to copy them into a safe buffer */ diff --git a/drivers/crypto/marvell/cesa.h b/drivers/crypto/marvell/cesa.h index bd985e725..74071e45a 100644 --- a/drivers/crypto/marvell/cesa.h +++ b/drivers/crypto/marvell/cesa.h @@ -588,6 +588,7 @@ struct mv_cesa_ahash_dma_req { struct mv_cesa_tdma_req base; u8 *padding; dma_addr_t padding_dma; + u8 *cache; dma_addr_t cache_dma; }; @@ -609,7 +610,7 @@ struct mv_cesa_ahash_req { struct mv_cesa_ahash_std_req std; } req; struct mv_cesa_op_ctx op_tmpl; - u8 *cache; + u8 cache[CESA_MAX_HASH_BLOCK_SIZE]; unsigned int cache_ptr; u64 len; int src_nents; diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index 683cca9ac..7ca2e0f9d 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -45,69 +45,25 @@ mv_cesa_ahash_req_iter_next_op(struct mv_cesa_ahash_dma_iter *iter) return mv_cesa_req_dma_iter_next_op(&iter->base); } -static inline int mv_cesa_ahash_dma_alloc_cache(struct mv_cesa_ahash_req *creq, - gfp_t flags) +static inline int +mv_cesa_ahash_dma_alloc_cache(struct mv_cesa_ahash_dma_req *req, gfp_t flags) { - struct mv_cesa_ahash_dma_req *dreq = &creq->req.dma; - - creq->cache = dma_pool_alloc(cesa_dev->dma->cache_pool, flags, - &dreq->cache_dma); - if (!creq->cache) - return -ENOMEM; - - return 0; -} - -static inline int mv_cesa_ahash_std_alloc_cache(struct mv_cesa_ahash_req *creq, - gfp_t flags) -{ - creq->cache = kzalloc(CESA_MAX_HASH_BLOCK_SIZE, flags); - if (!creq->cache) + req->cache = dma_pool_alloc(cesa_dev->dma->cache_pool, flags, + &req->cache_dma); + if (!req->cache) return -ENOMEM; return 0; } -static int mv_cesa_ahash_alloc_cache(struct ahash_request *req) -{ - struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); - gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? - GFP_KERNEL : GFP_ATOMIC; - int ret; - - if (creq->cache) - return 0; - - if (creq->req.base.type == CESA_DMA_REQ) - ret = mv_cesa_ahash_dma_alloc_cache(creq, flags); - else - ret = mv_cesa_ahash_std_alloc_cache(creq, flags); - - return ret; -} - -static inline void mv_cesa_ahash_dma_free_cache(struct mv_cesa_ahash_req *creq) -{ - dma_pool_free(cesa_dev->dma->cache_pool, creq->cache, - creq->req.dma.cache_dma); -} - -static inline void mv_cesa_ahash_std_free_cache(struct mv_cesa_ahash_req *creq) -{ - kfree(creq->cache); -} - -static void mv_cesa_ahash_free_cache(struct mv_cesa_ahash_req *creq) +static inline void +mv_cesa_ahash_dma_free_cache(struct mv_cesa_ahash_dma_req *req) { - if (!creq->cache) + if (!req->cache) return; - if (creq->req.base.type == CESA_DMA_REQ) - mv_cesa_ahash_dma_free_cache(creq); - else - mv_cesa_ahash_std_free_cache(creq); - - creq->cache = NULL; + dma_pool_free(cesa_dev->dma->cache_pool, req->cache, + req->cache_dma); } static int mv_cesa_ahash_dma_alloc_padding(struct mv_cesa_ahash_dma_req *req, @@ -146,6 +102,7 @@ static inline void mv_cesa_ahash_dma_cleanup(struct ahash_request *req) struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); dma_unmap_sg(cesa_dev->dev, req->src, creq->src_nents, DMA_TO_DEVICE); + mv_cesa_ahash_dma_free_cache(&creq->req.dma); mv_cesa_dma_cleanup(&creq->req.dma.base); } @@ -161,8 +118,6 @@ static void mv_cesa_ahash_last_cleanup(struct ahash_request *req) { struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); - mv_cesa_ahash_free_cache(creq); - if (creq->req.base.type == CESA_DMA_REQ) mv_cesa_ahash_dma_last_cleanup(req); } @@ -445,14 +400,6 @@ static inline int mv_cesa_ahash_cra_init(struct crypto_tfm *tfm) static int mv_cesa_ahash_cache_req(struct ahash_request *req, bool *cached) { struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); - int ret; - - if (((creq->cache_ptr + req->nbytes) & CESA_HASH_BLOCK_SIZE_MSK) && - !creq->last_req) { - ret = mv_cesa_ahash_alloc_cache(req); - if (ret) - return ret; - } if (creq->cache_ptr + req->nbytes < 64 && !creq->last_req) { *cached = true; @@ -505,10 +452,17 @@ mv_cesa_ahash_dma_add_cache(struct mv_cesa_tdma_chain *chain, gfp_t flags) { struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma; + int ret; if (!creq->cache_ptr) return 0; + ret = mv_cesa_ahash_dma_alloc_cache(ahashdreq, flags); + if (ret) + return ret; + + memcpy(ahashdreq->cache, creq->cache, creq->cache_ptr); + return mv_cesa_dma_add_data_transfer(chain, CESA_SA_DATA_SRAM_OFFSET, ahashdreq->cache_dma, @@ -848,10 +802,6 @@ static int mv_cesa_ahash_import(struct ahash_request *req, const void *hash, if (!cache_ptr) return 0; - ret = mv_cesa_ahash_alloc_cache(req); - if (ret) - return ret; - memcpy(creq->cache, cache, cache_ptr); creq->cache_ptr = cache_ptr; @@ -860,9 +810,14 @@ static int mv_cesa_ahash_import(struct ahash_request *req, const void *hash, static int mv_cesa_md5_init(struct ahash_request *req) { + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); struct mv_cesa_op_ctx tmpl = { }; mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_MD5); + creq->state[0] = MD5_H0; + creq->state[1] = MD5_H1; + creq->state[2] = MD5_H2; + creq->state[3] = MD5_H3; mv_cesa_ahash_init(req, &tmpl, true); @@ -923,9 +878,15 @@ struct ahash_alg mv_md5_alg = { static int mv_cesa_sha1_init(struct ahash_request *req) { + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); struct mv_cesa_op_ctx tmpl = { }; mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_SHA1); + creq->state[0] = SHA1_H0; + creq->state[1] = SHA1_H1; + creq->state[2] = SHA1_H2; + creq->state[3] = SHA1_H3; + creq->state[4] = SHA1_H4; mv_cesa_ahash_init(req, &tmpl, false); @@ -986,9 +947,18 @@ struct ahash_alg mv_sha1_alg = { static int mv_cesa_sha256_init(struct ahash_request *req) { + struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); struct mv_cesa_op_ctx tmpl = { }; mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_SHA256); + creq->state[0] = SHA256_H0; + creq->state[1] = SHA256_H1; + creq->state[2] = SHA256_H2; + creq->state[3] = SHA256_H3; + creq->state[4] = SHA256_H4; + creq->state[5] = SHA256_H5; + creq->state[6] = SHA256_H6; + creq->state[7] = SHA256_H7; mv_cesa_ahash_init(req, &tmpl, false); diff --git a/drivers/crypto/nx/nx-842.c b/drivers/crypto/nx/nx-842.c index 046c1c454..d94e25df5 100644 --- a/drivers/crypto/nx/nx-842.c +++ b/drivers/crypto/nx/nx-842.c @@ -308,7 +308,7 @@ int nx842_crypto_compress(struct crypto_tfm *tfm, h = !n && add_header ? hdrsize : 0; if (ignore) - pr_warn("interal error, ignore is set %x\n", ignore); + pr_warn("internal error, ignore is set %x\n", ignore); ret = compress(ctx, &p, &hdr->group[n], &c, &ignore, h); if (ret) diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index dd355bd19..d420ec751 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -36,6 +36,7 @@ #include <linux/interrupt.h> #include <crypto/scatterwalk.h> #include <crypto/aes.h> +#include <crypto/algapi.h> #define DST_MAXBURST 4 #define DMA_MIN (DST_MAXBURST * sizeof(u32)) @@ -152,13 +153,10 @@ struct omap_aes_dev { unsigned long flags; int err; - spinlock_t lock; - struct crypto_queue queue; - struct tasklet_struct done_task; - struct tasklet_struct queue_task; struct ablkcipher_request *req; + struct crypto_engine *engine; /* * total is used by PIO mode for book keeping so introduce @@ -532,9 +530,7 @@ static void omap_aes_finish_req(struct omap_aes_dev *dd, int err) pr_debug("err: %d\n", err); - dd->flags &= ~FLAGS_BUSY; - - req->base.complete(&req->base, err); + crypto_finalize_request(dd->engine, req, err); } static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd) @@ -604,34 +600,25 @@ static int omap_aes_copy_sgs(struct omap_aes_dev *dd) } static int omap_aes_handle_queue(struct omap_aes_dev *dd, - struct ablkcipher_request *req) + struct ablkcipher_request *req) { - struct crypto_async_request *async_req, *backlog; - struct omap_aes_ctx *ctx; - struct omap_aes_reqctx *rctx; - unsigned long flags; - int err, ret = 0, len; - - spin_lock_irqsave(&dd->lock, flags); if (req) - ret = ablkcipher_enqueue_request(&dd->queue, req); - if (dd->flags & FLAGS_BUSY) { - spin_unlock_irqrestore(&dd->lock, flags); - return ret; - } - backlog = crypto_get_backlog(&dd->queue); - async_req = crypto_dequeue_request(&dd->queue); - if (async_req) - dd->flags |= FLAGS_BUSY; - spin_unlock_irqrestore(&dd->lock, flags); + return crypto_transfer_request_to_engine(dd->engine, req); - if (!async_req) - return ret; + return 0; +} - if (backlog) - backlog->complete(backlog, -EINPROGRESS); +static int omap_aes_prepare_req(struct crypto_engine *engine, + struct ablkcipher_request *req) +{ + struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx( + crypto_ablkcipher_reqtfm(req)); + struct omap_aes_dev *dd = omap_aes_find_dev(ctx); + struct omap_aes_reqctx *rctx; + int len; - req = ablkcipher_request_cast(async_req); + if (!dd) + return -ENODEV; /* assign new request to device */ dd->req = req; @@ -662,16 +649,20 @@ static int omap_aes_handle_queue(struct omap_aes_dev *dd, dd->ctx = ctx; ctx->dd = dd; - err = omap_aes_write_ctrl(dd); - if (!err) - err = omap_aes_crypt_dma_start(dd); - if (err) { - /* aes_task will not finish it, so do it here */ - omap_aes_finish_req(dd, err); - tasklet_schedule(&dd->queue_task); - } + return omap_aes_write_ctrl(dd); +} - return ret; /* return ret, which is enqueue return value */ +static int omap_aes_crypt_req(struct crypto_engine *engine, + struct ablkcipher_request *req) +{ + struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx( + crypto_ablkcipher_reqtfm(req)); + struct omap_aes_dev *dd = omap_aes_find_dev(ctx); + + if (!dd) + return -ENODEV; + + return omap_aes_crypt_dma_start(dd); } static void omap_aes_done_task(unsigned long data) @@ -704,18 +695,10 @@ static void omap_aes_done_task(unsigned long data) } omap_aes_finish_req(dd, 0); - omap_aes_handle_queue(dd, NULL); pr_debug("exit\n"); } -static void omap_aes_queue_task(unsigned long data) -{ - struct omap_aes_dev *dd = (struct omap_aes_dev *)data; - - omap_aes_handle_queue(dd, NULL); -} - static int omap_aes_crypt(struct ablkcipher_request *req, unsigned long mode) { struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx( @@ -1175,9 +1158,6 @@ static int omap_aes_probe(struct platform_device *pdev) dd->dev = dev; platform_set_drvdata(pdev, dd); - spin_lock_init(&dd->lock); - crypto_init_queue(&dd->queue, OMAP_AES_QUEUE_LENGTH); - err = (dev->of_node) ? omap_aes_get_res_of(dd, dev, &res) : omap_aes_get_res_pdev(dd, pdev, &res); if (err) @@ -1209,7 +1189,6 @@ static int omap_aes_probe(struct platform_device *pdev) (reg & dd->pdata->minor_mask) >> dd->pdata->minor_shift); tasklet_init(&dd->done_task, omap_aes_done_task, (unsigned long)dd); - tasklet_init(&dd->queue_task, omap_aes_queue_task, (unsigned long)dd); err = omap_aes_dma_init(dd); if (err && AES_REG_IRQ_STATUS(dd) && AES_REG_IRQ_ENABLE(dd)) { @@ -1250,7 +1229,20 @@ static int omap_aes_probe(struct platform_device *pdev) } } + /* Initialize crypto engine */ + dd->engine = crypto_engine_alloc_init(dev, 1); + if (!dd->engine) + goto err_algs; + + dd->engine->prepare_request = omap_aes_prepare_req; + dd->engine->crypt_one_request = omap_aes_crypt_req; + err = crypto_engine_start(dd->engine); + if (err) + goto err_engine; + return 0; +err_engine: + crypto_engine_exit(dd->engine); err_algs: for (i = dd->pdata->algs_info_size - 1; i >= 0; i--) for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) @@ -1260,7 +1252,6 @@ err_algs: omap_aes_dma_cleanup(dd); err_irq: tasklet_kill(&dd->done_task); - tasklet_kill(&dd->queue_task); pm_runtime_disable(dev); err_res: dd = NULL; @@ -1286,8 +1277,8 @@ static int omap_aes_remove(struct platform_device *pdev) crypto_unregister_alg( &dd->pdata->algs_info[i].algs_list[j]); + crypto_engine_exit(dd->engine); tasklet_kill(&dd->done_task); - tasklet_kill(&dd->queue_task); omap_aes_dma_cleanup(dd); pm_runtime_disable(dd->dev); dd = NULL; diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index f96d427e5..5a07208ce 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -55,8 +55,8 @@ #define ADF_DH895XCC_DEVICE_NAME "dh895xcc" #define ADF_DH895XCCVF_DEVICE_NAME "dh895xccvf" -#define ADF_C62X_DEVICE_NAME "c62x" -#define ADF_C62XVF_DEVICE_NAME "c62xvf" +#define ADF_C62X_DEVICE_NAME "c6xx" +#define ADF_C62XVF_DEVICE_NAME "c6xxvf" #define ADF_C3XXX_DEVICE_NAME "c3xxx" #define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf" #define ADF_DH895XCC_PCI_DEVICE_ID 0x435 diff --git a/drivers/crypto/qat/qat_common/adf_aer.c b/drivers/crypto/qat/qat_common/adf_aer.c index e78a1d7d8..b40d9c8da 100644 --- a/drivers/crypto/qat/qat_common/adf_aer.c +++ b/drivers/crypto/qat/qat_common/adf_aer.c @@ -121,7 +121,6 @@ static void adf_device_reset_worker(struct work_struct *work) adf_dev_restarting_notify(accel_dev); adf_dev_stop(accel_dev); adf_dev_shutdown(accel_dev); - adf_dev_restore(accel_dev); if (adf_dev_init(accel_dev) || adf_dev_start(accel_dev)) { /* The device hanged and we can't restart it so stop here */ dev_err(&GET_DEV(accel_dev), "Restart device failed\n"); diff --git a/drivers/crypto/qat/qat_common/adf_cfg_user.h b/drivers/crypto/qat/qat_common/adf_cfg_user.h index ef5988afd..b5484bfa6 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg_user.h +++ b/drivers/crypto/qat/qat_common/adf_cfg_user.h @@ -58,7 +58,7 @@ struct adf_user_cfg_key_val { uint64_t padding3; }; enum adf_cfg_val_type type; -}; +} __packed; struct adf_user_cfg_section { char name[ADF_CFG_MAX_SECTION_LEN_IN_BYTES]; @@ -70,7 +70,7 @@ struct adf_user_cfg_section { struct adf_user_cfg_section *next; uint64_t padding3; }; -}; +} __packed; struct adf_user_cfg_ctl_data { union { @@ -78,5 +78,5 @@ struct adf_user_cfg_ctl_data { uint64_t padding; }; uint8_t device_id; -}; +} __packed; #endif diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index 0e82ce3c3..976b01e58 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -236,6 +236,8 @@ void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, uint32_t vf_mask); void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); +int adf_init_pf_wq(void); +void adf_exit_pf_wq(void); #else static inline int adf_sriov_configure(struct pci_dev *pdev, int numvfs) { @@ -253,5 +255,14 @@ static inline void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev) static inline void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev) { } + +static inline int adf_init_pf_wq(void) +{ + return 0; +} + +static inline void adf_exit_pf_wq(void) +{ +} #endif #endif diff --git a/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/qat/qat_common/adf_ctl_drv.c index 5c897e6e7..3c3f94829 100644 --- a/drivers/crypto/qat/qat_common/adf_ctl_drv.c +++ b/drivers/crypto/qat/qat_common/adf_ctl_drv.c @@ -462,12 +462,17 @@ static int __init adf_register_ctl_device_driver(void) if (adf_init_aer()) goto err_aer; + if (adf_init_pf_wq()) + goto err_pf_wq; + if (qat_crypto_register()) goto err_crypto_register; return 0; err_crypto_register: + adf_exit_pf_wq(); +err_pf_wq: adf_exit_aer(); err_aer: adf_chr_drv_destroy(); @@ -480,6 +485,7 @@ static void __exit adf_unregister_ctl_device_driver(void) { adf_chr_drv_destroy(); adf_exit_aer(); + adf_exit_pf_wq(); qat_crypto_unregister(); adf_clean_vf_map(false); mutex_destroy(&adf_ctl_lock); diff --git a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c index f267d9e42..d7dd18d9b 100644 --- a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c +++ b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c @@ -49,7 +49,6 @@ #include "adf_transport_internal.h" #define ADF_ARB_NUM 4 -#define ADF_ARB_REQ_RING_NUM 8 #define ADF_ARB_REG_SIZE 0x4 #define ADF_ARB_WTR_SIZE 0x20 #define ADF_ARB_OFFSET 0x30000 @@ -64,15 +63,6 @@ ADF_CSR_WR(csr_addr, ADF_ARB_RINGSRVARBEN_OFFSET + \ (ADF_ARB_REG_SLOT * index), value) -#define WRITE_CSR_ARB_RESPORDERING(csr_addr, index, value) \ - ADF_CSR_WR(csr_addr, (ADF_ARB_OFFSET + \ - ADF_ARB_RO_EN_OFFSET) + (ADF_ARB_REG_SIZE * index), value) - -#define WRITE_CSR_ARB_WEIGHT(csr_addr, arb, index, value) \ - ADF_CSR_WR(csr_addr, (ADF_ARB_OFFSET + \ - ADF_ARB_WTR_OFFSET) + (ADF_ARB_WTR_SIZE * arb) + \ - (ADF_ARB_REG_SIZE * index), value) - #define WRITE_CSR_ARB_SARCONFIG(csr_addr, index, value) \ ADF_CSR_WR(csr_addr, ADF_ARB_OFFSET + \ (ADF_ARB_REG_SIZE * index), value) @@ -99,15 +89,6 @@ int adf_init_arb(struct adf_accel_dev *accel_dev) for (arb = 0; arb < ADF_ARB_NUM; arb++) WRITE_CSR_ARB_SARCONFIG(csr, arb, arb_cfg); - /* Setup service weighting */ - for (arb = 0; arb < ADF_ARB_NUM; arb++) - for (i = 0; i < ADF_ARB_REQ_RING_NUM; i++) - WRITE_CSR_ARB_WEIGHT(csr, arb, i, 0xFFFFFFFF); - - /* Setup ring response ordering */ - for (i = 0; i < ADF_ARB_REQ_RING_NUM; i++) - WRITE_CSR_ARB_RESPORDERING(csr, i, 0xFFFFFFFF); - /* Setup worker queue registers */ for (i = 0; i < hw_data->num_engines; i++) WRITE_CSR_ARB_WQCFG(csr, i, i); diff --git a/drivers/crypto/qat/qat_common/adf_sriov.c b/drivers/crypto/qat/qat_common/adf_sriov.c index 1117a8b58..38a0415e7 100644 --- a/drivers/crypto/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/qat/qat_common/adf_sriov.c @@ -119,11 +119,6 @@ static int adf_enable_sriov(struct adf_accel_dev *accel_dev) int i; u32 reg; - /* Workqueue for PF2VF responses */ - pf2vf_resp_wq = create_workqueue("qat_pf2vf_resp_wq"); - if (!pf2vf_resp_wq) - return -ENOMEM; - for (i = 0, vf_info = accel_dev->pf.vf_info; i < totalvfs; i++, vf_info++) { /* This ptr will be populated when VFs will be created */ @@ -216,11 +211,6 @@ void adf_disable_sriov(struct adf_accel_dev *accel_dev) kfree(accel_dev->pf.vf_info); accel_dev->pf.vf_info = NULL; - - if (pf2vf_resp_wq) { - destroy_workqueue(pf2vf_resp_wq); - pf2vf_resp_wq = NULL; - } } EXPORT_SYMBOL_GPL(adf_disable_sriov); @@ -304,3 +294,19 @@ int adf_sriov_configure(struct pci_dev *pdev, int numvfs) return numvfs; } EXPORT_SYMBOL_GPL(adf_sriov_configure); + +int __init adf_init_pf_wq(void) +{ + /* Workqueue for PF2VF responses */ + pf2vf_resp_wq = create_workqueue("qat_pf2vf_resp_wq"); + + return !pf2vf_resp_wq ? -ENOMEM : 0; +} + +void adf_exit_pf_wq(void) +{ + if (pf2vf_resp_wq) { + destroy_workqueue(pf2vf_resp_wq); + pf2vf_resp_wq = NULL; + } +} diff --git a/drivers/crypto/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/qat/qat_common/icp_qat_uclo.h index d97db9909..5d1ee7e53 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_uclo.h +++ b/drivers/crypto/qat/qat_common/icp_qat_uclo.h @@ -112,27 +112,27 @@ enum icp_qat_uof_mem_region { }; enum icp_qat_uof_regtype { - ICP_NO_DEST, - ICP_GPA_REL, - ICP_GPA_ABS, - ICP_GPB_REL, - ICP_GPB_ABS, - ICP_SR_REL, - ICP_SR_RD_REL, - ICP_SR_WR_REL, - ICP_SR_ABS, - ICP_SR_RD_ABS, - ICP_SR_WR_ABS, - ICP_DR_REL, - ICP_DR_RD_REL, - ICP_DR_WR_REL, - ICP_DR_ABS, - ICP_DR_RD_ABS, - ICP_DR_WR_ABS, - ICP_LMEM, - ICP_LMEM0, - ICP_LMEM1, - ICP_NEIGH_REL, + ICP_NO_DEST = 0, + ICP_GPA_REL = 1, + ICP_GPA_ABS = 2, + ICP_GPB_REL = 3, + ICP_GPB_ABS = 4, + ICP_SR_REL = 5, + ICP_SR_RD_REL = 6, + ICP_SR_WR_REL = 7, + ICP_SR_ABS = 8, + ICP_SR_RD_ABS = 9, + ICP_SR_WR_ABS = 10, + ICP_DR_REL = 19, + ICP_DR_RD_REL = 20, + ICP_DR_WR_REL = 21, + ICP_DR_ABS = 22, + ICP_DR_RD_ABS = 23, + ICP_DR_WR_ABS = 24, + ICP_LMEM = 26, + ICP_LMEM0 = 27, + ICP_LMEM1 = 28, + ICP_NEIGH_REL = 31, }; enum icp_qat_css_fwtype { diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index 59e4c3af1..1e8852a8a 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -1064,8 +1064,7 @@ static int qat_alg_aead_init(struct crypto_aead *tfm, if (IS_ERR(ctx->hash_tfm)) return PTR_ERR(ctx->hash_tfm); ctx->qat_hash_alg = hash; - crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) + - sizeof(struct qat_crypto_request)); + crypto_aead_set_reqsize(tfm, sizeof(struct qat_crypto_request)); return 0; } @@ -1114,8 +1113,7 @@ static int qat_alg_ablkcipher_init(struct crypto_tfm *tfm) struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm); spin_lock_init(&ctx->lock); - tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) + - sizeof(struct qat_crypto_request); + tfm->crt_ablkcipher.reqsize = sizeof(struct qat_crypto_request); ctx->tfm = tfm; return 0; } diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c index 51c594fda..e5c0727d4 100644 --- a/drivers/crypto/qat/qat_common/qat_asym_algs.c +++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c @@ -340,14 +340,16 @@ static int qat_rsa_enc(struct akcipher_request *req) if (!ret) return -EINPROGRESS; -unmap_src: - if (qat_req->src_align) - dma_free_coherent(dev, ctx->key_sz, qat_req->src_align, - qat_req->in.enc.m); - else - if (!dma_mapping_error(dev, qat_req->in.enc.m)) - dma_unmap_single(dev, qat_req->in.enc.m, ctx->key_sz, - DMA_TO_DEVICE); + + if (!dma_mapping_error(dev, qat_req->phy_out)) + dma_unmap_single(dev, qat_req->phy_out, + sizeof(struct qat_rsa_output_params), + DMA_TO_DEVICE); +unmap_in_params: + if (!dma_mapping_error(dev, qat_req->phy_in)) + dma_unmap_single(dev, qat_req->phy_in, + sizeof(struct qat_rsa_input_params), + DMA_TO_DEVICE); unmap_dst: if (qat_req->dst_align) dma_free_coherent(dev, ctx->key_sz, qat_req->dst_align, @@ -356,15 +358,14 @@ unmap_dst: if (!dma_mapping_error(dev, qat_req->out.enc.c)) dma_unmap_single(dev, qat_req->out.enc.c, ctx->key_sz, DMA_FROM_DEVICE); -unmap_in_params: - if (!dma_mapping_error(dev, qat_req->phy_in)) - dma_unmap_single(dev, qat_req->phy_in, - sizeof(struct qat_rsa_input_params), - DMA_TO_DEVICE); - if (!dma_mapping_error(dev, qat_req->phy_out)) - dma_unmap_single(dev, qat_req->phy_out, - sizeof(struct qat_rsa_output_params), - DMA_TO_DEVICE); +unmap_src: + if (qat_req->src_align) + dma_free_coherent(dev, ctx->key_sz, qat_req->src_align, + qat_req->in.enc.m); + else + if (!dma_mapping_error(dev, qat_req->in.enc.m)) + dma_unmap_single(dev, qat_req->in.enc.m, ctx->key_sz, + DMA_TO_DEVICE); return ret; } @@ -472,14 +473,16 @@ static int qat_rsa_dec(struct akcipher_request *req) if (!ret) return -EINPROGRESS; -unmap_src: - if (qat_req->src_align) - dma_free_coherent(dev, ctx->key_sz, qat_req->src_align, - qat_req->in.dec.c); - else - if (!dma_mapping_error(dev, qat_req->in.dec.c)) - dma_unmap_single(dev, qat_req->in.dec.c, ctx->key_sz, - DMA_TO_DEVICE); + + if (!dma_mapping_error(dev, qat_req->phy_out)) + dma_unmap_single(dev, qat_req->phy_out, + sizeof(struct qat_rsa_output_params), + DMA_TO_DEVICE); +unmap_in_params: + if (!dma_mapping_error(dev, qat_req->phy_in)) + dma_unmap_single(dev, qat_req->phy_in, + sizeof(struct qat_rsa_input_params), + DMA_TO_DEVICE); unmap_dst: if (qat_req->dst_align) dma_free_coherent(dev, ctx->key_sz, qat_req->dst_align, @@ -488,15 +491,14 @@ unmap_dst: if (!dma_mapping_error(dev, qat_req->out.dec.m)) dma_unmap_single(dev, qat_req->out.dec.m, ctx->key_sz, DMA_FROM_DEVICE); -unmap_in_params: - if (!dma_mapping_error(dev, qat_req->phy_in)) - dma_unmap_single(dev, qat_req->phy_in, - sizeof(struct qat_rsa_input_params), - DMA_TO_DEVICE); - if (!dma_mapping_error(dev, qat_req->phy_out)) - dma_unmap_single(dev, qat_req->phy_out, - sizeof(struct qat_rsa_output_params), - DMA_TO_DEVICE); +unmap_src: + if (qat_req->src_align) + dma_free_coherent(dev, ctx->key_sz, qat_req->src_align, + qat_req->in.dec.c); + else + if (!dma_mapping_error(dev, qat_req->in.dec.c)) + dma_unmap_single(dev, qat_req->in.dec.c, ctx->key_sz, + DMA_TO_DEVICE); return ret; } diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 25d15f19c..9b961b37a 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -688,7 +688,7 @@ static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae) int mflag = 0; struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; - for (ae = 0; ae <= max_ae; ae++) { + for (ae = 0; ae < max_ae; ae++) { if (!test_bit(ae, (unsigned long *)&handle->hal_handle->ae_mask)) continue; diff --git a/drivers/crypto/rockchip/Makefile b/drivers/crypto/rockchip/Makefile index 7051c6c71..30f91297b 100644 --- a/drivers/crypto/rockchip/Makefile +++ b/drivers/crypto/rockchip/Makefile @@ -1,3 +1,4 @@ obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP) += rk_crypto.o rk_crypto-objs := rk3288_crypto.o \ rk3288_crypto_ablkcipher.o \ + rk3288_crypto_ahash.o diff --git a/drivers/crypto/rockchip/rk3288_crypto.c b/drivers/crypto/rockchip/rk3288_crypto.c index da9c73dce..af508258d 100644 --- a/drivers/crypto/rockchip/rk3288_crypto.c +++ b/drivers/crypto/rockchip/rk3288_crypto.c @@ -208,6 +208,8 @@ static void rk_crypto_tasklet_cb(unsigned long data) if (crypto_tfm_alg_type(async_req->tfm) == CRYPTO_ALG_TYPE_ABLKCIPHER) dev->ablk_req = ablkcipher_request_cast(async_req); + else + dev->ahash_req = ahash_request_cast(async_req); err = dev->start(dev); if (err) dev->complete(dev, err); @@ -220,6 +222,9 @@ static struct rk_crypto_tmp *rk_cipher_algs[] = { &rk_cbc_des_alg, &rk_ecb_des3_ede_alg, &rk_cbc_des3_ede_alg, + &rk_ahash_sha1, + &rk_ahash_sha256, + &rk_ahash_md5, }; static int rk_crypto_register(struct rk_crypto_info *crypto_info) @@ -229,15 +234,24 @@ static int rk_crypto_register(struct rk_crypto_info *crypto_info) for (i = 0; i < ARRAY_SIZE(rk_cipher_algs); i++) { rk_cipher_algs[i]->dev = crypto_info; - err = crypto_register_alg(&rk_cipher_algs[i]->alg); + if (rk_cipher_algs[i]->type == ALG_TYPE_CIPHER) + err = crypto_register_alg( + &rk_cipher_algs[i]->alg.crypto); + else + err = crypto_register_ahash( + &rk_cipher_algs[i]->alg.hash); if (err) goto err_cipher_algs; } return 0; err_cipher_algs: - for (k = 0; k < i; k++) - crypto_unregister_alg(&rk_cipher_algs[k]->alg); + for (k = 0; k < i; k++) { + if (rk_cipher_algs[i]->type == ALG_TYPE_CIPHER) + crypto_unregister_alg(&rk_cipher_algs[k]->alg.crypto); + else + crypto_unregister_ahash(&rk_cipher_algs[i]->alg.hash); + } return err; } @@ -245,8 +259,12 @@ static void rk_crypto_unregister(void) { unsigned int i; - for (i = 0; i < ARRAY_SIZE(rk_cipher_algs); i++) - crypto_unregister_alg(&rk_cipher_algs[i]->alg); + for (i = 0; i < ARRAY_SIZE(rk_cipher_algs); i++) { + if (rk_cipher_algs[i]->type == ALG_TYPE_CIPHER) + crypto_unregister_alg(&rk_cipher_algs[i]->alg.crypto); + else + crypto_unregister_ahash(&rk_cipher_algs[i]->alg.hash); + } } static void rk_crypto_action(void *data) diff --git a/drivers/crypto/rockchip/rk3288_crypto.h b/drivers/crypto/rockchip/rk3288_crypto.h index e499c2c6c..d7b71fea3 100644 --- a/drivers/crypto/rockchip/rk3288_crypto.h +++ b/drivers/crypto/rockchip/rk3288_crypto.h @@ -6,6 +6,10 @@ #include <crypto/algapi.h> #include <linux/interrupt.h> #include <linux/delay.h> +#include <crypto/internal/hash.h> + +#include <crypto/md5.h> +#include <crypto/sha.h> #define _SBF(v, f) ((v) << (f)) @@ -149,6 +153,28 @@ #define RK_CRYPTO_TDES_KEY3_0 0x0130 #define RK_CRYPTO_TDES_KEY3_1 0x0134 +/* HASH */ +#define RK_CRYPTO_HASH_CTRL 0x0180 +#define RK_CRYPTO_HASH_SWAP_DO BIT(3) +#define RK_CRYPTO_HASH_SWAP_DI BIT(2) +#define RK_CRYPTO_HASH_SHA1 _SBF(0x00, 0) +#define RK_CRYPTO_HASH_MD5 _SBF(0x01, 0) +#define RK_CRYPTO_HASH_SHA256 _SBF(0x02, 0) +#define RK_CRYPTO_HASH_PRNG _SBF(0x03, 0) + +#define RK_CRYPTO_HASH_STS 0x0184 +#define RK_CRYPTO_HASH_DONE BIT(0) + +#define RK_CRYPTO_HASH_MSG_LEN 0x0188 +#define RK_CRYPTO_HASH_DOUT_0 0x018c +#define RK_CRYPTO_HASH_DOUT_1 0x0190 +#define RK_CRYPTO_HASH_DOUT_2 0x0194 +#define RK_CRYPTO_HASH_DOUT_3 0x0198 +#define RK_CRYPTO_HASH_DOUT_4 0x019c +#define RK_CRYPTO_HASH_DOUT_5 0x01a0 +#define RK_CRYPTO_HASH_DOUT_6 0x01a4 +#define RK_CRYPTO_HASH_DOUT_7 0x01a8 + #define CRYPTO_READ(dev, offset) \ readl_relaxed(((dev)->reg + (offset))) #define CRYPTO_WRITE(dev, offset, val) \ @@ -166,6 +192,7 @@ struct rk_crypto_info { struct crypto_queue queue; struct tasklet_struct crypto_tasklet; struct ablkcipher_request *ablk_req; + struct ahash_request *ahash_req; /* device lock */ spinlock_t lock; @@ -195,15 +222,36 @@ struct rk_crypto_info { void (*unload_data)(struct rk_crypto_info *dev); }; +/* the private variable of hash */ +struct rk_ahash_ctx { + struct rk_crypto_info *dev; + /* for fallback */ + struct crypto_ahash *fallback_tfm; +}; + +/* the privete variable of hash for fallback */ +struct rk_ahash_rctx { + struct ahash_request fallback_req; +}; + /* the private variable of cipher */ struct rk_cipher_ctx { struct rk_crypto_info *dev; unsigned int keylen; }; +enum alg_type { + ALG_TYPE_HASH, + ALG_TYPE_CIPHER, +}; + struct rk_crypto_tmp { - struct rk_crypto_info *dev; - struct crypto_alg alg; + struct rk_crypto_info *dev; + union { + struct crypto_alg crypto; + struct ahash_alg hash; + } alg; + enum alg_type type; }; extern struct rk_crypto_tmp rk_ecb_aes_alg; @@ -213,4 +261,8 @@ extern struct rk_crypto_tmp rk_cbc_des_alg; extern struct rk_crypto_tmp rk_ecb_des3_ede_alg; extern struct rk_crypto_tmp rk_cbc_des3_ede_alg; +extern struct rk_crypto_tmp rk_ahash_sha1; +extern struct rk_crypto_tmp rk_ahash_sha256; +extern struct rk_crypto_tmp rk_ahash_md5; + #endif diff --git a/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c b/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c index d98b681f6..b5a3afe22 100644 --- a/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c +++ b/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c @@ -336,7 +336,7 @@ static int rk_ablk_cra_init(struct crypto_tfm *tfm) struct crypto_alg *alg = tfm->__crt_alg; struct rk_crypto_tmp *algt; - algt = container_of(alg, struct rk_crypto_tmp, alg); + algt = container_of(alg, struct rk_crypto_tmp, alg.crypto); ctx->dev = algt->dev; ctx->dev->align_size = crypto_tfm_alg_alignmask(tfm) + 1; @@ -357,7 +357,8 @@ static void rk_ablk_cra_exit(struct crypto_tfm *tfm) } struct rk_crypto_tmp rk_ecb_aes_alg = { - .alg = { + .type = ALG_TYPE_CIPHER, + .alg.crypto = { .cra_name = "ecb(aes)", .cra_driver_name = "ecb-aes-rk", .cra_priority = 300, @@ -381,7 +382,8 @@ struct rk_crypto_tmp rk_ecb_aes_alg = { }; struct rk_crypto_tmp rk_cbc_aes_alg = { - .alg = { + .type = ALG_TYPE_CIPHER, + .alg.crypto = { .cra_name = "cbc(aes)", .cra_driver_name = "cbc-aes-rk", .cra_priority = 300, @@ -406,7 +408,8 @@ struct rk_crypto_tmp rk_cbc_aes_alg = { }; struct rk_crypto_tmp rk_ecb_des_alg = { - .alg = { + .type = ALG_TYPE_CIPHER, + .alg.crypto = { .cra_name = "ecb(des)", .cra_driver_name = "ecb-des-rk", .cra_priority = 300, @@ -430,7 +433,8 @@ struct rk_crypto_tmp rk_ecb_des_alg = { }; struct rk_crypto_tmp rk_cbc_des_alg = { - .alg = { + .type = ALG_TYPE_CIPHER, + .alg.crypto = { .cra_name = "cbc(des)", .cra_driver_name = "cbc-des-rk", .cra_priority = 300, @@ -455,7 +459,8 @@ struct rk_crypto_tmp rk_cbc_des_alg = { }; struct rk_crypto_tmp rk_ecb_des3_ede_alg = { - .alg = { + .type = ALG_TYPE_CIPHER, + .alg.crypto = { .cra_name = "ecb(des3_ede)", .cra_driver_name = "ecb-des3-ede-rk", .cra_priority = 300, @@ -480,7 +485,8 @@ struct rk_crypto_tmp rk_ecb_des3_ede_alg = { }; struct rk_crypto_tmp rk_cbc_des3_ede_alg = { - .alg = { + .type = ALG_TYPE_CIPHER, + .alg.crypto = { .cra_name = "cbc(des3_ede)", .cra_driver_name = "cbc-des3-ede-rk", .cra_priority = 300, diff --git a/drivers/crypto/rockchip/rk3288_crypto_ahash.c b/drivers/crypto/rockchip/rk3288_crypto_ahash.c new file mode 100644 index 000000000..718588219 --- /dev/null +++ b/drivers/crypto/rockchip/rk3288_crypto_ahash.c @@ -0,0 +1,404 @@ +/* + * Crypto acceleration support for Rockchip RK3288 + * + * Copyright (c) 2015, Fuzhou Rockchip Electronics Co., Ltd + * + * Author: Zain Wang <zain.wang@rock-chips.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * Some ideas are from marvell/cesa.c and s5p-sss.c driver. + */ +#include "rk3288_crypto.h" + +/* + * IC can not process zero message hash, + * so we put the fixed hash out when met zero message. + */ + +static int zero_message_process(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + int rk_digest_size = crypto_ahash_digestsize(tfm); + + switch (rk_digest_size) { + case SHA1_DIGEST_SIZE: + memcpy(req->result, sha1_zero_message_hash, rk_digest_size); + break; + case SHA256_DIGEST_SIZE: + memcpy(req->result, sha256_zero_message_hash, rk_digest_size); + break; + case MD5_DIGEST_SIZE: + memcpy(req->result, md5_zero_message_hash, rk_digest_size); + break; + default: + return -EINVAL; + } + + return 0; +} + +static void rk_ahash_crypto_complete(struct rk_crypto_info *dev, int err) +{ + if (dev->ahash_req->base.complete) + dev->ahash_req->base.complete(&dev->ahash_req->base, err); +} + +static void rk_ahash_reg_init(struct rk_crypto_info *dev) +{ + int reg_status = 0; + + reg_status = CRYPTO_READ(dev, RK_CRYPTO_CTRL) | + RK_CRYPTO_HASH_FLUSH | _SBF(0xffff, 16); + CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, reg_status); + + reg_status = CRYPTO_READ(dev, RK_CRYPTO_CTRL); + reg_status &= (~RK_CRYPTO_HASH_FLUSH); + reg_status |= _SBF(0xffff, 16); + CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, reg_status); + + memset_io(dev->reg + RK_CRYPTO_HASH_DOUT_0, 0, 32); + + CRYPTO_WRITE(dev, RK_CRYPTO_INTENA, RK_CRYPTO_HRDMA_ERR_ENA | + RK_CRYPTO_HRDMA_DONE_ENA); + + CRYPTO_WRITE(dev, RK_CRYPTO_INTSTS, RK_CRYPTO_HRDMA_ERR_INT | + RK_CRYPTO_HRDMA_DONE_INT); + + CRYPTO_WRITE(dev, RK_CRYPTO_HASH_CTRL, dev->mode | + RK_CRYPTO_HASH_SWAP_DO); + + CRYPTO_WRITE(dev, RK_CRYPTO_CONF, RK_CRYPTO_BYTESWAP_HRFIFO | + RK_CRYPTO_BYTESWAP_BRFIFO | + RK_CRYPTO_BYTESWAP_BTFIFO); + + CRYPTO_WRITE(dev, RK_CRYPTO_HASH_MSG_LEN, dev->total); +} + +static int rk_ahash_init(struct ahash_request *req) +{ + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + rctx->fallback_req.base.flags = req->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + + return crypto_ahash_init(&rctx->fallback_req); +} + +static int rk_ahash_update(struct ahash_request *req) +{ + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + rctx->fallback_req.base.flags = req->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + rctx->fallback_req.nbytes = req->nbytes; + rctx->fallback_req.src = req->src; + + return crypto_ahash_update(&rctx->fallback_req); +} + +static int rk_ahash_final(struct ahash_request *req) +{ + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + rctx->fallback_req.base.flags = req->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + rctx->fallback_req.result = req->result; + + return crypto_ahash_final(&rctx->fallback_req); +} + +static int rk_ahash_finup(struct ahash_request *req) +{ + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + rctx->fallback_req.base.flags = req->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + + rctx->fallback_req.nbytes = req->nbytes; + rctx->fallback_req.src = req->src; + rctx->fallback_req.result = req->result; + + return crypto_ahash_finup(&rctx->fallback_req); +} + +static int rk_ahash_import(struct ahash_request *req, const void *in) +{ + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + rctx->fallback_req.base.flags = req->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + + return crypto_ahash_import(&rctx->fallback_req, in); +} + +static int rk_ahash_export(struct ahash_request *req, void *out) +{ + struct rk_ahash_rctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm); + + ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); + rctx->fallback_req.base.flags = req->base.flags & + CRYPTO_TFM_REQ_MAY_SLEEP; + + return crypto_ahash_export(&rctx->fallback_req, out); +} + +static int rk_ahash_digest(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct rk_ahash_ctx *tctx = crypto_tfm_ctx(req->base.tfm); + struct rk_crypto_info *dev = NULL; + unsigned long flags; + int ret; + + if (!req->nbytes) + return zero_message_process(req); + + dev = tctx->dev; + dev->total = req->nbytes; + dev->left_bytes = req->nbytes; + dev->aligned = 0; + dev->mode = 0; + dev->align_size = 4; + dev->sg_dst = NULL; + dev->sg_src = req->src; + dev->first = req->src; + dev->nents = sg_nents(req->src); + + switch (crypto_ahash_digestsize(tfm)) { + case SHA1_DIGEST_SIZE: + dev->mode = RK_CRYPTO_HASH_SHA1; + break; + case SHA256_DIGEST_SIZE: + dev->mode = RK_CRYPTO_HASH_SHA256; + break; + case MD5_DIGEST_SIZE: + dev->mode = RK_CRYPTO_HASH_MD5; + break; + default: + return -EINVAL; + } + + rk_ahash_reg_init(dev); + + spin_lock_irqsave(&dev->lock, flags); + ret = crypto_enqueue_request(&dev->queue, &req->base); + spin_unlock_irqrestore(&dev->lock, flags); + + tasklet_schedule(&dev->crypto_tasklet); + + /* + * it will take some time to process date after last dma transmission. + * + * waiting time is relative with the last date len, + * so cannot set a fixed time here. + * 10-50 makes system not call here frequently wasting + * efficiency, and make it response quickly when dma + * complete. + */ + while (!CRYPTO_READ(dev, RK_CRYPTO_HASH_STS)) + usleep_range(10, 50); + + memcpy_fromio(req->result, dev->reg + RK_CRYPTO_HASH_DOUT_0, + crypto_ahash_digestsize(tfm)); + + return 0; +} + +static void crypto_ahash_dma_start(struct rk_crypto_info *dev) +{ + CRYPTO_WRITE(dev, RK_CRYPTO_HRDMAS, dev->addr_in); + CRYPTO_WRITE(dev, RK_CRYPTO_HRDMAL, (dev->count + 3) / 4); + CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, RK_CRYPTO_HASH_START | + (RK_CRYPTO_HASH_START << 16)); +} + +static int rk_ahash_set_data_start(struct rk_crypto_info *dev) +{ + int err; + + err = dev->load_data(dev, dev->sg_src, NULL); + if (!err) + crypto_ahash_dma_start(dev); + return err; +} + +static int rk_ahash_start(struct rk_crypto_info *dev) +{ + return rk_ahash_set_data_start(dev); +} + +static int rk_ahash_crypto_rx(struct rk_crypto_info *dev) +{ + int err = 0; + + dev->unload_data(dev); + if (dev->left_bytes) { + if (dev->aligned) { + if (sg_is_last(dev->sg_src)) { + dev_warn(dev->dev, "[%s:%d], Lack of data\n", + __func__, __LINE__); + err = -ENOMEM; + goto out_rx; + } + dev->sg_src = sg_next(dev->sg_src); + } + err = rk_ahash_set_data_start(dev); + } else { + dev->complete(dev, 0); + } + +out_rx: + return err; +} + +static int rk_cra_hash_init(struct crypto_tfm *tfm) +{ + struct rk_ahash_ctx *tctx = crypto_tfm_ctx(tfm); + struct rk_crypto_tmp *algt; + struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg); + + const char *alg_name = crypto_tfm_alg_name(tfm); + + algt = container_of(alg, struct rk_crypto_tmp, alg.hash); + + tctx->dev = algt->dev; + tctx->dev->addr_vir = (void *)__get_free_page(GFP_KERNEL); + if (!tctx->dev->addr_vir) { + dev_err(tctx->dev->dev, "failed to kmalloc for addr_vir\n"); + return -ENOMEM; + } + tctx->dev->start = rk_ahash_start; + tctx->dev->update = rk_ahash_crypto_rx; + tctx->dev->complete = rk_ahash_crypto_complete; + + /* for fallback */ + tctx->fallback_tfm = crypto_alloc_ahash(alg_name, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(tctx->fallback_tfm)) { + dev_err(tctx->dev->dev, "Could not load fallback driver.\n"); + return PTR_ERR(tctx->fallback_tfm); + } + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct rk_ahash_rctx) + + crypto_ahash_reqsize(tctx->fallback_tfm)); + + return tctx->dev->enable_clk(tctx->dev); +} + +static void rk_cra_hash_exit(struct crypto_tfm *tfm) +{ + struct rk_ahash_ctx *tctx = crypto_tfm_ctx(tfm); + + free_page((unsigned long)tctx->dev->addr_vir); + return tctx->dev->disable_clk(tctx->dev); +} + +struct rk_crypto_tmp rk_ahash_sha1 = { + .type = ALG_TYPE_HASH, + .alg.hash = { + .init = rk_ahash_init, + .update = rk_ahash_update, + .final = rk_ahash_final, + .finup = rk_ahash_finup, + .export = rk_ahash_export, + .import = rk_ahash_import, + .digest = rk_ahash_digest, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "sha1", + .cra_driver_name = "rk-sha1", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct rk_ahash_ctx), + .cra_alignmask = 3, + .cra_init = rk_cra_hash_init, + .cra_exit = rk_cra_hash_exit, + .cra_module = THIS_MODULE, + } + } + } +}; + +struct rk_crypto_tmp rk_ahash_sha256 = { + .type = ALG_TYPE_HASH, + .alg.hash = { + .init = rk_ahash_init, + .update = rk_ahash_update, + .final = rk_ahash_final, + .finup = rk_ahash_finup, + .export = rk_ahash_export, + .import = rk_ahash_import, + .digest = rk_ahash_digest, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha256", + .cra_driver_name = "rk-sha256", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct rk_ahash_ctx), + .cra_alignmask = 3, + .cra_init = rk_cra_hash_init, + .cra_exit = rk_cra_hash_exit, + .cra_module = THIS_MODULE, + } + } + } +}; + +struct rk_crypto_tmp rk_ahash_md5 = { + .type = ALG_TYPE_HASH, + .alg.hash = { + .init = rk_ahash_init, + .update = rk_ahash_update, + .final = rk_ahash_final, + .finup = rk_ahash_finup, + .export = rk_ahash_export, + .import = rk_ahash_import, + .digest = rk_ahash_digest, + .halg = { + .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct md5_state), + .base = { + .cra_name = "md5", + .cra_driver_name = "rk-md5", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct rk_ahash_ctx), + .cra_alignmask = 3, + .cra_init = rk_cra_hash_init, + .cra_exit = rk_cra_hash_exit, + .cra_module = THIS_MODULE, + } + } + } +}; diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index f214a8755..5f161a977 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -224,6 +224,7 @@ static inline struct samsung_aes_variant *find_s5p_sss_version { if (IS_ENABLED(CONFIG_OF) && (pdev->dev.of_node)) { const struct of_device_id *match; + match = of_match_node(s5p_sss_dt_match, pdev->dev.of_node); return (struct samsung_aes_variant *)match->data; @@ -382,7 +383,7 @@ static void s5p_set_aes(struct s5p_aes_dev *dev, void __iomem *keystart; if (iv) - memcpy(dev->aes_ioaddr + SSS_REG_AES_IV_DATA(0), iv, 0x10); + memcpy_toio(dev->aes_ioaddr + SSS_REG_AES_IV_DATA(0), iv, 0x10); if (keylen == AES_KEYSIZE_256) keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(0); @@ -391,13 +392,12 @@ static void s5p_set_aes(struct s5p_aes_dev *dev, else keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(4); - memcpy(keystart, key, keylen); + memcpy_toio(keystart, key, keylen); } static void s5p_aes_crypt_start(struct s5p_aes_dev *dev, unsigned long mode) { struct ablkcipher_request *req = dev->req; - uint32_t aes_control; int err; unsigned long flags; @@ -518,7 +518,7 @@ static int s5p_aes_crypt(struct ablkcipher_request *req, unsigned long mode) struct s5p_aes_dev *dev = ctx->dev; if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) { - pr_err("request size is not exact amount of AES blocks\n"); + dev_err(dev->dev, "request size is not exact amount of AES blocks\n"); return -EINVAL; } @@ -566,7 +566,7 @@ static int s5p_aes_cbc_decrypt(struct ablkcipher_request *req) static int s5p_aes_cra_init(struct crypto_tfm *tfm) { - struct s5p_aes_ctx *ctx = crypto_tfm_ctx(tfm); + struct s5p_aes_ctx *ctx = crypto_tfm_ctx(tfm); ctx->dev = s5p_dev; tfm->crt_ablkcipher.reqsize = sizeof(struct s5p_aes_reqctx); @@ -701,7 +701,7 @@ static int s5p_aes_probe(struct platform_device *pdev) goto err_algs; } - pr_info("s5p-sss driver registered\n"); + dev_info(dev, "s5p-sss driver registered\n"); return 0; diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index 6c4f91c5e..c3f3d89e4 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -182,7 +182,6 @@ struct sahara_sha_reqctx { u8 buf[SAHARA_MAX_SHA_BLOCK_SIZE]; u8 rembuf[SAHARA_MAX_SHA_BLOCK_SIZE]; u8 context[SHA256_DIGEST_SIZE + 4]; - struct mutex mutex; unsigned int mode; unsigned int digest_size; unsigned int context_size; @@ -1096,7 +1095,6 @@ static int sahara_sha_enqueue(struct ahash_request *req, int last) if (!req->nbytes && !last) return 0; - mutex_lock(&rctx->mutex); rctx->last = last; if (!rctx->active) { @@ -1109,7 +1107,6 @@ static int sahara_sha_enqueue(struct ahash_request *req, int last) mutex_unlock(&dev->queue_mutex); wake_up_process(dev->kthread); - mutex_unlock(&rctx->mutex); return ret; } @@ -1137,8 +1134,6 @@ static int sahara_sha_init(struct ahash_request *req) rctx->context_size = rctx->digest_size + 4; rctx->active = 0; - mutex_init(&rctx->mutex); - return 0; } @@ -1167,26 +1162,18 @@ static int sahara_sha_digest(struct ahash_request *req) static int sahara_sha_export(struct ahash_request *req, void *out) { - struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); - struct sahara_ctx *ctx = crypto_ahash_ctx(ahash); struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); - memcpy(out, ctx, sizeof(struct sahara_ctx)); - memcpy(out + sizeof(struct sahara_sha_reqctx), rctx, - sizeof(struct sahara_sha_reqctx)); + memcpy(out, rctx, sizeof(struct sahara_sha_reqctx)); return 0; } static int sahara_sha_import(struct ahash_request *req, const void *in) { - struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); - struct sahara_ctx *ctx = crypto_ahash_ctx(ahash); struct sahara_sha_reqctx *rctx = ahash_request_ctx(req); - memcpy(ctx, in, sizeof(struct sahara_ctx)); - memcpy(rctx, in + sizeof(struct sahara_sha_reqctx), - sizeof(struct sahara_sha_reqctx)); + memcpy(rctx, in, sizeof(struct sahara_sha_reqctx)); return 0; } @@ -1272,6 +1259,7 @@ static struct ahash_alg sha_v3_algs[] = { .export = sahara_sha_export, .import = sahara_sha_import, .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.statesize = sizeof(struct sahara_sha_reqctx), .halg.base = { .cra_name = "sha1", .cra_driver_name = "sahara-sha1", @@ -1299,6 +1287,7 @@ static struct ahash_alg sha_v4_algs[] = { .export = sahara_sha_export, .import = sahara_sha_import, .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.statesize = sizeof(struct sahara_sha_reqctx), .halg.base = { .cra_name = "sha256", .cra_driver_name = "sahara-sha256", diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c index a19ee127e..3830d7c4e 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c @@ -35,6 +35,7 @@ static int sun4i_ss_opti_poll(struct ablkcipher_request *areq) unsigned int todo; struct sg_mapping_iter mi, mo; unsigned int oi, oo; /* offset for in and out */ + unsigned long flags; if (areq->nbytes == 0) return 0; @@ -49,7 +50,7 @@ static int sun4i_ss_opti_poll(struct ablkcipher_request *areq) return -EINVAL; } - spin_lock_bh(&ss->slock); + spin_lock_irqsave(&ss->slock, flags); for (i = 0; i < op->keylen; i += 4) writel(*(op->key + i / 4), ss->base + SS_KEY0 + i); @@ -117,7 +118,7 @@ release_ss: sg_miter_stop(&mi); sg_miter_stop(&mo); writel(0, ss->base + SS_CTL); - spin_unlock_bh(&ss->slock); + spin_unlock_irqrestore(&ss->slock, flags); return err; } @@ -149,6 +150,7 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq) unsigned int ob = 0; /* offset in buf */ unsigned int obo = 0; /* offset in bufo*/ unsigned int obl = 0; /* length of data in bufo */ + unsigned long flags; if (areq->nbytes == 0) return 0; @@ -181,7 +183,7 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq) if (no_chunk == 1) return sun4i_ss_opti_poll(areq); - spin_lock_bh(&ss->slock); + spin_lock_irqsave(&ss->slock, flags); for (i = 0; i < op->keylen; i += 4) writel(*(op->key + i / 4), ss->base + SS_KEY0 + i); @@ -251,11 +253,10 @@ static int sun4i_ss_cipher_poll(struct ablkcipher_request *areq) spaces = readl(ss->base + SS_FCSR); rx_cnt = SS_RXFIFO_SPACES(spaces); tx_cnt = SS_TXFIFO_SPACES(spaces); - dev_dbg(ss->dev, "%x %u/%u %u/%u cnt=%u %u/%u %u/%u cnt=%u %u %u\n", + dev_dbg(ss->dev, "%x %u/%u %u/%u cnt=%u %u/%u %u/%u cnt=%u %u\n", mode, oi, mi.length, ileft, areq->nbytes, rx_cnt, - oo, mo.length, oleft, areq->nbytes, tx_cnt, - todo, ob); + oo, mo.length, oleft, areq->nbytes, tx_cnt, ob); if (tx_cnt == 0) continue; @@ -308,7 +309,7 @@ release_ss: sg_miter_stop(&mi); sg_miter_stop(&mo); writel(0, ss->base + SS_CTL); - spin_unlock_bh(&ss->slock); + spin_unlock_irqrestore(&ss->slock, flags); return err; } diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index aae05547b..b7ee8d301 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -835,6 +835,16 @@ struct talitos_ahash_req_ctx { struct scatterlist *psrc; }; +struct talitos_export_state { + u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)]; + u8 buf[HASH_MAX_BLOCK_SIZE]; + unsigned int swinit; + unsigned int first; + unsigned int last; + unsigned int to_hash_later; + unsigned int nbuf; +}; + static int aead_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen) { @@ -1981,6 +1991,46 @@ static int ahash_digest(struct ahash_request *areq) return ahash_process_req(areq, areq->nbytes); } +static int ahash_export(struct ahash_request *areq, void *out) +{ + struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct talitos_export_state *export = out; + + memcpy(export->hw_context, req_ctx->hw_context, + req_ctx->hw_context_size); + memcpy(export->buf, req_ctx->buf, req_ctx->nbuf); + export->swinit = req_ctx->swinit; + export->first = req_ctx->first; + export->last = req_ctx->last; + export->to_hash_later = req_ctx->to_hash_later; + export->nbuf = req_ctx->nbuf; + + return 0; +} + +static int ahash_import(struct ahash_request *areq, const void *in) +{ + struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq); + const struct talitos_export_state *export = in; + + memset(req_ctx, 0, sizeof(*req_ctx)); + req_ctx->hw_context_size = + (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE) + ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 + : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512; + memcpy(req_ctx->hw_context, export->hw_context, + req_ctx->hw_context_size); + memcpy(req_ctx->buf, export->buf, export->nbuf); + req_ctx->swinit = export->swinit; + req_ctx->first = export->first; + req_ctx->last = export->last; + req_ctx->to_hash_later = export->to_hash_later; + req_ctx->nbuf = export->nbuf; + + return 0; +} + struct keyhash_result { struct completion completion; int err; @@ -2458,6 +2508,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = MD5_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "md5", .cra_driver_name = "md5-talitos", @@ -2473,6 +2524,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "sha1", .cra_driver_name = "sha1-talitos", @@ -2488,6 +2540,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = SHA224_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "sha224", .cra_driver_name = "sha224-talitos", @@ -2503,6 +2556,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "sha256", .cra_driver_name = "sha256-talitos", @@ -2518,6 +2572,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = SHA384_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "sha384", .cra_driver_name = "sha384-talitos", @@ -2533,6 +2588,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = SHA512_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "sha512", .cra_driver_name = "sha512-talitos", @@ -2548,6 +2604,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = MD5_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "hmac(md5)", .cra_driver_name = "hmac-md5-talitos", @@ -2563,6 +2620,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = SHA1_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "hmac(sha1)", .cra_driver_name = "hmac-sha1-talitos", @@ -2578,6 +2636,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = SHA224_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "hmac(sha224)", .cra_driver_name = "hmac-sha224-talitos", @@ -2593,6 +2652,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = SHA256_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "hmac(sha256)", .cra_driver_name = "hmac-sha256-talitos", @@ -2608,6 +2668,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = SHA384_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "hmac(sha384)", .cra_driver_name = "hmac-sha384-talitos", @@ -2623,6 +2684,7 @@ static struct talitos_alg_template driver_algs[] = { { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { .halg.digestsize = SHA512_DIGEST_SIZE, + .halg.statesize = sizeof(struct talitos_export_state), .halg.base = { .cra_name = "hmac(sha512)", .cra_driver_name = "hmac-sha512-talitos", @@ -2814,6 +2876,8 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, t_alg->algt.alg.hash.finup = ahash_finup; t_alg->algt.alg.hash.digest = ahash_digest; t_alg->algt.alg.hash.setkey = ahash_setkey; + t_alg->algt.alg.hash.import = ahash_import; + t_alg->algt.alg.hash.export = ahash_export; if (!(priv->features & TALITOS_FTR_HMAC_OK) && !strncmp(alg->cra_name, "hmac", 4)) { |