| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20221130055214.2416888-3-jiajie.ho@starfivetech.com>
Date: Wed, 30 Nov 2022 13:52:10 +0800
From: Jia Jie Ho <jiajie.ho@...rfivetech.com>
To: Herbert Xu <herbert@...dor.apana.org.au>,
"David S . Miller" <davem@...emloft.net>,
Rob Herring <robh+dt@...nel.org>,
Krzysztof Kozlowski <krzysztof.kozlowski+dt@...aro.org>
CC: <linux-crypto@...r.kernel.org>, <devicetree@...r.kernel.org>,
<linux-kernel@...r.kernel.org>, <linux-riscv@...ts.infradead.org>,
Jia Jie Ho <jiajie.ho@...rfivetech.com>
Subject: [PATCH 2/6] crypto: starfive - Add hash and HMAC support
Adding hash function and HMAC support for SHA-2 family
and SM3 to Starfive crypto hardware driver.
Signed-off-by: Jia Jie Ho <jiajie.ho@...rfivetech.com>
Signed-off-by: Huan Feng <huan.feng@...rfivetech.com>
---
drivers/crypto/starfive/Makefile | 2 +-
drivers/crypto/starfive/starfive-cryp.c | 22 +
drivers/crypto/starfive/starfive-hash.c | 1152 +++++++++++++++++++++++
drivers/crypto/starfive/starfive-regs.h | 45 +
drivers/crypto/starfive/starfive-str.h | 52 +-
5 files changed, 1271 insertions(+), 2 deletions(-)
create mode 100644 drivers/crypto/starfive/starfive-hash.c
diff --git a/drivers/crypto/starfive/Makefile b/drivers/crypto/starfive/Makefile
index 5a84f808a671..437b8f036038 100644
--- a/drivers/crypto/starfive/Makefile
+++ b/drivers/crypto/starfive/Makefile
@@ -1,4 +1,4 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_CRYPTO_DEV_STARFIVE) += starfive-crypto.o
-starfive-crypto-objs := starfive-cryp.o
+starfive-crypto-objs := starfive-cryp.o starfive-hash.o
diff --git a/drivers/crypto/starfive/starfive-cryp.c b/drivers/crypto/starfive/starfive-cryp.c
index 574f9e8f4cc1..9f77cae758ac 100644
--- a/drivers/crypto/starfive/starfive-cryp.c
+++ b/drivers/crypto/starfive/starfive-cryp.c
@@ -109,6 +109,8 @@ static int starfive_cryp_probe(struct platform_device *pdev)
if (!sdev)
return -ENOMEM;
+ mutex_init(&sdev->lock);
+
sdev->dev = dev;
res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "secreg");
@@ -117,6 +119,7 @@ static int starfive_cryp_probe(struct platform_device *pdev)
if (IS_ERR(sdev->io_base))
return PTR_ERR(sdev->io_base);
+ sdev->io_phys_base = res->start;
sdev->use_side_channel_mitigation =
device_property_read_bool(dev, "enable-side-channel-mitigation");
sdev->use_dma = device_property_read_bool(dev, "enable-dma");
@@ -160,6 +163,12 @@ static int starfive_cryp_probe(struct platform_device *pdev)
pages = get_order(STARFIVE_MSG_BUFFER_SIZE);
+ sdev->hash_data = (void *)__get_free_pages(GFP_KERNEL | GFP_DMA32, pages);
+ if (!sdev->hash_data) {
+ dev_err(sdev->dev, "Can't allocate hash buffer pages when unaligned\n");
+ goto err_hash_data;
+ }
+
sdev->pages_count = pages >> 1;
sdev->data_buf_len = STARFIVE_MSG_BUFFER_SIZE >> 1;
@@ -174,13 +183,21 @@ static int starfive_cryp_probe(struct platform_device *pdev)
if (ret)
goto err_engine_start;
+ ret = starfive_hash_register_algs();
+ if (ret)
+ goto err_algs_hash;
+
dev_info(dev, "Crypto engine started\n");
return 0;
+err_algs_hash:
+ crypto_engine_stop(sdev->engine);
err_engine_start:
crypto_engine_exit(sdev->engine);
err_engine:
+ free_pages((unsigned long)sdev->hash_data, pages);
+err_hash_data:
starfive_dma_cleanup(sdev);
err_dma_init:
spin_lock(&dev_list.lock);
@@ -197,11 +214,16 @@ static int starfive_cryp_remove(struct platform_device *pdev)
if (!sdev)
return -ENODEV;
+ starfive_hash_unregister_algs();
+
crypto_engine_stop(sdev->engine);
crypto_engine_exit(sdev->engine);
starfive_dma_cleanup(sdev);
+ free_pages((unsigned long)sdev->hash_data, sdev->pages_count);
+ sdev->hash_data = NULL;
+
spin_lock(&dev_list.lock);
list_del(&sdev->list);
spin_unlock(&dev_list.lock);
diff --git a/drivers/crypto/starfive/starfive-hash.c b/drivers/crypto/starfive/starfive-hash.c
new file mode 100644
index 000000000000..c85dec784df9
--- /dev/null
+++ b/drivers/crypto/starfive/starfive-hash.c
@@ -0,0 +1,1152 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Hash function and HMAC support for StarFive driver
+ *
+ * Copyright (c) 2022 StarFive Technology
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/crypto.h>
+#include <linux/dma-direct.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/reset.h>
+#include <linux/amba/pl080.h>
+
+#include <crypto/hash.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/internal/hash.h>
+
+#include "starfive-str.h"
+
+#define HASH_OP_UPDATE 1
+#define HASH_OP_FINAL 2
+
+#define HASH_FLAGS_INIT BIT(0)
+#define HASH_FLAGS_FINAL BIT(1)
+#define HASH_FLAGS_FINUP BIT(2)
+
+#define STARFIVE_MAX_ALIGN_SIZE SHA512_BLOCK_SIZE
+
+#define STARFIVE_HASH_BUFLEN 8192
+#define STARFIVE_HASH_THRES 2048
+
+static inline int starfive_hash_wait_hmac_done(struct starfive_sec_ctx *ctx)
+{
+ struct starfive_sec_dev *sdev = ctx->sdev;
+ u32 status;
+
+ return readl_relaxed_poll_timeout(sdev->io_base + STARFIVE_HASH_SHACSR, status,
+ (status & STARFIVE_HASH_HMAC_DONE), 10, 100000);
+}
+
+static inline int starfive_hash_wait_busy(struct starfive_sec_ctx *ctx)
+{
+ struct starfive_sec_dev *sdev = ctx->sdev;
+ u32 status;
+
+ return readl_relaxed_poll_timeout(sdev->io_base + STARFIVE_HASH_SHACSR, status,
+ !(status & STARFIVE_HASH_BUSY), 10, 100000);
+}
+
+static inline int starfive_hash_wait_key_done(struct starfive_sec_ctx *ctx)
+{
+ struct starfive_sec_dev *sdev = ctx->sdev;
+ u32 status;
+
+ return readl_relaxed_poll_timeout(sdev->io_base + STARFIVE_HASH_SHACSR, status,
+ (status & STARFIVE_HASH_KEY_DONE), 10, 100000);
+}
+
+static int starfive_get_hash_size(struct starfive_sec_ctx *ctx)
+{
+ unsigned int hashsize;
+
+ switch (ctx->hash_mode & STARFIVE_HASH_MODE_MASK) {
+ case STARFIVE_HASH_SHA224:
+ hashsize = SHA224_DIGEST_SIZE;
+ break;
+ case STARFIVE_HASH_SHA256:
+ hashsize = SHA256_DIGEST_SIZE;
+ break;
+ case STARFIVE_HASH_SHA384:
+ hashsize = SHA384_DIGEST_SIZE;
+ break;
+ case STARFIVE_HASH_SHA512:
+ hashsize = SHA512_DIGEST_SIZE;
+ break;
+ case STARFIVE_HASH_SM3:
+ hashsize = SM3_DIGEST_SIZE;
+ break;
+ default:
+ return 0;
+ }
+
+ return hashsize;
+}
+
+static void starfive_hash_start(struct starfive_sec_ctx *ctx, int flags)
+{
+ struct starfive_sec_request_ctx *rctx = ctx->rctx;
+ struct starfive_sec_dev *sdev = ctx->sdev;
+
+ rctx->csr.hash.v = starfive_sec_read(sdev, STARFIVE_HASH_SHACSR);
+ rctx->csr.hash.firstb = 0;
+
+ if (flags)
+ rctx->csr.hash.final = 1;
+
+ starfive_sec_write(sdev, STARFIVE_HASH_SHACSR, rctx->csr.hash.v);
+}
+
+static int starfive_hash_hmac_key(struct starfive_sec_ctx *ctx)
+{
+ struct starfive_sec_request_ctx *rctx = ctx->rctx;
+ struct starfive_sec_dev *sdev = ctx->sdev;
+ int klen = ctx->keylen, loop;
+ unsigned int *key = (unsigned int *)ctx->key;
+ unsigned char *cl;
+
+ starfive_sec_write(sdev, STARFIVE_HASH_SHAWKLEN, ctx->keylen);
+
+ rctx->csr.hash.hmac = !!(ctx->hash_mode & STARFIVE_HASH_HMAC_FLAGS);
+ rctx->csr.hash.key_flag = 1;
+
+ starfive_sec_write(sdev, STARFIVE_HASH_SHACSR, rctx->csr.hash.v);
+
+ for (loop = 0; loop < klen / sizeof(unsigned int); loop++, key++)
+ starfive_sec_write(sdev, STARFIVE_HASH_SHAWKR, *key);
+
+ if (klen & 0x3) {
+ cl = (unsigned char *)key;
+ for (loop = 0; loop < (klen & 0x3); loop++, cl++)
+ starfive_sec_writeb(sdev, STARFIVE_HASH_SHAWKR, *cl);
+ }
+
+ if (starfive_hash_wait_key_done(ctx)) {
+ dev_err(sdev->dev, " starfive_hash_wait_key_done error\n");
+ return -ETIMEDOUT;
+ }
+
+ return 0;
+}
+
+static void starfive_hash_dma_callback(void *param)
+{
+ struct starfive_sec_dev *sdev = param;
+
+ complete(&sdev->sec_comp_m);
+}
+
+static int starfive_hash_xmit_dma(struct starfive_sec_ctx *ctx, int flags)
+{
+ struct starfive_sec_request_ctx *rctx = ctx->rctx;
+ struct starfive_sec_dev *sdev = ctx->sdev;
+ struct dma_async_tx_descriptor *in_desc;
+ dma_cookie_t cookie;
+ union starfive_alg_cr alg_cr;
+ int total_len;
+ int ret;
+
+ if (!rctx->bufcnt)
+ return 0;
+
+ ctx->hash_len_total += rctx->bufcnt;
+
+ total_len = rctx->bufcnt;
+
+ starfive_sec_write(sdev, STARFIVE_DMA_IN_LEN_OFFSET, rctx->bufcnt);
+
+ total_len = (total_len & 0x3) ? (((total_len >> 2) + 1) << 2) : total_len;
+
+ memset(sdev->hash_data + rctx->bufcnt, 0, total_len - rctx->bufcnt);
+
+ alg_cr.v = 0;
+ alg_cr.start = 1;
+ alg_cr.hash_dma_en = 1;
+ starfive_sec_write(sdev, STARFIVE_ALG_CR_OFFSET, alg_cr.v);
+
+ sg_init_table(&ctx->sg[0], 1);
+ sg_set_buf(&ctx->sg[0], sdev->hash_data, total_len);
+ sg_dma_address(&ctx->sg[0]) = phys_to_dma(sdev->dev, (unsigned long long)(sdev->hash_data));
+ sg_dma_len(&ctx->sg[0]) = total_len;
+
+ ret = dma_map_sg(sdev->dev, &ctx->sg[0], 1, DMA_TO_DEVICE);
+ if (!ret) {
+ dev_err(sdev->dev, "dma_map_sg() error\n");
+ return -EINVAL;
+ }
+
+ sdev->cfg_in.direction = DMA_MEM_TO_DEV;
+ sdev->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+ sdev->cfg_in.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+ sdev->cfg_in.src_maxburst = sdev->dma_maxburst;
+ sdev->cfg_in.dst_maxburst = sdev->dma_maxburst;
+ sdev->cfg_in.dst_addr = sdev->io_phys_base + STARFIVE_ALG_FIFO_OFFSET;
+
+ dmaengine_slave_config(sdev->sec_xm_m, &sdev->cfg_in);
+
+ in_desc = dmaengine_prep_slave_sg(sdev->sec_xm_m, &ctx->sg[0],
+ 1, DMA_MEM_TO_DEV,
+ DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+ if (!in_desc)
+ return -EINVAL;
+
+ reinit_completion(&sdev->sec_comp_m);
+
+ in_desc->callback = starfive_hash_dma_callback;
+ in_desc->callback_param = sdev;
+
+ cookie = dmaengine_submit(in_desc);
+ dma_async_issue_pending(sdev->sec_xm_m);
+
+ if (!wait_for_completion_timeout(&sdev->sec_comp_m,
+ msecs_to_jiffies(10000))) {
+ dev_err(sdev->dev, "wait_for_completion_timeout error, cookie = %x\n",
+ dma_async_is_tx_complete(sdev->sec_xm_p, cookie,
+ NULL, NULL));
+ }
+
+ dma_unmap_sg(sdev->dev, &ctx->sg[0], 1, DMA_TO_DEVICE);
+
+ alg_cr.v = 0;
+ alg_cr.clear = 1;
+ starfive_sec_write(sdev, STARFIVE_ALG_CR_OFFSET, alg_cr.v);
+
+ return 0;
+}
+
+static int starfive_hash_xmit_cpu(struct starfive_sec_ctx *ctx, int flags)
+{
+ struct starfive_sec_request_ctx *rctx = ctx->rctx;
+ struct starfive_sec_dev *sdev = ctx->sdev;
+ int total_len, mlen, loop;
+ unsigned int *buffer;
+ unsigned char *cl;
+
+ if (!rctx->bufcnt)
+ return 0;
+
+ ctx->hash_len_total += rctx->bufcnt;
+
+ total_len = rctx->bufcnt;
+ mlen = total_len / sizeof(u32);
+ buffer = (unsigned int *)ctx->buffer;
+
+ for (loop = 0; loop < mlen; loop++, buffer++) {
+ starfive_sec_write(sdev, STARFIVE_HASH_SHAWDR, *buffer);
+ udelay(2);
+ }
+
+ if (total_len & 0x3) {
+ cl = (unsigned char *)buffer;
+ for (loop = 0; loop < (total_len & 0x3); loop++, cl++) {
+ starfive_sec_writeb(sdev, STARFIVE_HASH_SHAWDR, *cl);
+ udelay(2);
+ }
+ }
+
+ return 0;
+}
+
+static void starfive_hash_append_sg(struct starfive_sec_request_ctx *rctx)
+{
+ struct starfive_sec_ctx *ctx = rctx->ctx;
+ size_t count;
+
+ while ((rctx->bufcnt < rctx->buflen) && rctx->total) {
+ count = min(rctx->in_sg->length - rctx->offset, rctx->total);
+ count = min(count, rctx->buflen - rctx->bufcnt);
+
+ if (count <= 0) {
+ if (rctx->in_sg->length == 0 && !sg_is_last(rctx->in_sg)) {
+ rctx->in_sg = sg_next(rctx->in_sg);
+ continue;
+ } else {
+ break;
+ }
+ }
+
+ scatterwalk_map_and_copy(ctx->buffer + rctx->bufcnt, rctx->in_sg,
+ rctx->offset, count, 0);
+
+ rctx->bufcnt += count;
+ rctx->offset += count;
+ rctx->total -= count;
+
+ if (rctx->offset == rctx->in_sg->length) {
+ rctx->in_sg = sg_next(rctx->in_sg);
+ if (rctx->in_sg)
+ rctx->offset = 0;
+ else
+ rctx->total = 0;
+ }
+ }
+}
+
+static int starfive_hash_xmit(struct starfive_sec_ctx *ctx, int flags)
+{
+ struct starfive_sec_request_ctx *rctx = ctx->rctx;
+ struct starfive_sec_dev *sdev = ctx->sdev;
+ int ret;
+
+ rctx->csr.hash.v = 0;
+ rctx->csr.hash.reset = 1;
+ starfive_sec_write(sdev, STARFIVE_HASH_SHACSR, rctx->csr.hash.v);
+
+ if (starfive_hash_wait_busy(ctx)) {
+ dev_err(sdev->dev, "Error resetting engine.\n");
+ return -ETIMEDOUT;
+ }
+
+ rctx->csr.hash.v = 0;
+ rctx->csr.hash.mode = ctx->hash_mode & STARFIVE_HASH_MODE_MASK;
+
+ if (ctx->hash_mode & STARFIVE_HASH_HMAC_FLAGS) {
+ ret = starfive_hash_hmac_key(ctx);
+ if (ret)
+ return ret;
+ } else {
+ rctx->csr.hash.start = 1;
+ rctx->csr.hash.firstb = 1;
+ starfive_sec_write(sdev, STARFIVE_HASH_SHACSR, rctx->csr.hash.v);
+ }
+
+ if (ctx->sdev->use_dma)
+ ret = starfive_hash_xmit_dma(ctx, flags);
+ else
+ ret = starfive_hash_xmit_cpu(ctx, flags);
+
+ if (ret)
+ return ret;
+
+ rctx->flags |= HASH_FLAGS_FINAL;
+ starfive_hash_start(ctx, flags);
+
+ if (starfive_hash_wait_busy(ctx)) {
+ dev_err(sdev->dev, "Timeout waiting for hash completion\n");
+ return -ETIMEDOUT;
+ }
+
+ if (ctx->hash_mode & STARFIVE_HASH_HMAC_FLAGS)
+ if (starfive_hash_wait_hmac_done(ctx)) {
+ dev_err(sdev->dev, "Timeout waiting for hmac completion\n");
+ return -ETIMEDOUT;
+ }
+
+ return 0;
+}
+
+static int starfive_hash_update_req(struct starfive_sec_ctx *ctx)
+{
+ struct starfive_sec_request_ctx *rctx = ctx->rctx;
+ int ret, final;
+
+ final = (rctx->flags & HASH_FLAGS_FINUP);
+
+ while ((rctx->total >= rctx->buflen) ||
+ (rctx->bufcnt + rctx->total >= rctx->buflen)) {
+ starfive_hash_append_sg(rctx);
+ ret = starfive_hash_xmit(ctx, 0);
+ rctx->bufcnt = 0;
+ }
+
+ starfive_hash_append_sg(rctx);
+
+ if (final) {
+ ret = starfive_hash_xmit(ctx, (rctx->flags & HASH_FLAGS_FINUP));
+ rctx->bufcnt = 0;
+ }
+
+ return ret;
+}
+
+static int starfive_hash_final_req(struct starfive_sec_ctx *ctx)
+{
+ struct ahash_request *req = ctx->rctx->req.hreq;
+ struct starfive_sec_request_ctx *rctx = ahash_request_ctx(req);
+ int ret;
+
+ ret = starfive_hash_xmit(ctx, 1);
+ rctx->bufcnt = 0;
+
+ return ret;
+}
+
+static int starfive_hash_out_cpu(struct ahash_request *req)
+{
+ struct starfive_sec_request_ctx *rctx = ahash_request_ctx(req);
+ struct starfive_sec_ctx *ctx = rctx->ctx;
+ int count, *data;
+ int mlen;
+
+ if (!req->result)
+ return 0;
+
+ mlen = starfive_get_hash_size(ctx) / sizeof(u32);
+ data = (u32 *)req->result;
+
+ for (count = 0; count < mlen; count++)
+ data[count] = starfive_sec_read(ctx->sdev, STARFIVE_HASH_SHARDR);
+
+ return 0;
+}
+
+static int starfive_hash_copy_hash(struct ahash_request *req)
+{
+ struct starfive_sec_request_ctx *rctx = ahash_request_ctx(req);
+ struct starfive_sec_ctx *ctx = rctx->ctx;
+ int hashsize;
+ int ret;
+
+ hashsize = starfive_get_hash_size(ctx);
+
+ ret = starfive_hash_out_cpu(req);
+
+ if (ret)
+ return ret;
+
+ memcpy(rctx->hash_digest_mid, req->result, hashsize);
+ rctx->hash_digest_len = hashsize;
+
+ return ret;
+}
+
+static void starfive_hash_finish_req(struct ahash_request *req, int err)
+{
+ struct starfive_sec_request_ctx *rctx = ahash_request_ctx(req);
+ struct starfive_sec_dev *sdev = rctx->sdev;
+
+ if (!err && (HASH_FLAGS_FINAL & rctx->flags)) {
+ err = starfive_hash_copy_hash(req);
+ rctx->flags &= ~(HASH_FLAGS_FINAL |
+ HASH_FLAGS_INIT);
+ }
+
+ crypto_finalize_hash_request(sdev->engine, req, err);
+}
+
+static int starfive_hash_prepare_req(struct crypto_engine *engine, void *areq)
+{
+ struct ahash_request *req = container_of(areq, struct ahash_request,
+ base);
+ struct starfive_sec_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
+ struct starfive_sec_dev *sdev = ctx->sdev;
+ struct starfive_sec_request_ctx *rctx;
+
+ if (!sdev)
+ return -ENODEV;
+
+ rctx = ahash_request_ctx(req);
+
+ rctx->req.hreq = req;
+
+ return 0;
+}
+
+static int starfive_hash_one_request(struct crypto_engine *engine, void *areq)
+{
+ struct ahash_request *req = container_of(areq, struct ahash_request,
+ base);
+ struct starfive_sec_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
+ struct starfive_sec_dev *sdev = ctx->sdev;
+ struct starfive_sec_request_ctx *rctx;
+ int err = 0;
+
+ if (!sdev)
+ return -ENODEV;
+
+ rctx = ahash_request_ctx(req);
+
+ mutex_lock(&ctx->sdev->lock);
+
+ if (rctx->op == HASH_OP_UPDATE)
+ err = starfive_hash_update_req(ctx);
+ else if (rctx->op == HASH_OP_FINAL)
+ err = starfive_hash_final_req(ctx);
+
+ if (err != -EINPROGRESS)
+ /* done task will not finish it, so do it here */
+ starfive_hash_finish_req(req, err);
+
+ mutex_unlock(&ctx->sdev->lock);
+
+ return 0;
+}
+
+static int starfive_hash_handle_queue(struct starfive_sec_dev *sdev,
+ struct ahash_request *req)
+{
+ return crypto_transfer_hash_request_to_engine(sdev->engine, req);
+}
+
+static int starfive_hash_enqueue(struct ahash_request *req, unsigned int op)
+{
+ struct starfive_sec_request_ctx *rctx = ahash_request_ctx(req);
+ struct starfive_sec_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+ struct starfive_sec_dev *sdev = ctx->sdev;
+
+ rctx->op = op;
+
+ return starfive_hash_handle_queue(sdev, req);
+}
+
+static int starfive_hash_init(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct starfive_sec_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct starfive_sec_request_ctx *rctx = ahash_request_ctx(req);
+ struct starfive_sec_dev *sdev = ctx->sdev;
+
+ memset(rctx, 0, sizeof(struct starfive_sec_request_ctx));
+
+ rctx->sdev = sdev;
+ rctx->ctx = ctx;
+ rctx->req.hreq = req;
+ rctx->bufcnt = 0;
+
+ rctx->total = 0;
+ rctx->offset = 0;
+ rctx->bufcnt = 0;
+ rctx->buflen = STARFIVE_HASH_BUFLEN;
+
+ memset(ctx->buffer, 0, STARFIVE_HASH_BUFLEN);
+
+ ctx->rctx = rctx;
+
+ dev_dbg(sdev->dev, "%s Flags %lx\n", __func__, rctx->flags);
+
+ return 0;
+}
+
+static int starfive_hash_update(struct ahash_request *req)
+{
+ struct starfive_sec_request_ctx *rctx = ahash_request_ctx(req);
+
+ if (!req->nbytes)
+ return 0;
+
+ rctx->total = req->nbytes;
+ rctx->in_sg = req->src;
+ rctx->offset = 0;
+
+ if ((rctx->bufcnt + rctx->total < rctx->buflen)) {
+ starfive_hash_append_sg(rctx);
+ return 0;
+ }
+
+ return starfive_hash_enqueue(req, HASH_OP_UPDATE);
+}
+
+static int starfive_hash_final(struct ahash_request *req)
+{
+ struct starfive_sec_request_ctx *rctx = ahash_request_ctx(req);
+ struct starfive_sec_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+
+ rctx->flags |= HASH_FLAGS_FINUP;
+
+ if (ctx->fallback_available && rctx->bufcnt < STARFIVE_HASH_THRES) {
+ if (ctx->hash_mode & STARFIVE_HASH_HMAC_FLAGS)
+ crypto_shash_setkey(ctx->fallback.shash, ctx->key,
+ ctx->keylen);
+
+ return crypto_shash_tfm_digest(ctx->fallback.shash, ctx->buffer,
+ rctx->bufcnt, req->result);
+ }
+
+ return starfive_hash_enqueue(req, HASH_OP_FINAL);
+}
+
+static int starfive_hash_finup(struct ahash_request *req)
+{
+ struct starfive_sec_request_ctx *rctx = ahash_request_ctx(req);
+ int err1, err2;
+ int nents;
+
+ nents = sg_nents_for_len(req->src, req->nbytes);
+
+ rctx->flags |= HASH_FLAGS_FINUP;
+
+ err1 = starfive_hash_update(req);
+
+ if (err1 == -EINPROGRESS || err1 == -EBUSY)
+ return err1;
+
+ /*
+ * final() has to be always called to cleanup resources
+ * even if update() failed, except EINPROGRESS
+ */
+ err2 = starfive_hash_final(req);
+
+ return err1 ?: err2;
+}
+
+static int starfive_hash_digest(struct ahash_request *req)
+{
+ return starfive_hash_init(req) ?: starfive_hash_finup(req);
+}
+
+static int starfive_hash_export(struct ahash_request *req, void *out)
+{
+ struct starfive_sec_request_ctx *rctx = ahash_request_ctx(req);
+
+ memcpy(out, rctx, sizeof(*rctx));
+
+ return 0;
+}
+
+static int starfive_hash_import(struct ahash_request *req, const void *in)
+{
+ struct starfive_sec_request_ctx *rctx = ahash_request_ctx(req);
+
+ memcpy(rctx, in, sizeof(*rctx));
+
+ return 0;
+}
+
+static int starfive_hash_cra_init_algs(struct crypto_tfm *tfm,
+ const char *algs_hmac_name,
+ unsigned int mode)
+{
+ struct starfive_sec_ctx *ctx = crypto_tfm_ctx(tfm);
+ const char *alg_name = crypto_tfm_alg_name(tfm);
+
+ ctx->sdev = starfive_sec_find_dev(ctx);
+
+ if (!ctx->sdev)
+ return -ENODEV;
+
+ ctx->fallback_available = true;
+ ctx->fallback.shash = crypto_alloc_shash(alg_name, 0,
+ CRYPTO_ALG_NEED_FALLBACK);
+
+ if (IS_ERR(ctx->fallback.shash))
+ ctx->fallback_available = false;
+
+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+ sizeof(struct starfive_sec_request_ctx));
+
+ ctx->keylen = 0;
+ ctx->hash_mode = mode;
+ ctx->hash_len_total = 0;
+ ctx->buffer = ctx->sdev->hash_data;
+
+ if (algs_hmac_name)
+ ctx->hash_mode |= STARFIVE_HASH_HMAC_FLAGS;
+
+ ctx->enginectx.op.do_one_request = starfive_hash_one_request;
+ ctx->enginectx.op.prepare_request = starfive_hash_prepare_req;
+ ctx->enginectx.op.unprepare_request = NULL;
+
+ return 0;
+}
+
+static void starfive_hash_cra_exit(struct crypto_tfm *tfm)
+{
+ struct starfive_sec_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ crypto_free_shash(ctx->fallback.shash);
+
+ ctx->fallback.shash = NULL;
+ ctx->enginectx.op.do_one_request = NULL;
+ ctx->enginectx.op.prepare_request = NULL;
+ ctx->enginectx.op.unprepare_request = NULL;
+}
+
+static int starfive_hash_long_setkey(struct starfive_sec_ctx *ctx,
+ const u8 *key, unsigned int keylen,
+ const char *alg_name)
+{
+ struct crypto_wait wait;
+ struct ahash_request *req;
+ struct scatterlist sg;
+ struct crypto_ahash *ahash_tfm;
+ u8 *buf;
+ int ret;
+
+ ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0);
+ if (IS_ERR(ahash_tfm))
+ return PTR_ERR(ahash_tfm);
+
+ req = ahash_request_alloc(ahash_tfm, GFP_KERNEL);
+ if (!req) {
+ ret = -ENOMEM;
+ goto err_free_ahash;
+ }
+
+ crypto_init_wait(&wait);
+ ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+ crypto_req_done, &wait);
+ crypto_ahash_clear_flags(ahash_tfm, ~0);
+
+ buf = kzalloc(keylen + STARFIVE_MAX_ALIGN_SIZE, GFP_KERNEL);
+ if (!buf) {
+ ret = -ENOMEM;
+ goto err_free_req;
+ }
+
+ memcpy(buf, key, keylen);
+ sg_init_one(&sg, buf, keylen);
+ ahash_request_set_crypt(req, &sg, ctx->key, keylen);
+
+ ret = crypto_wait_req(crypto_ahash_digest(req), &wait);
+
+err_free_req:
+ ahash_request_free(req);
+err_free_ahash:
+ crypto_free_ahash(ahash_tfm);
+ return ret;
+}
+
+static int starfive_hash224_setkey(struct crypto_ahash *tfm,
+ const u8 *key, unsigned int keylen)
+{
+ struct starfive_sec_ctx *ctx = crypto_ahash_ctx(tfm);
+ unsigned int digestsize = crypto_ahash_digestsize(tfm);
+ unsigned int blocksize;
+ int ret = 0;
+
+ blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
+
+ if (keylen <= blocksize) {
+ memcpy(ctx->key, key, keylen);
+ ctx->keylen = keylen;
+ } else {
+ ctx->keylen = digestsize;
+ ret = starfive_hash_long_setkey(ctx, key, keylen, "starfive-sha224");
+ }
+
+ return ret;
+}
+
+static int starfive_hash256_setkey(struct crypto_ahash *tfm,
+ const u8 *key, unsigned int keylen)
+{
+ struct starfive_sec_ctx *ctx = crypto_ahash_ctx(tfm);
+ unsigned int digestsize = crypto_ahash_digestsize(tfm);
+ unsigned int blocksize;
+ int ret = 0;
+
+ blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
+
+ if (keylen <= blocksize) {
+ memcpy(ctx->key, key, keylen);
+ ctx->keylen = keylen;
+ } else {
+ ctx->keylen = digestsize;
+ ret = starfive_hash_long_setkey(ctx, key, keylen, "starfive-sha256");
+ }
+
+ return ret;
+}
+
+static int starfive_hash384_setkey(struct crypto_ahash *tfm,
+ const u8 *key, unsigned int keylen)
+{
+ struct starfive_sec_ctx *ctx = crypto_ahash_ctx(tfm);
+ unsigned int digestsize = crypto_ahash_digestsize(tfm);
+ unsigned int blocksize;
+ int ret = 0;
+
+ blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
+
+ if (keylen <= blocksize) {
+ memcpy(ctx->key, key, keylen);
+ ctx->keylen = keylen;
+ } else {
+ ctx->keylen = digestsize;
+ ret = starfive_hash_long_setkey(ctx, key, keylen, "starfive-sha384");
+ }
+
+ return ret;
+}
+
+static int starfive_hash512_setkey(struct crypto_ahash *tfm,
+ const u8 *key, unsigned int keylen)
+{
+ struct starfive_sec_ctx *ctx = crypto_ahash_ctx(tfm);
+ unsigned int digestsize = crypto_ahash_digestsize(tfm);
+ unsigned int blocksize;
+ int ret = 0;
+
+ blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
+
+ if (keylen <= blocksize) {
+ memcpy(ctx->key, key, keylen);
+ ctx->keylen = keylen;
+ } else {
+ ctx->keylen = digestsize;
+ ret = starfive_hash_long_setkey(ctx, key, keylen, "starfive-sha512");
+ }
+
+ return ret;
+}
+
+static int starfive_sm3_setkey(struct crypto_ahash *tfm,
+ const u8 *key, unsigned int keylen)
+{
+ struct starfive_sec_ctx *ctx = crypto_ahash_ctx(tfm);
+ unsigned int digestsize = crypto_ahash_digestsize(tfm);
+ unsigned int blocksize;
+ int ret = 0;
+
+ blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
+
+ if (keylen <= blocksize) {
+ memcpy(ctx->key, key, keylen);
+ ctx->keylen = keylen;
+ } else {
+ ctx->keylen = digestsize;
+ ret = starfive_hash_long_setkey(ctx, key, keylen, "starfive-sm3");
+ }
+
+ return ret;
+}
+
+static int starfive_hash_cra_sha224_init(struct crypto_tfm *tfm)
+{
+ return starfive_hash_cra_init_algs(tfm, NULL, STARFIVE_HASH_SHA224);
+}
+
+static int starfive_hash_cra_sha256_init(struct crypto_tfm *tfm)
+{
+ return starfive_hash_cra_init_algs(tfm, NULL, STARFIVE_HASH_SHA256);
+}
+
+static int starfive_hash_cra_sha384_init(struct crypto_tfm *tfm)
+{
+ return starfive_hash_cra_init_algs(tfm, NULL, STARFIVE_HASH_SHA384);
+}
+
+static int starfive_hash_cra_sha512_init(struct crypto_tfm *tfm)
+{
+ return starfive_hash_cra_init_algs(tfm, NULL, STARFIVE_HASH_SHA512);
+}
+
+static int starfive_hash_cra_sm3_init(struct crypto_tfm *tfm)
+{
+ return starfive_hash_cra_init_algs(tfm, NULL, STARFIVE_HASH_SM3);
+}
+
+static int starfive_hash_cra_hmac_sha224_init(struct crypto_tfm *tfm)
+{
+ return starfive_hash_cra_init_algs(tfm, "sha224", STARFIVE_HASH_SHA224);
+}
+
+static int starfive_hash_cra_hmac_sha256_init(struct crypto_tfm *tfm)
+{
+ return starfive_hash_cra_init_algs(tfm, "sha256", STARFIVE_HASH_SHA256);
+}
+
+static int starfive_hash_cra_hmac_sha384_init(struct crypto_tfm *tfm)
+{
+ return starfive_hash_cra_init_algs(tfm, "sha384", STARFIVE_HASH_SHA384);
+}
+
+static int starfive_hash_cra_hmac_sha512_init(struct crypto_tfm *tfm)
+{
+ return starfive_hash_cra_init_algs(tfm, "sha512", STARFIVE_HASH_SHA512);
+}
+
+static int starfive_hash_cra_hmac_sm3_init(struct crypto_tfm *tfm)
+{
+ return starfive_hash_cra_init_algs(tfm, "sm3", STARFIVE_HASH_SM3);
+}
+
+static struct ahash_alg algs_sha2_sm3[] = {
+{
+ .init = starfive_hash_init,
+ .update = starfive_hash_update,
+ .final = starfive_hash_final,
+ .finup = starfive_hash_finup,
+ .digest = starfive_hash_digest,
+ .export = starfive_hash_export,
+ .import = starfive_hash_import,
+ .halg = {
+ .digestsize = SHA224_DIGEST_SIZE,
+ .statesize = sizeof(struct starfive_sec_request_ctx),
+ .base = {
+ .cra_name = "sha224",
+ .cra_driver_name = "starfive-sha224",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA224_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct starfive_sec_ctx),
+ .cra_alignmask = 3,
+ .cra_init = starfive_hash_cra_sha224_init,
+ .cra_exit = starfive_hash_cra_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+},
+{
+ .init = starfive_hash_init,
+ .update = starfive_hash_update,
+ .final = starfive_hash_final,
+ .finup = starfive_hash_finup,
+ .digest = starfive_hash_digest,
+ .export = starfive_hash_export,
+ .import = starfive_hash_import,
+ .setkey = starfive_hash224_setkey,
+ .halg = {
+ .digestsize = SHA224_DIGEST_SIZE,
+ .statesize = sizeof(struct starfive_sec_request_ctx),
+ .base = {
+ .cra_name = "hmac(sha224)",
+ .cra_driver_name = "starfive-hmac-sha224",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA224_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct starfive_sec_ctx),
+ .cra_alignmask = 3,
+ .cra_init = starfive_hash_cra_hmac_sha224_init,
+ .cra_exit = starfive_hash_cra_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+},
+{
+ .init = starfive_hash_init,
+ .update = starfive_hash_update,
+ .final = starfive_hash_final,
+ .finup = starfive_hash_finup,
+ .digest = starfive_hash_digest,
+ .export = starfive_hash_export,
+ .import = starfive_hash_import,
+ .halg = {
+ .digestsize = SHA256_DIGEST_SIZE,
+ .statesize = sizeof(struct starfive_sec_request_ctx),
+ .base = {
+ .cra_name = "sha256",
+ .cra_driver_name = "starfive-sha256",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA256_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct starfive_sec_ctx),
+ .cra_alignmask = 3,
+ .cra_init = starfive_hash_cra_sha256_init,
+ .cra_exit = starfive_hash_cra_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+},
+{
+ .init = starfive_hash_init,
+ .update = starfive_hash_update,
+ .final = starfive_hash_final,
+ .finup = starfive_hash_finup,
+ .digest = starfive_hash_digest,
+ .export = starfive_hash_export,
+ .import = starfive_hash_import,
+ .setkey = starfive_hash256_setkey,
+ .halg = {
+ .digestsize = SHA256_DIGEST_SIZE,
+ .statesize = sizeof(struct starfive_sec_request_ctx),
+ .base = {
+ .cra_name = "hmac(sha256)",
+ .cra_driver_name = "starfive-hmac-sha256",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA256_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct starfive_sec_ctx),
+ .cra_alignmask = 3,
+ .cra_init = starfive_hash_cra_hmac_sha256_init,
+ .cra_exit = starfive_hash_cra_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+},
+{
+ .init = starfive_hash_init,
+ .update = starfive_hash_update,
+ .final = starfive_hash_final,
+ .finup = starfive_hash_finup,
+ .digest = starfive_hash_digest,
+ .export = starfive_hash_export,
+ .import = starfive_hash_import,
+ .halg = {
+ .digestsize = SHA384_DIGEST_SIZE,
+ .statesize = sizeof(struct starfive_sec_request_ctx),
+ .base = {
+ .cra_name = "sha384",
+ .cra_driver_name = "starfive-sha384",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA384_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct starfive_sec_ctx),
+ .cra_alignmask = 3,
+ .cra_init = starfive_hash_cra_sha384_init,
+ .cra_exit = starfive_hash_cra_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+},
+{
+ .init = starfive_hash_init,
+ .update = starfive_hash_update,
+ .final = starfive_hash_final,
+ .finup = starfive_hash_finup,
+ .digest = starfive_hash_digest,
+ .setkey = starfive_hash384_setkey,
+ .export = starfive_hash_export,
+ .import = starfive_hash_import,
+ .halg = {
+ .digestsize = SHA384_DIGEST_SIZE,
+ .statesize = sizeof(struct starfive_sec_request_ctx),
+ .base = {
+ .cra_name = "hmac(sha384)",
+ .cra_driver_name = "starfive-hmac-sha384",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA384_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct starfive_sec_ctx),
+ .cra_alignmask = 3,
+ .cra_init = starfive_hash_cra_hmac_sha384_init,
+ .cra_exit = starfive_hash_cra_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+},
+{
+ .init = starfive_hash_init,
+ .update = starfive_hash_update,
+ .final = starfive_hash_final,
+ .finup = starfive_hash_finup,
+ .digest = starfive_hash_digest,
+ .export = starfive_hash_export,
+ .import = starfive_hash_import,
+ .halg = {
+ .digestsize = SHA512_DIGEST_SIZE,
+ .statesize = sizeof(struct starfive_sec_request_ctx),
+ .base = {
+ .cra_name = "sha512",
+ .cra_driver_name = "starfive-sha512",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_TYPE_AHASH,
+ .cra_blocksize = SHA512_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct starfive_sec_ctx),
+ .cra_alignmask = 3,
+ .cra_init = starfive_hash_cra_sha512_init,
+ .cra_exit = starfive_hash_cra_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+},
+{
+ .init = starfive_hash_init,
+ .update = starfive_hash_update,
+ .final = starfive_hash_final,
+ .finup = starfive_hash_finup,
+ .digest = starfive_hash_digest,
+ .setkey = starfive_hash512_setkey,
+ .export = starfive_hash_export,
+ .import = starfive_hash_import,
+ .halg = {
+ .digestsize = SHA512_DIGEST_SIZE,
+ .statesize = sizeof(struct starfive_sec_request_ctx),
+ .base = {
+ .cra_name = "hmac(sha512)",
+ .cra_driver_name = "starfive-hmac-sha512",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA512_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct starfive_sec_ctx),
+ .cra_alignmask = 3,
+ .cra_init = starfive_hash_cra_hmac_sha512_init,
+ .cra_exit = starfive_hash_cra_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+},
+{
+ .init = starfive_hash_init,
+ .update = starfive_hash_update,
+ .final = starfive_hash_final,
+ .finup = starfive_hash_finup,
+ .digest = starfive_hash_digest,
+ .export = starfive_hash_export,
+ .import = starfive_hash_import,
+ .halg = {
+ .digestsize = SM3_DIGEST_SIZE,
+ .statesize = sizeof(struct starfive_sec_request_ctx),
+ .base = {
+ .cra_name = "sm3",
+ .cra_driver_name = "starfive-sm3",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SM3_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct starfive_sec_ctx),
+ .cra_alignmask = 3,
+ .cra_init = starfive_hash_cra_sm3_init,
+ .cra_exit = starfive_hash_cra_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+},
+{
+ .init = starfive_hash_init,
+ .update = starfive_hash_update,
+ .final = starfive_hash_final,
+ .finup = starfive_hash_finup,
+ .digest = starfive_hash_digest,
+ .setkey = starfive_sm3_setkey,
+ .export = starfive_hash_export,
+ .import = starfive_hash_import,
+ .halg = {
+ .digestsize = SM3_DIGEST_SIZE,
+ .statesize = sizeof(struct starfive_sec_request_ctx),
+ .base = {
+ .cra_name = "hmac(sm3)",
+ .cra_driver_name = "starfive-hmac-sm3",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SM3_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct starfive_sec_ctx),
+ .cra_alignmask = 3,
+ .cra_init = starfive_hash_cra_hmac_sm3_init,
+ .cra_exit = starfive_hash_cra_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+},
+};
+
+int starfive_hash_register_algs(void)
+{
+ int ret = 0;
+
+ ret = crypto_register_ahashes(algs_sha2_sm3, ARRAY_SIZE(algs_sha2_sm3));
+
+ return ret;
+}
+
+void starfive_hash_unregister_algs(void)
+{
+ crypto_unregister_ahashes(algs_sha2_sm3, ARRAY_SIZE(algs_sha2_sm3));
+}
diff --git a/drivers/crypto/starfive/starfive-regs.h b/drivers/crypto/starfive/starfive-regs.h
index 0d680cb1f502..3f5e8881b3c0 100644
--- a/drivers/crypto/starfive/starfive-regs.h
+++ b/drivers/crypto/starfive/starfive-regs.h
@@ -9,6 +9,8 @@
#define STARFIVE_DMA_IN_LEN_OFFSET 0x10
#define STARFIVE_DMA_OUT_LEN_OFFSET 0x14
+#define STARFIVE_HASH_REGS_OFFSET 0x300
+
union starfive_alg_cr {
u32 v;
struct {
@@ -23,4 +25,47 @@ union starfive_alg_cr {
};
};
+#define STARFIVE_HASH_SHACSR (STARFIVE_HASH_REGS_OFFSET + 0x0)
+#define STARFIVE_HASH_SHAWDR (STARFIVE_HASH_REGS_OFFSET + 0x4)
+#define STARFIVE_HASH_SHARDR (STARFIVE_HASH_REGS_OFFSET + 0x8)
+#define STARFIVE_HASH_SHAWSR (STARFIVE_HASH_REGS_OFFSET + 0xC)
+#define STARFIVE_HASH_SHAWLEN3 (STARFIVE_HASH_REGS_OFFSET + 0x10)
+#define STARFIVE_HASH_SHAWLEN2 (STARFIVE_HASH_REGS_OFFSET + 0x14)
+#define STARFIVE_HASH_SHAWLEN1 (STARFIVE_HASH_REGS_OFFSET + 0x18)
+#define STARFIVE_HASH_SHAWLEN0 (STARFIVE_HASH_REGS_OFFSET + 0x1C)
+#define STARFIVE_HASH_SHAWKR (STARFIVE_HASH_REGS_OFFSET + 0x20)
+#define STARFIVE_HASH_SHAWKLEN (STARFIVE_HASH_REGS_OFFSET + 0x24)
+
+union starfive_hash_csr {
+ u32 v;
+ struct {
+ u32 start :1;
+ u32 reset :1;
+ u32 rsvd_0 :1;
+ u32 firstb :1;
+#define STARFIVE_HASH_SM3 0x0
+#define STARFIVE_HASH_SHA224 0x3
+#define STARFIVE_HASH_SHA256 0x4
+#define STARFIVE_HASH_SHA384 0x5
+#define STARFIVE_HASH_SHA512 0x6
+#define STARFIVE_HASH_MODE_MASK 0x7
+ u32 mode :3;
+ u32 rsvd_1 :1;
+ u32 final :1;
+ u32 rsvd_2 :2;
+#define STARFIVE_HASH_HMAC_FLAGS 0x800
+ u32 hmac :1;
+ u32 rsvd_3 :1;
+#define STARFIVE_HASH_KEY_DONE BIT(13)
+ u32 key_done :1;
+ u32 key_flag :1;
+#define STARFIVE_HASH_HMAC_DONE BIT(15)
+ u32 hmac_done :1;
+#define STARFIVE_HASH_BUSY BIT(16)
+ u32 busy :1;
+ u32 hashdone :1;
+ u32 rsvd_4 :14;
+ };
+};
+
#endif
diff --git a/drivers/crypto/starfive/starfive-str.h b/drivers/crypto/starfive/starfive-str.h
index 4ba3c56f0573..a6fed48a0b19 100644
--- a/drivers/crypto/starfive/starfive-str.h
+++ b/drivers/crypto/starfive/starfive-str.h
@@ -7,16 +7,29 @@
#include <linux/dmaengine.h>
#include <crypto/engine.h>
+#include <crypto/sha2.h>
+#include <crypto/sm3.h>
#include "starfive-regs.h"
#define STARFIVE_MSG_BUFFER_SIZE SZ_16K
+#define MAX_KEY_SIZE SHA512_BLOCK_SIZE
struct starfive_sec_ctx {
struct crypto_engine_ctx enginectx;
struct starfive_sec_dev *sdev;
-
+ struct starfive_sec_request_ctx *rctx;
+ unsigned int hash_mode;
+ u8 key[MAX_KEY_SIZE];
+ int keylen;
+ struct scatterlist sg[2];
+ size_t hash_len_total;
u8 *buffer;
+
+ union {
+ struct crypto_shash *shash;
+ } fallback;
+ bool fallback_available;
};
struct starfive_sec_dev {
@@ -29,6 +42,7 @@ struct starfive_sec_dev {
void __iomem *io_base;
phys_addr_t io_phys_base;
+ void *hash_data;
size_t data_buf_len;
int pages_count;
@@ -41,12 +55,45 @@ struct starfive_sec_dev {
struct dma_slave_config cfg_out;
struct completion sec_comp_m;
struct completion sec_comp_p;
+ /* To synchronize concurrent request from different
+ * crypto module accessing the hardware engine.
+ */
+ struct mutex lock;
struct crypto_engine *engine;
union starfive_alg_cr alg_cr;
};
+struct starfive_sec_request_ctx {
+ struct starfive_sec_ctx *ctx;
+ struct starfive_sec_dev *sdev;
+
+ union {
+ struct ahash_request *hreq;
+ } req;
+#define STARFIVE_AHASH_REQ 0
+ unsigned int req_type;
+
+ union {
+ union starfive_hash_csr hash;
+ } csr;
+
+ struct scatterlist *in_sg;
+
+ unsigned long flags;
+ unsigned long op;
+
+ size_t bufcnt;
+ size_t buflen;
+ size_t total;
+ size_t offset;
+ size_t data_offset;
+
+ unsigned int hash_digest_len;
+ u8 hash_digest_mid[SHA512_DIGEST_SIZE]__aligned(sizeof(u32));
+};
+
static inline u32 starfive_sec_read(struct starfive_sec_dev *sdev, u32 offset)
{
return __raw_readl(sdev->io_base + offset);
@@ -71,4 +118,7 @@ static inline void starfive_sec_writeb(struct starfive_sec_dev *sdev,
struct starfive_sec_dev *starfive_sec_find_dev(struct starfive_sec_ctx *ctx);
+int starfive_hash_register_algs(void);
+void starfive_hash_unregister_algs(void);
+
#endif
--
2.25.1
Powered by blists - more mailing lists