Message-ID: <20231216141234.417498-3-jiajie.ho@starfivetech.com>
Date: Sat, 16 Dec 2023 22:12:31 +0800
From: Jia Jie Ho <jiajie.ho@...rfivetech.com>
To: Herbert Xu <herbert@...dor.apana.org.au>, "David S . Miller"
	<davem@...emloft.net>, Rob Herring <robh+dt@...nel.org>, Krzysztof Kozlowski
	<krzysztof.kozlowski+dt@...aro.org>, Conor Dooley <conor+dt@...nel.org>,
	<linux-crypto@...r.kernel.org>, <devicetree@...r.kernel.org>,
	<linux-kernel@...r.kernel.org>
Subject: [PATCH 2/5] crypto: starfive: Update hash dma usage

Current hash uses sw fallback for non-word aligned input scatterlists.
Add support for unaligned cases utilizing the data valid mask for dma.

Signed-off-by: Jia Jie Ho <jiajie.ho@...rfivetech.com>
---
 drivers/crypto/starfive/jh7110-cryp.h |   1 +
 drivers/crypto/starfive/jh7110-hash.c | 257 ++++++++++----------------
 2 files changed, 100 insertions(+), 158 deletions(-)

diff --git a/drivers/crypto/starfive/jh7110-cryp.h b/drivers/crypto/starfive/jh7110-cryp.h
index 6cdf6db5d904..4940cd1a3fbb 100644
--- a/drivers/crypto/starfive/jh7110-cryp.h
+++ b/drivers/crypto/starfive/jh7110-cryp.h
@@ -190,6 +190,7 @@ struct starfive_cryp_dev {
 	struct crypto_engine			*engine;
 	struct tasklet_struct			aes_done;
 	struct tasklet_struct			hash_done;
+	struct completion			dma_done;
 	size_t					assoclen;
 	size_t					total_in;
 	size_t					total_out;
diff --git a/drivers/crypto/starfive/jh7110-hash.c b/drivers/crypto/starfive/jh7110-hash.c
index b6d1808012ca..74e151b5f875 100644
--- a/drivers/crypto/starfive/jh7110-hash.c
+++ b/drivers/crypto/starfive/jh7110-hash.c
@@ -86,62 +86,31 @@ static int starfive_hash_hmac_key(struct starfive_cryp_ctx *ctx)
 
 static void starfive_hash_start(void *param)
 {
-	struct starfive_cryp_ctx *ctx = param;
-	struct starfive_cryp_request_ctx *rctx = ctx->rctx;
-	struct starfive_cryp_dev *cryp = ctx->cryp;
-	union starfive_alg_cr alg_cr;
+	struct starfive_cryp_dev *cryp = param;
 	union starfive_hash_csr csr;
-	u32 stat;
-
-	dma_unmap_sg(cryp->dev, rctx->in_sg, rctx->in_sg_len, DMA_TO_DEVICE);
-
-	alg_cr.v = 0;
-	alg_cr.clear = 1;
-
-	writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET);
+	u32 mask;
 
 	csr.v = readl(cryp->base + STARFIVE_HASH_SHACSR);
 	csr.firstb = 0;
 	csr.final = 1;
-
-	stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET);
-	stat &= ~STARFIVE_IE_MASK_HASH_DONE;
-	writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET);
+	csr.ie = 1;
 	writel(csr.v, cryp->base + STARFIVE_HASH_SHACSR);
+
+	mask = readl(cryp->base + STARFIVE_IE_MASK_OFFSET);
+	mask &= ~STARFIVE_IE_MASK_HASH_DONE;
+	writel(mask, cryp->base + STARFIVE_IE_MASK_OFFSET);
 }
 
-static int starfive_hash_xmit_dma(struct starfive_cryp_ctx *ctx)
+static void starfive_hash_dma_callback(void *param)
 {
-	struct starfive_cryp_request_ctx *rctx = ctx->rctx;
-	struct starfive_cryp_dev *cryp = ctx->cryp;
-	struct dma_async_tx_descriptor	*in_desc;
-	union  starfive_alg_cr alg_cr;
-	int total_len;
-	int ret;
-
-	if (!rctx->total) {
-		starfive_hash_start(ctx);
-		return 0;
-	}
+	struct starfive_cryp_dev *cryp = param;
 
-	writel(rctx->total, cryp->base + STARFIVE_DMA_IN_LEN_OFFSET);
-
-	total_len = rctx->total;
-	total_len = (total_len & 0x3) ? (((total_len >> 2) + 1) << 2) : total_len;
-	sg_dma_len(rctx->in_sg) = total_len;
-
-	alg_cr.v = 0;
-	alg_cr.start = 1;
-	alg_cr.hash_dma_en = 1;
-
-	writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET);
-
-	ret = dma_map_sg(cryp->dev, rctx->in_sg, rctx->in_sg_len, DMA_TO_DEVICE);
-	if (!ret)
-		return dev_err_probe(cryp->dev, -EINVAL, "dma_map_sg() error\n");
+	complete(&cryp->dma_done);
+}
 
-	cryp->cfg_in.direction = DMA_MEM_TO_DEV;
-	cryp->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+static void starfive_hash_dma_init(struct starfive_cryp_dev *cryp)
+{
+	cryp->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_16_BYTES;
 	cryp->cfg_in.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 	cryp->cfg_in.src_maxburst = cryp->dma_maxburst;
 	cryp->cfg_in.dst_maxburst = cryp->dma_maxburst;
@@ -149,50 +118,48 @@ static int starfive_hash_xmit_dma(struct starfive_cryp_ctx *ctx)
 
 	dmaengine_slave_config(cryp->tx, &cryp->cfg_in);
 
-	in_desc = dmaengine_prep_slave_sg(cryp->tx, rctx->in_sg,
-					  ret, DMA_MEM_TO_DEV,
-					  DMA_PREP_INTERRUPT  |  DMA_CTRL_ACK);
-
-	if (!in_desc)
-		return -EINVAL;
-
-	in_desc->callback = starfive_hash_start;
-	in_desc->callback_param = ctx;
-
-	dmaengine_submit(in_desc);
-	dma_async_issue_pending(cryp->tx);
-
-	return 0;
+	init_completion(&cryp->dma_done);
 }
 
-static int starfive_hash_xmit(struct starfive_cryp_ctx *ctx)
+static int starfive_hash_dma_xfer(struct starfive_cryp_dev *cryp,
+				  struct scatterlist *sg)
 {
-	struct starfive_cryp_request_ctx *rctx = ctx->rctx;
-	struct starfive_cryp_dev *cryp = ctx->cryp;
+	struct dma_async_tx_descriptor *in_desc;
+	union starfive_alg_cr alg_cr;
 	int ret = 0;
 
-	rctx->csr.hash.v = 0;
-	rctx->csr.hash.reset = 1;
-	writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR);
-
-	if (starfive_hash_wait_busy(ctx))
-		return dev_err_probe(cryp->dev, -ETIMEDOUT, "Error resetting engine.\n");
+	alg_cr.v = 0;
+	alg_cr.start = 1;
+	alg_cr.hash_dma_en = 1;
+	writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET);
 
-	rctx->csr.hash.v = 0;
-	rctx->csr.hash.mode = ctx->hash_mode;
-	rctx->csr.hash.ie = 1;
+	writel(sg_dma_len(sg), cryp->base + STARFIVE_DMA_IN_LEN_OFFSET);
+	sg_dma_len(sg) = ALIGN(sg_dma_len(sg), sizeof(u32));
 
-	if (ctx->is_hmac) {
-		ret = starfive_hash_hmac_key(ctx);
-		if (ret)
-			return ret;
-	} else {
-		rctx->csr.hash.start = 1;
-		rctx->csr.hash.firstb = 1;
-		writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR);
+	in_desc = dmaengine_prep_slave_sg(cryp->tx, sg, 1, DMA_MEM_TO_DEV,
+					  DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+	if (!in_desc) {
+		ret = -EINVAL;
+		goto end;
 	}
 
-	return starfive_hash_xmit_dma(ctx);
+	reinit_completion(&cryp->dma_done);
+	in_desc->callback = starfive_hash_dma_callback;
+	in_desc->callback_param = cryp;
+
+	dmaengine_submit(in_desc);
+	dma_async_issue_pending(cryp->tx);
+
+	if (!wait_for_completion_timeout(&cryp->dma_done,
+					 msecs_to_jiffies(1000)))
+		ret = -ETIMEDOUT;
+
+end:
+	alg_cr.v = 0;
+	alg_cr.clear = 1;
+	writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET);
+
+	return ret;
 }
 
 static int starfive_hash_copy_hash(struct ahash_request *req)
@@ -229,44 +196,56 @@ void starfive_hash_done_task(unsigned long param)
 	crypto_finalize_hash_request(cryp->engine, cryp->req.hreq, err);
 }
 
-static int starfive_hash_check_aligned(struct scatterlist *sg, size_t total, size_t align)
+static int starfive_hash_one_request(struct crypto_engine *engine, void *areq)
 {
-	int len = 0;
-
-	if (!total)
-		return 0;
+	struct ahash_request *req = container_of(areq, struct ahash_request,
+						 base);
+	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
+	struct starfive_cryp_request_ctx *rctx = ctx->rctx;
+	struct starfive_cryp_dev *cryp = ctx->cryp;
+	struct scatterlist *tsg;
+	int ret, src_nents, i;
 
-	if (!IS_ALIGNED(total, align))
-		return -EINVAL;
+	writel(STARFIVE_HASH_RESET, cryp->base + STARFIVE_HASH_SHACSR);
 
-	while (sg) {
-		if (!IS_ALIGNED(sg->offset, sizeof(u32)))
-			return -EINVAL;
+	if (starfive_hash_wait_busy(ctx))
+		return dev_err_probe(cryp->dev, -ETIMEDOUT, "Error resetting hardware.\n");
 
-		if (!IS_ALIGNED(sg->length, align))
-			return -EINVAL;
+	rctx->csr.hash.v = 0;
+	rctx->csr.hash.mode = ctx->hash_mode;
 
-		len += sg->length;
-		sg = sg_next(sg);
+	if (ctx->is_hmac) {
+		ret = starfive_hash_hmac_key(ctx);
+		if (ret)
+			return ret;
+	} else {
+		rctx->csr.hash.start = 1;
+		rctx->csr.hash.firstb = 1;
+		writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR);
 	}
 
-	if (len != total)
-		return -EINVAL;
+	/* No input message, get digest and end. */
+	if (!rctx->total)
+		goto hash_start;
 
-	return 0;
-}
+	starfive_hash_dma_init(cryp);
 
-static int starfive_hash_one_request(struct crypto_engine *engine, void *areq)
-{
-	struct ahash_request *req = container_of(areq, struct ahash_request,
-						 base);
-	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
-	struct starfive_cryp_dev *cryp = ctx->cryp;
+	for_each_sg(rctx->in_sg, tsg, rctx->in_sg_len, i) {
+		src_nents = dma_map_sg(cryp->dev, tsg, 1, DMA_TO_DEVICE);
+		if (src_nents == 0)
+			return dev_err_probe(cryp->dev, -ENOMEM,
+					     "dma_map_sg error\n");
 
-	if (!cryp)
-		return -ENODEV;
+		ret = starfive_hash_dma_xfer(cryp, tsg);
+		dma_unmap_sg(cryp->dev, tsg, 1, DMA_TO_DEVICE);
+		if (ret)
+			return ret;
+	}
+
+hash_start:
+	starfive_hash_start(cryp);
 
-	return starfive_hash_xmit(ctx);
+	return 0;
 }
 
 static int starfive_hash_init(struct ahash_request *req)
@@ -337,22 +316,6 @@ static int starfive_hash_finup(struct ahash_request *req)
 	return crypto_ahash_finup(&rctx->ahash_fbk_req);
 }
 
-static int starfive_hash_digest_fb(struct ahash_request *req)
-{
-	struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req);
-	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
-	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm);
-
-	ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk);
-	ahash_request_set_callback(&rctx->ahash_fbk_req, req->base.flags,
-				   req->base.complete, req->base.data);
-
-	ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src,
-				req->result, req->nbytes);
-
-	return crypto_ahash_digest(&rctx->ahash_fbk_req);
-}
-
 static int starfive_hash_digest(struct ahash_request *req)
 {
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
@@ -370,9 +333,6 @@ static int starfive_hash_digest(struct ahash_request *req)
 	rctx->in_sg_len = sg_nents_for_len(rctx->in_sg, rctx->total);
 	ctx->rctx = rctx;
 
-	if (starfive_hash_check_aligned(rctx->in_sg, rctx->total, rctx->blksize))
-		return starfive_hash_digest_fb(req);
-
 	return crypto_transfer_hash_request_to_engine(cryp->engine, req);
 }
 
@@ -406,7 +366,8 @@ static int starfive_hash_import(struct ahash_request *req, const void *in)
 
 static int starfive_hash_init_tfm(struct crypto_ahash *hash,
 				  const char *alg_name,
-				  unsigned int mode)
+				  unsigned int mode,
+				  bool is_hmac)
 {
 	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
 
@@ -426,7 +387,7 @@ static int starfive_hash_init_tfm(struct crypto_ahash *hash,
 	crypto_ahash_set_reqsize(hash, sizeof(struct starfive_cryp_request_ctx) +
 				 crypto_ahash_reqsize(ctx->ahash_fbk));
 
-	ctx->keylen = 0;
+	ctx->is_hmac = is_hmac;
 	ctx->hash_mode = mode;
 
 	return 0;
@@ -529,81 +490,61 @@ static int starfive_hash_setkey(struct crypto_ahash *hash,
 static int starfive_sha224_init_tfm(struct crypto_ahash *hash)
 {
 	return starfive_hash_init_tfm(hash, "sha224-generic",
-				      STARFIVE_HASH_SHA224);
+				      STARFIVE_HASH_SHA224, 0);
 }
 
 static int starfive_sha256_init_tfm(struct crypto_ahash *hash)
 {
 	return starfive_hash_init_tfm(hash, "sha256-generic",
-				      STARFIVE_HASH_SHA256);
+				      STARFIVE_HASH_SHA256, 0);
 }
 
 static int starfive_sha384_init_tfm(struct crypto_ahash *hash)
 {
 	return starfive_hash_init_tfm(hash, "sha384-generic",
-				      STARFIVE_HASH_SHA384);
+				      STARFIVE_HASH_SHA384, 0);
 }
 
 static int starfive_sha512_init_tfm(struct crypto_ahash *hash)
 {
 	return starfive_hash_init_tfm(hash, "sha512-generic",
-				      STARFIVE_HASH_SHA512);
+				      STARFIVE_HASH_SHA512, 0);
 }
 
 static int starfive_sm3_init_tfm(struct crypto_ahash *hash)
 {
 	return starfive_hash_init_tfm(hash, "sm3-generic",
-				      STARFIVE_HASH_SM3);
+				      STARFIVE_HASH_SM3, 0);
 }
 
 static int starfive_hmac_sha224_init_tfm(struct crypto_ahash *hash)
 {
-	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
-
-	ctx->is_hmac = true;
-
 	return starfive_hash_init_tfm(hash, "hmac(sha224-generic)",
-				      STARFIVE_HASH_SHA224);
+				      STARFIVE_HASH_SHA224, 1);
 }
 
 static int starfive_hmac_sha256_init_tfm(struct crypto_ahash *hash)
 {
-	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
-
-	ctx->is_hmac = true;
-
 	return starfive_hash_init_tfm(hash, "hmac(sha256-generic)",
-				      STARFIVE_HASH_SHA256);
+				      STARFIVE_HASH_SHA256, 1);
 }
 
 static int starfive_hmac_sha384_init_tfm(struct crypto_ahash *hash)
 {
-	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
-
-	ctx->is_hmac = true;
-
 	return starfive_hash_init_tfm(hash, "hmac(sha384-generic)",
-				      STARFIVE_HASH_SHA384);
+				      STARFIVE_HASH_SHA384, 1);
 }
 
 static int starfive_hmac_sha512_init_tfm(struct crypto_ahash *hash)
 {
-	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
-
-	ctx->is_hmac = true;
-
 	return starfive_hash_init_tfm(hash, "hmac(sha512-generic)",
-				      STARFIVE_HASH_SHA512);
+				      STARFIVE_HASH_SHA512, 1);
 }
 
 static int starfive_hmac_sm3_init_tfm(struct crypto_ahash *hash)
 {
-	struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
-
-	ctx->is_hmac = true;
-
 	return starfive_hash_init_tfm(hash, "hmac(sm3-generic)",
-				      STARFIVE_HASH_SM3);
+				      STARFIVE_HASH_SM3, 1);
 }
 
 static struct ahash_engine_alg algs_sha2_sm3[] = {
-- 
2.34.1

Powered by blists - more mailing lists