blob: a0806ba40c68deae338073635878e6140e0095d0 [file] [log] [blame]
Ryder Lee785e5c62016-12-19 10:20:44 +08001/*
2 * Cryptographic API.
3 *
4 * Driver for EIP97 SHA1/SHA2(HMAC) acceleration.
5 *
6 * Copyright (c) 2016 Ryder Lee <ryder.lee@mediatek.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 * Some ideas are from atmel-sha.c and omap-sham.c drivers.
13 */
14
Corentin LABBE1127eea2017-05-19 08:53:30 +020015#include <crypto/hmac.h>
Ryder Lee785e5c62016-12-19 10:20:44 +080016#include <crypto/sha.h>
17#include "mtk-platform.h"
18
19#define SHA_ALIGN_MSK (sizeof(u32) - 1)
20#define SHA_QUEUE_SIZE 512
Ryder Lee785e5c62016-12-19 10:20:44 +080021#define SHA_BUF_SIZE ((u32)PAGE_SIZE)
22
23#define SHA_OP_UPDATE 1
24#define SHA_OP_FINAL 2
25
26#define SHA_DATA_LEN_MSK cpu_to_le32(GENMASK(16, 0))
Ryder Lee9aa2fcb2017-03-09 10:11:19 +080027#define SHA_MAX_DIGEST_BUF_SIZE 32
Ryder Lee785e5c62016-12-19 10:20:44 +080028
29/* SHA command token */
30#define SHA_CT_SIZE 5
31#define SHA_CT_CTRL_HDR cpu_to_le32(0x02220000)
Ryder Leea87399622017-01-20 13:41:08 +080032#define SHA_CMD0 cpu_to_le32(0x03020000)
33#define SHA_CMD1 cpu_to_le32(0x21060000)
34#define SHA_CMD2 cpu_to_le32(0xe0e63802)
Ryder Lee785e5c62016-12-19 10:20:44 +080035
36/* SHA transform information */
37#define SHA_TFM_HASH cpu_to_le32(0x2 << 0)
Ryder Lee785e5c62016-12-19 10:20:44 +080038#define SHA_TFM_SIZE(x) cpu_to_le32((x) << 8)
39#define SHA_TFM_START cpu_to_le32(0x1 << 4)
40#define SHA_TFM_CONTINUE cpu_to_le32(0x1 << 5)
41#define SHA_TFM_HASH_STORE cpu_to_le32(0x1 << 19)
42#define SHA_TFM_SHA1 cpu_to_le32(0x2 << 23)
43#define SHA_TFM_SHA256 cpu_to_le32(0x3 << 23)
44#define SHA_TFM_SHA224 cpu_to_le32(0x4 << 23)
45#define SHA_TFM_SHA512 cpu_to_le32(0x5 << 23)
46#define SHA_TFM_SHA384 cpu_to_le32(0x6 << 23)
47#define SHA_TFM_DIGEST(x) cpu_to_le32(((x) & GENMASK(3, 0)) << 24)
48
49/* SHA flags */
50#define SHA_FLAGS_BUSY BIT(0)
51#define SHA_FLAGS_FINAL BIT(1)
52#define SHA_FLAGS_FINUP BIT(2)
53#define SHA_FLAGS_SG BIT(3)
54#define SHA_FLAGS_ALGO_MSK GENMASK(8, 4)
55#define SHA_FLAGS_SHA1 BIT(4)
56#define SHA_FLAGS_SHA224 BIT(5)
57#define SHA_FLAGS_SHA256 BIT(6)
58#define SHA_FLAGS_SHA384 BIT(7)
59#define SHA_FLAGS_SHA512 BIT(8)
60#define SHA_FLAGS_HMAC BIT(9)
61#define SHA_FLAGS_PAD BIT(10)
62
63/**
Ryder Lee9aa2fcb2017-03-09 10:11:19 +080064 * mtk_sha_info - hardware information of AES
65 * @cmd: command token, hardware instruction
66 * @tfm: transform state of cipher algorithm.
67 * @state: contains keys and initial vectors.
68 *
Ryder Lee785e5c62016-12-19 10:20:44 +080069 */
70struct mtk_sha_info {
Ryder Lee9aa2fcb2017-03-09 10:11:19 +080071 __le32 ctrl[2];
72 __le32 cmd[3];
73 __le32 tfm[2];
74 __le32 digest[SHA_MAX_DIGEST_BUF_SIZE];
Ryder Lee785e5c62016-12-19 10:20:44 +080075};
76
77struct mtk_sha_reqctx {
78 struct mtk_sha_info info;
79 unsigned long flags;
80 unsigned long op;
81
82 u64 digcnt;
Ryder Lee785e5c62016-12-19 10:20:44 +080083 size_t bufcnt;
84 dma_addr_t dma_addr;
85
Ryder Leea87399622017-01-20 13:41:08 +080086 __le32 ct_hdr;
87 u32 ct_size;
88 dma_addr_t ct_dma;
89 dma_addr_t tfm_dma;
90
Ryder Lee785e5c62016-12-19 10:20:44 +080091 /* Walk state */
92 struct scatterlist *sg;
93 u32 offset; /* Offset in current sg */
94 u32 total; /* Total request */
95 size_t ds;
96 size_t bs;
97
98 u8 *buffer;
99};
100
101struct mtk_sha_hmac_ctx {
102 struct crypto_shash *shash;
103 u8 ipad[SHA512_BLOCK_SIZE] __aligned(sizeof(u32));
104 u8 opad[SHA512_BLOCK_SIZE] __aligned(sizeof(u32));
105};
106
107struct mtk_sha_ctx {
108 struct mtk_cryp *cryp;
109 unsigned long flags;
110 u8 id;
111 u8 buf[SHA_BUF_SIZE] __aligned(sizeof(u32));
112
113 struct mtk_sha_hmac_ctx base[0];
114};
115
116struct mtk_sha_drv {
117 struct list_head dev_list;
118 /* Device list lock */
119 spinlock_t lock;
120};
121
122static struct mtk_sha_drv mtk_sha = {
123 .dev_list = LIST_HEAD_INIT(mtk_sha.dev_list),
124 .lock = __SPIN_LOCK_UNLOCKED(mtk_sha.lock),
125};
126
127static int mtk_sha_handle_queue(struct mtk_cryp *cryp, u8 id,
128 struct ahash_request *req);
129
130static inline u32 mtk_sha_read(struct mtk_cryp *cryp, u32 offset)
131{
132 return readl_relaxed(cryp->base + offset);
133}
134
135static inline void mtk_sha_write(struct mtk_cryp *cryp,
136 u32 offset, u32 value)
137{
138 writel_relaxed(value, cryp->base + offset);
139}
140
Ryder Lee3d21c412017-03-09 10:11:15 +0800141static inline void mtk_sha_ring_shift(struct mtk_ring *ring,
142 struct mtk_desc **cmd_curr,
143 struct mtk_desc **res_curr,
144 int *count)
145{
146 *cmd_curr = ring->cmd_next++;
147 *res_curr = ring->res_next++;
148 (*count)++;
149
150 if (ring->cmd_next == ring->cmd_base + MTK_DESC_NUM) {
151 ring->cmd_next = ring->cmd_base;
152 ring->res_next = ring->res_base;
153 }
154}
155
Ryder Lee785e5c62016-12-19 10:20:44 +0800156static struct mtk_cryp *mtk_sha_find_dev(struct mtk_sha_ctx *tctx)
157{
158 struct mtk_cryp *cryp = NULL;
159 struct mtk_cryp *tmp;
160
161 spin_lock_bh(&mtk_sha.lock);
162 if (!tctx->cryp) {
163 list_for_each_entry(tmp, &mtk_sha.dev_list, sha_list) {
164 cryp = tmp;
165 break;
166 }
167 tctx->cryp = cryp;
168 } else {
169 cryp = tctx->cryp;
170 }
171
172 /*
173 * Assign record id to tfm in round-robin fashion, and this
174 * will help tfm to bind to corresponding descriptor rings.
175 */
176 tctx->id = cryp->rec;
177 cryp->rec = !cryp->rec;
178
179 spin_unlock_bh(&mtk_sha.lock);
180
181 return cryp;
182}
183
184static int mtk_sha_append_sg(struct mtk_sha_reqctx *ctx)
185{
186 size_t count;
187
188 while ((ctx->bufcnt < SHA_BUF_SIZE) && ctx->total) {
189 count = min(ctx->sg->length - ctx->offset, ctx->total);
190 count = min(count, SHA_BUF_SIZE - ctx->bufcnt);
191
192 if (count <= 0) {
193 /*
194 * Check if count <= 0 because the buffer is full or
195 * because the sg length is 0. In the latest case,
196 * check if there is another sg in the list, a 0 length
197 * sg doesn't necessarily mean the end of the sg list.
198 */
199 if ((ctx->sg->length == 0) && !sg_is_last(ctx->sg)) {
200 ctx->sg = sg_next(ctx->sg);
201 continue;
202 } else {
203 break;
204 }
205 }
206
207 scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, ctx->sg,
208 ctx->offset, count, 0);
209
210 ctx->bufcnt += count;
211 ctx->offset += count;
212 ctx->total -= count;
213
214 if (ctx->offset == ctx->sg->length) {
215 ctx->sg = sg_next(ctx->sg);
216 if (ctx->sg)
217 ctx->offset = 0;
218 else
219 ctx->total = 0;
220 }
221 }
222
223 return 0;
224}
225
226/*
227 * The purpose of this padding is to ensure that the padded message is a
228 * multiple of 512 bits (SHA1/SHA224/SHA256) or 1024 bits (SHA384/SHA512).
229 * The bit "1" is appended at the end of the message followed by
230 * "padlen-1" zero bits. Then a 64 bits block (SHA1/SHA224/SHA256) or
231 * 128 bits block (SHA384/SHA512) equals to the message length in bits
232 * is appended.
233 *
234 * For SHA1/SHA224/SHA256, padlen is calculated as followed:
235 * - if message length < 56 bytes then padlen = 56 - message length
236 * - else padlen = 64 + 56 - message length
237 *
238 * For SHA384/SHA512, padlen is calculated as followed:
239 * - if message length < 112 bytes then padlen = 112 - message length
240 * - else padlen = 128 + 112 - message length
241 */
242static void mtk_sha_fill_padding(struct mtk_sha_reqctx *ctx, u32 len)
243{
244 u32 index, padlen;
245 u64 bits[2];
246 u64 size = ctx->digcnt;
247
248 size += ctx->bufcnt;
249 size += len;
250
251 bits[1] = cpu_to_be64(size << 3);
252 bits[0] = cpu_to_be64(size >> 61);
253
Ryder Lee9aa2fcb2017-03-09 10:11:19 +0800254 switch (ctx->flags & SHA_FLAGS_ALGO_MSK) {
255 case SHA_FLAGS_SHA384:
256 case SHA_FLAGS_SHA512:
Ryder Lee785e5c62016-12-19 10:20:44 +0800257 index = ctx->bufcnt & 0x7f;
258 padlen = (index < 112) ? (112 - index) : ((128 + 112) - index);
259 *(ctx->buffer + ctx->bufcnt) = 0x80;
260 memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen - 1);
261 memcpy(ctx->buffer + ctx->bufcnt + padlen, bits, 16);
262 ctx->bufcnt += padlen + 16;
263 ctx->flags |= SHA_FLAGS_PAD;
Ryder Lee9aa2fcb2017-03-09 10:11:19 +0800264 break;
265
266 default:
Ryder Lee785e5c62016-12-19 10:20:44 +0800267 index = ctx->bufcnt & 0x3f;
268 padlen = (index < 56) ? (56 - index) : ((64 + 56) - index);
269 *(ctx->buffer + ctx->bufcnt) = 0x80;
270 memset(ctx->buffer + ctx->bufcnt + 1, 0, padlen - 1);
271 memcpy(ctx->buffer + ctx->bufcnt + padlen, &bits[1], 8);
272 ctx->bufcnt += padlen + 8;
273 ctx->flags |= SHA_FLAGS_PAD;
Ryder Lee9aa2fcb2017-03-09 10:11:19 +0800274 break;
Ryder Lee785e5c62016-12-19 10:20:44 +0800275 }
276}
277
278/* Initialize basic transform information of SHA */
Ryder Leea87399622017-01-20 13:41:08 +0800279static void mtk_sha_info_init(struct mtk_sha_reqctx *ctx)
Ryder Lee785e5c62016-12-19 10:20:44 +0800280{
Ryder Lee9aa2fcb2017-03-09 10:11:19 +0800281 struct mtk_sha_info *info = &ctx->info;
Ryder Lee785e5c62016-12-19 10:20:44 +0800282
Ryder Leea87399622017-01-20 13:41:08 +0800283 ctx->ct_hdr = SHA_CT_CTRL_HDR;
284 ctx->ct_size = SHA_CT_SIZE;
Ryder Lee785e5c62016-12-19 10:20:44 +0800285
Ryder Lee9aa2fcb2017-03-09 10:11:19 +0800286 info->tfm[0] = SHA_TFM_HASH | SHA_TFM_SIZE(SIZE_IN_WORDS(ctx->ds));
Ryder Lee785e5c62016-12-19 10:20:44 +0800287
288 switch (ctx->flags & SHA_FLAGS_ALGO_MSK) {
289 case SHA_FLAGS_SHA1:
Ryder Lee9aa2fcb2017-03-09 10:11:19 +0800290 info->tfm[0] |= SHA_TFM_SHA1;
Ryder Lee785e5c62016-12-19 10:20:44 +0800291 break;
292 case SHA_FLAGS_SHA224:
Ryder Lee9aa2fcb2017-03-09 10:11:19 +0800293 info->tfm[0] |= SHA_TFM_SHA224;
Ryder Lee785e5c62016-12-19 10:20:44 +0800294 break;
295 case SHA_FLAGS_SHA256:
Ryder Lee9aa2fcb2017-03-09 10:11:19 +0800296 info->tfm[0] |= SHA_TFM_SHA256;
Ryder Lee785e5c62016-12-19 10:20:44 +0800297 break;
298 case SHA_FLAGS_SHA384:
Ryder Lee9aa2fcb2017-03-09 10:11:19 +0800299 info->tfm[0] |= SHA_TFM_SHA384;
Ryder Lee785e5c62016-12-19 10:20:44 +0800300 break;
301 case SHA_FLAGS_SHA512:
Ryder Lee9aa2fcb2017-03-09 10:11:19 +0800302 info->tfm[0] |= SHA_TFM_SHA512;
Ryder Lee785e5c62016-12-19 10:20:44 +0800303 break;
304
305 default:
306 /* Should not happen... */
307 return;
308 }
309
Ryder Lee9aa2fcb2017-03-09 10:11:19 +0800310 info->tfm[1] = SHA_TFM_HASH_STORE;
311 info->ctrl[0] = info->tfm[0] | SHA_TFM_CONTINUE | SHA_TFM_START;
312 info->ctrl[1] = info->tfm[1];
Ryder Lee785e5c62016-12-19 10:20:44 +0800313
Ryder Lee9aa2fcb2017-03-09 10:11:19 +0800314 info->cmd[0] = SHA_CMD0;
315 info->cmd[1] = SHA_CMD1;
316 info->cmd[2] = SHA_CMD2 | SHA_TFM_DIGEST(SIZE_IN_WORDS(ctx->ds));
Ryder Lee785e5c62016-12-19 10:20:44 +0800317}
318
319/*
320 * Update input data length field of transform information and
321 * map it to DMA region.
322 */
Ryder Lee059b1492017-01-20 13:41:13 +0800323static int mtk_sha_info_update(struct mtk_cryp *cryp,
324 struct mtk_sha_rec *sha,
Ryder Lee82445fe2017-03-09 10:11:14 +0800325 size_t len1, size_t len2)
Ryder Lee785e5c62016-12-19 10:20:44 +0800326{
327 struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req);
Ryder Leea87399622017-01-20 13:41:08 +0800328 struct mtk_sha_info *info = &ctx->info;
Ryder Lee785e5c62016-12-19 10:20:44 +0800329
Ryder Leea87399622017-01-20 13:41:08 +0800330 ctx->ct_hdr &= ~SHA_DATA_LEN_MSK;
Ryder Lee82445fe2017-03-09 10:11:14 +0800331 ctx->ct_hdr |= cpu_to_le32(len1 + len2);
Ryder Lee9aa2fcb2017-03-09 10:11:19 +0800332 info->cmd[0] &= ~SHA_DATA_LEN_MSK;
333 info->cmd[0] |= cpu_to_le32(len1 + len2);
334
335 /* Setting SHA_TFM_START only for the first iteration */
336 if (ctx->digcnt)
337 info->ctrl[0] &= ~SHA_TFM_START;
Ryder Lee785e5c62016-12-19 10:20:44 +0800338
Ryder Lee82445fe2017-03-09 10:11:14 +0800339 ctx->digcnt += len1;
Ryder Lee785e5c62016-12-19 10:20:44 +0800340
Ryder Leea87399622017-01-20 13:41:08 +0800341 ctx->ct_dma = dma_map_single(cryp->dev, info, sizeof(*info),
Ryder Lee059b1492017-01-20 13:41:13 +0800342 DMA_BIDIRECTIONAL);
Ryder Leea87399622017-01-20 13:41:08 +0800343 if (unlikely(dma_mapping_error(cryp->dev, ctx->ct_dma))) {
Arnd Bergmann41e05322017-01-11 14:55:20 +0100344 dev_err(cryp->dev, "dma %zu bytes error\n", sizeof(*info));
Ryder Lee785e5c62016-12-19 10:20:44 +0800345 return -EINVAL;
346 }
Ryder Lee9aa2fcb2017-03-09 10:11:19 +0800347
348 ctx->tfm_dma = ctx->ct_dma + sizeof(info->ctrl) + sizeof(info->cmd);
Ryder Lee785e5c62016-12-19 10:20:44 +0800349
350 return 0;
351}
352
353/*
354 * Because of hardware limitation, we must pre-calculate the inner
355 * and outer digest that need to be processed firstly by engine, then
356 * apply the result digest to the input message. These complex hashing
357 * procedures limits HMAC performance, so we use fallback SW encoding.
358 */
359static int mtk_sha_finish_hmac(struct ahash_request *req)
360{
361 struct mtk_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
362 struct mtk_sha_hmac_ctx *bctx = tctx->base;
363 struct mtk_sha_reqctx *ctx = ahash_request_ctx(req);
364
365 SHASH_DESC_ON_STACK(shash, bctx->shash);
366
367 shash->tfm = bctx->shash;
Ryder Lee785e5c62016-12-19 10:20:44 +0800368
369 return crypto_shash_init(shash) ?:
370 crypto_shash_update(shash, bctx->opad, ctx->bs) ?:
371 crypto_shash_finup(shash, req->result, ctx->ds, req->result);
372}
373
374/* Initialize request context */
375static int mtk_sha_init(struct ahash_request *req)
376{
377 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
378 struct mtk_sha_ctx *tctx = crypto_ahash_ctx(tfm);
379 struct mtk_sha_reqctx *ctx = ahash_request_ctx(req);
380
381 ctx->flags = 0;
382 ctx->ds = crypto_ahash_digestsize(tfm);
383
384 switch (ctx->ds) {
385 case SHA1_DIGEST_SIZE:
386 ctx->flags |= SHA_FLAGS_SHA1;
387 ctx->bs = SHA1_BLOCK_SIZE;
388 break;
389 case SHA224_DIGEST_SIZE:
390 ctx->flags |= SHA_FLAGS_SHA224;
391 ctx->bs = SHA224_BLOCK_SIZE;
392 break;
393 case SHA256_DIGEST_SIZE:
394 ctx->flags |= SHA_FLAGS_SHA256;
395 ctx->bs = SHA256_BLOCK_SIZE;
396 break;
397 case SHA384_DIGEST_SIZE:
398 ctx->flags |= SHA_FLAGS_SHA384;
399 ctx->bs = SHA384_BLOCK_SIZE;
400 break;
401 case SHA512_DIGEST_SIZE:
402 ctx->flags |= SHA_FLAGS_SHA512;
403 ctx->bs = SHA512_BLOCK_SIZE;
404 break;
405 default:
406 return -EINVAL;
407 }
408
409 ctx->bufcnt = 0;
410 ctx->digcnt = 0;
411 ctx->buffer = tctx->buf;
Ryder Lee785e5c62016-12-19 10:20:44 +0800412
413 if (tctx->flags & SHA_FLAGS_HMAC) {
414 struct mtk_sha_hmac_ctx *bctx = tctx->base;
415
416 memcpy(ctx->buffer, bctx->ipad, ctx->bs);
417 ctx->bufcnt = ctx->bs;
418 ctx->flags |= SHA_FLAGS_HMAC;
419 }
420
421 return 0;
422}
423
424static int mtk_sha_xmit(struct mtk_cryp *cryp, struct mtk_sha_rec *sha,
Ryder Lee82445fe2017-03-09 10:11:14 +0800425 dma_addr_t addr1, size_t len1,
426 dma_addr_t addr2, size_t len2)
Ryder Lee785e5c62016-12-19 10:20:44 +0800427{
Ryder Leea87399622017-01-20 13:41:08 +0800428 struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req);
Ryder Lee785e5c62016-12-19 10:20:44 +0800429 struct mtk_ring *ring = cryp->ring[sha->id];
Ryder Lee3d21c412017-03-09 10:11:15 +0800430 struct mtk_desc *cmd, *res;
Ryder Lee82445fe2017-03-09 10:11:14 +0800431 int err, count = 0;
Ryder Lee785e5c62016-12-19 10:20:44 +0800432
Ryder Lee82445fe2017-03-09 10:11:14 +0800433 err = mtk_sha_info_update(cryp, sha, len1, len2);
Ryder Lee785e5c62016-12-19 10:20:44 +0800434 if (err)
435 return err;
436
437 /* Fill in the command/result descriptors */
Ryder Lee3d21c412017-03-09 10:11:15 +0800438 mtk_sha_ring_shift(ring, &cmd, &res, &count);
439
Ryder Lee82445fe2017-03-09 10:11:14 +0800440 res->hdr = MTK_DESC_FIRST | MTK_DESC_BUF_LEN(len1);
441 cmd->hdr = MTK_DESC_FIRST | MTK_DESC_BUF_LEN(len1) |
Ryder Leea87399622017-01-20 13:41:08 +0800442 MTK_DESC_CT_LEN(ctx->ct_size);
Ryder Lee82445fe2017-03-09 10:11:14 +0800443 cmd->buf = cpu_to_le32(addr1);
Ryder Leea87399622017-01-20 13:41:08 +0800444 cmd->ct = cpu_to_le32(ctx->ct_dma);
445 cmd->ct_hdr = ctx->ct_hdr;
446 cmd->tfm = cpu_to_le32(ctx->tfm_dma);
Ryder Lee785e5c62016-12-19 10:20:44 +0800447
Ryder Lee82445fe2017-03-09 10:11:14 +0800448 if (len2) {
Ryder Lee3d21c412017-03-09 10:11:15 +0800449 mtk_sha_ring_shift(ring, &cmd, &res, &count);
Ryder Lee785e5c62016-12-19 10:20:44 +0800450
Ryder Lee82445fe2017-03-09 10:11:14 +0800451 res->hdr = MTK_DESC_BUF_LEN(len2);
452 cmd->hdr = MTK_DESC_BUF_LEN(len2);
453 cmd->buf = cpu_to_le32(addr2);
Ryder Lee82445fe2017-03-09 10:11:14 +0800454 }
Ryder Lee785e5c62016-12-19 10:20:44 +0800455
Ryder Lee82445fe2017-03-09 10:11:14 +0800456 cmd->hdr |= MTK_DESC_LAST;
457 res->hdr |= MTK_DESC_LAST;
Ryder Lee785e5c62016-12-19 10:20:44 +0800458
459 /*
460 * Make sure that all changes to the DMA ring are done before we
461 * start engine.
462 */
463 wmb();
464 /* Start DMA transfer */
Ryder Lee82445fe2017-03-09 10:11:14 +0800465 mtk_sha_write(cryp, RDR_PREP_COUNT(sha->id), MTK_DESC_CNT(count));
466 mtk_sha_write(cryp, CDR_PREP_COUNT(sha->id), MTK_DESC_CNT(count));
Ryder Lee785e5c62016-12-19 10:20:44 +0800467
468 return -EINPROGRESS;
469}
470
471static int mtk_sha_dma_map(struct mtk_cryp *cryp,
472 struct mtk_sha_rec *sha,
473 struct mtk_sha_reqctx *ctx,
474 size_t count)
475{
476 ctx->dma_addr = dma_map_single(cryp->dev, ctx->buffer,
Ryder Lee059b1492017-01-20 13:41:13 +0800477 SHA_BUF_SIZE, DMA_TO_DEVICE);
Ryder Lee785e5c62016-12-19 10:20:44 +0800478 if (unlikely(dma_mapping_error(cryp->dev, ctx->dma_addr))) {
479 dev_err(cryp->dev, "dma map error\n");
480 return -EINVAL;
481 }
482
483 ctx->flags &= ~SHA_FLAGS_SG;
484
Ryder Lee82445fe2017-03-09 10:11:14 +0800485 return mtk_sha_xmit(cryp, sha, ctx->dma_addr, count, 0, 0);
Ryder Lee785e5c62016-12-19 10:20:44 +0800486}
487
488static int mtk_sha_update_slow(struct mtk_cryp *cryp,
489 struct mtk_sha_rec *sha)
490{
491 struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req);
492 size_t count;
493 u32 final;
494
495 mtk_sha_append_sg(ctx);
496
497 final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total;
498
Arnd Bergmann41e05322017-01-11 14:55:20 +0100499 dev_dbg(cryp->dev, "slow: bufcnt: %zu\n", ctx->bufcnt);
Ryder Lee785e5c62016-12-19 10:20:44 +0800500
501 if (final) {
502 sha->flags |= SHA_FLAGS_FINAL;
503 mtk_sha_fill_padding(ctx, 0);
504 }
505
506 if (final || (ctx->bufcnt == SHA_BUF_SIZE && ctx->total)) {
507 count = ctx->bufcnt;
508 ctx->bufcnt = 0;
509
510 return mtk_sha_dma_map(cryp, sha, ctx, count);
511 }
512 return 0;
513}
514
515static int mtk_sha_update_start(struct mtk_cryp *cryp,
516 struct mtk_sha_rec *sha)
517{
518 struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req);
519 u32 len, final, tail;
520 struct scatterlist *sg;
521
522 if (!ctx->total)
523 return 0;
524
525 if (ctx->bufcnt || ctx->offset)
526 return mtk_sha_update_slow(cryp, sha);
527
528 sg = ctx->sg;
529
530 if (!IS_ALIGNED(sg->offset, sizeof(u32)))
531 return mtk_sha_update_slow(cryp, sha);
532
533 if (!sg_is_last(sg) && !IS_ALIGNED(sg->length, ctx->bs))
534 /* size is not ctx->bs aligned */
535 return mtk_sha_update_slow(cryp, sha);
536
537 len = min(ctx->total, sg->length);
538
539 if (sg_is_last(sg)) {
540 if (!(ctx->flags & SHA_FLAGS_FINUP)) {
541 /* not last sg must be ctx->bs aligned */
542 tail = len & (ctx->bs - 1);
543 len -= tail;
544 }
545 }
546
547 ctx->total -= len;
548 ctx->offset = len; /* offset where to start slow */
549
550 final = (ctx->flags & SHA_FLAGS_FINUP) && !ctx->total;
551
552 /* Add padding */
553 if (final) {
554 size_t count;
555
556 tail = len & (ctx->bs - 1);
557 len -= tail;
558 ctx->total += tail;
559 ctx->offset = len; /* offset where to start slow */
560
561 sg = ctx->sg;
562 mtk_sha_append_sg(ctx);
563 mtk_sha_fill_padding(ctx, len);
564
565 ctx->dma_addr = dma_map_single(cryp->dev, ctx->buffer,
Ryder Lee059b1492017-01-20 13:41:13 +0800566 SHA_BUF_SIZE, DMA_TO_DEVICE);
Ryder Lee785e5c62016-12-19 10:20:44 +0800567 if (unlikely(dma_mapping_error(cryp->dev, ctx->dma_addr))) {
568 dev_err(cryp->dev, "dma map bytes error\n");
569 return -EINVAL;
570 }
571
572 sha->flags |= SHA_FLAGS_FINAL;
573 count = ctx->bufcnt;
574 ctx->bufcnt = 0;
575
576 if (len == 0) {
577 ctx->flags &= ~SHA_FLAGS_SG;
Ryder Lee82445fe2017-03-09 10:11:14 +0800578 return mtk_sha_xmit(cryp, sha, ctx->dma_addr,
579 count, 0, 0);
Ryder Lee785e5c62016-12-19 10:20:44 +0800580
581 } else {
582 ctx->sg = sg;
583 if (!dma_map_sg(cryp->dev, ctx->sg, 1, DMA_TO_DEVICE)) {
584 dev_err(cryp->dev, "dma_map_sg error\n");
585 return -EINVAL;
586 }
587
588 ctx->flags |= SHA_FLAGS_SG;
Ryder Lee82445fe2017-03-09 10:11:14 +0800589 return mtk_sha_xmit(cryp, sha, sg_dma_address(ctx->sg),
590 len, ctx->dma_addr, count);
Ryder Lee785e5c62016-12-19 10:20:44 +0800591 }
592 }
593
594 if (!dma_map_sg(cryp->dev, ctx->sg, 1, DMA_TO_DEVICE)) {
595 dev_err(cryp->dev, "dma_map_sg error\n");
596 return -EINVAL;
597 }
598
599 ctx->flags |= SHA_FLAGS_SG;
600
Ryder Lee82445fe2017-03-09 10:11:14 +0800601 return mtk_sha_xmit(cryp, sha, sg_dma_address(ctx->sg),
602 len, 0, 0);
Ryder Lee785e5c62016-12-19 10:20:44 +0800603}
604
605static int mtk_sha_final_req(struct mtk_cryp *cryp,
606 struct mtk_sha_rec *sha)
607{
Ryder Lee059b1492017-01-20 13:41:13 +0800608 struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req);
Ryder Lee785e5c62016-12-19 10:20:44 +0800609 size_t count;
610
611 mtk_sha_fill_padding(ctx, 0);
612
613 sha->flags |= SHA_FLAGS_FINAL;
614 count = ctx->bufcnt;
615 ctx->bufcnt = 0;
616
617 return mtk_sha_dma_map(cryp, sha, ctx, count);
618}
619
620/* Copy ready hash (+ finalize hmac) */
621static int mtk_sha_finish(struct ahash_request *req)
622{
623 struct mtk_sha_reqctx *ctx = ahash_request_ctx(req);
Ryder Lee9aa2fcb2017-03-09 10:11:19 +0800624 __le32 *digest = ctx->info.digest;
Ryder Lee785e5c62016-12-19 10:20:44 +0800625 u32 *result = (u32 *)req->result;
626 int i;
627
628 /* Get the hash from the digest buffer */
629 for (i = 0; i < SIZE_IN_WORDS(ctx->ds); i++)
630 result[i] = le32_to_cpu(digest[i]);
631
632 if (ctx->flags & SHA_FLAGS_HMAC)
633 return mtk_sha_finish_hmac(req);
634
635 return 0;
636}
637
638static void mtk_sha_finish_req(struct mtk_cryp *cryp,
Ryder Lee059b1492017-01-20 13:41:13 +0800639 struct mtk_sha_rec *sha,
640 int err)
Ryder Lee785e5c62016-12-19 10:20:44 +0800641{
642 if (likely(!err && (SHA_FLAGS_FINAL & sha->flags)))
643 err = mtk_sha_finish(sha->req);
644
645 sha->flags &= ~(SHA_FLAGS_BUSY | SHA_FLAGS_FINAL);
646
647 sha->req->base.complete(&sha->req->base, err);
648
649 /* Handle new request */
Ryder Lee0d4a8262017-03-09 10:11:16 +0800650 tasklet_schedule(&sha->queue_task);
Ryder Lee785e5c62016-12-19 10:20:44 +0800651}
652
653static int mtk_sha_handle_queue(struct mtk_cryp *cryp, u8 id,
654 struct ahash_request *req)
655{
656 struct mtk_sha_rec *sha = cryp->sha[id];
657 struct crypto_async_request *async_req, *backlog;
658 struct mtk_sha_reqctx *ctx;
659 unsigned long flags;
660 int err = 0, ret = 0;
661
662 spin_lock_irqsave(&sha->lock, flags);
663 if (req)
664 ret = ahash_enqueue_request(&sha->queue, req);
665
666 if (SHA_FLAGS_BUSY & sha->flags) {
667 spin_unlock_irqrestore(&sha->lock, flags);
668 return ret;
669 }
670
671 backlog = crypto_get_backlog(&sha->queue);
672 async_req = crypto_dequeue_request(&sha->queue);
673 if (async_req)
674 sha->flags |= SHA_FLAGS_BUSY;
675 spin_unlock_irqrestore(&sha->lock, flags);
676
677 if (!async_req)
678 return ret;
679
680 if (backlog)
681 backlog->complete(backlog, -EINPROGRESS);
682
683 req = ahash_request_cast(async_req);
684 ctx = ahash_request_ctx(req);
685
686 sha->req = req;
Ryder Lee785e5c62016-12-19 10:20:44 +0800687
Ryder Leea87399622017-01-20 13:41:08 +0800688 mtk_sha_info_init(ctx);
Ryder Lee785e5c62016-12-19 10:20:44 +0800689
690 if (ctx->op == SHA_OP_UPDATE) {
691 err = mtk_sha_update_start(cryp, sha);
692 if (err != -EINPROGRESS && (ctx->flags & SHA_FLAGS_FINUP))
693 /* No final() after finup() */
694 err = mtk_sha_final_req(cryp, sha);
695 } else if (ctx->op == SHA_OP_FINAL) {
696 err = mtk_sha_final_req(cryp, sha);
697 }
698
699 if (unlikely(err != -EINPROGRESS))
700 /* Task will not finish it, so do it here */
701 mtk_sha_finish_req(cryp, sha, err);
702
703 return ret;
704}
705
706static int mtk_sha_enqueue(struct ahash_request *req, u32 op)
707{
708 struct mtk_sha_reqctx *ctx = ahash_request_ctx(req);
709 struct mtk_sha_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
710
711 ctx->op = op;
712
713 return mtk_sha_handle_queue(tctx->cryp, tctx->id, req);
714}
715
716static void mtk_sha_unmap(struct mtk_cryp *cryp, struct mtk_sha_rec *sha)
717{
718 struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req);
719
Ryder Leea87399622017-01-20 13:41:08 +0800720 dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->info),
721 DMA_BIDIRECTIONAL);
Ryder Lee785e5c62016-12-19 10:20:44 +0800722
723 if (ctx->flags & SHA_FLAGS_SG) {
724 dma_unmap_sg(cryp->dev, ctx->sg, 1, DMA_TO_DEVICE);
725 if (ctx->sg->length == ctx->offset) {
726 ctx->sg = sg_next(ctx->sg);
727 if (ctx->sg)
728 ctx->offset = 0;
729 }
730 if (ctx->flags & SHA_FLAGS_PAD) {
731 dma_unmap_single(cryp->dev, ctx->dma_addr,
732 SHA_BUF_SIZE, DMA_TO_DEVICE);
733 }
734 } else
735 dma_unmap_single(cryp->dev, ctx->dma_addr,
736 SHA_BUF_SIZE, DMA_TO_DEVICE);
737}
738
739static void mtk_sha_complete(struct mtk_cryp *cryp,
740 struct mtk_sha_rec *sha)
741{
742 int err = 0;
743
744 err = mtk_sha_update_start(cryp, sha);
745 if (err != -EINPROGRESS)
746 mtk_sha_finish_req(cryp, sha, err);
747}
748
749static int mtk_sha_update(struct ahash_request *req)
750{
751 struct mtk_sha_reqctx *ctx = ahash_request_ctx(req);
752
753 ctx->total = req->nbytes;
754 ctx->sg = req->src;
755 ctx->offset = 0;
756
757 if ((ctx->bufcnt + ctx->total < SHA_BUF_SIZE) &&
758 !(ctx->flags & SHA_FLAGS_FINUP))
759 return mtk_sha_append_sg(ctx);
760
761 return mtk_sha_enqueue(req, SHA_OP_UPDATE);
762}
763
764static int mtk_sha_final(struct ahash_request *req)
765{
766 struct mtk_sha_reqctx *ctx = ahash_request_ctx(req);
767
768 ctx->flags |= SHA_FLAGS_FINUP;
769
770 if (ctx->flags & SHA_FLAGS_PAD)
771 return mtk_sha_finish(req);
772
773 return mtk_sha_enqueue(req, SHA_OP_FINAL);
774}
775
776static int mtk_sha_finup(struct ahash_request *req)
777{
778 struct mtk_sha_reqctx *ctx = ahash_request_ctx(req);
779 int err1, err2;
780
781 ctx->flags |= SHA_FLAGS_FINUP;
782
783 err1 = mtk_sha_update(req);
784 if (err1 == -EINPROGRESS || err1 == -EBUSY)
785 return err1;
786 /*
787 * final() has to be always called to cleanup resources
788 * even if update() failed
789 */
790 err2 = mtk_sha_final(req);
791
792 return err1 ?: err2;
793}
794
795static int mtk_sha_digest(struct ahash_request *req)
796{
797 return mtk_sha_init(req) ?: mtk_sha_finup(req);
798}
799
Ryder Lee059b1492017-01-20 13:41:13 +0800800static int mtk_sha_setkey(struct crypto_ahash *tfm, const u8 *key,
801 u32 keylen)
Ryder Lee785e5c62016-12-19 10:20:44 +0800802{
803 struct mtk_sha_ctx *tctx = crypto_ahash_ctx(tfm);
804 struct mtk_sha_hmac_ctx *bctx = tctx->base;
805 size_t bs = crypto_shash_blocksize(bctx->shash);
806 size_t ds = crypto_shash_digestsize(bctx->shash);
807 int err, i;
808
809 SHASH_DESC_ON_STACK(shash, bctx->shash);
810
811 shash->tfm = bctx->shash;
Ryder Lee785e5c62016-12-19 10:20:44 +0800812
813 if (keylen > bs) {
814 err = crypto_shash_digest(shash, key, keylen, bctx->ipad);
815 if (err)
816 return err;
817 keylen = ds;
818 } else {
819 memcpy(bctx->ipad, key, keylen);
820 }
821
822 memset(bctx->ipad + keylen, 0, bs - keylen);
823 memcpy(bctx->opad, bctx->ipad, bs);
824
825 for (i = 0; i < bs; i++) {
Corentin LABBE1127eea2017-05-19 08:53:30 +0200826 bctx->ipad[i] ^= HMAC_IPAD_VALUE;
827 bctx->opad[i] ^= HMAC_OPAD_VALUE;
Ryder Lee785e5c62016-12-19 10:20:44 +0800828 }
829
Colin Ian Kingf2831482017-01-03 13:21:22 +0000830 return 0;
Ryder Lee785e5c62016-12-19 10:20:44 +0800831}
832
833static int mtk_sha_export(struct ahash_request *req, void *out)
834{
835 const struct mtk_sha_reqctx *ctx = ahash_request_ctx(req);
836
837 memcpy(out, ctx, sizeof(*ctx));
838 return 0;
839}
840
841static int mtk_sha_import(struct ahash_request *req, const void *in)
842{
843 struct mtk_sha_reqctx *ctx = ahash_request_ctx(req);
844
845 memcpy(ctx, in, sizeof(*ctx));
846 return 0;
847}
848
849static int mtk_sha_cra_init_alg(struct crypto_tfm *tfm,
850 const char *alg_base)
851{
852 struct mtk_sha_ctx *tctx = crypto_tfm_ctx(tfm);
853 struct mtk_cryp *cryp = NULL;
854
855 cryp = mtk_sha_find_dev(tctx);
856 if (!cryp)
857 return -ENODEV;
858
859 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
860 sizeof(struct mtk_sha_reqctx));
861
862 if (alg_base) {
863 struct mtk_sha_hmac_ctx *bctx = tctx->base;
864
865 tctx->flags |= SHA_FLAGS_HMAC;
866 bctx->shash = crypto_alloc_shash(alg_base, 0,
867 CRYPTO_ALG_NEED_FALLBACK);
868 if (IS_ERR(bctx->shash)) {
869 pr_err("base driver %s could not be loaded.\n",
870 alg_base);
871
872 return PTR_ERR(bctx->shash);
873 }
874 }
875 return 0;
876}
877
878static int mtk_sha_cra_init(struct crypto_tfm *tfm)
879{
880 return mtk_sha_cra_init_alg(tfm, NULL);
881}
882
883static int mtk_sha_cra_sha1_init(struct crypto_tfm *tfm)
884{
885 return mtk_sha_cra_init_alg(tfm, "sha1");
886}
887
888static int mtk_sha_cra_sha224_init(struct crypto_tfm *tfm)
889{
890 return mtk_sha_cra_init_alg(tfm, "sha224");
891}
892
893static int mtk_sha_cra_sha256_init(struct crypto_tfm *tfm)
894{
895 return mtk_sha_cra_init_alg(tfm, "sha256");
896}
897
898static int mtk_sha_cra_sha384_init(struct crypto_tfm *tfm)
899{
900 return mtk_sha_cra_init_alg(tfm, "sha384");
901}
902
903static int mtk_sha_cra_sha512_init(struct crypto_tfm *tfm)
904{
905 return mtk_sha_cra_init_alg(tfm, "sha512");
906}
907
908static void mtk_sha_cra_exit(struct crypto_tfm *tfm)
909{
910 struct mtk_sha_ctx *tctx = crypto_tfm_ctx(tfm);
911
912 if (tctx->flags & SHA_FLAGS_HMAC) {
913 struct mtk_sha_hmac_ctx *bctx = tctx->base;
914
915 crypto_free_shash(bctx->shash);
916 }
917}
918
919static struct ahash_alg algs_sha1_sha224_sha256[] = {
920{
921 .init = mtk_sha_init,
922 .update = mtk_sha_update,
923 .final = mtk_sha_final,
924 .finup = mtk_sha_finup,
925 .digest = mtk_sha_digest,
926 .export = mtk_sha_export,
927 .import = mtk_sha_import,
928 .halg.digestsize = SHA1_DIGEST_SIZE,
929 .halg.statesize = sizeof(struct mtk_sha_reqctx),
930 .halg.base = {
931 .cra_name = "sha1",
932 .cra_driver_name = "mtk-sha1",
933 .cra_priority = 400,
934 .cra_flags = CRYPTO_ALG_ASYNC,
935 .cra_blocksize = SHA1_BLOCK_SIZE,
936 .cra_ctxsize = sizeof(struct mtk_sha_ctx),
937 .cra_alignmask = SHA_ALIGN_MSK,
938 .cra_module = THIS_MODULE,
939 .cra_init = mtk_sha_cra_init,
940 .cra_exit = mtk_sha_cra_exit,
941 }
942},
943{
944 .init = mtk_sha_init,
945 .update = mtk_sha_update,
946 .final = mtk_sha_final,
947 .finup = mtk_sha_finup,
948 .digest = mtk_sha_digest,
949 .export = mtk_sha_export,
950 .import = mtk_sha_import,
951 .halg.digestsize = SHA224_DIGEST_SIZE,
952 .halg.statesize = sizeof(struct mtk_sha_reqctx),
953 .halg.base = {
954 .cra_name = "sha224",
955 .cra_driver_name = "mtk-sha224",
956 .cra_priority = 400,
957 .cra_flags = CRYPTO_ALG_ASYNC,
958 .cra_blocksize = SHA224_BLOCK_SIZE,
959 .cra_ctxsize = sizeof(struct mtk_sha_ctx),
960 .cra_alignmask = SHA_ALIGN_MSK,
961 .cra_module = THIS_MODULE,
962 .cra_init = mtk_sha_cra_init,
963 .cra_exit = mtk_sha_cra_exit,
964 }
965},
966{
967 .init = mtk_sha_init,
968 .update = mtk_sha_update,
969 .final = mtk_sha_final,
970 .finup = mtk_sha_finup,
971 .digest = mtk_sha_digest,
972 .export = mtk_sha_export,
973 .import = mtk_sha_import,
974 .halg.digestsize = SHA256_DIGEST_SIZE,
975 .halg.statesize = sizeof(struct mtk_sha_reqctx),
976 .halg.base = {
977 .cra_name = "sha256",
978 .cra_driver_name = "mtk-sha256",
979 .cra_priority = 400,
980 .cra_flags = CRYPTO_ALG_ASYNC,
981 .cra_blocksize = SHA256_BLOCK_SIZE,
982 .cra_ctxsize = sizeof(struct mtk_sha_ctx),
983 .cra_alignmask = SHA_ALIGN_MSK,
984 .cra_module = THIS_MODULE,
985 .cra_init = mtk_sha_cra_init,
986 .cra_exit = mtk_sha_cra_exit,
987 }
988},
989{
990 .init = mtk_sha_init,
991 .update = mtk_sha_update,
992 .final = mtk_sha_final,
993 .finup = mtk_sha_finup,
994 .digest = mtk_sha_digest,
995 .export = mtk_sha_export,
996 .import = mtk_sha_import,
997 .setkey = mtk_sha_setkey,
998 .halg.digestsize = SHA1_DIGEST_SIZE,
999 .halg.statesize = sizeof(struct mtk_sha_reqctx),
1000 .halg.base = {
1001 .cra_name = "hmac(sha1)",
1002 .cra_driver_name = "mtk-hmac-sha1",
1003 .cra_priority = 400,
1004 .cra_flags = CRYPTO_ALG_ASYNC |
1005 CRYPTO_ALG_NEED_FALLBACK,
1006 .cra_blocksize = SHA1_BLOCK_SIZE,
1007 .cra_ctxsize = sizeof(struct mtk_sha_ctx) +
1008 sizeof(struct mtk_sha_hmac_ctx),
1009 .cra_alignmask = SHA_ALIGN_MSK,
1010 .cra_module = THIS_MODULE,
1011 .cra_init = mtk_sha_cra_sha1_init,
1012 .cra_exit = mtk_sha_cra_exit,
1013 }
1014},
1015{
1016 .init = mtk_sha_init,
1017 .update = mtk_sha_update,
1018 .final = mtk_sha_final,
1019 .finup = mtk_sha_finup,
1020 .digest = mtk_sha_digest,
1021 .export = mtk_sha_export,
1022 .import = mtk_sha_import,
1023 .setkey = mtk_sha_setkey,
1024 .halg.digestsize = SHA224_DIGEST_SIZE,
1025 .halg.statesize = sizeof(struct mtk_sha_reqctx),
1026 .halg.base = {
1027 .cra_name = "hmac(sha224)",
1028 .cra_driver_name = "mtk-hmac-sha224",
1029 .cra_priority = 400,
1030 .cra_flags = CRYPTO_ALG_ASYNC |
1031 CRYPTO_ALG_NEED_FALLBACK,
1032 .cra_blocksize = SHA224_BLOCK_SIZE,
1033 .cra_ctxsize = sizeof(struct mtk_sha_ctx) +
1034 sizeof(struct mtk_sha_hmac_ctx),
1035 .cra_alignmask = SHA_ALIGN_MSK,
1036 .cra_module = THIS_MODULE,
1037 .cra_init = mtk_sha_cra_sha224_init,
1038 .cra_exit = mtk_sha_cra_exit,
1039 }
1040},
1041{
1042 .init = mtk_sha_init,
1043 .update = mtk_sha_update,
1044 .final = mtk_sha_final,
1045 .finup = mtk_sha_finup,
1046 .digest = mtk_sha_digest,
1047 .export = mtk_sha_export,
1048 .import = mtk_sha_import,
1049 .setkey = mtk_sha_setkey,
1050 .halg.digestsize = SHA256_DIGEST_SIZE,
1051 .halg.statesize = sizeof(struct mtk_sha_reqctx),
1052 .halg.base = {
1053 .cra_name = "hmac(sha256)",
1054 .cra_driver_name = "mtk-hmac-sha256",
1055 .cra_priority = 400,
1056 .cra_flags = CRYPTO_ALG_ASYNC |
1057 CRYPTO_ALG_NEED_FALLBACK,
1058 .cra_blocksize = SHA256_BLOCK_SIZE,
1059 .cra_ctxsize = sizeof(struct mtk_sha_ctx) +
1060 sizeof(struct mtk_sha_hmac_ctx),
1061 .cra_alignmask = SHA_ALIGN_MSK,
1062 .cra_module = THIS_MODULE,
1063 .cra_init = mtk_sha_cra_sha256_init,
1064 .cra_exit = mtk_sha_cra_exit,
1065 }
1066},
1067};
1068
1069static struct ahash_alg algs_sha384_sha512[] = {
1070{
1071 .init = mtk_sha_init,
1072 .update = mtk_sha_update,
1073 .final = mtk_sha_final,
1074 .finup = mtk_sha_finup,
1075 .digest = mtk_sha_digest,
1076 .export = mtk_sha_export,
1077 .import = mtk_sha_import,
1078 .halg.digestsize = SHA384_DIGEST_SIZE,
1079 .halg.statesize = sizeof(struct mtk_sha_reqctx),
1080 .halg.base = {
1081 .cra_name = "sha384",
1082 .cra_driver_name = "mtk-sha384",
1083 .cra_priority = 400,
1084 .cra_flags = CRYPTO_ALG_ASYNC,
1085 .cra_blocksize = SHA384_BLOCK_SIZE,
1086 .cra_ctxsize = sizeof(struct mtk_sha_ctx),
1087 .cra_alignmask = SHA_ALIGN_MSK,
1088 .cra_module = THIS_MODULE,
1089 .cra_init = mtk_sha_cra_init,
1090 .cra_exit = mtk_sha_cra_exit,
1091 }
1092},
1093{
1094 .init = mtk_sha_init,
1095 .update = mtk_sha_update,
1096 .final = mtk_sha_final,
1097 .finup = mtk_sha_finup,
1098 .digest = mtk_sha_digest,
1099 .export = mtk_sha_export,
1100 .import = mtk_sha_import,
1101 .halg.digestsize = SHA512_DIGEST_SIZE,
1102 .halg.statesize = sizeof(struct mtk_sha_reqctx),
1103 .halg.base = {
1104 .cra_name = "sha512",
1105 .cra_driver_name = "mtk-sha512",
1106 .cra_priority = 400,
1107 .cra_flags = CRYPTO_ALG_ASYNC,
1108 .cra_blocksize = SHA512_BLOCK_SIZE,
1109 .cra_ctxsize = sizeof(struct mtk_sha_ctx),
1110 .cra_alignmask = SHA_ALIGN_MSK,
1111 .cra_module = THIS_MODULE,
1112 .cra_init = mtk_sha_cra_init,
1113 .cra_exit = mtk_sha_cra_exit,
1114 }
1115},
1116{
1117 .init = mtk_sha_init,
1118 .update = mtk_sha_update,
1119 .final = mtk_sha_final,
1120 .finup = mtk_sha_finup,
1121 .digest = mtk_sha_digest,
1122 .export = mtk_sha_export,
1123 .import = mtk_sha_import,
1124 .setkey = mtk_sha_setkey,
1125 .halg.digestsize = SHA384_DIGEST_SIZE,
1126 .halg.statesize = sizeof(struct mtk_sha_reqctx),
1127 .halg.base = {
1128 .cra_name = "hmac(sha384)",
1129 .cra_driver_name = "mtk-hmac-sha384",
1130 .cra_priority = 400,
1131 .cra_flags = CRYPTO_ALG_ASYNC |
1132 CRYPTO_ALG_NEED_FALLBACK,
1133 .cra_blocksize = SHA384_BLOCK_SIZE,
1134 .cra_ctxsize = sizeof(struct mtk_sha_ctx) +
1135 sizeof(struct mtk_sha_hmac_ctx),
1136 .cra_alignmask = SHA_ALIGN_MSK,
1137 .cra_module = THIS_MODULE,
1138 .cra_init = mtk_sha_cra_sha384_init,
1139 .cra_exit = mtk_sha_cra_exit,
1140 }
1141},
1142{
1143 .init = mtk_sha_init,
1144 .update = mtk_sha_update,
1145 .final = mtk_sha_final,
1146 .finup = mtk_sha_finup,
1147 .digest = mtk_sha_digest,
1148 .export = mtk_sha_export,
1149 .import = mtk_sha_import,
1150 .setkey = mtk_sha_setkey,
1151 .halg.digestsize = SHA512_DIGEST_SIZE,
1152 .halg.statesize = sizeof(struct mtk_sha_reqctx),
1153 .halg.base = {
1154 .cra_name = "hmac(sha512)",
1155 .cra_driver_name = "mtk-hmac-sha512",
1156 .cra_priority = 400,
1157 .cra_flags = CRYPTO_ALG_ASYNC |
1158 CRYPTO_ALG_NEED_FALLBACK,
1159 .cra_blocksize = SHA512_BLOCK_SIZE,
1160 .cra_ctxsize = sizeof(struct mtk_sha_ctx) +
1161 sizeof(struct mtk_sha_hmac_ctx),
1162 .cra_alignmask = SHA_ALIGN_MSK,
1163 .cra_module = THIS_MODULE,
1164 .cra_init = mtk_sha_cra_sha512_init,
1165 .cra_exit = mtk_sha_cra_exit,
1166 }
1167},
1168};
1169
Ryder Lee0d4a8262017-03-09 10:11:16 +08001170static void mtk_sha_queue_task(unsigned long data)
1171{
1172 struct mtk_sha_rec *sha = (struct mtk_sha_rec *)data;
1173
1174 mtk_sha_handle_queue(sha->cryp, sha->id - MTK_RING2, NULL);
1175}
1176
Ryder Lee132c57c2017-03-09 10:11:12 +08001177static void mtk_sha_done_task(unsigned long data)
Ryder Lee785e5c62016-12-19 10:20:44 +08001178{
Ryder Lee132c57c2017-03-09 10:11:12 +08001179 struct mtk_sha_rec *sha = (struct mtk_sha_rec *)data;
1180 struct mtk_cryp *cryp = sha->cryp;
Ryder Lee785e5c62016-12-19 10:20:44 +08001181
1182 mtk_sha_unmap(cryp, sha);
1183 mtk_sha_complete(cryp, sha);
1184}
1185
Ryder Lee132c57c2017-03-09 10:11:12 +08001186static irqreturn_t mtk_sha_irq(int irq, void *dev_id)
Ryder Lee785e5c62016-12-19 10:20:44 +08001187{
Ryder Lee132c57c2017-03-09 10:11:12 +08001188 struct mtk_sha_rec *sha = (struct mtk_sha_rec *)dev_id;
1189 struct mtk_cryp *cryp = sha->cryp;
1190 u32 val = mtk_sha_read(cryp, RDR_STAT(sha->id));
Ryder Lee785e5c62016-12-19 10:20:44 +08001191
Ryder Lee132c57c2017-03-09 10:11:12 +08001192 mtk_sha_write(cryp, RDR_STAT(sha->id), val);
Ryder Lee785e5c62016-12-19 10:20:44 +08001193
1194 if (likely((SHA_FLAGS_BUSY & sha->flags))) {
Ryder Lee132c57c2017-03-09 10:11:12 +08001195 mtk_sha_write(cryp, RDR_PROC_COUNT(sha->id), MTK_CNT_RST);
1196 mtk_sha_write(cryp, RDR_THRESH(sha->id),
Ryder Lee785e5c62016-12-19 10:20:44 +08001197 MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE);
1198
Ryder Lee0d4a8262017-03-09 10:11:16 +08001199 tasklet_schedule(&sha->done_task);
Ryder Lee785e5c62016-12-19 10:20:44 +08001200 } else {
Ryder Lee132c57c2017-03-09 10:11:12 +08001201 dev_warn(cryp->dev, "SHA interrupt when no active requests.\n");
Ryder Lee785e5c62016-12-19 10:20:44 +08001202 }
1203 return IRQ_HANDLED;
1204}
1205
1206/*
1207 * The purpose of two SHA records is used to get extra performance.
1208 * It is similar to mtk_aes_record_init().
1209 */
1210static int mtk_sha_record_init(struct mtk_cryp *cryp)
1211{
1212 struct mtk_sha_rec **sha = cryp->sha;
1213 int i, err = -ENOMEM;
1214
1215 for (i = 0; i < MTK_REC_NUM; i++) {
1216 sha[i] = kzalloc(sizeof(**sha), GFP_KERNEL);
1217 if (!sha[i])
1218 goto err_cleanup;
1219
Ryder Lee132c57c2017-03-09 10:11:12 +08001220 sha[i]->cryp = cryp;
Ryder Lee785e5c62016-12-19 10:20:44 +08001221
1222 spin_lock_init(&sha[i]->lock);
1223 crypto_init_queue(&sha[i]->queue, SHA_QUEUE_SIZE);
Ryder Lee132c57c2017-03-09 10:11:12 +08001224
Ryder Lee0d4a8262017-03-09 10:11:16 +08001225 tasklet_init(&sha[i]->queue_task, mtk_sha_queue_task,
1226 (unsigned long)sha[i]);
1227 tasklet_init(&sha[i]->done_task, mtk_sha_done_task,
Ryder Lee132c57c2017-03-09 10:11:12 +08001228 (unsigned long)sha[i]);
Ryder Lee785e5c62016-12-19 10:20:44 +08001229 }
1230
Ryder Lee132c57c2017-03-09 10:11:12 +08001231 /* Link to ring2 and ring3 respectively */
Ryder Leeb7a2be32017-03-09 10:11:13 +08001232 sha[0]->id = MTK_RING2;
1233 sha[1]->id = MTK_RING3;
Ryder Lee785e5c62016-12-19 10:20:44 +08001234
1235 cryp->rec = 1;
1236
1237 return 0;
1238
1239err_cleanup:
1240 for (; i--; )
1241 kfree(sha[i]);
1242 return err;
1243}
1244
1245static void mtk_sha_record_free(struct mtk_cryp *cryp)
1246{
1247 int i;
1248
1249 for (i = 0; i < MTK_REC_NUM; i++) {
Ryder Lee0d4a8262017-03-09 10:11:16 +08001250 tasklet_kill(&cryp->sha[i]->done_task);
1251 tasklet_kill(&cryp->sha[i]->queue_task);
1252
Ryder Lee785e5c62016-12-19 10:20:44 +08001253 kfree(cryp->sha[i]);
1254 }
1255}
1256
1257static void mtk_sha_unregister_algs(void)
1258{
1259 int i;
1260
1261 for (i = 0; i < ARRAY_SIZE(algs_sha1_sha224_sha256); i++)
1262 crypto_unregister_ahash(&algs_sha1_sha224_sha256[i]);
1263
1264 for (i = 0; i < ARRAY_SIZE(algs_sha384_sha512); i++)
1265 crypto_unregister_ahash(&algs_sha384_sha512[i]);
1266}
1267
1268static int mtk_sha_register_algs(void)
1269{
1270 int err, i;
1271
1272 for (i = 0; i < ARRAY_SIZE(algs_sha1_sha224_sha256); i++) {
1273 err = crypto_register_ahash(&algs_sha1_sha224_sha256[i]);
1274 if (err)
1275 goto err_sha_224_256_algs;
1276 }
1277
1278 for (i = 0; i < ARRAY_SIZE(algs_sha384_sha512); i++) {
1279 err = crypto_register_ahash(&algs_sha384_sha512[i]);
1280 if (err)
1281 goto err_sha_384_512_algs;
1282 }
1283
1284 return 0;
1285
1286err_sha_384_512_algs:
1287 for (; i--; )
1288 crypto_unregister_ahash(&algs_sha384_sha512[i]);
1289 i = ARRAY_SIZE(algs_sha1_sha224_sha256);
1290err_sha_224_256_algs:
1291 for (; i--; )
1292 crypto_unregister_ahash(&algs_sha1_sha224_sha256[i]);
1293
1294 return err;
1295}
1296
1297int mtk_hash_alg_register(struct mtk_cryp *cryp)
1298{
1299 int err;
1300
1301 INIT_LIST_HEAD(&cryp->sha_list);
1302
1303 /* Initialize two hash records */
1304 err = mtk_sha_record_init(cryp);
1305 if (err)
1306 goto err_record;
1307
Ryder Leeb7a2be32017-03-09 10:11:13 +08001308 err = devm_request_irq(cryp->dev, cryp->irq[MTK_RING2], mtk_sha_irq,
Ryder Lee132c57c2017-03-09 10:11:12 +08001309 0, "mtk-sha", cryp->sha[0]);
Ryder Lee785e5c62016-12-19 10:20:44 +08001310 if (err) {
1311 dev_err(cryp->dev, "unable to request sha irq0.\n");
1312 goto err_res;
1313 }
1314
Ryder Leeb7a2be32017-03-09 10:11:13 +08001315 err = devm_request_irq(cryp->dev, cryp->irq[MTK_RING3], mtk_sha_irq,
Ryder Lee132c57c2017-03-09 10:11:12 +08001316 0, "mtk-sha", cryp->sha[1]);
Ryder Lee785e5c62016-12-19 10:20:44 +08001317 if (err) {
1318 dev_err(cryp->dev, "unable to request sha irq1.\n");
1319 goto err_res;
1320 }
1321
1322 /* Enable ring2 and ring3 interrupt for hash */
Ryder Leeb7a2be32017-03-09 10:11:13 +08001323 mtk_sha_write(cryp, AIC_ENABLE_SET(MTK_RING2), MTK_IRQ_RDR2);
1324 mtk_sha_write(cryp, AIC_ENABLE_SET(MTK_RING3), MTK_IRQ_RDR3);
Ryder Lee785e5c62016-12-19 10:20:44 +08001325
Ryder Lee785e5c62016-12-19 10:20:44 +08001326 spin_lock(&mtk_sha.lock);
1327 list_add_tail(&cryp->sha_list, &mtk_sha.dev_list);
1328 spin_unlock(&mtk_sha.lock);
1329
1330 err = mtk_sha_register_algs();
1331 if (err)
1332 goto err_algs;
1333
1334 return 0;
1335
1336err_algs:
1337 spin_lock(&mtk_sha.lock);
1338 list_del(&cryp->sha_list);
1339 spin_unlock(&mtk_sha.lock);
Ryder Lee785e5c62016-12-19 10:20:44 +08001340err_res:
1341 mtk_sha_record_free(cryp);
1342err_record:
1343
1344 dev_err(cryp->dev, "mtk-sha initialization failed.\n");
1345 return err;
1346}
1347
1348void mtk_hash_alg_release(struct mtk_cryp *cryp)
1349{
1350 spin_lock(&mtk_sha.lock);
1351 list_del(&cryp->sha_list);
1352 spin_unlock(&mtk_sha.lock);
1353
1354 mtk_sha_unregister_algs();
Ryder Lee785e5c62016-12-19 10:20:44 +08001355 mtk_sha_record_free(cryp);
1356}