blob: 7e96db75347a02f54e3cf7547f561b25f444215e [file] [log] [blame]
Kim Phillips9c4a7962008-06-23 19:50:15 +08001/*
2 * talitos - Freescale Integrated Security Engine (SEC) device driver
3 *
Kim Phillips5228f0f2011-07-15 11:21:38 +08004 * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
Kim Phillips9c4a7962008-06-23 19:50:15 +08005 *
6 * Scatterlist Crypto API glue code copied from files with the following:
7 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
8 *
9 * Crypto algorithm registration code copied from hifn driver:
10 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
11 * All rights reserved.
12 *
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * You should have received a copy of the GNU General Public License
24 * along with this program; if not, write to the Free Software
25 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 */
27
28#include <linux/kernel.h>
29#include <linux/module.h>
30#include <linux/mod_devicetable.h>
31#include <linux/device.h>
32#include <linux/interrupt.h>
33#include <linux/crypto.h>
34#include <linux/hw_random.h>
Rob Herring5af50732013-09-17 14:28:33 -050035#include <linux/of_address.h>
36#include <linux/of_irq.h>
Kim Phillips9c4a7962008-06-23 19:50:15 +080037#include <linux/of_platform.h>
38#include <linux/dma-mapping.h>
39#include <linux/io.h>
40#include <linux/spinlock.h>
41#include <linux/rtnetlink.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090042#include <linux/slab.h>
Kim Phillips9c4a7962008-06-23 19:50:15 +080043
44#include <crypto/algapi.h>
45#include <crypto/aes.h>
Lee Nipper3952f172008-07-10 18:29:18 +080046#include <crypto/des.h>
Kim Phillips9c4a7962008-06-23 19:50:15 +080047#include <crypto/sha.h>
Lee Nipper497f2e62010-05-19 19:20:36 +100048#include <crypto/md5.h>
Herbert Xue98014a2015-05-11 17:47:48 +080049#include <crypto/internal/aead.h>
Kim Phillips9c4a7962008-06-23 19:50:15 +080050#include <crypto/authenc.h>
Lee Nipper4de9d0b2009-03-29 15:52:32 +080051#include <crypto/skcipher.h>
Lee Nipperacbf7c622010-05-19 19:19:33 +100052#include <crypto/hash.h>
53#include <crypto/internal/hash.h>
Lee Nipper4de9d0b2009-03-29 15:52:32 +080054#include <crypto/scatterwalk.h>
Kim Phillips9c4a7962008-06-23 19:50:15 +080055
56#include "talitos.h"
57
LEROY Christophe922f9dc2015-04-17 16:32:07 +020058static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
LEROY Christopheda9de142017-10-06 15:04:57 +020059 unsigned int len, bool is_sec1)
Kim Phillips81eb0242009-08-13 11:51:51 +100060{
LEROY Christopheedc6bd62015-04-17 16:31:53 +020061 ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
LEROY Christopheda9de142017-10-06 15:04:57 +020062 if (is_sec1) {
63 ptr->len1 = cpu_to_be16(len);
64 } else {
65 ptr->len = cpu_to_be16(len);
LEROY Christophe922f9dc2015-04-17 16:32:07 +020066 ptr->eptr = upper_32_bits(dma_addr);
LEROY Christopheda9de142017-10-06 15:04:57 +020067 }
Kim Phillips81eb0242009-08-13 11:51:51 +100068}
69
Horia Geant?340ff602016-04-19 20:33:48 +030070static void copy_talitos_ptr(struct talitos_ptr *dst_ptr,
71 struct talitos_ptr *src_ptr, bool is_sec1)
72{
73 dst_ptr->ptr = src_ptr->ptr;
LEROY Christophe922f9dc2015-04-17 16:32:07 +020074 if (is_sec1) {
LEROY Christopheda9de142017-10-06 15:04:57 +020075 dst_ptr->len1 = src_ptr->len1;
LEROY Christophe922f9dc2015-04-17 16:32:07 +020076 } else {
LEROY Christopheda9de142017-10-06 15:04:57 +020077 dst_ptr->len = src_ptr->len;
78 dst_ptr->eptr = src_ptr->eptr;
LEROY Christophe922f9dc2015-04-17 16:32:07 +020079 }
LEROY Christophe538caf82015-04-17 16:31:59 +020080}
81
LEROY Christophe922f9dc2015-04-17 16:32:07 +020082static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
83 bool is_sec1)
LEROY Christophe538caf82015-04-17 16:31:59 +020084{
LEROY Christophe922f9dc2015-04-17 16:32:07 +020085 if (is_sec1)
86 return be16_to_cpu(ptr->len1);
87 else
88 return be16_to_cpu(ptr->len);
LEROY Christophe538caf82015-04-17 16:31:59 +020089}
90
LEROY Christopheb096b542016-06-06 13:20:34 +020091static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val,
92 bool is_sec1)
LEROY Christophe185eb792015-04-17 16:31:55 +020093{
LEROY Christophe922f9dc2015-04-17 16:32:07 +020094 if (!is_sec1)
LEROY Christopheb096b542016-06-06 13:20:34 +020095 ptr->j_extent = val;
96}
97
98static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1)
99{
100 if (!is_sec1)
101 ptr->j_extent |= val;
LEROY Christophe185eb792015-04-17 16:31:55 +0200102}
103
Kim Phillips9c4a7962008-06-23 19:50:15 +0800104/*
105 * map virtual single (contiguous) pointer to h/w descriptor pointer
106 */
107static void map_single_talitos_ptr(struct device *dev,
LEROY Christopheedc6bd62015-04-17 16:31:53 +0200108 struct talitos_ptr *ptr,
Horia Geant?42e8b0d2015-05-11 20:04:56 +0300109 unsigned int len, void *data,
Kim Phillips9c4a7962008-06-23 19:50:15 +0800110 enum dma_data_direction dir)
111{
Kim Phillips81eb0242009-08-13 11:51:51 +1000112 dma_addr_t dma_addr = dma_map_single(dev, data, len, dir);
LEROY Christophe922f9dc2015-04-17 16:32:07 +0200113 struct talitos_private *priv = dev_get_drvdata(dev);
114 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillips81eb0242009-08-13 11:51:51 +1000115
LEROY Christopheda9de142017-10-06 15:04:57 +0200116 to_talitos_ptr(ptr, dma_addr, len, is_sec1);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800117}
118
119/*
120 * unmap bus single (contiguous) h/w descriptor pointer
121 */
122static void unmap_single_talitos_ptr(struct device *dev,
LEROY Christopheedc6bd62015-04-17 16:31:53 +0200123 struct talitos_ptr *ptr,
Kim Phillips9c4a7962008-06-23 19:50:15 +0800124 enum dma_data_direction dir)
125{
LEROY Christophe922f9dc2015-04-17 16:32:07 +0200126 struct talitos_private *priv = dev_get_drvdata(dev);
127 bool is_sec1 = has_ftr_sec1(priv);
128
LEROY Christopheedc6bd62015-04-17 16:31:53 +0200129 dma_unmap_single(dev, be32_to_cpu(ptr->ptr),
LEROY Christophe922f9dc2015-04-17 16:32:07 +0200130 from_talitos_ptr_len(ptr, is_sec1), dir);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800131}
132
133static int reset_channel(struct device *dev, int ch)
134{
135 struct talitos_private *priv = dev_get_drvdata(dev);
136 unsigned int timeout = TALITOS_TIMEOUT;
LEROY Christophedd3c0982015-04-17 16:32:13 +0200137 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800138
LEROY Christophedd3c0982015-04-17 16:32:13 +0200139 if (is_sec1) {
140 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
141 TALITOS1_CCCR_LO_RESET);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800142
LEROY Christophedd3c0982015-04-17 16:32:13 +0200143 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) &
144 TALITOS1_CCCR_LO_RESET) && --timeout)
145 cpu_relax();
146 } else {
147 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
148 TALITOS2_CCCR_RESET);
149
150 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
151 TALITOS2_CCCR_RESET) && --timeout)
152 cpu_relax();
153 }
Kim Phillips9c4a7962008-06-23 19:50:15 +0800154
155 if (timeout == 0) {
156 dev_err(dev, "failed to reset channel %d\n", ch);
157 return -EIO;
158 }
159
Kim Phillips81eb0242009-08-13 11:51:51 +1000160 /* set 36-bit addressing, done writeback enable and done IRQ enable */
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800161 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
Kim Phillips81eb0242009-08-13 11:51:51 +1000162 TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800163
Kim Phillipsfe5720e2008-10-12 20:33:14 +0800164 /* and ICCR writeback, if available */
165 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800166 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
Kim Phillipsfe5720e2008-10-12 20:33:14 +0800167 TALITOS_CCCR_LO_IWSE);
168
Kim Phillips9c4a7962008-06-23 19:50:15 +0800169 return 0;
170}
171
172static int reset_device(struct device *dev)
173{
174 struct talitos_private *priv = dev_get_drvdata(dev);
175 unsigned int timeout = TALITOS_TIMEOUT;
LEROY Christophedd3c0982015-04-17 16:32:13 +0200176 bool is_sec1 = has_ftr_sec1(priv);
177 u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800178
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800179 setbits32(priv->reg + TALITOS_MCR, mcr);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800180
LEROY Christophedd3c0982015-04-17 16:32:13 +0200181 while ((in_be32(priv->reg + TALITOS_MCR) & mcr)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800182 && --timeout)
183 cpu_relax();
184
Kim Phillips2cdba3c2011-12-12 14:59:11 -0600185 if (priv->irq[1]) {
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800186 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
187 setbits32(priv->reg + TALITOS_MCR, mcr);
188 }
189
Kim Phillips9c4a7962008-06-23 19:50:15 +0800190 if (timeout == 0) {
191 dev_err(dev, "failed to reset device\n");
192 return -EIO;
193 }
194
195 return 0;
196}
197
198/*
199 * Reset and initialize the device
200 */
201static int init_device(struct device *dev)
202{
203 struct talitos_private *priv = dev_get_drvdata(dev);
204 int ch, err;
LEROY Christophedd3c0982015-04-17 16:32:13 +0200205 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800206
207 /*
208 * Master reset
209 * errata documentation: warning: certain SEC interrupts
210 * are not fully cleared by writing the MCR:SWR bit,
211 * set bit twice to completely reset
212 */
213 err = reset_device(dev);
214 if (err)
215 return err;
216
217 err = reset_device(dev);
218 if (err)
219 return err;
220
221 /* reset channels */
222 for (ch = 0; ch < priv->num_channels; ch++) {
223 err = reset_channel(dev, ch);
224 if (err)
225 return err;
226 }
227
228 /* enable channel done and error interrupts */
LEROY Christophedd3c0982015-04-17 16:32:13 +0200229 if (is_sec1) {
230 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT);
231 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);
232 /* disable parity error check in DEU (erroneous? test vect.) */
233 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE);
234 } else {
235 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT);
236 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);
237 }
Kim Phillips9c4a7962008-06-23 19:50:15 +0800238
Kim Phillipsfe5720e2008-10-12 20:33:14 +0800239 /* disable integrity check error interrupts (use writeback instead) */
240 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200241 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO,
Kim Phillipsfe5720e2008-10-12 20:33:14 +0800242 TALITOS_MDEUICR_LO_ICE);
243
Kim Phillips9c4a7962008-06-23 19:50:15 +0800244 return 0;
245}
246
247/**
248 * talitos_submit - submits a descriptor to the device for processing
249 * @dev: the SEC device to be used
Kim Phillips5228f0f2011-07-15 11:21:38 +0800250 * @ch: the SEC device channel to be used
Kim Phillips9c4a7962008-06-23 19:50:15 +0800251 * @desc: the descriptor to be processed by the device
252 * @callback: whom to call when processing is complete
253 * @context: a handle for use by caller (optional)
254 *
255 * desc must contain valid dma-mapped (bus physical) address pointers.
256 * callback must check err and feedback in descriptor header
257 * for device processing status.
258 */
Horia Geanta865d5062012-07-03 19:16:52 +0300259int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
260 void (*callback)(struct device *dev,
261 struct talitos_desc *desc,
262 void *context, int error),
263 void *context)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800264{
265 struct talitos_private *priv = dev_get_drvdata(dev);
266 struct talitos_request *request;
Kim Phillips5228f0f2011-07-15 11:21:38 +0800267 unsigned long flags;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800268 int head;
LEROY Christophe7d607c6a2015-04-17 16:32:09 +0200269 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800270
Kim Phillips4b9926282009-08-13 11:50:38 +1000271 spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800272
Kim Phillips4b9926282009-08-13 11:50:38 +1000273 if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
Kim Phillipsec6644d2008-07-17 20:16:40 +0800274 /* h/w fifo is full */
Kim Phillips4b9926282009-08-13 11:50:38 +1000275 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800276 return -EAGAIN;
277 }
278
Kim Phillips4b9926282009-08-13 11:50:38 +1000279 head = priv->chan[ch].head;
280 request = &priv->chan[ch].fifo[head];
Kim Phillipsec6644d2008-07-17 20:16:40 +0800281
Kim Phillips9c4a7962008-06-23 19:50:15 +0800282 /* map descriptor and save caller data */
LEROY Christophe7d607c6a2015-04-17 16:32:09 +0200283 if (is_sec1) {
284 desc->hdr1 = desc->hdr;
LEROY Christophe7d607c6a2015-04-17 16:32:09 +0200285 request->dma_desc = dma_map_single(dev, &desc->hdr1,
286 TALITOS_DESC_SIZE,
287 DMA_BIDIRECTIONAL);
288 } else {
289 request->dma_desc = dma_map_single(dev, desc,
290 TALITOS_DESC_SIZE,
291 DMA_BIDIRECTIONAL);
292 }
Kim Phillips9c4a7962008-06-23 19:50:15 +0800293 request->callback = callback;
294 request->context = context;
295
296 /* increment fifo head */
Kim Phillips4b9926282009-08-13 11:50:38 +1000297 priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800298
299 smp_wmb();
300 request->desc = desc;
301
302 /* GO! */
303 wmb();
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800304 out_be32(priv->chan[ch].reg + TALITOS_FF,
305 upper_32_bits(request->dma_desc));
306 out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
Kim Phillipsa7524472010-09-23 15:56:38 +0800307 lower_32_bits(request->dma_desc));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800308
Kim Phillips4b9926282009-08-13 11:50:38 +1000309 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800310
311 return -EINPROGRESS;
312}
Horia Geanta865d5062012-07-03 19:16:52 +0300313EXPORT_SYMBOL(talitos_submit);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800314
315/*
316 * process what was done, notify callback of error if not
317 */
318static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
319{
320 struct talitos_private *priv = dev_get_drvdata(dev);
321 struct talitos_request *request, saved_req;
322 unsigned long flags;
323 int tail, status;
LEROY Christophe7d607c6a2015-04-17 16:32:09 +0200324 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800325
Kim Phillips4b9926282009-08-13 11:50:38 +1000326 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800327
Kim Phillips4b9926282009-08-13 11:50:38 +1000328 tail = priv->chan[ch].tail;
329 while (priv->chan[ch].fifo[tail].desc) {
LEROY Christophe7d607c6a2015-04-17 16:32:09 +0200330 __be32 hdr;
331
Kim Phillips4b9926282009-08-13 11:50:38 +1000332 request = &priv->chan[ch].fifo[tail];
Kim Phillips9c4a7962008-06-23 19:50:15 +0800333
334 /* descriptors with their done bits set don't get the error */
335 rmb();
LEROY Christophe7d607c6a2015-04-17 16:32:09 +0200336 hdr = is_sec1 ? request->desc->hdr1 : request->desc->hdr;
337
338 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800339 status = 0;
Lee Nipperca38a812008-12-20 17:09:25 +1100340 else
Kim Phillips9c4a7962008-06-23 19:50:15 +0800341 if (!error)
342 break;
343 else
344 status = error;
345
346 dma_unmap_single(dev, request->dma_desc,
LEROY Christophe7d607c6a2015-04-17 16:32:09 +0200347 TALITOS_DESC_SIZE,
Kim Phillipse938e462009-03-29 15:53:23 +0800348 DMA_BIDIRECTIONAL);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800349
350 /* copy entries so we can call callback outside lock */
351 saved_req.desc = request->desc;
352 saved_req.callback = request->callback;
353 saved_req.context = request->context;
354
355 /* release request entry in fifo */
356 smp_wmb();
357 request->desc = NULL;
358
359 /* increment fifo tail */
Kim Phillips4b9926282009-08-13 11:50:38 +1000360 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800361
Kim Phillips4b9926282009-08-13 11:50:38 +1000362 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
Kim Phillipsec6644d2008-07-17 20:16:40 +0800363
Kim Phillips4b9926282009-08-13 11:50:38 +1000364 atomic_dec(&priv->chan[ch].submit_count);
Kim Phillipsec6644d2008-07-17 20:16:40 +0800365
Kim Phillips9c4a7962008-06-23 19:50:15 +0800366 saved_req.callback(dev, saved_req.desc, saved_req.context,
367 status);
368 /* channel may resume processing in single desc error case */
369 if (error && !reset_ch && status == error)
370 return;
Kim Phillips4b9926282009-08-13 11:50:38 +1000371 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
372 tail = priv->chan[ch].tail;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800373 }
374
Kim Phillips4b9926282009-08-13 11:50:38 +1000375 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800376}
377
378/*
379 * process completed requests for channels that have done status
380 */
LEROY Christophedd3c0982015-04-17 16:32:13 +0200381#define DEF_TALITOS1_DONE(name, ch_done_mask) \
382static void talitos1_done_##name(unsigned long data) \
383{ \
384 struct device *dev = (struct device *)data; \
385 struct talitos_private *priv = dev_get_drvdata(dev); \
386 unsigned long flags; \
387 \
388 if (ch_done_mask & 0x10000000) \
389 flush_channel(dev, 0, 0, 0); \
LEROY Christophedd3c0982015-04-17 16:32:13 +0200390 if (ch_done_mask & 0x40000000) \
391 flush_channel(dev, 1, 0, 0); \
392 if (ch_done_mask & 0x00010000) \
393 flush_channel(dev, 2, 0, 0); \
394 if (ch_done_mask & 0x00040000) \
395 flush_channel(dev, 3, 0, 0); \
396 \
LEROY Christophedd3c0982015-04-17 16:32:13 +0200397 /* At this point, all completed channels have been processed */ \
398 /* Unmask done interrupts for channels completed later on. */ \
399 spin_lock_irqsave(&priv->reg_lock, flags); \
400 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
401 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \
402 spin_unlock_irqrestore(&priv->reg_lock, flags); \
403}
404
405DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)
LEROY Christophe9c02e282017-10-06 15:04:55 +0200406DEF_TALITOS1_DONE(ch0, TALITOS1_ISR_CH_0_DONE)
LEROY Christophedd3c0982015-04-17 16:32:13 +0200407
408#define DEF_TALITOS2_DONE(name, ch_done_mask) \
409static void talitos2_done_##name(unsigned long data) \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800410{ \
411 struct device *dev = (struct device *)data; \
412 struct talitos_private *priv = dev_get_drvdata(dev); \
Horia Geanta511d63c2012-03-30 17:49:53 +0300413 unsigned long flags; \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800414 \
415 if (ch_done_mask & 1) \
416 flush_channel(dev, 0, 0, 0); \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800417 if (ch_done_mask & (1 << 2)) \
418 flush_channel(dev, 1, 0, 0); \
419 if (ch_done_mask & (1 << 4)) \
420 flush_channel(dev, 2, 0, 0); \
421 if (ch_done_mask & (1 << 6)) \
422 flush_channel(dev, 3, 0, 0); \
423 \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800424 /* At this point, all completed channels have been processed */ \
425 /* Unmask done interrupts for channels completed later on. */ \
Horia Geanta511d63c2012-03-30 17:49:53 +0300426 spin_lock_irqsave(&priv->reg_lock, flags); \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800427 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
LEROY Christophedd3c0982015-04-17 16:32:13 +0200428 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); \
Horia Geanta511d63c2012-03-30 17:49:53 +0300429 spin_unlock_irqrestore(&priv->reg_lock, flags); \
Kim Phillips9c4a7962008-06-23 19:50:15 +0800430}
LEROY Christophedd3c0982015-04-17 16:32:13 +0200431
432DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE)
LEROY Christophe9c02e282017-10-06 15:04:55 +0200433DEF_TALITOS2_DONE(ch0, TALITOS2_ISR_CH_0_DONE)
LEROY Christophedd3c0982015-04-17 16:32:13 +0200434DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE)
435DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800436
437/*
438 * locate current (offending) descriptor
439 */
Kim Phillips3e721ae2011-10-21 15:20:28 +0200440static u32 current_desc_hdr(struct device *dev, int ch)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800441{
442 struct talitos_private *priv = dev_get_drvdata(dev);
Horia Geantab62ffd82013-11-13 12:20:37 +0200443 int tail, iter;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800444 dma_addr_t cur_desc;
445
Horia Geantab62ffd82013-11-13 12:20:37 +0200446 cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
447 cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800448
Horia Geantab62ffd82013-11-13 12:20:37 +0200449 if (!cur_desc) {
450 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n");
451 return 0;
452 }
453
454 tail = priv->chan[ch].tail;
455
456 iter = tail;
457 while (priv->chan[ch].fifo[iter].dma_desc != cur_desc) {
458 iter = (iter + 1) & (priv->fifo_len - 1);
459 if (iter == tail) {
Kim Phillips9c4a7962008-06-23 19:50:15 +0800460 dev_err(dev, "couldn't locate current descriptor\n");
Kim Phillips3e721ae2011-10-21 15:20:28 +0200461 return 0;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800462 }
463 }
464
Horia Geantab62ffd82013-11-13 12:20:37 +0200465 return priv->chan[ch].fifo[iter].desc->hdr;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800466}
467
468/*
469 * user diagnostics; report root cause of error based on execution unit status
470 */
Kim Phillips3e721ae2011-10-21 15:20:28 +0200471static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800472{
473 struct talitos_private *priv = dev_get_drvdata(dev);
474 int i;
475
Kim Phillips3e721ae2011-10-21 15:20:28 +0200476 if (!desc_hdr)
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800477 desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);
Kim Phillips3e721ae2011-10-21 15:20:28 +0200478
479 switch (desc_hdr & DESC_HDR_SEL0_MASK) {
Kim Phillips9c4a7962008-06-23 19:50:15 +0800480 case DESC_HDR_SEL0_AFEU:
481 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200482 in_be32(priv->reg_afeu + TALITOS_EUISR),
483 in_be32(priv->reg_afeu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800484 break;
485 case DESC_HDR_SEL0_DEU:
486 dev_err(dev, "DEUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200487 in_be32(priv->reg_deu + TALITOS_EUISR),
488 in_be32(priv->reg_deu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800489 break;
490 case DESC_HDR_SEL0_MDEUA:
491 case DESC_HDR_SEL0_MDEUB:
492 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200493 in_be32(priv->reg_mdeu + TALITOS_EUISR),
494 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800495 break;
496 case DESC_HDR_SEL0_RNG:
497 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200498 in_be32(priv->reg_rngu + TALITOS_ISR),
499 in_be32(priv->reg_rngu + TALITOS_ISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800500 break;
501 case DESC_HDR_SEL0_PKEU:
502 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200503 in_be32(priv->reg_pkeu + TALITOS_EUISR),
504 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800505 break;
506 case DESC_HDR_SEL0_AESU:
507 dev_err(dev, "AESUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200508 in_be32(priv->reg_aesu + TALITOS_EUISR),
509 in_be32(priv->reg_aesu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800510 break;
511 case DESC_HDR_SEL0_CRCU:
512 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200513 in_be32(priv->reg_crcu + TALITOS_EUISR),
514 in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800515 break;
516 case DESC_HDR_SEL0_KEU:
517 dev_err(dev, "KEUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200518 in_be32(priv->reg_pkeu + TALITOS_EUISR),
519 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800520 break;
521 }
522
Kim Phillips3e721ae2011-10-21 15:20:28 +0200523 switch (desc_hdr & DESC_HDR_SEL1_MASK) {
Kim Phillips9c4a7962008-06-23 19:50:15 +0800524 case DESC_HDR_SEL1_MDEUA:
525 case DESC_HDR_SEL1_MDEUB:
526 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200527 in_be32(priv->reg_mdeu + TALITOS_EUISR),
528 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800529 break;
530 case DESC_HDR_SEL1_CRCU:
531 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200532 in_be32(priv->reg_crcu + TALITOS_EUISR),
533 in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800534 break;
535 }
536
537 for (i = 0; i < 8; i++)
538 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800539 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
540 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800541}
542
543/*
544 * recover from error interrupts
545 */
Kim Phillips5e718a02011-12-12 14:59:12 -0600546static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800547{
Kim Phillips9c4a7962008-06-23 19:50:15 +0800548 struct talitos_private *priv = dev_get_drvdata(dev);
549 unsigned int timeout = TALITOS_TIMEOUT;
LEROY Christophedd3c0982015-04-17 16:32:13 +0200550 int ch, error, reset_dev = 0;
Horia Geant?42e8b0d2015-05-11 20:04:56 +0300551 u32 v_lo;
LEROY Christophedd3c0982015-04-17 16:32:13 +0200552 bool is_sec1 = has_ftr_sec1(priv);
553 int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */
Kim Phillips9c4a7962008-06-23 19:50:15 +0800554
555 for (ch = 0; ch < priv->num_channels; ch++) {
556 /* skip channels without errors */
LEROY Christophedd3c0982015-04-17 16:32:13 +0200557 if (is_sec1) {
558 /* bits 29, 31, 17, 19 */
559 if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6))))
560 continue;
561 } else {
562 if (!(isr & (1 << (ch * 2 + 1))))
563 continue;
564 }
Kim Phillips9c4a7962008-06-23 19:50:15 +0800565
566 error = -EINVAL;
567
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800568 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800569
570 if (v_lo & TALITOS_CCPSR_LO_DOF) {
571 dev_err(dev, "double fetch fifo overflow error\n");
572 error = -EAGAIN;
573 reset_ch = 1;
574 }
575 if (v_lo & TALITOS_CCPSR_LO_SOF) {
576 /* h/w dropped descriptor */
577 dev_err(dev, "single fetch fifo overflow error\n");
578 error = -EAGAIN;
579 }
580 if (v_lo & TALITOS_CCPSR_LO_MDTE)
581 dev_err(dev, "master data transfer error\n");
582 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
Colin Ian King4d9b3a52016-11-01 20:14:04 -0600583 dev_err(dev, is_sec1 ? "pointer not complete error\n"
LEROY Christophedd3c0982015-04-17 16:32:13 +0200584 : "s/g data length zero error\n");
Kim Phillips9c4a7962008-06-23 19:50:15 +0800585 if (v_lo & TALITOS_CCPSR_LO_FPZ)
LEROY Christophedd3c0982015-04-17 16:32:13 +0200586 dev_err(dev, is_sec1 ? "parity error\n"
587 : "fetch pointer zero error\n");
Kim Phillips9c4a7962008-06-23 19:50:15 +0800588 if (v_lo & TALITOS_CCPSR_LO_IDH)
589 dev_err(dev, "illegal descriptor header error\n");
590 if (v_lo & TALITOS_CCPSR_LO_IEU)
LEROY Christophedd3c0982015-04-17 16:32:13 +0200591 dev_err(dev, is_sec1 ? "static assignment error\n"
592 : "invalid exec unit error\n");
Kim Phillips9c4a7962008-06-23 19:50:15 +0800593 if (v_lo & TALITOS_CCPSR_LO_EU)
Kim Phillips3e721ae2011-10-21 15:20:28 +0200594 report_eu_error(dev, ch, current_desc_hdr(dev, ch));
LEROY Christophedd3c0982015-04-17 16:32:13 +0200595 if (!is_sec1) {
596 if (v_lo & TALITOS_CCPSR_LO_GB)
597 dev_err(dev, "gather boundary error\n");
598 if (v_lo & TALITOS_CCPSR_LO_GRL)
599 dev_err(dev, "gather return/length error\n");
600 if (v_lo & TALITOS_CCPSR_LO_SB)
601 dev_err(dev, "scatter boundary error\n");
602 if (v_lo & TALITOS_CCPSR_LO_SRL)
603 dev_err(dev, "scatter return/length error\n");
604 }
Kim Phillips9c4a7962008-06-23 19:50:15 +0800605
606 flush_channel(dev, ch, error, reset_ch);
607
608 if (reset_ch) {
609 reset_channel(dev, ch);
610 } else {
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800611 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
LEROY Christophedd3c0982015-04-17 16:32:13 +0200612 TALITOS2_CCCR_CONT);
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800613 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
614 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
LEROY Christophedd3c0982015-04-17 16:32:13 +0200615 TALITOS2_CCCR_CONT) && --timeout)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800616 cpu_relax();
617 if (timeout == 0) {
618 dev_err(dev, "failed to restart channel %d\n",
619 ch);
620 reset_dev = 1;
621 }
622 }
623 }
LEROY Christophedd3c0982015-04-17 16:32:13 +0200624 if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) ||
625 (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) {
626 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR))
627 dev_err(dev, "TEA error: ISR 0x%08x_%08x\n",
628 isr, isr_lo);
629 else
630 dev_err(dev, "done overflow, internal time out, or "
631 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800632
633 /* purge request queues */
634 for (ch = 0; ch < priv->num_channels; ch++)
635 flush_channel(dev, ch, -EIO, 1);
636
637 /* reset and reinitialize the device */
638 init_device(dev);
639 }
640}
641
LEROY Christophedd3c0982015-04-17 16:32:13 +0200642#define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
643static irqreturn_t talitos1_interrupt_##name(int irq, void *data) \
644{ \
645 struct device *dev = data; \
646 struct talitos_private *priv = dev_get_drvdata(dev); \
647 u32 isr, isr_lo; \
648 unsigned long flags; \
649 \
650 spin_lock_irqsave(&priv->reg_lock, flags); \
651 isr = in_be32(priv->reg + TALITOS_ISR); \
652 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
653 /* Acknowledge interrupt */ \
654 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
655 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
656 \
657 if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) { \
658 spin_unlock_irqrestore(&priv->reg_lock, flags); \
659 talitos_error(dev, isr & ch_err_mask, isr_lo); \
660 } \
661 else { \
662 if (likely(isr & ch_done_mask)) { \
663 /* mask further done interrupts. */ \
664 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
665 /* done_task will unmask done interrupts at exit */ \
666 tasklet_schedule(&priv->done_task[tlet]); \
667 } \
668 spin_unlock_irqrestore(&priv->reg_lock, flags); \
669 } \
670 \
671 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
672 IRQ_NONE; \
673}
674
675DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
676
677#define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
678static irqreturn_t talitos2_interrupt_##name(int irq, void *data) \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800679{ \
680 struct device *dev = data; \
681 struct talitos_private *priv = dev_get_drvdata(dev); \
682 u32 isr, isr_lo; \
Horia Geanta511d63c2012-03-30 17:49:53 +0300683 unsigned long flags; \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800684 \
Horia Geanta511d63c2012-03-30 17:49:53 +0300685 spin_lock_irqsave(&priv->reg_lock, flags); \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800686 isr = in_be32(priv->reg + TALITOS_ISR); \
687 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
688 /* Acknowledge interrupt */ \
689 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
690 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
691 \
Horia Geanta511d63c2012-03-30 17:49:53 +0300692 if (unlikely(isr & ch_err_mask || isr_lo)) { \
693 spin_unlock_irqrestore(&priv->reg_lock, flags); \
694 talitos_error(dev, isr & ch_err_mask, isr_lo); \
695 } \
696 else { \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800697 if (likely(isr & ch_done_mask)) { \
698 /* mask further done interrupts. */ \
699 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
700 /* done_task will unmask done interrupts at exit */ \
701 tasklet_schedule(&priv->done_task[tlet]); \
702 } \
Horia Geanta511d63c2012-03-30 17:49:53 +0300703 spin_unlock_irqrestore(&priv->reg_lock, flags); \
704 } \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800705 \
706 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
707 IRQ_NONE; \
Kim Phillips9c4a7962008-06-23 19:50:15 +0800708}
LEROY Christophedd3c0982015-04-17 16:32:13 +0200709
710DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0)
711DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR,
712 0)
713DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR,
714 1)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800715
716/*
717 * hwrng
718 */
719static int talitos_rng_data_present(struct hwrng *rng, int wait)
720{
721 struct device *dev = (struct device *)rng->priv;
722 struct talitos_private *priv = dev_get_drvdata(dev);
723 u32 ofl;
724 int i;
725
726 for (i = 0; i < 20; i++) {
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200727 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) &
Kim Phillips9c4a7962008-06-23 19:50:15 +0800728 TALITOS_RNGUSR_LO_OFL;
729 if (ofl || !wait)
730 break;
731 udelay(10);
732 }
733
734 return !!ofl;
735}
736
737static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
738{
739 struct device *dev = (struct device *)rng->priv;
740 struct talitos_private *priv = dev_get_drvdata(dev);
741
742 /* rng fifo requires 64-bit accesses */
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200743 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO);
744 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800745
746 return sizeof(u32);
747}
748
749static int talitos_rng_init(struct hwrng *rng)
750{
751 struct device *dev = (struct device *)rng->priv;
752 struct talitos_private *priv = dev_get_drvdata(dev);
753 unsigned int timeout = TALITOS_TIMEOUT;
754
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200755 setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR);
756 while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO)
757 & TALITOS_RNGUSR_LO_RD)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800758 && --timeout)
759 cpu_relax();
760 if (timeout == 0) {
761 dev_err(dev, "failed to reset rng hw\n");
762 return -ENODEV;
763 }
764
765 /* start generating */
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200766 setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800767
768 return 0;
769}
770
771static int talitos_register_rng(struct device *dev)
772{
773 struct talitos_private *priv = dev_get_drvdata(dev);
Aaron Sierra35a3bb32015-08-05 16:52:08 -0500774 int err;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800775
776 priv->rng.name = dev_driver_string(dev),
777 priv->rng.init = talitos_rng_init,
778 priv->rng.data_present = talitos_rng_data_present,
779 priv->rng.data_read = talitos_rng_data_read,
780 priv->rng.priv = (unsigned long)dev;
781
Aaron Sierra35a3bb32015-08-05 16:52:08 -0500782 err = hwrng_register(&priv->rng);
783 if (!err)
784 priv->rng_registered = true;
785
786 return err;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800787}
788
789static void talitos_unregister_rng(struct device *dev)
790{
791 struct talitos_private *priv = dev_get_drvdata(dev);
792
Aaron Sierra35a3bb32015-08-05 16:52:08 -0500793 if (!priv->rng_registered)
794 return;
795
Kim Phillips9c4a7962008-06-23 19:50:15 +0800796 hwrng_unregister(&priv->rng);
Aaron Sierra35a3bb32015-08-05 16:52:08 -0500797 priv->rng_registered = false;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800798}
799
800/*
801 * crypto alg
802 */
803#define TALITOS_CRA_PRIORITY 3000
LEROY Christophe7405c8d2016-06-06 13:20:46 +0200804/*
805 * Defines a priority for doing AEAD with descriptors type
806 * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP
807 */
808#define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1)
Martin Hicks03d2c512017-05-02 09:38:35 -0400809#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE)
Lee Nipper3952f172008-07-10 18:29:18 +0800810#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
Lee Nipper70bcaca2008-07-03 19:08:46 +0800811
Kim Phillips9c4a7962008-06-23 19:50:15 +0800812struct talitos_ctx {
813 struct device *dev;
Kim Phillips5228f0f2011-07-15 11:21:38 +0800814 int ch;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800815 __be32 desc_hdr_template;
816 u8 key[TALITOS_MAX_KEY_SIZE];
Lee Nipper70bcaca2008-07-03 19:08:46 +0800817 u8 iv[TALITOS_MAX_IV_LENGTH];
Kim Phillips9c4a7962008-06-23 19:50:15 +0800818 unsigned int keylen;
819 unsigned int enckeylen;
820 unsigned int authkeylen;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800821};
822
Lee Nipper497f2e62010-05-19 19:20:36 +1000823#define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
824#define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
825
826struct talitos_ahash_req_ctx {
Kim Phillips60f208d2010-05-19 19:21:53 +1000827 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
Lee Nipper497f2e62010-05-19 19:20:36 +1000828 unsigned int hw_context_size;
829 u8 buf[HASH_MAX_BLOCK_SIZE];
830 u8 bufnext[HASH_MAX_BLOCK_SIZE];
Kim Phillips60f208d2010-05-19 19:21:53 +1000831 unsigned int swinit;
Lee Nipper497f2e62010-05-19 19:20:36 +1000832 unsigned int first;
833 unsigned int last;
834 unsigned int to_hash_later;
Horia Geant?42e8b0d2015-05-11 20:04:56 +0300835 unsigned int nbuf;
Lee Nipper497f2e62010-05-19 19:20:36 +1000836 struct scatterlist bufsl[2];
837 struct scatterlist *psrc;
838};
839
Horia Geant?3639ca82016-04-21 19:24:55 +0300840struct talitos_export_state {
841 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
842 u8 buf[HASH_MAX_BLOCK_SIZE];
843 unsigned int swinit;
844 unsigned int first;
845 unsigned int last;
846 unsigned int to_hash_later;
847 unsigned int nbuf;
848};
849
Lee Nipper56af8cd2009-03-29 15:50:50 +0800850static int aead_setkey(struct crypto_aead *authenc,
851 const u8 *key, unsigned int keylen)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800852{
853 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
Mathias Krausec306a982013-10-15 13:49:34 +0200854 struct crypto_authenc_keys keys;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800855
Mathias Krausec306a982013-10-15 13:49:34 +0200856 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800857 goto badkey;
858
Mathias Krausec306a982013-10-15 13:49:34 +0200859 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800860 goto badkey;
861
Mathias Krausec306a982013-10-15 13:49:34 +0200862 memcpy(ctx->key, keys.authkey, keys.authkeylen);
863 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800864
Mathias Krausec306a982013-10-15 13:49:34 +0200865 ctx->keylen = keys.authkeylen + keys.enckeylen;
866 ctx->enckeylen = keys.enckeylen;
867 ctx->authkeylen = keys.authkeylen;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800868
869 return 0;
870
871badkey:
872 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
873 return -EINVAL;
874}
875
876/*
Lee Nipper56af8cd2009-03-29 15:50:50 +0800877 * talitos_edesc - s/w-extended descriptor
Kim Phillips9c4a7962008-06-23 19:50:15 +0800878 * @src_nents: number of segments in input scatterlist
879 * @dst_nents: number of segments in output scatterlist
Herbert Xuaeb4c132015-07-30 17:53:22 +0800880 * @icv_ool: whether ICV is out-of-line
Horia Geanta79fd31d2012-08-02 17:16:40 +0300881 * @iv_dma: dma address of iv for checking continuity and link table
Kim Phillips9c4a7962008-06-23 19:50:15 +0800882 * @dma_len: length of dma mapped link_tbl space
LEROY Christophe6f65f6a2015-04-17 16:32:15 +0200883 * @dma_link_tbl: bus physical address of link_tbl/buf
Kim Phillips9c4a7962008-06-23 19:50:15 +0800884 * @desc: h/w descriptor
LEROY Christophe6f65f6a2015-04-17 16:32:15 +0200885 * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
886 * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800887 *
888 * if decrypting (with authcheck), or either one of src_nents or dst_nents
889 * is greater than 1, an integrity check value is concatenated to the end
890 * of link_tbl data
891 */
Lee Nipper56af8cd2009-03-29 15:50:50 +0800892struct talitos_edesc {
Kim Phillips9c4a7962008-06-23 19:50:15 +0800893 int src_nents;
894 int dst_nents;
Herbert Xuaeb4c132015-07-30 17:53:22 +0800895 bool icv_ool;
Horia Geanta79fd31d2012-08-02 17:16:40 +0300896 dma_addr_t iv_dma;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800897 int dma_len;
898 dma_addr_t dma_link_tbl;
899 struct talitos_desc desc;
LEROY Christophe6f65f6a2015-04-17 16:32:15 +0200900 union {
901 struct talitos_ptr link_tbl[0];
902 u8 buf[0];
903 };
Kim Phillips9c4a7962008-06-23 19:50:15 +0800904};
905
Lee Nipper4de9d0b2009-03-29 15:52:32 +0800906static void talitos_sg_unmap(struct device *dev,
907 struct talitos_edesc *edesc,
908 struct scatterlist *src,
LEROY Christophe6a1e8d12016-06-06 13:20:38 +0200909 struct scatterlist *dst,
910 unsigned int len, unsigned int offset)
LEROY Christophe246a87c2016-06-06 13:20:36 +0200911{
912 struct talitos_private *priv = dev_get_drvdata(dev);
913 bool is_sec1 = has_ftr_sec1(priv);
LEROY Christophe6a1e8d12016-06-06 13:20:38 +0200914 unsigned int src_nents = edesc->src_nents ? : 1;
915 unsigned int dst_nents = edesc->dst_nents ? : 1;
LEROY Christophe246a87c2016-06-06 13:20:36 +0200916
LEROY Christophe6a1e8d12016-06-06 13:20:38 +0200917 if (is_sec1 && dst && dst_nents > 1) {
918 dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset,
919 len, DMA_FROM_DEVICE);
920 sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len,
921 offset);
922 }
923 if (src != dst) {
924 if (src_nents == 1 || !is_sec1)
925 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
926
927 if (dst && (dst_nents == 1 || !is_sec1))
928 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
929 } else if (src_nents == 1 || !is_sec1) {
930 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
LEROY Christophe246a87c2016-06-06 13:20:36 +0200931 }
932}
933
Kim Phillips9c4a7962008-06-23 19:50:15 +0800934static void ipsec_esp_unmap(struct device *dev,
Lee Nipper56af8cd2009-03-29 15:50:50 +0800935 struct talitos_edesc *edesc,
Kim Phillips9c4a7962008-06-23 19:50:15 +0800936 struct aead_request *areq)
937{
LEROY Christophe549bd8b2016-06-06 13:20:40 +0200938 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
939 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
940 unsigned int ivsize = crypto_aead_ivsize(aead);
941
942 if (edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP)
943 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6],
944 DMA_FROM_DEVICE);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800945 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
946 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
947 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
948
LEROY Christophe6a1e8d12016-06-06 13:20:38 +0200949 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen,
950 areq->assoclen);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800951
952 if (edesc->dma_len)
953 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
954 DMA_BIDIRECTIONAL);
LEROY Christophe549bd8b2016-06-06 13:20:40 +0200955
956 if (!(edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP)) {
957 unsigned int dst_nents = edesc->dst_nents ? : 1;
958
959 sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize,
960 areq->assoclen + areq->cryptlen - ivsize);
961 }
Kim Phillips9c4a7962008-06-23 19:50:15 +0800962}
963
964/*
965 * ipsec_esp descriptor callbacks
966 */
967static void ipsec_esp_encrypt_done(struct device *dev,
968 struct talitos_desc *desc, void *context,
969 int err)
970{
LEROY Christophe549bd8b2016-06-06 13:20:40 +0200971 struct talitos_private *priv = dev_get_drvdata(dev);
972 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800973 struct aead_request *areq = context;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800974 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
Herbert Xuaeb4c132015-07-30 17:53:22 +0800975 unsigned int authsize = crypto_aead_authsize(authenc);
Kim Phillips19bbbc62009-03-29 15:53:59 +0800976 struct talitos_edesc *edesc;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800977 struct scatterlist *sg;
978 void *icvdata;
979
Kim Phillips19bbbc62009-03-29 15:53:59 +0800980 edesc = container_of(desc, struct talitos_edesc, desc);
981
Kim Phillips9c4a7962008-06-23 19:50:15 +0800982 ipsec_esp_unmap(dev, edesc, areq);
983
984 /* copy the generated ICV to dst */
Herbert Xuaeb4c132015-07-30 17:53:22 +0800985 if (edesc->icv_ool) {
LEROY Christophe549bd8b2016-06-06 13:20:40 +0200986 if (is_sec1)
987 icvdata = edesc->buf + areq->assoclen + areq->cryptlen;
988 else
989 icvdata = &edesc->link_tbl[edesc->src_nents +
990 edesc->dst_nents + 2];
Kim Phillips9c4a7962008-06-23 19:50:15 +0800991 sg = sg_last(areq->dst, edesc->dst_nents);
Herbert Xuaeb4c132015-07-30 17:53:22 +0800992 memcpy((char *)sg_virt(sg) + sg->length - authsize,
993 icvdata, authsize);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800994 }
995
996 kfree(edesc);
997
998 aead_request_complete(areq, err);
999}
1000
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001001static void ipsec_esp_decrypt_swauth_done(struct device *dev,
Kim Phillipse938e462009-03-29 15:53:23 +08001002 struct talitos_desc *desc,
1003 void *context, int err)
Kim Phillips9c4a7962008-06-23 19:50:15 +08001004{
1005 struct aead_request *req = context;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001006 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001007 unsigned int authsize = crypto_aead_authsize(authenc);
Kim Phillips19bbbc62009-03-29 15:53:59 +08001008 struct talitos_edesc *edesc;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001009 struct scatterlist *sg;
Herbert Xuaeb4c132015-07-30 17:53:22 +08001010 char *oicv, *icv;
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001011 struct talitos_private *priv = dev_get_drvdata(dev);
1012 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001013
Kim Phillips19bbbc62009-03-29 15:53:59 +08001014 edesc = container_of(desc, struct talitos_edesc, desc);
1015
Kim Phillips9c4a7962008-06-23 19:50:15 +08001016 ipsec_esp_unmap(dev, edesc, req);
1017
1018 if (!err) {
1019 /* auth check */
Kim Phillips9c4a7962008-06-23 19:50:15 +08001020 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001021 icv = (char *)sg_virt(sg) + sg->length - authsize;
1022
1023 if (edesc->dma_len) {
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001024 if (is_sec1)
1025 oicv = (char *)&edesc->dma_link_tbl +
1026 req->assoclen + req->cryptlen;
1027 else
1028 oicv = (char *)
1029 &edesc->link_tbl[edesc->src_nents +
Herbert Xuaeb4c132015-07-30 17:53:22 +08001030 edesc->dst_nents + 2];
1031 if (edesc->icv_ool)
1032 icv = oicv + authsize;
1033 } else
1034 oicv = (char *)&edesc->link_tbl[0];
1035
David Gstir79960942015-11-15 17:14:42 +01001036 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001037 }
1038
1039 kfree(edesc);
1040
1041 aead_request_complete(req, err);
1042}
1043
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001044static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
Kim Phillipse938e462009-03-29 15:53:23 +08001045 struct talitos_desc *desc,
1046 void *context, int err)
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001047{
1048 struct aead_request *req = context;
Kim Phillips19bbbc62009-03-29 15:53:59 +08001049 struct talitos_edesc *edesc;
1050
1051 edesc = container_of(desc, struct talitos_edesc, desc);
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001052
1053 ipsec_esp_unmap(dev, edesc, req);
1054
1055 /* check ICV auth status */
Kim Phillipse938e462009-03-29 15:53:23 +08001056 if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
1057 DESC_HDR_LO_ICCR1_PASS))
1058 err = -EBADMSG;
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001059
1060 kfree(edesc);
1061
1062 aead_request_complete(req, err);
1063}
1064
Kim Phillips9c4a7962008-06-23 19:50:15 +08001065/*
1066 * convert scatterlist to SEC h/w link table format
1067 * stop at cryptlen bytes
1068 */
Herbert Xuaeb4c132015-07-30 17:53:22 +08001069static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
1070 unsigned int offset, int cryptlen,
1071 struct talitos_ptr *link_tbl_ptr)
Kim Phillips9c4a7962008-06-23 19:50:15 +08001072{
Lee Nipper70bcaca2008-07-03 19:08:46 +08001073 int n_sg = sg_count;
Herbert Xuaeb4c132015-07-30 17:53:22 +08001074 int count = 0;
Lee Nipper70bcaca2008-07-03 19:08:46 +08001075
Herbert Xuaeb4c132015-07-30 17:53:22 +08001076 while (cryptlen && sg && n_sg--) {
1077 unsigned int len = sg_dma_len(sg);
1078
1079 if (offset >= len) {
1080 offset -= len;
1081 goto next;
1082 }
1083
1084 len -= offset;
1085
1086 if (len > cryptlen)
1087 len = cryptlen;
1088
1089 to_talitos_ptr(link_tbl_ptr + count,
LEROY Christopheda9de142017-10-06 15:04:57 +02001090 sg_dma_address(sg) + offset, len, 0);
LEROY Christopheb096b542016-06-06 13:20:34 +02001091 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001092 count++;
1093 cryptlen -= len;
1094 offset = 0;
1095
1096next:
Cristian Stoica5be4d4c2015-01-20 10:06:16 +02001097 sg = sg_next(sg);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001098 }
1099
Kim Phillips9c4a7962008-06-23 19:50:15 +08001100 /* tag end of link table */
Herbert Xuaeb4c132015-07-30 17:53:22 +08001101 if (count > 0)
LEROY Christopheb096b542016-06-06 13:20:34 +02001102 to_talitos_ptr_ext_set(link_tbl_ptr + count - 1,
1103 DESC_PTR_LNKTBL_RETURN, 0);
Lee Nipper70bcaca2008-07-03 19:08:46 +08001104
Herbert Xuaeb4c132015-07-30 17:53:22 +08001105 return count;
1106}
1107
LEROY Christophe5b2cf262017-10-06 15:04:47 +02001108static int talitos_sg_map(struct device *dev, struct scatterlist *src,
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001109 unsigned int len, struct talitos_edesc *edesc,
1110 struct talitos_ptr *ptr,
1111 int sg_count, unsigned int offset, int tbl_off)
LEROY Christophe246a87c2016-06-06 13:20:36 +02001112{
LEROY Christophe246a87c2016-06-06 13:20:36 +02001113 struct talitos_private *priv = dev_get_drvdata(dev);
1114 bool is_sec1 = has_ftr_sec1(priv);
1115
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001116 if (sg_count == 1) {
LEROY Christopheda9de142017-10-06 15:04:57 +02001117 to_talitos_ptr(ptr, sg_dma_address(src) + offset, len, is_sec1);
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001118 return sg_count;
LEROY Christophe246a87c2016-06-06 13:20:36 +02001119 }
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001120 if (is_sec1) {
LEROY Christopheda9de142017-10-06 15:04:57 +02001121 to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, len, is_sec1);
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001122 return sg_count;
1123 }
1124 sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len,
1125 &edesc->link_tbl[tbl_off]);
1126 if (sg_count == 1) {
1127 /* Only one segment now, so no link tbl needed*/
1128 copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1);
1129 return sg_count;
1130 }
1131 to_talitos_ptr(ptr, edesc->dma_link_tbl +
LEROY Christopheda9de142017-10-06 15:04:57 +02001132 tbl_off * sizeof(struct talitos_ptr), len, is_sec1);
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001133 to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1);
1134
LEROY Christophe246a87c2016-06-06 13:20:36 +02001135 return sg_count;
1136}
1137
Kim Phillips9c4a7962008-06-23 19:50:15 +08001138/*
1139 * fill in and submit ipsec_esp descriptor
1140 */
Lee Nipper56af8cd2009-03-29 15:50:50 +08001141static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
Herbert Xuaeb4c132015-07-30 17:53:22 +08001142 void (*callback)(struct device *dev,
1143 struct talitos_desc *desc,
1144 void *context, int error))
Kim Phillips9c4a7962008-06-23 19:50:15 +08001145{
1146 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001147 unsigned int authsize = crypto_aead_authsize(aead);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001148 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1149 struct device *dev = ctx->dev;
1150 struct talitos_desc *desc = &edesc->desc;
1151 unsigned int cryptlen = areq->cryptlen;
Kim Phillipse41256f2009-08-13 11:49:06 +10001152 unsigned int ivsize = crypto_aead_ivsize(aead);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001153 int tbl_off = 0;
Kim Phillipsfa86a262008-07-17 20:20:06 +08001154 int sg_count, ret;
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001155 int sg_link_tbl_len;
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001156 bool sync_needed = false;
1157 struct talitos_private *priv = dev_get_drvdata(dev);
1158 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001159
1160 /* hmac key */
1161 map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
LEROY Christophea2b35aa2015-04-17 16:31:57 +02001162 DMA_TO_DEVICE);
Horia Geanta79fd31d2012-08-02 17:16:40 +03001163
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001164 sg_count = edesc->src_nents ?: 1;
1165 if (is_sec1 && sg_count > 1)
1166 sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1167 areq->assoclen + cryptlen);
1168 else
1169 sg_count = dma_map_sg(dev, areq->src, sg_count,
1170 (areq->src == areq->dst) ?
1171 DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1172
Kim Phillips9c4a7962008-06-23 19:50:15 +08001173 /* hmac data */
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001174 ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc,
1175 &desc->ptr[1], sg_count, 0, tbl_off);
Horia Geanta79fd31d2012-08-02 17:16:40 +03001176
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001177 if (ret > 1) {
Horia Geant?340ff602016-04-19 20:33:48 +03001178 tbl_off += ret;
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001179 sync_needed = true;
Horia Geanta79fd31d2012-08-02 17:16:40 +03001180 }
1181
Kim Phillips9c4a7962008-06-23 19:50:15 +08001182 /* cipher iv */
LEROY Christopheda9de142017-10-06 15:04:57 +02001183 if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)
1184 to_talitos_ptr(&desc->ptr[2], edesc->iv_dma, ivsize, is_sec1);
1185 else
1186 to_talitos_ptr(&desc->ptr[3], edesc->iv_dma, ivsize, is_sec1);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001187
1188 /* cipher key */
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001189 if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)
1190 map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
1191 (char *)&ctx->key + ctx->authkeylen,
1192 DMA_TO_DEVICE);
1193 else
1194 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->enckeylen,
1195 (char *)&ctx->key + ctx->authkeylen,
1196 DMA_TO_DEVICE);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001197
1198 /*
1199 * cipher in
1200 * map and adjust cipher len to aead request cryptlen.
1201 * extent is bytes of HMAC postpended to ciphertext,
1202 * typically 12 for ipsec
1203 */
Herbert Xuaeb4c132015-07-30 17:53:22 +08001204 sg_link_tbl_len = cryptlen;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001205
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001206 if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) {
1207 to_talitos_ptr_ext_set(&desc->ptr[4], authsize, is_sec1);
1208
1209 if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV)
1210 sg_link_tbl_len += authsize;
1211 }
1212
LEROY Christophefbb22132017-10-06 15:04:41 +02001213 ret = talitos_sg_map(dev, areq->src, sg_link_tbl_len, edesc,
1214 &desc->ptr[4], sg_count, areq->assoclen, tbl_off);
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001215
LEROY Christopheec8c7d12017-10-06 15:04:33 +02001216 if (ret > 1) {
1217 tbl_off += ret;
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001218 sync_needed = true;
Horia Geant?340ff602016-04-19 20:33:48 +03001219 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08001220
1221 /* cipher out */
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001222 if (areq->src != areq->dst) {
1223 sg_count = edesc->dst_nents ? : 1;
1224 if (!is_sec1 || sg_count == 1)
1225 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1226 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08001227
LEROY Christophee04a61b2017-10-06 15:04:35 +02001228 ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[5],
1229 sg_count, areq->assoclen, tbl_off);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001230
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001231 if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)
1232 to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001233
LEROY Christophee04a61b2017-10-06 15:04:35 +02001234 /* ICV data */
1235 if (ret > 1) {
1236 tbl_off += ret;
Herbert Xuaeb4c132015-07-30 17:53:22 +08001237 edesc->icv_ool = true;
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001238 sync_needed = true;
1239
1240 if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP) {
1241 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1242 int offset = (edesc->src_nents + edesc->dst_nents + 2) *
1243 sizeof(struct talitos_ptr) + authsize;
1244
1245 /* Add an entry to the link table for ICV data */
LEROY Christophee04a61b2017-10-06 15:04:35 +02001246 to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1);
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001247 to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RETURN,
1248 is_sec1);
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001249
1250 /* icv data follows link tables */
1251 to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + offset,
LEROY Christopheda9de142017-10-06 15:04:57 +02001252 authsize, is_sec1);
LEROY Christophee04a61b2017-10-06 15:04:35 +02001253 } else {
1254 dma_addr_t addr = edesc->dma_link_tbl;
1255
1256 if (is_sec1)
1257 addr += areq->assoclen + cryptlen;
1258 else
1259 addr += sizeof(struct talitos_ptr) * tbl_off;
1260
LEROY Christopheda9de142017-10-06 15:04:57 +02001261 to_talitos_ptr(&desc->ptr[6], addr, authsize, is_sec1);
LEROY Christophee04a61b2017-10-06 15:04:35 +02001262 }
1263 } else if (!(desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)) {
1264 ret = talitos_sg_map(dev, areq->dst, authsize, edesc,
1265 &desc->ptr[6], sg_count, areq->assoclen +
1266 cryptlen,
1267 tbl_off);
1268 if (ret > 1) {
1269 tbl_off += ret;
1270 edesc->icv_ool = true;
1271 sync_needed = true;
1272 } else {
1273 edesc->icv_ool = false;
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001274 }
Horia Geant?340ff602016-04-19 20:33:48 +03001275 } else {
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001276 edesc->icv_ool = false;
1277 }
1278
Kim Phillips9c4a7962008-06-23 19:50:15 +08001279 /* iv out */
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001280 if (desc->hdr & DESC_HDR_TYPE_IPSEC_ESP)
1281 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv,
1282 DMA_FROM_DEVICE);
1283
1284 if (sync_needed)
1285 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1286 edesc->dma_len,
1287 DMA_BIDIRECTIONAL);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001288
Kim Phillips5228f0f2011-07-15 11:21:38 +08001289 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
Kim Phillipsfa86a262008-07-17 20:20:06 +08001290 if (ret != -EINPROGRESS) {
1291 ipsec_esp_unmap(dev, edesc, areq);
1292 kfree(edesc);
1293 }
1294 return ret;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001295}
1296
Kim Phillips9c4a7962008-06-23 19:50:15 +08001297/*
Lee Nipper56af8cd2009-03-29 15:50:50 +08001298 * allocate and map the extended descriptor
Kim Phillips9c4a7962008-06-23 19:50:15 +08001299 */
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001300static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1301 struct scatterlist *src,
1302 struct scatterlist *dst,
Horia Geanta79fd31d2012-08-02 17:16:40 +03001303 u8 *iv,
1304 unsigned int assoclen,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001305 unsigned int cryptlen,
1306 unsigned int authsize,
Horia Geanta79fd31d2012-08-02 17:16:40 +03001307 unsigned int ivsize,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001308 int icv_stashing,
Horia Geanta62293a32013-11-28 15:11:17 +02001309 u32 cryptoflags,
1310 bool encrypt)
Kim Phillips9c4a7962008-06-23 19:50:15 +08001311{
Lee Nipper56af8cd2009-03-29 15:50:50 +08001312 struct talitos_edesc *edesc;
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001313 int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len;
Horia Geanta79fd31d2012-08-02 17:16:40 +03001314 dma_addr_t iv_dma = 0;
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001315 gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
Kim Phillips586725f2008-07-17 20:19:18 +08001316 GFP_ATOMIC;
LEROY Christophe6f65f6a2015-04-17 16:32:15 +02001317 struct talitos_private *priv = dev_get_drvdata(dev);
1318 bool is_sec1 = has_ftr_sec1(priv);
1319 int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN;
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001320 void *err;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001321
LEROY Christophe6f65f6a2015-04-17 16:32:15 +02001322 if (cryptlen + authsize > max_len) {
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001323 dev_err(dev, "length exceeds h/w max limit\n");
Kim Phillips9c4a7962008-06-23 19:50:15 +08001324 return ERR_PTR(-EINVAL);
1325 }
1326
Horia Geanta935e99a2013-11-19 14:57:49 +02001327 if (ivsize)
Horia Geanta79fd31d2012-08-02 17:16:40 +03001328 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
1329
Horia Geanta62293a32013-11-28 15:11:17 +02001330 if (!dst || dst == src) {
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001331 src_len = assoclen + cryptlen + authsize;
1332 src_nents = sg_nents_for_len(src, src_len);
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001333 if (src_nents < 0) {
1334 dev_err(dev, "Invalid number of src SG.\n");
1335 err = ERR_PTR(-EINVAL);
1336 goto error_sg;
1337 }
Horia Geanta62293a32013-11-28 15:11:17 +02001338 src_nents = (src_nents == 1) ? 0 : src_nents;
1339 dst_nents = dst ? src_nents : 0;
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001340 dst_len = 0;
Horia Geanta62293a32013-11-28 15:11:17 +02001341 } else { /* dst && dst != src*/
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001342 src_len = assoclen + cryptlen + (encrypt ? 0 : authsize);
1343 src_nents = sg_nents_for_len(src, src_len);
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001344 if (src_nents < 0) {
1345 dev_err(dev, "Invalid number of src SG.\n");
1346 err = ERR_PTR(-EINVAL);
1347 goto error_sg;
1348 }
Horia Geanta62293a32013-11-28 15:11:17 +02001349 src_nents = (src_nents == 1) ? 0 : src_nents;
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001350 dst_len = assoclen + cryptlen + (encrypt ? authsize : 0);
1351 dst_nents = sg_nents_for_len(dst, dst_len);
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001352 if (dst_nents < 0) {
1353 dev_err(dev, "Invalid number of dst SG.\n");
1354 err = ERR_PTR(-EINVAL);
1355 goto error_sg;
1356 }
Horia Geanta62293a32013-11-28 15:11:17 +02001357 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001358 }
1359
1360 /*
1361 * allocate space for base edesc plus the link tables,
Herbert Xuaeb4c132015-07-30 17:53:22 +08001362 * allowing for two separate entries for AD and generated ICV (+ 2),
1363 * and space for two sets of ICVs (stashed and generated)
Kim Phillips9c4a7962008-06-23 19:50:15 +08001364 */
Lee Nipper56af8cd2009-03-29 15:50:50 +08001365 alloc_len = sizeof(struct talitos_edesc);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001366 if (src_nents || dst_nents) {
LEROY Christophe6f65f6a2015-04-17 16:32:15 +02001367 if (is_sec1)
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001368 dma_len = (src_nents ? src_len : 0) +
1369 (dst_nents ? dst_len : 0);
LEROY Christophe6f65f6a2015-04-17 16:32:15 +02001370 else
Herbert Xuaeb4c132015-07-30 17:53:22 +08001371 dma_len = (src_nents + dst_nents + 2) *
1372 sizeof(struct talitos_ptr) + authsize * 2;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001373 alloc_len += dma_len;
1374 } else {
1375 dma_len = 0;
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001376 alloc_len += icv_stashing ? authsize : 0;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001377 }
1378
Kim Phillips586725f2008-07-17 20:19:18 +08001379 edesc = kmalloc(alloc_len, GFP_DMA | flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001380 if (!edesc) {
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001381 dev_err(dev, "could not allocate edescriptor\n");
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001382 err = ERR_PTR(-ENOMEM);
1383 goto error_sg;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001384 }
LEROY Christophee4a647c2017-10-06 15:04:45 +02001385 memset(&edesc->desc, 0, sizeof(edesc->desc));
Kim Phillips9c4a7962008-06-23 19:50:15 +08001386
1387 edesc->src_nents = src_nents;
1388 edesc->dst_nents = dst_nents;
Horia Geanta79fd31d2012-08-02 17:16:40 +03001389 edesc->iv_dma = iv_dma;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001390 edesc->dma_len = dma_len;
Lee Nipper497f2e62010-05-19 19:20:36 +10001391 if (dma_len)
1392 edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
1393 edesc->dma_len,
1394 DMA_BIDIRECTIONAL);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001395
1396 return edesc;
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001397error_sg:
1398 if (iv_dma)
1399 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
1400 return err;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001401}
1402
Horia Geanta79fd31d2012-08-02 17:16:40 +03001403static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
Horia Geanta62293a32013-11-28 15:11:17 +02001404 int icv_stashing, bool encrypt)
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001405{
1406 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001407 unsigned int authsize = crypto_aead_authsize(authenc);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001408 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
Horia Geanta79fd31d2012-08-02 17:16:40 +03001409 unsigned int ivsize = crypto_aead_ivsize(authenc);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001410
Herbert Xuaeb4c132015-07-30 17:53:22 +08001411 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
Horia Geanta79fd31d2012-08-02 17:16:40 +03001412 iv, areq->assoclen, areq->cryptlen,
Herbert Xuaeb4c132015-07-30 17:53:22 +08001413 authsize, ivsize, icv_stashing,
Horia Geanta62293a32013-11-28 15:11:17 +02001414 areq->base.flags, encrypt);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001415}
1416
Lee Nipper56af8cd2009-03-29 15:50:50 +08001417static int aead_encrypt(struct aead_request *req)
Kim Phillips9c4a7962008-06-23 19:50:15 +08001418{
1419 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1420 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
Lee Nipper56af8cd2009-03-29 15:50:50 +08001421 struct talitos_edesc *edesc;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001422
1423 /* allocate extended descriptor */
Horia Geanta62293a32013-11-28 15:11:17 +02001424 edesc = aead_edesc_alloc(req, req->iv, 0, true);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001425 if (IS_ERR(edesc))
1426 return PTR_ERR(edesc);
1427
1428 /* set encrypt */
Lee Nipper70bcaca2008-07-03 19:08:46 +08001429 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001430
Herbert Xuaeb4c132015-07-30 17:53:22 +08001431 return ipsec_esp(edesc, req, ipsec_esp_encrypt_done);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001432}
1433
Lee Nipper56af8cd2009-03-29 15:50:50 +08001434static int aead_decrypt(struct aead_request *req)
Kim Phillips9c4a7962008-06-23 19:50:15 +08001435{
1436 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001437 unsigned int authsize = crypto_aead_authsize(authenc);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001438 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001439 struct talitos_private *priv = dev_get_drvdata(ctx->dev);
Lee Nipper56af8cd2009-03-29 15:50:50 +08001440 struct talitos_edesc *edesc;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001441 struct scatterlist *sg;
1442 void *icvdata;
1443
1444 req->cryptlen -= authsize;
1445
1446 /* allocate extended descriptor */
Horia Geanta62293a32013-11-28 15:11:17 +02001447 edesc = aead_edesc_alloc(req, req->iv, 1, false);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001448 if (IS_ERR(edesc))
1449 return PTR_ERR(edesc);
1450
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001451 if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
Kim Phillipse938e462009-03-29 15:53:23 +08001452 ((!edesc->src_nents && !edesc->dst_nents) ||
1453 priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
Kim Phillips9c4a7962008-06-23 19:50:15 +08001454
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001455 /* decrypt and check the ICV */
Kim Phillipse938e462009-03-29 15:53:23 +08001456 edesc->desc.hdr = ctx->desc_hdr_template |
1457 DESC_HDR_DIR_INBOUND |
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001458 DESC_HDR_MODE1_MDEU_CICV;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001459
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001460 /* reset integrity check result bits */
Kim Phillips9c4a7962008-06-23 19:50:15 +08001461
Herbert Xuaeb4c132015-07-30 17:53:22 +08001462 return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done);
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001463 }
Kim Phillipse938e462009-03-29 15:53:23 +08001464
1465 /* Have to check the ICV with software */
1466 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1467
1468 /* stash incoming ICV for later cmp with ICV generated by the h/w */
1469 if (edesc->dma_len)
Herbert Xuaeb4c132015-07-30 17:53:22 +08001470 icvdata = (char *)&edesc->link_tbl[edesc->src_nents +
1471 edesc->dst_nents + 2];
Kim Phillipse938e462009-03-29 15:53:23 +08001472 else
1473 icvdata = &edesc->link_tbl[0];
1474
1475 sg = sg_last(req->src, edesc->src_nents ? : 1);
1476
Herbert Xuaeb4c132015-07-30 17:53:22 +08001477 memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize);
Kim Phillipse938e462009-03-29 15:53:23 +08001478
Herbert Xuaeb4c132015-07-30 17:53:22 +08001479 return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001480}
1481
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001482static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1483 const u8 *key, unsigned int keylen)
1484{
1485 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
LEROY Christophef384cdc2017-10-06 15:04:37 +02001486 u32 tmp[DES_EXPKEY_WORDS];
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001487
Martin Hicks03d2c512017-05-02 09:38:35 -04001488 if (keylen > TALITOS_MAX_KEY_SIZE) {
1489 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
1490 return -EINVAL;
1491 }
1492
LEROY Christophef384cdc2017-10-06 15:04:37 +02001493 if (unlikely(crypto_ablkcipher_get_flags(cipher) &
1494 CRYPTO_TFM_REQ_WEAK_KEY) &&
1495 !des_ekey(tmp, key)) {
1496 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_WEAK_KEY);
1497 return -EINVAL;
1498 }
1499
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001500 memcpy(&ctx->key, key, keylen);
1501 ctx->keylen = keylen;
1502
1503 return 0;
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001504}
1505
1506static void common_nonsnoop_unmap(struct device *dev,
1507 struct talitos_edesc *edesc,
1508 struct ablkcipher_request *areq)
1509{
1510 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
LEROY Christophe032d1972015-04-17 16:31:51 +02001511
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001512 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->nbytes, 0);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001513 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
1514 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1515
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001516 if (edesc->dma_len)
1517 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1518 DMA_BIDIRECTIONAL);
1519}
1520
1521static void ablkcipher_done(struct device *dev,
1522 struct talitos_desc *desc, void *context,
1523 int err)
1524{
1525 struct ablkcipher_request *areq = context;
Kim Phillips19bbbc62009-03-29 15:53:59 +08001526 struct talitos_edesc *edesc;
1527
1528 edesc = container_of(desc, struct talitos_edesc, desc);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001529
1530 common_nonsnoop_unmap(dev, edesc, areq);
1531
1532 kfree(edesc);
1533
1534 areq->base.complete(&areq->base, err);
1535}
1536
1537static int common_nonsnoop(struct talitos_edesc *edesc,
1538 struct ablkcipher_request *areq,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001539 void (*callback) (struct device *dev,
1540 struct talitos_desc *desc,
1541 void *context, int error))
1542{
1543 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1544 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1545 struct device *dev = ctx->dev;
1546 struct talitos_desc *desc = &edesc->desc;
1547 unsigned int cryptlen = areq->nbytes;
Horia Geanta79fd31d2012-08-02 17:16:40 +03001548 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001549 int sg_count, ret;
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001550 bool sync_needed = false;
LEROY Christophe922f9dc2015-04-17 16:32:07 +02001551 struct talitos_private *priv = dev_get_drvdata(dev);
1552 bool is_sec1 = has_ftr_sec1(priv);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001553
1554 /* first DWORD empty */
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001555
1556 /* cipher iv */
LEROY Christopheda9de142017-10-06 15:04:57 +02001557 to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, ivsize, is_sec1);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001558
1559 /* cipher key */
1560 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
LEROY Christophea2b35aa2015-04-17 16:31:57 +02001561 (char *)&ctx->key, DMA_TO_DEVICE);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001562
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001563 sg_count = edesc->src_nents ?: 1;
1564 if (is_sec1 && sg_count > 1)
1565 sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1566 cryptlen);
1567 else
1568 sg_count = dma_map_sg(dev, areq->src, sg_count,
1569 (areq->src == areq->dst) ?
1570 DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001571 /*
1572 * cipher in
1573 */
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001574 sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc,
1575 &desc->ptr[3], sg_count, 0, 0);
1576 if (sg_count > 1)
1577 sync_needed = true;
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001578
1579 /* cipher out */
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001580 if (areq->src != areq->dst) {
1581 sg_count = edesc->dst_nents ? : 1;
1582 if (!is_sec1 || sg_count == 1)
1583 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1584 }
1585
1586 ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4],
1587 sg_count, 0, (edesc->src_nents + 1));
1588 if (ret > 1)
1589 sync_needed = true;
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001590
1591 /* iv out */
LEROY Christophea2b35aa2015-04-17 16:31:57 +02001592 map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001593 DMA_FROM_DEVICE);
1594
1595 /* last DWORD empty */
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001596
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001597 if (sync_needed)
1598 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1599 edesc->dma_len, DMA_BIDIRECTIONAL);
1600
Kim Phillips5228f0f2011-07-15 11:21:38 +08001601 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001602 if (ret != -EINPROGRESS) {
1603 common_nonsnoop_unmap(dev, edesc, areq);
1604 kfree(edesc);
1605 }
1606 return ret;
1607}
1608
Kim Phillipse938e462009-03-29 15:53:23 +08001609static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
Horia Geanta62293a32013-11-28 15:11:17 +02001610 areq, bool encrypt)
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001611{
1612 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1613 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
Horia Geanta79fd31d2012-08-02 17:16:40 +03001614 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001615
Herbert Xuaeb4c132015-07-30 17:53:22 +08001616 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
Horia Geanta79fd31d2012-08-02 17:16:40 +03001617 areq->info, 0, areq->nbytes, 0, ivsize, 0,
Horia Geanta62293a32013-11-28 15:11:17 +02001618 areq->base.flags, encrypt);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001619}
1620
1621static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1622{
1623 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1624 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1625 struct talitos_edesc *edesc;
1626
1627 /* allocate extended descriptor */
Horia Geanta62293a32013-11-28 15:11:17 +02001628 edesc = ablkcipher_edesc_alloc(areq, true);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001629 if (IS_ERR(edesc))
1630 return PTR_ERR(edesc);
1631
1632 /* set encrypt */
1633 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1634
Kim Phillipsfebec542011-07-15 11:21:39 +08001635 return common_nonsnoop(edesc, areq, ablkcipher_done);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001636}
1637
1638static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1639{
1640 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1641 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1642 struct talitos_edesc *edesc;
1643
1644 /* allocate extended descriptor */
Horia Geanta62293a32013-11-28 15:11:17 +02001645 edesc = ablkcipher_edesc_alloc(areq, false);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001646 if (IS_ERR(edesc))
1647 return PTR_ERR(edesc);
1648
1649 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1650
Kim Phillipsfebec542011-07-15 11:21:39 +08001651 return common_nonsnoop(edesc, areq, ablkcipher_done);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001652}
1653
Lee Nipper497f2e62010-05-19 19:20:36 +10001654static void common_nonsnoop_hash_unmap(struct device *dev,
1655 struct talitos_edesc *edesc,
1656 struct ahash_request *areq)
1657{
1658 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
LEROY Christophe922f9dc2015-04-17 16:32:07 +02001659 struct talitos_private *priv = dev_get_drvdata(dev);
1660 bool is_sec1 = has_ftr_sec1(priv);
Lee Nipper497f2e62010-05-19 19:20:36 +10001661
1662 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1663
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001664 talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0);
LEROY Christophe032d1972015-04-17 16:31:51 +02001665
Lee Nipper497f2e62010-05-19 19:20:36 +10001666 /* When using hashctx-in, must unmap it. */
LEROY Christophe922f9dc2015-04-17 16:32:07 +02001667 if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1))
Lee Nipper497f2e62010-05-19 19:20:36 +10001668 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],
1669 DMA_TO_DEVICE);
1670
LEROY Christophe922f9dc2015-04-17 16:32:07 +02001671 if (from_talitos_ptr_len(&edesc->desc.ptr[2], is_sec1))
Lee Nipper497f2e62010-05-19 19:20:36 +10001672 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2],
1673 DMA_TO_DEVICE);
1674
Lee Nipper497f2e62010-05-19 19:20:36 +10001675 if (edesc->dma_len)
1676 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1677 DMA_BIDIRECTIONAL);
1678
1679}
1680
1681static void ahash_done(struct device *dev,
1682 struct talitos_desc *desc, void *context,
1683 int err)
1684{
1685 struct ahash_request *areq = context;
1686 struct talitos_edesc *edesc =
1687 container_of(desc, struct talitos_edesc, desc);
1688 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1689
1690 if (!req_ctx->last && req_ctx->to_hash_later) {
1691 /* Position any partial block for next update/final/finup */
1692 memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later);
Lee Nipper5e833bc2010-06-16 15:29:15 +10001693 req_ctx->nbuf = req_ctx->to_hash_later;
Lee Nipper497f2e62010-05-19 19:20:36 +10001694 }
1695 common_nonsnoop_hash_unmap(dev, edesc, areq);
1696
1697 kfree(edesc);
1698
1699 areq->base.complete(&areq->base, err);
1700}
1701
LEROY Christophe2d029052015-04-17 16:32:18 +02001702/*
1703 * SEC1 doesn't like hashing of 0 sized message, so we do the padding
1704 * ourself and submit a padded block
1705 */
LEROY Christophe5b2cf262017-10-06 15:04:47 +02001706static void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
LEROY Christophe2d029052015-04-17 16:32:18 +02001707 struct talitos_edesc *edesc,
1708 struct talitos_ptr *ptr)
1709{
1710 static u8 padded_hash[64] = {
1711 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1712 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1713 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1714 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1715 };
1716
1717 pr_err_once("Bug in SEC1, padding ourself\n");
1718 edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1719 map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash),
1720 (char *)padded_hash, DMA_TO_DEVICE);
1721}
1722
Lee Nipper497f2e62010-05-19 19:20:36 +10001723static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1724 struct ahash_request *areq, unsigned int length,
1725 void (*callback) (struct device *dev,
1726 struct talitos_desc *desc,
1727 void *context, int error))
1728{
1729 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1730 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1731 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1732 struct device *dev = ctx->dev;
1733 struct talitos_desc *desc = &edesc->desc;
LEROY Christophe032d1972015-04-17 16:31:51 +02001734 int ret;
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001735 bool sync_needed = false;
LEROY Christophe922f9dc2015-04-17 16:32:07 +02001736 struct talitos_private *priv = dev_get_drvdata(dev);
1737 bool is_sec1 = has_ftr_sec1(priv);
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001738 int sg_count;
Lee Nipper497f2e62010-05-19 19:20:36 +10001739
1740 /* first DWORD empty */
Lee Nipper497f2e62010-05-19 19:20:36 +10001741
Kim Phillips60f208d2010-05-19 19:21:53 +10001742 /* hash context in */
1743 if (!req_ctx->first || req_ctx->swinit) {
Lee Nipper497f2e62010-05-19 19:20:36 +10001744 map_single_talitos_ptr(dev, &desc->ptr[1],
1745 req_ctx->hw_context_size,
LEROY Christophea2b35aa2015-04-17 16:31:57 +02001746 (char *)req_ctx->hw_context,
Lee Nipper497f2e62010-05-19 19:20:36 +10001747 DMA_TO_DEVICE);
Kim Phillips60f208d2010-05-19 19:21:53 +10001748 req_ctx->swinit = 0;
Lee Nipper497f2e62010-05-19 19:20:36 +10001749 }
LEROY Christopheafd62fa2017-09-13 12:44:51 +02001750 /* Indicate next op is not the first. */
1751 req_ctx->first = 0;
Lee Nipper497f2e62010-05-19 19:20:36 +10001752
1753 /* HMAC key */
1754 if (ctx->keylen)
1755 map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
LEROY Christophea2b35aa2015-04-17 16:31:57 +02001756 (char *)&ctx->key, DMA_TO_DEVICE);
Lee Nipper497f2e62010-05-19 19:20:36 +10001757
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001758 sg_count = edesc->src_nents ?: 1;
1759 if (is_sec1 && sg_count > 1)
LEROY Christophe886a27c2017-09-13 12:44:57 +02001760 sg_copy_to_buffer(req_ctx->psrc, sg_count, edesc->buf, length);
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001761 else
1762 sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count,
1763 DMA_TO_DEVICE);
Lee Nipper497f2e62010-05-19 19:20:36 +10001764 /*
1765 * data in
1766 */
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001767 sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
1768 &desc->ptr[3], sg_count, 0, 0);
1769 if (sg_count > 1)
1770 sync_needed = true;
Lee Nipper497f2e62010-05-19 19:20:36 +10001771
1772 /* fifth DWORD empty */
Lee Nipper497f2e62010-05-19 19:20:36 +10001773
1774 /* hash/HMAC out -or- hash context out */
1775 if (req_ctx->last)
1776 map_single_talitos_ptr(dev, &desc->ptr[5],
1777 crypto_ahash_digestsize(tfm),
LEROY Christophea2b35aa2015-04-17 16:31:57 +02001778 areq->result, DMA_FROM_DEVICE);
Lee Nipper497f2e62010-05-19 19:20:36 +10001779 else
1780 map_single_talitos_ptr(dev, &desc->ptr[5],
1781 req_ctx->hw_context_size,
LEROY Christophea2b35aa2015-04-17 16:31:57 +02001782 req_ctx->hw_context, DMA_FROM_DEVICE);
Lee Nipper497f2e62010-05-19 19:20:36 +10001783
1784 /* last DWORD empty */
Lee Nipper497f2e62010-05-19 19:20:36 +10001785
LEROY Christophe2d029052015-04-17 16:32:18 +02001786 if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0)
1787 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
1788
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001789 if (sync_needed)
1790 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1791 edesc->dma_len, DMA_BIDIRECTIONAL);
1792
Kim Phillips5228f0f2011-07-15 11:21:38 +08001793 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
Lee Nipper497f2e62010-05-19 19:20:36 +10001794 if (ret != -EINPROGRESS) {
1795 common_nonsnoop_hash_unmap(dev, edesc, areq);
1796 kfree(edesc);
1797 }
1798 return ret;
1799}
1800
1801static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1802 unsigned int nbytes)
1803{
1804 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1805 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1806 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1807
Herbert Xuaeb4c132015-07-30 17:53:22 +08001808 return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0,
Horia Geanta62293a32013-11-28 15:11:17 +02001809 nbytes, 0, 0, 0, areq->base.flags, false);
Lee Nipper497f2e62010-05-19 19:20:36 +10001810}
1811
1812static int ahash_init(struct ahash_request *areq)
1813{
1814 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1815 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1816
1817 /* Initialize the context */
Lee Nipper5e833bc2010-06-16 15:29:15 +10001818 req_ctx->nbuf = 0;
Kim Phillips60f208d2010-05-19 19:21:53 +10001819 req_ctx->first = 1; /* first indicates h/w must init its context */
1820 req_ctx->swinit = 0; /* assume h/w init of context */
Lee Nipper497f2e62010-05-19 19:20:36 +10001821 req_ctx->hw_context_size =
1822 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1823 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1824 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
1825
1826 return 0;
1827}
1828
Kim Phillips60f208d2010-05-19 19:21:53 +10001829/*
1830 * on h/w without explicit sha224 support, we initialize h/w context
1831 * manually with sha224 constants, and tell it to run sha256.
1832 */
1833static int ahash_init_sha224_swinit(struct ahash_request *areq)
1834{
1835 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1836
1837 ahash_init(areq);
1838 req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1839
Kim Phillipsa7524472010-09-23 15:56:38 +08001840 req_ctx->hw_context[0] = SHA224_H0;
1841 req_ctx->hw_context[1] = SHA224_H1;
1842 req_ctx->hw_context[2] = SHA224_H2;
1843 req_ctx->hw_context[3] = SHA224_H3;
1844 req_ctx->hw_context[4] = SHA224_H4;
1845 req_ctx->hw_context[5] = SHA224_H5;
1846 req_ctx->hw_context[6] = SHA224_H6;
1847 req_ctx->hw_context[7] = SHA224_H7;
Kim Phillips60f208d2010-05-19 19:21:53 +10001848
1849 /* init 64-bit count */
1850 req_ctx->hw_context[8] = 0;
1851 req_ctx->hw_context[9] = 0;
1852
1853 return 0;
1854}
1855
Lee Nipper497f2e62010-05-19 19:20:36 +10001856static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1857{
1858 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1859 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1860 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1861 struct talitos_edesc *edesc;
1862 unsigned int blocksize =
1863 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1864 unsigned int nbytes_to_hash;
1865 unsigned int to_hash_later;
Lee Nipper5e833bc2010-06-16 15:29:15 +10001866 unsigned int nsg;
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001867 int nents;
Lee Nipper497f2e62010-05-19 19:20:36 +10001868
Lee Nipper5e833bc2010-06-16 15:29:15 +10001869 if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
1870 /* Buffer up to one whole block */
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001871 nents = sg_nents_for_len(areq->src, nbytes);
1872 if (nents < 0) {
1873 dev_err(ctx->dev, "Invalid number of src SG.\n");
1874 return nents;
1875 }
1876 sg_copy_to_buffer(areq->src, nents,
Lee Nipper5e833bc2010-06-16 15:29:15 +10001877 req_ctx->buf + req_ctx->nbuf, nbytes);
1878 req_ctx->nbuf += nbytes;
Lee Nipper497f2e62010-05-19 19:20:36 +10001879 return 0;
1880 }
1881
Lee Nipper5e833bc2010-06-16 15:29:15 +10001882 /* At least (blocksize + 1) bytes are available to hash */
1883 nbytes_to_hash = nbytes + req_ctx->nbuf;
1884 to_hash_later = nbytes_to_hash & (blocksize - 1);
1885
1886 if (req_ctx->last)
1887 to_hash_later = 0;
1888 else if (to_hash_later)
1889 /* There is a partial block. Hash the full block(s) now */
1890 nbytes_to_hash -= to_hash_later;
1891 else {
1892 /* Keep one block buffered */
1893 nbytes_to_hash -= blocksize;
1894 to_hash_later = blocksize;
1895 }
1896
1897 /* Chain in any previously buffered data */
1898 if (req_ctx->nbuf) {
1899 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
1900 sg_init_table(req_ctx->bufsl, nsg);
1901 sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf);
1902 if (nsg > 1)
Dan Williamsc56f6d12015-08-07 18:15:13 +02001903 sg_chain(req_ctx->bufsl, 2, areq->src);
Lee Nipper497f2e62010-05-19 19:20:36 +10001904 req_ctx->psrc = req_ctx->bufsl;
Lee Nipper5e833bc2010-06-16 15:29:15 +10001905 } else
Lee Nipper497f2e62010-05-19 19:20:36 +10001906 req_ctx->psrc = areq->src;
Lee Nipper497f2e62010-05-19 19:20:36 +10001907
Lee Nipper5e833bc2010-06-16 15:29:15 +10001908 if (to_hash_later) {
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001909 nents = sg_nents_for_len(areq->src, nbytes);
1910 if (nents < 0) {
1911 dev_err(ctx->dev, "Invalid number of src SG.\n");
1912 return nents;
1913 }
Akinobu Mitad0525722013-07-08 16:01:55 -07001914 sg_pcopy_to_buffer(areq->src, nents,
Lee Nipper5e833bc2010-06-16 15:29:15 +10001915 req_ctx->bufnext,
1916 to_hash_later,
1917 nbytes - to_hash_later);
Lee Nipper497f2e62010-05-19 19:20:36 +10001918 }
Lee Nipper5e833bc2010-06-16 15:29:15 +10001919 req_ctx->to_hash_later = to_hash_later;
Lee Nipper497f2e62010-05-19 19:20:36 +10001920
Lee Nipper5e833bc2010-06-16 15:29:15 +10001921 /* Allocate extended descriptor */
Lee Nipper497f2e62010-05-19 19:20:36 +10001922 edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
1923 if (IS_ERR(edesc))
1924 return PTR_ERR(edesc);
1925
1926 edesc->desc.hdr = ctx->desc_hdr_template;
1927
1928 /* On last one, request SEC to pad; otherwise continue */
1929 if (req_ctx->last)
1930 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
1931 else
1932 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
1933
Kim Phillips60f208d2010-05-19 19:21:53 +10001934 /* request SEC to INIT hash. */
1935 if (req_ctx->first && !req_ctx->swinit)
Lee Nipper497f2e62010-05-19 19:20:36 +10001936 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
1937
1938 /* When the tfm context has a keylen, it's an HMAC.
1939 * A first or last (ie. not middle) descriptor must request HMAC.
1940 */
1941 if (ctx->keylen && (req_ctx->first || req_ctx->last))
1942 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
1943
1944 return common_nonsnoop_hash(edesc, areq, nbytes_to_hash,
1945 ahash_done);
1946}
1947
1948static int ahash_update(struct ahash_request *areq)
1949{
1950 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1951
1952 req_ctx->last = 0;
1953
1954 return ahash_process_req(areq, areq->nbytes);
1955}
1956
1957static int ahash_final(struct ahash_request *areq)
1958{
1959 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1960
1961 req_ctx->last = 1;
1962
1963 return ahash_process_req(areq, 0);
1964}
1965
1966static int ahash_finup(struct ahash_request *areq)
1967{
1968 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1969
1970 req_ctx->last = 1;
1971
1972 return ahash_process_req(areq, areq->nbytes);
1973}
1974
1975static int ahash_digest(struct ahash_request *areq)
1976{
1977 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
Kim Phillips60f208d2010-05-19 19:21:53 +10001978 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
Lee Nipper497f2e62010-05-19 19:20:36 +10001979
Kim Phillips60f208d2010-05-19 19:21:53 +10001980 ahash->init(areq);
Lee Nipper497f2e62010-05-19 19:20:36 +10001981 req_ctx->last = 1;
1982
1983 return ahash_process_req(areq, areq->nbytes);
1984}
1985
Horia Geant?3639ca82016-04-21 19:24:55 +03001986static int ahash_export(struct ahash_request *areq, void *out)
1987{
1988 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1989 struct talitos_export_state *export = out;
1990
1991 memcpy(export->hw_context, req_ctx->hw_context,
1992 req_ctx->hw_context_size);
1993 memcpy(export->buf, req_ctx->buf, req_ctx->nbuf);
1994 export->swinit = req_ctx->swinit;
1995 export->first = req_ctx->first;
1996 export->last = req_ctx->last;
1997 export->to_hash_later = req_ctx->to_hash_later;
1998 export->nbuf = req_ctx->nbuf;
1999
2000 return 0;
2001}
2002
2003static int ahash_import(struct ahash_request *areq, const void *in)
2004{
2005 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2006 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2007 const struct talitos_export_state *export = in;
2008
2009 memset(req_ctx, 0, sizeof(*req_ctx));
2010 req_ctx->hw_context_size =
2011 (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
2012 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
2013 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
2014 memcpy(req_ctx->hw_context, export->hw_context,
2015 req_ctx->hw_context_size);
2016 memcpy(req_ctx->buf, export->buf, export->nbuf);
2017 req_ctx->swinit = export->swinit;
2018 req_ctx->first = export->first;
2019 req_ctx->last = export->last;
2020 req_ctx->to_hash_later = export->to_hash_later;
2021 req_ctx->nbuf = export->nbuf;
2022
2023 return 0;
2024}
2025
Lee Nipper79b3a412011-11-21 16:13:25 +08002026struct keyhash_result {
2027 struct completion completion;
2028 int err;
2029};
2030
2031static void keyhash_complete(struct crypto_async_request *req, int err)
2032{
2033 struct keyhash_result *res = req->data;
2034
2035 if (err == -EINPROGRESS)
2036 return;
2037
2038 res->err = err;
2039 complete(&res->completion);
2040}
2041
2042static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
2043 u8 *hash)
2044{
2045 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2046
2047 struct scatterlist sg[1];
2048 struct ahash_request *req;
2049 struct keyhash_result hresult;
2050 int ret;
2051
2052 init_completion(&hresult.completion);
2053
2054 req = ahash_request_alloc(tfm, GFP_KERNEL);
2055 if (!req)
2056 return -ENOMEM;
2057
2058 /* Keep tfm keylen == 0 during hash of the long key */
2059 ctx->keylen = 0;
2060 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
2061 keyhash_complete, &hresult);
2062
2063 sg_init_one(&sg[0], key, keylen);
2064
2065 ahash_request_set_crypt(req, sg, hash, keylen);
2066 ret = crypto_ahash_digest(req);
2067 switch (ret) {
2068 case 0:
2069 break;
2070 case -EINPROGRESS:
2071 case -EBUSY:
2072 ret = wait_for_completion_interruptible(
2073 &hresult.completion);
2074 if (!ret)
2075 ret = hresult.err;
2076 break;
2077 default:
2078 break;
2079 }
2080 ahash_request_free(req);
2081
2082 return ret;
2083}
2084
2085static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2086 unsigned int keylen)
2087{
2088 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2089 unsigned int blocksize =
2090 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2091 unsigned int digestsize = crypto_ahash_digestsize(tfm);
2092 unsigned int keysize = keylen;
2093 u8 hash[SHA512_DIGEST_SIZE];
2094 int ret;
2095
2096 if (keylen <= blocksize)
2097 memcpy(ctx->key, key, keysize);
2098 else {
2099 /* Must get the hash of the long key */
2100 ret = keyhash(tfm, key, keylen, hash);
2101
2102 if (ret) {
2103 crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
2104 return -EINVAL;
2105 }
2106
2107 keysize = digestsize;
2108 memcpy(ctx->key, hash, digestsize);
2109 }
2110
2111 ctx->keylen = keysize;
2112
2113 return 0;
2114}
2115
2116
Kim Phillips9c4a7962008-06-23 19:50:15 +08002117struct talitos_alg_template {
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002118 u32 type;
LEROY Christopheb0057762016-06-06 13:20:44 +02002119 u32 priority;
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002120 union {
2121 struct crypto_alg crypto;
Lee Nipperacbf7c622010-05-19 19:19:33 +10002122 struct ahash_alg hash;
Herbert Xuaeb4c132015-07-30 17:53:22 +08002123 struct aead_alg aead;
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002124 } alg;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002125 __be32 desc_hdr_template;
2126};
2127
2128static struct talitos_alg_template driver_algs[] = {
Horia Geanta991155b2013-03-20 16:31:38 +02002129 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002130 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002131 .alg.aead = {
2132 .base = {
2133 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2134 .cra_driver_name = "authenc-hmac-sha1-"
2135 "cbc-aes-talitos",
2136 .cra_blocksize = AES_BLOCK_SIZE,
2137 .cra_flags = CRYPTO_ALG_ASYNC,
2138 },
2139 .ivsize = AES_BLOCK_SIZE,
2140 .maxauthsize = SHA1_DIGEST_SIZE,
Lee Nipper56af8cd2009-03-29 15:50:50 +08002141 },
Kim Phillips9c4a7962008-06-23 19:50:15 +08002142 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2143 DESC_HDR_SEL0_AESU |
2144 DESC_HDR_MODE0_AESU_CBC |
2145 DESC_HDR_SEL1_MDEUA |
2146 DESC_HDR_MODE1_MDEU_INIT |
2147 DESC_HDR_MODE1_MDEU_PAD |
2148 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
Lee Nipper70bcaca2008-07-03 19:08:46 +08002149 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002150 { .type = CRYPTO_ALG_TYPE_AEAD,
LEROY Christophe7405c8d2016-06-06 13:20:46 +02002151 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2152 .alg.aead = {
2153 .base = {
2154 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2155 .cra_driver_name = "authenc-hmac-sha1-"
2156 "cbc-aes-talitos",
2157 .cra_blocksize = AES_BLOCK_SIZE,
2158 .cra_flags = CRYPTO_ALG_ASYNC,
2159 },
2160 .ivsize = AES_BLOCK_SIZE,
2161 .maxauthsize = SHA1_DIGEST_SIZE,
2162 },
2163 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2164 DESC_HDR_SEL0_AESU |
2165 DESC_HDR_MODE0_AESU_CBC |
2166 DESC_HDR_SEL1_MDEUA |
2167 DESC_HDR_MODE1_MDEU_INIT |
2168 DESC_HDR_MODE1_MDEU_PAD |
2169 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2170 },
2171 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002172 .alg.aead = {
2173 .base = {
2174 .cra_name = "authenc(hmac(sha1),"
2175 "cbc(des3_ede))",
2176 .cra_driver_name = "authenc-hmac-sha1-"
2177 "cbc-3des-talitos",
2178 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2179 .cra_flags = CRYPTO_ALG_ASYNC,
2180 },
2181 .ivsize = DES3_EDE_BLOCK_SIZE,
2182 .maxauthsize = SHA1_DIGEST_SIZE,
Lee Nipper56af8cd2009-03-29 15:50:50 +08002183 },
Lee Nipper70bcaca2008-07-03 19:08:46 +08002184 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2185 DESC_HDR_SEL0_DEU |
2186 DESC_HDR_MODE0_DEU_CBC |
2187 DESC_HDR_MODE0_DEU_3DES |
2188 DESC_HDR_SEL1_MDEUA |
2189 DESC_HDR_MODE1_MDEU_INIT |
2190 DESC_HDR_MODE1_MDEU_PAD |
2191 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
Lee Nipper3952f172008-07-10 18:29:18 +08002192 },
LEROY Christophe7405c8d2016-06-06 13:20:46 +02002193 { .type = CRYPTO_ALG_TYPE_AEAD,
2194 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2195 .alg.aead = {
2196 .base = {
2197 .cra_name = "authenc(hmac(sha1),"
2198 "cbc(des3_ede))",
2199 .cra_driver_name = "authenc-hmac-sha1-"
2200 "cbc-3des-talitos",
2201 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2202 .cra_flags = CRYPTO_ALG_ASYNC,
2203 },
2204 .ivsize = DES3_EDE_BLOCK_SIZE,
2205 .maxauthsize = SHA1_DIGEST_SIZE,
2206 },
2207 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2208 DESC_HDR_SEL0_DEU |
2209 DESC_HDR_MODE0_DEU_CBC |
2210 DESC_HDR_MODE0_DEU_3DES |
2211 DESC_HDR_SEL1_MDEUA |
2212 DESC_HDR_MODE1_MDEU_INIT |
2213 DESC_HDR_MODE1_MDEU_PAD |
2214 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2215 },
Horia Geanta357fb602012-07-03 19:16:53 +03002216 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002217 .alg.aead = {
2218 .base = {
2219 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2220 .cra_driver_name = "authenc-hmac-sha224-"
2221 "cbc-aes-talitos",
2222 .cra_blocksize = AES_BLOCK_SIZE,
2223 .cra_flags = CRYPTO_ALG_ASYNC,
2224 },
2225 .ivsize = AES_BLOCK_SIZE,
2226 .maxauthsize = SHA224_DIGEST_SIZE,
Horia Geanta357fb602012-07-03 19:16:53 +03002227 },
2228 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2229 DESC_HDR_SEL0_AESU |
2230 DESC_HDR_MODE0_AESU_CBC |
2231 DESC_HDR_SEL1_MDEUA |
2232 DESC_HDR_MODE1_MDEU_INIT |
2233 DESC_HDR_MODE1_MDEU_PAD |
2234 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2235 },
LEROY Christophe7405c8d2016-06-06 13:20:46 +02002236 { .type = CRYPTO_ALG_TYPE_AEAD,
2237 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2238 .alg.aead = {
2239 .base = {
2240 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2241 .cra_driver_name = "authenc-hmac-sha224-"
2242 "cbc-aes-talitos",
2243 .cra_blocksize = AES_BLOCK_SIZE,
2244 .cra_flags = CRYPTO_ALG_ASYNC,
2245 },
2246 .ivsize = AES_BLOCK_SIZE,
2247 .maxauthsize = SHA224_DIGEST_SIZE,
2248 },
2249 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2250 DESC_HDR_SEL0_AESU |
2251 DESC_HDR_MODE0_AESU_CBC |
2252 DESC_HDR_SEL1_MDEUA |
2253 DESC_HDR_MODE1_MDEU_INIT |
2254 DESC_HDR_MODE1_MDEU_PAD |
2255 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2256 },
Horia Geanta357fb602012-07-03 19:16:53 +03002257 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002258 .alg.aead = {
2259 .base = {
2260 .cra_name = "authenc(hmac(sha224),"
2261 "cbc(des3_ede))",
2262 .cra_driver_name = "authenc-hmac-sha224-"
2263 "cbc-3des-talitos",
2264 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2265 .cra_flags = CRYPTO_ALG_ASYNC,
2266 },
2267 .ivsize = DES3_EDE_BLOCK_SIZE,
2268 .maxauthsize = SHA224_DIGEST_SIZE,
Horia Geanta357fb602012-07-03 19:16:53 +03002269 },
2270 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2271 DESC_HDR_SEL0_DEU |
2272 DESC_HDR_MODE0_DEU_CBC |
2273 DESC_HDR_MODE0_DEU_3DES |
2274 DESC_HDR_SEL1_MDEUA |
2275 DESC_HDR_MODE1_MDEU_INIT |
2276 DESC_HDR_MODE1_MDEU_PAD |
2277 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2278 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002279 { .type = CRYPTO_ALG_TYPE_AEAD,
LEROY Christophe7405c8d2016-06-06 13:20:46 +02002280 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2281 .alg.aead = {
2282 .base = {
2283 .cra_name = "authenc(hmac(sha224),"
2284 "cbc(des3_ede))",
2285 .cra_driver_name = "authenc-hmac-sha224-"
2286 "cbc-3des-talitos",
2287 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2288 .cra_flags = CRYPTO_ALG_ASYNC,
2289 },
2290 .ivsize = DES3_EDE_BLOCK_SIZE,
2291 .maxauthsize = SHA224_DIGEST_SIZE,
2292 },
2293 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2294 DESC_HDR_SEL0_DEU |
2295 DESC_HDR_MODE0_DEU_CBC |
2296 DESC_HDR_MODE0_DEU_3DES |
2297 DESC_HDR_SEL1_MDEUA |
2298 DESC_HDR_MODE1_MDEU_INIT |
2299 DESC_HDR_MODE1_MDEU_PAD |
2300 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2301 },
2302 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002303 .alg.aead = {
2304 .base = {
2305 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2306 .cra_driver_name = "authenc-hmac-sha256-"
2307 "cbc-aes-talitos",
2308 .cra_blocksize = AES_BLOCK_SIZE,
2309 .cra_flags = CRYPTO_ALG_ASYNC,
2310 },
2311 .ivsize = AES_BLOCK_SIZE,
2312 .maxauthsize = SHA256_DIGEST_SIZE,
Lee Nipper56af8cd2009-03-29 15:50:50 +08002313 },
Lee Nipper3952f172008-07-10 18:29:18 +08002314 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2315 DESC_HDR_SEL0_AESU |
2316 DESC_HDR_MODE0_AESU_CBC |
2317 DESC_HDR_SEL1_MDEUA |
2318 DESC_HDR_MODE1_MDEU_INIT |
2319 DESC_HDR_MODE1_MDEU_PAD |
2320 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2321 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002322 { .type = CRYPTO_ALG_TYPE_AEAD,
LEROY Christophe7405c8d2016-06-06 13:20:46 +02002323 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2324 .alg.aead = {
2325 .base = {
2326 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2327 .cra_driver_name = "authenc-hmac-sha256-"
2328 "cbc-aes-talitos",
2329 .cra_blocksize = AES_BLOCK_SIZE,
2330 .cra_flags = CRYPTO_ALG_ASYNC,
2331 },
2332 .ivsize = AES_BLOCK_SIZE,
2333 .maxauthsize = SHA256_DIGEST_SIZE,
2334 },
2335 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2336 DESC_HDR_SEL0_AESU |
2337 DESC_HDR_MODE0_AESU_CBC |
2338 DESC_HDR_SEL1_MDEUA |
2339 DESC_HDR_MODE1_MDEU_INIT |
2340 DESC_HDR_MODE1_MDEU_PAD |
2341 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2342 },
2343 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002344 .alg.aead = {
2345 .base = {
2346 .cra_name = "authenc(hmac(sha256),"
2347 "cbc(des3_ede))",
2348 .cra_driver_name = "authenc-hmac-sha256-"
2349 "cbc-3des-talitos",
2350 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2351 .cra_flags = CRYPTO_ALG_ASYNC,
2352 },
2353 .ivsize = DES3_EDE_BLOCK_SIZE,
2354 .maxauthsize = SHA256_DIGEST_SIZE,
Lee Nipper56af8cd2009-03-29 15:50:50 +08002355 },
Lee Nipper3952f172008-07-10 18:29:18 +08002356 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2357 DESC_HDR_SEL0_DEU |
2358 DESC_HDR_MODE0_DEU_CBC |
2359 DESC_HDR_MODE0_DEU_3DES |
2360 DESC_HDR_SEL1_MDEUA |
2361 DESC_HDR_MODE1_MDEU_INIT |
2362 DESC_HDR_MODE1_MDEU_PAD |
2363 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2364 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002365 { .type = CRYPTO_ALG_TYPE_AEAD,
LEROY Christophe7405c8d2016-06-06 13:20:46 +02002366 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2367 .alg.aead = {
2368 .base = {
2369 .cra_name = "authenc(hmac(sha256),"
2370 "cbc(des3_ede))",
2371 .cra_driver_name = "authenc-hmac-sha256-"
2372 "cbc-3des-talitos",
2373 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2374 .cra_flags = CRYPTO_ALG_ASYNC,
2375 },
2376 .ivsize = DES3_EDE_BLOCK_SIZE,
2377 .maxauthsize = SHA256_DIGEST_SIZE,
2378 },
2379 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2380 DESC_HDR_SEL0_DEU |
2381 DESC_HDR_MODE0_DEU_CBC |
2382 DESC_HDR_MODE0_DEU_3DES |
2383 DESC_HDR_SEL1_MDEUA |
2384 DESC_HDR_MODE1_MDEU_INIT |
2385 DESC_HDR_MODE1_MDEU_PAD |
2386 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2387 },
2388 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002389 .alg.aead = {
2390 .base = {
2391 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2392 .cra_driver_name = "authenc-hmac-sha384-"
2393 "cbc-aes-talitos",
2394 .cra_blocksize = AES_BLOCK_SIZE,
2395 .cra_flags = CRYPTO_ALG_ASYNC,
2396 },
2397 .ivsize = AES_BLOCK_SIZE,
2398 .maxauthsize = SHA384_DIGEST_SIZE,
Horia Geanta357fb602012-07-03 19:16:53 +03002399 },
2400 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2401 DESC_HDR_SEL0_AESU |
2402 DESC_HDR_MODE0_AESU_CBC |
2403 DESC_HDR_SEL1_MDEUB |
2404 DESC_HDR_MODE1_MDEU_INIT |
2405 DESC_HDR_MODE1_MDEU_PAD |
2406 DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2407 },
2408 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002409 .alg.aead = {
2410 .base = {
2411 .cra_name = "authenc(hmac(sha384),"
2412 "cbc(des3_ede))",
2413 .cra_driver_name = "authenc-hmac-sha384-"
2414 "cbc-3des-talitos",
2415 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2416 .cra_flags = CRYPTO_ALG_ASYNC,
2417 },
2418 .ivsize = DES3_EDE_BLOCK_SIZE,
2419 .maxauthsize = SHA384_DIGEST_SIZE,
Horia Geanta357fb602012-07-03 19:16:53 +03002420 },
2421 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2422 DESC_HDR_SEL0_DEU |
2423 DESC_HDR_MODE0_DEU_CBC |
2424 DESC_HDR_MODE0_DEU_3DES |
2425 DESC_HDR_SEL1_MDEUB |
2426 DESC_HDR_MODE1_MDEU_INIT |
2427 DESC_HDR_MODE1_MDEU_PAD |
2428 DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2429 },
2430 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002431 .alg.aead = {
2432 .base = {
2433 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2434 .cra_driver_name = "authenc-hmac-sha512-"
2435 "cbc-aes-talitos",
2436 .cra_blocksize = AES_BLOCK_SIZE,
2437 .cra_flags = CRYPTO_ALG_ASYNC,
2438 },
2439 .ivsize = AES_BLOCK_SIZE,
2440 .maxauthsize = SHA512_DIGEST_SIZE,
Horia Geanta357fb602012-07-03 19:16:53 +03002441 },
2442 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2443 DESC_HDR_SEL0_AESU |
2444 DESC_HDR_MODE0_AESU_CBC |
2445 DESC_HDR_SEL1_MDEUB |
2446 DESC_HDR_MODE1_MDEU_INIT |
2447 DESC_HDR_MODE1_MDEU_PAD |
2448 DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2449 },
2450 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002451 .alg.aead = {
2452 .base = {
2453 .cra_name = "authenc(hmac(sha512),"
2454 "cbc(des3_ede))",
2455 .cra_driver_name = "authenc-hmac-sha512-"
2456 "cbc-3des-talitos",
2457 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2458 .cra_flags = CRYPTO_ALG_ASYNC,
2459 },
2460 .ivsize = DES3_EDE_BLOCK_SIZE,
2461 .maxauthsize = SHA512_DIGEST_SIZE,
Horia Geanta357fb602012-07-03 19:16:53 +03002462 },
2463 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2464 DESC_HDR_SEL0_DEU |
2465 DESC_HDR_MODE0_DEU_CBC |
2466 DESC_HDR_MODE0_DEU_3DES |
2467 DESC_HDR_SEL1_MDEUB |
2468 DESC_HDR_MODE1_MDEU_INIT |
2469 DESC_HDR_MODE1_MDEU_PAD |
2470 DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2471 },
2472 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002473 .alg.aead = {
2474 .base = {
2475 .cra_name = "authenc(hmac(md5),cbc(aes))",
2476 .cra_driver_name = "authenc-hmac-md5-"
2477 "cbc-aes-talitos",
2478 .cra_blocksize = AES_BLOCK_SIZE,
2479 .cra_flags = CRYPTO_ALG_ASYNC,
2480 },
2481 .ivsize = AES_BLOCK_SIZE,
2482 .maxauthsize = MD5_DIGEST_SIZE,
Lee Nipper56af8cd2009-03-29 15:50:50 +08002483 },
Lee Nipper3952f172008-07-10 18:29:18 +08002484 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2485 DESC_HDR_SEL0_AESU |
2486 DESC_HDR_MODE0_AESU_CBC |
2487 DESC_HDR_SEL1_MDEUA |
2488 DESC_HDR_MODE1_MDEU_INIT |
2489 DESC_HDR_MODE1_MDEU_PAD |
2490 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2491 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002492 { .type = CRYPTO_ALG_TYPE_AEAD,
LEROY Christophe7405c8d2016-06-06 13:20:46 +02002493 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2494 .alg.aead = {
2495 .base = {
2496 .cra_name = "authenc(hmac(md5),cbc(aes))",
2497 .cra_driver_name = "authenc-hmac-md5-"
2498 "cbc-aes-talitos",
2499 .cra_blocksize = AES_BLOCK_SIZE,
2500 .cra_flags = CRYPTO_ALG_ASYNC,
2501 },
2502 .ivsize = AES_BLOCK_SIZE,
2503 .maxauthsize = MD5_DIGEST_SIZE,
2504 },
2505 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2506 DESC_HDR_SEL0_AESU |
2507 DESC_HDR_MODE0_AESU_CBC |
2508 DESC_HDR_SEL1_MDEUA |
2509 DESC_HDR_MODE1_MDEU_INIT |
2510 DESC_HDR_MODE1_MDEU_PAD |
2511 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2512 },
2513 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002514 .alg.aead = {
2515 .base = {
2516 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2517 .cra_driver_name = "authenc-hmac-md5-"
2518 "cbc-3des-talitos",
2519 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2520 .cra_flags = CRYPTO_ALG_ASYNC,
2521 },
2522 .ivsize = DES3_EDE_BLOCK_SIZE,
2523 .maxauthsize = MD5_DIGEST_SIZE,
Lee Nipper56af8cd2009-03-29 15:50:50 +08002524 },
Lee Nipper3952f172008-07-10 18:29:18 +08002525 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2526 DESC_HDR_SEL0_DEU |
2527 DESC_HDR_MODE0_DEU_CBC |
2528 DESC_HDR_MODE0_DEU_3DES |
2529 DESC_HDR_SEL1_MDEUA |
2530 DESC_HDR_MODE1_MDEU_INIT |
2531 DESC_HDR_MODE1_MDEU_PAD |
2532 DESC_HDR_MODE1_MDEU_MD5_HMAC,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002533 },
LEROY Christophe7405c8d2016-06-06 13:20:46 +02002534 { .type = CRYPTO_ALG_TYPE_AEAD,
2535 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2536 .alg.aead = {
2537 .base = {
2538 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2539 .cra_driver_name = "authenc-hmac-md5-"
2540 "cbc-3des-talitos",
2541 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2542 .cra_flags = CRYPTO_ALG_ASYNC,
2543 },
2544 .ivsize = DES3_EDE_BLOCK_SIZE,
2545 .maxauthsize = MD5_DIGEST_SIZE,
2546 },
2547 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2548 DESC_HDR_SEL0_DEU |
2549 DESC_HDR_MODE0_DEU_CBC |
2550 DESC_HDR_MODE0_DEU_3DES |
2551 DESC_HDR_SEL1_MDEUA |
2552 DESC_HDR_MODE1_MDEU_INIT |
2553 DESC_HDR_MODE1_MDEU_PAD |
2554 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2555 },
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002556 /* ABLKCIPHER algorithms. */
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002557 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2558 .alg.crypto = {
LEROY Christophe5e75ae12015-12-01 12:44:15 +01002559 .cra_name = "ecb(aes)",
2560 .cra_driver_name = "ecb-aes-talitos",
2561 .cra_blocksize = AES_BLOCK_SIZE,
2562 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2563 CRYPTO_ALG_ASYNC,
2564 .cra_ablkcipher = {
2565 .min_keysize = AES_MIN_KEY_SIZE,
2566 .max_keysize = AES_MAX_KEY_SIZE,
2567 .ivsize = AES_BLOCK_SIZE,
2568 }
2569 },
2570 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2571 DESC_HDR_SEL0_AESU,
2572 },
2573 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2574 .alg.crypto = {
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002575 .cra_name = "cbc(aes)",
2576 .cra_driver_name = "cbc-aes-talitos",
2577 .cra_blocksize = AES_BLOCK_SIZE,
2578 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2579 CRYPTO_ALG_ASYNC,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002580 .cra_ablkcipher = {
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002581 .min_keysize = AES_MIN_KEY_SIZE,
2582 .max_keysize = AES_MAX_KEY_SIZE,
2583 .ivsize = AES_BLOCK_SIZE,
2584 }
2585 },
2586 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2587 DESC_HDR_SEL0_AESU |
2588 DESC_HDR_MODE0_AESU_CBC,
2589 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002590 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2591 .alg.crypto = {
LEROY Christophe5e75ae12015-12-01 12:44:15 +01002592 .cra_name = "ctr(aes)",
2593 .cra_driver_name = "ctr-aes-talitos",
2594 .cra_blocksize = AES_BLOCK_SIZE,
2595 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2596 CRYPTO_ALG_ASYNC,
2597 .cra_ablkcipher = {
2598 .min_keysize = AES_MIN_KEY_SIZE,
2599 .max_keysize = AES_MAX_KEY_SIZE,
2600 .ivsize = AES_BLOCK_SIZE,
2601 }
2602 },
LEROY Christophe70d355c2017-10-06 15:04:43 +02002603 .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP |
LEROY Christophe5e75ae12015-12-01 12:44:15 +01002604 DESC_HDR_SEL0_AESU |
2605 DESC_HDR_MODE0_AESU_CTR,
2606 },
2607 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2608 .alg.crypto = {
2609 .cra_name = "ecb(des)",
2610 .cra_driver_name = "ecb-des-talitos",
2611 .cra_blocksize = DES_BLOCK_SIZE,
2612 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2613 CRYPTO_ALG_ASYNC,
2614 .cra_ablkcipher = {
2615 .min_keysize = DES_KEY_SIZE,
2616 .max_keysize = DES_KEY_SIZE,
2617 .ivsize = DES_BLOCK_SIZE,
2618 }
2619 },
2620 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2621 DESC_HDR_SEL0_DEU,
2622 },
2623 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2624 .alg.crypto = {
2625 .cra_name = "cbc(des)",
2626 .cra_driver_name = "cbc-des-talitos",
2627 .cra_blocksize = DES_BLOCK_SIZE,
2628 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2629 CRYPTO_ALG_ASYNC,
2630 .cra_ablkcipher = {
2631 .min_keysize = DES_KEY_SIZE,
2632 .max_keysize = DES_KEY_SIZE,
2633 .ivsize = DES_BLOCK_SIZE,
2634 }
2635 },
2636 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2637 DESC_HDR_SEL0_DEU |
2638 DESC_HDR_MODE0_DEU_CBC,
2639 },
2640 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2641 .alg.crypto = {
2642 .cra_name = "ecb(des3_ede)",
2643 .cra_driver_name = "ecb-3des-talitos",
2644 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2645 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2646 CRYPTO_ALG_ASYNC,
2647 .cra_ablkcipher = {
2648 .min_keysize = DES3_EDE_KEY_SIZE,
2649 .max_keysize = DES3_EDE_KEY_SIZE,
2650 .ivsize = DES3_EDE_BLOCK_SIZE,
2651 }
2652 },
2653 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2654 DESC_HDR_SEL0_DEU |
2655 DESC_HDR_MODE0_DEU_3DES,
2656 },
2657 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2658 .alg.crypto = {
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002659 .cra_name = "cbc(des3_ede)",
2660 .cra_driver_name = "cbc-3des-talitos",
2661 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2662 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2663 CRYPTO_ALG_ASYNC,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002664 .cra_ablkcipher = {
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002665 .min_keysize = DES3_EDE_KEY_SIZE,
2666 .max_keysize = DES3_EDE_KEY_SIZE,
2667 .ivsize = DES3_EDE_BLOCK_SIZE,
2668 }
2669 },
2670 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2671 DESC_HDR_SEL0_DEU |
2672 DESC_HDR_MODE0_DEU_CBC |
2673 DESC_HDR_MODE0_DEU_3DES,
Lee Nipper497f2e62010-05-19 19:20:36 +10002674 },
2675 /* AHASH algorithms. */
2676 { .type = CRYPTO_ALG_TYPE_AHASH,
2677 .alg.hash = {
Lee Nipper497f2e62010-05-19 19:20:36 +10002678 .halg.digestsize = MD5_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002679 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper497f2e62010-05-19 19:20:36 +10002680 .halg.base = {
2681 .cra_name = "md5",
2682 .cra_driver_name = "md5-talitos",
Martin Hicksb3988612015-03-03 08:21:34 -05002683 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
Lee Nipper497f2e62010-05-19 19:20:36 +10002684 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2685 CRYPTO_ALG_ASYNC,
Lee Nipper497f2e62010-05-19 19:20:36 +10002686 }
2687 },
2688 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2689 DESC_HDR_SEL0_MDEUA |
2690 DESC_HDR_MODE0_MDEU_MD5,
2691 },
2692 { .type = CRYPTO_ALG_TYPE_AHASH,
2693 .alg.hash = {
Lee Nipper497f2e62010-05-19 19:20:36 +10002694 .halg.digestsize = SHA1_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002695 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper497f2e62010-05-19 19:20:36 +10002696 .halg.base = {
2697 .cra_name = "sha1",
2698 .cra_driver_name = "sha1-talitos",
2699 .cra_blocksize = SHA1_BLOCK_SIZE,
2700 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2701 CRYPTO_ALG_ASYNC,
Lee Nipper497f2e62010-05-19 19:20:36 +10002702 }
2703 },
2704 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2705 DESC_HDR_SEL0_MDEUA |
2706 DESC_HDR_MODE0_MDEU_SHA1,
2707 },
2708 { .type = CRYPTO_ALG_TYPE_AHASH,
2709 .alg.hash = {
Kim Phillips60f208d2010-05-19 19:21:53 +10002710 .halg.digestsize = SHA224_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002711 .halg.statesize = sizeof(struct talitos_export_state),
Kim Phillips60f208d2010-05-19 19:21:53 +10002712 .halg.base = {
2713 .cra_name = "sha224",
2714 .cra_driver_name = "sha224-talitos",
2715 .cra_blocksize = SHA224_BLOCK_SIZE,
2716 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2717 CRYPTO_ALG_ASYNC,
Kim Phillips60f208d2010-05-19 19:21:53 +10002718 }
2719 },
2720 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2721 DESC_HDR_SEL0_MDEUA |
2722 DESC_HDR_MODE0_MDEU_SHA224,
2723 },
2724 { .type = CRYPTO_ALG_TYPE_AHASH,
2725 .alg.hash = {
Lee Nipper497f2e62010-05-19 19:20:36 +10002726 .halg.digestsize = SHA256_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002727 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper497f2e62010-05-19 19:20:36 +10002728 .halg.base = {
2729 .cra_name = "sha256",
2730 .cra_driver_name = "sha256-talitos",
2731 .cra_blocksize = SHA256_BLOCK_SIZE,
2732 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2733 CRYPTO_ALG_ASYNC,
Lee Nipper497f2e62010-05-19 19:20:36 +10002734 }
2735 },
2736 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2737 DESC_HDR_SEL0_MDEUA |
2738 DESC_HDR_MODE0_MDEU_SHA256,
2739 },
2740 { .type = CRYPTO_ALG_TYPE_AHASH,
2741 .alg.hash = {
Lee Nipper497f2e62010-05-19 19:20:36 +10002742 .halg.digestsize = SHA384_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002743 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper497f2e62010-05-19 19:20:36 +10002744 .halg.base = {
2745 .cra_name = "sha384",
2746 .cra_driver_name = "sha384-talitos",
2747 .cra_blocksize = SHA384_BLOCK_SIZE,
2748 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2749 CRYPTO_ALG_ASYNC,
Lee Nipper497f2e62010-05-19 19:20:36 +10002750 }
2751 },
2752 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2753 DESC_HDR_SEL0_MDEUB |
2754 DESC_HDR_MODE0_MDEUB_SHA384,
2755 },
2756 { .type = CRYPTO_ALG_TYPE_AHASH,
2757 .alg.hash = {
Lee Nipper497f2e62010-05-19 19:20:36 +10002758 .halg.digestsize = SHA512_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002759 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper497f2e62010-05-19 19:20:36 +10002760 .halg.base = {
2761 .cra_name = "sha512",
2762 .cra_driver_name = "sha512-talitos",
2763 .cra_blocksize = SHA512_BLOCK_SIZE,
2764 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2765 CRYPTO_ALG_ASYNC,
Lee Nipper497f2e62010-05-19 19:20:36 +10002766 }
2767 },
2768 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2769 DESC_HDR_SEL0_MDEUB |
2770 DESC_HDR_MODE0_MDEUB_SHA512,
2771 },
Lee Nipper79b3a412011-11-21 16:13:25 +08002772 { .type = CRYPTO_ALG_TYPE_AHASH,
2773 .alg.hash = {
Lee Nipper79b3a412011-11-21 16:13:25 +08002774 .halg.digestsize = MD5_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002775 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper79b3a412011-11-21 16:13:25 +08002776 .halg.base = {
2777 .cra_name = "hmac(md5)",
2778 .cra_driver_name = "hmac-md5-talitos",
Martin Hicksb3988612015-03-03 08:21:34 -05002779 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
Lee Nipper79b3a412011-11-21 16:13:25 +08002780 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2781 CRYPTO_ALG_ASYNC,
Lee Nipper79b3a412011-11-21 16:13:25 +08002782 }
2783 },
2784 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2785 DESC_HDR_SEL0_MDEUA |
2786 DESC_HDR_MODE0_MDEU_MD5,
2787 },
2788 { .type = CRYPTO_ALG_TYPE_AHASH,
2789 .alg.hash = {
Lee Nipper79b3a412011-11-21 16:13:25 +08002790 .halg.digestsize = SHA1_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002791 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper79b3a412011-11-21 16:13:25 +08002792 .halg.base = {
2793 .cra_name = "hmac(sha1)",
2794 .cra_driver_name = "hmac-sha1-talitos",
2795 .cra_blocksize = SHA1_BLOCK_SIZE,
2796 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2797 CRYPTO_ALG_ASYNC,
Lee Nipper79b3a412011-11-21 16:13:25 +08002798 }
2799 },
2800 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2801 DESC_HDR_SEL0_MDEUA |
2802 DESC_HDR_MODE0_MDEU_SHA1,
2803 },
2804 { .type = CRYPTO_ALG_TYPE_AHASH,
2805 .alg.hash = {
Lee Nipper79b3a412011-11-21 16:13:25 +08002806 .halg.digestsize = SHA224_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002807 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper79b3a412011-11-21 16:13:25 +08002808 .halg.base = {
2809 .cra_name = "hmac(sha224)",
2810 .cra_driver_name = "hmac-sha224-talitos",
2811 .cra_blocksize = SHA224_BLOCK_SIZE,
2812 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2813 CRYPTO_ALG_ASYNC,
Lee Nipper79b3a412011-11-21 16:13:25 +08002814 }
2815 },
2816 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2817 DESC_HDR_SEL0_MDEUA |
2818 DESC_HDR_MODE0_MDEU_SHA224,
2819 },
2820 { .type = CRYPTO_ALG_TYPE_AHASH,
2821 .alg.hash = {
Lee Nipper79b3a412011-11-21 16:13:25 +08002822 .halg.digestsize = SHA256_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002823 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper79b3a412011-11-21 16:13:25 +08002824 .halg.base = {
2825 .cra_name = "hmac(sha256)",
2826 .cra_driver_name = "hmac-sha256-talitos",
2827 .cra_blocksize = SHA256_BLOCK_SIZE,
2828 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2829 CRYPTO_ALG_ASYNC,
Lee Nipper79b3a412011-11-21 16:13:25 +08002830 }
2831 },
2832 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2833 DESC_HDR_SEL0_MDEUA |
2834 DESC_HDR_MODE0_MDEU_SHA256,
2835 },
2836 { .type = CRYPTO_ALG_TYPE_AHASH,
2837 .alg.hash = {
Lee Nipper79b3a412011-11-21 16:13:25 +08002838 .halg.digestsize = SHA384_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002839 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper79b3a412011-11-21 16:13:25 +08002840 .halg.base = {
2841 .cra_name = "hmac(sha384)",
2842 .cra_driver_name = "hmac-sha384-talitos",
2843 .cra_blocksize = SHA384_BLOCK_SIZE,
2844 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2845 CRYPTO_ALG_ASYNC,
Lee Nipper79b3a412011-11-21 16:13:25 +08002846 }
2847 },
2848 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2849 DESC_HDR_SEL0_MDEUB |
2850 DESC_HDR_MODE0_MDEUB_SHA384,
2851 },
2852 { .type = CRYPTO_ALG_TYPE_AHASH,
2853 .alg.hash = {
Lee Nipper79b3a412011-11-21 16:13:25 +08002854 .halg.digestsize = SHA512_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002855 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper79b3a412011-11-21 16:13:25 +08002856 .halg.base = {
2857 .cra_name = "hmac(sha512)",
2858 .cra_driver_name = "hmac-sha512-talitos",
2859 .cra_blocksize = SHA512_BLOCK_SIZE,
2860 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2861 CRYPTO_ALG_ASYNC,
Lee Nipper79b3a412011-11-21 16:13:25 +08002862 }
2863 },
2864 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2865 DESC_HDR_SEL0_MDEUB |
2866 DESC_HDR_MODE0_MDEUB_SHA512,
2867 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08002868};
2869
2870struct talitos_crypto_alg {
2871 struct list_head entry;
2872 struct device *dev;
Lee Nipperacbf7c622010-05-19 19:19:33 +10002873 struct talitos_alg_template algt;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002874};
2875
Jonas Eymann89d124c2016-04-19 20:33:47 +03002876static int talitos_init_common(struct talitos_ctx *ctx,
2877 struct talitos_crypto_alg *talitos_alg)
Kim Phillips9c4a7962008-06-23 19:50:15 +08002878{
Kim Phillips5228f0f2011-07-15 11:21:38 +08002879 struct talitos_private *priv;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002880
2881 /* update context with ptr to dev */
2882 ctx->dev = talitos_alg->dev;
Kim Phillips19bbbc62009-03-29 15:53:59 +08002883
Kim Phillips5228f0f2011-07-15 11:21:38 +08002884 /* assign SEC channel to tfm in round-robin fashion */
2885 priv = dev_get_drvdata(ctx->dev);
2886 ctx->ch = atomic_inc_return(&priv->last_chan) &
2887 (priv->num_channels - 1);
2888
Kim Phillips9c4a7962008-06-23 19:50:15 +08002889 /* copy descriptor header template value */
Lee Nipperacbf7c622010-05-19 19:19:33 +10002890 ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002891
Kim Phillips602dba52011-07-15 11:21:39 +08002892 /* select done notification */
2893 ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
2894
Lee Nipper497f2e62010-05-19 19:20:36 +10002895 return 0;
2896}
2897
Jonas Eymann89d124c2016-04-19 20:33:47 +03002898static int talitos_cra_init(struct crypto_tfm *tfm)
2899{
2900 struct crypto_alg *alg = tfm->__crt_alg;
2901 struct talitos_crypto_alg *talitos_alg;
2902 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2903
2904 if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
2905 talitos_alg = container_of(__crypto_ahash_alg(alg),
2906 struct talitos_crypto_alg,
2907 algt.alg.hash);
2908 else
2909 talitos_alg = container_of(alg, struct talitos_crypto_alg,
2910 algt.alg.crypto);
2911
2912 return talitos_init_common(ctx, talitos_alg);
2913}
2914
Herbert Xuaeb4c132015-07-30 17:53:22 +08002915static int talitos_cra_init_aead(struct crypto_aead *tfm)
Lee Nipper497f2e62010-05-19 19:20:36 +10002916{
Jonas Eymann89d124c2016-04-19 20:33:47 +03002917 struct aead_alg *alg = crypto_aead_alg(tfm);
2918 struct talitos_crypto_alg *talitos_alg;
2919 struct talitos_ctx *ctx = crypto_aead_ctx(tfm);
2920
2921 talitos_alg = container_of(alg, struct talitos_crypto_alg,
2922 algt.alg.aead);
2923
2924 return talitos_init_common(ctx, talitos_alg);
Kim Phillips9c4a7962008-06-23 19:50:15 +08002925}
2926
Lee Nipper497f2e62010-05-19 19:20:36 +10002927static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
2928{
2929 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
2930
2931 talitos_cra_init(tfm);
2932
2933 ctx->keylen = 0;
2934 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2935 sizeof(struct talitos_ahash_req_ctx));
2936
2937 return 0;
2938}
2939
Kim Phillips9c4a7962008-06-23 19:50:15 +08002940/*
2941 * given the alg's descriptor header template, determine whether descriptor
2942 * type and primary/secondary execution units required match the hw
2943 * capabilities description provided in the device tree node.
2944 */
2945static int hw_supports(struct device *dev, __be32 desc_hdr_template)
2946{
2947 struct talitos_private *priv = dev_get_drvdata(dev);
2948 int ret;
2949
2950 ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
2951 (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
2952
2953 if (SECONDARY_EU(desc_hdr_template))
2954 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
2955 & priv->exec_units);
2956
2957 return ret;
2958}
2959
Grant Likely2dc11582010-08-06 09:25:50 -06002960static int talitos_remove(struct platform_device *ofdev)
Kim Phillips9c4a7962008-06-23 19:50:15 +08002961{
2962 struct device *dev = &ofdev->dev;
2963 struct talitos_private *priv = dev_get_drvdata(dev);
2964 struct talitos_crypto_alg *t_alg, *n;
2965 int i;
2966
2967 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
Lee Nipperacbf7c622010-05-19 19:19:33 +10002968 switch (t_alg->algt.type) {
2969 case CRYPTO_ALG_TYPE_ABLKCIPHER:
Lee Nipperacbf7c622010-05-19 19:19:33 +10002970 break;
Herbert Xuaeb4c132015-07-30 17:53:22 +08002971 case CRYPTO_ALG_TYPE_AEAD:
2972 crypto_unregister_aead(&t_alg->algt.alg.aead);
Lee Nipperacbf7c622010-05-19 19:19:33 +10002973 case CRYPTO_ALG_TYPE_AHASH:
2974 crypto_unregister_ahash(&t_alg->algt.alg.hash);
2975 break;
2976 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08002977 list_del(&t_alg->entry);
Kim Phillips9c4a7962008-06-23 19:50:15 +08002978 }
2979
2980 if (hw_supports(dev, DESC_HDR_SEL0_RNG))
2981 talitos_unregister_rng(dev);
2982
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002983 for (i = 0; i < 2; i++)
Kim Phillips2cdba3c2011-12-12 14:59:11 -06002984 if (priv->irq[i]) {
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002985 free_irq(priv->irq[i], dev);
2986 irq_dispose_mapping(priv->irq[i]);
2987 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08002988
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002989 tasklet_kill(&priv->done_task[0]);
Kim Phillips2cdba3c2011-12-12 14:59:11 -06002990 if (priv->irq[1])
Kim Phillipsc3e337f2011-11-21 16:13:27 +08002991 tasklet_kill(&priv->done_task[1]);
Kim Phillips9c4a7962008-06-23 19:50:15 +08002992
Kim Phillips9c4a7962008-06-23 19:50:15 +08002993 return 0;
2994}
2995
2996static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
2997 struct talitos_alg_template
2998 *template)
2999{
Kim Phillips60f208d2010-05-19 19:21:53 +10003000 struct talitos_private *priv = dev_get_drvdata(dev);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003001 struct talitos_crypto_alg *t_alg;
3002 struct crypto_alg *alg;
3003
LEROY Christophe24b92ff2017-10-06 15:04:49 +02003004 t_alg = devm_kzalloc(dev, sizeof(struct talitos_crypto_alg),
3005 GFP_KERNEL);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003006 if (!t_alg)
3007 return ERR_PTR(-ENOMEM);
3008
Lee Nipperacbf7c622010-05-19 19:19:33 +10003009 t_alg->algt = *template;
3010
3011 switch (t_alg->algt.type) {
3012 case CRYPTO_ALG_TYPE_ABLKCIPHER:
Lee Nipper497f2e62010-05-19 19:20:36 +10003013 alg = &t_alg->algt.alg.crypto;
3014 alg->cra_init = talitos_cra_init;
Kim Phillipsd4cd3282012-08-08 20:32:00 -05003015 alg->cra_type = &crypto_ablkcipher_type;
Kim Phillipsb286e002012-08-08 20:33:34 -05003016 alg->cra_ablkcipher.setkey = ablkcipher_setkey;
3017 alg->cra_ablkcipher.encrypt = ablkcipher_encrypt;
3018 alg->cra_ablkcipher.decrypt = ablkcipher_decrypt;
3019 alg->cra_ablkcipher.geniv = "eseqiv";
Lee Nipper497f2e62010-05-19 19:20:36 +10003020 break;
Lee Nipperacbf7c622010-05-19 19:19:33 +10003021 case CRYPTO_ALG_TYPE_AEAD:
Herbert Xuaeb4c132015-07-30 17:53:22 +08003022 alg = &t_alg->algt.alg.aead.base;
Herbert Xuaeb4c132015-07-30 17:53:22 +08003023 t_alg->algt.alg.aead.init = talitos_cra_init_aead;
3024 t_alg->algt.alg.aead.setkey = aead_setkey;
3025 t_alg->algt.alg.aead.encrypt = aead_encrypt;
3026 t_alg->algt.alg.aead.decrypt = aead_decrypt;
LEROY Christophe6cda0752017-10-06 15:04:39 +02003027 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
3028 !strncmp(alg->cra_name, "authenc(hmac(sha224)", 20)) {
LEROY Christophe24b92ff2017-10-06 15:04:49 +02003029 devm_kfree(dev, t_alg);
LEROY Christophe6cda0752017-10-06 15:04:39 +02003030 return ERR_PTR(-ENOTSUPP);
3031 }
Lee Nipperacbf7c622010-05-19 19:19:33 +10003032 break;
3033 case CRYPTO_ALG_TYPE_AHASH:
3034 alg = &t_alg->algt.alg.hash.halg.base;
Lee Nipper497f2e62010-05-19 19:20:36 +10003035 alg->cra_init = talitos_cra_init_ahash;
Kim Phillipsd4cd3282012-08-08 20:32:00 -05003036 alg->cra_type = &crypto_ahash_type;
Kim Phillipsb286e002012-08-08 20:33:34 -05003037 t_alg->algt.alg.hash.init = ahash_init;
3038 t_alg->algt.alg.hash.update = ahash_update;
3039 t_alg->algt.alg.hash.final = ahash_final;
3040 t_alg->algt.alg.hash.finup = ahash_finup;
3041 t_alg->algt.alg.hash.digest = ahash_digest;
LEROY Christophe56136632017-09-12 11:03:39 +02003042 if (!strncmp(alg->cra_name, "hmac", 4))
3043 t_alg->algt.alg.hash.setkey = ahash_setkey;
Horia Geant?3639ca82016-04-21 19:24:55 +03003044 t_alg->algt.alg.hash.import = ahash_import;
3045 t_alg->algt.alg.hash.export = ahash_export;
Kim Phillipsb286e002012-08-08 20:33:34 -05003046
Lee Nipper79b3a412011-11-21 16:13:25 +08003047 if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
Kim Phillips0b2730d2011-12-12 14:59:10 -06003048 !strncmp(alg->cra_name, "hmac", 4)) {
LEROY Christophe24b92ff2017-10-06 15:04:49 +02003049 devm_kfree(dev, t_alg);
Lee Nipper79b3a412011-11-21 16:13:25 +08003050 return ERR_PTR(-ENOTSUPP);
Kim Phillips0b2730d2011-12-12 14:59:10 -06003051 }
Kim Phillips60f208d2010-05-19 19:21:53 +10003052 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
Lee Nipper79b3a412011-11-21 16:13:25 +08003053 (!strcmp(alg->cra_name, "sha224") ||
3054 !strcmp(alg->cra_name, "hmac(sha224)"))) {
Kim Phillips60f208d2010-05-19 19:21:53 +10003055 t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
3056 t_alg->algt.desc_hdr_template =
3057 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3058 DESC_HDR_SEL0_MDEUA |
3059 DESC_HDR_MODE0_MDEU_SHA256;
3060 }
Lee Nipper497f2e62010-05-19 19:20:36 +10003061 break;
Kim Phillips1d119112010-09-23 15:55:27 +08003062 default:
3063 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
LEROY Christophe24b92ff2017-10-06 15:04:49 +02003064 devm_kfree(dev, t_alg);
Kim Phillips1d119112010-09-23 15:55:27 +08003065 return ERR_PTR(-EINVAL);
Lee Nipperacbf7c622010-05-19 19:19:33 +10003066 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08003067
Kim Phillips9c4a7962008-06-23 19:50:15 +08003068 alg->cra_module = THIS_MODULE;
LEROY Christopheb0057762016-06-06 13:20:44 +02003069 if (t_alg->algt.priority)
3070 alg->cra_priority = t_alg->algt.priority;
3071 else
3072 alg->cra_priority = TALITOS_CRA_PRIORITY;
Kim Phillips9c4a7962008-06-23 19:50:15 +08003073 alg->cra_alignmask = 0;
Kim Phillips9c4a7962008-06-23 19:50:15 +08003074 alg->cra_ctxsize = sizeof(struct talitos_ctx);
Nikos Mavrogiannopoulosd912bb72011-11-01 13:39:56 +01003075 alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
Kim Phillips9c4a7962008-06-23 19:50:15 +08003076
Kim Phillips9c4a7962008-06-23 19:50:15 +08003077 t_alg->dev = dev;
3078
3079 return t_alg;
3080}
3081
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003082static int talitos_probe_irq(struct platform_device *ofdev)
3083{
3084 struct device *dev = &ofdev->dev;
3085 struct device_node *np = ofdev->dev.of_node;
3086 struct talitos_private *priv = dev_get_drvdata(dev);
3087 int err;
LEROY Christophedd3c0982015-04-17 16:32:13 +02003088 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003089
3090 priv->irq[0] = irq_of_parse_and_map(np, 0);
Kim Phillips2cdba3c2011-12-12 14:59:11 -06003091 if (!priv->irq[0]) {
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003092 dev_err(dev, "failed to map irq\n");
3093 return -EINVAL;
3094 }
LEROY Christophedd3c0982015-04-17 16:32:13 +02003095 if (is_sec1) {
3096 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0,
3097 dev_driver_string(dev), dev);
3098 goto primary_out;
3099 }
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003100
3101 priv->irq[1] = irq_of_parse_and_map(np, 1);
3102
3103 /* get the primary irq line */
Kim Phillips2cdba3c2011-12-12 14:59:11 -06003104 if (!priv->irq[1]) {
LEROY Christophedd3c0982015-04-17 16:32:13 +02003105 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0,
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003106 dev_driver_string(dev), dev);
3107 goto primary_out;
3108 }
3109
LEROY Christophedd3c0982015-04-17 16:32:13 +02003110 err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0,
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003111 dev_driver_string(dev), dev);
3112 if (err)
3113 goto primary_out;
3114
3115 /* get the secondary irq line */
LEROY Christophedd3c0982015-04-17 16:32:13 +02003116 err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0,
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003117 dev_driver_string(dev), dev);
3118 if (err) {
3119 dev_err(dev, "failed to request secondary irq\n");
3120 irq_dispose_mapping(priv->irq[1]);
Kim Phillips2cdba3c2011-12-12 14:59:11 -06003121 priv->irq[1] = 0;
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003122 }
3123
3124 return err;
3125
3126primary_out:
3127 if (err) {
3128 dev_err(dev, "failed to request primary irq\n");
3129 irq_dispose_mapping(priv->irq[0]);
Kim Phillips2cdba3c2011-12-12 14:59:11 -06003130 priv->irq[0] = 0;
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003131 }
3132
3133 return err;
3134}
3135
Grant Likely1c48a5c2011-02-17 02:43:24 -07003136static int talitos_probe(struct platform_device *ofdev)
Kim Phillips9c4a7962008-06-23 19:50:15 +08003137{
3138 struct device *dev = &ofdev->dev;
Grant Likely61c7a082010-04-13 16:12:29 -07003139 struct device_node *np = ofdev->dev.of_node;
Kim Phillips9c4a7962008-06-23 19:50:15 +08003140 struct talitos_private *priv;
Kim Phillips9c4a7962008-06-23 19:50:15 +08003141 int i, err;
LEROY Christophe5fa7fa12015-04-17 16:32:11 +02003142 int stride;
LEROY Christophefd5ea7f2017-10-06 15:04:53 +02003143 struct resource *res;
Kim Phillips9c4a7962008-06-23 19:50:15 +08003144
LEROY Christophe24b92ff2017-10-06 15:04:49 +02003145 priv = devm_kzalloc(dev, sizeof(struct talitos_private), GFP_KERNEL);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003146 if (!priv)
3147 return -ENOMEM;
3148
Kevin Haof3de9cb2014-01-28 20:17:23 +08003149 INIT_LIST_HEAD(&priv->alg_list);
3150
Kim Phillips9c4a7962008-06-23 19:50:15 +08003151 dev_set_drvdata(dev, priv);
3152
3153 priv->ofdev = ofdev;
3154
Horia Geanta511d63c2012-03-30 17:49:53 +03003155 spin_lock_init(&priv->reg_lock);
3156
LEROY Christophefd5ea7f2017-10-06 15:04:53 +02003157 res = platform_get_resource(ofdev, IORESOURCE_MEM, 0);
3158 if (!res)
3159 return -ENXIO;
3160 priv->reg = devm_ioremap(dev, res->start, resource_size(res));
Kim Phillips9c4a7962008-06-23 19:50:15 +08003161 if (!priv->reg) {
3162 dev_err(dev, "failed to of_iomap\n");
3163 err = -ENOMEM;
3164 goto err_out;
3165 }
3166
3167 /* get SEC version capabilities from device tree */
LEROY Christophefa14c6c2017-10-06 15:04:51 +02003168 of_property_read_u32(np, "fsl,num-channels", &priv->num_channels);
3169 of_property_read_u32(np, "fsl,channel-fifo-len", &priv->chfifo_len);
3170 of_property_read_u32(np, "fsl,exec-units-mask", &priv->exec_units);
3171 of_property_read_u32(np, "fsl,descriptor-types-mask",
3172 &priv->desc_types);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003173
3174 if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
3175 !priv->exec_units || !priv->desc_types) {
3176 dev_err(dev, "invalid property data in device tree node\n");
3177 err = -EINVAL;
3178 goto err_out;
3179 }
3180
Lee Nipperf3c85bc2008-07-30 16:26:57 +08003181 if (of_device_is_compatible(np, "fsl,sec3.0"))
3182 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
3183
Kim Phillipsfe5720e2008-10-12 20:33:14 +08003184 if (of_device_is_compatible(np, "fsl,sec2.1"))
Kim Phillips60f208d2010-05-19 19:21:53 +10003185 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
Lee Nipper79b3a412011-11-21 16:13:25 +08003186 TALITOS_FTR_SHA224_HWINIT |
3187 TALITOS_FTR_HMAC_OK;
Kim Phillipsfe5720e2008-10-12 20:33:14 +08003188
LEROY Christophe21590882015-04-17 16:32:05 +02003189 if (of_device_is_compatible(np, "fsl,sec1.0"))
3190 priv->features |= TALITOS_FTR_SEC1;
3191
LEROY Christophe5fa7fa12015-04-17 16:32:11 +02003192 if (of_device_is_compatible(np, "fsl,sec1.2")) {
3193 priv->reg_deu = priv->reg + TALITOS12_DEU;
3194 priv->reg_aesu = priv->reg + TALITOS12_AESU;
3195 priv->reg_mdeu = priv->reg + TALITOS12_MDEU;
3196 stride = TALITOS1_CH_STRIDE;
3197 } else if (of_device_is_compatible(np, "fsl,sec1.0")) {
3198 priv->reg_deu = priv->reg + TALITOS10_DEU;
3199 priv->reg_aesu = priv->reg + TALITOS10_AESU;
3200 priv->reg_mdeu = priv->reg + TALITOS10_MDEU;
3201 priv->reg_afeu = priv->reg + TALITOS10_AFEU;
3202 priv->reg_rngu = priv->reg + TALITOS10_RNGU;
3203 priv->reg_pkeu = priv->reg + TALITOS10_PKEU;
3204 stride = TALITOS1_CH_STRIDE;
3205 } else {
3206 priv->reg_deu = priv->reg + TALITOS2_DEU;
3207 priv->reg_aesu = priv->reg + TALITOS2_AESU;
3208 priv->reg_mdeu = priv->reg + TALITOS2_MDEU;
3209 priv->reg_afeu = priv->reg + TALITOS2_AFEU;
3210 priv->reg_rngu = priv->reg + TALITOS2_RNGU;
3211 priv->reg_pkeu = priv->reg + TALITOS2_PKEU;
3212 priv->reg_keu = priv->reg + TALITOS2_KEU;
3213 priv->reg_crcu = priv->reg + TALITOS2_CRCU;
3214 stride = TALITOS2_CH_STRIDE;
3215 }
3216
LEROY Christophedd3c0982015-04-17 16:32:13 +02003217 err = talitos_probe_irq(ofdev);
3218 if (err)
3219 goto err_out;
3220
3221 if (of_device_is_compatible(np, "fsl,sec1.0")) {
LEROY Christophe9c02e282017-10-06 15:04:55 +02003222 if (priv->num_channels == 1)
3223 tasklet_init(&priv->done_task[0], talitos1_done_ch0,
LEROY Christophedd3c0982015-04-17 16:32:13 +02003224 (unsigned long)dev);
LEROY Christophe9c02e282017-10-06 15:04:55 +02003225 else
3226 tasklet_init(&priv->done_task[0], talitos1_done_4ch,
3227 (unsigned long)dev);
3228 } else {
3229 if (priv->irq[1]) {
LEROY Christophedd3c0982015-04-17 16:32:13 +02003230 tasklet_init(&priv->done_task[0], talitos2_done_ch0_2,
3231 (unsigned long)dev);
3232 tasklet_init(&priv->done_task[1], talitos2_done_ch1_3,
3233 (unsigned long)dev);
LEROY Christophe9c02e282017-10-06 15:04:55 +02003234 } else if (priv->num_channels == 1) {
3235 tasklet_init(&priv->done_task[0], talitos2_done_ch0,
3236 (unsigned long)dev);
3237 } else {
3238 tasklet_init(&priv->done_task[0], talitos2_done_4ch,
3239 (unsigned long)dev);
LEROY Christophedd3c0982015-04-17 16:32:13 +02003240 }
3241 }
3242
LEROY Christophe24b92ff2017-10-06 15:04:49 +02003243 priv->chan = devm_kzalloc(dev, sizeof(struct talitos_channel) *
3244 priv->num_channels, GFP_KERNEL);
Kim Phillips4b9926282009-08-13 11:50:38 +10003245 if (!priv->chan) {
3246 dev_err(dev, "failed to allocate channel management space\n");
Kim Phillips9c4a7962008-06-23 19:50:15 +08003247 err = -ENOMEM;
3248 goto err_out;
3249 }
3250
Martin Hicksf641ddd2015-03-03 08:21:33 -05003251 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
3252
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003253 for (i = 0; i < priv->num_channels; i++) {
LEROY Christophe5fa7fa12015-04-17 16:32:11 +02003254 priv->chan[i].reg = priv->reg + stride * (i + 1);
Kim Phillips2cdba3c2011-12-12 14:59:11 -06003255 if (!priv->irq[1] || !(i & 1))
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003256 priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
Kim Phillipsad42d5f2011-11-21 16:13:27 +08003257
Kim Phillips4b9926282009-08-13 11:50:38 +10003258 spin_lock_init(&priv->chan[i].head_lock);
3259 spin_lock_init(&priv->chan[i].tail_lock);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003260
LEROY Christophe24b92ff2017-10-06 15:04:49 +02003261 priv->chan[i].fifo = devm_kzalloc(dev,
3262 sizeof(struct talitos_request) *
3263 priv->fifo_len, GFP_KERNEL);
Kim Phillips4b9926282009-08-13 11:50:38 +10003264 if (!priv->chan[i].fifo) {
Kim Phillips9c4a7962008-06-23 19:50:15 +08003265 dev_err(dev, "failed to allocate request fifo %d\n", i);
3266 err = -ENOMEM;
3267 goto err_out;
3268 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08003269
Kim Phillips4b9926282009-08-13 11:50:38 +10003270 atomic_set(&priv->chan[i].submit_count,
3271 -(priv->chfifo_len - 1));
Martin Hicksf641ddd2015-03-03 08:21:33 -05003272 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08003273
Kim Phillips81eb0242009-08-13 11:51:51 +10003274 dma_set_mask(dev, DMA_BIT_MASK(36));
3275
Kim Phillips9c4a7962008-06-23 19:50:15 +08003276 /* reset and initialize the h/w */
3277 err = init_device(dev);
3278 if (err) {
3279 dev_err(dev, "failed to initialize device\n");
3280 goto err_out;
3281 }
3282
3283 /* register the RNG, if available */
3284 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
3285 err = talitos_register_rng(dev);
3286 if (err) {
3287 dev_err(dev, "failed to register hwrng: %d\n", err);
3288 goto err_out;
3289 } else
3290 dev_info(dev, "hwrng\n");
3291 }
3292
3293 /* register crypto algorithms the device supports */
Kim Phillips9c4a7962008-06-23 19:50:15 +08003294 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3295 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
3296 struct talitos_crypto_alg *t_alg;
Herbert Xuaeb4c132015-07-30 17:53:22 +08003297 struct crypto_alg *alg = NULL;
Kim Phillips9c4a7962008-06-23 19:50:15 +08003298
3299 t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
3300 if (IS_ERR(t_alg)) {
3301 err = PTR_ERR(t_alg);
Kim Phillips0b2730d2011-12-12 14:59:10 -06003302 if (err == -ENOTSUPP)
Lee Nipper79b3a412011-11-21 16:13:25 +08003303 continue;
Kim Phillips9c4a7962008-06-23 19:50:15 +08003304 goto err_out;
3305 }
3306
Lee Nipperacbf7c622010-05-19 19:19:33 +10003307 switch (t_alg->algt.type) {
3308 case CRYPTO_ALG_TYPE_ABLKCIPHER:
Lee Nipperacbf7c622010-05-19 19:19:33 +10003309 err = crypto_register_alg(
3310 &t_alg->algt.alg.crypto);
Herbert Xuaeb4c132015-07-30 17:53:22 +08003311 alg = &t_alg->algt.alg.crypto;
Lee Nipperacbf7c622010-05-19 19:19:33 +10003312 break;
Herbert Xuaeb4c132015-07-30 17:53:22 +08003313
3314 case CRYPTO_ALG_TYPE_AEAD:
3315 err = crypto_register_aead(
3316 &t_alg->algt.alg.aead);
3317 alg = &t_alg->algt.alg.aead.base;
3318 break;
3319
Lee Nipperacbf7c622010-05-19 19:19:33 +10003320 case CRYPTO_ALG_TYPE_AHASH:
3321 err = crypto_register_ahash(
3322 &t_alg->algt.alg.hash);
Herbert Xuaeb4c132015-07-30 17:53:22 +08003323 alg = &t_alg->algt.alg.hash.halg.base;
Lee Nipperacbf7c622010-05-19 19:19:33 +10003324 break;
3325 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08003326 if (err) {
3327 dev_err(dev, "%s alg registration failed\n",
Herbert Xuaeb4c132015-07-30 17:53:22 +08003328 alg->cra_driver_name);
LEROY Christophe24b92ff2017-10-06 15:04:49 +02003329 devm_kfree(dev, t_alg);
Horia Geanta991155b2013-03-20 16:31:38 +02003330 } else
Kim Phillips9c4a7962008-06-23 19:50:15 +08003331 list_add_tail(&t_alg->entry, &priv->alg_list);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003332 }
3333 }
Kim Phillips5b859b6e2011-11-21 16:13:26 +08003334 if (!list_empty(&priv->alg_list))
3335 dev_info(dev, "%s algorithms registered in /proc/crypto\n",
3336 (char *)of_get_property(np, "compatible", NULL));
Kim Phillips9c4a7962008-06-23 19:50:15 +08003337
3338 return 0;
3339
3340err_out:
3341 talitos_remove(ofdev);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003342
3343 return err;
3344}
3345
Márton Németh6c3f9752010-01-17 21:54:01 +11003346static const struct of_device_id talitos_match[] = {
LEROY Christophe0635b7db2015-04-17 16:32:20 +02003347#ifdef CONFIG_CRYPTO_DEV_TALITOS1
3348 {
3349 .compatible = "fsl,sec1.0",
3350 },
3351#endif
3352#ifdef CONFIG_CRYPTO_DEV_TALITOS2
Kim Phillips9c4a7962008-06-23 19:50:15 +08003353 {
3354 .compatible = "fsl,sec2.0",
3355 },
LEROY Christophe0635b7db2015-04-17 16:32:20 +02003356#endif
Kim Phillips9c4a7962008-06-23 19:50:15 +08003357 {},
3358};
3359MODULE_DEVICE_TABLE(of, talitos_match);
3360
Grant Likely1c48a5c2011-02-17 02:43:24 -07003361static struct platform_driver talitos_driver = {
Grant Likely40182942010-04-13 16:13:02 -07003362 .driver = {
3363 .name = "talitos",
Grant Likely40182942010-04-13 16:13:02 -07003364 .of_match_table = talitos_match,
3365 },
Kim Phillips9c4a7962008-06-23 19:50:15 +08003366 .probe = talitos_probe,
Al Viro596f1032008-11-22 17:34:24 +00003367 .remove = talitos_remove,
Kim Phillips9c4a7962008-06-23 19:50:15 +08003368};
3369
Axel Lin741e8c22011-11-26 21:26:19 +08003370module_platform_driver(talitos_driver);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003371
3372MODULE_LICENSE("GPL");
3373MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
3374MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");